@voidwire/lore 1.3.0 → 1.5.0

package/cli.ts

@@ -41,6 +41,9 @@ import {
   hybridSearch,
   formatBriefSearch,
   hasEmbeddings,
+  findPurgeMatches,
+  deleteEntries,
+  PURGEABLE_SOURCES,
   SOURCES,
   type SearchResult,
   type HybridResult,
@@ -55,6 +58,7 @@ import {
   type ObservationInput,
   type ObservationSubtype,
   type ObservationConfidence,
+  type PurgeableSource,
 } from "./index";
 import { isValidLoreType, LORE_TYPES } from "./lib/types";
 import { runIndexer } from "./lib/indexer";
@@ -103,6 +107,7 @@ const BOOLEAN_FLAGS = new Set([
   "brief",
   "list",
   "rebuild",
+  "force",
 ]);
 
 function getPositionalArgs(args: string[]): string[] {
@@ -611,6 +616,133 @@ Examples:
   process.exit(0);
 }
 
+// ============================================================================
+// Purge Command
+// ============================================================================
+
+async function handlePurge(args: string[]): Promise<void> {
+  if (hasFlag(args, "help")) {
+    showPurgeHelp();
+  }
+
+  const parsed = parseArgs(args);
+  const matchQuery = parsed.get("match");
+
+  if (!matchQuery) {
+    fail('Missing --match flag. Use: lore purge --match "content to find"');
+  }
+
+  const force = hasFlag(args, "force");
+  const sourceFilter = parsed.get("source") as PurgeableSource | undefined;
+
+  // Validate source filter if provided
+  if (
+    sourceFilter &&
+    !PURGEABLE_SOURCES.includes(sourceFilter as PurgeableSource)
+  ) {
+    fail(
+      `Invalid source: ${sourceFilter}. Purgeable sources: ${PURGEABLE_SOURCES.join(", ")}`,
+    );
+  }
+
+  try {
+    const matches = findPurgeMatches(matchQuery, { source: sourceFilter });
+
+    if (matches.length === 0) {
+      output({
+        success: true,
+        matches: 0,
+        deleted: 0,
+        message: "No matching entries found",
+      });
+      console.error("No matching entries found.");
+      process.exit(0);
+    }
+
+    // Display matches
+    console.error(`\nFound ${matches.length} matching entries:\n`);
+    for (const m of matches) {
+      const preview =
+        m.content.length > 100 ? m.content.slice(0, 100) + "..." : m.content;
+      console.error(`  [${m.rowid}] ${m.source}/${m.type}: ${preview}`);
+    }
+    console.error("");
+
+    // Confirm unless --force
+    if (!force) {
+      process.stderr.write(`Delete ${matches.length} entries? (y/N) `);
+
+      // Read confirmation from stdin
+      const buf = Buffer.alloc(16);
+      const fd = require("fs").openSync("/dev/tty", "r");
+      const bytesRead = require("fs").readSync(fd, buf, 0, 16, null);
+      require("fs").closeSync(fd);
+      const answer = buf.toString("utf8", 0, bytesRead).trim().toLowerCase();
+
+      if (answer !== "y" && answer !== "yes") {
+        output({
+          success: true,
+          matches: matches.length,
+          deleted: 0,
+          message: "Aborted",
+        });
+        console.error("Aborted.");
+        process.exit(0);
+      }
+    }
+
+    const result = deleteEntries(matches.map((m) => m.rowid));
+
+    output({
+      success: true,
+      matches: matches.length,
+      deleted: result.deleted,
+      rowids: result.rowids,
+    });
+    console.error(`Purged ${result.deleted} entries from search + embeddings.`);
+    process.exit(0);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    fail(message, 2);
+  }
+}
+
+function showPurgeHelp(): void {
+  console.log(`
+lore purge - Delete entries from purgeable sources
+
+Usage:
+  lore purge --match "content string"         Find and delete matching entries
+  lore purge --match "content" --force        Delete without confirmation
+  lore purge --match "content" --source captures   Filter by source
+
+Purgeable Sources:
+  captures          Quick captures (knowledge, decisions, gotchas, etc.)
+  observations      Model observations about user patterns
+  teachings         Teaching moments
+
+Non-purgeable sources (blogs, commits, obsidian, etc.) are never affected.
+
+Options:
+  --match <text>    Content to search for (required, uses LIKE matching)
+  --source <src>    Limit to specific purgeable source
+  --force           Skip confirmation prompt
+  --help            Show this help
+
+Process:
+  1. Searches purgeable sources for entries containing the match text
+  2. Displays matches with rowid, source, type, and content preview
+  3. Asks for confirmation (unless --force)
+  4. Deletes from both FTS5 search table and vec0 embeddings table
+
+Examples:
+  lore purge --match "install.sh overwrites config.toml"
+  lore purge --match "bracket prefix" --force
+  lore purge --match "stale gotcha" --source captures
+`);
+  process.exit(0);
+}
+
 // ============================================================================
 // Capture Command
 // ============================================================================
@@ -916,6 +1048,7 @@ Usage:
   lore about <project>                  Aggregate view of project knowledge
   lore about <project> --brief          Compact project summary
   lore capture task|knowledge|note|teaching  Capture knowledge
+  lore purge --match "content"              Delete matching entries
   lore index [source] [--rebuild] [--list]  Run indexers
 
 Search Options:
@@ -1303,9 +1436,12 @@ async function main(): Promise<void> {
     case "index":
       await handleIndex(commandArgs);
       break;
+    case "purge":
+      await handlePurge(commandArgs);
+      break;
     default:
       fail(
-        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, or index`,
+        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, purge, or index`,
       );
   }
 }

package/index.ts

@@ -102,3 +102,13 @@ export { LoreType, LORE_TYPES, isValidLoreType } from "./lib/types";
 
 // Real-time indexing
 export { indexAndEmbed } from "./lib/realtime";
+
+// Purge
+export {
+  findPurgeMatches,
+  deleteEntries,
+  PURGEABLE_SOURCES,
+  type PurgeableSource,
+  type PurgeMatch,
+  type PurgeResult,
+} from "./lib/purge";

package/lib/db.ts

@@ -45,7 +45,9 @@ export function openDatabase(readonly = false): Database {
     throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
   }
 
-  const db = new Database(dbPath, { readonly });
+  const db = readonly
+    ? new Database(dbPath, { readonly: true })
+    : new Database(dbPath);
 
   // Load sqlite-vec extension
   const vecPath = process.env.SQLITE_VEC_PATH;
@@ -71,5 +73,7 @@ export function openDatabaseBasic(readonly = false): Database {
     throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
   }
 
-  return new Database(dbPath, { readonly });
+  return readonly
+    ? new Database(dbPath, { readonly: true })
+    : new Database(dbPath);
 }

package/lib/purge.ts

@@ -0,0 +1,125 @@
+/**
+ * lib/purge.ts - Purge entries from lore database
+ *
+ * Search and delete entries from purgeable sources (captures, observations, teachings).
+ * Deletes from both FTS5 search table and vec0 embeddings table.
+ *
+ * Usage:
+ *   const matches = findPurgeMatches(query);
+ *   deleteEntries(matches.map(m => m.rowid));
+ */
+
+import { Database } from "bun:sqlite";
+import { existsSync } from "fs";
+import { getDatabasePath, openDatabase } from "./db.js";
+
+// Only these sources can be purged — indexed sources (blogs, commits, etc.) are never purgeable
+export const PURGEABLE_SOURCES = [
+  "captures",
+  "observations",
+  "teachings",
+] as const;
+
+export type PurgeableSource = (typeof PURGEABLE_SOURCES)[number];
+
+export interface PurgeMatch {
+  rowid: number;
+  source: string;
+  title: string;
+  content: string;
+  type: string;
+}
+
+export interface PurgeResult {
+  deleted: number;
+  rowids: number[];
+}
+
+/**
+ * Find entries matching query in purgeable sources using LIKE.
+ *
+ * Uses SQL LIKE instead of FTS5 MATCH because content strings
+ * often contain dots, dashes, and special characters that break
+ * FTS5 query syntax.
+ */
+export function findPurgeMatches(
+  query: string,
+  options: { source?: PurgeableSource } = {},
+): PurgeMatch[] {
+  const db = openDatabase(true);
+
+  try {
+    const conditions: string[] = ["content LIKE ?"];
+    const params: (string | number)[] = [`%${query}%`];
+
+    if (options.source) {
+      conditions.push("source = ?");
+      params.push(options.source);
+    } else {
+      const placeholders = PURGEABLE_SOURCES.map(() => "?").join(", ");
+      conditions.push(`source IN (${placeholders})`);
+      params.push(...PURGEABLE_SOURCES);
+    }
+
+    const sql = `
+      SELECT rowid, source, title, content, type
+      FROM search
+      WHERE ${conditions.join(" AND ")}
+      ORDER BY rowid DESC
+    `;
+
+    const stmt = db.prepare(sql);
+    return stmt.all(...params) as PurgeMatch[];
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Delete entries from both FTS5 search table and vec0 embeddings table.
+ *
+ * FTS5 DELETE: DELETE FROM search WHERE rowid = ?
+ * vec0 DELETE: DELETE FROM embeddings WHERE doc_id = ?
+ */
+export function deleteEntries(rowids: number[]): PurgeResult {
+  if (rowids.length === 0) {
+    return { deleted: 0, rowids: [] };
+  }
+
+  // Open DB directly for read-write (matches realtime.ts pattern —
+  // openDatabase(false) triggers SQLITE_MISUSE with custom_sqlite)
+  const dbPath = getDatabasePath();
+  if (!existsSync(dbPath)) {
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+  }
+
+  const db = new Database(dbPath);
+
+  try {
+    // Load sqlite-vec extension for embeddings table access
+    const vecPath = process.env.SQLITE_VEC_PATH;
+    if (!vecPath) {
+      throw new Error(
+        'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
+      );
+    }
+    db.loadExtension(vecPath);
+
+    const deleteSearch = db.prepare("DELETE FROM search WHERE rowid = ?");
+    const deleteEmbedding = db.prepare(
+      "DELETE FROM embeddings WHERE doc_id = ?",
+    );
+
+    let deleted = 0;
+
+    for (const rowid of rowids) {
+      deleteSearch.run(rowid);
+      deleteEmbedding.run(rowid);
+      deleted++;
+    }
+
+    return { deleted, rowids };
+  } finally {
+    db.close();
+  }
+}

package/lib/semantic.ts

@@ -13,6 +13,44 @@ import { getDatabasePath, openDatabase } from "./db.js";
 import { search as keywordSearch, type SearchResult } from "./search.js";
 import { getConfig } from "./config.js";
 
+// ─── Embedding Server (server-first, in-process fallback) ────────────────────
+
+const EMBED_SERVER = process.env.EMBED_SERVER_URL || "http://localhost:8090";
+
+/**
+ * Try the persistent embedding server first (warm: ~9ms vs 244ms in-process).
+ * Returns null on any failure — caller falls back to in-process.
+ */
+async function serverEmbed(
+  text: string,
+  prefix: string,
+): Promise<number[] | null> {
+  try {
+    const resp = await fetch(`${EMBED_SERVER}/embed`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ text, prefix }),
+      signal: AbortSignal.timeout(500),
+    });
+    if (!resp.ok) return null;
+    const data = (await resp.json()) as {
+      embedding?: number[];
+      dims?: number;
+    };
+    if (
+      !Array.isArray(data.embedding) ||
+      data.embedding.length !== EMBEDDING_DIM
+    ) {
+      return null;
+    }
+    return data.embedding;
+  } catch {
+    return null; // Server not running or timed out — fall back silently
+  }
+}
+
+// ─── Types ───────────────────────────────────────────────────────────────────
+
 export interface SemanticResult {
   rowid: number;
   source: string;
@@ -76,6 +114,11 @@ async function getEmbeddingPipeline(): Promise<EmbeddingPipeline> {
  * @returns 768-dimensional embedding vector
  */
 export async function embedQuery(query: string): Promise<number[]> {
+  // Try persistent server first (~9ms warm vs 244ms in-process)
+  const serverResult = await serverEmbed(query, "search_query");
+  if (serverResult) return serverResult;
+
+  // Fall back to in-process model loading
   const embedder = await getEmbeddingPipeline();
 
   // nomic model requires "search_query: " prefix for queries
@@ -104,6 +147,11 @@ export async function embedQuery(query: string): Promise<number[]> {
  * @returns 768-dimensional embedding vector
  */
 export async function embedDocument(text: string): Promise<number[]> {
+  // Try persistent server first (~9ms warm vs 244ms in-process)
+  const serverResult = await serverEmbed(text, "search_document");
+  if (serverResult) return serverResult;
+
+  // Fall back to in-process model loading
   const embedder = await getEmbeddingPipeline();
 
   const prefixedText = `search_document: ${text}`;
@@ -131,12 +179,23 @@ export async function embedDocument(text: string): Promise<number[]> {
 export async function embedDocuments(texts: string[]): Promise<number[][]> {
   if (texts.length === 0) return [];
 
-  const embedder = await getEmbeddingPipeline();
   const results: number[][] = [];
 
-  // Process one at a time (transformers.js doesn't batch well)
-  // But we benefit from cached pipeline
+  // Try persistent server first for each document
+  let serverAvailable = true;
   for (const text of texts) {
+    if (serverAvailable) {
+      const serverResult = await serverEmbed(text, "search_document");
+      if (serverResult) {
+        results.push(serverResult);
+        continue;
+      }
+      // Server failed — stop trying and fall back for remaining
+      serverAvailable = false;
+    }
+
+    // Fall back to in-process
+    const embedder = await getEmbeddingPipeline();
     const prefixedText = `search_document: ${text}`;
     const output = await embedder(prefixedText, {
       pooling: "mean",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",