@voidwire/lore 0.1.2 → 0.1.4

package/README.md

@@ -35,6 +35,20 @@ lore capture task|knowledge|note       # Capture knowledge
 - `--since <date>` — Filter by date (today, yesterday, this-week, YYYY-MM-DD)
 - `--sources` — List indexed sources with counts
 
+### Passthrough Sources
+
+Some sources query external services rather than the local index:
+
+```bash
+lore search prismis "kubernetes security"    # Semantic search via prismis
+```
+
+| Source | Description | Requires |
+|--------|-------------|----------|
+| `prismis` | Semantic search across saved articles | prismis-daemon running |
+
+Passthrough sources appear in `lore search --sources` with `type: "passthrough"`.
+
 ### List Options
 
 - `--limit <n>` — Maximum entries

package/cli.ts

@@ -22,12 +22,16 @@
 import {
   search,
   searchPrismis,
+  searchAtuin,
   listSources,
   list,
   listDomains,
   captureTask,
   captureKnowledge,
   captureNote,
+  semanticSearch,
+  isOllamaAvailable,
+  hasEmbeddings,
   DOMAINS,
   type SearchResult,
   type ListResult,
@@ -72,8 +76,29 @@ function parseArgs(args: string[]): Map<string, string> {
   return parsed;
 }
 
+// Boolean flags that don't take values
+const BOOLEAN_FLAGS = new Set(["help", "sources", "domains", "exact"]);
+
 function getPositionalArgs(args: string[]): string[] {
-  return args.filter((arg) => !arg.startsWith("--"));
+  const result: string[] = [];
+  let i = 0;
+  while (i < args.length) {
+    const arg = args[i];
+    if (arg.startsWith("--")) {
+      const flag = arg.slice(2).split("=")[0]; // Handle --flag=value format
+      if (BOOLEAN_FLAGS.has(flag) || arg.includes("=")) {
+        i += 1; // Boolean flag or --flag=value, skip only the flag
+      } else if (i + 1 < args.length && !args[i + 1].startsWith("--")) {
+        i += 2; // Flag with separate value, skip both
+      } else {
+        i += 1; // Flag at end or followed by another flag
+      }
+      continue;
+    }
+    result.push(arg);
+    i++;
+  }
+  return result;
 }
 
 function hasFlag(args: string[], flag: string): boolean {
@@ -112,19 +137,21 @@ function fail(error: string, code: number = 1): never {
 // Search Command
 // ============================================================================
 
-function handleSearch(args: string[]): void {
+async function handleSearch(args: string[]): Promise<void> {
   if (hasFlag(args, "help")) {
     showSearchHelp();
   }
 
   const parsed = parseArgs(args);
   const positional = getPositionalArgs(args);
+  const exact = hasFlag(args, "exact");
 
   // Handle --sources flag
   if (hasFlag(args, "sources")) {
     const indexed = listSources();
     const passthrough = [
       { source: "prismis", count: null, type: "passthrough" },
+      { source: "atuin", count: null, type: "passthrough" },
     ];
     const sources = [
       ...indexed.map((s) => ({ ...s, type: "indexed" })),
@@ -177,15 +204,63 @@ function handleSearch(args: string[]): void {
     return;
   }
 
+  // Handle atuin passthrough
+  if (source === "atuin") {
+    try {
+      const results = searchAtuin(query, { limit });
+      output({
+        success: true,
+        results,
+        count: results.length,
+      });
+      console.error(
+        `✅ ${results.length} result${results.length !== 1 ? "s" : ""} found`,
+      );
+      process.exit(0);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      fail(message, 2);
+    }
+    return;
+  }
+
+  // Route semantic vs FTS5 based on --exact flag and availability
+  if (!exact) {
+    try {
+      const canUseSemantic = hasEmbeddings() && (await isOllamaAvailable());
+      if (canUseSemantic) {
+        const results = await semanticSearch(query, { source, limit });
+        output({
+          success: true,
+          results,
+          count: results.length,
+          mode: "semantic",
+        });
+        console.error(
+          `✅ ${results.length} result${results.length !== 1 ? "s" : ""} found (semantic)`,
+        );
+        process.exit(0);
+      }
+      // Fall through to FTS5 if semantic not available
+    } catch (error) {
+      // Semantic search failed, fall back to FTS5
+      console.error(
+        `⚠️ Semantic search unavailable: ${error instanceof Error ? error.message : "Unknown error"}`,
+      );
+    }
+  }
+
+  // FTS5 path (default fallback or explicit --exact)
   try {
     const results = search(query, { source, limit, since });
     output({
       success: true,
       results,
       count: results.length,
+      mode: exact ? "exact" : "fts5",
     });
     console.error(
-      `✅ ${results.length} result${results.length !== 1 ? "s" : ""} found`,
+      `✅ ${results.length} result${results.length !== 1 ? "s" : ""} found (${exact ? "exact" : "fts5"})`,
     );
     process.exit(0);
   } catch (error) {
@@ -409,12 +484,14 @@ Usage:
   lore capture task|knowledge|note      Capture knowledge
 
 Search Options:
+  --exact           Use FTS5 text search (bypasses semantic search)
   --limit <n>       Maximum results (default: 20)
   --since <date>    Filter by date (today, yesterday, this-week, YYYY-MM-DD)
   --sources         List indexed sources with counts
 
 Passthrough Sources:
   prismis           Semantic search via prismis daemon (requires prismis-daemon running)
+  atuin             Shell history search (queries ~/.local/share/atuin/history.db directly)
 
 List Options:
   --limit <n>       Maximum entries
@@ -458,6 +535,7 @@ Usage:
   lore search --sources                 List indexed sources
 
 Options:
+  --exact           Use FTS5 text search (bypasses semantic search)
   --limit <n>       Maximum results (default: 20)
   --since <date>    Filter by date (today, yesterday, this-week, YYYY-MM-DD)
   --sources         List indexed sources with counts
@@ -479,12 +557,16 @@ Indexed Sources:
 Passthrough Sources:
   prismis           Semantic search via prismis daemon
                     (requires prismis-daemon running)
+  atuin             Shell history search
+                    (queries ~/.local/share/atuin/history.db directly)
 
 Examples:
   lore search "authentication"
   lore search blogs "typescript patterns"
   lore search commits --since this-week "refactor"
+  lore search --exact "def process_data"
   lore search prismis "kubernetes security"
+  lore search atuin "docker build"
 `);
   process.exit(0);
 }

package/index.ts

@@ -34,6 +34,13 @@ export {
   type PrismisSearchOptions,
 } from "./lib/prismis";
 
+// Atuin integration
+export {
+  searchAtuin,
+  type AtuinSearchResult,
+  type AtuinSearchOptions,
+} from "./lib/atuin";
+
 // Capture
 export {
   captureKnowledge,
@@ -46,3 +53,13 @@ export {
   type NoteInput,
   type CaptureEvent,
 } from "./lib/capture";
+
+// Semantic search
+export {
+  semanticSearch,
+  embedQuery,
+  isOllamaAvailable,
+  hasEmbeddings,
+  type SemanticResult,
+  type SemanticSearchOptions,
+} from "./lib/semantic";

package/lib/atuin.ts

@@ -0,0 +1,160 @@
+/**
+ * Atuin shell history integration
+ *
+ * Queries Atuin SQLite database directly for shell command history.
+ * Filters sensitive commands containing passwords, tokens, secrets, API keys.
+ */
+
+import { existsSync } from "fs";
+import { join } from "path";
+import { Database } from "bun:sqlite";
+
+export interface AtuinSearchResult {
+  source: string;
+  title: string;
+  content: string;
+  metadata: string;
+  rank: number;
+}
+
+export interface AtuinSearchOptions {
+  limit?: number;
+  cwd?: string;
+  exitCode?: number;
+}
+
+const ATUIN_DB_PATH = join(
+  process.env.HOME ?? "",
+  ".local",
+  "share",
+  "atuin",
+  "history.db",
+);
+
+// Patterns that indicate sensitive data - exclude these commands
+const SENSITIVE_PATTERNS = [
+  "%--password%",
+  "%--token%",
+  "%--secret%",
+  "%PASSWORD=%",
+  "%TOKEN=%",
+  "%SECRET=%",
+  "%API_KEY=%",
+  "%APIKEY=%",
+  "%_KEY=%",
+  "export %KEY%",
+  "export %TOKEN%",
+  "export %SECRET%",
+  "export %PASSWORD%",
+  "%X-API-Key:%",
+  "%Authorization:%",
+  "echo $%KEY%",
+  "echo $%TOKEN%",
+  "echo $%SECRET%",
+];
+
+interface AtuinRow {
+  command: string;
+  cwd: string;
+  exit: number;
+  duration: number;
+  timestamp: number;
+  hostname: string;
+}
+
+/**
+ * Search Atuin shell history
+ */
+export function searchAtuin(
+  query: string,
+  options: AtuinSearchOptions = {},
+): AtuinSearchResult[] {
+  if (!existsSync(ATUIN_DB_PATH)) {
+    throw new Error(
+      `Atuin database not found at ${ATUIN_DB_PATH}. ` +
+        "Install Atuin: https://atuin.sh",
+    );
+  }
+
+  const db = new Database(ATUIN_DB_PATH, { readonly: true });
+  const limit = options.limit ?? 20;
+
+  try {
+    // Build WHERE clause with sensitive data filtering
+    const conditions = [
+      "deleted_at IS NULL",
+      "command LIKE ?",
+      ...SENSITIVE_PATTERNS.map(() => "command NOT LIKE ?"),
+    ];
+
+    if (options.cwd) {
+      conditions.push("cwd = ?");
+    }
+
+    if (options.exitCode !== undefined) {
+      conditions.push("exit = ?");
+    }
+
+    const whereClause = conditions.join(" AND ");
+
+    const sql = `
+      SELECT command, cwd, exit, duration, timestamp, hostname
+      FROM history
+      WHERE ${whereClause}
+      ORDER BY timestamp DESC
+      LIMIT ?
+    `;
+
+    // Build parameters
+    const params: (string | number)[] = [`%${query}%`, ...SENSITIVE_PATTERNS];
+
+    if (options.cwd) {
+      params.push(options.cwd);
+    }
+
+    if (options.exitCode !== undefined) {
+      params.push(options.exitCode);
+    }
+
+    params.push(limit);
+
+    const stmt = db.prepare(sql);
+    const rows = stmt.all(...params) as AtuinRow[];
+
+    return rows.map((row, index) => {
+      // Convert nanosecond timestamp to ISO date
+      const timestampSec = Math.floor(row.timestamp / 1_000_000_000);
+      const date = new Date(timestampSec * 1000);
+      const dateStr = date.toISOString().split("T")[0];
+
+      // Convert duration from nanoseconds to milliseconds
+      const durationMs = Math.floor(row.duration / 1_000_000);
+
+      // Build title: truncate command to 80 chars
+      const title =
+        row.command.length > 80
+          ? `[shell] ${row.command.slice(0, 77)}...`
+          : `[shell] ${row.command}`;
+
+      // Normalize cwd
+      const cwd = row.cwd === "unknown" ? "" : row.cwd;
+
+      return {
+        source: "atuin",
+        title,
+        content: row.command,
+        metadata: JSON.stringify({
+          command: row.command,
+          cwd,
+          exit_code: row.exit,
+          duration_ms: durationMs,
+          date: dateStr,
+          hostname: row.hostname,
+        }),
+        rank: -index, // Simple ranking by recency
+      };
+    });
+  } finally {
+    db.close();
+  }
+}

package/lib/semantic.ts

@@ -0,0 +1,278 @@
+/**
+ * lib/semantic.ts - Semantic search via Ollama embeddings
+ *
+ * Query embedding and KNN search against sqlite-vec virtual table.
+ * Uses Bun's built-in SQLite with sqlite-vec extension.
+ *
+ * Note: macOS ships Apple's SQLite which disables extension loading.
+ * We use Homebrew's SQLite via setCustomSQLite() to enable sqlite-vec.
+ */
+
+import { Database } from "bun:sqlite";
+import { homedir } from "os";
+import { existsSync, readFileSync } from "fs";
+
+// Use Homebrew SQLite on macOS to enable extension loading
+// Must be called before any Database instances are created
+const HOMEBREW_SQLITE = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib";
+if (existsSync(HOMEBREW_SQLITE)) {
+  Database.setCustomSQLite(HOMEBREW_SQLITE);
+}
+
+export interface SemanticResult {
+  source: string;
+  title: string;
+  content: string;
+  metadata: string;
+  distance: number;
+}
+
+export interface SemanticSearchOptions {
+  source?: string;
+  limit?: number;
+}
+
+interface EmbeddingConfig {
+  endpoint: string;
+  model: string;
+}
+
+const DEFAULT_CONFIG: EmbeddingConfig = {
+  endpoint: "http://localhost:11434",
+  model: "nomic-embed-text",
+};
+
+function getDatabasePath(): string {
+  return `${homedir()}/.local/share/lore/lore.db`;
+}
+
+function getConfigPath(): string {
+  return `${homedir()}/.config/lore/config.toml`;
+}
+
+/**
+ * Load embedding config from config.toml
+ * Falls back to [llm].api_base if [embedding].endpoint not set
+ */
+function loadEmbeddingConfig(): EmbeddingConfig {
+  const configPath = getConfigPath();
+
+  if (!existsSync(configPath)) {
+    return DEFAULT_CONFIG;
+  }
+
+  try {
+    const content = readFileSync(configPath, "utf-8");
+
+    // Extract [embedding].endpoint first
+    const endpointMatch = content.match(
+      /\[embedding\][^[]*endpoint\s*=\s*"([^"]+)"/s,
+    );
+    if (endpointMatch) {
+      const modelMatch = content.match(
+        /\[embedding\][^[]*model\s*=\s*"([^"]+)"/s,
+      );
+      return {
+        endpoint: endpointMatch[1],
+        model: modelMatch?.[1] ?? DEFAULT_CONFIG.model,
+      };
+    }
+
+    // Fall back to [llm].api_base
+    const apiBaseMatch = content.match(/\[llm\][^[]*api_base\s*=\s*"([^"]+)"/s);
+    if (apiBaseMatch) {
+      const modelMatch = content.match(
+        /\[embedding\][^[]*model\s*=\s*"([^"]+)"/s,
+      );
+      return {
+        endpoint: apiBaseMatch[1],
+        model: modelMatch?.[1] ?? DEFAULT_CONFIG.model,
+      };
+    }
+
+    return DEFAULT_CONFIG;
+  } catch {
+    return DEFAULT_CONFIG;
+  }
+}
+
+/**
+ * Check if Ollama is available at configured endpoint
+ */
+export async function isOllamaAvailable(): Promise<boolean> {
+  const config = loadEmbeddingConfig();
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 2000);
+
+    const response = await fetch(`${config.endpoint}/api/tags`, {
+      method: "GET",
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Embed a query string using Ollama
+ * @returns 768-dimensional embedding vector
+ */
+export async function embedQuery(query: string): Promise<number[]> {
+  const config = loadEmbeddingConfig();
+  const url = `${config.endpoint}/api/embeddings`;
+
+  const response = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      model: config.model,
+      prompt: query,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Ollama API error: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const result = (await response.json()) as { embedding?: number[] };
+  const embedding = result.embedding;
+
+  if (!Array.isArray(embedding) || embedding.length !== 768) {
+    throw new Error(
+      `Invalid embedding: expected 768 dims, got ${embedding?.length ?? 0}`,
+    );
+  }
+
+  return embedding;
+}
+
+/**
+ * Check if embeddings table has any data
+ */
+export function hasEmbeddings(): boolean {
+  const dbPath = getDatabasePath();
+
+  if (!existsSync(dbPath)) {
+    return false;
+  }
+
+  const db = new Database(dbPath, { readonly: true });
+
+  try {
+    // Load sqlite-vec extension
+    const vecPath = process.env.SQLITE_VEC_PATH;
+    if (!vecPath) {
+      return false;
+    }
+
+    db.loadExtension(vecPath);
+
+    const stmt = db.prepare("SELECT COUNT(*) as count FROM embeddings");
+    const result = stmt.get() as { count: number };
+    return result.count > 0;
+  } catch {
+    return false;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Serialize embedding to blob format for sqlite-vec
+ */
+function serializeEmbedding(embedding: number[]): Uint8Array {
+  const buffer = new Float32Array(embedding);
+  return new Uint8Array(buffer.buffer);
+}
+
+/**
+ * Perform semantic search using KNN against embeddings table
+ */
+export async function semanticSearch(
+  query: string,
+  options: SemanticSearchOptions = {},
+): Promise<SemanticResult[]> {
+  const dbPath = getDatabasePath();
+
+  if (!existsSync(dbPath)) {
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+  }
+
+  // Get query embedding
+  const queryEmbedding = await embedQuery(query);
+  const queryBlob = serializeEmbedding(queryEmbedding);
+
+  const db = new Database(dbPath, { readonly: true });
+
+  try {
+    // Load sqlite-vec extension
+    const vecPath = process.env.SQLITE_VEC_PATH;
+    if (!vecPath) {
+      throw new Error(
+        'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
+      );
+    }
+
+    db.loadExtension(vecPath);
+
+    const limit = options.limit ?? 20;
+
+    // KNN query with join to search table
+    // Group by doc_id to return best chunk per document
+    let sql: string;
+    const params: (Uint8Array | string | number)[] = [queryBlob];
+
+    if (options.source) {
+      sql = `
+        SELECT
+          s.source,
+          s.title,
+          s.content,
+          s.metadata,
+          MIN(e.distance) as distance
+        FROM embeddings e
+        JOIN search s ON e.doc_id = s.rowid
+        WHERE e.embedding MATCH ?
+          AND k = ?
+          AND s.source = ?
+        GROUP BY s.rowid
+        ORDER BY distance
+        LIMIT ?
+      `;
+      params.push(limit * 3); // Fetch more for grouping
+      params.push(options.source);
+      params.push(limit);
+    } else {
+      sql = `
+        SELECT
+          s.source,
+          s.title,
+          s.content,
+          s.metadata,
+          MIN(e.distance) as distance
+        FROM embeddings e
+        JOIN search s ON e.doc_id = s.rowid
+        WHERE e.embedding MATCH ?
+          AND k = ?
+        GROUP BY s.rowid
+        ORDER BY distance
+        LIMIT ?
+      `;
+      params.push(limit * 3); // Fetch more for grouping
+      params.push(limit);
+    }
+
+    const stmt = db.prepare(sql);
+    const results = stmt.all(...params) as SemanticResult[];
+
+    return results;
+  } finally {
+    db.close();
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",
@@ -42,7 +42,7 @@
     "bun": ">=1.0.0"
   },
   "devDependencies": {
-    "bun-types": "latest"
+    "bun-types": "1.3.5"
   },
   "scripts": {
     "test": "bun test"