@voidwire/lore 1.8.5 → 2.0.0

package/cli.ts

@@ -64,6 +64,52 @@ import {
 import { isValidLoreType, LORE_TYPES } from "./lib/types";
 import { runIndexer } from "./lib/indexer";
 import { indexers } from "./lib/indexers/index";
+import { runInit } from "./lib/init";
+import { runEmbed } from "./lib/embed";
+
+// ============================================================================
+// Import Command Handler
+// ============================================================================
+
+async function handleImport(args: string[]): Promise<void> {
+  const subcommand = args[0];
+  if (!subcommand) {
+    fail(
+      "Usage: lore import <goodreads|letterboxd|apple-podcasts|podcasts> <file>",
+    );
+  }
+  const file = args[1];
+  switch (subcommand) {
+    case "goodreads": {
+      if (!file) fail(`Usage: lore import goodreads <file>`);
+      const { importGoodreads } = await import("./lib/importers/goodreads");
+      await importGoodreads(file);
+      break;
+    }
+    case "letterboxd": {
+      if (!file) fail(`Usage: lore import letterboxd <file>`);
+      const { importLetterboxd } = await import("./lib/importers/letterboxd");
+      await importLetterboxd(file);
+      break;
+    }
+    case "apple-podcasts": {
+      const { importApplePodcasts } =
+        await import("./lib/importers/apple-podcasts");
+      await importApplePodcasts(file);
+      break;
+    }
+    case "podcasts": {
+      if (!file) fail(`Usage: lore import podcasts <file>`);
+      const { importPodcasts } = await import("./lib/importers/podcasts");
+      await importPodcasts(file);
+      break;
+    }
+    default:
+      fail(
+        `Unknown import source: ${subcommand}. Use: goodreads, letterboxd, apple-podcasts, podcasts`,
+      );
+  }
+}
 
 // ============================================================================
 // Argument Parsing
@@ -1474,9 +1520,21 @@ async function main(): Promise<void> {
     case "purge":
       await handlePurge(commandArgs);
       break;
+    case "init":
+      await runInit();
+      break;
+    case "embed":
+      await runEmbed({
+        rebuild: commandArgs.includes("--rebuild"),
+        dryRun: commandArgs.includes("--dry-run"),
+      });
+      break;
+    case "import":
+      await handleImport(commandArgs);
+      break;
     default:
       fail(
-        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, purge, or index`,
+        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, purge, index, init, embed, or import`,
       );
   }
 }

package/lib/config.ts

@@ -18,11 +18,11 @@ import { parse as parseToml } from "@iarna/toml";
 export interface LoreConfig {
   paths: {
     data: string;
-    obsidian: string;
-    explorations: string;
-    blogs: string;
-    projects: string;
     personal: string;
+    obsidian?: string;
+    explorations?: string;
+    blogs?: string;
+    projects?: string;
     session_events?: string;
     sable_events?: string;
     flux?: string;
@@ -32,6 +32,7 @@ export interface LoreConfig {
   database: {
     sqlite: string;
     custom_sqlite?: string;
+    sqlite_vec?: string;
   };
   embedding: {
     model: string;
@@ -64,7 +65,7 @@ export function getConfig(): LoreConfig {
     throw new Error(
       `Config file not found: ${configPath}\n` +
         `Create it with [paths] and [database] sections.\n` +
-        `See: https://github.com/nickpending/llmcli-tools/tree/main/packages/lore#configuration`,
+        `See: https://github.com/nickpending/lore#configuration`,
     );
   }
 
@@ -107,15 +108,8 @@ export function getConfig(): LoreConfig {
     );
   }
 
-  // Validate required path fields
-  const requiredPaths = [
-    "data",
-    "obsidian",
-    "explorations",
-    "blogs",
-    "projects",
-    "personal",
-  ];
+  // Validate required path fields (data + personal are always created by init)
+  const requiredPaths = ["data", "personal"];
   for (const field of requiredPaths) {
     if (typeof paths[field] !== "string") {
       throw new Error(
@@ -134,11 +128,21 @@ export function getConfig(): LoreConfig {
   cachedConfig = {
     paths: {
       data: resolvePath(paths.data as string),
-      obsidian: resolvePath(paths.obsidian as string),
-      explorations: resolvePath(paths.explorations as string),
-      blogs: resolvePath(paths.blogs as string),
-      projects: resolvePath(paths.projects as string),
       personal: resolvePath(paths.personal as string),
+      obsidian:
+        typeof paths.obsidian === "string"
+          ? resolvePath(paths.obsidian)
+          : undefined,
+      explorations:
+        typeof paths.explorations === "string"
+          ? resolvePath(paths.explorations)
+          : undefined,
+      blogs:
+        typeof paths.blogs === "string" ? resolvePath(paths.blogs) : undefined,
+      projects:
+        typeof paths.projects === "string"
+          ? resolvePath(paths.projects)
+          : undefined,
       session_events:
         typeof paths.session_events === "string"
           ? resolvePath(paths.session_events)
@@ -161,6 +165,10 @@ export function getConfig(): LoreConfig {
         typeof database.custom_sqlite === "string"
           ? resolvePath(database.custom_sqlite)
           : undefined,
+      sqlite_vec:
+        typeof database.sqlite_vec === "string"
+          ? resolvePath(database.sqlite_vec)
+          : undefined,
     },
     embedding: {
       model: embedding.model as string,

package/lib/db.ts

@@ -9,22 +9,20 @@ import { Database } from "bun:sqlite";
 import { existsSync } from "fs";
 import { getConfig } from "./config";
 
-// Load custom SQLite from config to enable extension loading
-// Must be called before any Database instances are created
-const config = getConfig();
-if (config.database.custom_sqlite) {
-  if (!existsSync(config.database.custom_sqlite)) {
-    throw new Error(
-      `database.custom_sqlite path does not exist: ${config.database.custom_sqlite}`,
-    );
+// Lazy initialization — deferred until first database open
+// This allows `lore init` to run before config.toml exists
+let initialized = false;
+
+function ensureConfig(): void {
+  if (initialized) return;
+  const config = getConfig();
+  if (
+    config.database.custom_sqlite &&
+    existsSync(config.database.custom_sqlite)
+  ) {
+    Database.setCustomSQLite(config.database.custom_sqlite);
   }
-  Database.setCustomSQLite(config.database.custom_sqlite);
-} else {
-  throw new Error(
-    "database.custom_sqlite not set in ~/.config/lore/config.toml.\n" +
-      "Required for sqlite-vec extension loading.\n" +
-      'macOS: custom_sqlite = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib"',
-  );
+  initialized = true;
 }
 
 /**
@@ -39,10 +37,11 @@ export function getDatabasePath(): string {
  * @param readonly - Open in readonly mode (default: false)
  */
 export function openDatabase(readonly = false): Database {
+  ensureConfig();
   const dbPath = getDatabasePath();
 
   if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+    throw new Error(`Database not found: ${dbPath}. Run lore init first.`);
   }
 
   const db = readonly
@@ -50,10 +49,10 @@ export function openDatabase(readonly = false): Database {
     : new Database(dbPath);
 
   // Load sqlite-vec extension
-  const vecPath = process.env.SQLITE_VEC_PATH;
+  const vecPath = getConfig().database.sqlite_vec;
   if (!vecPath) {
     throw new Error(
-      'SQLITE_VEC_PATH not set. Get path with: python3 -c "import sqlite_vec; print(sqlite_vec.loadable_path())"',
+      "sqlite-vec path not configured. Run lore init to detect and configure it.",
     );
   }
 
@@ -67,10 +66,11 @@ export function openDatabase(readonly = false): Database {
  * @param readonly - Open in readonly mode (default: false)
  */
 export function openDatabaseBasic(readonly = false): Database {
+  ensureConfig();
   const dbPath = getDatabasePath();
 
   if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
+    throw new Error(`Database not found: ${dbPath}. Run lore init first.`);
   }
 
   return readonly

package/lib/embed.ts

@@ -0,0 +1,142 @@
+/**
+ * lib/embed.ts - Batch embedding command
+ *
+ * Reads unembedded FTS5 entries, generates embeddings via HTTP
+ * call to the embed server using @voidwire/llm-core's embed(),
+ * writes to vec0 table with SHA256 cache dedup.
+ * Replaces bin/lore-embed-all (Python).
+ */
+
+import { embed } from "@voidwire/llm-core";
+import { openDatabase } from "./db";
+import { hashContent, getCachedEmbedding, cacheEmbedding } from "./cache";
+import { serializeEmbedding } from "./semantic";
+import type { Database } from "bun:sqlite";
+
+const MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5";
+const EMBEDDING_DIM = 768;
+const BATCH_SIZE = 50;
+
+interface EmbedOptions {
+  rebuild?: boolean;
+  dryRun?: boolean;
+}
+
+interface FTSEntry {
+  rowid: number;
+  source: string;
+  content: string;
+  topic: string;
+  type: string;
+  timestamp: string;
+}
+
+export async function runEmbed(options: EmbedOptions = {}): Promise<void> {
+  const db = openDatabase();
+
+  try {
+    // If rebuild: delete all embeddings first
+    if (options.rebuild && !options.dryRun) {
+      db.exec("DELETE FROM embeddings");
+      db.exec("DELETE FROM embedding_cache");
+      console.log("Cleared all embeddings for rebuild");
+    }
+
+    // Find unembedded entries
+    const entries = getUnembeddedEntries(db);
+
+    if (options.dryRun) {
+      if (entries.length === 0) {
+        console.log("All entries embedded");
+      } else {
+        console.log(`${entries.length} entries need embedding`);
+      }
+      return;
+    }
+
+    if (entries.length === 0) {
+      console.log("All entries embedded");
+      return;
+    }
+
+    console.log(`Embedding ${entries.length} entries...`);
+
+    // Process in batches for throughput
+    let processed = 0;
+    for (let i = 0; i < entries.length; i += BATCH_SIZE) {
+      const batch = entries.slice(i, i + BATCH_SIZE);
+      await processBatch(db, batch);
+      processed += batch.length;
+      process.stdout.write(`\r${processed}/${entries.length}`);
+    }
+    console.log(`\nDone. Embedded ${processed} entries.`);
+  } finally {
+    db.close();
+  }
+}
+
+function getUnembeddedEntries(db: Database): FTSEntry[] {
+  // NOT IN subquery instead of LEFT JOIN: vec0 tables don't support efficient
+  // JOIN operations and would hang on large datasets with the JOIN approach.
+  const stmt = db.prepare(`
+    SELECT s.rowid, s.source, s.content, s.topic, s.type, s.timestamp
+    FROM search s
+    WHERE s.rowid NOT IN (SELECT doc_id FROM embeddings)
+    ORDER BY s.rowid
+  `);
+  return stmt.all() as FTSEntry[];
+}
+
+function buildContentString(entry: FTSEntry): string {
+  // Same format as realtime.ts getContentForEmbedding()
+  return [entry.type, entry.topic, entry.content].filter(Boolean).join(" ");
+}
+
+async function processBatch(db: Database, batch: FTSEntry[]): Promise<void> {
+  // Check cache first, collect misses
+  const toEmbed: { idx: number; contentString: string; hash: string }[] = [];
+  const embeddings: (number[] | null)[] = new Array(batch.length).fill(null);
+
+  for (let i = 0; i < batch.length; i++) {
+    const contentString = buildContentString(batch[i]);
+    const hash = hashContent(contentString);
+    const cached = getCachedEmbedding(db, hash);
+    if (cached) {
+      embeddings[i] = cached;
+    } else {
+      toEmbed.push({ idx: i, contentString, hash });
+    }
+  }
+
+  // Embed cache misses concurrently
+  if (toEmbed.length > 0) {
+    const results = await Promise.all(
+      toEmbed.map(({ contentString }) =>
+        embed({ text: contentString, prefix: "search_document" }),
+      ),
+    );
+    for (let i = 0; i < toEmbed.length; i++) {
+      const { idx, hash } = toEmbed[i];
+      embeddings[idx] = results[i].embedding;
+      cacheEmbedding(db, hash, results[i].embedding, MODEL_NAME);
+    }
+  }
+
+  // Insert all embeddings
+  const stmt = db.prepare(`
+    INSERT INTO embeddings (doc_id, chunk_idx, source, topic, type, timestamp, embedding)
+    VALUES (?, 0, ?, ?, ?, ?, ?)
+  `);
+  for (let i = 0; i < batch.length; i++) {
+    const entry = batch[i];
+    const embedding = embeddings[i]!;
+    stmt.run(
+      entry.rowid,
+      entry.source,
+      entry.topic,
+      entry.type,
+      entry.timestamp,
+      serializeEmbedding(embedding),
+    );
+  }
+}

package/lib/importers/apple-podcasts.ts

@@ -0,0 +1,98 @@
+/**
+ * lib/importers/apple-podcasts.ts - Apple Podcasts SQLite importer
+ *
+ * Reads the Apple Podcasts SQLite database and writes podcasts.json
+ * to the personal data directory.
+ *
+ * Default DB path:
+ *   ~/Library/Group Containers/243LU875E5.groups.com.apple.podcasts/Documents/MTLibrary.sqlite
+ *
+ * Output schema matches what lib/indexers/personal.ts reads.
+ */
+
+import { existsSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { Database } from "bun:sqlite";
+import { getConfig } from "../config";
+import { atomicWrite, mkdirSafe } from "../utils";
+
+const DEFAULT_DB_PATH = join(
+  homedir(),
+  "Library/Group Containers/243LU875E5.groups.com.apple.podcasts/Documents/MTLibrary.sqlite",
+);
+
+export async function importApplePodcasts(dbPath?: string): Promise<void> {
+  const resolvedPath = dbPath ?? DEFAULT_DB_PATH;
+
+  if (!existsSync(resolvedPath)) {
+    console.error("Apple Podcasts database not found");
+    console.error(`Expected: ${resolvedPath}`);
+    console.error(
+      "Make sure Apple Podcasts is installed and has been launched at least once.",
+    );
+    process.exit(1);
+  }
+
+  let db: Database;
+  try {
+    db = new Database(resolvedPath, { readonly: true });
+  } catch (e) {
+    console.error(`Error opening database: ${e}`);
+    process.exit(1);
+    return; // unreachable but satisfies TypeScript
+  }
+
+  const podcasts: {
+    title: string;
+    url: string;
+    description: string | null;
+    categories: string[] | null;
+  }[] = [];
+
+  try {
+    const rows = db
+      .prepare(
+        `SELECT ZTITLE as title, ZFEEDURL as url, ZITEMDESCRIPTION as description, ZCATEGORY as category
+       FROM ZMTPODCAST
+       WHERE ZSUBSCRIBED = 1
+       ORDER BY ZTITLE`,
+      )
+      .all() as {
+      title: string | null;
+      url: string | null;
+      description: string | null;
+      category: string | null;
+    }[];
+
+    for (const row of rows) {
+      const title = row.title ?? "";
+      const url = row.url ?? "";
+
+      // Skip podcasts without title or URL
+      if (!title || !url) continue;
+
+      podcasts.push({
+        title,
+        url,
+        description: row.description || null,
+        categories: row.category ? [row.category] : null,
+      });
+    }
+  } catch (e) {
+    console.error(`Error reading database: ${e}`);
+    db.close();
+    process.exit(1);
+  }
+
+  db.close();
+
+  const config = getConfig();
+  const personalDir = config.paths.personal;
+  mkdirSafe(personalDir);
+
+  const outPath = join(personalDir, "podcasts.json");
+  atomicWrite(outPath, podcasts);
+
+  console.log(`Imported ${podcasts.length} podcasts \u2192 ${outPath}`);
+}

package/lib/importers/goodreads.ts

@@ -0,0 +1,79 @@
+/**
+ * lib/importers/goodreads.ts - Goodreads CSV importer
+ *
+ * Reads a Goodreads library CSV export and writes books.json
+ * to the personal data directory.
+ *
+ * CSV columns: Title, Author, ISBN13, My Rating, Date Read, Bookshelves
+ * Output schema matches what lib/indexers/personal.ts reads.
+ */
+
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import { getConfig } from "../config";
+import { atomicWrite, mkdirSafe, parseCSV } from "../utils";
+
+export async function importGoodreads(filePath: string): Promise<void> {
+  if (!existsSync(filePath)) {
+    console.error(`File not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  const content = readFileSync(filePath, "utf-8");
+  const rows = parseCSV(content);
+
+  let skipped = 0;
+  const books: {
+    title: string;
+    author: string;
+    isbn: string | null;
+    rating: number | null;
+    date_read: string | null;
+    shelf: string | null;
+  }[] = [];
+
+  for (const row of rows) {
+    const title = (row["Title"] ?? "").trim();
+    if (!title) {
+      skipped++;
+      continue;
+    }
+
+    const author = (row["Author"] ?? "").trim();
+    let isbn = (row["ISBN13"] ?? "").trim();
+    const ratingStr = (row["My Rating"] ?? "").trim();
+    const dateRead = (row["Date Read"] ?? "").trim();
+    const shelf = (row["Bookshelves"] ?? "").trim();
+
+    // Clean ISBN Excel formula wrapper like ="1234567890123"
+    // CSV parser strips surrounding quotes, leaving =9780132350884 (no leading quote)
+    if (isbn.startsWith("=")) {
+      isbn = isbn.slice(1);
+    }
+
+    // Goodreads uses 0 to mean "not rated" — convert to null
+    const parsed = ratingStr ? parseInt(ratingStr, 10) : null;
+    const rating = parsed && !isNaN(parsed) ? parsed : null;
+
+    books.push({
+      title,
+      author,
+      isbn: isbn || null,
+      rating,
+      date_read: dateRead || null,
+      shelf: shelf || null,
+    });
+  }
+
+  const config = getConfig();
+  const personalDir = config.paths.personal;
+  mkdirSafe(personalDir);
+
+  const outPath = join(personalDir, "books.json");
+  atomicWrite(outPath, books);
+
+  console.log(`Imported ${books.length} books \u2192 ${outPath}`);
+  if (skipped > 0) {
+    console.log(`Skipped ${skipped} rows (empty title)`);
+  }
+}

package/lib/importers/letterboxd.ts

@@ -0,0 +1,70 @@
+/**
+ * lib/importers/letterboxd.ts - Letterboxd CSV importer
+ *
+ * Reads a Letterboxd ratings CSV export and writes movies.json
+ * to the personal data directory.
+ *
+ * CSV columns: Date, Name, Year, Letterboxd URI, Rating
+ * Output schema matches what lib/indexers/personal.ts reads.
+ *
+ * Note: Letterboxd uses 'Name' not 'Title', and ratings are floats
+ * (half-star increments, e.g. 3.5). The output field is 'date_watched'
+ * to match what personal.ts indexer reads as movie.date_watched.
+ */
+
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import { getConfig } from "../config";
+import { atomicWrite, mkdirSafe, parseCSV } from "../utils";
+
+export async function importLetterboxd(filePath: string): Promise<void> {
+  if (!existsSync(filePath)) {
+    console.error(`File not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  const content = readFileSync(filePath, "utf-8");
+  const rows = parseCSV(content);
+
+  let skipped = 0;
+  const movies: {
+    title: string;
+    year: number | null;
+    rating: number | null;
+    date_watched: string | null;
+  }[] = [];
+
+  for (const row of rows) {
+    const title = (row["Name"] ?? "").trim();
+    if (!title) {
+      skipped++;
+      continue;
+    }
+
+    const yearStr = (row["Year"] ?? "").trim();
+    const ratingStr = (row["Rating"] ?? "").trim();
+    const date = (row["Date"] ?? "").trim();
+
+    const year = yearStr ? parseInt(yearStr, 10) : null;
+    const rating = ratingStr ? parseFloat(ratingStr) : null;
+
+    movies.push({
+      title,
+      year: year !== null && !isNaN(year) ? year : null,
+      rating: rating !== null && !isNaN(rating) ? rating : null,
+      date_watched: date || null,
+    });
+  }
+
+  const config = getConfig();
+  const personalDir = config.paths.personal;
+  mkdirSafe(personalDir);
+
+  const outPath = join(personalDir, "movies.json");
+  atomicWrite(outPath, movies);
+
+  console.log(`Imported ${movies.length} movies \u2192 ${outPath}`);
+  if (skipped > 0) {
+    console.log(`Skipped ${skipped} rows (empty title)`);
+  }
+}

package/lib/importers/podcasts.ts

@@ -0,0 +1,151 @@
+/**
+ * lib/importers/podcasts.ts - OPML podcast importer with RSS enrichment
+ *
+ * Reads a podcast OPML export, extracts feed URLs and titles,
+ * then optionally enriches each entry by fetching its RSS feed
+ * for description and categories. Writes podcasts.json to the
+ * personal data directory.
+ *
+ * Output schema matches what lib/indexers/personal.ts reads.
+ */
+
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import { getConfig } from "../config";
+import { atomicWrite, mkdirSafe } from "../utils";
+
+// ---------------------------------------------------------------------------
+// OPML parsing (regex-based, no XML library needed)
+// ---------------------------------------------------------------------------
+
+interface PodcastEntry {
+  title: string;
+  url: string;
+  description: string | null;
+  categories: string[] | null;
+}
+
+function extractAttr(chunk: string, attr: string): string | null {
+  // Match attr="value" — handles both single and double quotes
+  const re = new RegExp(`${attr}=["']([^"']*)["']`);
+  const m = chunk.match(re);
+  return m ? m[1] : null;
+}
+
+function parseOPML(content: string): {
+  entries: PodcastEntry[];
+  skipped: number;
+} {
+  const entries: PodcastEntry[] = [];
+  let skipped = 0;
+
+  // Split on <outline to get each outline element as a chunk
+  const chunks = content.split(/<outline\b/);
+
+  for (let i = 1; i < chunks.length; i++) {
+    const chunk = chunks[i];
+
+    const xmlUrl = extractAttr(chunk, "xmlUrl");
+    // Skip folder/category nodes (no xmlUrl)
+    if (!xmlUrl) continue;
+
+    const title = extractAttr(chunk, "text") ?? extractAttr(chunk, "title");
+    if (!title) {
+      skipped++;
+      continue;
+    }
+
+    entries.push({
+      title,
+      url: xmlUrl,
+      description: null,
+      categories: null,
+    });
+  }
+
+  return { entries, skipped };
+}
+
+// ---------------------------------------------------------------------------
+// RSS enrichment (best-effort, per-feed)
+// ---------------------------------------------------------------------------
+
+async function enrichFromRSS(entry: PodcastEntry): Promise<PodcastEntry> {
+  try {
+    const resp = await fetch(entry.url, {
+      signal: AbortSignal.timeout(5000),
+      headers: { "User-Agent": "lore-import-podcasts/1.0" },
+    });
+    const xml = await resp.text();
+
+    // Extract channel block (RSS 2.0)
+    const channelMatch = xml.match(/<channel>([\s\S]*?)<\/channel>/);
+    if (!channelMatch) return entry;
+    const channel = channelMatch[1];
+
+    // Description: first <description> in channel
+    const descMatch = channel.match(/<description>([\s\S]*?)<\/description>/);
+    const description = descMatch
+      ? descMatch[1].replace(/<!\[CDATA\[|\]\]>/g, "").trim()
+      : null;
+
+    // Categories: all <category> elements in channel
+    const catMatches = [...channel.matchAll(/<category>(.*?)<\/category>/g)];
+    const categories =
+      catMatches.length > 0
+        ? catMatches.map((m) => m[1].replace(/<!\[CDATA\[|\]\]>/g, "").trim())
+        : null;
+
+    return {
+      ...entry,
+      description: description || entry.description,
+      categories: categories || entry.categories,
+    };
+  } catch {
+    // Network error, timeout, parse error — graceful degradation
+    return entry;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Importer
+// ---------------------------------------------------------------------------
+
+export async function importPodcasts(filePath: string): Promise<void> {
+  if (!existsSync(filePath)) {
+    console.error(`File not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  const content = readFileSync(filePath, "utf-8");
+  const { entries, skipped } = parseOPML(content);
+
+  // Enrich from RSS feeds (concurrent, best-effort)
+  let enrichedCount = 0;
+  const podcasts = await Promise.all(
+    entries.map(async (entry) => {
+      const enriched = await enrichFromRSS(entry);
+      if (enriched.description || enriched.categories) {
+        enrichedCount++;
+      }
+      return enriched;
+    }),
+  );
+
+  const config = getConfig();
+  const personalDir = config.paths.personal;
+  mkdirSafe(personalDir);
+
+  const outPath = join(personalDir, "podcasts.json");
+  atomicWrite(outPath, podcasts);
+
+  console.log(`Imported ${podcasts.length} podcasts \u2192 ${outPath}`);
+  if (enrichedCount > 0) {
+    console.log(
+      `Enriched ${enrichedCount} feeds (with description/categories)`,
+    );
+  }
+  if (skipped > 0) {
+    console.log(`Skipped ${skipped} entries (no title)`);
+  }
+}

package/lib/indexers/blogs.ts

@@ -35,8 +35,9 @@ function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
 
 export async function indexBlogs(ctx: IndexerContext): Promise<void> {
   const blogsDir = ctx.config.paths.blogs;
-  const postsDir = join(blogsDir, "content", "posts");
+  if (!checkPath("blogs", "paths.blogs", blogsDir)) return;
 
+  const postsDir = join(blogsDir, "content", "posts");
   if (!checkPath("blogs", "content/posts", postsDir)) return;
 
   if (!ctx.config.paths.blog_url) {

package/lib/indexers/personal.ts

@@ -38,16 +38,10 @@ Example: {"name":"Jade","relationship":"child"} → Jade is a child, a kid and o
 Example: {"name":"Sansa","relationship":"cat"} → Sansa is a cat, a pet and feline companion in the household.
 ${ENRICH_SHARED}`,
   book: `You are enriching a book entry for search indexing.
-Generate a natural language description starting with the book's title and author.
-Include genre, themes, and related topics naturally in the sentence.
-Example: {"title":"The Odyssey","author":"Homer"} → The Odyssey by Homer is an epic poem exploring journey, homecoming, fate, and loyalty through Greek mythology.
-Example: {"title":"Dune","author":"Frank Herbert"} → Dune by Frank Herbert is a sci-fi novel about power, ecology, religion, and survival on a desert planet.
+Generate: genre, themes, and related topics based on the title.
 ${ENRICH_SHARED}`,
   movie: `You are enriching a movie entry for search indexing.
-Generate a natural language description starting with the movie's title.
-Include genre, themes, and related topics naturally in the sentence.
-Example: {"title":"The Matrix","year":1999} → The Matrix (1999) is a sci-fi action film exploring reality, free will, and technology through cyberpunk themes.
-Example: {"title":"Pan's Labyrinth","year":2006} → Pan's Labyrinth (2006) is a dark fantasy drama about childhood, political oppression, and magical escape during the Spanish Civil War.
+Generate: genre, themes, and related topics based on the title.
 ${ENRICH_SHARED}`,
   interest: `You are enriching a personal interest entry for search indexing.
 Generate: related activities, domains, synonyms, and common alternative phrasings.
@@ -159,7 +153,7 @@ export async function indexPersonal(ctx: IndexerContext): Promise<void> {
           title: person.name,
           content,
           topic: "",
-          type: "contact",
+          type: "person",
           timestamp: peopleTs,
           metadata: { name: person.name },
         });

package/lib/init.ts

@@ -0,0 +1,254 @@
+/**
+ * lib/init.ts - lore init command
+ *
+ * Auto-detects environment, generates ~/.config/lore/config.toml,
+ * creates data directories, and initializes the database schema.
+ *
+ * IMPORTANT: This file must NOT import ./db or ./config to avoid
+ * bootstrap circularity — config.toml may not exist yet when this runs.
+ * Uses bun:sqlite and fs directly.
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { Database } from "bun:sqlite";
+
+interface DetectedPaths {
+  obsidian?: string;
+  explorations?: string;
+  projects?: string;
+  sableEvents?: string;
+  customSqlite?: string;
+  sqliteVec?: string;
+}
+
+export async function runInit(homeOverride?: string): Promise<void> {
+  console.log("lore init — detecting environment...");
+
+  const home = homeOverride ?? homedir();
+  const configDir = `${home}/.config/lore`;
+  const configPath = `${configDir}/config.toml`;
+  const dataDir = `${home}/.local/share/lore`;
+  const dbPath = `${dataDir}/lore.db`;
+
+  // 1. Create required directories
+  mkdirSync(configDir, { recursive: true });
+  mkdirSync(dataDir, { recursive: true });
+  mkdirSync(`${home}/.cache/lore`, { recursive: true });
+
+  // 2. Auto-detect paths
+  const detected = await detectPaths(home);
+
+  // 3. Report detection results
+  console.log("\nDetected paths:");
+  const reportPath = (label: string, value: string | undefined): void => {
+    if (value) {
+      console.log(`  \u2713 ${label}: ${value}`);
+    } else {
+      console.log(`  \u2717 ${label}: not detected`);
+    }
+  };
+  reportPath("obsidian", detected.obsidian);
+  reportPath("explorations", detected.explorations);
+  reportPath("projects", detected.projects);
+  reportPath("sable_events", detected.sableEvents);
+  reportPath("custom_sqlite", detected.customSqlite);
+  reportPath("sqlite_vec", detected.sqliteVec);
+
+  // 4. Generate or verify config.toml
+  if (existsSync(configPath)) {
+    console.log(`\nConfig exists: ${configPath} (skipping)`);
+  } else {
+    const toml = generateConfig(detected, dataDir, dbPath);
+    writeFileSync(configPath, toml, "utf-8");
+    console.log(`\nConfig created: ${configPath}`);
+  }
+
+  const missingPaths = [
+    !detected.obsidian && "obsidian",
+    !detected.explorations && "explorations",
+    !detected.projects && "projects",
+  ].filter(Boolean);
+  if (missingPaths.length > 0) {
+    console.log(
+      `\nTo configure missing paths, edit ${configPath} and add them under [paths].`,
+    );
+  }
+
+  // 5. Initialize or verify database
+  initDatabase(dbPath, detected.customSqlite, detected.sqliteVec);
+}
+
+async function detectPaths(home: string): Promise<DetectedPaths> {
+  const detected: DetectedPaths = {};
+
+  // Obsidian vault
+  const obsidianPath = `${home}/obsidian`;
+  if (existsSync(obsidianPath)) {
+    detected.obsidian = obsidianPath;
+  }
+
+  // Explorations
+  const explorationsPath = `${home}/obsidian/reference/technical/explorations`;
+  if (existsSync(explorationsPath)) {
+    detected.explorations = explorationsPath;
+  }
+
+  // Dev projects
+  const projectsPath = `${home}/development/projects`;
+  if (existsSync(projectsPath)) {
+    detected.projects = projectsPath;
+  }
+
+  // Sable events
+  const xdgDataHome = process.env.XDG_DATA_HOME ?? `${home}/.local/share`;
+  const sableEventsPath = `${xdgDataHome}/sable/events`;
+  if (existsSync(sableEventsPath)) {
+    detected.sableEvents = sableEventsPath;
+  }
+
+  // Homebrew custom SQLite (macOS)
+  const customSqlitePath = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib";
+  if (existsSync(customSqlitePath)) {
+    detected.customSqlite = customSqlitePath;
+  }
+
+  // sqlite-vec extension
+  detected.sqliteVec = detectSqliteVec();
+
+  return detected;
+}
+
+function detectSqliteVec(): string | undefined {
+  const ext = process.platform === "darwin" ? "dylib" : "so";
+
+  // Strategy 1: brew --prefix sqlite-vec
+  const result = Bun.spawnSync(["brew", "--prefix", "sqlite-vec"]);
+  if (result.exitCode === 0) {
+    const prefix = new TextDecoder().decode(result.stdout).trim();
+    const candidate = `${prefix}/lib/sqlite-vec/vec0.${ext}`;
+    if (existsSync(candidate)) return candidate;
+  }
+
+  // Strategy 2: npm-installed sqlite-vec (node_modules)
+  const platformPkg = `sqlite-vec-${process.platform}-${process.arch}`;
+  const nmCandidate = `${import.meta.dir}/../node_modules/${platformPkg}/vec0.${ext}`;
+  if (existsSync(nmCandidate)) return nmCandidate;
+
+  // Strategy 3: common macOS/Linux system paths
+  for (const p of [
+    "/opt/homebrew/lib/sqlite-vec/vec0.dylib",
+    "/usr/local/lib/sqlite-vec/vec0.dylib",
+    "/opt/homebrew/opt/sqlite-vec/lib/vec0.dylib",
+    "/usr/lib/sqlite-vec/vec0.so",
+    "/usr/local/lib/vec0.so",
+  ]) {
+    if (existsSync(p)) return p;
+  }
+
+  // Strategy 4: pip-installed sqlite-vec (Python site-packages)
+  const pipGlob =
+    process.platform === "darwin"
+      ? "/opt/homebrew/lib/python3.*/site-packages/sqlite_vec/vec0.dylib"
+      : "/usr/lib/python3*/dist-packages/sqlite_vec/vec0.so";
+  const glob = new Bun.Glob(pipGlob);
+  for (const match of glob.scanSync("/")) {
+    const fullPath = `/${match}`;
+    if (existsSync(fullPath)) return fullPath;
+  }
+
+  return undefined;
+}
+
+function generateConfig(
+  detected: DetectedPaths,
+  dataDir: string,
+  dbPath: string,
+): string {
+  const timestamp = new Date().toISOString().replace(/\.\d+Z$/, "Z");
+
+  const pathLine = (key: string, value: string | undefined): string => {
+    if (value) return `${key} = "${value}"`;
+    return `# ${key} = "" # not detected`;
+  };
+
+  return `# Lore Configuration (TypeScript indexers)
+# Generated: ${timestamp}
+# Paths that were not found on this machine are commented out.
+
+[paths]
+data = "${dataDir}"
+personal = "${dataDir}/personal"
+${pathLine("obsidian", detected.obsidian ? "~/obsidian" : undefined)}
+${pathLine("explorations", detected.explorations ? "~/obsidian/reference/technical/explorations" : undefined)}
+# blogs = "" # not detected
+# blog_url = "" # not detected
+${pathLine("projects", detected.projects ? "~/development/projects" : undefined)}
+${pathLine("sable_events", detected.sableEvents ? "~/.local/share/sable/events" : undefined)}
+
+[database]
+sqlite = "${dbPath}"
+${pathLine("custom_sqlite", detected.customSqlite)}
+${pathLine("sqlite_vec", detected.sqliteVec)}
+
+[embedding]
+model = "nomic-ai/nomic-embed-text-v1.5"
+dimensions = 768
+`;
+}
+
+function initDatabase(
+  dbPath: string,
+  customSqlite: string | undefined,
+  vecPath: string | undefined,
+): void {
+  if (customSqlite && existsSync(customSqlite)) {
+    try {
+      Database.setCustomSQLite(customSqlite);
+    } catch (e) {
+      if (!(e instanceof Error && e.message.includes("already loaded"))) {
+        throw e;
+      }
+    }
+  }
+
+  const db = new Database(dbPath);
+  db.exec("PRAGMA journal_mode=WAL");
+
+  // Load sqlite-vec for vec0 table creation
+  let vecLoaded = false;
+  if (vecPath && existsSync(vecPath)) {
+    try {
+      db.loadExtension(vecPath);
+      vecLoaded = true;
+    } catch (e) {
+      // Extension loading fails if custom_sqlite wasn't set (Bun's built-in
+      // sqlite doesn't support extensions). Warn instead of crashing.
+      console.warn(
+        `sqlite-vec found at ${vecPath} but extension loading not supported — need custom_sqlite`,
+      );
+    }
+  } else {
+    console.warn("sqlite-vec not found — embeddings table not created");
+  }
+
+  // Create tables (IF NOT EXISTS = idempotent)
+  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(
+    source, title, content, metadata, topic, type, timestamp UNINDEXED
+  )`);
+
+  if (vecLoaded) {
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+      doc_id INTEGER, chunk_idx INTEGER, source TEXT, topic TEXT, type TEXT,
+      timestamp TEXT, embedding float[768]
+    )`);
+  }
+
+  db.exec(`CREATE TABLE IF NOT EXISTS embedding_cache (
+    hash TEXT PRIMARY KEY, embedding BLOB NOT NULL, model TEXT NOT NULL,
+    dims INTEGER NOT NULL, created_at INTEGER NOT NULL
+  )`);
+
+  console.log(`Database verified: ${dbPath}`);
+  db.close();
+}

package/lib/utils.ts

@@ -0,0 +1,65 @@
+/**
+ * lib/utils.ts - Shared utilities for importers
+ *
+ * Provides atomic file writing, safe directory creation, and CSV parsing
+ * used by all importers in lib/importers/.
+ */
+
+import { writeFileSync, renameSync, mkdirSync } from "fs";
+import { tmpdir } from "os";
+
+// ---------------------------------------------------------------------------
+// CSV parsing (handles quoted fields with commas and escaped quotes)
+// ---------------------------------------------------------------------------
+
+export function parseCSVLine(line: string): string[] {
+  const result: string[] = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    if (line[i] === '"') {
+      if (inQuotes && line[i + 1] === '"') {
+        current += '"';
+        i++;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (line[i] === "," && !inQuotes) {
+      result.push(current);
+      current = "";
+    } else {
+      current += line[i];
+    }
+  }
+  result.push(current);
+  return result;
+}
+
+export function parseCSV(content: string): Record<string, string>[] {
+  const lines = content.split("\n");
+  const headers = parseCSVLine(lines[0]);
+  return lines
+    .slice(1)
+    .filter((line) => line.trim())
+    .map((line) => {
+      const values = parseCSVLine(line);
+      return Object.fromEntries(headers.map((h, i) => [h, values[i] ?? ""]));
+    });
+}
+
+/**
+ * Write data to a file atomically using temp-file + rename.
+ * Prevents partial writes if the process is interrupted.
+ */
+export function atomicWrite(filePath: string, data: unknown): void {
+  const tmp = `${tmpdir()}/lore-import-${Date.now()}-${Math.random().toString(36).slice(2)}.tmp`;
+  writeFileSync(tmp, JSON.stringify(data, null, 2), "utf-8");
+  renameSync(tmp, filePath);
+}
+
+/**
+ * Create a directory (and parents) if it does not exist.
+ */
+export function mkdirSafe(dir: string): void {
+  mkdirSync(dir, { recursive: true });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "1.8.5",
+  "version": "2.0.0",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",
@@ -37,19 +37,18 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/nickpending/llmcli-tools.git",
-    "directory": "packages/lore"
+    "url": "git+https://github.com/nickpending/lore.git"
   },
-  "homepage": "https://github.com/nickpending/llmcli-tools/tree/main/packages/lore#readme",
+  "homepage": "https://github.com/nickpending/lore#readme",
   "bugs": {
-    "url": "https://github.com/nickpending/llmcli-tools/issues"
+    "url": "https://github.com/nickpending/lore/issues"
   },
   "engines": {
     "bun": ">=1.0.0"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
-    "@voidwire/llm-core": "0.4.0"
+    "@voidwire/llm-core": "^0.4.0"
   },
   "devDependencies": {
     "bun-types": "1.3.5"

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Rudy Ruiz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/README.md

@@ -1,173 +0,0 @@
-# lore
-
-Unified knowledge CLI — search, list, and capture your indexed knowledge fabric.
-
-## Philosophy
-
-- **Unified** — Single entry point for all knowledge operations
-- **Library-first** — Import functions directly, CLI is a thin wrapper
-- **Composable** — JSON output pipes to jq, grep, other Unix tools
-- **Zero duplication** — Re-exports from lore-search and lore-capture
-
-## Installation
-
-```bash
-cd llmcli-tools/packages/lore
-bun link
-```
-
-## CLI Usage
-
-```bash
-lore search <query>                    # Search all sources
-lore search <source> <query>           # Search specific source
-lore search --sources                  # List indexed sources
-
-lore list <domain>                     # List domain entries
-lore list --domains                    # List available domains
-
-lore capture task|knowledge|note       # Capture knowledge
-```
-
-### Search Options
-
-- `--limit <n>` — Maximum results (default: 20)
-- `--since <date>` — Filter by date (today, yesterday, this-week, YYYY-MM-DD)
-- `--sources` — List indexed sources with counts
-
-### Passthrough Sources
-
-Some sources query external services rather than the local index:
-
-```bash
-lore search prismis "kubernetes security"    # Semantic search via prismis
-```
-
-| Source | Description | Requires |
-|--------|-------------|----------|
-| `prismis` | Semantic search across saved articles | prismis-daemon running |
-
-Passthrough sources appear in `lore search --sources` with `type: "passthrough"`.
-
-### List Options
-
-- `--limit <n>` — Maximum entries
-- `--format <fmt>` — Output format: json (default), jsonl, human
-- `--domains` — List available domains
-
-### Capture Types
-
-```bash
-# Task completion
-lore capture task --project=myproject --name="Task name" \
-  --problem="What was solved" --solution="How it was solved"
-
-# Knowledge insight
-lore capture knowledge --context=myproject \
-  --text="Insight learned" --type=learning
-
-# Quick note
-lore capture note --text="Remember this" --tags=tag1,tag2
-```
-
-## Library Usage
-
-The real power is programmatic access:
-
-```typescript
-import {
-  // Search (from lore-search)
-  search,
-  listSources,
-  type SearchResult,
-  type SearchOptions,
-
-  // List (local)
-  list,
-  listDomains,
-  DOMAINS,
-  type Domain,
-  type ListResult,
-
-  // Capture (from lore-capture)
-  captureKnowledge,
-  captureTask,
-  captureNote,
-  type KnowledgeInput,
-  type TaskInput,
-  type NoteInput,
-} from "lore";
-
-// Search
-const results = search("authentication", { limit: 10 });
-
-// List
-const devProjects = list("development");
-
-// Capture
-captureKnowledge({
-  context: "myproject",
-  text: "Important insight",
-  type: "learning",
-});
-```
-
-## Domains
-
-15 domains available for `lore list`:
-
-| Domain | Description |
-|--------|-------------|
-| development | Development projects |
-| tasks | Flux tasks and ideas |
-| events | Events by project |
-| blogs | Blog posts |
-| commits | Git commits |
-| explorations | Project explorations |
-| readmes | Project READMEs |
-| obsidian | Obsidian vault notes |
-| captures | Quick captures |
-| books | Books read |
-| movies | Movies watched |
-| podcasts | Podcast subscriptions |
-| interests | Personal interests |
-| people | People and relationships |
-| habits | Habit tracking |
-
-## Knowledge Types
-
-For `lore capture knowledge --type`:
-
-- `decision` — Architectural or design decisions
-- `learning` — Something learned during work
-- `gotcha` — Pitfall or gotcha to remember
-- `preference` — User preference discovered
-- `project` — Project-level insight
-- `conversation` — Insight from conversation
-- `knowledge` — General knowledge
-
-## Architecture
-
-```
-lore/
-├── index.ts              # Re-exports from lib/
-├── cli.ts                # Unified CLI (search|list|capture)
-├── lib/
-│   ├── search.ts         # FTS5 search (SQLite)
-│   ├── list.ts           # Domain listing
-│   └── capture.ts        # JSONL capture
-└── package.json          # Zero dependencies
-```
-
-Self-contained package. No workspace dependencies. Ready for npm publish.
-
-## Data Locations
-
-- `~/.local/share/lore/lore.db` — SQLite FTS5 database (search, list)
-- `~/.local/share/lore/log.jsonl` — Capture event log
-
-## Exit Codes
-
-- `0` — Success
-- `1` — Validation error (missing args, invalid domain)
-- `2` — Runtime error (database not found)