@voidwire/lore 1.8.6 → 2.0.1

package/lib/importers/podcasts.ts

@@ -0,0 +1,151 @@
+/**
+ * lib/importers/podcasts.ts - OPML podcast importer with RSS enrichment
+ *
+ * Reads a podcast OPML export, extracts feed URLs and titles,
+ * then optionally enriches each entry by fetching its RSS feed
+ * for description and categories. Writes podcasts.json to the
+ * personal data directory.
+ *
+ * Output schema matches what lib/indexers/personal.ts reads.
+ */
+
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import { getConfig } from "../config";
+import { atomicWrite, mkdirSafe } from "../utils";
+
+// ---------------------------------------------------------------------------
+// OPML parsing (regex-based, no XML library needed)
+// ---------------------------------------------------------------------------
+
+interface PodcastEntry {
+  title: string;
+  url: string;
+  description: string | null;
+  categories: string[] | null;
+}
+
+function extractAttr(chunk: string, attr: string): string | null {
+  // Match attr="value" — handles both single and double quotes
+  const re = new RegExp(`${attr}=["']([^"']*)["']`);
+  const m = chunk.match(re);
+  return m ? m[1] : null;
+}
+
+function parseOPML(content: string): {
+  entries: PodcastEntry[];
+  skipped: number;
+} {
+  const entries: PodcastEntry[] = [];
+  let skipped = 0;
+
+  // Split on <outline to get each outline element as a chunk
+  const chunks = content.split(/<outline\b/);
+
+  for (let i = 1; i < chunks.length; i++) {
+    const chunk = chunks[i];
+
+    const xmlUrl = extractAttr(chunk, "xmlUrl");
+    // Skip folder/category nodes (no xmlUrl)
+    if (!xmlUrl) continue;
+
+    const title = extractAttr(chunk, "text") ?? extractAttr(chunk, "title");
+    if (!title) {
+      skipped++;
+      continue;
+    }
+
+    entries.push({
+      title,
+      url: xmlUrl,
+      description: null,
+      categories: null,
+    });
+  }
+
+  return { entries, skipped };
+}
+
+// ---------------------------------------------------------------------------
+// RSS enrichment (best-effort, per-feed)
+// ---------------------------------------------------------------------------
+
+async function enrichFromRSS(entry: PodcastEntry): Promise<PodcastEntry> {
+  try {
+    const resp = await fetch(entry.url, {
+      signal: AbortSignal.timeout(5000),
+      headers: { "User-Agent": "lore-import-podcasts/1.0" },
+    });
+    const xml = await resp.text();
+
+    // Extract channel block (RSS 2.0)
+    const channelMatch = xml.match(/<channel>([\s\S]*?)<\/channel>/);
+    if (!channelMatch) return entry;
+    const channel = channelMatch[1];
+
+    // Description: first <description> in channel
+    const descMatch = channel.match(/<description>([\s\S]*?)<\/description>/);
+    const description = descMatch
+      ? descMatch[1].replace(/<!\[CDATA\[|\]\]>/g, "").trim()
+      : null;
+
+    // Categories: all <category> elements in channel
+    const catMatches = [...channel.matchAll(/<category>(.*?)<\/category>/g)];
+    const categories =
+      catMatches.length > 0
+        ? catMatches.map((m) => m[1].replace(/<!\[CDATA\[|\]\]>/g, "").trim())
+        : null;
+
+    return {
+      ...entry,
+      description: description || entry.description,
+      categories: categories || entry.categories,
+    };
+  } catch {
+    // Network error, timeout, parse error — graceful degradation
+    return entry;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Importer
+// ---------------------------------------------------------------------------
+
+export async function importPodcasts(filePath: string): Promise<void> {
+  if (!existsSync(filePath)) {
+    console.error(`File not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  const content = readFileSync(filePath, "utf-8");
+  const { entries, skipped } = parseOPML(content);
+
+  // Enrich from RSS feeds (concurrent, best-effort)
+  let enrichedCount = 0;
+  const podcasts = await Promise.all(
+    entries.map(async (entry) => {
+      const enriched = await enrichFromRSS(entry);
+      if (enriched.description || enriched.categories) {
+        enrichedCount++;
+      }
+      return enriched;
+    }),
+  );
+
+  const config = getConfig();
+  const personalDir = config.paths.personal;
+  mkdirSafe(personalDir);
+
+  const outPath = join(personalDir, "podcasts.json");
+  atomicWrite(outPath, podcasts);
+
+  console.log(`Imported ${podcasts.length} podcasts \u2192 ${outPath}`);
+  if (enrichedCount > 0) {
+    console.log(
+      `Enriched ${enrichedCount} feeds (with description/categories)`,
+    );
+  }
+  if (skipped > 0) {
+    console.log(`Skipped ${skipped} entries (no title)`);
+  }
+}

package/lib/indexers/blogs.ts

@@ -35,8 +35,9 @@ function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
 
 export async function indexBlogs(ctx: IndexerContext): Promise<void> {
   const blogsDir = ctx.config.paths.blogs;
-  const postsDir = join(blogsDir, "content", "posts");
+  if (!checkPath("blogs", "paths.blogs", blogsDir)) return;
 
+  const postsDir = join(blogsDir, "content", "posts");
   if (!checkPath("blogs", "content/posts", postsDir)) return;
 
   if (!ctx.config.paths.blog_url) {

package/lib/indexers/personal.ts

@@ -38,28 +38,16 @@ Example: {"name":"Jade","relationship":"child"} → Jade is a child, a kid and o
 Example: {"name":"Sansa","relationship":"cat"} → Sansa is a cat, a pet and feline companion in the household.
 ${ENRICH_SHARED}`,
   book: `You are enriching a book entry for search indexing.
-Generate a natural language description starting with the book's title and author.
-Include genre, themes, and related topics naturally in the sentence.
-Example: {"title":"The Odyssey","author":"Homer"} → The Odyssey by Homer is an epic poem exploring journey, homecoming, fate, and loyalty through Greek mythology.
-Example: {"title":"Dune","author":"Frank Herbert"} → Dune by Frank Herbert is a sci-fi novel about power, ecology, religion, and survival on a desert planet.
+Generate: genre, themes, and related topics based on the title.
 ${ENRICH_SHARED}`,
   movie: `You are enriching a movie entry for search indexing.
-Generate a natural language description starting with the movie's title.
-Include genre, themes, and related topics naturally in the sentence.
-Example: {"title":"The Matrix","year":1999} → The Matrix (1999) is a sci-fi action film exploring reality, free will, and technology through cyberpunk themes.
-Example: {"title":"Pan's Labyrinth","year":2006} → Pan's Labyrinth (2006) is a dark fantasy drama about childhood, political oppression, and magical escape during the Spanish Civil War.
+Generate: genre, themes, and related topics based on the title.
 ${ENRICH_SHARED}`,
   interest: `You are enriching a personal interest entry for search indexing.
-Generate a natural language description starting with the interest name.
-Include related activities, domains, and common alternative phrasings naturally in the sentence.
-Example: {"name":"woodworking"} → Woodworking is a hands-on craft involving carpentry, furniture making, and wood carving using hand tools and power tools.
-Example: {"name":"cybersecurity"} → Cybersecurity is a technical field covering network security, penetration testing, threat analysis, and digital defense.
+Generate: related activities, domains, synonyms, and common alternative phrasings.
 ${ENRICH_SHARED}`,
   habit: `You are enriching a personal habit/routine entry for search indexing.
-Generate a natural language description starting with the habit name.
-Include frequency, related routines, and benefits naturally in the sentence.
-Example: {"habit":"morning meditation","frequency":"daily"} → Morning meditation is a daily mindfulness practice involving focused breathing and mental clarity exercises.
-Example: {"habit":"meal prep","frequency":"weekly"} → Meal prep is a weekly cooking routine involving batch preparation of meals for the upcoming days.
+Generate: related routines, synonyms, categories, and common alternative phrasings.
 ${ENRICH_SHARED}`,
 };
 
@@ -165,7 +153,7 @@ export async function indexPersonal(ctx: IndexerContext): Promise<void> {
           title: person.name,
           content,
           topic: "",
-          type: "contact",
+          type: "person",
           timestamp: peopleTs,
           metadata: { name: person.name },
         });

package/lib/init.ts

@@ -0,0 +1,254 @@
+/**
+ * lib/init.ts - lore init command
+ *
+ * Auto-detects environment, generates ~/.config/lore/config.toml,
+ * creates data directories, and initializes the database schema.
+ *
+ * IMPORTANT: This file must NOT import ./db or ./config to avoid
+ * bootstrap circularity — config.toml may not exist yet when this runs.
+ * Uses bun:sqlite and fs directly.
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { Database } from "bun:sqlite";
+
+interface DetectedPaths {
+  obsidian?: string;
+  explorations?: string;
+  projects?: string;
+  sableEvents?: string;
+  customSqlite?: string;
+  sqliteVec?: string;
+}
+
+export async function runInit(homeOverride?: string): Promise<void> {
+  console.log("lore init — detecting environment...");
+
+  const home = homeOverride ?? homedir();
+  const configDir = `${home}/.config/lore`;
+  const configPath = `${configDir}/config.toml`;
+  const dataDir = `${home}/.local/share/lore`;
+  const dbPath = `${dataDir}/lore.db`;
+
+  // 1. Create required directories
+  mkdirSync(configDir, { recursive: true });
+  mkdirSync(dataDir, { recursive: true });
+  mkdirSync(`${home}/.cache/lore`, { recursive: true });
+
+  // 2. Auto-detect paths
+  const detected = await detectPaths(home);
+
+  // 3. Report detection results
+  console.log("\nDetected paths:");
+  const reportPath = (label: string, value: string | undefined): void => {
+    if (value) {
+      console.log(`  \u2713 ${label}: ${value}`);
+    } else {
+      console.log(`  \u2717 ${label}: not detected`);
+    }
+  };
+  reportPath("obsidian", detected.obsidian);
+  reportPath("explorations", detected.explorations);
+  reportPath("projects", detected.projects);
+  reportPath("sable_events", detected.sableEvents);
+  reportPath("custom_sqlite", detected.customSqlite);
+  reportPath("sqlite_vec", detected.sqliteVec);
+
+  // 4. Generate or verify config.toml
+  if (existsSync(configPath)) {
+    console.log(`\nConfig exists: ${configPath} (skipping)`);
+  } else {
+    const toml = generateConfig(detected, dataDir, dbPath);
+    writeFileSync(configPath, toml, "utf-8");
+    console.log(`\nConfig created: ${configPath}`);
+  }
+
+  const missingPaths = [
+    !detected.obsidian && "obsidian",
+    !detected.explorations && "explorations",
+    !detected.projects && "projects",
+  ].filter(Boolean);
+  if (missingPaths.length > 0) {
+    console.log(
+      `\nTo configure missing paths, edit ${configPath} and add them under [paths].`,
+    );
+  }
+
+  // 5. Initialize or verify database
+  initDatabase(dbPath, detected.customSqlite, detected.sqliteVec);
+}
+
+async function detectPaths(home: string): Promise<DetectedPaths> {
+  const detected: DetectedPaths = {};
+
+  // Obsidian vault
+  const obsidianPath = `${home}/obsidian`;
+  if (existsSync(obsidianPath)) {
+    detected.obsidian = obsidianPath;
+  }
+
+  // Explorations
+  const explorationsPath = `${home}/obsidian/reference/technical/explorations`;
+  if (existsSync(explorationsPath)) {
+    detected.explorations = explorationsPath;
+  }
+
+  // Dev projects
+  const projectsPath = `${home}/development/projects`;
+  if (existsSync(projectsPath)) {
+    detected.projects = projectsPath;
+  }
+
+  // Sable events
+  const xdgDataHome = process.env.XDG_DATA_HOME ?? `${home}/.local/share`;
+  const sableEventsPath = `${xdgDataHome}/sable/events`;
+  if (existsSync(sableEventsPath)) {
+    detected.sableEvents = sableEventsPath;
+  }
+
+  // Homebrew custom SQLite (macOS)
+  const customSqlitePath = "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib";
+  if (existsSync(customSqlitePath)) {
+    detected.customSqlite = customSqlitePath;
+  }
+
+  // sqlite-vec extension
+  detected.sqliteVec = detectSqliteVec();
+
+  return detected;
+}
+
+function detectSqliteVec(): string | undefined {
+  const ext = process.platform === "darwin" ? "dylib" : "so";
+
+  // Strategy 1: brew --prefix sqlite-vec
+  const result = Bun.spawnSync(["brew", "--prefix", "sqlite-vec"]);
+  if (result.exitCode === 0) {
+    const prefix = new TextDecoder().decode(result.stdout).trim();
+    const candidate = `${prefix}/lib/sqlite-vec/vec0.${ext}`;
+    if (existsSync(candidate)) return candidate;
+  }
+
+  // Strategy 2: npm-installed sqlite-vec (node_modules)
+  const platformPkg = `sqlite-vec-${process.platform}-${process.arch}`;
+  const nmCandidate = `${import.meta.dir}/../node_modules/${platformPkg}/vec0.${ext}`;
+  if (existsSync(nmCandidate)) return nmCandidate;
+
+  // Strategy 3: common macOS/Linux system paths
+  for (const p of [
+    "/opt/homebrew/lib/sqlite-vec/vec0.dylib",
+    "/usr/local/lib/sqlite-vec/vec0.dylib",
+    "/opt/homebrew/opt/sqlite-vec/lib/vec0.dylib",
+    "/usr/lib/sqlite-vec/vec0.so",
+    "/usr/local/lib/vec0.so",
+  ]) {
+    if (existsSync(p)) return p;
+  }
+
+  // Strategy 4: pip-installed sqlite-vec (Python site-packages)
+  const pipGlob =
+    process.platform === "darwin"
+      ? "/opt/homebrew/lib/python3.*/site-packages/sqlite_vec/vec0.dylib"
+      : "/usr/lib/python3*/dist-packages/sqlite_vec/vec0.so";
+  const glob = new Bun.Glob(pipGlob);
+  for (const match of glob.scanSync("/")) {
+    const fullPath = `/${match}`;
+    if (existsSync(fullPath)) return fullPath;
+  }
+
+  return undefined;
+}
+
+function generateConfig(
+  detected: DetectedPaths,
+  dataDir: string,
+  dbPath: string,
+): string {
+  const timestamp = new Date().toISOString().replace(/\.\d+Z$/, "Z");
+
+  const pathLine = (key: string, value: string | undefined): string => {
+    if (value) return `${key} = "${value}"`;
+    return `# ${key} = "" # not detected`;
+  };
+
+  return `# Lore Configuration (TypeScript indexers)
+# Generated: ${timestamp}
+# Paths that were not found on this machine are commented out.
+
+[paths]
+data = "${dataDir}"
+personal = "${dataDir}/personal"
+${pathLine("obsidian", detected.obsidian ? "~/obsidian" : undefined)}
+${pathLine("explorations", detected.explorations ? "~/obsidian/reference/technical/explorations" : undefined)}
+# blogs = "" # not detected
+# blog_url = "" # not detected
+${pathLine("projects", detected.projects ? "~/development/projects" : undefined)}
+${pathLine("sable_events", detected.sableEvents ? "~/.local/share/sable/events" : undefined)}
+
+[database]
+sqlite = "${dbPath}"
+${pathLine("custom_sqlite", detected.customSqlite)}
+${pathLine("sqlite_vec", detected.sqliteVec)}
+
+[embedding]
+model = "nomic-ai/nomic-embed-text-v1.5"
+dimensions = 768
+`;
+}
+
+function initDatabase(
+  dbPath: string,
+  customSqlite: string | undefined,
+  vecPath: string | undefined,
+): void {
+  if (customSqlite && existsSync(customSqlite)) {
+    try {
+      Database.setCustomSQLite(customSqlite);
+    } catch (e) {
+      if (!(e instanceof Error && e.message.includes("already loaded"))) {
+        throw e;
+      }
+    }
+  }
+
+  const db = new Database(dbPath);
+  db.exec("PRAGMA journal_mode=WAL");
+
+  // Load sqlite-vec for vec0 table creation
+  let vecLoaded = false;
+  if (vecPath && existsSync(vecPath)) {
+    try {
+      db.loadExtension(vecPath);
+      vecLoaded = true;
+    } catch (e) {
+      // Extension loading fails if custom_sqlite wasn't set (Bun's built-in
+      // sqlite doesn't support extensions). Warn instead of crashing.
+      console.warn(
+        `sqlite-vec found at ${vecPath} but extension loading not supported — need custom_sqlite`,
+      );
+    }
+  } else {
+    console.warn("sqlite-vec not found — embeddings table not created");
+  }
+
+  // Create tables (IF NOT EXISTS = idempotent)
+  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(
+    source, title, content, metadata, topic, type, timestamp UNINDEXED
+  )`);
+
+  if (vecLoaded) {
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+      doc_id INTEGER, chunk_idx INTEGER, source TEXT, topic TEXT, type TEXT,
+      timestamp TEXT, embedding float[768]
+    )`);
+  }
+
+  db.exec(`CREATE TABLE IF NOT EXISTS embedding_cache (
+    hash TEXT PRIMARY KEY, embedding BLOB NOT NULL, model TEXT NOT NULL,
+    dims INTEGER NOT NULL, created_at INTEGER NOT NULL
+  )`);
+
+  console.log(`Database verified: ${dbPath}`);
+  db.close();
+}

package/lib/utils.ts

@@ -0,0 +1,65 @@
+/**
+ * lib/utils.ts - Shared utilities for importers
+ *
+ * Provides atomic file writing, safe directory creation, and CSV parsing
+ * used by all importers in lib/importers/.
+ */
+
+import { writeFileSync, renameSync, mkdirSync } from "fs";
+import { tmpdir } from "os";
+
+// ---------------------------------------------------------------------------
+// CSV parsing (handles quoted fields with commas and escaped quotes)
+// ---------------------------------------------------------------------------
+
+export function parseCSVLine(line: string): string[] {
+  const result: string[] = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    if (line[i] === '"') {
+      if (inQuotes && line[i + 1] === '"') {
+        current += '"';
+        i++;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (line[i] === "," && !inQuotes) {
+      result.push(current);
+      current = "";
+    } else {
+      current += line[i];
+    }
+  }
+  result.push(current);
+  return result;
+}
+
+export function parseCSV(content: string): Record<string, string>[] {
+  const lines = content.split("\n");
+  const headers = parseCSVLine(lines[0]);
+  return lines
+    .slice(1)
+    .filter((line) => line.trim())
+    .map((line) => {
+      const values = parseCSVLine(line);
+      return Object.fromEntries(headers.map((h, i) => [h, values[i] ?? ""]));
+    });
+}
+
+/**
+ * Write data to a file atomically using temp-file + rename.
+ * Prevents partial writes if the process is interrupted.
+ */
+export function atomicWrite(filePath: string, data: unknown): void {
+  const tmp = `${tmpdir()}/lore-import-${Date.now()}-${Math.random().toString(36).slice(2)}.tmp`;
+  writeFileSync(tmp, JSON.stringify(data, null, 2), "utf-8");
+  renameSync(tmp, filePath);
+}
+
+/**
+ * Create a directory (and parents) if it does not exist.
+ */
+export function mkdirSafe(dir: string): void {
+  mkdirSync(dir, { recursive: true });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "1.8.6",
+  "version": "2.0.1",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",
@@ -37,19 +37,18 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/nickpending/llmcli-tools.git",
-    "directory": "packages/lore"
+    "url": "git+https://github.com/nickpending/lore.git"
   },
-  "homepage": "https://github.com/nickpending/llmcli-tools/tree/main/packages/lore#readme",
+  "homepage": "https://github.com/nickpending/lore#readme",
   "bugs": {
-    "url": "https://github.com/nickpending/llmcli-tools/issues"
+    "url": "https://github.com/nickpending/lore/issues"
   },
   "engines": {
     "bun": ">=1.0.0"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
-    "@voidwire/llm-core": "0.4.0"
+    "@voidwire/llm-core": "^0.4.0"
   },
   "devDependencies": {
     "bun-types": "1.3.5"

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Rudy Ruiz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.