@voidwire/lore 2.0.0 → 2.1.0

package/cli.ts

@@ -61,8 +61,10 @@ import {
   type PurgeableSource,
   type ContradictionDecision,
 } from "./index";
+import { findStaleEntries, applyAging, type StaleEntry } from "./lib/age.js";
 import { isValidLoreType, LORE_TYPES } from "./lib/types";
-import { runIndexer } from "./lib/indexer";
+import { runIndexer, getNestedValue } from "./lib/indexer";
+import { getConfig } from "./lib/config";
 import { indexers } from "./lib/indexers/index";
 import { runInit } from "./lib/init";
 import { runEmbed } from "./lib/embed";
@@ -155,6 +157,8 @@ const BOOLEAN_FLAGS = new Set([
   "list",
   "rebuild",
   "force",
+  "dry-run",
+  "apply",
 ]);
 
 function getPositionalArgs(args: string[]): string[] {
@@ -819,6 +823,113 @@ Examples:
   process.exit(0);
 }
 
+// ============================================================================
+// Age Command
+// ============================================================================
+
+async function handleAge(args: string[]): Promise<void> {
+  if (hasFlag(args, "help")) {
+    showAgeHelp();
+  }
+
+  const dryRun = hasFlag(args, "dry-run") || !hasFlag(args, "apply");
+
+  try {
+    const config = getConfig();
+    const staleEntries = findStaleEntries(config);
+
+    if (staleEntries.length === 0) {
+      output({ success: true, stale: 0, message: "Nothing to age" });
+      console.error("Nothing to age — no entries exceed their TTL.");
+      process.exit(0);
+    }
+
+    // Group by source/type for display
+    const bySource: Record<string, Record<string, number>> = {};
+    for (const entry of staleEntries) {
+      if (!bySource[entry.source]) bySource[entry.source] = {};
+      const typeKey = entry.type || "(no type)";
+      bySource[entry.source][typeKey] =
+        (bySource[entry.source][typeKey] || 0) + 1;
+    }
+
+    // Print summary to stderr
+    console.error(`\nStale entries (${staleEntries.length} total):\n`);
+    for (const [source, types] of Object.entries(bySource)) {
+      const sourceTotal = Object.values(types).reduce((a, b) => a + b, 0);
+      console.error(`  ${source} (${sourceTotal}):`);
+      for (const [type, count] of Object.entries(types)) {
+        console.error(`    ${type}: ${count}`);
+      }
+    }
+    console.error("");
+
+    if (dryRun) {
+      output({
+        success: true,
+        dryRun: true,
+        stale: staleEntries.length,
+        bySource,
+      });
+      console.error("Dry run — no entries deleted. Use --apply to execute.");
+      process.exit(0);
+    }
+
+    // Apply aging
+    const deleted = applyAging(staleEntries);
+    output({
+      success: true,
+      dryRun: false,
+      deleted,
+    });
+    console.error(`Aged ${deleted} entries from search + embeddings.`);
+    process.exit(0);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    fail(message, 2);
+  }
+}
+
+function showAgeHelp(): void {
+  console.log(`
+lore age - Remove stale purgeable entries based on TTL
+
+Usage:
+  lore age                  Preview stale entries (dry run, default)
+  lore age --dry-run        Preview stale entries without deleting
+  lore age --apply          Delete stale entries
+
+Purgeable Sources:
+  captures          Quick captures (knowledge, decisions, gotchas, etc.)
+  observations      Model observations about user patterns
+  teachings         Teaching moments
+
+Non-purgeable sources (blogs, commits, obsidian, etc.) are never affected.
+
+TTL Defaults (days):
+  observations:   inferred=180, stated=365, verified=365
+  captures:       gotcha=365, learning=365, decision=730,
+                  preference=730, pattern=365, context=180, general=365
+  teachings:      365
+
+Override defaults in config.toml:
+  [aging]
+  observations.inferred = 180
+  captures.decision = 730
+  teachings = 365
+
+Options:
+  --dry-run       Preview only (default if neither flag given)
+  --apply         Execute deletions
+  --help          Show this help
+
+Examples:
+  lore age                    # Preview stale entries
+  lore age --apply            # Delete stale entries
+`);
+  process.exit(0);
+}
+
 // ============================================================================
 // Capture Command
 // ============================================================================
@@ -1059,13 +1170,31 @@ async function handleIndex(args: string[]): Promise<void> {
   }
 
   if (hasFlag(args, "list")) {
-    console.log("Registered indexers:");
-    const names = Object.keys(indexers);
-    if (names.length === 0) {
-      console.log("  (none)");
-    } else {
-      names.forEach((name) => console.log(`  - ${name}`));
-    }
+    const config = getConfig();
+    const pluginList = Object.values(indexers).map((plugin) => {
+      const available = plugin.canRun(config);
+      const missingKeys = available
+        ? null
+        : plugin.manifest.requiredConfig
+            .filter(
+              (k) =>
+                !getNestedValue(
+                  config as unknown as Record<string, unknown>,
+                  k,
+                ),
+            )
+            .join(", ");
+      return {
+        name: plugin.manifest.name,
+        description: plugin.manifest.description,
+        requiredConfig: plugin.manifest.requiredConfig,
+        optionalConfig: plugin.manifest.optionalConfig,
+        rebuildExcluded: plugin.manifest.rebuildExcluded,
+        available,
+        reason: missingKeys ? `missing: ${missingKeys}` : null,
+      };
+    });
+    output({ success: true, indexers: pluginList });
     process.exit(0);
   }
 
@@ -1130,6 +1259,7 @@ Usage:
   lore about <project> --brief          Compact project summary
   lore capture task|knowledge|note|teaching  Capture knowledge
   lore purge --match "content"              Delete matching entries
+  lore age [--dry-run] [--apply]            Age out stale entries by TTL
   lore index [source] [--rebuild] [--list]  Run indexers
 
 Search Options:
@@ -1520,6 +1650,9 @@ async function main(): Promise<void> {
     case "purge":
       await handlePurge(commandArgs);
       break;
+    case "age":
+      await handleAge(commandArgs);
+      break;
     case "init":
       await runInit();
       break;
@@ -1534,7 +1667,7 @@ async function main(): Promise<void> {
       break;
     default:
       fail(
-        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, purge, index, init, embed, or import`,
+        `Unknown command: ${command}. Use: search, list, sources, info, projects, about, capture, purge, age, index, init, embed, or import`,
       );
   }
 }

package/lib/age.ts

@@ -0,0 +1,162 @@
+/**
+ * lib/age.ts - TTL-based aging for purgeable knowledge entries
+ *
+ * Finds and deletes stale entries from purgeable sources (captures, observations, teachings)
+ * based on per-source/subtype TTL defaults with config.toml overrides.
+ *
+ * Usage:
+ *   const stale = findStaleEntries(config);
+ *   const deleted = applyAging(stale);
+ */
+
+import { openDatabaseBasic } from "./db.js";
+import { deleteEntries } from "./purge.js";
+import { indexers } from "./indexers/index.js";
+import type { LoreConfig } from "./config.js";
+
+// Purgeable sources derived from indexer manifests (rebuildExcluded === true)
+const PURGEABLE_SOURCES = Object.entries(indexers)
+  .filter(([, plugin]) => plugin.manifest.rebuildExcluded)
+  .map(([key]) => key);
+
+// Default TTLs (days) — conservative, override via config.toml [aging]
+const DEFAULT_TTLS: Record<string, Record<string, number>> = {
+  observations: { inferred: 180, stated: 365, verified: 365 },
+  captures: {
+    gotcha: 365,
+    learning: 365,
+    decision: 730,
+    preference: 730,
+    pattern: 365,
+    context: 180,
+    general: 365,
+  },
+  teachings: { teaching: 365 },
+};
+
+export interface StaleEntry {
+  rowid: number;
+  source: string;
+  type: string;
+  topic: string;
+  timestamp: string;
+  ageDays: number;
+  ttlDays: number;
+}
+
+/**
+ * Resolve TTL for a given source/type using config overrides with DEFAULT_TTLS fallback.
+ *
+ * Returns null if:
+ * - Source is not in DEFAULT_TTLS (safety guard — non-purgeable sources skipped)
+ * - TTL is 0 (never-age sentinel)
+ */
+function resolveTtl(
+  source: string,
+  type: string,
+  config: LoreConfig,
+): number | null {
+  // Safety guard: skip sources not in DEFAULT_TTLS (non-purgeable)
+  if (!DEFAULT_TTLS[source]) return null;
+
+  // Check config override first
+  if (config.aging) {
+    if (source === "teachings") {
+      // teachings config is a flat number, not nested
+      const val = config.aging.teachings;
+      if (typeof val === "number") {
+        return val === 0 ? null : val;
+      }
+    } else {
+      // observations and captures are nested objects
+      const sourceConfig = config.aging[source as keyof typeof config.aging];
+      if (sourceConfig && typeof sourceConfig === "object") {
+        const typeConfig = (sourceConfig as Record<string, unknown>)[type];
+        if (typeof typeConfig === "number") {
+          return typeConfig === 0 ? null : typeConfig;
+        }
+      }
+    }
+  }
+
+  // Fall back to DEFAULT_TTLS
+  const sourceDefaults = DEFAULT_TTLS[source];
+  const ttl = sourceDefaults[type];
+  if (ttl === undefined) return null;
+  return ttl === 0 ? null : ttl;
+}
+
+/**
+ * Find stale entries from purgeable sources based on TTL.
+ *
+ * Queries only captures, observations, teachings (purgeable sources).
+ * Compares entry timestamp against TTL — entries older than their TTL are stale.
+ */
+export function findStaleEntries(config: LoreConfig): StaleEntry[] {
+  const db = openDatabaseBasic(true);
+
+  try {
+    const placeholders = PURGEABLE_SOURCES.map(() => "?").join(", ");
+    const sql = `
+      SELECT rowid, source, type, topic, timestamp
+      FROM search
+      WHERE source IN (${placeholders})
+        AND timestamp IS NOT NULL
+        AND timestamp != ''
+      ORDER BY timestamp ASC
+    `;
+
+    const rows = db.prepare(sql).all(...PURGEABLE_SOURCES) as Array<{
+      rowid: number;
+      source: string;
+      type: string;
+      topic: string;
+      timestamp: string;
+    }>;
+
+    const now = Date.now();
+    const stale: StaleEntry[] = [];
+
+    for (const row of rows) {
+      const parsed = new Date(row.timestamp).getTime();
+      if (isNaN(parsed)) continue; // Malformed timestamp — skip
+
+      const ageDays = (now - parsed) / 86400000;
+      if (ageDays <= 0) continue; // Future timestamp — skip
+
+      const ttlDays = resolveTtl(row.source, row.type, config);
+      if (ttlDays === null) continue; // No TTL or non-purgeable — skip
+
+      if (ageDays > ttlDays) {
+        stale.push({
+          rowid: row.rowid,
+          source: row.source,
+          type: row.type,
+          topic: row.topic,
+          timestamp: row.timestamp,
+          ageDays: Math.floor(ageDays),
+          ttlDays,
+        });
+      }
+    }
+
+    return stale;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Delete stale entries via deleteEntries() from purge.ts.
+ *
+ * No matchContents provided — log.jsonl cleanup is skipped for TTL-based aging.
+ *
+ * @returns Count of deleted entries
+ */
+export function applyAging(staleEntries: StaleEntry[]): number {
+  if (staleEntries.length === 0) return 0;
+
+  const rowids = staleEntries.map((e) => e.rowid);
+  const result = deleteEntries(rowids);
+  return result.deleted;
+}

package/lib/config.ts

@@ -15,6 +15,24 @@ import { readFileSync } from "fs";
 import { homedir } from "os";
 import { parse as parseToml } from "@iarna/toml";
 
+export interface AgingConfig {
+  observations?: {
+    inferred?: number;
+    stated?: number;
+    verified?: number;
+  };
+  captures?: {
+    gotcha?: number;
+    learning?: number;
+    decision?: number;
+    preference?: number;
+    pattern?: number;
+    context?: number;
+    general?: number;
+  };
+  teachings?: number;
+}
+
 export interface LoreConfig {
   paths: {
     data: string;
@@ -38,6 +56,7 @@ export interface LoreConfig {
     model: string;
     dimensions: number;
   };
+  aging?: AgingConfig;
 }
 
 let cachedConfig: LoreConfig | null = null;
@@ -176,5 +195,10 @@ export function getConfig(): LoreConfig {
     },
   };
 
+  // Aging config (optional section) — missing [aging] uses DEFAULT_TTLS in age.ts
+  if (parsed.aging && typeof parsed.aging === "object") {
+    cachedConfig.aging = parsed.aging as AgingConfig;
+  }
+
   return cachedConfig!;
 }

package/lib/contradiction.ts

@@ -14,7 +14,8 @@
  *   // result.deleteRowid: number (only set for DELETE+ADD)
  */
 
-import { hybridSearch, type HybridResult } from "./semantic.js";
+import type { HybridResult } from "./semantic.js";
+import { rawSearch } from "./search.js";
 import { PURGEABLE_SOURCES } from "./purge.js";
 import { complete } from "@voidwire/llm-core";
 import type { CaptureEvent } from "./capture.js";
@@ -57,9 +58,9 @@ export function isContradictionCheckable(source: string): boolean {
 
 /**
  * Find existing entries that may contradict or duplicate the new event.
- * Uses hybrid search scoped by source + topic for precision.
+ * Uses raw FTS5 search scoped by source for precision.
  *
- * Opens its own read connection (hybridSearch uses openDatabase(true)).
+ * Opens its own read connection (rawSearch uses openDatabaseBasic(true)).
  * Safe in WAL mode — concurrent reads with the write connection in indexAndEmbed.
  *
  * Note: entries inserted in the current indexAndEmbed batch are not visible
@@ -76,10 +77,23 @@ export async function findCandidates(
 
   if (!content) return [];
 
-  return hybridSearch(content, {
+  const raw = rawSearch(content, {
     source,
     limit: CANDIDATE_LIMIT,
   });
+
+  return raw.map((r) => ({
+    rowid: r.rowid,
+    source: r.source,
+    title: r.title,
+    content: r.content,
+    metadata: r.metadata,
+    topic: r.topic,
+    type: r.type,
+    score: 0,
+    vectorScore: 0,
+    textScore: 0,
+  }));
 }
 
 /**

package/lib/db.ts

@@ -7,6 +7,7 @@
 
 import { Database } from "bun:sqlite";
 import { existsSync } from "fs";
+import { load } from "sqlite-vec";
 import { getConfig } from "./config";
 
 // Lazy initialization — deferred until first database open
@@ -48,15 +49,7 @@ export function openDatabase(readonly = false): Database {
     ? new Database(dbPath, { readonly: true })
     : new Database(dbPath);
 
-  // Load sqlite-vec extension
-  const vecPath = getConfig().database.sqlite_vec;
-  if (!vecPath) {
-    throw new Error(
-      "sqlite-vec path not configured. Run lore init to detect and configure it.",
-    );
-  }
-
-  db.loadExtension(vecPath);
+  load(db); // loads vec0 from npm package
 
   return db;
 }

package/lib/embed.ts

@@ -108,18 +108,14 @@ async function processBatch(db: Database, batch: FTSEntry[]): Promise<void> {
     }
   }
 
-  // Embed cache misses concurrently
-  if (toEmbed.length > 0) {
-    const results = await Promise.all(
-      toEmbed.map(({ contentString }) =>
-        embed({ text: contentString, prefix: "search_document" }),
-      ),
-    );
-    for (let i = 0; i < toEmbed.length; i++) {
-      const { idx, hash } = toEmbed[i];
-      embeddings[idx] = results[i].embedding;
-      cacheEmbedding(db, hash, results[i].embedding, MODEL_NAME);
-    }
+  // Embed cache misses sequentially (server is single-threaded)
+  for (const { idx, contentString, hash } of toEmbed) {
+    const result = await embed({
+      text: contentString,
+      prefix: "search_document",
+    });
+    embeddings[idx] = result.embedding;
+    cacheEmbedding(db, hash, result.embedding, MODEL_NAME);
   }
 
   // Insert all embeddings

package/lib/indexer.ts

@@ -10,13 +10,12 @@
  * - Orchestration (runIndexer)
  *
  * Usage:
- *   import { runIndexer, type IndexerFunction } from "./indexer";
- *   const myIndexer: IndexerFunction = async (ctx) => {
- *     ctx.insert({ source: "mySource", title: "...", content: "...", topic: "..." });
- *   };
+ *   import { runIndexer, type IndexerPlugin } from "./indexer";
+ *   // Register plugins via lib/indexers/index.ts barrel file
  */
 
-import { Database } from "bun:sqlite";
+import type { Database } from "bun:sqlite";
+import { openDatabaseBasic } from "./db.js";
 import { createHash } from "crypto";
 import { existsSync } from "fs";
 import { getConfig, type LoreConfig } from "./config";
@@ -40,17 +39,34 @@ export interface IndexerContext {
 
 export type IndexerFunction = (ctx: IndexerContext) => Promise<void>;
 
+export interface IndexerManifest {
+  name: string;
+  description: string;
+  requiredConfig: string[];
+  optionalConfig: string[];
+  rebuildExcluded: boolean;
+}
+
+export interface IndexerPlugin {
+  manifest: IndexerManifest;
+  canRun: (config: LoreConfig) => boolean;
+  run: IndexerFunction;
+}
+
 /**
- * Sources excluded from --rebuild because the DB is their source of truth.
- * These sources use real-time indexing (indexAndEmbed) with contradiction
- * resolution. Rebuilding from log.jsonl would resurrect purged/superseded
- * entries, breaking the contradiction resolution invariant.
+ * Resolve a dot-notation config path against a nested object.
+ * Used to derive human-readable missing config keys from manifest.requiredConfig.
  */
-export const REBUILD_EXCLUDED_SOURCES = new Set([
-  "captures",
-  "observations",
-  "teachings",
-]);
+export function getNestedValue(
+  obj: Record<string, unknown>,
+  path: string,
+): unknown {
+  return path.split(".").reduce<unknown>((acc, key) => {
+    if (acc && typeof acc === "object")
+      return (acc as Record<string, unknown>)[key];
+    return undefined;
+  }, obj as unknown);
+}
 
 /**
  * Check if a path is configured and exists on disk.
@@ -216,14 +232,15 @@ export function createIndexerContext(
 /**
  * Main indexing orchestrator.
  * Runs registered indexers for the given source (or all).
+ * Reports per-indexer status to stderr: checkmark ran, skip skipped, cross failed.
  */
 export async function runIndexer(
   source: string | "all",
   rebuild: boolean,
-  registry: Record<string, IndexerFunction>,
+  registry: Record<string, IndexerPlugin>,
 ): Promise<void> {
   const config = getConfig();
-  const db = new Database(config.database.sqlite);
+  const db = openDatabaseBasic(false);
 
   try {
     db.run("PRAGMA busy_timeout = 5000");
@@ -237,31 +254,59 @@ export async function runIndexer(
     const toRun = source === "all" ? Object.keys(registry) : [source];
 
     for (const src of toRun) {
-      const indexer = registry[src];
-      if (!indexer) {
+      const plugin = registry[src];
+      if (!plugin) {
         console.error(`Unknown source: ${src}`);
         continue;
       }
 
+      // Check canRun gate — skip indexers with missing config
+      if (!plugin.canRun(config)) {
+        const missing = plugin.manifest.requiredConfig
+          .filter(
+            (k) =>
+              !getNestedValue(config as unknown as Record<string, unknown>, k),
+          )
+          .join(", ");
+        console.error(`\u23ED\uFE0F  ${src}: skipped (missing: ${missing})`);
+        continue;
+      }
+
       // Skip rebuild-excluded sources — DB is source of truth, not log.jsonl
-      if (rebuild && REBUILD_EXCLUDED_SOURCES.has(src)) {
-        console.log(
-          `${src}: skipped (DB is source of truth — not rebuilt from log)`,
+      if (rebuild && plugin.manifest.rebuildExcluded) {
+        console.error(
+          `\u23ED\uFE0F  ${src}: skipped (DB is source of truth \u2014 not rebuilt from log)`,
         );
         continue;
       }
 
-      console.log(`Indexing ${src}...`);
-
       // Clear source if rebuilding
       if (rebuild) {
         db.run("DELETE FROM search WHERE source = ?", [src]);
       }
 
-      await indexer(ctx);
+      const countBefore = (
+        db
+          .query("SELECT COUNT(*) as n FROM search WHERE source = ?")
+          .get(src) as { n: number }
+      ).n;
+
+      try {
+        await plugin.run(ctx);
+        const countAfter = (
+          db
+            .query("SELECT COUNT(*) as n FROM search WHERE source = ?")
+            .get(src) as { n: number }
+        ).n;
+        console.error(`\u2705  ${src}: ${countAfter - countBefore} entries`);
+      } catch (err) {
+        console.error(
+          `\u274C  ${src}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
     }
 
-    console.log("Indexing complete");
+    console.error("Indexing complete");
   } finally {
     db.close();
   }

package/lib/indexers/blogs.ts

@@ -13,7 +13,7 @@
 
 import { readdirSync, readFileSync, statSync, existsSync } from "fs";
 import { join, basename } from "path";
-import { checkPath, type IndexerContext } from "../indexer";
+import { checkPath, type IndexerContext, type IndexerPlugin } from "../indexer";
 
 function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
   if (!existsSync(dir)) return files;
@@ -33,7 +33,7 @@ function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
   return files;
 }
 
-export async function indexBlogs(ctx: IndexerContext): Promise<void> {
+async function indexBlogs(ctx: IndexerContext): Promise<void> {
   const blogsDir = ctx.config.paths.blogs;
   if (!checkPath("blogs", "paths.blogs", blogsDir)) return;
 
@@ -149,3 +149,15 @@ export async function indexBlogs(ctx: IndexerContext): Promise<void> {
     }
   }
 }
+
+export const blogsPlugin: IndexerPlugin = {
+  manifest: {
+    name: "blogs",
+    description: "Indexes Hugo blog posts from the content directory",
+    requiredConfig: ["paths.blogs"],
+    optionalConfig: [],
+    rebuildExcluded: false,
+  },
+  canRun: (config) => !!config.paths?.blogs,
+  run: indexBlogs,
+};