@voidwire/lore 0.9.1 → 1.0.1

package/lib/indexers/teachings.ts

@@ -0,0 +1,52 @@
+/**
+ * lib/indexers/teachings.ts - Teachings indexer
+ *
+ * Reads log.jsonl and indexes teaching captures.
+ * Filters for event=captured AND type=teaching.
+ *
+ * Source: teachings
+ * Topic: data.topic (AI-written)
+ * Type: teaching (fixed)
+ * Timestamp: event timestamp
+ */
+
+import { readFileSync, existsSync } from "fs";
+import type { IndexerContext } from "../indexer";
+
+export async function indexTeachings(ctx: IndexerContext): Promise<void> {
+  const logPath = `${ctx.config.paths.data}/log.jsonl`;
+  if (!existsSync(logPath)) {
+    console.log("No log.jsonl found, skipping teachings");
+    return;
+  }
+
+  const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.event !== "captured" || event.type !== "teaching") continue;
+
+      const topic = event.data?.topic || "general";
+      const content = event.data?.content || "";
+      const confidence = event.data?.confidence;
+
+      if (!content) continue;
+
+      const metadata: Record<string, unknown> = {};
+      if (confidence) metadata.confidence = confidence;
+
+      ctx.insert({
+        source: "teachings",
+        title: `[teaching] ${topic}`,
+        content,
+        topic,
+        type: "teaching",
+        timestamp: event.timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      continue;
+    }
+  }
+}

package/lib/info.ts

@@ -6,8 +6,8 @@
  */
 
 import { Database } from "bun:sqlite";
-import { homedir } from "os";
 import { existsSync } from "fs";
+import { getDatabasePath } from "./db.js";
 import { projects as getProjects } from "./projects.js";
 
 export interface SourceInfo {
@@ -22,10 +22,6 @@ export interface InfoOutput {
   total_entries: number;
 }
 
-function getDatabasePath(): string {
-  return `${homedir()}/.local/share/lore/lore.db`;
-}
-
 /**
  * Get info about indexed sources
  *
@@ -62,11 +58,11 @@ export function info(): InfoOutput {
     const totalResult = totalStmt.get() as { total: number };
     const total_entries = totalResult?.total ?? 0;
 
-    // Get last indexed timestamp from metadata
+    // Get last indexed timestamp from column
     const tsStmt = db.prepare(`
-      SELECT MAX(json_extract(metadata, '$.timestamp')) as ts
+      SELECT MAX(timestamp) as ts
       FROM search
-      WHERE json_extract(metadata, '$.timestamp') IS NOT NULL
+      WHERE timestamp IS NOT NULL AND timestamp != ''
     `);
     const tsResult = tsStmt.get() as { ts: string | null };
     const last_indexed = tsResult?.ts ?? new Date().toISOString();

package/lib/list.ts

@@ -6,8 +6,8 @@
  */
 
 import { Database } from "bun:sqlite";
-import { homedir } from "os";
 import { existsSync } from "fs";
+import { getDatabasePath } from "./db.js";
 
 // Source types - data sources that can be listed
 export type Source =
@@ -65,19 +65,6 @@ const PERSONAL_SUBTYPES: Partial<Record<Source, string>> = {
   habits: "habit",
 };
 
-// Maps source to metadata field for --project filter
-// Project-based domains use "project", topic-based domains use "topic"
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  flux: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
-
 export interface ListOptions {
   limit?: number;
   project?: string;
@@ -87,6 +74,8 @@ export interface ListOptions {
 export interface ListEntry {
   title: string;
   content: string;
+  topic: string;
+  type: string;
   metadata: Record<string, unknown>;
 }
 
@@ -96,14 +85,11 @@ export interface ListResult {
   count: number;
 }
 
-// Database path following XDG spec
-function getDatabasePath(): string {
-  return `${homedir()}/.local/share/lore/lore.db`;
-}
-
 interface RawRow {
   title: string;
   content: string;
+  topic: string;
+  type: string;
   metadata: string;
 }
 
@@ -117,26 +103,24 @@ function queryBySource(
   project?: string,
   type?: string,
 ): ListEntry[] {
-  let sql = "SELECT title, content, metadata FROM search WHERE source = ?";
+  let sql =
+    "SELECT title, content, topic, type, metadata FROM search WHERE source = ?";
   const params: (string | number)[] = [source];
 
-  // Add project filter if provided and source has a project field
+  // Add project filter if provided — uses topic column directly
   if (project) {
-    const field = PROJECT_FIELD[source];
-    if (field) {
-      sql += ` AND json_extract(metadata, '$.${field}') = ?`;
-      params.push(project);
-    }
+    sql += " AND topic = ?";
+    params.push(project);
   }
 
-  // Add type filter if provided (captures source only)
-  if (type && source === "captures") {
-    sql += ` AND json_extract(metadata, '$.type') = ?`;
+  // Add type filter if provided — uses type column directly
+  if (type) {
+    sql += " AND type = ?";
     params.push(type);
   }
 
   // Order by timestamp descending (most recent first)
-  sql += " ORDER BY json_extract(metadata, '$.timestamp') DESC";
+  sql += " ORDER BY timestamp DESC";
 
   if (limit) {
     sql += " LIMIT ?";
@@ -149,6 +133,8 @@ function queryBySource(
   return rows.map((row) => ({
     title: row.title,
     content: row.content,
+    topic: row.topic,
+    type: row.type,
     metadata: JSON.parse(row.metadata || "{}"),
   }));
 }
@@ -163,10 +149,10 @@ function queryPersonalType(
 ): ListEntry[] {
   // Filter by type in SQL, not JS - avoids LIMIT truncation bug
   let sql = `
-    SELECT title, content, metadata FROM search
+    SELECT title, content, topic, type, metadata FROM search
     WHERE source = 'personal'
-      AND json_extract(metadata, '$.type') = ?
-    ORDER BY json_extract(metadata, '$.timestamp') DESC
+      AND type = ?
+    ORDER BY timestamp DESC
   `;
   const params: (string | number)[] = [type];
 
@@ -181,6 +167,8 @@ function queryPersonalType(
   return rows.map((row) => ({
     title: row.title,
     content: row.content,
+    topic: row.topic,
+    type: row.type,
     metadata: JSON.parse(row.metadata || "{}"),
   }));
 }
@@ -203,7 +191,7 @@ export function list(source: Source, options: ListOptions = {}): ListResult {
   const dbPath = getDatabasePath();
 
   if (!existsSync(dbPath)) {
-    throw new Error(`Database not found: ${dbPath}. Run lore-index-all first.`);
+    throw new Error(`Database not found: ${dbPath}. Run lore-db-init first.`);
   }
 
   const db = new Database(dbPath, { readonly: true });
@@ -243,12 +231,10 @@ export function listSources(): Source[] {
 }
 
 /**
- * Extract project name from entry metadata
+ * Extract project name from entry
  */
-function extractProjectFromEntry(entry: ListEntry, source: string): string {
-  const field = PROJECT_FIELD[source];
-  if (!field) return "unknown";
-  return (entry.metadata[field] as string) || "unknown";
+function extractProjectFromEntry(entry: ListEntry, _source: string): string {
+  return entry.topic || "unknown";
 }
 
 /**

package/lib/projects.ts

@@ -1,29 +1,24 @@
 /**
  * lib/projects.ts - List all known projects across sources
  *
- * Queries distinct project values from metadata fields, handling
- * different field names per source type.
+ * Queries distinct topic values from the topic column across
+ * project-scoped sources.
  */
 
 import { Database } from "bun:sqlite";
-import { homedir } from "os";
 import { existsSync } from "fs";
-
-// Project-based domains use "project", topic-based domains use "topic"
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  tasks: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
-
-function getDatabasePath(): string {
-  return `${homedir()}/.local/share/lore/lore.db`;
-}
+import { getDatabasePath } from "./db.js";
+
+const PROJECT_SOURCES = [
+  "commits",
+  "sessions",
+  "flux",
+  "insights",
+  "captures",
+  "teachings",
+  "learnings",
+  "observations",
+];
 
 /**
  * Get all unique project names across sources
@@ -40,24 +35,20 @@ export function projects(): string[] {
   const db = new Database(dbPath, { readonly: true });
 
   try {
-    const allProjects = new Set<string>();
-
-    for (const [source, field] of Object.entries(PROJECT_FIELD)) {
-      const stmt = db.prepare(`
-        SELECT DISTINCT json_extract(metadata, '$.${field}') as proj
-        FROM search
-        WHERE source = ? AND json_extract(metadata, '$.${field}') IS NOT NULL
-      `);
-      const results = stmt.all(source) as { proj: string | null }[];
-
-      for (const r of results) {
-        if (r.proj) {
-          allProjects.add(r.proj);
-        }
-      }
-    }
-
-    return Array.from(allProjects).sort();
+    const placeholders = PROJECT_SOURCES.map(() => "?").join(", ");
+    const stmt = db.prepare(`
+      SELECT DISTINCT topic
+      FROM search
+      WHERE source IN (${placeholders})
+        AND topic IS NOT NULL
+        AND topic != ''
+    `);
+    const results = stmt.all(...PROJECT_SOURCES) as { topic: string }[];
+
+    return results
+      .map((r) => r.topic)
+      .filter(Boolean)
+      .sort();
   } finally {
     db.close();
   }

package/lib/realtime.ts

@@ -23,12 +23,7 @@ import {
   EMBEDDING_DIM,
   serializeEmbedding,
 } from "./semantic.js";
-import {
-  hashContent,
-  getCachedEmbedding,
-  cacheEmbedding,
-  getMissingHashes,
-} from "./cache.js";
+import { hashContent, getCachedEmbedding, cacheEmbedding } from "./cache.js";
 import type { CaptureEvent } from "./capture.js";
 
 /**
@@ -89,13 +84,23 @@ function insertSearchEntry(db: Database, event: CaptureEvent): number {
   const metadata = buildMetadata(event);
   const data = event.data as Record<string, unknown>;
   const topic = String(data.topic || "");
+  const type = extractType(event);
+  const timestamp = event.timestamp || new Date().toISOString();
 
   const stmt = db.prepare(`
-    INSERT INTO search (source, title, content, metadata, topic)
-    VALUES (?, ?, ?, ?, ?)
+    INSERT INTO search (source, title, content, metadata, topic, type, timestamp)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
   `);
 
-  const result = stmt.run(source, title, content, metadata, topic);
+  const result = stmt.run(
+    source,
+    title,
+    content,
+    metadata,
+    topic,
+    type,
+    timestamp,
+  );
   return Number(result.lastInsertRowid);
 }
 
@@ -163,31 +168,19 @@ function getContentForEmbedding(event: CaptureEvent): string {
  */
 function buildMetadata(event: CaptureEvent): string {
   const data = event.data as Record<string, unknown>;
-  const timestamp = event.timestamp;
-  const date = timestamp ? timestamp.substring(0, 10) : "";
-
-  const content = getContentForEmbedding(event);
-  const metadata: Record<string, unknown> = {
-    topic: data.topic || "general",
-    timestamp,
-    date,
-    content,
-    content_hash: hashContent(content),
-  };
+  const metadata: Record<string, unknown> = {};
 
-  // Add type-specific fields
+  // Add type-specific fields only (no topic, content, content_hash, date, timestamp)
   switch (event.type) {
     case "knowledge":
       metadata.subtype = data.subtype;
       break;
     case "teaching":
       metadata.confidence = data.confidence;
-      metadata.capture_source = data.source || "manual";
       break;
     case "observation":
       metadata.subtype = data.subtype;
       metadata.confidence = data.confidence;
-      metadata.capture_source = data.source || "auto";
       break;
     case "insight":
       metadata.subtype = data.subtype;

package/lib/search.ts

@@ -6,8 +6,8 @@
  */
 
 import { Database } from "bun:sqlite";
-import { homedir } from "os";
 import { existsSync } from "fs";
+import { getDatabasePath } from "./db.js";
 
 export interface SearchResult {
   rowid: number;
@@ -15,6 +15,7 @@ export interface SearchResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   rank: number;
 }
 
@@ -25,10 +26,6 @@ export interface SearchOptions {
   type?: string | string[];
 }
 
-function getDatabasePath(): string {
-  return `${homedir()}/.local/share/lore/lore.db`;
-}
-
 /**
  * Escape a query for safe FTS5 MATCH
  * Wraps terms in double quotes to prevent FTS5 syntax interpretation
@@ -85,19 +82,16 @@ export function search(
 
     if (options.type) {
       const types = Array.isArray(options.type) ? options.type : [options.type];
-      const typeClauses = types.map(
-        () =>
-          "(json_extract(metadata, '$.type') = ? OR json_extract(metadata, '$.subtype') = ?)",
-      );
+      const typeClauses = types.map(() => "type = ?");
       conditions.push(`(${typeClauses.join(" OR ")})`);
       types.forEach((t) => {
-        params.push(t, t);
+        params.push(t);
       });
     }
 
     if (options.since) {
       conditions.push(
-        "json_extract(metadata, '$.date') IS NOT NULL AND json_extract(metadata, '$.date') != 'unknown' AND json_extract(metadata, '$.date') >= ?",
+        "timestamp IS NOT NULL AND timestamp != '' AND timestamp >= ?",
       );
       params.push(options.since);
     }
@@ -105,7 +99,7 @@ export function search(
     params.push(limit);
 
     const sql = `
-      SELECT rowid, source, title, snippet(search, 2, '→', '←', '...', 32) as content, metadata, rank
+      SELECT rowid, source, title, snippet(search, 2, '→', '←', '...', 32) as content, metadata, topic, rank
       FROM search
       WHERE ${conditions.join(" AND ")}
       ORDER BY rank

package/lib/semantic.ts

@@ -18,6 +18,7 @@ export interface SemanticResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   distance: number;
 }
 
@@ -28,21 +29,6 @@ export interface SemanticSearchOptions {
   type?: string | string[];
 }
 
-/**
- * Maps source types to their project/topic field name in metadata JSON.
- * Project-based domains use "project", topic-based domains use "topic".
- */
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  tasks: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
-
 const MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5";
 const EMBEDDING_DIM = 768;
 
@@ -254,6 +240,7 @@ export async function semanticSearch(
         s.title,
         s.content,
         s.metadata,
+        s.topic,
         e.distance
       FROM embeddings e
       JOIN search s ON e.doc_id = s.rowid
@@ -281,6 +268,7 @@ export interface HybridResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   score: number;
   vectorScore: number;
   textScore: number;
@@ -367,6 +355,7 @@ export async function hybridSearch(
       title: r.title,
       content: r.content,
       metadata: r.metadata,
+      topic: r.topic,
       vectorScore,
       textScore: 0,
       score: vectorWeight * vectorScore,
@@ -393,6 +382,7 @@ export async function hybridSearch(
         title: r.title,
         content: r.content,
         metadata: r.metadata,
+        topic: r.topic,
         vectorScore: 0,
         textScore,
         score: textWeight * textScore,
@@ -408,21 +398,6 @@ export async function hybridSearch(
   return results;
 }
 
-/**
- * Extract project from result metadata
- */
-function extractProjectFromMetadata(metadata: string, source: string): string {
-  const field = PROJECT_FIELD[source];
-  if (!field) return "unknown";
-
-  try {
-    const parsed = JSON.parse(metadata);
-    return parsed[field] || "unknown";
-  } catch {
-    return "unknown";
-  }
-}
-
 /**
  * Extract identifier from semantic result
  */
@@ -478,7 +453,7 @@ export function formatBriefSearch(results: SemanticResult[]): string {
     const lines = [`${source} (${sourceResults.length}):`];
 
     sourceResults.forEach((r) => {
-      const project = extractProjectFromMetadata(r.metadata, r.source);
+      const project = r.topic || "unknown";
       const identifier = extractIdentifierFromResult(r);
       const displayText = getDisplayText(r);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "0.9.1",
+  "version": "1.0.1",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",
@@ -43,7 +43,8 @@
     "bun": ">=1.0.0"
   },
   "dependencies": {
-    "@huggingface/transformers": "^3.2.6"
+    "@huggingface/transformers": "^3.2.6",
+    "@iarna/toml": "^2.2.5"
   },
   "devDependencies": {
     "bun-types": "1.3.5"