@voidwire/lore 0.9.0 → 1.0.0

package/lib/list.ts

@@ -65,19 +65,6 @@ const PERSONAL_SUBTYPES: Partial<Record<Source, string>> = {
   habits: "habit",
 };
 
-// Maps source to metadata field for --project filter
-// Project-based domains use "project", topic-based domains use "topic"
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  flux: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
-
 export interface ListOptions {
   limit?: number;
   project?: string;
@@ -87,6 +74,8 @@ export interface ListOptions {
 export interface ListEntry {
   title: string;
   content: string;
+  topic: string;
+  type: string;
   metadata: Record<string, unknown>;
 }
 
@@ -104,6 +93,8 @@ function getDatabasePath(): string {
 interface RawRow {
   title: string;
   content: string;
+  topic: string;
+  type: string;
   metadata: string;
 }
 
@@ -117,26 +108,24 @@ function queryBySource(
   project?: string,
   type?: string,
 ): ListEntry[] {
-  let sql = "SELECT title, content, metadata FROM search WHERE source = ?";
+  let sql =
+    "SELECT title, content, topic, type, metadata FROM search WHERE source = ?";
   const params: (string | number)[] = [source];
 
-  // Add project filter if provided and source has a project field
+  // Add project filter if provided — uses topic column directly
   if (project) {
-    const field = PROJECT_FIELD[source];
-    if (field) {
-      sql += ` AND json_extract(metadata, '$.${field}') = ?`;
-      params.push(project);
-    }
+    sql += " AND topic = ?";
+    params.push(project);
   }
 
-  // Add type filter if provided (captures source only)
-  if (type && source === "captures") {
-    sql += ` AND json_extract(metadata, '$.type') = ?`;
+  // Add type filter if provided — uses type column directly
+  if (type) {
+    sql += " AND type = ?";
     params.push(type);
   }
 
   // Order by timestamp descending (most recent first)
-  sql += " ORDER BY json_extract(metadata, '$.timestamp') DESC";
+  sql += " ORDER BY timestamp DESC";
 
   if (limit) {
     sql += " LIMIT ?";
@@ -149,6 +138,8 @@ function queryBySource(
   return rows.map((row) => ({
     title: row.title,
     content: row.content,
+    topic: row.topic,
+    type: row.type,
     metadata: JSON.parse(row.metadata || "{}"),
   }));
 }
@@ -163,10 +154,10 @@ function queryPersonalType(
 ): ListEntry[] {
   // Filter by type in SQL, not JS - avoids LIMIT truncation bug
   let sql = `
-    SELECT title, content, metadata FROM search
+    SELECT title, content, topic, type, metadata FROM search
     WHERE source = 'personal'
-      AND json_extract(metadata, '$.type') = ?
-    ORDER BY json_extract(metadata, '$.timestamp') DESC
+      AND type = ?
+    ORDER BY timestamp DESC
   `;
   const params: (string | number)[] = [type];
 
@@ -181,6 +172,8 @@ function queryPersonalType(
   return rows.map((row) => ({
     title: row.title,
     content: row.content,
+    topic: row.topic,
+    type: row.type,
     metadata: JSON.parse(row.metadata || "{}"),
   }));
 }
@@ -243,12 +236,10 @@ export function listSources(): Source[] {
 }
 
 /**
- * Extract project name from entry metadata
+ * Extract project name from entry
  */
-function extractProjectFromEntry(entry: ListEntry, source: string): string {
-  const field = PROJECT_FIELD[source];
-  if (!field) return "unknown";
-  return (entry.metadata[field] as string) || "unknown";
+function extractProjectFromEntry(entry: ListEntry, _source: string): string {
+  return entry.topic || "unknown";
 }
 
 /**

package/lib/projects.ts

@@ -1,25 +1,24 @@
 /**
  * lib/projects.ts - List all known projects across sources
  *
- * Queries distinct project values from metadata fields, handling
- * different field names per source type.
+ * Queries distinct topic values from the topic column across
+ * project-scoped sources.
  */
 
 import { Database } from "bun:sqlite";
 import { homedir } from "os";
 import { existsSync } from "fs";
 
-// Project-based domains use "project", topic-based domains use "topic"
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  tasks: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
+const PROJECT_SOURCES = [
+  "commits",
+  "sessions",
+  "flux",
+  "insights",
+  "captures",
+  "teachings",
+  "learnings",
+  "observations",
+];
 
 function getDatabasePath(): string {
   return `${homedir()}/.local/share/lore/lore.db`;
@@ -40,24 +39,20 @@ export function projects(): string[] {
   const db = new Database(dbPath, { readonly: true });
 
   try {
-    const allProjects = new Set<string>();
-
-    for (const [source, field] of Object.entries(PROJECT_FIELD)) {
-      const stmt = db.prepare(`
-        SELECT DISTINCT json_extract(metadata, '$.${field}') as proj
-        FROM search
-        WHERE source = ? AND json_extract(metadata, '$.${field}') IS NOT NULL
-      `);
-      const results = stmt.all(source) as { proj: string | null }[];
-
-      for (const r of results) {
-        if (r.proj) {
-          allProjects.add(r.proj);
-        }
-      }
-    }
-
-    return Array.from(allProjects).sort();
+    const placeholders = PROJECT_SOURCES.map(() => "?").join(", ");
+    const stmt = db.prepare(`
+      SELECT DISTINCT topic
+      FROM search
+      WHERE source IN (${placeholders})
+        AND topic IS NOT NULL
+        AND topic != ''
+    `);
+    const results = stmt.all(...PROJECT_SOURCES) as { topic: string }[];
+
+    return results
+      .map((r) => r.topic)
+      .filter(Boolean)
+      .sort();
   } finally {
     db.close();
   }

package/lib/realtime.ts

@@ -87,13 +87,25 @@ function insertSearchEntry(db: Database, event: CaptureEvent): number {
   const title = buildTitle(event);
   const content = getContentForEmbedding(event);
   const metadata = buildMetadata(event);
+  const data = event.data as Record<string, unknown>;
+  const topic = String(data.topic || "");
+  const type = extractType(event);
+  const timestamp = event.timestamp || new Date().toISOString();
 
   const stmt = db.prepare(`
-    INSERT INTO search (source, title, content, metadata)
-    VALUES (?, ?, ?, ?)
+    INSERT INTO search (source, title, content, metadata, topic, type, timestamp)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
   `);
 
-  const result = stmt.run(source, title, content, metadata);
+  const result = stmt.run(
+    source,
+    title,
+    content,
+    metadata,
+    topic,
+    type,
+    timestamp,
+  );
   return Number(result.lastInsertRowid);
 }
 
@@ -147,10 +159,13 @@ function buildTitle(event: CaptureEvent): string {
 
 /**
  * Extract content for embedding from event
+ * Concatenates topic+content for richer embeddings (matches lore-embed-all)
  */
 function getContentForEmbedding(event: CaptureEvent): string {
   const data = event.data as Record<string, unknown>;
-  return String(data.content || data.text || "");
+  const content = String(data.content || data.text || "");
+  const topic = String(data.topic || "").trim();
+  return topic ? `${topic} ${content}`.trim() : content;
 }
 
 /**
@@ -158,19 +173,9 @@ function getContentForEmbedding(event: CaptureEvent): string {
  */
 function buildMetadata(event: CaptureEvent): string {
   const data = event.data as Record<string, unknown>;
-  const timestamp = event.timestamp;
-  const date = timestamp ? timestamp.substring(0, 10) : "";
-
-  const content = getContentForEmbedding(event);
-  const metadata: Record<string, unknown> = {
-    topic: data.topic || "general",
-    timestamp,
-    date,
-    content,
-    content_hash: hashContent(content),
-  };
+  const metadata: Record<string, unknown> = {};
 
-  // Add type-specific fields
+  // Add type-specific fields only (no topic, content, content_hash, date, timestamp)
   switch (event.type) {
     case "knowledge":
       metadata.subtype = data.subtype;
@@ -255,13 +260,40 @@ function insertEmbedding(
   const source = getSourceForEvent(event);
   const data = event.data as Record<string, unknown>;
   const topic = String(data.topic || "");
+  const type = extractType(event);
 
   const embeddingBlob = serializeEmbedding(embedding);
 
   const stmt = db.prepare(`
-    INSERT INTO embeddings (doc_id, chunk_idx, source, topic, embedding)
-    VALUES (?, 0, ?, ?, ?)
+    INSERT INTO embeddings (doc_id, chunk_idx, source, topic, type, embedding)
+    VALUES (?, 0, ?, ?, ?, ?)
   `);
 
-  stmt.run(docId, source, topic, embeddingBlob);
+  stmt.run(docId, source, topic, type, embeddingBlob);
+}
+
+/**
+ * Extract type value for embeddings partition column
+ */
+function extractType(event: CaptureEvent): string {
+  const data = event.data as Record<string, unknown>;
+
+  switch (event.type) {
+    case "knowledge":
+      return String(data.subtype || "general");
+    case "teaching":
+      return "teaching";
+    case "observation":
+      return String(data.subtype || "pattern");
+    case "insight":
+      return String(data.subtype || "insight");
+    case "learning":
+      return "learning";
+    case "task":
+      return "task";
+    case "note":
+      return "note";
+    default:
+      return "general";
+  }
 }

package/lib/search.ts

@@ -15,6 +15,7 @@ export interface SearchResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   rank: number;
 }
 
@@ -85,19 +86,16 @@ export function search(
 
     if (options.type) {
       const types = Array.isArray(options.type) ? options.type : [options.type];
-      const typeClauses = types.map(
-        () =>
-          "(json_extract(metadata, '$.type') = ? OR json_extract(metadata, '$.subtype') = ?)",
-      );
+      const typeClauses = types.map(() => "type = ?");
       conditions.push(`(${typeClauses.join(" OR ")})`);
       types.forEach((t) => {
-        params.push(t, t);
+        params.push(t);
       });
     }
 
     if (options.since) {
       conditions.push(
-        "json_extract(metadata, '$.date') IS NOT NULL AND json_extract(metadata, '$.date') != 'unknown' AND json_extract(metadata, '$.date') >= ?",
+        "timestamp IS NOT NULL AND timestamp != '' AND timestamp >= ?",
       );
       params.push(options.since);
     }
@@ -105,7 +103,7 @@ export function search(
     params.push(limit);
 
     const sql = `
-      SELECT rowid, source, title, snippet(search, 2, '→', '←', '...', 32) as content, metadata, rank
+      SELECT rowid, source, title, snippet(search, 2, '→', '←', '...', 32) as content, metadata, topic, rank
       FROM search
       WHERE ${conditions.join(" AND ")}
       ORDER BY rank

package/lib/semantic.ts

@@ -18,6 +18,7 @@ export interface SemanticResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   distance: number;
 }
 
@@ -28,21 +29,6 @@ export interface SemanticSearchOptions {
   type?: string | string[];
 }
 
-/**
- * Maps source types to their project/topic field name in metadata JSON.
- * Project-based domains use "project", topic-based domains use "topic".
- */
-const PROJECT_FIELD: Record<string, string> = {
-  commits: "project",
-  sessions: "project",
-  tasks: "project",
-  insights: "topic",
-  captures: "topic",
-  teachings: "topic",
-  learnings: "topic",
-  observations: "topic",
-};
-
 const MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5";
 const EMBEDDING_DIM = 768;
 
@@ -254,6 +240,7 @@ export async function semanticSearch(
         s.title,
         s.content,
         s.metadata,
+        s.topic,
         e.distance
       FROM embeddings e
       JOIN search s ON e.doc_id = s.rowid
@@ -281,6 +268,7 @@ export interface HybridResult {
   title: string;
   content: string;
   metadata: string;
+  topic: string;
   score: number;
   vectorScore: number;
   textScore: number;
@@ -367,6 +355,7 @@ export async function hybridSearch(
       title: r.title,
       content: r.content,
       metadata: r.metadata,
+      topic: r.topic,
       vectorScore,
       textScore: 0,
       score: vectorWeight * vectorScore,
@@ -393,6 +382,7 @@ export async function hybridSearch(
         title: r.title,
         content: r.content,
         metadata: r.metadata,
+        topic: r.topic,
         vectorScore: 0,
         textScore,
         score: textWeight * textScore,
@@ -408,21 +398,6 @@ export async function hybridSearch(
   return results;
 }
 
-/**
- * Extract project from result metadata
- */
-function extractProjectFromMetadata(metadata: string, source: string): string {
-  const field = PROJECT_FIELD[source];
-  if (!field) return "unknown";
-
-  try {
-    const parsed = JSON.parse(metadata);
-    return parsed[field] || "unknown";
-  } catch {
-    return "unknown";
-  }
-}
-
 /**
  * Extract identifier from semantic result
  */
@@ -478,7 +453,7 @@ export function formatBriefSearch(results: SemanticResult[]): string {
     const lines = [`${source} (${sourceResults.length}):`];
 
     sourceResults.forEach((r) => {
-      const project = extractProjectFromMetadata(r.metadata, r.source);
+      const project = r.topic || "unknown";
       const identifier = extractIdentifierFromResult(r);
       const displayText = getDisplayText(r);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "0.9.0",
+  "version": "1.0.0",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",
@@ -19,11 +19,6 @@
     "README.md",
     "LICENSE"
   ],
-  "scripts": {
-    "build": "tsc --noEmit false --outDir dist --declaration",
-    "typecheck": "tsc --noEmit",
-    "test": "bun test"
-  },
   "keywords": [
     "knowledge",
     "search",
@@ -48,9 +43,15 @@
     "bun": ">=1.0.0"
   },
   "dependencies": {
-    "@huggingface/transformers": "^3.2.6"
+    "@huggingface/transformers": "^3.2.6",
+    "@iarna/toml": "^2.2.5"
   },
   "devDependencies": {
     "bun-types": "1.3.5"
+  },
+  "scripts": {
+    "build": "tsc --noEmit false --outDir dist --declaration",
+    "typecheck": "tsc --noEmit",
+    "test": "bun test"
   }
-}
+}