@voidwire/lore 1.5.0 → 1.6.0

package/cli.ts

@@ -691,15 +691,24 @@ async function handlePurge(args: string[]): Promise<void> {
       }
     }
 
-    const result = deleteEntries(matches.map((m) => m.rowid));
+    const result = deleteEntries(
+      matches.map((m) => m.rowid),
+      matches.map((m) => m.content),
+    );
 
     output({
       success: true,
       matches: matches.length,
       deleted: result.deleted,
       rowids: result.rowids,
+      logEntriesRemoved: result.logEntriesRemoved,
     });
-    console.error(`Purged ${result.deleted} entries from search + embeddings.`);
+    console.error(
+      `Purged ${result.deleted} entries from search + embeddings` +
+        (result.logEntriesRemoved > 0
+          ? ` + ${result.logEntriesRemoved} from log.jsonl.`
+          : `.`),
+    );
     process.exit(0);
   } catch (error) {
     const message = error instanceof Error ? error.message : "Unknown error";

package/lib/contradiction.ts

@@ -0,0 +1,214 @@
+/**
+ * lib/contradiction.ts - Write-time contradiction detection
+ *
+ * Uses hybrid search + local MLX model to classify new captures against
+ * existing entries. Determines whether a new entry should ADD (new info),
+ * NOOP (redundant), or DELETE+ADD (supersedes existing).
+ *
+ * Fail-open design: any error defaults to ADD — never blocks a capture.
+ *
+ * Usage:
+ *   const candidates = await findCandidates(event);
+ *   const result = await classifyContradiction(event, candidates);
+ *   // result.action: "ADD" | "NOOP" | "DELETE+ADD"
+ *   // result.deleteRowid: number (only set for DELETE+ADD)
+ */
+
+import { hybridSearch, type HybridResult } from "./semantic.js";
+import { PURGEABLE_SOURCES } from "./purge.js";
+import type { CaptureEvent } from "./capture.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export type ContradictionAction = "ADD" | "DELETE+ADD" | "NOOP";
+
+export interface ContradictionResult {
+  action: ContradictionAction;
+  deleteRowid?: number;
+}
+
+// ─── Constants ──────────────────────────────────────────────────────────────
+
+const MLX_URL = "http://localhost:8080/v1/chat/completions";
+const MLX_MODEL = "mlx-community/Qwen2.5-7B-Instruct-4bit";
+const MLX_TIMEOUT_MS = 1500;
+
+const CANDIDATE_LIMIT = 5;
+
+// Sources eligible for contradiction checking (same as purgeable)
+const CONTRADICTION_SOURCES = new Set<string>(PURGEABLE_SOURCES);
+
+// ─── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * Check if an event's source is eligible for contradiction detection.
+ */
+export function isContradictionCheckable(source: string): boolean {
+  return CONTRADICTION_SOURCES.has(source);
+}
+
+/**
+ * Find existing entries that may contradict or duplicate the new event.
+ * Uses hybrid search scoped by source + topic for precision.
+ *
+ * Opens its own read connection (hybridSearch uses openDatabase(true)).
+ * Safe in WAL mode — concurrent reads with the write connection in indexAndEmbed.
+ *
+ * Note: entries inserted in the current indexAndEmbed batch are not visible
+ * to this read connection (uncommitted). This is acceptable — same-batch
+ * entries are unlikely to contradict each other.
+ */
+export async function findCandidates(
+  event: CaptureEvent,
+): Promise<HybridResult[]> {
+  const data = event.data as Record<string, unknown>;
+  const content = String(data.content || data.text || "");
+  const topic = String(data.topic || "");
+  const source = getSourceForEvent(event);
+
+  if (!content) return [];
+
+  return hybridSearch(content, {
+    source,
+    project: topic || undefined,
+    limit: CANDIDATE_LIMIT,
+  });
+}
+
+/**
+ * Classify a new event against existing candidates using local MLX model.
+ *
+ * LLM returns one of:
+ *   ADD     — new information not covered by candidates
+ *   NOOP    — duplicate/redundant (already captured)
+ *   DELETE <rowid> — new info supersedes a specific candidate
+ *
+ * Fail-open: any error (timeout, parse failure, model down) defaults to ADD.
+ */
+export async function classifyContradiction(
+  event: CaptureEvent,
+  candidates: HybridResult[],
+): Promise<ContradictionResult> {
+  if (candidates.length === 0) {
+    return { action: "ADD" };
+  }
+
+  const data = event.data as Record<string, unknown>;
+  const content = String(data.content || data.text || "");
+  const topic = String(data.topic || "");
+  const source = getSourceForEvent(event);
+
+  const candidateLines = candidates
+    .map((c) => `[rowid: ${c.rowid}] ${c.content}`)
+    .join("\n");
+
+  const systemPrompt = `You classify knowledge contradictions. Reply with exactly one word: ADD, NOOP, or DELETE.
+  ADD: new information not covered by candidates
+  NOOP: duplicate or redundant (already captured)
+  DELETE: new information supersedes a candidate (also provide rowid)`;
+
+  const userPrompt = `New entry (source: ${source}, topic: ${topic}):
+${content}
+
+Existing entries:
+${candidateLines}
+
+If DELETE, reply: DELETE <rowid>
+Otherwise reply: ADD or NOOP`;
+
+  try {
+    const resp = await fetch(MLX_URL, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: MLX_MODEL,
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: userPrompt },
+        ],
+        max_tokens: 20,
+        temperature: 0,
+      }),
+      signal: AbortSignal.timeout(MLX_TIMEOUT_MS),
+    });
+
+    if (!resp.ok) {
+      console.error(
+        `[contradiction] MLX returned ${resp.status} — defaulting to ADD`,
+      );
+      return { action: "ADD" };
+    }
+
+    const json = (await resp.json()) as {
+      choices?: Array<{ message?: { content?: string } }>;
+    };
+
+    const raw = json.choices?.[0]?.message?.content?.trim() || "";
+    return parseClassification(raw);
+  } catch (err) {
+    // Timeout, network error, or model unavailable — fail open
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(
+      `[contradiction] classification failed (${message}) — defaulting to ADD`,
+    );
+    return { action: "ADD" };
+  }
+}
+
+// ─── Internal Helpers ───────────────────────────────────────────────────────
+
+/**
+ * Parse LLM response into a ContradictionResult.
+ * Accepts: "ADD", "NOOP", "DELETE 123", "DELETE <rowid>"
+ * Any parse failure defaults to ADD.
+ */
+function parseClassification(raw: string): ContradictionResult {
+  const normalized = raw.toUpperCase().trim();
+
+  if (normalized === "NOOP") {
+    return { action: "NOOP" };
+  }
+
+  if (normalized === "ADD") {
+    return { action: "ADD" };
+  }
+
+  // Match "DELETE <number>" pattern
+  const deleteMatch = normalized.match(/^DELETE\s+(\d+)/);
+  if (deleteMatch) {
+    const rowid = parseInt(deleteMatch[1], 10);
+    if (!isNaN(rowid) && rowid > 0) {
+      return { action: "DELETE+ADD", deleteRowid: rowid };
+    }
+  }
+
+  // Unparseable — default to ADD
+  console.error(
+    `[contradiction] unparseable response "${raw}" — defaulting to ADD`,
+  );
+  return { action: "ADD" };
+}
+
+/**
+ * Map event type to source name (mirrors realtime.ts getSourceForEvent).
+ */
+function getSourceForEvent(event: CaptureEvent): string {
+  switch (event.type) {
+    case "knowledge":
+      return "captures";
+    case "teaching":
+      return "teachings";
+    case "observation":
+      return "observations";
+    case "insight":
+      return "insights";
+    case "learning":
+      return "learnings";
+    case "task":
+      return "flux";
+    case "note":
+      return "captures";
+    default:
+      return "captures";
+  }
+}

package/lib/indexer.ts

@@ -40,6 +40,18 @@ export interface IndexerContext {
 
 export type IndexerFunction = (ctx: IndexerContext) => Promise<void>;
 
+/**
+ * Sources excluded from --rebuild because the DB is their source of truth.
+ * These sources use real-time indexing (indexAndEmbed) with contradiction
+ * resolution. Rebuilding from log.jsonl would resurrect purged/superseded
+ * entries, breaking the contradiction resolution invariant.
+ */
+export const REBUILD_EXCLUDED_SOURCES = new Set([
+  "captures",
+  "observations",
+  "teachings",
+]);
+
 /**
  * Check if a path is configured and exists on disk.
  * Logs a specific reason when the check fails:
@@ -231,6 +243,14 @@ export async function runIndexer(
         continue;
       }
 
+      // Skip rebuild-excluded sources — DB is source of truth, not log.jsonl
+      if (rebuild && REBUILD_EXCLUDED_SOURCES.has(src)) {
+        console.log(
+          `${src}: skipped (DB is source of truth — not rebuilt from log)`,
+        );
+        continue;
+      }
+
       console.log(`Indexing ${src}...`);
 
       // Clear source if rebuilding

package/lib/purge.ts

@@ -10,8 +10,16 @@
  */
 
 import { Database } from "bun:sqlite";
-import { existsSync } from "fs";
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  renameSync,
+  unlinkSync,
+} from "fs";
+import { join } from "path";
 import { getDatabasePath, openDatabase } from "./db.js";
+import { getConfig } from "./config.js";
 
 // Only these sources can be purged — indexed sources (blogs, commits, etc.) are never purgeable
 export const PURGEABLE_SOURCES = [
@@ -33,6 +41,7 @@ export interface PurgeMatch {
 export interface PurgeResult {
   deleted: number;
   rowids: number[];
+  logEntriesRemoved: number;
 }
 
 /**
@@ -76,14 +85,20 @@ export function findPurgeMatches(
 }
 
 /**
- * Delete entries from both FTS5 search table and vec0 embeddings table.
+ * Delete entries from FTS5 search table, vec0 embeddings table,
+ * and optionally clean matching lines from log.jsonl.
  *
- * FTS5 DELETE: DELETE FROM search WHERE rowid = ?
- * vec0 DELETE: DELETE FROM embeddings WHERE doc_id = ?
+ * @param rowids - Row IDs to delete from search + embeddings
+ * @param matchContents - Content strings from findPurgeMatches for log.jsonl filtering.
+ *   Optional (default []); when empty, log.jsonl cleanup is skipped.
+ *   This avoids a breaking change for existing callers.
  */
-export function deleteEntries(rowids: number[]): PurgeResult {
+export function deleteEntries(
+  rowids: number[],
+  matchContents: string[] = [],
+): PurgeResult {
   if (rowids.length === 0) {
-    return { deleted: 0, rowids: [] };
+    return { deleted: 0, rowids: [], logEntriesRemoved: 0 };
   }
 
   // Open DB directly for read-write (matches realtime.ts pattern —
@@ -118,8 +133,66 @@ export function deleteEntries(rowids: number[]): PurgeResult {
       deleted++;
     }
 
-    return { deleted, rowids };
+    // Clean matching lines from log.jsonl (best-effort)
+    const logEntriesRemoved = purgeLogEntries(matchContents);
+
+    return { deleted, rowids, logEntriesRemoved };
   } finally {
     db.close();
   }
 }
+
+/**
+ * Remove lines from log.jsonl whose content matches any of the given strings.
+ *
+ * Uses atomic write: writes filtered content to a temp file, then renames.
+ * Matches on event.data.content (the raw capture content), not the assembled
+ * search table content — task entries may use assembled content that differs.
+ * This is acceptable: the rebuild exclusion (Change 2) is the hard guard;
+ * log.jsonl cleanup is best-effort.
+ *
+ * @param matchContents - Content strings to filter out
+ * @returns Number of lines removed
+ */
+function purgeLogEntries(matchContents: string[]): number {
+  if (matchContents.length === 0) return 0;
+
+  const logPath = join(getConfig().paths.data, "log.jsonl");
+  const tmpPath = logPath + ".tmp";
+
+  if (!existsSync(logPath)) return 0;
+
+  try {
+    // Clean up stale temp file from a prior crash
+    if (existsSync(tmpPath)) {
+      unlinkSync(tmpPath);
+    }
+
+    const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+    const filtered = lines.filter((line) => {
+      try {
+        const event = JSON.parse(line) as {
+          data?: { content?: string; text?: string };
+        };
+        const content = event.data?.content || event.data?.text || "";
+        return !matchContents.some((mc) => content.includes(mc));
+      } catch {
+        return true; // Keep unparseable lines
+      }
+    });
+
+    writeFileSync(
+      tmpPath,
+      filtered.join("\n") + (filtered.length > 0 ? "\n" : ""),
+      "utf-8",
+    );
+    renameSync(tmpPath, logPath);
+
+    return lines.length - filtered.length;
+  } catch (err) {
+    // log.jsonl cleanup is best-effort — never fail the purge
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`[purge] log.jsonl cleanup failed (${message})`);
+    return 0;
+  }
+}

package/lib/realtime.ts

@@ -25,6 +25,11 @@ import {
 } from "./semantic.js";
 import { hashContent, getCachedEmbedding, cacheEmbedding } from "./cache.js";
 import type { CaptureEvent } from "./capture.js";
+import {
+  isContradictionCheckable,
+  findCandidates,
+  classifyContradiction,
+} from "./contradiction.js";
 
 /**
  * Index and embed capture events for immediate searchability
@@ -53,20 +58,67 @@ export async function indexAndEmbed(events: CaptureEvent[]): Promise<void> {
     }
     db.loadExtension(vecPath);
 
+    // 0. Contradiction detection — filter events before insert
+    //    For purgeable sources, check if the new event contradicts or
+    //    duplicates existing entries. NOOP skips the event, DELETE+ADD
+    //    removes the old entry before inserting the new one.
+    const eventsToIndex: CaptureEvent[] = [];
+    for (const event of events) {
+      const source = getSourceForEvent(event);
+
+      if (isContradictionCheckable(source)) {
+        try {
+          const candidates = await findCandidates(event);
+          if (candidates.length > 0) {
+            const result = await classifyContradiction(event, candidates);
+
+            if (result.action === "NOOP") {
+              const data = event.data as Record<string, unknown>;
+              const topic = String(data.topic || "");
+              console.error(
+                `[contradiction] NOOP: skipped as redundant (topic: ${topic})`,
+              );
+              continue;
+            }
+
+            if (result.action === "DELETE+ADD" && result.deleteRowid) {
+              deleteSearchAndEmbedding(db, result.deleteRowid);
+              const data = event.data as Record<string, unknown>;
+              const topic = String(data.topic || "");
+              console.error(
+                `[contradiction] DELETE+ADD: removed rowid ${result.deleteRowid}, topic: ${topic}`,
+              );
+            }
+            // ADD falls through to normal insert
+          }
+        } catch (err) {
+          // Fail open — if contradiction check fails, proceed with ADD
+          const message = err instanceof Error ? err.message : String(err);
+          console.error(
+            `[contradiction] check failed (${message}) — proceeding with ADD`,
+          );
+        }
+      }
+
+      eventsToIndex.push(event);
+    }
+
+    if (eventsToIndex.length === 0) return;
+
     // 1. Insert into FTS5 and collect doc IDs
     const docIds: number[] = [];
-    for (const event of events) {
+    for (const event of eventsToIndex) {
       const docId = insertSearchEntry(db, event);
       docIds.push(docId);
     }
 
     // 2. Generate embeddings with cache
-    const contents = events.map((e) => getContentForEmbedding(e));
+    const contents = eventsToIndex.map((e) => getContentForEmbedding(e));
     const embeddings = await embedWithCache(db, contents);
 
     // 3. Insert embeddings
-    for (let i = 0; i < events.length; i++) {
-      insertEmbedding(db, docIds[i], embeddings[i], events[i]);
+    for (let i = 0; i < eventsToIndex.length; i++) {
+      insertEmbedding(db, docIds[i], embeddings[i], eventsToIndex[i]);
     }
   } finally {
     db.close();
@@ -104,6 +156,16 @@ function insertSearchEntry(db: Database, event: CaptureEvent): number {
   return Number(result.lastInsertRowid);
 }
 
+/**
+ * Delete an entry from both FTS5 search and vec0 embeddings tables.
+ * Used by contradiction resolution to remove superseded entries.
+ * Reuses the same prepared statement pattern as purge.ts:108-114.
+ */
+function deleteSearchAndEmbedding(db: Database, rowid: number): void {
+  db.prepare("DELETE FROM search WHERE rowid = ?").run(rowid);
+  db.prepare("DELETE FROM embeddings WHERE doc_id = ?").run(rowid);
+}
+
 /**
  * Map event type to source name used in search table
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voidwire/lore",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Unified knowledge CLI - Search, list, and capture your indexed knowledge",
   "type": "module",
   "main": "./index.ts",