@voidwire/lore 0.9.0 → 1.0.0

package/lib/indexers/obsidian.ts

@@ -0,0 +1,151 @@
+/**
+ * lib/indexers/obsidian.ts - Obsidian vault indexer
+ *
+ * Recursively scans obsidian directory for markdown files.
+ * Extracts project, domain, and status from frontmatter.
+ * Skips personal subdirectory for privacy.
+ *
+ * Source: obsidian
+ * Topic: frontmatter project > domain > parent directory name > empty
+ * Type: (empty)
+ * Timestamp: file mtime as ISO 8601
+ */
+
+import { readdirSync, readFileSync, statSync, existsSync } from "fs";
+import { join, basename, dirname } from "path";
+import type { IndexerContext } from "../indexer";
+
+function walkMarkdownFiles(
+  dir: string,
+  rootDir: string,
+  files: string[] = [],
+): string[] {
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      // Skip personal directory (privacy filter)
+      if (entry.name === "personal") continue;
+
+      walkMarkdownFiles(fullPath, rootDir, files);
+    } else if (entry.isFile() && entry.name.endsWith(".md")) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+export async function indexObsidian(ctx: IndexerContext): Promise<void> {
+  const obsidianDir = ctx.config.paths.obsidian;
+
+  if (!existsSync(obsidianDir)) {
+    console.log(`Obsidian directory not found: ${obsidianDir}`);
+    return;
+  }
+
+  const files = walkMarkdownFiles(obsidianDir, obsidianDir);
+
+  for (const filePath of files) {
+    try {
+      const raw = readFileSync(filePath, "utf-8");
+      const mtime = statSync(filePath).mtime;
+
+      let content = raw;
+      let project: string | undefined;
+      let domain: string | undefined;
+      let status: string | undefined;
+      let fmDate: string | undefined;
+      let fmStarted: string | undefined;
+
+      // Extract frontmatter
+      const frontmatterMatch = raw.match(/^---\n([\s\S]*?)\n---\n/);
+      if (frontmatterMatch) {
+        const frontmatter = frontmatterMatch[1];
+
+        // Skip private notes
+        const privateMatch = frontmatter.match(/^private:\s*(true|yes)$/m);
+        if (privateMatch) continue;
+
+        const projectMatch = frontmatter.match(/^project:\s*(.+)$/m);
+        const domainMatch = frontmatter.match(/^domain:\s*(.+)$/m);
+        const statusMatch = frontmatter.match(/^status:\s*(.+)$/m);
+        const dateMatch = frontmatter.match(/^date:\s*(.+)$/m);
+        const startedMatch = frontmatter.match(/^started:\s*(.+)$/m);
+
+        const tagsMatch = frontmatter.match(/^tags:\s*\[(.+)\]$/m);
+        const tagsMultiMatch = !tagsMatch
+          ? frontmatter.match(/^tags:\s*\n((?:\s+-\s+.+\n?)+)/m)
+          : null;
+
+        if (projectMatch) project = projectMatch[1].trim();
+        if (domainMatch) domain = domainMatch[1].trim();
+        if (statusMatch) status = statusMatch[1].trim();
+        if (dateMatch) fmDate = dateMatch[1].trim();
+        if (startedMatch) fmStarted = startedMatch[1].trim();
+
+        let tags: string[] = [];
+        if (tagsMatch) {
+          tags = tagsMatch[1].split(",").map((t) => t.trim().replace(/"/g, ""));
+        } else if (tagsMultiMatch) {
+          tags = tagsMultiMatch[1]
+            .split("\n")
+            .map((l) => l.replace(/^\s+-\s+/, "").trim())
+            .filter(Boolean);
+        }
+
+        content = raw.slice(frontmatterMatch[0].length);
+
+        if (tags.length > 0) {
+          content += `\nTags: ${tags.join(", ")}`;
+        }
+      }
+
+      // Topic derivation: project > domain > parent directory > empty
+      let topic = "";
+      if (project) {
+        topic = project;
+      } else if (domain) {
+        topic = domain;
+      } else {
+        const parentDir = basename(dirname(filePath));
+        if (parentDir !== basename(obsidianDir)) {
+          topic = parentDir;
+        }
+      }
+
+      // Timestamp cascade: frontmatter date > started > file mtime
+      let timestamp: string;
+      if (fmDate) {
+        timestamp = fmDate.includes("T")
+          ? fmDate
+          : `${fmDate.slice(0, 10)}T00:00:00Z`;
+      } else if (fmStarted) {
+        timestamp = fmStarted.includes("T")
+          ? fmStarted
+          : `${fmStarted.slice(0, 10)}T00:00:00Z`;
+      } else {
+        timestamp = mtime.toISOString();
+      }
+
+      const metadata: Record<string, unknown> = {};
+      if (status) metadata.status = status;
+
+      const title = basename(filePath, ".md");
+
+      ctx.insert({
+        source: "obsidian",
+        title,
+        content,
+        topic,
+        timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      console.warn(`Failed to read ${filePath}: ${e}`);
+      continue;
+    }
+  }
+}

package/lib/indexers/personal.ts

@@ -0,0 +1,262 @@
+/**
+ * lib/indexers/personal.ts - Personal data indexer
+ *
+ * Reads JSON files from the personal data directory and indexes
+ * 8 types: book, person, movie, podcast, interest, habit, profile, preference.
+ *
+ * Source: personal
+ * Topic: (empty - type handles categorization)
+ * Type: derived from JSON filename
+ * Timestamp: file mtime, or date_read/date_watched when available
+ */
+
+import { readFileSync, statSync, existsSync } from "fs";
+import { join } from "path";
+import type { IndexerContext } from "../indexer";
+
+function fileMtime(path: string): string {
+  return statSync(path).mtime.toISOString();
+}
+
+function toISO(dateStr: string, fallback: string): string {
+  if (!dateStr) return fallback;
+  const s = String(dateStr);
+  return s.includes("T") ? s : `${s.slice(0, 10)}T00:00:00Z`;
+}
+
+export async function indexPersonal(ctx: IndexerContext): Promise<void> {
+  const personalDir = ctx.config.paths.personal;
+
+  if (!existsSync(personalDir)) {
+    console.log(`Personal data directory not found: ${personalDir}`);
+    return;
+  }
+
+  // Books
+  const booksPath = join(personalDir, "books.json");
+  if (existsSync(booksPath)) {
+    try {
+      const booksTs = fileMtime(booksPath);
+      const books = JSON.parse(readFileSync(booksPath, "utf-8"));
+      for (const book of books) {
+        if (!book.title) continue;
+        const content = `${book.title} by ${book.author || "unknown"}\n${book.notes || ""}`;
+        const timestamp = book.date_read
+          ? toISO(book.date_read, booksTs)
+          : booksTs;
+
+        ctx.insert({
+          source: "personal",
+          title: `[book] ${book.title}`,
+          content,
+          topic: "",
+          type: "book",
+          timestamp,
+          metadata: { name: book.title },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read books.json: ${e}`);
+    }
+  }
+
+  // People
+  const peoplePath = join(personalDir, "people.json");
+  if (existsSync(peoplePath)) {
+    try {
+      const peopleTs = fileMtime(peoplePath);
+      const people = JSON.parse(readFileSync(peoplePath, "utf-8"));
+      for (const person of people) {
+        if (!person.name) continue;
+        const content = `${person.name}\n${person.relationship || ""}\n${person.notes || ""}`;
+
+        ctx.insert({
+          source: "personal",
+          title: `[person] ${person.name}`,
+          content,
+          topic: "",
+          type: "person",
+          timestamp: peopleTs,
+          metadata: { name: person.name },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read people.json: ${e}`);
+    }
+  }
+
+  // Movies
+  const moviesPath = join(personalDir, "movies.json");
+  if (existsSync(moviesPath)) {
+    try {
+      const moviesTs = fileMtime(moviesPath);
+      const movies = JSON.parse(readFileSync(moviesPath, "utf-8"));
+      for (const movie of movies) {
+        if (!movie.title) continue;
+        const year = movie.year || "";
+        const content = year
+          ? `${movie.title} (${year})\n${movie.notes || ""}`
+          : `${movie.title}\n${movie.notes || ""}`;
+        const timestamp = movie.date_watched
+          ? toISO(movie.date_watched, moviesTs)
+          : moviesTs;
+
+        ctx.insert({
+          source: "personal",
+          title: `[movie] ${movie.title}`,
+          content,
+          topic: "",
+          type: "movie",
+          timestamp,
+          metadata: { name: movie.title },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read movies.json: ${e}`);
+    }
+  }
+
+  // Podcasts (field is "title", not "name")
+  const podcastsPath = join(personalDir, "podcasts.json");
+  if (existsSync(podcastsPath)) {
+    try {
+      const podcastsTs = fileMtime(podcastsPath);
+      const podcasts = JSON.parse(readFileSync(podcastsPath, "utf-8"));
+      for (const podcast of podcasts) {
+        const name = podcast.title || "";
+        if (!name) continue;
+        const content = `${name}\n${podcast.description || ""}`;
+
+        ctx.insert({
+          source: "personal",
+          title: `[podcast] ${name}`,
+          content,
+          topic: "",
+          type: "podcast",
+          timestamp: podcastsTs,
+          metadata: { name },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read podcasts.json: ${e}`);
+    }
+  }
+
+  // Interests (array of strings, not objects)
+  const interestsPath = join(personalDir, "interests.json");
+  if (existsSync(interestsPath)) {
+    try {
+      const interestsTs = fileMtime(interestsPath);
+      const interests = JSON.parse(readFileSync(interestsPath, "utf-8"));
+      for (const interest of interests) {
+        if (typeof interest !== "string" || !interest) continue;
+
+        ctx.insert({
+          source: "personal",
+          title: `[interest] ${interest}`,
+          content: interest,
+          topic: "",
+          type: "interest",
+          timestamp: interestsTs,
+          metadata: { name: interest },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read interests.json: ${e}`);
+    }
+  }
+
+  // Habits (field is "habit" not "name", "frequency" not "description")
+  const habitsPath = join(personalDir, "habits.json");
+  if (existsSync(habitsPath)) {
+    try {
+      const habitsTs = fileMtime(habitsPath);
+      const habits = JSON.parse(readFileSync(habitsPath, "utf-8"));
+      for (const habit of habits) {
+        const habitName = habit.habit || "";
+        if (!habitName) continue;
+        const frequency = habit.frequency || "";
+        const content = frequency ? `${habitName} (${frequency})` : habitName;
+
+        ctx.insert({
+          source: "personal",
+          title: `[habit] ${habitName}`,
+          content,
+          topic: "",
+          type: "habit",
+          timestamp: habitsTs,
+          metadata: { name: habitName },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read habits.json: ${e}`);
+    }
+  }
+
+  // Profile (one entry per field)
+  const profilePath = join(personalDir, "profile.json");
+  if (existsSync(profilePath)) {
+    try {
+      const profileTs = fileMtime(profilePath);
+      const profile = JSON.parse(readFileSync(profilePath, "utf-8"));
+      for (const [key, value] of Object.entries(profile)) {
+        if (value === null || value === undefined) continue;
+        const content = `${key}: ${String(value)}`;
+
+        ctx.insert({
+          source: "personal",
+          title: `[profile] ${key}`,
+          content,
+          topic: "",
+          type: "profile",
+          timestamp: profileTs,
+          metadata: { name: key },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to read profile.json: ${e}`);
+    }
+  }
+
+  // Preferences (one entry per category/key)
+  const preferencesPath = join(personalDir, "preferences.json");
+  if (existsSync(preferencesPath)) {
+    try {
+      const prefsTs = fileMtime(preferencesPath);
+      const preferences = JSON.parse(readFileSync(preferencesPath, "utf-8"));
+      for (const [category, settings] of Object.entries(preferences)) {
+        if (typeof settings === "object" && settings !== null) {
+          for (const [key, value] of Object.entries(
+            settings as Record<string, unknown>,
+          )) {
+            const content = `${category}/${key}: ${String(value)}`;
+
+            ctx.insert({
+              source: "personal",
+              title: `[preference] ${category}/${key}`,
+              content,
+              topic: "",
+              type: "preference",
+              timestamp: prefsTs,
+              metadata: { name: `${category}/${key}` },
+            });
+          }
+        } else {
+          const content = `${category}: ${String(settings)}`;
+
+          ctx.insert({
+            source: "personal",
+            title: `[preference] ${category}`,
+            content,
+            topic: "",
+            type: "preference",
+            timestamp: prefsTs,
+            metadata: { name: category },
+          });
+        }
+      }
+    } catch (e) {
+      console.warn(`Failed to read preferences.json: ${e}`);
+    }
+  }
+}

package/lib/indexers/readmes.ts

@@ -0,0 +1,49 @@
+/**
+ * lib/indexers/readmes.ts - READMEs indexer
+ *
+ * Scans project directories for README.md and indexes content.
+ * Framework handles chunking for large READMEs.
+ *
+ * Source: readmes, Topic: project directory name,
+ * Type: empty, Timestamp: file mtime as ISO 8601
+ */
+
+import { readdirSync, readFileSync, statSync, existsSync } from "fs";
+import { join } from "path";
+import type { IndexerContext } from "../indexer";
+
+export async function indexReadmes(ctx: IndexerContext): Promise<void> {
+  const projectsDir = ctx.config.paths.projects;
+
+  if (!existsSync(projectsDir)) {
+    console.log(`Projects directory not found: ${projectsDir}`);
+    return;
+  }
+
+  const projects = readdirSync(projectsDir, { withFileTypes: true })
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  for (const project of projects) {
+    const readmePath = join(projectsDir, project, "README.md");
+
+    if (!existsSync(readmePath)) continue;
+
+    try {
+      const content = readFileSync(readmePath, "utf-8");
+      const mtime = statSync(readmePath).mtime;
+      const timestamp = mtime.toISOString();
+
+      ctx.insert({
+        source: "readmes",
+        title: `README: ${project}`,
+        content,
+        topic: project,
+        timestamp,
+      });
+    } catch (e) {
+      console.warn(`Failed to read ${readmePath}: ${e}`);
+      continue;
+    }
+  }
+}

package/lib/indexers/sessions.ts

@@ -0,0 +1,127 @@
+/**
+ * lib/indexers/sessions.ts - Sessions indexer
+ *
+ * Reads session event JSONL files (hook telemetry) and aggregates by session_id.
+ * Each session gets one entry with summary, tools, model, and token counts.
+ *
+ * Source: sessions
+ * Topic: event.project (project the session was in)
+ * Type: (empty)
+ * Timestamp: first event timestamp per session
+ */
+
+import { readdirSync, readFileSync, existsSync } from "fs";
+import { join } from "path";
+import type { IndexerContext } from "../indexer";
+
+interface SessionData {
+  project: string;
+  totalInput: number;
+  totalOutput: number;
+  toolsUsed: Set<string>;
+  model: string | null;
+  summary: string | null;
+  eventCount: number;
+  firstTs: string;
+  lastTs: string;
+}
+
+export async function indexSessions(ctx: IndexerContext): Promise<void> {
+  const eventsDir = ctx.config.paths.session_events;
+  if (!eventsDir || !existsSync(eventsDir)) {
+    console.log("No session events directory found, skipping sessions");
+    return;
+  }
+
+  const eventFiles = readdirSync(eventsDir)
+    .filter((f) => f.endsWith(".jsonl"))
+    .sort()
+    .map((f) => join(eventsDir, f));
+
+  if (eventFiles.length === 0) {
+    console.log("No session event files found, skipping sessions");
+    return;
+  }
+
+  const sessions = new Map<string, SessionData>();
+
+  for (const eventFile of eventFiles) {
+    try {
+      const lines = readFileSync(eventFile, "utf-8")
+        .split("\n")
+        .filter(Boolean);
+      for (const line of lines) {
+        try {
+          const event = JSON.parse(line);
+          const sessionId = event.session_id;
+          if (!sessionId) continue;
+
+          if (!sessions.has(sessionId)) {
+            sessions.set(sessionId, {
+              project: event.project || "unknown",
+              totalInput: 0,
+              totalOutput: 0,
+              toolsUsed: new Set(),
+              model: null,
+              summary: null,
+              eventCount: 0,
+              firstTs: event.ts || "",
+              lastTs: event.ts || "",
+            });
+          }
+
+          const session = sessions.get(sessionId)!;
+          session.eventCount++;
+          if (event.ts) session.lastTs = event.ts;
+
+          const data = event.data || {};
+          const tokens = data.tokens || {};
+          session.totalInput += tokens.input || 0;
+          session.totalOutput += tokens.output || 0;
+
+          const tools: string[] = data.tools_used || [];
+          for (const tool of tools) session.toolsUsed.add(tool);
+
+          if (!session.model && data.model) session.model = data.model;
+          if (data.summary) session.summary = data.summary;
+        } catch {
+          continue;
+        }
+      }
+    } catch (e) {
+      console.warn(`Failed to read ${eventFile}: ${e}`);
+      continue;
+    }
+  }
+
+  for (const [sessionId, session] of sessions) {
+    const tools = Array.from(session.toolsUsed).sort();
+    const model = session.model || "unknown";
+    const totalTokens = session.totalInput + session.totalOutput;
+    const date = session.firstTs.slice(0, 10) || "unknown";
+
+    let content: string;
+    if (session.summary) {
+      content = `${session.summary} Tools: ${tools.join(", ")}. Model: ${model}. Tokens: ${session.totalInput} input, ${session.totalOutput} output.`;
+    } else {
+      content = `Session for ${session.project} on ${date}. Tools: ${tools.join(", ")}. Model: ${model}. Events: ${session.eventCount}. Tokens: ${session.totalInput} input, ${session.totalOutput} output.`;
+    }
+
+    if (!content) continue;
+
+    ctx.insert({
+      source: "sessions",
+      title: `[session] ${session.project} (${date})`,
+      content,
+      topic: session.project,
+      timestamp: session.firstTs,
+      metadata: {
+        session_id: sessionId,
+        model,
+        tools_used: tools,
+        total_tokens: totalTokens,
+        event_count: session.eventCount,
+      },
+    });
+  }
+}

package/lib/indexers/teachings.ts

@@ -0,0 +1,52 @@
+/**
+ * lib/indexers/teachings.ts - Teachings indexer
+ *
+ * Reads log.jsonl and indexes teaching captures.
+ * Filters for event=captured AND type=teaching.
+ *
+ * Source: teachings
+ * Topic: data.topic (AI-written)
+ * Type: teaching (fixed)
+ * Timestamp: event timestamp
+ */
+
+import { readFileSync, existsSync } from "fs";
+import type { IndexerContext } from "../indexer";
+
+export async function indexTeachings(ctx: IndexerContext): Promise<void> {
+  const logPath = `${ctx.config.paths.data}/log.jsonl`;
+  if (!existsSync(logPath)) {
+    console.log("No log.jsonl found, skipping teachings");
+    return;
+  }
+
+  const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.event !== "captured" || event.type !== "teaching") continue;
+
+      const topic = event.data?.topic || "general";
+      const content = event.data?.content || "";
+      const confidence = event.data?.confidence;
+
+      if (!content) continue;
+
+      const metadata: Record<string, unknown> = {};
+      if (confidence) metadata.confidence = confidence;
+
+      ctx.insert({
+        source: "teachings",
+        title: `[teaching] ${topic}`,
+        content,
+        topic,
+        type: "teaching",
+        timestamp: event.timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      continue;
+    }
+  }
+}

package/lib/info.ts

@@ -62,11 +62,11 @@ export function info(): InfoOutput {
     const totalResult = totalStmt.get() as { total: number };
     const total_entries = totalResult?.total ?? 0;
 
-    // Get last indexed timestamp from metadata
+    // Get last indexed timestamp from column
     const tsStmt = db.prepare(`
-      SELECT MAX(json_extract(metadata, '$.timestamp')) as ts
+      SELECT MAX(timestamp) as ts
       FROM search
-      WHERE json_extract(metadata, '$.timestamp') IS NOT NULL
+      WHERE timestamp IS NOT NULL AND timestamp != ''
     `);
     const tsResult = tsStmt.get() as { ts: string | null };
     const last_indexed = tsResult?.ts ?? new Date().toISOString();