@voidwire/lore 0.9.1 → 1.0.1

package/lib/indexers/captures.ts

@@ -0,0 +1,105 @@
+/**
+ * lib/indexers/captures.ts - Captures indexer
+ *
+ * Reads log.jsonl and indexes knowledge, task, and note captures.
+ * Filters for event=captured AND type in [knowledge, task, note].
+ *
+ * Source: captures
+ * Topic: data.topic (AI-written)
+ * Type: completion (task), note (note), or data.subtype (knowledge)
+ * Timestamp: event timestamp
+ */
+
+import { readFileSync, existsSync } from "fs";
+import type { IndexerContext } from "../indexer";
+
+export async function indexCaptures(ctx: IndexerContext): Promise<void> {
+  const logPath = `${ctx.config.paths.data}/log.jsonl`;
+  if (!existsSync(logPath)) {
+    console.log("No log.jsonl found, skipping captures");
+    return;
+  }
+
+  const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.event !== "captured") continue;
+
+      const eventType = event.type;
+      if (!["knowledge", "task", "note"].includes(eventType)) continue;
+
+      const topic = event.data?.topic || "general";
+      let content = "";
+      let title = "";
+      const metadata: Record<string, unknown> = {};
+      let type = "";
+
+      if (eventType === "task") {
+        const name = event.data?.name || "untitled";
+        const problem = event.data?.problem || "";
+        const solution = event.data?.solution || "";
+        const code = event.data?.code || "";
+        const discoveries = event.data?.discoveries?.join(", ") || "";
+        const deviations = event.data?.deviations || "";
+        const pattern = event.data?.pattern || "";
+        const difficulty = event.data?.difficulty || "";
+
+        content = `Problem: ${problem}`;
+        if (solution) content += `\nSolution: ${solution}`;
+        if (code) content += `\nCode: ${code}`;
+        if (discoveries) content += `\nDiscoveries: ${discoveries}`;
+        if (deviations) content += `\nDeviations: ${deviations}`;
+        if (pattern) content += `\nPattern: ${pattern}`;
+
+        const techTags: string[] = [];
+        if (event.data?.tech) techTags.push(`Tech: ${event.data.tech}`);
+        if (event.data?.tags) {
+          const tags = Array.isArray(event.data.tags)
+            ? event.data.tags
+            : [event.data.tags];
+          techTags.push(`Tags: ${tags.join(", ")}`);
+        }
+        if (techTags.length > 0) content += `\n${techTags.join("\n")}`;
+
+        title = `[completion] ${topic}: ${name}`;
+        type = "completion";
+
+        if (event.data?.tags) metadata.tags = event.data.tags;
+        if (event.data?.tech) metadata.tech = event.data.tech;
+        if (difficulty) metadata.difficulty = difficulty;
+      } else if (eventType === "note") {
+        content = event.data?.content || "";
+        const tags = event.data?.tags || [];
+        const tagsStr = tags.join(", ");
+
+        title = tagsStr ? `[note] ${tagsStr}` : "[note] untagged";
+        type = "note";
+
+        if (tags.length > 0) metadata.tags = tags;
+      } else {
+        // knowledge
+        content = event.data?.content || "";
+        const subtype = event.data?.subtype || "insight";
+
+        title = `[${subtype}] ${topic}`;
+        type = subtype;
+      }
+
+      if (!content) continue;
+
+      ctx.insert({
+        source: "captures",
+        title,
+        content,
+        topic,
+        type,
+        timestamp: event.timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      continue;
+    }
+  }
+}

package/lib/indexers/commits.ts

@@ -0,0 +1,90 @@
+/**
+ * lib/indexers/commits.ts - Git commits indexer
+ *
+ * Scans all project directories for git repos and indexes commit history.
+ * Uses record/unit separators to avoid delimiter collisions in messages.
+ *
+ * Source: commits
+ * Topic: project directory name (repo name)
+ * Type: (empty)
+ * Timestamp: commit author date as ISO 8601
+ */
+
+import { readdirSync, existsSync } from "fs";
+import { join } from "path";
+import { spawnSync } from "child_process";
+import type { IndexerContext } from "../indexer";
+
+export async function indexCommits(ctx: IndexerContext): Promise<void> {
+  const projectsDir = ctx.config.paths.projects;
+
+  if (!existsSync(projectsDir)) {
+    console.log(`Projects directory not found: ${projectsDir}`);
+    return;
+  }
+
+  const projects = readdirSync(projectsDir, { withFileTypes: true })
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  for (const project of projects) {
+    const repoPath = join(projectsDir, project);
+    const gitDir = join(repoPath, ".git");
+
+    if (!existsSync(gitDir)) continue;
+
+    try {
+      // Use record separator (%x1e) and unit separator (%x1f) to avoid
+      // delimiter collisions with commit message content
+      const SEP = "\x1e"; // Record separator between commits
+      const UNIT = "\x1f"; // Unit separator between fields
+      const result = spawnSync(
+        "git",
+        [
+          "log",
+          "--all",
+          `--format=${SEP}%H${UNIT}%an${UNIT}%aI${UNIT}%s${UNIT}%b`,
+        ],
+        {
+          cwd: repoPath,
+          encoding: "utf-8",
+          maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+        },
+      );
+
+      if (result.error || result.status !== 0) {
+        console.warn(`Failed to read git log for ${project}: ${result.stderr}`);
+        continue;
+      }
+
+      const records = result.stdout.split(SEP).filter(Boolean);
+
+      for (const record of records) {
+        const parts = record.split(UNIT);
+        if (parts.length < 4) continue;
+
+        const [sha, author, timestamp, subject, ...bodyParts] = parts;
+        const body = bodyParts.join("").trim();
+        const baseContent = body || subject;
+        const content = author
+          ? `Author: ${author}\n${baseContent}`
+          : baseContent;
+
+        ctx.insert({
+          source: "commits",
+          title: `[commit] ${subject}`,
+          content,
+          topic: project,
+          timestamp,
+          metadata: {
+            sha: sha.trim(),
+            author,
+          },
+        });
+      }
+    } catch (e) {
+      console.warn(`Failed to index commits for ${project}: ${e}`);
+      continue;
+    }
+  }
+}

package/lib/indexers/development.ts

@@ -0,0 +1,68 @@
+/**
+ * lib/indexers/development.ts - Development projects indexer
+ *
+ * Scans project directories for .workflow/artifacts/PROJECT_SUMMARY.md
+ * and indexes project summaries.
+ *
+ * Source: development, Topic: project directory name,
+ * Type: empty, Timestamp: file mtime as ISO 8601
+ */
+
+import { readdirSync, readFileSync, statSync, existsSync } from "fs";
+import { join } from "path";
+import type { IndexerContext } from "../indexer";
+
+export async function indexDevelopment(ctx: IndexerContext): Promise<void> {
+  const projectsDir = ctx.config.paths.projects;
+
+  if (!existsSync(projectsDir)) {
+    console.log(`Projects directory not found: ${projectsDir}`);
+    return;
+  }
+
+  const projects = readdirSync(projectsDir, { withFileTypes: true })
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  for (const project of projects) {
+    const summaryPath = join(
+      projectsDir,
+      project,
+      ".workflow",
+      "artifacts",
+      "PROJECT_SUMMARY.md",
+    );
+
+    if (!existsSync(summaryPath)) continue;
+
+    try {
+      const raw = readFileSync(summaryPath, "utf-8");
+      const mtime = statSync(summaryPath).mtime;
+      const timestamp = mtime.toISOString();
+
+      // Extract tech from **Stack:** line (matches bash script behavior)
+      let content = raw;
+      let tech: string | undefined;
+
+      const techMatch = raw.match(/^\*\*Stack:\*\*\s*(.+)$/m);
+      if (techMatch) {
+        tech = techMatch[1].trim();
+      }
+
+      const metadata: Record<string, unknown> = {};
+      if (tech) metadata.tech = tech;
+
+      ctx.insert({
+        source: "development",
+        title: `Project: ${project}`,
+        content,
+        topic: project,
+        timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      console.warn(`Failed to read ${summaryPath}: ${e}`);
+      continue;
+    }
+  }
+}

package/lib/indexers/events.ts

@@ -0,0 +1,61 @@
+/**
+ * lib/indexers/events.ts - Events indexer
+ *
+ * Aggregates development events from log.jsonl by project.
+ * Each project gets one entry with all event lines.
+ *
+ * Source: events
+ * Topic: project name
+ * Type: (empty)
+ * Timestamp: last event timestamp per project
+ */
+
+import { readFileSync, existsSync } from "fs";
+import type { IndexerContext } from "../indexer";
+
+export async function indexEvents(ctx: IndexerContext): Promise<void> {
+  const logPath = `${ctx.config.paths.data}/log.jsonl`;
+
+  if (!existsSync(logPath)) {
+    console.log("No log.jsonl found, skipping events");
+    return;
+  }
+
+  const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+  const projectData = new Map<
+    string,
+    { lines: string[]; lastTimestamp: string }
+  >();
+
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      const project = event.data?.topic || "general";
+      if (!projectData.has(project)) {
+        projectData.set(project, { lines: [], lastTimestamp: "" });
+      }
+      const data = projectData.get(project)!;
+      data.lines.push(
+        `[${event.timestamp}] ${event.event}: ${event.type || ""}`,
+      );
+      if (event.timestamp) {
+        data.lastTimestamp = event.timestamp;
+      }
+    } catch {
+      // Skip malformed JSON
+      continue;
+    }
+  }
+
+  for (const [project, data] of projectData) {
+    const content = data.lines.join("\n");
+
+    ctx.insert({
+      source: "events",
+      title: `Development events: ${project}`,
+      content,
+      topic: project,
+      timestamp: data.lastTimestamp,
+    });
+  }
+}

package/lib/indexers/explorations.ts

@@ -0,0 +1,89 @@
+/**
+ * lib/indexers/explorations.ts - Explorations indexer
+ *
+ * Recursively scans explorations directory for markdown files.
+ * Extracts project and status from frontmatter when available.
+ *
+ * Source: explorations
+ * Topic: frontmatter project or parent directory name
+ * Type: (empty)
+ * Timestamp: file mtime as ISO 8601
+ */
+
+import { readdirSync, readFileSync, statSync, existsSync } from "fs";
+import { join, basename, dirname } from "path";
+import type { IndexerContext } from "../indexer";
+
+function walkMarkdownFiles(dir: string, files: string[] = []): string[] {
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      walkMarkdownFiles(fullPath, files);
+    } else if (entry.isFile() && entry.name.endsWith(".md")) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+export async function indexExplorations(ctx: IndexerContext): Promise<void> {
+  const explorationsDir = ctx.config.paths.explorations;
+
+  if (!existsSync(explorationsDir)) {
+    console.log(`Explorations directory not found: ${explorationsDir}`);
+    return;
+  }
+
+  const files = walkMarkdownFiles(explorationsDir);
+
+  for (const filePath of files) {
+    try {
+      const raw = readFileSync(filePath, "utf-8");
+      const mtime = statSync(filePath).mtime;
+      const timestamp = mtime.toISOString();
+
+      let content = raw;
+      let project: string | undefined;
+      let status: string | undefined;
+
+      // Extract frontmatter
+      const frontmatterMatch = raw.match(/^---\n([\s\S]*?)\n---\n/);
+      if (frontmatterMatch) {
+        const frontmatter = frontmatterMatch[1];
+        const projectMatch = frontmatter.match(/^project:\s*(.+)$/m);
+        const statusMatch = frontmatter.match(/^status:\s*(.+)$/m);
+
+        if (projectMatch) project = projectMatch[1].trim();
+        if (statusMatch) status = statusMatch[1].trim();
+
+        content = raw.slice(frontmatterMatch[0].length);
+      }
+
+      // Fallback: use parent directory name as project
+      if (!project) {
+        project = basename(dirname(filePath));
+      }
+
+      const metadata: Record<string, unknown> = {};
+      if (status) metadata.status = status;
+
+      const title = basename(filePath, ".md");
+
+      ctx.insert({
+        source: "explorations",
+        title: `[exploration] ${title}`,
+        content,
+        topic: project,
+        timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      console.warn(`Failed to read ${filePath}: ${e}`);
+      continue;
+    }
+  }
+}

package/lib/indexers/flux.ts

@@ -0,0 +1,142 @@
+/**
+ * lib/indexers/flux.ts - Flux indexer
+ *
+ * Scans flux markdown files for todo and idea entries.
+ * Two passes: general flux (no project) and per-project flux.
+ * Line format: `- todo:: description id::xxx captured::date`
+ *              `- idea:: description id::xxx`
+ *
+ * Source: flux
+ * Topic: project directory name, or "general" for non-project items
+ * Type: todo or idea
+ * Timestamp: captured date if present, otherwise empty
+ */
+
+import { readdirSync, readFileSync, existsSync } from "fs";
+import { join, basename } from "path";
+import type { IndexerContext } from "../indexer";
+
+export async function indexFlux(ctx: IndexerContext): Promise<void> {
+  const fluxDir = ctx.config.paths.flux;
+  const fluxProjectsDir = ctx.config.paths.flux_projects;
+  let found = false;
+
+  // Pass 1: General flux files (no project association)
+  if (fluxDir && existsSync(fluxDir)) {
+    found = true;
+    const files = readdirSync(fluxDir).filter((f) => f.endsWith(".md"));
+    for (const file of files) {
+      const filePath = join(fluxDir, file);
+      const status = statusFromFilename(basename(file, ".md"));
+      try {
+        parseFluxFile(ctx, filePath, "general", status);
+      } catch (e) {
+        console.warn(`Failed to read ${filePath}: ${e}`);
+      }
+    }
+  }
+
+  // Pass 2: Per-project flux files (active.md, later.md)
+  if (fluxProjectsDir && existsSync(fluxProjectsDir)) {
+    found = true;
+    const projects = readdirSync(fluxProjectsDir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+
+    for (const project of projects) {
+      for (const filename of ["active.md", "later.md"]) {
+        const filePath = join(fluxProjectsDir, project, filename);
+        if (!existsSync(filePath)) continue;
+        const status = statusFromFilename(basename(filename, ".md"));
+        try {
+          parseFluxFile(ctx, filePath, project, status);
+        } catch (e) {
+          console.warn(`Failed to read ${filePath}: ${e}`);
+        }
+      }
+    }
+  }
+
+  if (!found) {
+    console.log("No flux directories found, skipping flux");
+  }
+}
+
+function statusFromFilename(name: string): string {
+  switch (name) {
+    case "active":
+      return "active";
+    case "later":
+      return "later";
+    case "inbox":
+      return "inbox";
+    default:
+      return "other";
+  }
+}
+
+function parseFluxFile(
+  ctx: IndexerContext,
+  filePath: string,
+  topic: string,
+  status: string,
+): void {
+  const raw = readFileSync(filePath, "utf-8");
+  const lines = raw.split("\n");
+
+  for (const line of lines) {
+    const match = line.match(/^- (todo|idea):: (.+)/);
+    if (!match) continue;
+
+    const type = match[1];
+    let rest = match[2];
+
+    // Extract id if present
+    let id = "";
+    const idMatch = rest.match(/\bid::(\S+)/);
+    if (idMatch) {
+      id = idMatch[1];
+      rest = rest.replace(/\s*id::\S+/, "");
+    }
+
+    // Extract captured date if present (may include time: captured:: 2025-08-13 10:52)
+    let timestamp = "";
+    const capturedMatch = rest.match(
+      /\bcaptured::\s*(\d{4}-\d{2}-\d{2})(?:\s+\d{2}:\d{2})?/,
+    );
+    if (capturedMatch) {
+      timestamp = `${capturedMatch[1]}T00:00:00Z`;
+      rest = rest.replace(
+        /\s*captured::\s*\d{4}-\d{2}-\d{2}(?:\s+\d{2}:\d{2})?/,
+        "",
+      );
+    }
+
+    // Extract archived date if present (strip from description)
+    rest = rest.replace(/\s*archived::\s*\S+/, "");
+
+    // Extract any remaining key::value pairs (like last::date)
+    rest = rest.replace(/\s*\w+::\s*\S+/g, "");
+
+    const description = rest.trim();
+    if (!description) continue;
+
+    const title =
+      topic !== "general"
+        ? `[${topic}] [${type}] ${description.slice(0, 80)}`
+        : `[${type}] ${description.slice(0, 80)}`;
+
+    ctx.insert({
+      source: "flux",
+      title,
+      content: description,
+      topic,
+      type,
+      timestamp,
+      metadata: {
+        id,
+        status,
+      },
+    });
+  }
+}

package/lib/indexers/index.ts

@@ -0,0 +1,41 @@
+/**
+ * lib/indexers/index.ts - Indexer registry
+ *
+ * Maps source names to indexer functions.
+ * Populated by tasks 3.1, 3.2, 3.3.
+ */
+
+import type { IndexerFunction } from "../indexer";
+import { indexEvents } from "./events";
+import { indexLearnings } from "./learnings";
+import { indexReadmes } from "./readmes";
+import { indexDevelopment } from "./development";
+import { indexCaptures } from "./captures";
+import { indexTeachings } from "./teachings";
+import { indexInsights } from "./insights";
+import { indexObservations } from "./observations";
+import { indexExplorations } from "./explorations";
+import { indexSessions } from "./sessions";
+import { indexFlux } from "./flux";
+import { indexObsidian } from "./obsidian";
+import { indexCommits } from "./commits";
+import { indexBlogs } from "./blogs";
+import { indexPersonal } from "./personal";
+
+export const indexers: Record<string, IndexerFunction> = {
+  events: indexEvents,
+  learnings: indexLearnings,
+  readmes: indexReadmes,
+  development: indexDevelopment,
+  captures: indexCaptures,
+  teachings: indexTeachings,
+  insights: indexInsights,
+  observations: indexObservations,
+  explorations: indexExplorations,
+  sessions: indexSessions,
+  flux: indexFlux,
+  obsidian: indexObsidian,
+  commits: indexCommits,
+  blogs: indexBlogs,
+  personal: indexPersonal,
+};

package/lib/indexers/insights.ts

@@ -0,0 +1,53 @@
+/**
+ * lib/indexers/insights.ts - Insights indexer
+ *
+ * Reads log.jsonl and indexes insight summary captures.
+ * Filters for event=captured AND type=insight AND data.subtype=summary.
+ *
+ * Source: insights
+ * Topic: data.topic or "assistant"
+ * Type: summary (fixed)
+ * Timestamp: event timestamp
+ */
+
+import { readFileSync, existsSync } from "fs";
+import type { IndexerContext } from "../indexer";
+
+export async function indexInsights(ctx: IndexerContext): Promise<void> {
+  const logPath = `${ctx.config.paths.data}/log.jsonl`;
+  if (!existsSync(logPath)) {
+    console.log("No log.jsonl found, skipping insights");
+    return;
+  }
+
+  const lines = readFileSync(logPath, "utf-8").split("\n").filter(Boolean);
+
+  for (const line of lines) {
+    try {
+      const event = JSON.parse(line);
+      if (event.event !== "captured" || event.type !== "insight") continue;
+      if (event.data?.subtype !== "summary") continue;
+
+      const topic = event.data?.topic || "assistant";
+      const content = event.data?.content || "";
+      const sessionId = event.data?.session_id;
+
+      if (!content) continue;
+
+      const metadata: Record<string, unknown> = {};
+      if (sessionId) metadata.session_id = sessionId;
+
+      ctx.insert({
+        source: "insights",
+        title: `[summary] ${topic}`,
+        content,
+        topic,
+        type: "summary",
+        timestamp: event.timestamp,
+        metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+      });
+    } catch (e) {
+      continue;
+    }
+  }
+}