@geoql/mdr 0.0.1

package/opencode/journal.ts

@@ -0,0 +1,153 @@
+/**
+ * Journal operations for OpenCode plugin
+ *
+ * Write journal entries and search memory
+ */
+
+import { existsSync, appendFileSync, mkdirSync, readFileSync, readdirSync } from "fs";
+import { join } from "path";
+import { getStateRoot } from "./context.js";
+import { indexJournalEntry } from "./search.js";
+import { logger } from "./logger.js";
+
+interface JournalEntry {
+  timestamp: string;
+  topic: string;
+  content: string;
+  metadata?: {
+    source?: string;
+    intent?: string;
+  };
+}
+
+function ensureDirectories(): void {
+  const stateRoot = getStateRoot();
+  const dirs = [
+    stateRoot,
+    join(stateRoot, "state"),
+    join(stateRoot, "entities"),
+    join(stateRoot, "entities", "people"),
+    join(stateRoot, "entities", "projects"),
+    join(stateRoot, "journal"),
+  ];
+
+  for (const dir of dirs) {
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+function getTodayJournalPath(): string {
+  const stateRoot = getStateRoot();
+  const today = new Date().toISOString().split("T")[0];
+  return join(stateRoot, "journal", `${today}.jsonl`);
+}
+
+/**
+ * Log an entry to the journal
+ */
+export async function logJournal(
+  topic: string,
+  content: string,
+  metadata?: { source?: string; intent?: string },
+): Promise<void> {
+  ensureDirectories();
+
+  const entry: JournalEntry = {
+    timestamp: new Date().toISOString(),
+    topic,
+    content,
+    metadata: metadata || { source: "opencode-plugin" },
+  };
+
+  const journalPath = getTodayJournalPath();
+  appendFileSync(journalPath, JSON.stringify(entry) + "\n");
+
+  // Index the entry for semantic search
+  try {
+    await indexJournalEntry(entry);
+  } catch (err) {
+    logger.error(`Failed to index journal entry: ${String(err)}`);
+  }
+}
+
+/**
+ * Get recent journal entries
+ */
+export function getRecentJournal(count: number, topic?: string): JournalEntry[] {
+  const stateRoot = getStateRoot();
+  const journalDir = join(stateRoot, "journal");
+  let entries: JournalEntry[] = [];
+
+  if (!existsSync(journalDir)) return entries;
+
+  try {
+    const files = readdirSync(journalDir)
+      .filter((f) => f.endsWith(".jsonl"))
+      .sort()
+      .reverse();
+
+    for (const file of files) {
+      if (entries.length >= count * 2) break; // Get more for filtering
+
+      const content = readFileSync(join(journalDir, file), "utf-8");
+      const lines = content.trim().split("\n").filter(Boolean);
+
+      for (const line of lines.reverse()) {
+        try {
+          const entry = JSON.parse(line) as JournalEntry;
+          entries.push(entry);
+        } catch {
+          // Skip malformed lines
+        }
+      }
+    }
+  } catch {
+    // Ignore errors
+  }
+
+  // Filter by topic if specified
+  if (topic) {
+    entries = entries.filter((e) => e.topic === topic);
+  }
+
+  return entries.slice(0, count);
+}
+
+/**
+ * Get recent conversation summaries
+ */
+export function getRecentSummaries(count: number): JournalEntry[] {
+  return getRecentJournal(count, "conversation-summary");
+}
+
+/**
+ * Save a conversation summary
+ */
+export async function saveConversationSummary(options: {
+  summary: string;
+  keyDecisions?: string[];
+  openThreads?: string[];
+  learnedPatterns?: string[];
+  notes?: string;
+}): Promise<void> {
+  const parts = [options.summary];
+
+  if (options.keyDecisions?.length) {
+    parts.push(`Decisions: ${options.keyDecisions.join(", ")}`);
+  }
+  if (options.openThreads?.length) {
+    parts.push(`Open threads: ${options.openThreads.join(", ")}`);
+  }
+  if (options.learnedPatterns?.length) {
+    parts.push(`Learned: ${options.learnedPatterns.join(", ")}`);
+  }
+  if (options.notes) {
+    parts.push(`Notes: ${options.notes}`);
+  }
+
+  await logJournal("conversation-summary", parts.join("\n"), {
+    source: "opencode-plugin",
+  });
+}

package/opencode/logger.ts

@@ -0,0 +1,32 @@
+/**
+ * File-based logger for Macrodata
+ * Writes to .macrodata.log in state root instead of console
+ */
+
+import { appendFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+const LOG_FILE = join(homedir(), ".config", "macrodata", ".macrodata.log");
+
+// Ensure directory exists
+mkdirSync(join(homedir(), ".config", "macrodata"), { recursive: true });
+
+function formatMessage(level: string, message: string): string {
+  const timestamp = new Date().toISOString();
+  return `${timestamp} [${level}] ${message}\n`;
+}
+
+export const logger = {
+  log(message: string): void {
+    appendFileSync(LOG_FILE, formatMessage("INFO", message));
+  },
+
+  error(message: string): void {
+    appendFileSync(LOG_FILE, formatMessage("ERROR", message));
+  },
+
+  warn(message: string): void {
+    appendFileSync(LOG_FILE, formatMessage("WARN", message));
+  },
+};

package/opencode/search.ts

@@ -0,0 +1,288 @@
+/**
+ * Semantic search for OpenCode plugin
+ *
+ * Uses Vectra for vector search, with embeddings from src/embeddings.ts
+ * (local Transformers.js by default, remote provider when configured).
+ */
+
+import { LocalIndex } from "vectra";
+import { existsSync, readFileSync, readdirSync, mkdirSync } from "fs";
+import { join } from "path";
+import { embed, embedBatch, embedQuery } from "../src/embeddings.js";
+import { getStateRoot } from "./context.js";
+import { logger } from "./logger.js";
+
+// Memory index singleton
+let memoryIndex: LocalIndex | null = null;
+
+// Test seam: drop the cached index so a new MACRODATA_ROOT is picked up.
+export function resetMemoryIndexForTests(): void {
+  memoryIndex = null;
+}
+
+async function getMemoryIndex(): Promise<LocalIndex> {
+  if (memoryIndex) return memoryIndex;
+
+  const stateRoot = getStateRoot();
+  const indexPath = join(stateRoot, ".index", "vectors");
+
+  const indexDir = join(stateRoot, ".index");
+  if (!existsSync(indexDir)) {
+    mkdirSync(indexDir, { recursive: true });
+  }
+
+  memoryIndex = new LocalIndex(indexPath);
+
+  if (!(await memoryIndex.isIndexCreated())) {
+    logger.log("Creating new memory index...");
+    await memoryIndex.createIndex();
+  }
+
+  return memoryIndex;
+}
+
+export type MemoryItemType = "journal" | "person" | "project" | "topic";
+
+export interface SearchResult {
+  content: string;
+  source: string;
+  section?: string;
+  timestamp?: string;
+  type: MemoryItemType;
+  score: number;
+}
+
+/**
+ * Search memory index
+ */
+export async function searchMemory(
+  query: string,
+  options: {
+    limit?: number;
+    type?: MemoryItemType;
+    since?: string;
+  } = {},
+): Promise<SearchResult[]> {
+  const { limit = 5, type, since } = options;
+
+  const idx = await getMemoryIndex();
+  const stats = await idx.listItems();
+
+  if (stats.length === 0) {
+    return [];
+  }
+
+  const queryVector = await embedQuery(query);
+  const results = await idx.queryItems(queryVector, query, limit * 2);
+
+  let filtered = results;
+  if (type || since) {
+    filtered = results.filter((item) => {
+      const meta = item.item.metadata as Record<string, unknown>;
+      if (type && meta.type !== type) return false;
+      if (since && meta.timestamp && (meta.timestamp as string) < since) return false;
+      return true;
+    });
+  }
+
+  return filtered.slice(0, limit).map((r) => {
+    const meta = r.item.metadata as Record<string, unknown>;
+    return {
+      content: meta.content as string,
+      source: meta.source as string,
+      section: meta.section as string | undefined,
+      timestamp: meta.timestamp as string | undefined,
+      type: meta.type as MemoryItemType,
+      score: r.score,
+    };
+  });
+}
+
+/**
+ * Rebuild memory index from journal and entity files
+ */
+export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
+  logger.log("Rebuilding memory index...");
+  const startTime = Date.now();
+  const stateRoot = getStateRoot();
+
+  interface MemoryItem {
+    id: string;
+    type: MemoryItemType;
+    content: string;
+    source: string;
+    section?: string;
+    timestamp?: string;
+  }
+
+  const allItems: MemoryItem[] = [];
+
+  // Index journal entries
+  const journalDir = join(stateRoot, "journal");
+  if (existsSync(journalDir)) {
+    const files = readdirSync(journalDir).filter((f) => f.endsWith(".jsonl"));
+    for (const file of files) {
+      try {
+        const content = readFileSync(join(journalDir, file), "utf-8");
+        const lines = content.trim().split("\n").filter(Boolean);
+        for (let i = 0; i < lines.length; i++) {
+          try {
+            const entry = JSON.parse(lines[i]);
+            allItems.push({
+              id: `journal-${file}-${i}`,
+              type: "journal",
+              content: `[${entry.topic}] ${entry.content}`,
+              source: file,
+              timestamp: entry.timestamp,
+            });
+          } catch {
+            // Skip malformed lines
+          }
+        }
+      } catch {
+        // Skip unreadable files
+      }
+    }
+  }
+
+  // Index entity files (people, projects)
+  const entitiesDir = join(stateRoot, "entities");
+  for (const [subdir, type] of [
+    ["people", "person"],
+    ["projects", "project"],
+  ] as const) {
+    const dir = join(entitiesDir, subdir);
+    if (!existsSync(dir)) continue;
+
+    const files = readdirSync(dir).filter((f) => f.endsWith(".md"));
+    for (const file of files) {
+      try {
+        const content = readFileSync(join(dir, file), "utf-8");
+        const filename = file.replace(".md", "");
+
+        // Split by ## headers
+        const sections = content.split(/^## /m);
+
+        if (sections[0].trim()) {
+          allItems.push({
+            id: `${type}-${filename}-preamble`,
+            type,
+            content: sections[0].trim(),
+            source: `${subdir}/${file}`,
+            section: "preamble",
+          });
+        }
+
+        for (let i = 1; i < sections.length; i++) {
+          const section = sections[i];
+          const firstLine = section.split("\n")[0];
+          const sectionTitle = firstLine.trim();
+          const sectionContent = section.slice(firstLine.length).trim();
+
+          if (sectionContent) {
+            allItems.push({
+              id: `${type}-${filename}-${i}`,
+              type,
+              content: `## ${sectionTitle}\n\n${sectionContent}`,
+              source: `${subdir}/${file}`,
+              section: sectionTitle,
+            });
+          }
+        }
+      } catch {
+        // Skip unreadable files
+      }
+    }
+  }
+
+  // Index topics
+  const topicsDir = join(stateRoot, "topics");
+  if (existsSync(topicsDir)) {
+    const files = readdirSync(topicsDir).filter((f) => f.endsWith(".md"));
+    for (const file of files) {
+      try {
+        const content = readFileSync(join(topicsDir, file), "utf-8");
+        const filename = file.replace(".md", "");
+        allItems.push({
+          id: `topic-${filename}`,
+          type: "topic",
+          content: content.trim(),
+          source: `topics/${file}`,
+        });
+      } catch {
+        // Skip
+      }
+    }
+  }
+
+  if (allItems.length === 0) {
+    return { itemCount: 0 };
+  }
+
+  // Generate embeddings
+  logger.log(`Generating embeddings for ${allItems.length} items...`);
+  const vectors = await embedBatch(allItems.map((i) => i.content));
+
+  // Index all items in a single update transaction (one index.json write)
+  const idx = await getMemoryIndex();
+  await idx.beginUpdate();
+  try {
+    for (let i = 0; i < allItems.length; i++) {
+      const item = allItems[i];
+      const metadata: Record<string, string | number | boolean> = {
+        type: item.type,
+        content: item.content,
+        source: item.source,
+      };
+      if (item.section) metadata.section = item.section;
+      if (item.timestamp) metadata.timestamp = item.timestamp;
+
+      await idx.upsertItem({
+        id: item.id,
+        vector: vectors[i],
+        metadata,
+      });
+    }
+    await idx.endUpdate();
+  } catch (err) {
+    idx.cancelUpdate();
+    throw err;
+  }
+
+  const duration = Date.now() - startTime;
+  logger.log(`Index rebuilt in ${duration}ms`);
+
+  return { itemCount: allItems.length };
+}
+
+/**
+ * Get memory index stats
+ */
+export async function getMemoryIndexStats(): Promise<{ itemCount: number }> {
+  const idx = await getMemoryIndex();
+  const items = await idx.listItems();
+  return { itemCount: items.length };
+}
+
+/**
+ * Index a single journal entry (incremental)
+ */
+export async function indexJournalEntry(entry: {
+  timestamp: string;
+  topic: string;
+  content: string;
+}): Promise<void> {
+  const idx = await getMemoryIndex();
+  const vector = await embed(`[${entry.topic}] ${entry.content}`);
+
+  await idx.upsertItem({
+    id: `journal-${entry.timestamp}`,
+    vector,
+    metadata: {
+      type: "journal",
+      content: `[${entry.topic}] ${entry.content}`,
+      source: "journal",
+      timestamp: entry.timestamp,
+    },
+  });
+}

package/opencode/skills/macrodata-distill/SKILL.md

@@ -0,0 +1,171 @@
+---
+name: macrodata-distill
+description: Extract distilled actions and facts from today's conversations. Spawns sub-agents per conversation to avoid context blowup.
+---
+
+# Distill Conversations
+
+Process today's conversations to extract actionable knowledge. This is the core of memory consolidation.
+
+**Important:** This runs as a coordinator. Spawn sub-agents for each conversation to avoid loading full transcripts into your context.
+
+## Storage Format
+
+OpenCode stores all session data in a SQLite database at `~/.local/share/opencode/opencode.db`.
+
+**Schema:**
+
+- `session` — id, project_id, parent_id, title, time_created, time_updated
+- `message` — id, session_id, time_created, data (JSON: role, agent, modelID, etc.)
+- `part` — id, message_id, session_id, time_created, data (JSON: type, text, etc.)
+- `project` — id, worktree, name
+
+**Part types:** text, tool, step-start, step-finish, patch, reasoning, compaction, file, subtask
+
+**Key JSON paths:**
+
+- `message.data` → `$.role` (user/assistant), `$.summary` (set on compaction messages)
+- `part.data` → `$.type` (text/tool/etc.), `$.text` (for text parts)
+
+## Process
+
+### 1. Find Today's Sessions
+
+Query the SQLite database for sessions with activity today. Exclude subtask sessions (parent_id IS NOT NULL).
+
+```bash
+sqlite3 ~/.local/share/opencode/opencode.db "
+  SELECT s.id, s.title, p.worktree, s.time_created
+  FROM session s
+  LEFT JOIN project p ON p.id = s.project_id
+  WHERE s.parent_id IS NULL
+    AND s.time_updated > unixepoch('now', '-1 day') * 1000
+  ORDER BY s.time_updated DESC
+"
+```
+
+### 2. Process Each Session
+
+For **each** session, spawn a sub-agent with the Task tool:
+
+```
+Task(subagent_type="general", prompt=`
+Read an OpenCode conversation from the SQLite database at ~/.local/share/opencode/opencode.db.
+
+Session ID: {sessionId}
+Session title: {sessionTitle}
+Project: {projectWorktree}
+
+Use this query to extract the conversation (user prompts and assistant text responses):
+
+sqlite3 ~/.local/share/opencode/opencode.db "
+  SELECT
+    m.id AS message_id,
+    json_extract(m.data, '$.role') AS role,
+    m.time_created,
+    GROUP_CONCAT(
+      CASE WHEN json_extract(p.data, '$.type') = 'text'
+        THEN json_extract(p.data, '$.text')
+      END,
+      char(10)
+    ) AS text_content
+  FROM message m
+  JOIN part p ON p.message_id = m.id
+  WHERE m.session_id = '{sessionId}'
+    AND json_extract(m.data, '$.role') IN ('user', 'assistant')
+    AND json_extract(m.data, '$.summary') IS NULL
+  GROUP BY m.id
+  HAVING text_content IS NOT NULL AND text_content != ''
+  ORDER BY m.time_created ASC
+"
+
+Filter to actual conversation content:
+- Include: user messages, assistant text responses
+- Exclude: tool calls, tool results, system content, compaction summaries
+
+Extract and return as JSON:
+{
+  "distilled_actions": [
+    {
+      "summary": "Fixed auth bug in src/auth.ts where token refresh was racing",
+      "files": ["src/auth.ts"],
+      "outcome": "Added mutex lock around refresh"
+    }
+  ],
+  "facts": [
+    {
+      "topic": "project-name",
+      "content": "Uses JWT tokens with 15min expiry"
+    },
+    {
+      "topic": "person-name",
+      "content": "Prefers explicit error handling over try/catch"
+    }
+  ],
+  "decisions": [
+    "Chose Redis over in-memory cache for session storage because of multi-instance deployment"
+  ]
+}
+
+Focus on:
+- What was accomplished (not just discussed)
+- Decisions made and their rationale
+- New information about projects, people, or preferences
+- File paths and specific technical details that should survive compression
+
+Return ONLY the JSON, no explanation.
+`)
+```
+
+### 3. Collect and Write Results
+
+After all sub-agents complete:
+
+**Write distilled actions to journal:**
+
+```
+For each action in all results:
+  macrodata_log_journal(topic="distilled", content=action.summary + " Files: " + action.files.join(", "))
+```
+
+**Write overall summary to journal:**
+
+```
+macrodata_log_journal(topic="distill-summary", content="Processed N sessions. Extracted X actions, Y facts.")
+```
+
+**Update entity files with facts:**
+
+- Group facts by topic
+- For each topic, read existing entity file (if any)
+- Integrate new facts, removing duplicates
+- Write updated file
+
+### 4. Example Sub-Agent Output
+
+```json
+{
+  "distilled_actions": [
+    {
+      "summary": "Added /distill skill to macrodata plugin",
+      "files": ["plugins/macrodata/skills/distill/SKILL.md"],
+      "outcome": "Skill extracts facts from conversations via sub-agents"
+    }
+  ],
+  "facts": [
+    {
+      "topic": "macrodata",
+      "content": "Distillation separates narrative context from retained facts for better compression"
+    }
+  ],
+  "decisions": [
+    "Coordinator updates state directly to prevent race conditions from parallel sub-agents"
+  ]
+}
+```
+
+## Notes
+
+- Sub-agents should be spawned in parallel for efficiency
+- Empty results are fine - not every conversation has extractable knowledge
+- Facts should be concise and specific, not narrative summaries