@geoql/mdr 0.0.1

package/opencode/conversations.ts

@@ -0,0 +1,467 @@
+/**
+ * OpenCode Conversation Indexer
+ *
+ * Indexes past OpenCode sessions for semantic search.
+ * Reads from the OpenCode SQLite database at ~/.local/share/opencode/opencode.db
+ *
+ * Schema (relevant tables):
+ *   - session: id, project_id, title, time_created, time_updated, parent_id
+ *   - message: id, session_id, time_created, data (JSON with role, agent, etc.)
+ *   - part: id, message_id, session_id, data (JSON with type, text, etc.)
+ *   - project: id, worktree
+ */
+
+import { existsSync, mkdirSync } from "fs";
+import { join, basename } from "path";
+import { homedir } from "os";
+import { DatabaseSync } from "node:sqlite";
+import { LocalIndex } from "vectra";
+import { embedBatch, embedQuery } from "../src/embeddings.js";
+import { getStateRoot } from "./context.js";
+import { logger } from "./logger.js";
+
+const OPENCODE_DB_PATH =
+  process.env.MACRODATA_OPENCODE_DB_PATH ||
+  join(homedir(), ".local", "share", "opencode", "opencode.db");
+
+// Conversation index singleton
+let convIndex: LocalIndex | null = null;
+
+// Test seam: drop the cached index so a new MACRODATA_ROOT is picked up.
+export function resetConversationIndexForTests(): void {
+  convIndex = null;
+}
+
+async function getConversationIndex(): Promise<LocalIndex> {
+  if (convIndex) return convIndex;
+
+  const stateRoot = getStateRoot();
+  const indexPath = join(stateRoot, ".index", "oc-conversations");
+
+  const indexDir = join(stateRoot, ".index");
+  if (!existsSync(indexDir)) {
+    mkdirSync(indexDir, { recursive: true });
+  }
+
+  convIndex = new LocalIndex(indexPath);
+
+  if (!(await convIndex.isIndexCreated())) {
+    logger.log("Creating new conversation index...");
+    await convIndex.createIndex();
+  }
+
+  return convIndex;
+}
+
+export interface ConversationExchange {
+  id: string;
+  userPrompt: string;
+  assistantSummary: string;
+  project: string;
+  projectPath: string;
+  timestamp: string;
+  sessionId: string;
+  messageId: string;
+}
+
+export interface ConversationSearchResult {
+  exchange: ConversationExchange;
+  score: number;
+  adjustedScore: number;
+}
+
+/**
+ * Open the OpenCode SQLite database (read-only)
+ */
+function openDb(): DatabaseSync | null {
+  if (!existsSync(OPENCODE_DB_PATH)) {
+    logger.log(`OpenCode database not found at ${OPENCODE_DB_PATH}`);
+    return null;
+  }
+
+  try {
+    return new DatabaseSync(OPENCODE_DB_PATH, { readOnly: true });
+  } catch (err) {
+    logger.error(`Failed to open OpenCode database: ${String(err)}`);
+    return null;
+  }
+}
+
+interface ExchangeRow {
+  user_msg_id: string;
+  session_id: string;
+  user_time: number;
+  user_text: string;
+  assistant_text: string;
+  worktree: string | null;
+  directory: string | null;
+}
+
+/**
+ * Query exchanges from the SQLite database.
+ *
+ * This runs a single query that:
+ * 1. Finds user messages (role = 'user') that aren't compaction summaries
+ * 2. Finds the next assistant message in the same session
+ * 3. Aggregates text parts for both user and assistant messages
+ * 4. Joins to project for worktree path
+ * 5. Excludes subtask sessions (parent_id IS NULL)
+ */
+export function queryExchanges(db: DatabaseSync, sinceMs?: number): ExchangeRow[] {
+  // Interpolated inside the user_messages CTE body, where only `m` and `s`
+  // are in scope (`um` is the outer query's alias and must not be used here).
+  const whereClause = sinceMs ? "AND m.time_created > ?" : "";
+  const params = sinceMs ? [sinceMs] : [];
+
+  // Get user-assistant pairs with their text content.
+  // We use a CTE to match each user message with its subsequent assistant message,
+  // then aggregate text parts for both.
+  const sql = `
+    WITH user_messages AS (
+      SELECT
+        m.id AS user_msg_id,
+        m.session_id,
+        m.time_created AS user_time,
+        m.data AS user_data,
+        -- Find the next assistant message by time in the same session
+        (
+          SELECT am.id FROM message am
+          WHERE am.session_id = m.session_id
+            AND am.time_created > m.time_created
+            AND json_extract(am.data, '$.role') = 'assistant'
+          ORDER BY am.time_created ASC
+          LIMIT 1
+        ) AS assistant_msg_id
+      FROM message m
+      JOIN session s ON s.id = m.session_id
+      WHERE json_extract(m.data, '$.role') = 'user'
+        AND s.parent_id IS NULL
+        ${whereClause}
+    )
+    SELECT
+      um.user_msg_id,
+      um.session_id,
+      um.user_time,
+      COALESCE(
+        GROUP_CONCAT(
+          CASE WHEN up.message_id = um.user_msg_id AND json_extract(up.data, '$.type') = 'text'
+            THEN json_extract(up.data, '$.text')
+          END,
+          '\n'
+        ),
+        ''
+      ) AS user_text,
+      COALESCE(
+        GROUP_CONCAT(
+          CASE WHEN up.message_id = um.assistant_msg_id AND json_extract(up.data, '$.type') = 'text'
+            THEN json_extract(up.data, '$.text')
+          END,
+          '\n'
+        ),
+        ''
+      ) AS assistant_text,
+      p.worktree,
+      s.directory
+    FROM user_messages um
+    LEFT JOIN part up ON up.message_id IN (um.user_msg_id, um.assistant_msg_id)
+    LEFT JOIN session s ON s.id = um.session_id
+    LEFT JOIN project p ON p.id = s.project_id
+    WHERE um.assistant_msg_id IS NOT NULL
+    GROUP BY um.user_msg_id
+    HAVING user_text != ''
+    ORDER BY um.user_time ASC
+  `;
+
+  try {
+    return db.prepare(sql).all(...params) as unknown as ExchangeRow[];
+  } catch (err) {
+    // Propagate instead of returning [] so callers can't mistake a schema
+    // mismatch for "no new exchanges" (a silent no-op that previously
+    // disabled indexing for weeks, see #25).
+    logger.error(`Query failed: ${String(err)}`);
+    throw err;
+  }
+}
+
+/**
+ * Convert raw DB rows to ConversationExchange objects
+ */
+function rowsToExchanges(rows: ExchangeRow[]): ConversationExchange[] {
+  return rows.map((row) => {
+    // Use project worktree, but fall back to session directory for "global" sessions
+    // where worktree is "/" (the root filesystem, not a real project)
+    const worktree = row.worktree && row.worktree !== "/" ? row.worktree : "";
+    const directory = row.directory || "";
+    const projectPath = worktree || directory;
+    const name = projectPath ? basename(projectPath) : "";
+    const projectName = name || "unknown";
+
+    return {
+      id: `oc-${row.session_id}-${row.user_msg_id}`,
+      userPrompt: row.user_text.slice(0, 1000),
+      assistantSummary: row.assistant_text.slice(0, 500),
+      project: projectName,
+      projectPath,
+      timestamp: new Date(row.user_time).toISOString(),
+      sessionId: row.session_id,
+      messageId: row.user_msg_id,
+    };
+  });
+}
+
+// Guard against concurrent rebuilds
+let rebuildInProgress: Promise<{ exchangeCount: number }> | null = null;
+
+/**
+ * Rebuild conversation index from scratch
+ */
+export async function rebuildConversationIndex(): Promise<{ exchangeCount: number }> {
+  if (rebuildInProgress) {
+    logger.log("Conversation index rebuild already in progress, waiting...");
+    return rebuildInProgress;
+  }
+
+  rebuildInProgress = doRebuildConversationIndex();
+  try {
+    return await rebuildInProgress;
+  } finally {
+    rebuildInProgress = null;
+  }
+}
+
+async function doRebuildConversationIndex(): Promise<{ exchangeCount: number }> {
+  logger.log("Rebuilding OpenCode conversation index...");
+  const startTime = Date.now();
+
+  const db = openDb();
+  if (!db) return { exchangeCount: 0 };
+
+  try {
+    const rows = queryExchanges(db);
+    const exchanges = rowsToExchanges(rows);
+
+    logger.log(`Found ${exchanges.length} exchanges`);
+    if (exchanges.length === 0) return { exchangeCount: 0 };
+
+    // Generate all embeddings BEFORE touching the index
+    const texts = exchanges.map((e) => e.userPrompt);
+    logger.log(`Generating embeddings for ${texts.length} exchanges...`);
+    const vectors = await embedBatch(texts);
+    logger.log(`Embeddings generated, inserting into index...`);
+
+    // Only delete after embeddings succeed
+    // Reset singleton since deleteIndex invalidates the cached instance
+    convIndex = null;
+    const idx = await getConversationIndex();
+    if (await idx.isIndexCreated()) {
+      await idx.deleteIndex();
+    }
+    await idx.createIndex();
+
+    // Batch inside a single update transaction: without it, vectra rewrites
+    // the entire index.json on every upsert, which is O(n^2) and takes hours
+    // for tens of thousands of items.
+    await idx.beginUpdate();
+    try {
+      for (let i = 0; i < exchanges.length; i++) {
+        const ex = exchanges[i];
+        await idx.upsertItem({
+          id: ex.id,
+          vector: vectors[i],
+          metadata: {
+            userPrompt: ex.userPrompt,
+            assistantSummary: ex.assistantSummary,
+            project: ex.project,
+            projectPath: ex.projectPath,
+            timestamp: ex.timestamp,
+            sessionId: ex.sessionId,
+            messageId: ex.messageId,
+          },
+        });
+
+        /* v8 ignore next 3 -- progress log that only fires past 500 indexed
+           exchanges; seeding 500 real embedded rows per test is impractical and
+           the line has no behavioural effect. */
+        if (i > 0 && i % 500 === 0) {
+          logger.log(`  ...inserted ${i}/${exchanges.length}`);
+        }
+      }
+      await idx.endUpdate();
+    } catch (err) {
+      idx.cancelUpdate();
+      throw err;
+    }
+
+    const duration = Date.now() - startTime;
+    logger.log(`Conversation index rebuilt: ${exchanges.length} exchanges in ${duration}ms`);
+    return { exchangeCount: exchanges.length };
+  } catch (err) {
+    logger.error(`Conversation index rebuild failed: ${String(err)}`);
+    throw err;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Time-based weight for scoring
+ */
+function getTimeWeight(timestamp: string): number {
+  const ts = new Date(timestamp);
+  if (isNaN(ts.getTime())) return 0.5;
+
+  const age = Date.now() - ts.getTime();
+  const dayMs = 24 * 60 * 60 * 1000;
+
+  if (age < 7 * dayMs) return 1.0;
+  if (age < 30 * dayMs) return 0.9;
+  if (age < 90 * dayMs) return 0.7;
+  if (age < 365 * dayMs) return 0.5;
+  return 0.3;
+}
+
+/**
+ * Search past conversations
+ */
+export async function searchConversations(
+  query: string,
+  options: {
+    currentProject?: string;
+    limit?: number;
+    projectOnly?: boolean;
+  } = {},
+): Promise<ConversationSearchResult[]> {
+  const { currentProject, limit = 5, projectOnly = false } = options;
+
+  const idx = await getConversationIndex();
+  const stats = await idx.listItems();
+
+  if (stats.length === 0) {
+    return [];
+  }
+
+  const queryVector = await embedQuery(query);
+  const results = await idx.queryItems(queryVector, query, limit * 3);
+
+  const searchResults: ConversationSearchResult[] = results.map((r) => {
+    const meta = r.item.metadata as Record<string, string>;
+
+    const exchange: ConversationExchange = {
+      id: r.item.id,
+      userPrompt: meta.userPrompt,
+      assistantSummary: meta.assistantSummary,
+      project: meta.project,
+      projectPath: meta.projectPath,
+      timestamp: meta.timestamp,
+      sessionId: meta.sessionId,
+      messageId: meta.messageId,
+    };
+
+    let adjustedScore = r.score;
+    adjustedScore *= getTimeWeight(exchange.timestamp);
+
+    if (currentProject && exchange.projectPath === currentProject) {
+      adjustedScore *= 1.5;
+    }
+
+    return {
+      exchange,
+      score: r.score,
+      adjustedScore,
+    };
+  });
+
+  let filtered = searchResults;
+  if (projectOnly && currentProject) {
+    filtered = searchResults.filter((r) => r.exchange.projectPath === currentProject);
+  }
+
+  return filtered.sort((a, b) => b.adjustedScore - a.adjustedScore).slice(0, limit);
+}
+
+/**
+ * Get conversation index stats
+ */
+export async function getConversationIndexStats(): Promise<{ exchangeCount: number }> {
+  const idx = await getConversationIndex();
+  const items = await idx.listItems();
+  return { exchangeCount: items.length };
+}
+
+/**
+ * Incrementally update conversation index (only new exchanges)
+ */
+export async function updateConversationIndex(): Promise<{ newCount: number; totalCount: number }> {
+  logger.log("Updating OpenCode conversation index...");
+  const startTime = Date.now();
+
+  const db = openDb();
+  if (!db) return { newCount: 0, totalCount: 0 };
+
+  try {
+    const idx = await getConversationIndex();
+    const existingItems = await idx.listItems();
+    const existingIds = new Set(existingItems.map((item) => item.id));
+
+    // Find the most recent timestamp in the index to narrow the query
+    let latestMs = 0;
+    for (const item of existingItems) {
+      const meta = item.metadata as Record<string, string>;
+      if (meta.timestamp) {
+        const ms = new Date(meta.timestamp).getTime();
+        if (ms > latestMs) latestMs = ms;
+      }
+    }
+
+    // Query only exchanges after the latest indexed timestamp (with some overlap for safety)
+    const sinceMs = latestMs > 0 ? latestMs - 60_000 : undefined;
+    const rows = queryExchanges(db, sinceMs);
+    const allExchanges = rowsToExchanges(rows);
+
+    // Filter to truly new exchanges
+    const newExchanges = allExchanges.filter((ex) => !existingIds.has(ex.id));
+
+    logger.log(`Found ${newExchanges.length} new exchanges (${existingIds.size} already indexed)`);
+
+    if (newExchanges.length === 0) {
+      return { newCount: 0, totalCount: existingIds.size };
+    }
+
+    const texts = newExchanges.map((e) => e.userPrompt);
+    logger.log(`Generating embeddings for ${texts.length} new exchanges...`);
+    const vectors = await embedBatch(texts);
+
+    // Single update transaction: one index.json write for the whole batch
+    await idx.beginUpdate();
+    try {
+      for (let i = 0; i < newExchanges.length; i++) {
+        const ex = newExchanges[i];
+        await idx.upsertItem({
+          id: ex.id,
+          vector: vectors[i],
+          metadata: {
+            userPrompt: ex.userPrompt,
+            assistantSummary: ex.assistantSummary,
+            project: ex.project,
+            projectPath: ex.projectPath,
+            timestamp: ex.timestamp,
+            sessionId: ex.sessionId,
+            messageId: ex.messageId,
+          },
+        });
+      }
+      await idx.endUpdate();
+    } catch (err) {
+      idx.cancelUpdate();
+      throw err;
+    }
+
+    const duration = Date.now() - startTime;
+    const totalCount = existingIds.size + newExchanges.length;
+    logger.log(`Added ${newExchanges.length} exchanges in ${duration}ms (total: ${totalCount})`);
+
+    return { newCount: newExchanges.length, totalCount };
+  } finally {
+    db.close();
+  }
+}

package/opencode/index.ts

@@ -0,0 +1,208 @@
+/**
+ * OpenCode Macrodata Plugin
+ *
+ * Provides persistent local memory for OpenCode agents:
+ * - Context injection via system prompt transform
+ * - Compaction hook to preserve memory context
+ * - Custom `macrodata` tool for memory operations
+ */
+
+import type { Plugin, PluginInput } from "@opencode-ai/plugin";
+import { existsSync, mkdirSync, cpSync, readdirSync, readFileSync, openSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { spawn } from "child_process";
+import { memoryTools } from "./tools.js";
+import {
+  formatContextForPrompt,
+  consumePendingContext,
+  initializeStateRoot,
+  getStateRoot,
+} from "./context.js";
+import { logger } from "./logger.js";
+
+/**
+ * Check if a process with given PID is running
+ */
+function isProcessRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Send SIGHUP to the daemon to reload config
+ */
+function signalDaemonReload(): void {
+  const pidFile = join(homedir(), ".config", "macrodata", ".daemon.pid");
+  if (!existsSync(pidFile)) return;
+
+  try {
+    const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+    if (isProcessRunning(pid)) {
+      process.kill(pid, "SIGHUP");
+    }
+  } catch {
+    // Ignore errors
+  }
+}
+
+const HEARTBEAT_STALE_MS = 15 * 60_000;
+
+/**
+ * Ensure the macrodata daemon is running and healthy.
+ * Starts it when the PID is dead, and restarts it when the PID is alive but
+ * the heartbeat file is stale (wedged daemon, see #25).
+ */
+function ensureDaemonRunning(): void {
+  const configDir = join(homedir(), ".config", "macrodata");
+  const pidFile = join(configDir, ".daemon.pid");
+  const stateRoot = getStateRoot();
+  const heartbeatFile = join(stateRoot, ".daemon.heartbeat");
+  const daemonScript = join(import.meta.dirname, "..", "bin", "macrodata-daemon.js");
+
+  if (existsSync(pidFile)) {
+    try {
+      const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+      if (isProcessRunning(pid)) {
+        if (!isHeartbeatStale(heartbeatFile)) {
+          return;
+        }
+        logger.warn(`Daemon PID ${pid} alive but heartbeat stale, restarting`);
+        try {
+          process.kill(pid, "SIGKILL");
+        } catch {
+          // Already gone
+        }
+      }
+    } catch {
+      // Invalid PID file, continue to start daemon
+    }
+  }
+
+  // Start daemon - it writes its own PID file
+  try {
+    // Ensure config dir exists for PID file
+    mkdirSync(configDir, { recursive: true });
+
+    const logFile = join(getStateRoot(), ".daemon.log");
+    const out = openSync(logFile, "a");
+    const err = openSync(logFile, "a");
+
+    const child = spawn(process.execPath, [daemonScript], {
+      detached: true,
+      stdio: ["ignore", out, err],
+      env: { ...process.env, MACRODATA_ROOT: stateRoot },
+    });
+    child.unref();
+  } catch (err) {
+    logger.error(`Failed to start daemon: ${String(err)}`);
+  }
+}
+
+function isHeartbeatStale(heartbeatFile: string): boolean {
+  if (!existsSync(heartbeatFile)) {
+    return false;
+  }
+  try {
+    const lastBeat = parseInt(readFileSync(heartbeatFile, "utf-8").trim(), 10);
+    return Number.isFinite(lastBeat) && Date.now() - lastBeat > HEARTBEAT_STALE_MS;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Install plugin skills to ~/.config/opencode/skills/
+ * Skills are copied from the plugin's skills directory on first load
+ */
+function installSkills(): void {
+  const globalSkillsDir = join(homedir(), ".config", "opencode", "skills");
+  // import.meta.dirname is the opencode/ folder
+  const pluginSkillsDir = join(import.meta.dirname, "skills");
+
+  /* v8 ignore next 3 -- the skills/ directory always ships next to the built
+     plugin, so this missing-dir bail-out is defensive only. */
+  if (!existsSync(pluginSkillsDir)) {
+    return;
+  }
+
+  // Ensure global skills directory exists
+  if (!existsSync(globalSkillsDir)) {
+    mkdirSync(globalSkillsDir, { recursive: true });
+  }
+
+  // Copy each skill directory
+  const skills = readdirSync(pluginSkillsDir, { withFileTypes: true })
+    .filter((d) => d.isDirectory())
+    .map((d) => d.name);
+
+  for (const skill of skills) {
+    const src = join(pluginSkillsDir, skill);
+    const dest = join(globalSkillsDir, skill);
+
+    // Always update skills (overwrite existing)
+    try {
+      cpSync(src, dest, { recursive: true });
+    } catch {
+      // Silently fail - non-critical
+    }
+  }
+}
+
+export const MacrodataPlugin: Plugin = async (ctx: PluginInput) => {
+  // Initialize state directories
+  initializeStateRoot();
+
+  // Ensure daemon is running for scheduled reminders
+  ensureDaemonRunning();
+
+  // Signal daemon to reload config (in case it was started with old config)
+  signalDaemonReload();
+
+  // Install skills to global config on plugin load
+  installSkills();
+
+  return {
+    // Inject memory context into system prompt
+    "experimental.chat.system.transform": async (_input, output) => {
+      try {
+        const pendingContext = consumePendingContext();
+        if (pendingContext) {
+          output.system.push(pendingContext);
+        }
+
+        const memoryContext = await formatContextForPrompt({ client: ctx.client });
+        /* v8 ignore next 3 -- outside compaction formatContextForPrompt always
+           returns a string (onboarding or full context), never null. */
+        if (memoryContext) {
+          output.system.push(memoryContext);
+        }
+      } catch (err) {
+        logger.error(`System context injection error: ${String(err)}`);
+      }
+    },
+
+    // Inject memory context before compaction
+    "experimental.session.compacting": async (_input, output) => {
+      try {
+        const memoryContext = await formatContextForPrompt({ forCompaction: true });
+
+        if (memoryContext) {
+          output.context.push(memoryContext);
+        }
+      } catch (err) {
+        logger.error(`Compaction hook error: ${String(err)}`);
+      }
+    },
+
+    // Provide memory tools
+    tool: memoryTools,
+  };
+};
+
+// Default export for OpenCode plugin system
+export default MacrodataPlugin;