@geoql/mdr 0.0.1

package/dist/opencode/index.js

@@ -0,0 +1,155 @@
+import { getStateRoot } from "../src/config.js";
+import { consumePendingContext, formatContextForPrompt, initializeStateRoot } from "./context.js";
+import { logger } from "./logger.js";
+import { memoryTools } from "./tools.js";
+import { cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { spawn } from "child_process";
+
+//#region opencode/index.ts
+/**
+* Check if a process with given PID is running
+*/
+function isProcessRunning(pid) {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/**
+* Send SIGHUP to the daemon to reload config
+*/
+function signalDaemonReload() {
+	const pidFile = join(homedir(), ".config", "macrodata", ".daemon.pid");
+	if (!existsSync(pidFile)) return;
+	try {
+		const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+		if (isProcessRunning(pid)) {
+			process.kill(pid, "SIGHUP");
+		}
+	} catch {}
+}
+const HEARTBEAT_STALE_MS = 15 * 6e4;
+/**
+* Ensure the macrodata daemon is running and healthy.
+* Starts it when the PID is dead, and restarts it when the PID is alive but
+* the heartbeat file is stale (wedged daemon, see #25).
+*/
+function ensureDaemonRunning() {
+	const configDir = join(homedir(), ".config", "macrodata");
+	const pidFile = join(configDir, ".daemon.pid");
+	const stateRoot = getStateRoot();
+	const heartbeatFile = join(stateRoot, ".daemon.heartbeat");
+	const daemonScript = join(import.meta.dirname, "..", "bin", "macrodata-daemon.js");
+	if (existsSync(pidFile)) {
+		try {
+			const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+			if (isProcessRunning(pid)) {
+				if (!isHeartbeatStale(heartbeatFile)) {
+					return;
+				}
+				logger.warn(`Daemon PID ${pid} alive but heartbeat stale, restarting`);
+				try {
+					process.kill(pid, "SIGKILL");
+				} catch {}
+			}
+		} catch {}
+	}
+	try {
+		mkdirSync(configDir, { recursive: true });
+		const logFile = join(getStateRoot(), ".daemon.log");
+		const out = openSync(logFile, "a");
+		const err = openSync(logFile, "a");
+		const child = spawn(process.execPath, [daemonScript], {
+			detached: true,
+			stdio: [
+				"ignore",
+				out,
+				err
+			],
+			env: {
+				...process.env,
+				MACRODATA_ROOT: stateRoot
+			}
+		});
+		child.unref();
+	} catch (err) {
+		logger.error(`Failed to start daemon: ${String(err)}`);
+	}
+}
+function isHeartbeatStale(heartbeatFile) {
+	if (!existsSync(heartbeatFile)) {
+		return false;
+	}
+	try {
+		const lastBeat = parseInt(readFileSync(heartbeatFile, "utf-8").trim(), 10);
+		return Number.isFinite(lastBeat) && Date.now() - lastBeat > HEARTBEAT_STALE_MS;
+	} catch {
+		return false;
+	}
+}
+/**
+* Install plugin skills to ~/.config/opencode/skills/
+* Skills are copied from the plugin's skills directory on first load
+*/
+function installSkills() {
+	const globalSkillsDir = join(homedir(), ".config", "opencode", "skills");
+	const pluginSkillsDir = join(import.meta.dirname, "skills");
+	/* v8 ignore next 3 -- the skills/ directory always ships next to the built
+	plugin, so this missing-dir bail-out is defensive only. */
+	if (!existsSync(pluginSkillsDir)) {
+		return;
+	}
+	if (!existsSync(globalSkillsDir)) {
+		mkdirSync(globalSkillsDir, { recursive: true });
+	}
+	const skills = readdirSync(pluginSkillsDir, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name);
+	for (const skill of skills) {
+		const src = join(pluginSkillsDir, skill);
+		const dest = join(globalSkillsDir, skill);
+		try {
+			cpSync(src, dest, { recursive: true });
+		} catch {}
+	}
+}
+const MacrodataPlugin = async (ctx) => {
+	initializeStateRoot();
+	ensureDaemonRunning();
+	signalDaemonReload();
+	installSkills();
+	return {
+		"experimental.chat.system.transform": async (_input, output) => {
+			try {
+				const pendingContext = consumePendingContext();
+				if (pendingContext) {
+					output.system.push(pendingContext);
+				}
+				const memoryContext = await formatContextForPrompt({ client: ctx.client });
+				/* v8 ignore next 3 -- outside compaction formatContextForPrompt always
+				returns a string (onboarding or full context), never null. */
+				if (memoryContext) {
+					output.system.push(memoryContext);
+				}
+			} catch (err) {
+				logger.error(`System context injection error: ${String(err)}`);
+			}
+		},
+		"experimental.session.compacting": async (_input, output) => {
+			try {
+				const memoryContext = await formatContextForPrompt({ forCompaction: true });
+				if (memoryContext) {
+					output.context.push(memoryContext);
+				}
+			} catch (err) {
+				logger.error(`Compaction hook error: ${String(err)}`);
+			}
+		},
+		tool: memoryTools
+	};
+};
+
+//#endregion
+export { MacrodataPlugin, MacrodataPlugin as default };

package/dist/opencode/journal.js

@@ -0,0 +1,108 @@
+import { getStateRoot } from "../src/config.js";
+import "./context.js";
+import { logger } from "./logger.js";
+import { indexJournalEntry } from "./search.js";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "fs";
+import { join } from "path";
+
+//#region opencode/journal.ts
+/**
+* Journal operations for OpenCode plugin
+*
+* Write journal entries and search memory
+*/
+function ensureDirectories() {
+	const stateRoot = getStateRoot();
+	const dirs = [
+		stateRoot,
+		join(stateRoot, "state"),
+		join(stateRoot, "entities"),
+		join(stateRoot, "entities", "people"),
+		join(stateRoot, "entities", "projects"),
+		join(stateRoot, "journal")
+	];
+	for (const dir of dirs) {
+		if (!existsSync(dir)) {
+			mkdirSync(dir, { recursive: true });
+		}
+	}
+}
+function getTodayJournalPath() {
+	const stateRoot = getStateRoot();
+	const today = new Date().toISOString().split("T")[0];
+	return join(stateRoot, "journal", `${today}.jsonl`);
+}
+/**
+* Log an entry to the journal
+*/
+async function logJournal(topic, content, metadata) {
+	ensureDirectories();
+	const entry = {
+		timestamp: new Date().toISOString(),
+		topic,
+		content,
+		metadata: metadata || { source: "opencode-plugin" }
+	};
+	const journalPath = getTodayJournalPath();
+	appendFileSync(journalPath, JSON.stringify(entry) + "\n");
+	try {
+		await indexJournalEntry(entry);
+	} catch (err) {
+		logger.error(`Failed to index journal entry: ${String(err)}`);
+	}
+}
+/**
+* Get recent journal entries
+*/
+function getRecentJournal(count, topic) {
+	const stateRoot = getStateRoot();
+	const journalDir = join(stateRoot, "journal");
+	let entries = [];
+	if (!existsSync(journalDir)) return entries;
+	try {
+		const files = readdirSync(journalDir).filter((f) => f.endsWith(".jsonl")).sort().reverse();
+		for (const file of files) {
+			if (entries.length >= count * 2) break;
+			const content = readFileSync(join(journalDir, file), "utf-8");
+			const lines = content.trim().split("\n").filter(Boolean);
+			for (const line of lines.reverse()) {
+				try {
+					const entry = JSON.parse(line);
+					entries.push(entry);
+				} catch {}
+			}
+		}
+	} catch {}
+	if (topic) {
+		entries = entries.filter((e) => e.topic === topic);
+	}
+	return entries.slice(0, count);
+}
+/**
+* Get recent conversation summaries
+*/
+function getRecentSummaries(count) {
+	return getRecentJournal(count, "conversation-summary");
+}
+/**
+* Save a conversation summary
+*/
+async function saveConversationSummary(options) {
+	const parts = [options.summary];
+	if (options.keyDecisions?.length) {
+		parts.push(`Decisions: ${options.keyDecisions.join(", ")}`);
+	}
+	if (options.openThreads?.length) {
+		parts.push(`Open threads: ${options.openThreads.join(", ")}`);
+	}
+	if (options.learnedPatterns?.length) {
+		parts.push(`Learned: ${options.learnedPatterns.join(", ")}`);
+	}
+	if (options.notes) {
+		parts.push(`Notes: ${options.notes}`);
+	}
+	await logJournal("conversation-summary", parts.join("\n"), { source: "opencode-plugin" });
+}
+
+//#endregion
+export { getRecentJournal, getRecentSummaries, logJournal, saveConversationSummary };

package/dist/opencode/logger.js

@@ -0,0 +1,29 @@
+import { appendFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+//#region opencode/logger.ts
+/**
+* File-based logger for Macrodata
+* Writes to .macrodata.log in state root instead of console
+*/
+const LOG_FILE = join(homedir(), ".config", "macrodata", ".macrodata.log");
+mkdirSync(join(homedir(), ".config", "macrodata"), { recursive: true });
+function formatMessage(level, message) {
+	const timestamp = new Date().toISOString();
+	return `${timestamp} [${level}] ${message}\n`;
+}
+const logger = {
+	log(message) {
+		appendFileSync(LOG_FILE, formatMessage("INFO", message));
+	},
+	error(message) {
+		appendFileSync(LOG_FILE, formatMessage("ERROR", message));
+	},
+	warn(message) {
+		appendFileSync(LOG_FILE, formatMessage("WARN", message));
+	}
+};
+
+//#endregion
+export { logger };

package/dist/opencode/search.js

@@ -0,0 +1,210 @@
+import { getStateRoot } from "../src/config.js";
+import "./context.js";
+import { embed, embedBatch, embedQuery } from "../src/embeddings.js";
+import { logger } from "./logger.js";
+import { existsSync, mkdirSync, readFileSync, readdirSync } from "fs";
+import { join } from "path";
+import { LocalIndex } from "vectra";
+
+//#region opencode/search.ts
+/**
+* Semantic search for OpenCode plugin
+*
+* Uses Vectra for vector search, with embeddings from src/embeddings.ts
+* (local Transformers.js by default, remote provider when configured).
+*/
+let memoryIndex = null;
+function resetMemoryIndexForTests() {
+	memoryIndex = null;
+}
+async function getMemoryIndex() {
+	if (memoryIndex) return memoryIndex;
+	const stateRoot = getStateRoot();
+	const indexPath = join(stateRoot, ".index", "vectors");
+	const indexDir = join(stateRoot, ".index");
+	if (!existsSync(indexDir)) {
+		mkdirSync(indexDir, { recursive: true });
+	}
+	memoryIndex = new LocalIndex(indexPath);
+	if (!await memoryIndex.isIndexCreated()) {
+		logger.log("Creating new memory index...");
+		await memoryIndex.createIndex();
+	}
+	return memoryIndex;
+}
+/**
+* Search memory index
+*/
+async function searchMemory(query, options = {}) {
+	const { limit = 5, type, since } = options;
+	const idx = await getMemoryIndex();
+	const stats = await idx.listItems();
+	if (stats.length === 0) {
+		return [];
+	}
+	const queryVector = await embedQuery(query);
+	const results = await idx.queryItems(queryVector, query, limit * 2);
+	let filtered = results;
+	if (type || since) {
+		filtered = results.filter((item) => {
+			const meta = item.item.metadata;
+			if (type && meta.type !== type) return false;
+			if (since && meta.timestamp && meta.timestamp < since) return false;
+			return true;
+		});
+	}
+	return filtered.slice(0, limit).map((r) => {
+		const meta = r.item.metadata;
+		return {
+			content: meta.content,
+			source: meta.source,
+			section: meta.section,
+			timestamp: meta.timestamp,
+			type: meta.type,
+			score: r.score
+		};
+	});
+}
+/**
+* Rebuild memory index from journal and entity files
+*/
+async function rebuildMemoryIndex() {
+	logger.log("Rebuilding memory index...");
+	const startTime = Date.now();
+	const stateRoot = getStateRoot();
+	const allItems = [];
+	const journalDir = join(stateRoot, "journal");
+	if (existsSync(journalDir)) {
+		const files = readdirSync(journalDir).filter((f) => f.endsWith(".jsonl"));
+		for (const file of files) {
+			try {
+				const content = readFileSync(join(journalDir, file), "utf-8");
+				const lines = content.trim().split("\n").filter(Boolean);
+				for (let i = 0; i < lines.length; i++) {
+					try {
+						const entry = JSON.parse(lines[i]);
+						allItems.push({
+							id: `journal-${file}-${i}`,
+							type: "journal",
+							content: `[${entry.topic}] ${entry.content}`,
+							source: file,
+							timestamp: entry.timestamp
+						});
+					} catch {}
+				}
+			} catch {}
+		}
+	}
+	const entitiesDir = join(stateRoot, "entities");
+	for (const [subdir, type] of [["people", "person"], ["projects", "project"]]) {
+		const dir = join(entitiesDir, subdir);
+		if (!existsSync(dir)) continue;
+		const files = readdirSync(dir).filter((f) => f.endsWith(".md"));
+		for (const file of files) {
+			try {
+				const content = readFileSync(join(dir, file), "utf-8");
+				const filename = file.replace(".md", "");
+				const sections = content.split(/^## /m);
+				if (sections[0].trim()) {
+					allItems.push({
+						id: `${type}-${filename}-preamble`,
+						type,
+						content: sections[0].trim(),
+						source: `${subdir}/${file}`,
+						section: "preamble"
+					});
+				}
+				for (let i = 1; i < sections.length; i++) {
+					const section = sections[i];
+					const firstLine = section.split("\n")[0];
+					const sectionTitle = firstLine.trim();
+					const sectionContent = section.slice(firstLine.length).trim();
+					if (sectionContent) {
+						allItems.push({
+							id: `${type}-${filename}-${i}`,
+							type,
+							content: `## ${sectionTitle}\n\n${sectionContent}`,
+							source: `${subdir}/${file}`,
+							section: sectionTitle
+						});
+					}
+				}
+			} catch {}
+		}
+	}
+	const topicsDir = join(stateRoot, "topics");
+	if (existsSync(topicsDir)) {
+		const files = readdirSync(topicsDir).filter((f) => f.endsWith(".md"));
+		for (const file of files) {
+			try {
+				const content = readFileSync(join(topicsDir, file), "utf-8");
+				const filename = file.replace(".md", "");
+				allItems.push({
+					id: `topic-${filename}`,
+					type: "topic",
+					content: content.trim(),
+					source: `topics/${file}`
+				});
+			} catch {}
+		}
+	}
+	if (allItems.length === 0) {
+		return { itemCount: 0 };
+	}
+	logger.log(`Generating embeddings for ${allItems.length} items...`);
+	const vectors = await embedBatch(allItems.map((i) => i.content));
+	const idx = await getMemoryIndex();
+	await idx.beginUpdate();
+	try {
+		for (let i = 0; i < allItems.length; i++) {
+			const item = allItems[i];
+			const metadata = {
+				type: item.type,
+				content: item.content,
+				source: item.source
+			};
+			if (item.section) metadata.section = item.section;
+			if (item.timestamp) metadata.timestamp = item.timestamp;
+			await idx.upsertItem({
+				id: item.id,
+				vector: vectors[i],
+				metadata
+			});
+		}
+		await idx.endUpdate();
+	} catch (err) {
+		idx.cancelUpdate();
+		throw err;
+	}
+	const duration = Date.now() - startTime;
+	logger.log(`Index rebuilt in ${duration}ms`);
+	return { itemCount: allItems.length };
+}
+/**
+* Get memory index stats
+*/
+async function getMemoryIndexStats() {
+	const idx = await getMemoryIndex();
+	const items = await idx.listItems();
+	return { itemCount: items.length };
+}
+/**
+* Index a single journal entry (incremental)
+*/
+async function indexJournalEntry(entry) {
+	const idx = await getMemoryIndex();
+	const vector = await embed(`[${entry.topic}] ${entry.content}`);
+	await idx.upsertItem({
+		id: `journal-${entry.timestamp}`,
+		vector,
+		metadata: {
+			type: "journal",
+			content: `[${entry.topic}] ${entry.content}`,
+			source: "journal",
+			timestamp: entry.timestamp
+		}
+	});
+}
+
+//#endregion
+export { getMemoryIndexStats, indexJournalEntry, rebuildMemoryIndex, searchMemory };

package/dist/opencode/skills/macrodata-distill/SKILL.md

@@ -0,0 +1,171 @@
+---
+name: macrodata-distill
+description: Extract distilled actions and facts from today's conversations. Spawns sub-agents per conversation to avoid context blowup.
+---
+
+# Distill Conversations
+
+Process today's conversations to extract actionable knowledge. This is the core of memory consolidation.
+
+**Important:** This runs as a coordinator. Spawn sub-agents for each conversation to avoid loading full transcripts into your context.
+
+## Storage Format
+
+OpenCode stores all session data in a SQLite database at `~/.local/share/opencode/opencode.db`.
+
+**Schema:**
+
+- `session` — id, project_id, parent_id, title, time_created, time_updated
+- `message` — id, session_id, time_created, data (JSON: role, agent, modelID, etc.)
+- `part` — id, message_id, session_id, time_created, data (JSON: type, text, etc.)
+- `project` — id, worktree, name
+
+**Part types:** text, tool, step-start, step-finish, patch, reasoning, compaction, file, subtask
+
+**Key JSON paths:**
+
+- `message.data` → `$.role` (user/assistant), `$.summary` (set on compaction messages)
+- `part.data` → `$.type` (text/tool/etc.), `$.text` (for text parts)
+
+## Process
+
+### 1. Find Today's Sessions
+
+Query the SQLite database for sessions with activity today. Exclude subtask sessions (parent_id IS NOT NULL).
+
+```bash
+sqlite3 ~/.local/share/opencode/opencode.db "
+  SELECT s.id, s.title, p.worktree, s.time_created
+  FROM session s
+  LEFT JOIN project p ON p.id = s.project_id
+  WHERE s.parent_id IS NULL
+    AND s.time_updated > unixepoch('now', '-1 day') * 1000
+  ORDER BY s.time_updated DESC
+"
+```
+
+### 2. Process Each Session
+
+For **each** session, spawn a sub-agent with the Task tool:
+
+```
+Task(subagent_type="general", prompt=`
+Read an OpenCode conversation from the SQLite database at ~/.local/share/opencode/opencode.db.
+
+Session ID: {sessionId}
+Session title: {sessionTitle}
+Project: {projectWorktree}
+
+Use this query to extract the conversation (user prompts and assistant text responses):
+
+sqlite3 ~/.local/share/opencode/opencode.db "
+  SELECT
+    m.id AS message_id,
+    json_extract(m.data, '$.role') AS role,
+    m.time_created,
+    GROUP_CONCAT(
+      CASE WHEN json_extract(p.data, '$.type') = 'text'
+        THEN json_extract(p.data, '$.text')
+      END,
+      char(10)
+    ) AS text_content
+  FROM message m
+  JOIN part p ON p.message_id = m.id
+  WHERE m.session_id = '{sessionId}'
+    AND json_extract(m.data, '$.role') IN ('user', 'assistant')
+    AND json_extract(m.data, '$.summary') IS NULL
+  GROUP BY m.id
+  HAVING text_content IS NOT NULL AND text_content != ''
+  ORDER BY m.time_created ASC
+"
+
+Filter to actual conversation content:
+- Include: user messages, assistant text responses
+- Exclude: tool calls, tool results, system content, compaction summaries
+
+Extract and return as JSON:
+{
+  "distilled_actions": [
+    {
+      "summary": "Fixed auth bug in src/auth.ts where token refresh was racing",
+      "files": ["src/auth.ts"],
+      "outcome": "Added mutex lock around refresh"
+    }
+  ],
+  "facts": [
+    {
+      "topic": "project-name",
+      "content": "Uses JWT tokens with 15min expiry"
+    },
+    {
+      "topic": "person-name",
+      "content": "Prefers explicit error handling over try/catch"
+    }
+  ],
+  "decisions": [
+    "Chose Redis over in-memory cache for session storage because of multi-instance deployment"
+  ]
+}
+
+Focus on:
+- What was accomplished (not just discussed)
+- Decisions made and their rationale
+- New information about projects, people, or preferences
+- File paths and specific technical details that should survive compression
+
+Return ONLY the JSON, no explanation.
+`)
+```
+
+### 3. Collect and Write Results
+
+After all sub-agents complete:
+
+**Write distilled actions to journal:**
+
+```
+For each action in all results:
+  macrodata_log_journal(topic="distilled", content=action.summary + " Files: " + action.files.join(", "))
+```
+
+**Write overall summary to journal:**
+
+```
+macrodata_log_journal(topic="distill-summary", content="Processed N sessions. Extracted X actions, Y facts.")
+```
+
+**Update entity files with facts:**
+
+- Group facts by topic
+- For each topic, read existing entity file (if any)
+- Integrate new facts, removing duplicates
+- Write updated file
+
+### 4. Example Sub-Agent Output
+
+```json
+{
+  "distilled_actions": [
+    {
+      "summary": "Added /distill skill to macrodata plugin",
+      "files": ["plugins/macrodata/skills/distill/SKILL.md"],
+      "outcome": "Skill extracts facts from conversations via sub-agents"
+    }
+  ],
+  "facts": [
+    {
+      "topic": "macrodata",
+      "content": "Distillation separates narrative context from retained facts for better compression"
+    }
+  ],
+  "decisions": [
+    "Coordinator updates state directly to prevent race conditions from parallel sub-agents"
+  ]
+}
+```
+
+## Notes
+
+- Sub-agents should be spawned in parallel for efficiency
+- Empty results are fine - not every conversation has extractable knowledge
+- Facts should be concise and specific, not narrative summaries