@geoql/mdr 0.0.1

package/dist/src/embeddings.js

@@ -0,0 +1,190 @@
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+//#region src/embeddings.ts
+/**
+* Embeddings module
+*
+* Default: local embedding generation via Transformers.js
+* (all-MiniLM-L6-v2, 384-dim), no API calls, fully offline.
+*
+* Optional: a remote OpenAI-compatible embeddings endpoint configured in
+* ~/.config/macrodata/config.json, which offloads embedding to an API and
+* avoids loading the local model entirely:
+*
+*   {
+*     "embedding": {
+*       "provider": "openai-compatible",
+*       "endpoint": "https://api.example.com/v1",
+*       "api_key": "sk-...",            // or "api_key_env": "MY_KEY_VAR"
+*       "model": "baai/bge-m3",
+*       "input_type": "passage",        // optional, for models that need it
+*       "query_input_type": "query",    // optional
+*       "batch_size": 64,               // optional, default 64
+*       "extra_body": {}                // optional, merged into the request
+*     }
+*   }
+*/
+const EMBEDDING_DIMENSIONS = 384;
+let cachedRemoteConfig;
+function getRemoteEmbeddingConfig() {
+	if (cachedRemoteConfig !== undefined) return cachedRemoteConfig;
+	const configPath = process.env.MACRODATA_CONFIG_PATH || join(homedir(), ".config", "macrodata", "config.json");
+	cachedRemoteConfig = null;
+	if (existsSync(configPath)) {
+		try {
+			const config = JSON.parse(readFileSync(configPath, "utf-8"));
+			const embedding = config.embedding;
+			if (embedding && embedding.provider === "openai-compatible" && typeof embedding.endpoint === "string" && typeof embedding.model === "string") {
+				cachedRemoteConfig = embedding;
+			}
+		} catch (err) {
+			console.error(`[Embeddings] Failed to parse config.json, using local model: ${String(err)}`);
+		}
+	}
+	return cachedRemoteConfig;
+}
+function resetEmbeddingConfigCache() {
+	cachedRemoteConfig = undefined;
+}
+function resetLocalPipelineForTests() {
+	embeddingPipeline = null;
+	pipelineLoading = null;
+}
+function resolveApiKey(config) {
+	if (config.api_key_env) {
+		const fromEnv = process.env[config.api_key_env];
+		if (fromEnv) return fromEnv;
+	}
+	return config.api_key;
+}
+async function embedRemote(texts, config, kind) {
+	const url = `${config.endpoint.replace(/\/$/, "")}/embeddings`;
+	const apiKey = resolveApiKey(config);
+	const inputType = kind === "query" ? config.query_input_type : config.input_type;
+	const body = {
+		...config.extra_body,
+		model: config.model,
+		input: texts
+	};
+	if (inputType) {
+		body.input_type = inputType;
+	}
+	const response = await fetch(url, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
+		},
+		body: JSON.stringify(body)
+	});
+	if (!response.ok) {
+		const errText = await response.text().catch(() => "");
+		throw new Error(`Remote embedding request failed: ${response.status} ${response.statusText} ${errText.slice(0, 300)}`);
+	}
+	const result = await response.json();
+	if (!result.data || result.data.length !== texts.length) {
+		throw new Error(`Remote embedding response mismatch: expected ${texts.length} embeddings, got ${result.data?.length ?? 0}`);
+	}
+	const sorted = [...result.data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
+	return sorted.map((d) => d.embedding);
+}
+async function embedRemoteBatched(texts, config, kind) {
+	const batchSize = config.batch_size && config.batch_size > 0 ? config.batch_size : 64;
+	const results = [];
+	for (let i = 0; i < texts.length; i += batchSize) {
+		const batch = texts.slice(i, i + batchSize);
+		results.push(...await embedRemote(batch, config, kind));
+	}
+	return results;
+}
+let embeddingPipeline = null;
+let pipelineLoading = null;
+/**
+* Get or create the local embedding pipeline
+* Uses all-MiniLM-L6-v2 – good balance of quality and speed
+*/
+async function getEmbeddingPipeline() {
+	if (embeddingPipeline) {
+		return embeddingPipeline;
+	}
+	if (pipelineLoading) {
+		return pipelineLoading;
+	}
+	pipelineLoading = (async () => {
+		const { pipeline } = await import("@huggingface/transformers");
+		return pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", { dtype: "q8" });
+	})();
+	try {
+		embeddingPipeline = await pipelineLoading;
+		console.log("[Embeddings] Model loaded successfully");
+		return embeddingPipeline;
+	} finally {
+		pipelineLoading = null;
+	}
+}
+async function embedLocal(texts) {
+	const pipe = await getEmbeddingPipeline();
+	const batchSize = 32;
+	const results = [];
+	for (let i = 0; i < texts.length; i += batchSize) {
+		const batch = texts.slice(i, i + batchSize);
+		const outputs = await pipe(batch, {
+			pooling: "mean",
+			normalize: true
+		});
+		const dims = outputs.dims?.at(-1) || 384;
+		for (let j = 0; j < batch.length; j++) {
+			const start = j * dims;
+			const end = start + dims;
+			results.push(Array.from(outputs.data.slice(start, end)));
+		}
+	}
+	return results;
+}
+/**
+* Generate an embedding for a single text (indexed content)
+*/
+async function embed(text) {
+	const [vector] = await embedBatch([text]);
+	return vector;
+}
+/**
+* Generate an embedding for a search query.
+* Remote providers may use a different input_type for queries (e.g. BGE-M3).
+*/
+async function embedQuery(text) {
+	const remote = getRemoteEmbeddingConfig();
+	if (remote) {
+		const [vector] = await embedRemoteBatched([text], remote, "query");
+		return vector;
+	}
+	const [vector] = await embedLocal([text]);
+	return vector;
+}
+/**
+* Generate embeddings for multiple texts (batched, indexed content)
+*/
+async function embedBatch(texts) {
+	if (texts.length === 0) return [];
+	const remote = getRemoteEmbeddingConfig();
+	if (remote) {
+		return embedRemoteBatched(texts, remote, "passage");
+	}
+	return embedLocal(texts);
+}
+/**
+* Preload the model (call during startup to avoid first-query delay).
+* No-op when a remote embedding provider is configured.
+*/
+async function preloadModel() {
+	if (getRemoteEmbeddingConfig()) {
+		console.log("[Embeddings] Remote embedding provider configured, skipping local model load");
+		return;
+	}
+	await getEmbeddingPipeline();
+}
+
+//#endregion
+export { embed, embedBatch, embedQuery, getRemoteEmbeddingConfig, preloadModel };

package/dist/src/index.js

@@ -0,0 +1,413 @@
+import { getEntitiesDir, getIndexDir, getJournalDir, getRemindersDir, getStateDir, getStateRoot } from "./config.js";
+import { expandConversation, getConversationIndexStats, rebuildConversationIndex, searchConversations, updateConversationIndex } from "./conversations.js";
+import { getIndexStats, indexJournalEntry, rebuildIndex, searchMemory } from "./indexer.js";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from "fs";
+import { join } from "path";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+
+//#region src/index.ts
+/**
+* Macrodata Local MCP Server
+*
+* Provides tools for local file-based memory:
+* - log_journal: Append timestamped entries (with auto-indexing)
+* - get_recent_journal: Get recent entries
+* - search_memory: Semantic search using Transformers.js
+* - manage_index: Rebuild or get stats for memory/conversation indexes
+* - schedule: Create cron or one-shot reminders
+* - list_reminders: List active schedules
+* - remove_reminder: Delete a reminder
+* - save_conversation_summary: Save session summaries
+* - get_recent_summaries: Get past summaries
+* - search_conversations: Search past Claude Code sessions
+* - expand_conversation: Load full context from a session
+*/
+function ensureDirectories() {
+	const entitiesDir = getEntitiesDir();
+	const dirs = [
+		getStateRoot(),
+		getStateDir(),
+		entitiesDir,
+		join(entitiesDir, "people"),
+		join(entitiesDir, "projects"),
+		getJournalDir(),
+		getIndexDir()
+	];
+	for (const dir of dirs) {
+		if (!existsSync(dir)) {
+			mkdirSync(dir, { recursive: true });
+		}
+	}
+}
+function loadAllSchedules() {
+	const remindersDir = getRemindersDir();
+	const schedules = [];
+	try {
+		if (!existsSync(remindersDir)) return schedules;
+		const files = readdirSync(remindersDir).filter((f) => f.endsWith(".json"));
+		for (const file of files) {
+			try {
+				const content = readFileSync(join(remindersDir, file), "utf-8");
+				schedules.push(JSON.parse(content));
+			} catch {}
+		}
+	} catch {}
+	return schedules;
+}
+function saveSchedule(schedule) {
+	const remindersDir = getRemindersDir();
+	if (!existsSync(remindersDir)) {
+		mkdirSync(remindersDir, { recursive: true });
+	}
+	const filePath = join(remindersDir, `${schedule.id}.json`);
+	writeFileSync(filePath, JSON.stringify(schedule, null, 2));
+}
+function deleteScheduleFile(id) {
+	const remindersDir = getRemindersDir();
+	const filePath = join(remindersDir, `${id}.json`);
+	try {
+		if (existsSync(filePath)) {
+			unlinkSync(filePath);
+			return true;
+		}
+	} catch {}
+	return false;
+}
+function getTodayJournalPath() {
+	const today = new Date().toISOString().split("T")[0];
+	return join(getJournalDir(), `${today}.jsonl`);
+}
+function getRecentJournalEntries(count) {
+	const entries = [];
+	const journalDir = getJournalDir();
+	if (!existsSync(journalDir)) return entries;
+	const files = readdirSync(journalDir).filter((f) => f.endsWith(".jsonl")).sort().reverse();
+	for (const file of files) {
+		if (entries.length >= count) break;
+		const content = readFileSync(join(journalDir, file), "utf-8");
+		const lines = content.trim().split("\n").filter(Boolean);
+		for (const line of lines.reverse()) {
+			if (entries.length >= count) break;
+			try {
+				entries.push(JSON.parse(line));
+			} catch {}
+		}
+	}
+	return entries;
+}
+function createServer() {
+	const server = new McpServer({
+		name: "macrodata-local",
+		version: "0.1.0"
+	});
+	server.tool("log_journal", "Append a timestamped entry to the journal", {
+		topic: z.string().describe("Category or tag for this entry"),
+		content: z.string().describe("The actual note or observation"),
+		source: z.string().optional().describe("Where this came from (conversation, cron, etc.)"),
+		intent: z.string().optional().describe("What you were doing when logging this")
+	}, async ({ topic, content, source, intent }) => {
+		ensureDirectories();
+		const entry = {
+			timestamp: new Date().toISOString(),
+			topic,
+			content,
+			metadata: {
+				...source && { source },
+				...intent && { intent }
+			}
+		};
+		const journalPath = getTodayJournalPath();
+		appendFileSync(journalPath, JSON.stringify(entry) + "\n");
+		try {
+			await indexJournalEntry(entry);
+		} catch (err) {
+			console.error("[log_journal] Failed to index entry:", err);
+		}
+		return { content: [{
+			type: "text",
+			text: `Logged to journal: ${topic}`
+		}] };
+	});
+	server.tool("get_recent_journal", "Get the N most recent journal entries, optionally filtered by topic", {
+		count: z.number().default(10).describe("Number of entries to retrieve"),
+		topic: z.string().optional().describe("Filter by specific topic")
+	}, async ({ count, topic }) => {
+		let entries = getRecentJournalEntries(Math.min(count * 2, 100));
+		if (topic) {
+			entries = entries.filter((e) => e.topic === topic);
+		}
+		entries = entries.slice(0, count);
+		return { content: [{
+			type: "text",
+			text: JSON.stringify(entries, null, 2)
+		}] };
+	});
+	server.tool("search_memory", "Semantic search across journal entries and entity files. Returns ranked results.", {
+		query: z.string().describe("Natural language search query"),
+		type: z.enum([
+			"journal",
+			"person",
+			"project",
+			"all"
+		]).default("all").describe("Filter by content type"),
+		since: z.string().optional().describe("Only include items after this ISO date"),
+		limit: z.number().default(5).describe("Maximum results to return")
+	}, async ({ query, type, since, limit }) => {
+		try {
+			const results = await searchMemory(query, {
+				limit,
+				type: type === "all" ? undefined : type,
+				since
+			});
+			if (results.length === 0) {
+				return { content: [{
+					type: "text",
+					text: "(no matches found)"
+				}] };
+			}
+			const formatted = results.map((r, i) => {
+				const header = `[${i + 1}] ${r.type}${r.section ? ` / ${r.section}` : ""} (score: ${r.score.toFixed(3)})`;
+				const meta = r.timestamp ? `  Date: ${r.timestamp}` : "";
+				const source = `  Source: ${r.source}`;
+				const content = r.content.slice(0, 500) + (r.content.length > 500 ? "..." : "");
+				return [
+					header,
+					meta,
+					source,
+					"",
+					content
+				].filter(Boolean).join("\n");
+			}).join("\n\n---\n\n");
+			return { content: [{
+				type: "text",
+				text: formatted
+			}] };
+		} catch (err) {
+			return { content: [{
+				type: "text",
+				text: `Search error: ${String(err)}`
+			}] };
+		}
+	});
+	server.tool("manage_index", "Manage search indexes. Target 'memory' for journal/entities, 'conversations' for past Claude Code sessions.", {
+		target: z.enum(["memory", "conversations"]).describe("Which index to manage"),
+		action: z.enum([
+			"rebuild",
+			"update",
+			"stats"
+		]).describe("'rebuild' to reindex from scratch, 'update' for incremental (conversations only), 'stats' to get counts")
+	}, async ({ target, action }) => {
+		try {
+			if (target === "memory") {
+				if (action === "rebuild" || action === "update") {
+					const result = await rebuildIndex();
+					return { content: [{
+						type: "text",
+						text: `Memory index rebuilt. Indexed ${result.itemCount} items.`
+					}] };
+				} else {
+					const stats = await getIndexStats();
+					return { content: [{
+						type: "text",
+						text: `Memory index contains ${stats.itemCount} items.`
+					}] };
+				}
+			} else {
+				if (action === "rebuild") {
+					rebuildConversationIndex().then((result) => console.log(`[Macrodata] Conversation index rebuilt: ${result.exchangeCount} exchanges`)).catch((err) => console.error(`[Macrodata] Conversation index rebuild failed: ${err}`));
+					return { content: [{
+						type: "text",
+						text: `Conversation index rebuild started in background.`
+					}] };
+				} else if (action === "update") {
+					updateConversationIndex().then((result) => console.log(`[Macrodata] Conversation index updated: ${result.filesUpdated} files (${result.skipped} skipped, total: ${result.exchangeCount})`)).catch((err) => console.error(`[Macrodata] Conversation index update failed: ${err}`));
+					return { content: [{
+						type: "text",
+						text: `Conversation index update started in background.`
+					}] };
+				} else {
+					const stats = await getConversationIndexStats();
+					return { content: [{
+						type: "text",
+						text: `Conversation index contains ${stats.exchangeCount} exchanges.`
+					}] };
+				}
+			}
+		} catch (err) {
+			return { content: [{
+				type: "text",
+				text: `Failed to ${action} ${target} index: ${String(err)}`
+			}] };
+		}
+	});
+	server.tool("schedule", "Create a reminder. Use type 'cron' for recurring (expression is cron syntax) or 'once' for one-shot (expression is ISO datetime).", {
+		type: z.enum(["cron", "once"]).describe("'cron' for recurring, 'once' for one-shot"),
+		id: z.string().describe("Unique identifier for this reminder"),
+		expression: z.string().describe("Cron expression (e.g., '0 9 * * *') or ISO datetime (e.g., '2026-01-31T10:00:00')"),
+		description: z.string().describe("What this reminder is for"),
+		payload: z.string().describe("Message to process when reminder fires"),
+		model: z.string().optional().describe("Model to use (e.g., 'anthropic/claude-opus-4-6' for deep thinking tasks)")
+	}, async ({ type, id, expression, description, payload, model }) => {
+		const schedule = {
+			id,
+			type,
+			expression,
+			description,
+			payload,
+			agent: "claude",
+			model,
+			createdAt: new Date().toISOString()
+		};
+		saveSchedule(schedule);
+		const typeLabel = type === "cron" ? "recurring" : "one-shot";
+		return { content: [{
+			type: "text",
+			text: `Created ${typeLabel} reminder: ${id} (${expression})${model ? ` with model ${model}` : ""}`
+		}] };
+	});
+	server.tool("list_reminders", "List all active scheduled reminders", {}, async () => {
+		const schedules = loadAllSchedules();
+		return { content: [{
+			type: "text",
+			text: JSON.stringify(schedules, null, 2)
+		}] };
+	});
+	server.tool("remove_reminder", "Remove a scheduled reminder", { id: z.string().describe("ID of the reminder to remove") }, async ({ id }) => {
+		const removed = deleteScheduleFile(id);
+		return { content: [{
+			type: "text",
+			text: removed ? `Removed reminder: ${id}` : `Reminder not found: ${id}`
+		}] };
+	});
+	server.tool("save_conversation_summary", "Save a summary of the current conversation for context recovery in future sessions", {
+		summary: z.string().describe("Brief summary of what was discussed/accomplished"),
+		keyDecisions: z.array(z.string()).optional().describe("Important decisions made"),
+		openThreads: z.array(z.string()).optional().describe("Topics to follow up on"),
+		learnedPatterns: z.array(z.string()).optional().describe("New patterns learned about the user"),
+		notes: z.string().optional().describe("Freeform notes")
+	}, async ({ summary, keyDecisions, openThreads, learnedPatterns, notes }) => {
+		ensureDirectories();
+		const parts = [summary];
+		if (keyDecisions?.length) parts.push(`Decisions: ${keyDecisions.join(", ")}`);
+		if (openThreads?.length) parts.push(`Open threads: ${openThreads.join(", ")}`);
+		if (learnedPatterns?.length) parts.push(`Learned: ${learnedPatterns.join(", ")}`);
+		if (notes) parts.push(`Notes: ${notes}`);
+		const entry = {
+			timestamp: new Date().toISOString(),
+			topic: "conversation-summary",
+			content: parts.join("\n"),
+			metadata: { source: "conversation" }
+		};
+		const journalPath = getTodayJournalPath();
+		appendFileSync(journalPath, JSON.stringify(entry) + "\n");
+		try {
+			await indexJournalEntry(entry);
+		} catch (err) {
+			console.error("[save_conversation_summary] Failed to index:", err);
+		}
+		return { content: [{
+			type: "text",
+			text: "Conversation summary saved."
+		}] };
+	});
+	server.tool("get_recent_summaries", "Get recent conversation summaries for context recovery", { count: z.number().default(7).describe("Number of summaries to retrieve") }, async ({ count }) => {
+		let entries = getRecentJournalEntries(count * 3);
+		entries = entries.filter((e) => e.topic === "conversation-summary").slice(0, count);
+		if (entries.length === 0) {
+			return { content: [{
+				type: "text",
+				text: "No conversation summaries yet."
+			}] };
+		}
+		return { content: [{
+			type: "text",
+			text: JSON.stringify(entries, null, 2)
+		}] };
+	});
+	server.tool("search_conversations", "Search past Claude Code conversations for similar problems/solutions. By default searches current project first, with recent conversations weighted higher.", {
+		query: z.string().describe("What to search for (e.g., 'fixing TypeScript errors', 'performance optimization')"),
+		projectOnly: z.boolean().default(false).describe("Only search current project (default: search all but boost current)"),
+		limit: z.number().default(5).describe("Maximum results to return")
+	}, async ({ query, projectOnly, limit }) => {
+		try {
+			const currentProject = process.env.CLAUDE_PROJECT_DIR;
+			const results = await searchConversations(query, {
+				currentProject,
+				projectOnly,
+				limit
+			});
+			if (results.length === 0) {
+				return { content: [{
+					type: "text",
+					text: "No matching conversations found. Try manage_index(target: 'conversations', action: 'update')."
+				}] };
+			}
+			const formatted = results.map((r, i) => {
+				const date = new Date(r.exchange.timestamp).toLocaleDateString();
+				const branch = r.exchange.branch ? ` (${r.exchange.branch})` : "";
+				return `[${i + 1}] ${r.exchange.project}${branch} - ${date}
+    "${r.exchange.userPrompt.slice(0, 200)}${r.exchange.userPrompt.length > 200 ? "..." : ""}"
+    Session: ${r.exchange.sessionId}`;
+			}).join("\n\n");
+			return { content: [{
+				type: "text",
+				text: `Found ${results.length} relevant conversation(s):\n\n${formatted}\n\nUse expand_conversation to see full context.`
+			}] };
+		} catch (err) {
+			return { content: [{
+				type: "text",
+				text: `Search error: ${String(err)}`
+			}] };
+		}
+	});
+	server.tool("expand_conversation", "Load full context from a past conversation. Use after search_conversations to see the complete exchange.", {
+		sessionPath: z.string().describe("Session file path from search results"),
+		messageUuid: z.string().optional().describe("Specific message UUID to center on"),
+		contextMessages: z.number().default(10).describe("Number of messages to include")
+	}, async ({ sessionPath, messageUuid, contextMessages }) => {
+		try {
+			let fullPath = sessionPath;
+			if (!sessionPath.startsWith("/")) {
+				return { content: [{
+					type: "text",
+					text: "Please provide the full session path from search results."
+				}] };
+			}
+			const result = await expandConversation(fullPath, messageUuid || "", contextMessages);
+			const formatted = result.messages.map((m) => {
+				const prefix = m.role === "user" ? "User" : "Assistant";
+				return `**${prefix}**: ${m.content}`;
+			}).join("\n\n---\n\n");
+			const header = `Project: ${result.project}${result.branch ? ` (${result.branch})` : ""}\n\n`;
+			return { content: [{
+				type: "text",
+				text: header + formatted
+			}] };
+		} catch (err) {
+			return { content: [{
+				type: "text",
+				text: `Failed to expand conversation: ${String(err)}`
+			}] };
+		}
+	});
+	return server;
+}
+async function main() {
+	const server = createServer();
+	const transport = new StdioServerTransport();
+	await server.connect(transport);
+}
+function isRunAsMain(argv1, moduleUrl) {
+	return Boolean(argv1) && moduleUrl === `file://${argv1}`;
+}
+/* v8 ignore next 3 -- entry-point glue: only runs when this file is the process
+entry (node dist/src/index.js); that path is a subprocess vitest cannot
+instrument. main() and isRunAsMain() are each covered directly above. */
+if (isRunAsMain(process.argv[1], import.meta.url)) {
+	main().catch(console.error);
+}
+
+//#endregion
+export { createServer, isRunAsMain, main };