@geoql/mdr 0.0.1

package/dist/src/indexer.js

@@ -0,0 +1,286 @@
+import { getEntitiesDir, getIndexDir, getJournalDir } from "./config.js";
+import { embed, embedBatch, embedQuery, preloadModel as preloadModel$1 } from "./embeddings.js";
+import { existsSync, mkdirSync, readFileSync, readdirSync } from "fs";
+import { basename, join } from "path";
+import { LocalIndex } from "vectra";
+
+//#region src/indexer.ts
+/**
+* Memory Indexer
+*
+* Manages the vector index for semantic search over:
+* - Journal entries
+* - People files
+* - Project files
+*
+* Uses Vectra for storage and embeddings.ts for vector generation.
+*/
+let index = null;
+let indexPath = null;
+/**
+* Get or create the vector index
+* Re-creates if the configured path has changed
+*/
+async function getIndex() {
+	const currentIndexDir = getIndexDir();
+	const currentIndexPath = join(currentIndexDir, "vectors");
+	if (index && indexPath !== currentIndexPath) {
+		index = null;
+		indexPath = null;
+	}
+	if (index) return index;
+	if (!existsSync(currentIndexDir)) {
+		mkdirSync(currentIndexDir, { recursive: true });
+	}
+	index = new LocalIndex(currentIndexPath);
+	indexPath = currentIndexPath;
+	if (!await index.isIndexCreated()) {
+		console.log("[Indexer] Creating new index...");
+		await index.createIndex();
+	}
+	return index;
+}
+/**
+* Add or update a single item in the index
+*/
+async function indexItem(item) {
+	const idx = await getIndex();
+	const vector = await embed(item.content);
+	const metadata = {
+		type: item.type,
+		content: item.content,
+		source: item.source
+	};
+	if (item.section) metadata.section = item.section;
+	if (item.timestamp) metadata.timestamp = item.timestamp;
+	await idx.upsertItem({
+		id: item.id,
+		vector,
+		metadata
+	});
+}
+/**
+* Add or update multiple items (batched for efficiency)
+*/
+async function indexItems(items) {
+	if (items.length === 0) return;
+	const idx = await getIndex();
+	const vectors = await embedBatch(items.map((i) => i.content));
+	for (let i = 0; i < items.length; i++) {
+		const item = items[i];
+		const metadata = {
+			type: item.type,
+			content: item.content,
+			source: item.source
+		};
+		if (item.section) metadata.section = item.section;
+		if (item.timestamp) metadata.timestamp = item.timestamp;
+		await idx.upsertItem({
+			id: item.id,
+			vector: vectors[i],
+			metadata
+		});
+	}
+}
+/**
+* Search the index
+*/
+async function searchMemory(query, options = {}) {
+	const { limit = 5, type, since } = options;
+	const idx = await getIndex();
+	const stats = await idx.listItems();
+	if (stats.length === 0) {
+		console.log("[Indexer] Index is empty");
+		return [];
+	}
+	const queryVector = await embedQuery(query);
+	const results = await idx.queryItems(queryVector, query, limit * 2);
+	let filtered = results;
+	if (type || since) {
+		filtered = results.filter((item) => {
+			const meta = item.item.metadata;
+			if (type && meta.type !== type) return false;
+			if (since && meta.timestamp && meta.timestamp < since) return false;
+			return true;
+		});
+	}
+	return filtered.slice(0, limit).map((r) => {
+		const meta = r.item.metadata;
+		return {
+			content: meta.content,
+			source: meta.source,
+			section: meta.section,
+			timestamp: meta.timestamp,
+			type: meta.type,
+			score: r.score
+		};
+	});
+}
+/**
+* Parse journal files and return items for indexing
+*/
+function parseJournalForIndexing() {
+	const items = [];
+	const journalDir = getJournalDir();
+	if (!existsSync(journalDir)) return items;
+	const files = readdirSync(journalDir).filter((f) => f.endsWith(".jsonl"));
+	for (const file of files) {
+		try {
+			const content = readFileSync(join(journalDir, file), "utf-8");
+			const lines = content.trim().split("\n").filter(Boolean);
+			for (let i = 0; i < lines.length; i++) {
+				try {
+					const entry = JSON.parse(lines[i]);
+					items.push({
+						id: `journal-${file}-${i}`,
+						type: "journal",
+						content: `[${entry.topic}] ${entry.content}`,
+						source: file,
+						timestamp: entry.timestamp
+					});
+				} catch {}
+			}
+		} catch {}
+	}
+	return items;
+}
+/**
+* Parse entity files (people, projects) for indexing
+*/
+function parseEntitiesForIndexing(subdir, type) {
+	const items = [];
+	const dir = join(getEntitiesDir(), subdir);
+	if (!existsSync(dir)) return items;
+	const files = readdirSync(dir).filter((f) => f.endsWith(".md"));
+	for (const file of files) {
+		try {
+			const content = readFileSync(join(dir, file), "utf-8");
+			const filename = file.replace(".md", "");
+			const sections = content.split(/^## /m);
+			if (sections[0].trim()) {
+				items.push({
+					id: `${type}-${filename}-preamble`,
+					type,
+					content: sections[0].trim(),
+					source: `${subdir}/${file}`,
+					section: "preamble"
+				});
+			}
+			for (let i = 1; i < sections.length; i++) {
+				const section = sections[i];
+				const firstLine = section.split("\n")[0];
+				const sectionTitle = firstLine.trim();
+				const sectionContent = section.slice(firstLine.length).trim();
+				if (sectionContent) {
+					items.push({
+						id: `${type}-${filename}-${i}`,
+						type,
+						content: `## ${sectionTitle}\n\n${sectionContent}`,
+						source: `${subdir}/${file}`,
+						section: sectionTitle
+					});
+				}
+			}
+		} catch {}
+	}
+	return items;
+}
+/**
+* Rebuild the entire index from scratch
+*/
+async function rebuildIndex() {
+	console.log("[Indexer] Starting full index rebuild...");
+	const startTime = Date.now();
+	const allItems = [];
+	console.log("[Indexer] Parsing journal...");
+	allItems.push(...parseJournalForIndexing());
+	console.log("[Indexer] Parsing people...");
+	allItems.push(...parseEntitiesForIndexing("people", "person"));
+	console.log("[Indexer] Parsing projects...");
+	allItems.push(...parseEntitiesForIndexing("projects", "project"));
+	console.log(`[Indexer] Indexing ${allItems.length} items...`);
+	await indexItems(allItems);
+	const duration = Date.now() - startTime;
+	console.log(`[Indexer] Index rebuild complete in ${duration}ms`);
+	return { itemCount: allItems.length };
+}
+/**
+* Index a single journal entry (for incremental updates)
+*/
+async function indexJournalEntry(entry) {
+	const item = {
+		id: `journal-${entry.timestamp}`,
+		type: "journal",
+		content: `[${entry.topic}] ${entry.content}`,
+		source: "journal",
+		timestamp: entry.timestamp
+	};
+	await indexItem(item);
+}
+/**
+* Get index stats
+*/
+async function getIndexStats() {
+	const idx = await getIndex();
+	const items = await idx.listItems();
+	return { itemCount: items.length };
+}
+/**
+* Index a single entity file (person or project)
+* Called by daemon when files change
+*/
+async function indexEntityFile(filePath) {
+	const filename = basename(filePath, ".md");
+	let type;
+	if (filePath.includes("/people/")) {
+		type = "person";
+	} else if (filePath.includes("/projects/")) {
+		type = "project";
+	} else {
+		console.error(`[Indexer] Unknown entity type for: ${filePath}`);
+		return;
+	}
+	try {
+		const content = readFileSync(filePath, "utf-8");
+		const items = [];
+		const subdir = type === "person" ? "people" : "projects";
+		const sections = content.split(/^## /m);
+		if (sections[0].trim()) {
+			items.push({
+				id: `${type}-${filename}-preamble`,
+				type,
+				content: sections[0].trim(),
+				source: `${subdir}/${basename(filePath)}`,
+				section: "preamble"
+			});
+		}
+		for (let i = 1; i < sections.length; i++) {
+			const section = sections[i];
+			const firstLine = section.split("\n")[0];
+			const sectionTitle = firstLine.trim();
+			const sectionContent = section.slice(firstLine.length).trim();
+			if (sectionContent) {
+				items.push({
+					id: `${type}-${filename}-${i}`,
+					type,
+					content: `## ${sectionTitle}\n\n${sectionContent}`,
+					source: `${subdir}/${basename(filePath)}`,
+					section: sectionTitle
+				});
+			}
+		}
+		await indexItems(items);
+		console.log(`[Indexer] Indexed ${items.length} sections from ${basename(filePath)}`);
+	} catch (err) {
+		console.error(`[Indexer] Failed to index ${filePath}: ${String(err)}`);
+	}
+}
+/**
+* Preload the embedding model (call during startup)
+*/
+async function preloadModel() {
+	await preloadModel$1();
+}
+
+//#endregion
+export { getIndexStats, indexEntityFile, indexItem, indexItems, indexJournalEntry, preloadModel, rebuildIndex, searchMemory };

package/opencode/context.ts

@@ -0,0 +1,322 @@
+/**
+ * Context formatting for OpenCode plugin
+ *
+ * Reads state files and formats them for injection into conversations
+ */
+
+import { existsSync, readFileSync, readdirSync, mkdirSync, unlinkSync } from "fs";
+
+import { join } from "path";
+import { getStateRoot, getJournalDir, getRemindersDir } from "../src/config.js";
+import { detectUser } from "../src/detect-user.js";
+
+/**
+ * Read and clear pending context from daemon
+ */
+export function consumePendingContext(): string | null {
+  const pendingPath = join(getStateRoot(), ".pending-context");
+  if (!existsSync(pendingPath)) return null;
+
+  try {
+    const content = readFileSync(pendingPath, "utf-8").trim();
+    unlinkSync(pendingPath);
+    return content || null;
+  } catch {
+    return null;
+  }
+}
+
+// Re-export for compatibility
+export { getStateRoot } from "../src/config.js";
+
+/**
+ * Initialize state directory structure (directories only, no default files)
+ * Files are created during onboarding.
+ */
+export function initializeStateRoot(): void {
+  const stateRoot = getStateRoot();
+
+  // Create directories only - files created during onboarding
+  const dirs = [
+    stateRoot,
+    join(stateRoot, "state"),
+    join(stateRoot, "journal"),
+    join(stateRoot, "entities"),
+    join(stateRoot, "entities", "people"),
+    join(stateRoot, "entities", "projects"),
+    join(stateRoot, "topics"),
+  ];
+
+  for (const dir of dirs) {
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+function readFileOrEmpty(path: string): string {
+  try {
+    if (existsSync(path)) {
+      return readFileSync(path, "utf-8");
+    }
+  } catch {
+    // Ignore
+  }
+  return "";
+}
+
+interface JournalEntry {
+  timestamp: string;
+  topic: string;
+  content: string;
+  metadata?: Record<string, unknown>;
+}
+
+function getRecentJournal(count: number): JournalEntry[] {
+  const entries: JournalEntry[] = [];
+  const journalDir = getJournalDir();
+
+  if (!existsSync(journalDir)) return entries;
+
+  try {
+    const files = readdirSync(journalDir)
+      .filter((f) => f.endsWith(".jsonl"))
+      .sort()
+      .reverse();
+
+    for (const file of files) {
+      if (entries.length >= count) break;
+
+      const content = readFileSync(join(journalDir, file), "utf-8");
+      const lines = content.trim().split("\n").filter(Boolean);
+
+      for (const line of lines.reverse()) {
+        if (entries.length >= count) break;
+        try {
+          entries.push(JSON.parse(line));
+        } catch {
+          // Skip malformed lines
+        }
+      }
+    }
+  } catch {
+    // Ignore errors
+  }
+
+  return entries;
+}
+
+interface Schedule {
+  id: string;
+  type: "cron" | "once";
+  expression: string;
+  description: string;
+  payload: string;
+  createdAt: string;
+}
+
+function getSchedules(): Schedule[] {
+  const remindersDir = getRemindersDir();
+  if (!existsSync(remindersDir)) return [];
+
+  const schedules: Schedule[] = [];
+  try {
+    const files = readdirSync(remindersDir).filter((f) => f.endsWith(".json"));
+    for (const file of files) {
+      try {
+        const content = readFileSync(join(remindersDir, file), "utf-8");
+        schedules.push(JSON.parse(content));
+      } catch {
+        // Skip malformed files
+      }
+    }
+  } catch {
+    return [];
+  }
+  return schedules;
+}
+
+interface FormatOptions {
+  forCompaction?: boolean;
+  client?: {
+    config: {
+      providers: () => Promise<{
+        data?: { providers?: Array<{ id: string; models?: Record<string, unknown> }> };
+      }>;
+    };
+  };
+}
+
+/**
+ * Format memory context for injection into conversation
+ */
+export async function formatContextForPrompt(options: FormatOptions = {}): Promise<string | null> {
+  const { forCompaction = false, client } = options;
+  const stateRoot = getStateRoot();
+  const identityPath = join(stateRoot, "state", "identity.md");
+  const isFirstRun = !existsSync(identityPath);
+
+  // First run - return minimal context with onboarding pointer and detected user info
+  if (isFirstRun) {
+    if (forCompaction) return null;
+
+    // Detect user info to avoid multiple permission prompts during onboarding
+    const userInfo = detectUser();
+
+    return `[MACRODATA]
+
+## Status: First Run
+
+Memory is not yet configured. Load the \`macrodata-onboarding\` skill to set up.
+
+## Detected User Info
+
+\`\`\`json
+${JSON.stringify(userInfo, null, 2)}
+\`\`\`
+
+Use this pre-detected info during onboarding instead of running detection scripts.`;
+  }
+
+  const identity = readFileOrEmpty(identityPath);
+  const today = readFileOrEmpty(join(stateRoot, "state", "today.md"));
+  const human = readFileOrEmpty(join(stateRoot, "state", "human.md"));
+  const workspace = readFileOrEmpty(join(stateRoot, "state", "workspace.md"));
+
+  // Get recent journal
+  const journalEntries = getRecentJournal(forCompaction ? 10 : 5);
+  const journalFormatted = journalEntries
+    .map((e) => {
+      const ts = new Date(e.timestamp);
+      const date = isNaN(ts.getTime()) ? "unknown" : ts.toLocaleDateString();
+      return `- [${e.topic}] ${e.content.split("\n")[0]} (${date})`;
+    })
+    .join("\n");
+
+  // Get schedules
+  const schedules = getSchedules();
+  const schedulesFormatted =
+    schedules.length > 0
+      ? schedules.map((s) => `- ${s.description} (${s.type}: ${s.expression})`).join("\n")
+      : "_No active schedules_";
+
+  const sections = [
+    `<macrodata-identity>\n${identity || "_Not configured_"}\n</macrodata-identity>`,
+    `<macrodata-today>\n${today || "_Empty_"}\n</macrodata-today>`,
+    `<macrodata-human>\n${human || "_Empty_"}\n</macrodata-human>`,
+  ];
+
+  if (workspace) {
+    sections.push(`<macrodata-workspace>\n${workspace}\n</macrodata-workspace>`);
+  }
+
+  sections.push(`<macrodata-journal>\n${journalFormatted || "_No entries_"}\n</macrodata-journal>`);
+
+  if (!forCompaction) {
+    sections.push(`<macrodata-schedules>\n${schedulesFormatted}\n</macrodata-schedules>`);
+
+    // List state files
+    const stateDir = join(stateRoot, "state");
+    /* v8 ignore next -- unreachable: this path only runs post-first-run, which
+       means identity.md exists under stateDir, so stateDir always exists. */
+    const stateFiles = existsSync(stateDir)
+      ? readdirSync(stateDir)
+          .filter((f) => f.endsWith(".md"))
+          .map((f) => `state/${f}`)
+      : [];
+
+    // List entity files (scan all subdirs dynamically)
+    const entitiesDir = join(stateRoot, "entities");
+    const entityFiles: string[] = [];
+    if (existsSync(entitiesDir)) {
+      for (const subdir of readdirSync(entitiesDir)) {
+        const dir = join(entitiesDir, subdir);
+        try {
+          /* v8 ignore next -- redundant guard: subdir came from readdirSync so it
+             exists, and a non-directory throws below and is caught, not skipped here. */
+          if (!existsSync(dir) || !readdirSync(dir)) continue;
+          for (const f of readdirSync(dir).filter((f) => f.endsWith(".md"))) {
+            entityFiles.push(`entities/${subdir}/${f}`);
+          }
+        } catch {
+          // Skip non-directories
+        }
+      }
+    }
+
+    const allFiles = [...stateFiles, ...entityFiles];
+    /* v8 ignore next -- unreachable: post-first-run always has state files
+       (identity.md etc.), so allFiles is never empty here. */
+    const filesFormatted =
+      allFiles.length > 0 ? allFiles.map((f) => `- ${f}`).join("\n") : "_No files yet_";
+
+    // Read usage from shared file
+    const usagePath = new URL("../USAGE.md", import.meta.url).pathname;
+    /* v8 ignore next -- USAGE.md is always shipped alongside the built plugin,
+       so the empty-usage fallback is defensive only. */
+    const usage = existsSync(usagePath) ? readFileSync(usagePath, "utf-8").trim() : "";
+
+    /* v8 ignore next 3 -- usage is always populated (USAGE.md ships with the
+       plugin), so the no-usage skip is defensive only. */
+    if (usage) {
+      sections.push(`<macrodata-usage>\n${usage}\n</macrodata-usage>`);
+    }
+
+    sections.push(`<macrodata-files root="${stateRoot}">\n${filesFormatted}\n</macrodata-files>`);
+
+    // Fetch available models for scheduling tools
+    if (client) {
+      try {
+        const { data } = await client.config.providers();
+        if (data?.providers) {
+          // Collect all models with toolcall capability, excluding dated versions
+          type ModelInfo = {
+            id: string;
+            family?: string;
+            release_date?: string;
+            capabilities?: { toolcall?: boolean };
+          };
+          const allModels: { fullId: string; family: string; releaseDate: string }[] = [];
+
+          for (const provider of data.providers) {
+            if (provider.models) {
+              for (const [modelId, model] of Object.entries(provider.models)) {
+                const m = model as ModelInfo;
+                // Skip dated versions and models without toolcall
+                if (/-\d{8}$/.test(modelId) || !m.capabilities?.toolcall) continue;
+
+                allModels.push({
+                  fullId: `${provider.id}/${modelId}`,
+                  family: m.family || `${provider.id}/${modelId}`,
+                  releaseDate: m.release_date || "1970-01-01",
+                });
+              }
+            }
+          }
+
+          // Group by family and pick latest per family
+          const byFamily = new Map<string, (typeof allModels)[0]>();
+          for (const model of allModels) {
+            const existing = byFamily.get(model.family);
+            if (!existing || model.releaseDate > existing.releaseDate) {
+              byFamily.set(model.family, model);
+            }
+          }
+
+          const models = Array.from(byFamily.values())
+            .map((m) => m.fullId)
+            .sort();
+          if (models.length > 0) {
+            sections.push(
+              `<macrodata-models>\nAvailable models for scheduling: ${models.join(", ")}\n</macrodata-models>`,
+            );
+          }
+        }
+      } catch {
+        // Ignore - models just won't be in context
+      }
+    }
+  }
+
+  return `<macrodata>\n${sections.join("\n\n")}\n</macrodata>`;
+}