@geoql/mdr 0.0.1

package/dist/src/conversations.js

@@ -0,0 +1,513 @@
+import { getIndexDir } from "./config.js";
+import { embedBatch, embedQuery } from "./embeddings.js";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
+import { basename, join } from "path";
+import { homedir } from "os";
+import { LocalIndex } from "vectra";
+
+//#region src/conversations.ts
+/**
+* Claude Conversation Log Parser and Indexer
+*
+* Indexes conversation "exchanges" from Claude Code's log files for semantic search.
+* Each exchange = user prompt + assistant's first text response.
+*
+* Features:
+* - Project-biased search (current project first, then global)
+* - Time-weighted scoring (recent > old)
+* - Metadata: project, branch, timestamp, session
+*/
+function getIndexStatePath() {
+	return join(getIndexDir(), "conversations-state.json");
+}
+function loadIndexState() {
+	const statePath = getIndexStatePath();
+	if (existsSync(statePath)) {
+		try {
+			return JSON.parse(readFileSync(statePath, "utf-8"));
+		} catch {
+			console.warn("[Conversations] Index state corrupted, starting fresh");
+		}
+	}
+	return {
+		files: {},
+		lastUpdate: ""
+	};
+}
+function saveIndexState(state) {
+	const statePath = getIndexStatePath();
+	const dir = getIndexDir();
+	if (!existsSync(dir)) {
+		mkdirSync(dir, { recursive: true });
+	}
+	writeFileSync(statePath, JSON.stringify(state, null, 2));
+}
+const CLAUDE_DIR = join(homedir(), ".claude");
+const PROJECTS_DIR = join(CLAUDE_DIR, "projects");
+let convIndex = null;
+let convIndexPath = null;
+async function getConversationIndex() {
+	const currentIndexDir = getIndexDir();
+	const currentIndexPath = join(currentIndexDir, "conversations");
+	if (convIndex && convIndexPath !== currentIndexPath) {
+		convIndex = null;
+		convIndexPath = null;
+	}
+	if (convIndex) return convIndex;
+	if (!existsSync(currentIndexDir)) {
+		const { mkdirSync } = await import("fs");
+		mkdirSync(currentIndexDir, { recursive: true });
+	}
+	convIndex = new LocalIndex(currentIndexPath);
+	convIndexPath = currentIndexPath;
+	if (!await convIndex.isIndexCreated()) {
+		console.log("[Conversations] Creating new conversation index...");
+		await convIndex.createIndex();
+	}
+	return convIndex;
+}
+/**
+* Decode project directory name back to path
+* e.g., "-Users-alex-Repos-my-project" -> "/Users/alex/Repos/my-project"
+*/
+function decodeProjectPath(encoded) {
+	return encoded.replace(/^-/, "/").replace(/-/g, "/");
+}
+/**
+* Extract project name from path
+*/
+function getProjectName(projectPath) {
+	return basename(projectPath);
+}
+/**
+* Extract first text content from assistant message
+*/
+function extractAssistantText(content) {
+	if (typeof content === "string") {
+		return content.slice(0, 500);
+	}
+	for (const block of content) {
+		if (block.type === "text" && block.text) {
+			return block.text.slice(0, 500);
+		}
+	}
+	return "";
+}
+/**
+* Check if user message content is a tool result (not an actual user prompt)
+*/
+function isToolResult(content) {
+	if (Array.isArray(content)) {
+		return content.some((item) => item.type === "tool_result" || item.tool_use_id !== undefined);
+	}
+	return false;
+}
+/**
+* Check if content is noise we should skip indexing
+*/
+function isNoiseContent(content) {
+	if (content.startsWith("This session is being continued from a previous conversation")) {
+		return true;
+	}
+	if (content.includes("<local-command-stdout>") || content.includes("<local-command-caveat>") || content.includes("<command-name>")) {
+		return true;
+	}
+	if (content.startsWith("<current_time>") || content.startsWith("<context_status>") || content.startsWith("<state_files>") || content.startsWith("<system-reminder>") || content.startsWith("## Current State Files") || content.startsWith("Base directory for this skill:")) {
+		return true;
+	}
+	if (content.trim().length < 1) {
+		return true;
+	}
+	return false;
+}
+/**
+* Extract actual user text from message content, filtering out tool results
+*/
+function extractUserText(content) {
+	let text = "";
+	if (typeof content === "string") {
+		text = content;
+	} else {
+		for (const block of content) {
+			const b = block;
+			/* v8 ignore next 3 -- defensive: callers pre-filter tool-result arrays via
+			isToolResult(), so this redundant guard is never the deciding skip. */
+			if (b.type === "tool_result" || b.tool_use_id !== undefined) {
+				continue;
+			}
+			if (b.type === "text" && typeof b.text === "string") {
+				text = b.text;
+				break;
+			}
+		}
+	}
+	if (!text) return "";
+	if (text.startsWith("# Agent Context") || text.includes("\nUser message: ")) {
+		const userMsgMatch = text.match(/\nUser message: (.+)$/s);
+		if (userMsgMatch) {
+			return userMsgMatch[1].trim();
+		}
+		return "";
+	}
+	return text;
+}
+/**
+* Parse a conversation file and extract exchanges
+*/
+function parseConversationFile(filePath, projectPath) {
+	const exchanges = [];
+	const projectName = getProjectName(projectPath);
+	try {
+		const content = readFileSync(filePath, "utf-8");
+		const lines = content.trim().split("\n").filter(Boolean);
+		let currentUser = null;
+		let malformedLines = 0;
+		for (const line of lines) {
+			try {
+				const msg = JSON.parse(line);
+				if (msg.type === "user" && msg.message?.content) {
+					if (isToolResult(msg.message.content)) {
+						continue;
+					}
+					const userText = extractUserText(msg.message.content);
+					if (!userText || isNoiseContent(userText)) {
+						continue;
+					}
+					currentUser = {
+						msg,
+						text: userText
+					};
+				} else if (msg.type === "assistant" && currentUser && msg.message?.content) {
+					const assistantText = extractAssistantText(msg.message.content);
+					exchanges.push({
+						id: `conv-${currentUser.msg.sessionId}-${currentUser.msg.uuid}`,
+						userPrompt: currentUser.text.slice(0, 1e3),
+						assistantSummary: assistantText,
+						project: projectName,
+						projectPath,
+						branch: currentUser.msg.gitBranch,
+						timestamp: currentUser.msg.timestamp || new Date().toISOString(),
+						sessionId: currentUser.msg.sessionId || basename(filePath, ".jsonl"),
+						sessionPath: filePath,
+						messageUuid: currentUser.msg.uuid || ""
+					});
+					currentUser = null;
+				}
+			} catch {
+				malformedLines++;
+			}
+		}
+		if (malformedLines > 0) {
+			console.warn(`[Conversations] Skipped ${malformedLines} malformed lines in ${filePath}`);
+		}
+	} catch (err) {
+		console.error(`[Conversations] Failed to parse ${filePath}: ${String(err)}`);
+	}
+	return exchanges;
+}
+/**
+* Scan all Claude project directories for conversation files
+*/
+function* scanConversationFiles() {
+	if (!existsSync(PROJECTS_DIR)) {
+		return;
+	}
+	const projectDirs = readdirSync(PROJECTS_DIR);
+	for (const projectDir of projectDirs) {
+		if (projectDir.startsWith(".")) continue;
+		const projectPath = decodeProjectPath(projectDir);
+		const projectFullPath = join(PROJECTS_DIR, projectDir);
+		if (!statSync(projectFullPath).isDirectory()) continue;
+		const files = readdirSync(projectFullPath);
+		for (const file of files) {
+			if (!file.endsWith(".jsonl") || file.startsWith("agent-")) continue;
+			const filePath = join(projectFullPath, file);
+			const mtime = statSync(filePath).mtimeMs;
+			yield {
+				filePath,
+				projectPath,
+				mtime
+			};
+		}
+	}
+}
+/**
+* Rebuild the conversation index from scratch
+*/
+async function rebuildConversationIndex() {
+	console.log("[Conversations] Starting full index rebuild...");
+	const startTime = Date.now();
+	const allExchanges = [];
+	const newState = {
+		files: {},
+		lastUpdate: new Date().toISOString()
+	};
+	for (const { filePath, projectPath, mtime } of scanConversationFiles()) {
+		const exchanges = parseConversationFile(filePath, projectPath);
+		allExchanges.push(...exchanges);
+		newState.files[filePath] = {
+			mtime,
+			exchangeIds: exchanges.map((e) => e.id)
+		};
+	}
+	console.log(`[Conversations] Found ${allExchanges.length} exchanges`);
+	if (allExchanges.length === 0) {
+		saveIndexState(newState);
+		return { exchangeCount: 0 };
+	}
+	const texts = allExchanges.map((e) => `${e.project}${e.branch ? ` (${e.branch})` : ""}: ${e.userPrompt}`);
+	console.log(`[Conversations] Generating embeddings...`);
+	const vectors = await embedBatch(texts);
+	const idx = await getConversationIndex();
+	await idx.beginUpdate();
+	try {
+		for (let i = 0; i < allExchanges.length; i++) {
+			const exchange = allExchanges[i];
+			await idx.upsertItem({
+				id: exchange.id,
+				vector: vectors[i],
+				metadata: {
+					userPrompt: exchange.userPrompt,
+					assistantSummary: exchange.assistantSummary,
+					project: exchange.project,
+					projectPath: exchange.projectPath,
+					branch: exchange.branch || "",
+					timestamp: exchange.timestamp,
+					sessionId: exchange.sessionId,
+					sessionPath: exchange.sessionPath,
+					messageUuid: exchange.messageUuid
+				}
+			});
+		}
+		await idx.endUpdate();
+	} catch (err) {
+		idx.cancelUpdate();
+		throw err;
+	}
+	saveIndexState(newState);
+	const duration = Date.now() - startTime;
+	console.log(`[Conversations] Full rebuild complete in ${duration}ms`);
+	return { exchangeCount: allExchanges.length };
+}
+/**
+* Incrementally update the conversation index (only changed files)
+*/
+async function updateConversationIndex() {
+	console.log("[Conversations] Starting incremental update...");
+	const startTime = Date.now();
+	const state = loadIndexState();
+	const idx = await getConversationIndex();
+	/* v8 ignore next 5 -- defensive: getConversationIndex() above always creates
+	the index, so isIndexCreated() is never false here; this full-rebuild
+	fallback only guards a future change to that contract. */
+	if (!await idx.isIndexCreated()) {
+		console.log("[Conversations] No existing index, doing full rebuild");
+		const result = await rebuildConversationIndex();
+		return {
+			exchangeCount: result.exchangeCount,
+			filesUpdated: 0,
+			skipped: 0
+		};
+	}
+	let filesUpdated = 0;
+	let skipped = 0;
+	let totalExchanges = 0;
+	const currentFiles = new Set();
+	for (const { filePath, projectPath, mtime } of scanConversationFiles()) {
+		currentFiles.add(filePath);
+		const cached = state.files[filePath];
+		if (cached && cached.mtime === mtime) {
+			skipped++;
+			totalExchanges += cached.exchangeIds.length;
+			continue;
+		}
+		const exchanges = parseConversationFile(filePath, projectPath);
+		if (exchanges.length > 0) {
+			const texts = exchanges.map((e) => `${e.project}${e.branch ? ` (${e.branch})` : ""}: ${e.userPrompt}`);
+			const vectors = await embedBatch(texts);
+			await idx.beginUpdate();
+			try {
+				for (let i = 0; i < exchanges.length; i++) {
+					const exchange = exchanges[i];
+					await idx.upsertItem({
+						id: exchange.id,
+						vector: vectors[i],
+						metadata: {
+							userPrompt: exchange.userPrompt,
+							assistantSummary: exchange.assistantSummary,
+							project: exchange.project,
+							projectPath: exchange.projectPath,
+							branch: exchange.branch || "",
+							timestamp: exchange.timestamp,
+							sessionId: exchange.sessionId,
+							sessionPath: exchange.sessionPath,
+							messageUuid: exchange.messageUuid
+						}
+					});
+				}
+				await idx.endUpdate();
+			} catch (err) {
+				idx.cancelUpdate();
+				throw err;
+			}
+		}
+		state.files[filePath] = {
+			mtime,
+			exchangeIds: exchanges.map((e) => e.id)
+		};
+		filesUpdated++;
+		totalExchanges += exchanges.length;
+	}
+	for (const filePath of Object.keys(state.files)) {
+		if (!currentFiles.has(filePath)) {
+			delete state.files[filePath];
+		}
+	}
+	state.lastUpdate = new Date().toISOString();
+	saveIndexState(state);
+	const duration = Date.now() - startTime;
+	console.log(`[Conversations] Incremental update complete in ${duration}ms (${filesUpdated} files updated, ${skipped} skipped)`);
+	return {
+		exchangeCount: totalExchanges,
+		filesUpdated,
+		skipped
+	};
+}
+/**
+* Calculate time-based weight for scoring
+* Recent = higher weight
+*/
+function getTimeWeight(timestamp) {
+	const age = Date.now() - new Date(timestamp).getTime();
+	const dayMs = 24 * 60 * 60 * 1e3;
+	if (age < 7 * dayMs) return 1;
+	if (age < 30 * dayMs) return .9;
+	if (age < 90 * dayMs) return .7;
+	if (age < 365 * dayMs) return .5;
+	return .3;
+}
+/**
+* Search conversations with project bias and time weighting
+*/
+async function searchConversations(query, options = {}) {
+	const { currentProject, limit = 5, projectOnly = false } = options;
+	const idx = await getConversationIndex();
+	const stats = await idx.listItems();
+	if (stats.length === 0) {
+		console.log("[Conversations] Index is empty");
+		return [];
+	}
+	const queryVector = await embedQuery(query);
+	const results = await idx.queryItems(queryVector, query, limit * 3);
+	const searchResults = results.map((r) => {
+		const meta = r.item.metadata;
+		const exchange = {
+			id: r.item.id,
+			userPrompt: meta.userPrompt,
+			assistantSummary: meta.assistantSummary,
+			project: meta.project,
+			projectPath: meta.projectPath,
+			branch: meta.branch || undefined,
+			timestamp: meta.timestamp,
+			sessionId: meta.sessionId,
+			sessionPath: meta.sessionPath,
+			messageUuid: meta.messageUuid
+		};
+		let adjustedScore = r.score;
+		adjustedScore *= getTimeWeight(exchange.timestamp);
+		if (currentProject && exchange.projectPath === currentProject) {
+			adjustedScore *= 1.5;
+		}
+		return {
+			exchange,
+			score: r.score,
+			adjustedScore
+		};
+	});
+	let filtered = searchResults;
+	if (projectOnly && currentProject) {
+		filtered = searchResults.filter((r) => r.exchange.projectPath === currentProject);
+	}
+	return filtered.sort((a, b) => b.adjustedScore - a.adjustedScore).slice(0, limit);
+}
+/**
+* Load full conversation context around a specific message
+*/
+async function expandConversation(sessionPath, messageUuid, contextMessages = 10) {
+	if (!existsSync(sessionPath)) {
+		throw new Error(`Session file not found: ${sessionPath}`);
+	}
+	const content = readFileSync(sessionPath, "utf-8");
+	const lines = content.trim().split("\n").filter(Boolean);
+	const messages = [];
+	let project = "";
+	let branch;
+	let malformedLines = 0;
+	for (const line of lines) {
+		try {
+			const msg = JSON.parse(line);
+			if (msg.type === "user" && msg.message?.content) {
+				if (isToolResult(msg.message.content)) {
+					continue;
+				}
+				const text = extractUserText(msg.message.content);
+				if (!text || isNoiseContent(text)) {
+					continue;
+				}
+				messages.push({
+					role: "user",
+					content: text,
+					timestamp: msg.timestamp,
+					uuid: msg.uuid
+				});
+				if (!project && msg.cwd) {
+					project = getProjectName(msg.cwd);
+				}
+				if (!branch && msg.gitBranch) {
+					branch = msg.gitBranch;
+				}
+			} else if (msg.type === "assistant" && msg.message?.content) {
+				const text = extractAssistantText(msg.message.content);
+				if (text) {
+					messages.push({
+						role: "assistant",
+						content: text,
+						timestamp: msg.timestamp,
+						uuid: msg.uuid
+					});
+				}
+			}
+		} catch {
+			malformedLines++;
+		}
+	}
+	if (malformedLines > 0) {
+		console.warn(`[Conversations] Skipped ${malformedLines} malformed lines in ${sessionPath}`);
+	}
+	const targetIdx = messages.findIndex((m) => m.uuid === messageUuid);
+	if (targetIdx === -1) {
+		return {
+			messages: messages.slice(-contextMessages).map(({ uuid: _uuid, ...rest }) => rest),
+			project,
+			branch
+		};
+	}
+	const start = Math.max(0, targetIdx - Math.floor(contextMessages / 2));
+	const end = Math.min(messages.length, start + contextMessages);
+	return {
+		messages: messages.slice(start, end).map(({ uuid: _uuid, ...rest }) => rest),
+		project,
+		branch
+	};
+}
+/**
+* Get conversation index stats
+*/
+async function getConversationIndexStats() {
+	const idx = await getConversationIndex();
+	const items = await idx.listItems();
+	return { exchangeCount: items.length };
+}
+
+//#endregion
+export { expandConversation, getConversationIndexStats, rebuildConversationIndex, searchConversations, updateConversationIndex };