@aeriondyseti/vector-memory-mcp 1.0.2-dev.0 → 1.1.0-dev.2

package/src/services/conversation-history.service.ts

@@ -0,0 +1,320 @@
+import { readdir, stat } from "fs/promises";
+import { join } from "path";
+import type { ConversationHistoryRepository } from "../db/conversation-history.repository.js";
+import type { EmbeddingsService } from "./embeddings.service.js";
+import {
+  parseSessionFile,
+  discoverSessionFiles,
+  detectSessionPath,
+  type ParsedMessage,
+  type ParseResult,
+  type SessionFileInfo,
+} from "./session-parser.js";
+import type {
+  ConversationHistoryEntry,
+  HistorySearchResult,
+  IndexedSession,
+  IndexedSessionSummary,
+  IndexingSummary,
+} from "../types/conversation-history.js";
+
+const EMBED_BATCH_SIZE = 50;
+
+export class ConversationHistoryService {
+  constructor(
+    private repository: ConversationHistoryRepository,
+    private embeddings: EmbeddingsService,
+    private sessionPath: string | null, // null = auto-detect
+  ) {}
+
+  /**
+   * Index all conversation sessions found in the session directory.
+   *
+   * For each .jsonl file discovered:
+   * - New (not tracked): full parse from byte 0
+   * - Grown (fileSize increased): incremental parse from last-known size
+   * - Shrunk (fileSize decreased — file replaced): delete + full reindex
+   * - Unchanged (same fileSize): skip
+   */
+  async indexConversations(sessionDir?: string): Promise<IndexingSummary> {
+    const allFiles = await this.discoverAllFiles(sessionDir);
+
+    // Bulk-fetch all tracked sessions into a Map to avoid N+1 lookups
+    const trackedSessions = await this.buildSessionIndex();
+
+    const summary: IndexingSummary = {
+      sessionsDiscovered: allFiles.length,
+      sessionsIndexed: 0,
+      sessionsSkipped: 0,
+      messagesIndexed: 0,
+    };
+
+    for (const file of allFiles) {
+      const indexed = trackedSessions.get(file.sessionId) ?? null;
+
+      if (indexed && indexed.fileSize === file.fileSize) {
+        // Unchanged — skip
+        summary.sessionsSkipped++;
+        continue;
+      }
+
+      if (indexed && file.fileSize < indexed.fileSize) {
+        // Shrunk — file was replaced, full reindex
+        await this.repository.deleteBySessionId(file.sessionId);
+        await this.repository.deleteIndexedSession(file.sessionId);
+        const count = await this.indexFile(file, 0, 0, null);
+        summary.sessionsIndexed++;
+        summary.messagesIndexed += count;
+        continue;
+      }
+
+      if (indexed && file.fileSize > indexed.fileSize) {
+        // Grown — incremental parse from where we left off
+        const count = await this.indexFile(
+          file,
+          indexed.fileSize,
+          indexed.messageCount,
+          indexed,
+        );
+        summary.sessionsIndexed++;
+        summary.messagesIndexed += count;
+        continue;
+      }
+
+      // New — full parse
+      const count = await this.indexFile(file, 0, 0, null);
+      summary.sessionsIndexed++;
+      summary.messagesIndexed += count;
+    }
+
+    return summary;
+  }
+
+  /**
+   * Search conversation history using hybrid (vector + FTS) search.
+   */
+  async search(query: string, limit: number): Promise<HistorySearchResult[]> {
+    const embedding = await this.embeddings.embed(query);
+    const rows = await this.repository.findHybrid(embedding, query, limit);
+
+    return rows.map((row) => ({
+      source: "conversation_history" as const,
+      id: row.id,
+      content: row.content,
+      metadata: row.metadata,
+      score: row.rrfScore,
+      sessionId: row.sessionId,
+      role: row.role,
+      messageIndex: row.messageIndex,
+      timestamp: row.timestamp,
+    }));
+  }
+
+  /**
+   * List all indexed sessions (pass-through to repository).
+   */
+  async listIndexedSessions(): Promise<IndexedSessionSummary[]> {
+    return this.repository.listIndexedSessions();
+  }
+
+  /**
+   * Force a full reindex of a specific session.
+   * Deletes all existing entries and tracking, then re-parses from byte 0.
+   */
+  async reindexSession(sessionId: string): Promise<IndexingSummary> {
+    const indexed = await this.repository.getIndexedSession(sessionId);
+
+    const summary: IndexingSummary = {
+      sessionsDiscovered: 1,
+      sessionsIndexed: 0,
+      sessionsSkipped: 0,
+      messagesIndexed: 0,
+    };
+
+    if (!indexed) {
+      // Nothing to reindex — no tracking record means we don't know the file path
+      summary.sessionsSkipped = 1;
+      return summary;
+    }
+
+    // Delete existing data
+    await this.repository.deleteBySessionId(sessionId);
+    await this.repository.deleteIndexedSession(sessionId);
+
+    // Get current file size (file may have changed since last index)
+    let fileSize: number;
+    try {
+      const stats = await stat(indexed.filePath);
+      fileSize = stats.size;
+    } catch {
+      // File no longer exists
+      summary.sessionsSkipped = 1;
+      return summary;
+    }
+
+    const fileInfo: SessionFileInfo = {
+      sessionId,
+      filePath: indexed.filePath,
+      fileSize,
+    };
+
+    const count = await this.indexFile(fileInfo, 0, 0, null);
+    summary.sessionsIndexed = 1;
+    summary.messagesIndexed = count;
+    return summary;
+  }
+
+  // --- Private helpers ---
+
+  /**
+   * Bulk-fetch all tracked sessions into a Map for O(1) lookups.
+   * Uses listIndexedSessions() which returns summaries, but we need full
+   * IndexedSession records. We call getIndexedSession() is avoided by using
+   * a repository method that returns all sessions with full details.
+   *
+   * Note: listIndexedSessions returns IndexedSessionSummary (no filePath/fileSize),
+   * so we use getIndexedSession per unique session. However, we batch this via
+   * the list + individual fetches only when needed. For now, we fetch all as
+   * summaries and promote to full records via individual lookups grouped upfront.
+   */
+  private async buildSessionIndex(): Promise<Map<string, IndexedSession>> {
+    const summaries = await this.repository.listIndexedSessions();
+    const sessionMap = new Map<string, IndexedSession>();
+
+    // Fetch full records in parallel for all known sessions
+    const fullRecords = await Promise.all(
+      summaries.map((s) => this.repository.getIndexedSession(s.sessionId)),
+    );
+
+    for (const record of fullRecords) {
+      if (record) {
+        sessionMap.set(record.sessionId, record);
+      }
+    }
+
+    return sessionMap;
+  }
+
+  /**
+   * Discover all .jsonl files across resolved session directories.
+   * Resolves dirs and discovers files in one pass to avoid double-scanning.
+   */
+  private async discoverAllFiles(sessionDir?: string): Promise<SessionFileInfo[]> {
+    const base = sessionDir ?? this.sessionPath ?? detectSessionPath();
+    if (!base) return [];
+
+    // Check if base dir itself has .jsonl files
+    const rootFiles = await discoverSessionFiles(base);
+    if (rootFiles.length > 0) return rootFiles;
+
+    // Otherwise enumerate subdirectories and discover files in each
+    const dirs = await this.listSubdirectories(base);
+    const nested = await Promise.all(dirs.map((d) => discoverSessionFiles(d)));
+    return nested.flat();
+  }
+
+  /**
+   * List immediate subdirectories of a path. Stat calls are parallelized.
+   */
+  private async listSubdirectories(base: string): Promise<string[]> {
+    let entries: string[];
+    try {
+      entries = await readdir(base);
+    } catch {
+      return [];
+    }
+
+    const results = await Promise.allSettled(
+      entries.map(async (entry) => {
+        const fullPath = join(base, entry);
+        const stats = await stat(fullPath);
+        return stats.isDirectory() ? fullPath : null;
+      }),
+    );
+
+    return results
+      .filter((r): r is PromiseFulfilledResult<string | null> => r.status === "fulfilled")
+      .map((r) => r.value)
+      .filter((v): v is string => v != null);
+  }
+
+  /**
+   * Parse a session file, embed messages in batches, insert into repository,
+   * and upsert the tracking record. Returns count of messages indexed.
+   */
+  private async indexFile(
+    file: SessionFileInfo,
+    fromByte: number,
+    startIndex: number,
+    existing: IndexedSession | null,
+  ): Promise<number> {
+    const parseResult = await parseSessionFile(
+      file.filePath,
+      fromByte,
+      startIndex,
+      file.fileSize,
+    );
+
+    if (parseResult.messages.length === 0) {
+      // Still upsert tracking so we don't re-parse an empty/no-new-content file
+      await this.upsertTracking(file, parseResult.messages, startIndex, parseResult, existing);
+      return 0;
+    }
+
+    // Embed and insert in batches
+    for (let i = 0; i < parseResult.messages.length; i += EMBED_BATCH_SIZE) {
+      const batch = parseResult.messages.slice(i, i + EMBED_BATCH_SIZE);
+      const texts = batch.map((m) => m.content);
+      const embeddings = await this.embeddings.embedBatch(texts);
+
+      const entries: ConversationHistoryEntry[] = batch.map((msg, idx) => ({
+        id: msg.id,
+        content: msg.content,
+        embedding: embeddings[idx],
+        sessionId: msg.sessionId,
+        role: msg.role,
+        messageIndex: msg.messageIndex,
+        timestamp: msg.timestamp,
+        metadata: msg.metadata,
+        createdAt: new Date(),
+      }));
+
+      await this.repository.insert(entries);
+    }
+
+    await this.upsertTracking(file, parseResult.messages, startIndex, parseResult, existing);
+    return parseResult.messages.length;
+  }
+
+  /**
+   * Upsert the indexed session tracking record.
+   * For incremental indexing, merges timestamps with the existing record.
+   */
+  private async upsertTracking(
+    file: SessionFileInfo,
+    newMessages: ParsedMessage[],
+    startIndex: number,
+    parseResult: ParseResult,
+    existing: IndexedSession | null,
+  ): Promise<void> {
+    const totalMessageCount = startIndex + newMessages.length;
+    const firstMessageAt =
+      existing?.firstMessageAt ?? parseResult.firstMessageAt ?? new Date();
+    const lastMessageAt =
+      parseResult.lastMessageAt ?? existing?.lastMessageAt ?? new Date();
+
+    const session: IndexedSession = {
+      sessionId: file.sessionId,
+      filePath: file.filePath,
+      fileSize: file.fileSize,
+      messageCount: totalMessageCount,
+      firstMessageAt,
+      lastMessageAt,
+      indexedAt: new Date(),
+      ...(parseResult.project ? { project: parseResult.project } : existing?.project ? { project: existing.project } : {}),
+      ...(parseResult.gitBranch ? { gitBranch: parseResult.gitBranch } : existing?.gitBranch ? { gitBranch: existing.gitBranch } : {}),
+    };
+
+    await this.repository.upsertIndexedSession(session);
+  }
+}

package/src/services/memory.service.ts

@@ -3,6 +3,8 @@ import type { Memory, SearchIntent, IntentProfile } from "../types/memory.js";
 import { isDeleted } from "../types/memory.js";
 import type { MemoryRepository } from "../db/memory.repository.js";
 import type { EmbeddingsService } from "./embeddings.service.js";
+import type { ConversationHistoryService } from "./conversation-history.service.js";
+import type { SearchResult, MemorySearchResult } from "../types/conversation-history.js";
 
 const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
   continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.02 },
@@ -15,11 +17,31 @@ const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
 const sigmoid = (x: number): number => 1 / (1 + Math.exp(-x));
 
 export class MemoryService {
+  private historyService: ConversationHistoryService | null = null;
+  private historyWeight: number = 0.5;
+
   constructor(
     private repository: MemoryRepository,
     private embeddings: EmbeddingsService
   ) { }
 
+  /**
+   * Optionally wire conversation history for unified search.
+   * Called from index.ts when conversationHistory.enabled is true.
+   */
+  setConversationHistory(service: ConversationHistoryService, weight: number): void {
+    this.historyService = service;
+    this.historyWeight = weight;
+  }
+
+  /**
+   * Access the conversation history service (if wired).
+   * Used by MCP handlers for index/list/reindex tools.
+   */
+  getConversationHistory(): ConversationHistoryService | null {
+    return this.historyService;
+  }
+
   async store(
     content: string,
     metadata: Record<string, unknown> = {},
@@ -169,6 +191,58 @@ export class MemoryService {
     return scored.slice(0, limit).map((s) => s.memory);
   }
 
+  /**
+   * Search across both memories and conversation history (if enabled).
+   * Returns a merged, score-normalized list sorted by relevance.
+   *
+   * If no history service is configured, falls back to memory-only search
+   * wrapped as MemorySearchResult[].
+   */
+  async searchUnified(
+    query: string,
+    intent: SearchIntent,
+    limit: number = 10,
+    includeDeleted: boolean = false,
+  ): Promise<SearchResult[]> {
+    if (!this.historyService) {
+      const memories = await this.search(query, intent, limit, includeDeleted);
+      return this.toMemoryResults(memories, limit);
+    }
+
+    // Run both searches in parallel
+    const [memories, historyResults] = await Promise.all([
+      this.search(query, intent, limit, includeDeleted),
+      this.historyService.search(query, limit),
+    ]);
+
+    const memoryResults = this.toMemoryResults(memories, limit);
+
+    // Apply history weight to RRF scores
+    const weightedHistory: SearchResult[] = historyResults.map((h) => ({
+      ...h,
+      score: h.score * this.historyWeight,
+    }));
+
+    // Merge, sort by score descending, take top N
+    const merged = [...memoryResults, ...weightedHistory];
+    merged.sort((a, b) => b.score - a.score);
+    return merged.slice(0, limit);
+  }
+
+  /** Convert pre-sorted Memory[] to MemorySearchResult[] with positional scores. */
+  private toMemoryResults(memories: Memory[], limit: number): MemorySearchResult[] {
+    return memories.map((m, rank) => ({
+      source: "memory" as const,
+      id: m.id,
+      content: m.content,
+      metadata: m.metadata,
+      score: (limit - rank) / limit,
+      createdAt: m.createdAt,
+      updatedAt: m.updatedAt,
+      supersededBy: m.supersededBy,
+    }));
+  }
+
   async trackAccess(ids: string[]): Promise<void> {
     const now = new Date();
     for (const id of ids) {

package/src/services/session-parser.ts

@@ -0,0 +1,232 @@
+import { createReadStream } from "fs";
+import { readdir, stat } from "fs/promises";
+import { createInterface } from "readline";
+import { basename, join } from "path";
+import { randomUUID } from "crypto";
+import type { MessageRole } from "../types/conversation-history.js";
+
+/**
+ * A parsed message extracted from a Claude Code JSONL session file.
+ * Does not include embedding — that's added by the service layer.
+ */
+export interface ParsedMessage {
+  id: string;
+  sessionId: string;
+  role: MessageRole;
+  messageIndex: number;
+  content: string;
+  timestamp: Date;
+  metadata: Record<string, unknown>;
+}
+
+/**
+ * Info about a discovered session file (before parsing).
+ */
+export interface SessionFileInfo {
+  sessionId: string;
+  filePath: string;
+  fileSize: number;
+}
+
+/**
+ * Result of parsing a single session file.
+ * Structurally mirrors IndexedSession but with nullable timestamps/metadata
+ * (a file with 0 messages has no timestamps). The service layer maps this to
+ * IndexedSession at write time, supplying messageCount and indexedAt.
+ */
+export interface ParseResult {
+  sessionId: string;
+  filePath: string;
+  fileSize: number;
+  messages: ParsedMessage[];
+  firstMessageAt: Date | null;
+  lastMessageAt: Date | null;
+  gitBranch: string | null;
+  project: string | null;
+}
+
+// -- JSONL line shapes (only the fields we need) --
+
+interface JournalLine {
+  type: string;
+  sessionId?: string;
+  timestamp?: string;
+  gitBranch?: string;
+  cwd?: string;
+  message?: {
+    role?: string;
+    content?: string | ContentBlock[];
+  };
+}
+
+interface ContentBlock {
+  type: string;
+  text?: string;
+}
+
+/**
+ * Extract the text content from a JSONL message line.
+ * - User messages: content is string OR array (skip tool_result blocks)
+ * - Assistant messages: content is array of blocks (keep only type=text)
+ * Returns null if no usable text content.
+ */
+function extractTextContent(line: JournalLine): string | null {
+  const content = line.message?.content;
+  if (content == null) return null;
+
+  // User messages can be a plain string
+  if (typeof content === "string") {
+    return content.trim() || null;
+  }
+
+  // Array of content blocks — extract text blocks only
+  const textParts: string[] = [];
+  for (const block of content) {
+    if (block.type === "text" && block.text) {
+      textParts.push(block.text);
+    }
+  }
+
+  const joined = textParts.join("\n").trim();
+  return joined || null;
+}
+
+/**
+ * Parse a single Claude Code JSONL session file.
+ *
+ * @param filePath - Absolute path to the .jsonl file
+ * @param fromByte - Byte offset to start reading from (for incremental parsing).
+ *                   When non-zero, messageIndex starts from startIndex.
+ * @param startIndex - Starting message index (for incremental parsing).
+ * @param knownFileSize - If already known (e.g. from discoverSessionFiles), avoids a redundant stat().
+ */
+export async function parseSessionFile(
+  filePath: string,
+  fromByte: number = 0,
+  startIndex: number = 0,
+  knownFileSize?: number,
+): Promise<ParseResult> {
+  const fileSize = knownFileSize ?? (await stat(filePath)).size;
+
+  // Extract session ID from filename (e.g. "abc-123.jsonl" → "abc-123")
+  const sessionId = basename(filePath, ".jsonl");
+
+  const messages: ParsedMessage[] = [];
+  let messageIndex = startIndex;
+  let firstMessageAt: Date | null = null;
+  let lastMessageAt: Date | null = null;
+  let gitBranch: string | null = null;
+  let project: string | null = null;
+
+  const stream = createReadStream(filePath, {
+    encoding: "utf-8",
+    start: fromByte,
+  });
+
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+
+  for await (const rawLine of rl) {
+    if (!rawLine.trim()) continue;
+
+    let line: JournalLine;
+    try {
+      line = JSON.parse(rawLine);
+    } catch {
+      // Skip malformed lines
+      continue;
+    }
+
+    // Only process user and assistant messages
+    if (line.type !== "user" && line.type !== "assistant") continue;
+
+    // Capture metadata from first line that has it
+    if (gitBranch == null && line.gitBranch) gitBranch = line.gitBranch;
+    if (project == null && line.cwd) project = line.cwd;
+
+    const role: MessageRole = line.type === "user" ? "user" : "assistant";
+    const text = extractTextContent(line);
+    if (!text) continue;
+
+    const timestamp = line.timestamp ? new Date(line.timestamp) : new Date();
+
+    if (firstMessageAt == null) firstMessageAt = timestamp;
+    lastMessageAt = timestamp;
+
+    messages.push({
+      id: randomUUID(),
+      sessionId,
+      role,
+      messageIndex,
+      content: text,
+      timestamp,
+      metadata: {
+        ...(line.gitBranch ? { gitBranch: line.gitBranch } : {}),
+        ...(line.cwd ? { cwd: line.cwd } : {}),
+      },
+    });
+
+    messageIndex++;
+  }
+
+  return {
+    sessionId,
+    filePath,
+    fileSize,
+    messages,
+    firstMessageAt,
+    lastMessageAt,
+    gitBranch,
+    project,
+  };
+}
+
+/**
+ * Discover all .jsonl session files in a directory.
+ * Stat calls are parallelized for efficiency.
+ */
+export async function discoverSessionFiles(
+  sessionDir: string,
+): Promise<SessionFileInfo[]> {
+  let entries: string[];
+  try {
+    entries = await readdir(sessionDir);
+  } catch {
+    return [];
+  }
+
+  const jsonlEntries = entries.filter((e) => e.endsWith(".jsonl"));
+
+  const settled = await Promise.allSettled(
+    jsonlEntries.map(async (entry) => {
+      const filePath = join(sessionDir, entry);
+      const fileStats = await stat(filePath);
+      if (!fileStats.isFile()) return null;
+      return {
+        sessionId: entry.replace(/\.jsonl$/, ""),
+        filePath,
+        fileSize: fileStats.size,
+      } satisfies SessionFileInfo;
+    }),
+  );
+
+  return settled
+    .filter(
+      (r): r is PromiseFulfilledResult<SessionFileInfo | null> =>
+        r.status === "fulfilled",
+    )
+    .map((r) => r.value)
+    .filter((v): v is SessionFileInfo => v != null);
+}
+
+/**
+ * Auto-detect the Claude Code sessions directory.
+ * Returns null if not found.
+ */
+export function detectSessionPath(): string | null {
+  const home = process.env.HOME ?? process.env.USERPROFILE;
+  if (!home) return null;
+
+  // Claude Code stores sessions at ~/.claude/projects/<project-slug>/<session-id>.jsonl
+  // We return the projects dir — the caller iterates project subdirs
+  return join(home, ".claude", "projects");
+}