@aeriondyseti/vector-memory-mcp 1.1.0-dev.2 → 1.1.0-dev.3

package/src/services/conversation.service.ts

@@ -0,0 +1,354 @@
+import { createHash } from "crypto";
+import { readFile, writeFile, mkdir } from "fs/promises";
+import { dirname, join } from "path";
+import type { ConversationRepository } from "../db/conversation.repository.js";
+import type {
+  ConversationChunk,
+  ConversationHybridRow,
+  HistoryFilters,
+  IndexedSession,
+  ParsedMessage,
+  SessionFileInfo,
+} from "../types/conversation.js";
+import type { ConversationHistoryConfig } from "../config/index.js";
+import { resolveSessionLogPath } from "../config/index.js";
+import type { EmbeddingsService } from "./embeddings.service.js";
+import type { SessionLogParser } from "./parsers/types.js";
+import { ClaudeCodeSessionParser } from "./parsers/claude-code.parser.js";
+
+/**
+ * Generate a deterministic chunk ID from session ID and message indices.
+ */
+function chunkId(
+  sessionId: string,
+  startIdx: number,
+  endIdx: number
+): string {
+  return createHash("sha256")
+    .update(`${sessionId}:${startIdx}:${endIdx}`)
+    .digest("hex")
+    .slice(0, 32);
+}
+
+/**
+ * Group parsed messages into embeddable chunks.
+ */
+export function chunkMessages(
+  messages: ParsedMessage[],
+  maxChunkMessages: number,
+  overlap: number
+): ConversationChunk[] {
+  if (messages.length === 0) return [];
+
+  const chunks: ConversationChunk[] = [];
+  let startIdx = 0;
+
+  while (startIdx < messages.length) {
+    const endIdx = Math.min(startIdx + maxChunkMessages, messages.length);
+    const chunkMsgs = messages.slice(startIdx, endIdx);
+
+    const firstRole = chunkMsgs[0].role;
+    const role = chunkMsgs.every((m) => m.role === firstRole)
+      ? firstRole
+      : "mixed";
+
+    const content = chunkMsgs
+      .map(
+        (m) =>
+          `[${m.role} @ ${m.timestamp.toISOString()}]: ${m.content}`
+      )
+      .join("\n\n");
+
+    const firstMsg = chunkMsgs[0];
+    const lastMsg = chunkMsgs[chunkMsgs.length - 1];
+
+    chunks.push({
+      id: chunkId(
+        firstMsg.sessionId,
+        firstMsg.messageIndex,
+        lastMsg.messageIndex
+      ),
+      content,
+      sessionId: firstMsg.sessionId,
+      timestamp: firstMsg.timestamp,
+      endTimestamp: lastMsg.timestamp,
+      role,
+      messageIndexStart: firstMsg.messageIndex,
+      messageIndexEnd: lastMsg.messageIndex,
+      project: firstMsg.project,
+      metadata: {
+        session_id: firstMsg.sessionId,
+        timestamp: firstMsg.timestamp.toISOString(),
+        role,
+        message_index_start: firstMsg.messageIndex,
+        message_index_end: lastMsg.messageIndex,
+        project: firstMsg.project,
+        git_branch: firstMsg.gitBranch,
+        is_subagent: firstMsg.isSubagent,
+        agent_id: firstMsg.agentId,
+      },
+    });
+
+    // Advance by (chunkSize - overlap), but always advance at least 1
+    const advance = Math.max(1, endIdx - startIdx - overlap);
+    startIdx += advance;
+  }
+
+  return chunks;
+}
+
+/** Serializable index state format */
+interface IndexStateEntry {
+  sessionId: string;
+  filePath: string;
+  project: string;
+  lastModified: number;
+  chunkCount: number;
+  messageCount: number;
+  indexedAt: string;
+  firstMessageAt: string;
+  lastMessageAt: string;
+}
+
+export class ConversationHistoryService {
+  private indexStatePath: string;
+  private indexStateCache: Map<string, IndexedSession> | null = null;
+
+  constructor(
+    private repository: ConversationRepository,
+    private embeddings: EmbeddingsService,
+    public readonly config: ConversationHistoryConfig,
+    private dbPath: string,
+    private parser: SessionLogParser = new ClaudeCodeSessionParser()
+  ) {
+    this.indexStatePath = join(
+      dirname(dbPath),
+      "conversation_index_state.json"
+    );
+  }
+
+  private async loadIndexState(): Promise<Map<string, IndexedSession>> {
+    if (this.indexStateCache) return this.indexStateCache;
+    try {
+      const raw = await readFile(this.indexStatePath, "utf-8");
+      const entries: IndexStateEntry[] = JSON.parse(raw);
+      const map = new Map<string, IndexedSession>();
+      for (const e of entries) {
+        map.set(e.sessionId, {
+          sessionId: e.sessionId,
+          filePath: e.filePath,
+          project: e.project,
+          lastModified: e.lastModified,
+          chunkCount: e.chunkCount,
+          messageCount: e.messageCount,
+          indexedAt: new Date(e.indexedAt),
+          firstMessageAt: new Date(e.firstMessageAt),
+          lastMessageAt: new Date(e.lastMessageAt),
+        });
+      }
+      this.indexStateCache = map;
+      return map;
+    } catch {
+      const map = new Map<string, IndexedSession>();
+      this.indexStateCache = map;
+      return map;
+    }
+  }
+
+  private async saveIndexState(state: Map<string, IndexedSession>): Promise<void> {
+    const entries: IndexStateEntry[] = [...state.values()].map((s) => ({
+      sessionId: s.sessionId,
+      filePath: s.filePath,
+      project: s.project,
+      lastModified: s.lastModified,
+      chunkCount: s.chunkCount,
+      messageCount: s.messageCount,
+      indexedAt: s.indexedAt.toISOString(),
+      firstMessageAt: s.firstMessageAt.toISOString(),
+      lastMessageAt: s.lastMessageAt.toISOString(),
+    }));
+    await mkdir(dirname(this.indexStatePath), { recursive: true });
+    await writeFile(this.indexStatePath, JSON.stringify(entries, null, 2));
+    this.indexStateCache = state;
+  }
+
+  async indexConversations(
+    path?: string,
+    since?: Date
+  ): Promise<{ indexed: number; skipped: number; errors: string[] }> {
+    if (!this.config.enabled) {
+      return {
+        indexed: 0,
+        skipped: 0,
+        errors: ["Conversation history indexing is not enabled"],
+      };
+    }
+
+    const logPath = path ?? resolveSessionLogPath(this.config);
+    if (!logPath) {
+      return {
+        indexed: 0,
+        skipped: 0,
+        errors: ["No session log path configured or detected"],
+      };
+    }
+
+    const sessionFiles = await this.parser.findSessionFiles(
+      logPath,
+      since,
+      this.config.indexSubagents
+    );
+    const indexState = await this.loadIndexState();
+
+    let indexed = 0;
+    let skipped = 0;
+    const errors: string[] = [];
+
+    for (const file of sessionFiles) {
+      const existing = indexState.get(file.sessionId);
+      if (existing && existing.lastModified >= file.lastModified.getTime()) {
+        skipped++;
+        continue;
+      }
+
+      try {
+        await this.indexSession(file, indexState);
+        indexed++;
+      } catch (err) {
+        errors.push(
+          `${file.sessionId}: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+
+    await this.saveIndexState(indexState);
+    return { indexed, skipped, errors };
+  }
+
+  private async indexSession(
+    file: SessionFileInfo,
+    indexState: Map<string, IndexedSession>
+  ): Promise<void> {
+    const messages = await this.parser.parse(
+      file.filePath,
+      this.config.indexSubagents
+    );
+    if (messages.length === 0) {
+      // Still track it so we don't re-attempt
+      indexState.set(file.sessionId, {
+        sessionId: file.sessionId,
+        filePath: file.filePath,
+        project: file.project,
+        lastModified: file.lastModified.getTime(),
+        chunkCount: 0,
+        messageCount: 0,
+        indexedAt: new Date(),
+        firstMessageAt: file.lastModified,
+        lastMessageAt: file.lastModified,
+      });
+      return;
+    }
+
+    const chunks = chunkMessages(
+      messages,
+      this.config.maxChunkMessages,
+      this.config.chunkOverlap
+    );
+
+    // Delete existing chunks for re-indexing
+    await this.repository.deleteBySessionId(file.sessionId);
+
+    // Embed all chunks
+    const embeddings = await this.embeddings.embedBatch(
+      chunks.map((c) => c.content)
+    );
+
+    // Insert all chunks
+    const rows = chunks.map((chunk, i) => ({
+      id: chunk.id,
+      vector: embeddings[i],
+      content: chunk.content,
+      metadata: JSON.stringify(chunk.metadata),
+      created_at: chunk.timestamp.getTime(),
+      session_id: chunk.sessionId,
+      role: chunk.role,
+      message_index_start: chunk.messageIndexStart,
+      message_index_end: chunk.messageIndexEnd,
+      project: chunk.project,
+    }));
+
+    await this.repository.insertBatch(rows);
+
+    // Update index state
+    indexState.set(file.sessionId, {
+      sessionId: file.sessionId,
+      filePath: file.filePath,
+      project: file.project,
+      lastModified: file.lastModified.getTime(),
+      chunkCount: chunks.length,
+      messageCount: messages.length,
+      indexedAt: new Date(),
+      firstMessageAt: messages[0].timestamp,
+      lastMessageAt: messages[messages.length - 1].timestamp,
+    });
+  }
+
+  async reindexSession(
+    sessionId: string
+  ): Promise<{ success: boolean; chunkCount: number; error?: string }> {
+    if (!this.config.enabled) {
+      return {
+        success: false,
+        chunkCount: 0,
+        error: "Conversation history indexing is not enabled",
+      };
+    }
+
+    const indexState = await this.loadIndexState();
+    const existing = indexState.get(sessionId);
+    if (!existing) {
+      return {
+        success: false,
+        chunkCount: 0,
+        error: "Session not found in index state",
+      };
+    }
+
+    // Construct session info for re-indexing
+    const file: SessionFileInfo = {
+      filePath: existing.filePath,
+      sessionId,
+      project: existing.project,
+      lastModified: new Date(),
+    };
+
+    await this.indexSession(file, indexState);
+    await this.saveIndexState(indexState);
+
+    const updated = indexState.get(sessionId)!;
+    return { success: true, chunkCount: updated.chunkCount };
+  }
+
+  async listIndexedSessions(
+    limit: number = 20,
+    offset: number = 0
+  ): Promise<{ sessions: IndexedSession[]; total: number }> {
+    const indexState = await this.loadIndexState();
+    const sessions = [...indexState.values()].sort(
+      (a, b) => b.indexedAt.getTime() - a.indexedAt.getTime()
+    );
+    return {
+      sessions: sessions.slice(offset, offset + limit),
+      total: sessions.length,
+    };
+  }
+
+  async searchHistory(
+    query: string,
+    embedding: number[],
+    limit: number,
+    filters?: HistoryFilters
+  ): Promise<ConversationHybridRow[]> {
+    return this.repository.findHybrid(embedding, query, limit, filters);
+  }
+}

package/src/services/memory.service.ts

@@ -1,10 +1,10 @@
 import { randomUUID } from "crypto";
-import type { Memory, SearchIntent, IntentProfile } from "../types/memory.js";
+import type { Memory, SearchIntent, IntentProfile, HybridRow } from "../types/memory.js";
 import { isDeleted } from "../types/memory.js";
+import type { SearchResult, SearchOptions } from "../types/conversation.js";
 import type { MemoryRepository } from "../db/memory.repository.js";
 import type { EmbeddingsService } from "./embeddings.service.js";
-import type { ConversationHistoryService } from "./conversation-history.service.js";
-import type { SearchResult, MemorySearchResult } from "../types/conversation-history.js";
+import type { ConversationHistoryService } from "./conversation.service.js";
 
 const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
   continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.02 },
@@ -17,29 +17,19 @@ const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
 const sigmoid = (x: number): number => 1 / (1 + Math.exp(-x));
 
 export class MemoryService {
-  private historyService: ConversationHistoryService | null = null;
-  private historyWeight: number = 0.5;
+  private conversationService: ConversationHistoryService | null = null;
 
   constructor(
     private repository: MemoryRepository,
     private embeddings: EmbeddingsService
-  ) { }
-
-  /**
-   * Optionally wire conversation history for unified search.
-   * Called from index.ts when conversationHistory.enabled is true.
-   */
-  setConversationHistory(service: ConversationHistoryService, weight: number): void {
-    this.historyService = service;
-    this.historyWeight = weight;
+  ) {}
+
+  setConversationService(service: ConversationHistoryService): void {
+    this.conversationService = service;
   }
 
-  /**
-   * Access the conversation history service (if wired).
-   * Used by MCP handlers for index/list/reindex tools.
-   */
-  getConversationHistory(): ConversationHistoryService | null {
-    return this.historyService;
+  getConversationService(): ConversationHistoryService | null {
+    return this.conversationService;
   }
 
   async store(
@@ -144,103 +134,109 @@ export class MemoryService {
     return updatedMemory;
   }
 
-  async search(
-    query: string,
-    intent: SearchIntent,
-    limit: number = 10,
-    includeDeleted: boolean = false
-  ): Promise<Memory[]> {
-    const queryEmbedding = await this.embeddings.embed(query);
-    const fetchLimit = limit * 5; // Fetch more for re-ranking
-
-    const candidates = await this.repository.findHybrid(queryEmbedding, query, fetchLimit);
-    const profile = INTENT_PROFILES[intent];
-    const now = new Date();
-
-    const scored = candidates
-      .filter((m) => includeDeleted || !isDeleted(m))
-      .map((candidate) => {
-        // Relevance: RRF score (already normalized ~0-1)
-        const relevance = candidate.rrfScore;
-
-        // Recency: exponential decay
-        const lastAccessed = candidate.lastAccessed ?? candidate.createdAt;
-        const hoursSinceAccess = Math.max(0, (now.getTime() - lastAccessed.getTime()) / (1000 * 60 * 60));
-        const recency = Math.pow(0.995, hoursSinceAccess);
-
-        // Utility: sigmoid of usefulness + log(accessCount)
-        const utility = sigmoid((candidate.usefulness + Math.log(candidate.accessCount + 1)) / 5);
-
-        // Weighted score
-        const { weights, jitter } = profile;
-        const score =
-          weights.relevance * relevance +
-          weights.recency * recency +
-          weights.utility * utility;
-
-        // Apply jitter
-        const finalScore = score * (1 + (Math.random() * 2 - 1) * jitter);
-
-        return { memory: candidate as Memory, finalScore };
-      });
-
-    // Sort by final score descending
-    scored.sort((a, b) => b.finalScore - a.finalScore);
-
-    // Return top N (read-only - no access tracking)
-    return scored.slice(0, limit).map((s) => s.memory);
+  private computeMemoryScore(
+    candidate: HybridRow,
+    profile: IntentProfile,
+    now: Date
+  ): number {
+    const relevance = candidate.rrfScore;
+    const lastAccessed = candidate.lastAccessed ?? candidate.createdAt;
+    const hoursSinceAccess = Math.max(
+      0,
+      (now.getTime() - lastAccessed.getTime()) / (1000 * 60 * 60)
+    );
+    const recency = Math.pow(0.995, hoursSinceAccess);
+    const utility = sigmoid(
+      (candidate.usefulness + Math.log(candidate.accessCount + 1)) / 5
+    );
+    const { weights, jitter } = profile;
+    const score =
+      weights.relevance * relevance +
+      weights.recency * recency +
+      weights.utility * utility;
+    return score * (1 + (Math.random() * 2 - 1) * jitter);
   }
 
-  /**
-   * Search across both memories and conversation history (if enabled).
-   * Returns a merged, score-normalized list sorted by relevance.
-   *
-   * If no history service is configured, falls back to memory-only search
-   * wrapped as MemorySearchResult[].
-   */
-  async searchUnified(
+  async search(
     query: string,
     intent: SearchIntent,
     limit: number = 10,
     includeDeleted: boolean = false,
+    options?: SearchOptions
   ): Promise<SearchResult[]> {
-    if (!this.historyService) {
-      const memories = await this.search(query, intent, limit, includeDeleted);
-      return this.toMemoryResults(memories, limit);
-    }
+    const queryEmbedding = await this.embeddings.embed(query);
+    const profile = INTENT_PROFILES[intent];
+    const now = new Date();
 
-    // Run both searches in parallel
-    const [memories, historyResults] = await Promise.all([
-      this.search(query, intent, limit, includeDeleted),
-      this.historyService.search(query, limit),
+    const hasConversationService = this.conversationService !== null;
+    const historyOnly = (options?.historyOnly ?? false) && hasConversationService;
+    const includeHistory =
+      (options?.includeHistory ?? true) && hasConversationService;
+    const historyWeight =
+      options?.historyWeight ??
+      this.conversationService?.config.historyWeight ??
+      0.75;
+
+    // Run memory + history queries in parallel
+    const memoryPromise =
+      !historyOnly
+        ? this.repository
+            .findHybrid(queryEmbedding, query, limit * 5)
+            .then((candidates) =>
+              candidates
+                .filter((m) => includeDeleted || !isDeleted(m))
+                .map((candidate) => ({
+                  id: candidate.id,
+                  content: candidate.content,
+                  metadata: candidate.metadata,
+                  createdAt: candidate.createdAt,
+                  updatedAt: candidate.updatedAt,
+                  source: "memory" as const,
+                  score: this.computeMemoryScore(candidate, profile, now),
+                  supersededBy: candidate.supersededBy,
+                  usefulness: candidate.usefulness,
+                  accessCount: candidate.accessCount,
+                  lastAccessed: candidate.lastAccessed,
+                }))
+            )
+        : Promise.resolve([] as SearchResult[]);
+
+    const historyPromise =
+      includeHistory || historyOnly
+        ? this.conversationService!
+            .searchHistory(
+              query,
+              queryEmbedding,
+              historyOnly ? limit * 5 : limit * 3,
+              options?.historyFilters
+            )
+            .then((historyRows) =>
+              historyRows.map((row) => ({
+                id: row.id,
+                content: row.content,
+                metadata: row.metadata,
+                createdAt: row.createdAt,
+                updatedAt: row.createdAt,
+                source: "conversation_history" as const,
+                score: row.rrfScore * historyWeight,
+                sessionId: (row.metadata?.session_id as string) ?? "",
+                role: (row.metadata?.role as string) ?? "unknown",
+                messageIndexStart: (row.metadata?.message_index_start as number) ?? 0,
+                messageIndexEnd: (row.metadata?.message_index_end as number) ?? 0,
+              }))
+            )
+        : Promise.resolve([] as SearchResult[]);
+
+    const [memoryResults, historyResults] = await Promise.all([
+      memoryPromise,
+      historyPromise,
     ]);
 
-    const memoryResults = this.toMemoryResults(memories, limit);
-
-    // Apply history weight to RRF scores
-    const weightedHistory: SearchResult[] = historyResults.map((h) => ({
-      ...h,
-      score: h.score * this.historyWeight,
-    }));
-
-    // Merge, sort by score descending, take top N
-    const merged = [...memoryResults, ...weightedHistory];
+    // Merge and sort by score descending
+    const merged = [...memoryResults, ...historyResults];
     merged.sort((a, b) => b.score - a.score);
-    return merged.slice(0, limit);
-  }
 
-  /** Convert pre-sorted Memory[] to MemorySearchResult[] with positional scores. */
-  private toMemoryResults(memories: Memory[], limit: number): MemorySearchResult[] {
-    return memories.map((m, rank) => ({
-      source: "memory" as const,
-      id: m.id,
-      content: m.content,
-      metadata: m.metadata,
-      score: (limit - rank) / limit,
-      createdAt: m.createdAt,
-      updatedAt: m.updatedAt,
-      supersededBy: m.supersededBy,
-    }));
+    return merged.slice(0, limit);
   }
 
   async trackAccess(ids: string[]): Promise<void> {