@aeriondyseti/vector-memory-mcp 2.4.4 → 2.5.0-dev.1

package/server/core/conversation.repository.ts

@@ -2,12 +2,13 @@ import type { Database } from "bun:sqlite";
 import type {
   ConversationHybridRow,
   HistoryFilters,
+  IndexedSession,
 } from "./conversation";
 import {
   serializeVector,
   safeParseJsonObject,
   sanitizeFtsQuery,
-  hybridRRF,
+  hybridRRFWithSignals,
   topByRRF,
   knnSearch,
 } from "./sqlite-utils";
@@ -15,6 +16,75 @@ import {
 export class ConversationRepository {
   constructor(private db: Database) {}
 
+  // ---------------------------------------------------------------------------
+  // Index state (replaces conversation_index_state.json — lives in the db so
+  // concurrent server processes share one consistent view)
+  // ---------------------------------------------------------------------------
+
+  loadIndexState(): Map<string, IndexedSession> {
+    const rows = this.db
+      .prepare("SELECT * FROM conversation_index_state")
+      .all() as Array<{
+      session_id: string;
+      file_path: string;
+      project: string;
+      last_modified: number;
+      chunk_count: number;
+      message_count: number;
+      indexed_at: number;
+      first_message_at: number;
+      last_message_at: number;
+    }>;
+
+    const map = new Map<string, IndexedSession>();
+    for (const r of rows) {
+      map.set(r.session_id, {
+        sessionId: r.session_id,
+        filePath: r.file_path,
+        project: r.project,
+        lastModified: r.last_modified,
+        chunkCount: r.chunk_count,
+        messageCount: r.message_count,
+        indexedAt: new Date(r.indexed_at),
+        firstMessageAt: new Date(r.first_message_at),
+        lastMessageAt: new Date(r.last_message_at),
+      });
+    }
+    return map;
+  }
+
+  upsertIndexState(sessions: IndexedSession[]): void {
+    if (sessions.length === 0) return;
+    const upsert = this.db.prepare(
+      `INSERT OR REPLACE INTO conversation_index_state
+        (session_id, file_path, project, last_modified, chunk_count, message_count, indexed_at, first_message_at, last_message_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    );
+    const tx = this.db.transaction(() => {
+      for (const s of sessions) {
+        upsert.run(
+          s.sessionId,
+          s.filePath,
+          s.project,
+          s.lastModified,
+          s.chunkCount,
+          s.messageCount,
+          s.indexedAt.getTime(),
+          s.firstMessageAt.getTime(),
+          s.lastMessageAt.getTime()
+        );
+      }
+    });
+    tx();
+  }
+
+  countIndexState(): number {
+    const row = this.db
+      .prepare("SELECT COUNT(*) AS n FROM conversation_index_state")
+      .get() as { n: number };
+    return row.n;
+  }
+
   async insertBatch(
     rows: Array<{
       id: string;
@@ -188,11 +258,14 @@ export class ConversationRepository {
   /**
    * Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
    *
-   * NOTE: Filters (session, role, project, date) are applied AFTER candidate selection
-   * and RRF scoring, not pushed into the KNN/FTS queries. This is an intentional
-   * performance tradeoff — KNN is brute-force JS-side (no SQL pre-filter possible),
-   * and filtering post-RRF avoids duplicating filter logic across both retrieval paths.
-   * The consequence is that filtered queries may return fewer than `limit` results.
+   * The project filter is applied PRE-candidate-selection (pushed into both
+   * the KNN scan and the FTS query) so project-scoped searches rank within
+   * the project's own chunks — post-filtering a global top-K would return
+   * false-empty results for projects with few chunks in a shared database.
+   *
+   * Remaining filters (session, role, date) are applied AFTER candidate
+   * selection and RRF scoring, so those filtered queries may return fewer
+   * than `limit` results.
    */
   async findHybrid(
     embedding: number[],
@@ -200,24 +273,52 @@ export class ConversationRepository {
     limit: number,
     filters?: HistoryFilters
   ): Promise<ConversationHybridRow[]> {
-    const candidateCount = limit * 3;
+    const candidateCount = limit * 5;
+    const project = filters?.project;
 
-    // Vector KNN search (brute-force cosine similarity in JS)
-    const vecResults = knnSearch(this.db, "conversation_history_vec", embedding, candidateCount);
+    // Vector KNN search (brute-force cosine similarity in JS), pre-filtered
+    // by project when scoped
+    const vecResults = knnSearch(
+      this.db,
+      "conversation_history_vec",
+      embedding,
+      candidateCount,
+      project !== undefined
+        ? {
+            sql: `SELECT v.id, v.vector FROM conversation_history_vec v
+                  JOIN conversation_history c ON v.id = c.id WHERE c.project = ?`,
+            params: [project],
+          }
+        : undefined,
+    );
 
-    // FTS5 search
+    // FTS5 search, pre-filtered by project when scoped
     const ftsQuery = sanitizeFtsQuery(query);
-    const ftsResults = this.db
-      .prepare(
-        `SELECT id FROM conversation_history_fts
-         WHERE conversation_history_fts MATCH ?
-         ORDER BY rank
-         LIMIT ?`
-      )
-      .all(ftsQuery, candidateCount) as Array<{ id: string }>;
+    const ftsResults = (
+      project !== undefined
+        ? this.db
+            .prepare(
+              `SELECT conversation_history_fts.id FROM conversation_history_fts
+               JOIN conversation_history c ON conversation_history_fts.id = c.id
+               WHERE conversation_history_fts MATCH ? AND c.project = ?
+               ORDER BY rank
+               LIMIT ?`
+            )
+            .all(ftsQuery, project, candidateCount)
+        : this.db
+            .prepare(
+              `SELECT id FROM conversation_history_fts
+               WHERE conversation_history_fts MATCH ?
+               ORDER BY rank
+               LIMIT ?`
+            )
+            .all(ftsQuery, candidateCount)
+    ) as Array<{ id: string }>;
 
-    // Compute RRF scores and get top ids
-    const rrfScores = hybridRRF(vecResults, ftsResults);
+    // Compute RRF scores with search signals for confidence scoring
+    const signalsMap = hybridRRFWithSignals(vecResults, ftsResults);
+    const rrfScores = new Map<string, number>();
+    for (const [id, s] of signalsMap) rrfScores.set(id, s.rrfScore);
     const topIds = topByRRF(rrfScores, limit);
 
     if (topIds.length === 0) return [];
@@ -274,17 +375,23 @@ export class ConversationRepository {
       project: string;
     }>;
 
-    // Build a lookup for ordering by RRF score
-    const scoreMap = new Map(topIds.map((id) => [id, rrfScores.get(id)!]));
-
     return fullRows
-      .map((row) => ({
-        id: row.id,
-        content: row.content,
-        metadata: safeParseJsonObject(row.metadata),
-        createdAt: new Date(row.created_at),
-        rrfScore: scoreMap.get(row.id) ?? 0,
-      }))
+      .map((row) => {
+        const signals = signalsMap.get(row.id)!;
+        return {
+          id: row.id,
+          content: row.content,
+          metadata: safeParseJsonObject(row.metadata),
+          createdAt: new Date(row.created_at),
+          rrfScore: signals.rrfScore,
+          signals: {
+            cosineSimilarity: signals.cosineSimilarity,
+            ftsMatch: signals.ftsMatch,
+            knnRank: signals.knnRank,
+            ftsRank: signals.ftsRank,
+          },
+        };
+      })
       .sort((a, b) => b.rrfScore - a.rrfScore);
   }
 }

package/server/core/conversation.service.ts

@@ -1,5 +1,5 @@
 import { createHash } from "crypto";
-import { readFile, writeFile, mkdir } from "fs/promises";
+import { readFile } from "fs/promises";
 import { dirname, join } from "path";
 import type { ConversationRepository } from "./conversation.repository";
 import type {
@@ -93,8 +93,8 @@ export function chunkMessages(
   return chunks;
 }
 
-/** Serializable index state format */
-interface IndexStateEntry {
+/** Legacy JSON index state format (pre-table), imported on first run. */
+interface LegacyIndexStateEntry {
   sessionId: string;
   filePath: string;
   project: string;
@@ -107,65 +107,63 @@ interface IndexStateEntry {
 }
 
 export class ConversationHistoryService {
-  private indexStatePath: string;
-  private indexStateCache: Map<string, IndexedSession> | null = null;
+  private legacyIndexStatePath: string;
+  private legacyImportAttempted = false;
 
   constructor(
     private repository: ConversationRepository,
     private embeddings: EmbeddingsService,
     public readonly config: ConversationHistoryConfig,
-    private dbPath: string,
+    dbPath: string,
     private parser: SessionLogParser = new ClaudeCodeSessionParser()
   ) {
-    this.indexStatePath = join(
+    this.legacyIndexStatePath = join(
       dirname(dbPath),
       "conversation_index_state.json"
     );
   }
 
+  /**
+   * Index state lives in the conversation_index_state table (shared by all
+   * server processes) — read fresh each time, never cached per-process.
+   * A pre-existing conversation_index_state.json is imported once.
+   */
   private async loadIndexState(): Promise<Map<string, IndexedSession>> {
-    if (this.indexStateCache) return this.indexStateCache;
-    try {
-      const raw = await readFile(this.indexStatePath, "utf-8");
-      const entries: IndexStateEntry[] = JSON.parse(raw);
-      const map = new Map<string, IndexedSession>();
-      for (const e of entries) {
-        map.set(e.sessionId, {
-          sessionId: e.sessionId,
-          filePath: e.filePath,
-          project: e.project,
-          lastModified: e.lastModified,
-          chunkCount: e.chunkCount,
-          messageCount: e.messageCount,
-          indexedAt: new Date(e.indexedAt),
-          firstMessageAt: new Date(e.firstMessageAt),
-          lastMessageAt: new Date(e.lastMessageAt),
-        });
+    if (!this.legacyImportAttempted) {
+      this.legacyImportAttempted = true;
+      if (this.repository.countIndexState() === 0) {
+        await this.importLegacyIndexState();
       }
-      this.indexStateCache = map;
-      return map;
+    }
+    return this.repository.loadIndexState();
+  }
+
+  private async importLegacyIndexState(): Promise<void> {
+    let entries: LegacyIndexStateEntry[];
+    try {
+      const raw = await readFile(this.legacyIndexStatePath, "utf-8");
+      entries = JSON.parse(raw);
     } catch {
-      const map = new Map<string, IndexedSession>();
-      this.indexStateCache = map;
-      return map;
+      return; // no legacy state — fine
     }
+
+    this.repository.upsertIndexState(
+      entries.map((e) => ({
+        sessionId: e.sessionId,
+        filePath: e.filePath,
+        project: e.project,
+        lastModified: e.lastModified,
+        chunkCount: e.chunkCount,
+        messageCount: e.messageCount,
+        indexedAt: new Date(e.indexedAt),
+        firstMessageAt: new Date(e.firstMessageAt),
+        lastMessageAt: new Date(e.lastMessageAt),
+      }))
+    );
   }
 
-  private async saveIndexState(state: Map<string, IndexedSession>): Promise<void> {
-    const entries: IndexStateEntry[] = [...state.values()].map((s) => ({
-      sessionId: s.sessionId,
-      filePath: s.filePath,
-      project: s.project,
-      lastModified: s.lastModified,
-      chunkCount: s.chunkCount,
-      messageCount: s.messageCount,
-      indexedAt: s.indexedAt.toISOString(),
-      firstMessageAt: s.firstMessageAt.toISOString(),
-      lastMessageAt: s.lastMessageAt.toISOString(),
-    }));
-    await mkdir(dirname(this.indexStatePath), { recursive: true });
-    await writeFile(this.indexStatePath, JSON.stringify(entries, null, 2));
-    this.indexStateCache = state;
+  private saveIndexState(sessions: IndexedSession[]): void {
+    this.repository.upsertIndexState(sessions);
   }
 
   async indexConversations(
@@ -207,6 +205,7 @@ export class ConversationHistoryService {
     let skipped = 0;
     const errors: string[] = [];
     const details: SessionIndexDetail[] = [];
+    const updated: IndexedSession[] = [];
 
     for (const file of sessionFiles) {
       const existing = indexState.get(file.sessionId);
@@ -218,6 +217,7 @@ export class ConversationHistoryService {
 
       try {
         const state = await this.indexSession(file, indexState);
+        updated.push(state);
         indexed++;
         details.push({
           sessionId: file.sessionId,
@@ -233,7 +233,7 @@ export class ConversationHistoryService {
       }
     }
 
-    await this.saveIndexState(indexState);
+    this.saveIndexState(updated);
     return { indexed, skipped, errors, details };
   }
 
@@ -297,11 +297,12 @@ export class ConversationHistoryService {
     // Atomically replace old chunks with new ones
     await this.repository.replaceSession(file.sessionId, rows);
 
-    // Update index state
+    // Update index state. Prefer the parsed project (cwd-derived) over the
+    // lossy directory-name decode carried by the file listing.
     const session: IndexedSession = {
       sessionId: file.sessionId,
       filePath: file.filePath,
-      project: file.project,
+      project: messages[0].project,
       lastModified: file.lastModified.getTime(),
       chunkCount: chunks.length,
       messageCount: messages.length,
@@ -342,11 +343,10 @@ export class ConversationHistoryService {
       lastModified: new Date(),
     };
 
-    await this.indexSession(file, indexState);
-    await this.saveIndexState(indexState);
+    const state = await this.indexSession(file, indexState);
+    this.saveIndexState([state]);
 
-    const updated = indexState.get(sessionId)!;
-    return { success: true, chunkCount: updated.chunkCount };
+    return { success: true, chunkCount: state.chunkCount };
   }
 
   async listIndexedSessions(

package/server/core/conversation.ts

@@ -47,6 +47,8 @@ export interface IndexedSession {
   lastMessageAt: Date;
 }
 
+import type { SearchSignals } from "./memory";
+
 /** Raw row from conversation_history table with RRF score */
 export interface ConversationHybridRow {
   id: string;
@@ -54,6 +56,7 @@ export interface ConversationHybridRow {
   metadata: Record<string, unknown>;
   createdAt: Date;
   rrfScore: number;
+  signals: SearchSignals;
 }
 
 /** Unified search result with source provenance */
@@ -65,6 +68,10 @@ export interface SearchResult {
   updatedAt: Date;
   source: "memory" | "conversation_history";
   score: number;
+  /** Absolute relevance confidence (0.0-1.0). Based on cosine similarity + retrieval agreement. */
+  confidence: number;
+  /** Canonical project path this result belongs to (null = untagged/legacy). */
+  project: string | null;
   // Memory-specific fields
   supersededBy: string | null;
   usefulness?: number;
@@ -110,10 +117,20 @@ export interface HistoryFilters {
 /** Options for the integrated search across both sources */
 export interface SearchOptions {
   limit?: number;
+  /**
+   * Project scope: "all" (default) searches every project with a ranking
+   * boost for the current one; "project" restricts to the current project;
+   * any other string is an explicit canonical project path to restrict to.
+   */
+  scope?: string;
   includeDeleted?: boolean;
   includeHistory?: boolean;
   historyOnly?: boolean;
   historyWeight?: number;
   historyFilters?: HistoryFilters;
   offset?: number;
+  /** Filter both memories and history created after this date. Merged into historyFilters; explicit historyFilters.after takes precedence. */
+  after?: Date;
+  /** Filter both memories and history created before this date. Merged into historyFilters; explicit historyFilters.before takes precedence. */
+  before?: Date;
 }

package/server/core/memory.repository.ts

@@ -4,7 +4,7 @@ import {
   deserializeVector,
   safeParseJsonObject,
   sanitizeFtsQuery,
-  hybridRRF,
+  hybridRRFWithSignals,
   topByRRF,
   knnSearch,
   batchedQuery,
@@ -49,6 +49,7 @@ export class MemoryRepository {
         row.last_accessed != null
           ? new Date(row.last_accessed as number)
           : null,
+      project: (row.project as string) ?? null,
     };
   }
 
@@ -70,8 +71,8 @@ export class MemoryRepository {
     const tx = this.db.transaction(() => {
       this.db
         .prepare(
-          `INSERT INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+          `INSERT INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed, project)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
         )
         .run(
           memory.id,
@@ -83,6 +84,7 @@ export class MemoryRepository {
           memory.usefulness,
           memory.accessCount,
           memory.lastAccessed?.getTime() ?? null,
+          memory.project,
         );
 
       this.db
@@ -102,8 +104,8 @@ export class MemoryRepository {
       // Main table supports INSERT OR REPLACE
       this.db
         .prepare(
-          `INSERT OR REPLACE INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+          `INSERT OR REPLACE INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed, project)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
         )
         .run(
           memory.id,
@@ -115,6 +117,7 @@ export class MemoryRepository {
           memory.usefulness,
           memory.accessCount,
           memory.lastAccessed?.getTime() ?? null,
+          memory.project,
         );
 
       this.db.prepare("DELETE FROM memories_vec WHERE id = ?").run(memory.id);
@@ -193,40 +196,88 @@ export class MemoryRepository {
 
   /**
    * Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
+   *
+   * The project filter is applied PRE-candidate-selection (pushed into both
+   * the KNN scan and the FTS query) so project-scoped searches rank within
+   * the project's own corpus — post-filtering a global top-K would return
+   * false-empty results for small projects in a large shared database.
+   *
+   * Date filters remain post-RRF on the final row fetch, so date-filtered
+   * queries may return fewer than `limit` results.
    */
   async findHybrid(
     embedding: number[],
     query: string,
     limit: number,
+    filters?: { after?: Date; before?: Date; project?: string },
   ): Promise<HybridRow[]> {
-    const candidateLimit = limit * 3;
-
-    // Vector KNN search (brute-force cosine similarity in JS)
-    const vectorResults = knnSearch(this.db, "memories_vec", embedding, candidateLimit);
-
-    // Full-text search
+    const candidateLimit = limit * 5;
+    const project = filters?.project;
+
+    // Vector KNN search (brute-force cosine similarity in JS), pre-filtered
+    // by project when scoped
+    const vectorResults = knnSearch(
+      this.db,
+      "memories_vec",
+      embedding,
+      candidateLimit,
+      project !== undefined
+        ? {
+            sql: `SELECT v.id, v.vector FROM memories_vec v
+                  JOIN memories m ON v.id = m.id WHERE m.project = ?`,
+            params: [project],
+          }
+        : undefined,
+    );
+
+    // Full-text search, pre-filtered by project when scoped
     const ftsQuery = sanitizeFtsQuery(query);
     const ftsResults: Array<{ id: string }> = ftsQuery
-      ? (this.db
-          .prepare(
-            "SELECT id FROM memories_fts WHERE memories_fts MATCH ? LIMIT ?",
-          )
-          .all(ftsQuery, candidateLimit) as Array<{ id: string }>)
+      ? project !== undefined
+        ? (this.db
+            .prepare(
+              `SELECT memories_fts.id FROM memories_fts
+               JOIN memories m ON memories_fts.id = m.id
+               WHERE memories_fts MATCH ? AND m.project = ? LIMIT ?`,
+            )
+            .all(ftsQuery, project, candidateLimit) as Array<{ id: string }>)
+        : (this.db
+            .prepare(
+              "SELECT id FROM memories_fts WHERE memories_fts MATCH ? LIMIT ?",
+            )
+            .all(ftsQuery, candidateLimit) as Array<{ id: string }>)
       : [];
 
-    // Compute RRF scores and pick top ids
-    const rrfScores = hybridRRF(vectorResults, ftsResults);
+    // Compute RRF scores with search signals for confidence scoring
+    const signalsMap = hybridRRFWithSignals(vectorResults, ftsResults);
+    const rrfScores = new Map<string, number>();
+    for (const [id, s] of signalsMap) rrfScores.set(id, s.rrfScore);
     const topIds = topByRRF(rrfScores, limit);
 
     if (topIds.length === 0) return [];
 
-    // Fetch full rows for the winning ids (service layer handles deleted filtering)
+    // Fetch full rows for the winning ids, applying date filters if present
+    const conditions: string[] = [];
+    const params: (string | number)[] = [];
+
     const placeholders = topIds.map(() => "?").join(", ");
+    conditions.push(`id IN (${placeholders})`);
+    params.push(...topIds);
+
+    if (filters?.after) {
+      conditions.push("created_at > ?");
+      params.push(filters.after.getTime());
+    }
+    if (filters?.before) {
+      conditions.push("created_at < ?");
+      params.push(filters.before.getTime());
+    }
+
     const rows = this.db
       .prepare(
-        `SELECT * FROM memories WHERE id IN (${placeholders})`,
+        `SELECT * FROM memories WHERE ${conditions.join(" AND ")}`,
       )
-      .all(...topIds) as Array<Record<string, unknown>>;
+      .all(...params) as Array<Record<string, unknown>>;
 
     // Build a lookup for quick access
     const rowMap = new Map<string, Record<string, unknown>>();
@@ -242,9 +293,16 @@ export class MemoryRepository {
 
       const memEmbedding = this.getEmbedding(id);
       const memory = this.rowToMemory(row, memEmbedding);
+      const signals = signalsMap.get(id)!;
       results.push({
         ...memory,
-        rrfScore: rrfScores.get(id) ?? 0,
+        rrfScore: signals.rrfScore,
+        signals: {
+          cosineSimilarity: signals.cosineSimilarity,
+          ftsMatch: signals.ftsMatch,
+          knnRank: signals.knnRank,
+          ftsRank: signals.ftsRank,
+        },
       });
     }