@aeriondyseti/vector-memory-mcp 2.4.0 → 2.4.4-dev.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.4.0",
+  "version": "2.4.4-dev.1",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",

package/server/core/connection.ts

@@ -1,7 +1,7 @@
 import { Database } from "bun:sqlite";
 import { existsSync, mkdirSync } from "fs";
 import { dirname } from "path";
-import { removeVec0Tables, runMigrations } from "./migrations.js";
+import { removeVec0Tables, runMigrations } from "./migrations";
 
 /**
  * Open (or create) a SQLite database at the given path

package/server/core/conversation.repository.ts

@@ -2,15 +2,15 @@ import type { Database } from "bun:sqlite";
 import type {
   ConversationHybridRow,
   HistoryFilters,
-} from "./conversation.js";
+} from "./conversation";
 import {
   serializeVector,
   safeParseJsonObject,
   sanitizeFtsQuery,
-  hybridRRF,
+  hybridRRFWithSignals,
   topByRRF,
   knnSearch,
-} from "./sqlite-utils.js";
+} from "./sqlite-utils";
 
 export class ConversationRepository {
   constructor(private db: Database) {}
@@ -105,13 +105,102 @@ export class ConversationRepository {
     tx();
   }
 
+  async replaceSession(
+    sessionId: string,
+    rows: Array<{
+      id: string;
+      vector: number[];
+      content: string;
+      metadata: string;
+      created_at: number;
+      session_id: string;
+      role: string;
+      message_index_start: number;
+      message_index_end: number;
+      project: string;
+    }>
+  ): Promise<void> {
+    const insertMain = this.db.prepare(
+      `INSERT OR REPLACE INTO conversation_history
+        (id, content, metadata, created_at, session_id, role, message_index_start, message_index_end, project)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    );
+    const deleteVec = this.db.prepare(
+      `DELETE FROM conversation_history_vec WHERE id = ?`
+    );
+    const insertVec = this.db.prepare(
+      `INSERT INTO conversation_history_vec (id, vector) VALUES (?, ?)`
+    );
+    const deleteFts = this.db.prepare(
+      `DELETE FROM conversation_history_fts WHERE id = ?`
+    );
+    const insertFts = this.db.prepare(
+      `INSERT INTO conversation_history_fts (id, content) VALUES (?, ?)`
+    );
+
+    const tx = this.db.transaction(() => {
+      // Delete old chunks first
+      const idRows = this.db
+        .prepare(`SELECT id FROM conversation_history WHERE session_id = ?`)
+        .all(sessionId) as Array<{ id: string }>;
+
+      if (idRows.length > 0) {
+        const ids = idRows.map((r) => r.id);
+        const placeholders = ids.map(() => "?").join(", ");
+        this.db
+          .prepare(
+            `DELETE FROM conversation_history_vec WHERE id IN (${placeholders})`
+          )
+          .run(...ids);
+        this.db
+          .prepare(
+            `DELETE FROM conversation_history_fts WHERE id IN (${placeholders})`
+          )
+          .run(...ids);
+        this.db
+          .prepare(`DELETE FROM conversation_history WHERE session_id = ?`)
+          .run(sessionId);
+      }
+
+      // Insert new chunks
+      for (const row of rows) {
+        insertMain.run(
+          row.id,
+          row.content,
+          row.metadata,
+          row.created_at,
+          row.session_id,
+          row.role,
+          row.message_index_start,
+          row.message_index_end,
+          row.project
+        );
+        deleteVec.run(row.id);
+        insertVec.run(row.id, serializeVector(row.vector));
+        deleteFts.run(row.id);
+        insertFts.run(row.id, row.content);
+      }
+    });
+
+    tx();
+  }
+
+  /**
+   * Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
+   *
+   * NOTE: Filters (session, role, project, date) are applied AFTER candidate selection
+   * and RRF scoring, not pushed into the KNN/FTS queries. This is an intentional
+   * performance tradeoff — KNN is brute-force JS-side (no SQL pre-filter possible),
+   * and filtering post-RRF avoids duplicating filter logic across both retrieval paths.
+   * The consequence is that filtered queries may return fewer than `limit` results.
+   */
   async findHybrid(
     embedding: number[],
     query: string,
     limit: number,
     filters?: HistoryFilters
   ): Promise<ConversationHybridRow[]> {
-    const candidateCount = limit * 3;
+    const candidateCount = limit * 5;
 
     // Vector KNN search (brute-force cosine similarity in JS)
     const vecResults = knnSearch(this.db, "conversation_history_vec", embedding, candidateCount);
@@ -127,8 +216,10 @@ export class ConversationRepository {
       )
       .all(ftsQuery, candidateCount) as Array<{ id: string }>;
 
-    // Compute RRF scores and get top ids
-    const rrfScores = hybridRRF(vecResults, ftsResults);
+    // Compute RRF scores with search signals for confidence scoring
+    const signalsMap = hybridRRFWithSignals(vecResults, ftsResults);
+    const rrfScores = new Map<string, number>();
+    for (const [id, s] of signalsMap) rrfScores.set(id, s.rrfScore);
     const topIds = topByRRF(rrfScores, limit);
 
     if (topIds.length === 0) return [];
@@ -185,17 +276,23 @@ export class ConversationRepository {
       project: string;
     }>;
 
-    // Build a lookup for ordering by RRF score
-    const scoreMap = new Map(topIds.map((id) => [id, rrfScores.get(id)!]));
-
     return fullRows
-      .map((row) => ({
-        id: row.id,
-        content: row.content,
-        metadata: safeParseJsonObject(row.metadata),
-        createdAt: new Date(row.created_at),
-        rrfScore: scoreMap.get(row.id) ?? 0,
-      }))
+      .map((row) => {
+        const signals = signalsMap.get(row.id)!;
+        return {
+          id: row.id,
+          content: row.content,
+          metadata: safeParseJsonObject(row.metadata),
+          createdAt: new Date(row.created_at),
+          rrfScore: signals.rrfScore,
+          signals: {
+            cosineSimilarity: signals.cosineSimilarity,
+            ftsMatch: signals.ftsMatch,
+            knnRank: signals.knnRank,
+            ftsRank: signals.ftsRank,
+          },
+        };
+      })
       .sort((a, b) => b.rrfScore - a.rrfScore);
   }
 }

package/server/core/conversation.service.ts

@@ -1,7 +1,7 @@
 import { createHash } from "crypto";
 import { readFile, writeFile, mkdir } from "fs/promises";
 import { dirname, join } from "path";
-import type { ConversationRepository } from "./conversation.repository.js";
+import type { ConversationRepository } from "./conversation.repository";
 import type {
   ConversationChunk,
   ConversationHybridRow,
@@ -10,12 +10,12 @@ import type {
   ParsedMessage,
   SessionFileInfo,
   SessionIndexDetail,
-} from "./conversation.js";
-import type { ConversationHistoryConfig } from "../config/index.js";
-import { resolveSessionLogPath } from "../config/index.js";
-import type { EmbeddingsService } from "./embeddings.service.js";
-import type { SessionLogParser } from "./parsers/types.js";
-import { ClaudeCodeSessionParser } from "./parsers/claude-code.parser.js";
+} from "./conversation";
+import type { ConversationHistoryConfig } from "../config/index";
+import { resolveSessionLogPath } from "../config/index";
+import type { EmbeddingsService } from "./embeddings.service";
+import type { SessionLogParser } from "./parsers/types";
+import { ClaudeCodeSessionParser } from "./parsers/claude-code.parser";
 
 /**
  * Generate a deterministic chunk ID from session ID and message indices.
@@ -78,12 +78,7 @@ export function chunkMessages(
       messageIndexEnd: lastMsg.messageIndex,
       project: firstMsg.project,
       metadata: {
-        session_id: firstMsg.sessionId,
         timestamp: firstMsg.timestamp.toISOString(),
-        role,
-        message_index_start: firstMsg.messageIndex,
-        message_index_end: lastMsg.messageIndex,
-        project: firstMsg.project,
         git_branch: firstMsg.gitBranch,
         is_subagent: firstMsg.isSubagent,
         agent_id: firstMsg.agentId,
@@ -273,20 +268,24 @@ export class ConversationHistoryService {
       this.config.chunkOverlap
     );
 
-    // Delete existing chunks for re-indexing
-    await this.repository.deleteBySessionId(file.sessionId);
-
-    // Embed all chunks
+    // Embed all chunks FIRST (pure computation, no DB side effects)
     const embeddings = await this.embeddings.embedBatch(
       chunks.map((c) => c.content)
     );
 
-    // Insert all chunks
+    // Build rows
     const rows = chunks.map((chunk, i) => ({
       id: chunk.id,
       vector: embeddings[i],
       content: chunk.content,
-      metadata: JSON.stringify(chunk.metadata),
+      metadata: JSON.stringify({
+        ...chunk.metadata,
+        session_id: chunk.sessionId,
+        role: chunk.role,
+        message_index_start: chunk.messageIndexStart,
+        message_index_end: chunk.messageIndexEnd,
+        project: chunk.project,
+      }),
       created_at: chunk.timestamp.getTime(),
       session_id: chunk.sessionId,
       role: chunk.role,
@@ -295,7 +294,8 @@ export class ConversationHistoryService {
       project: chunk.project,
     }));
 
-    await this.repository.insertBatch(rows);
+    // Atomically replace old chunks with new ones
+    await this.repository.replaceSession(file.sessionId, rows);
 
     // Update index state
     const session: IndexedSession = {

package/server/core/conversation.ts

@@ -14,12 +14,7 @@ export interface ParsedMessage {
 
 /** Metadata stored per conversation chunk in the database */
 export interface ConversationChunkMetadata {
-  session_id: string;
   timestamp: string;
-  role: string;
-  message_index_start: number;
-  message_index_end: number;
-  project: string;
   git_branch?: string;
   is_subagent: boolean;
   agent_id?: string;
@@ -52,6 +47,8 @@ export interface IndexedSession {
   lastMessageAt: Date;
 }
 
+import type { SearchSignals } from "./memory";
+
 /** Raw row from conversation_history table with RRF score */
 export interface ConversationHybridRow {
   id: string;
@@ -59,6 +56,7 @@ export interface ConversationHybridRow {
   metadata: Record<string, unknown>;
   createdAt: Date;
   rrfScore: number;
+  signals: SearchSignals;
 }
 
 /** Unified search result with source provenance */
@@ -70,6 +68,8 @@ export interface SearchResult {
   updatedAt: Date;
   source: "memory" | "conversation_history";
   score: number;
+  /** Absolute relevance confidence (0.0-1.0). Based on cosine similarity + retrieval agreement. */
+  confidence: number;
   // Memory-specific fields
   supersededBy: string | null;
   usefulness?: number;
@@ -114,6 +114,8 @@ export interface HistoryFilters {
 
 /** Options for the integrated search across both sources */
 export interface SearchOptions {
+  limit?: number;
+  includeDeleted?: boolean;
   includeHistory?: boolean;
   historyOnly?: boolean;
   historyWeight?: number;

package/server/core/embeddings.service.ts

@@ -108,6 +108,12 @@ export class EmbeddingsService {
 
   private meanPool(data: Float32Array, mask: number[], seqLen: number): number[] {
     const dim = this._dimension;
+    const expectedLen = seqLen * dim;
+    if (data.length < expectedLen) {
+      throw new Error(
+        `ONNX output size ${data.length} < expected ${expectedLen} (seqLen=${seqLen}, dim=${dim}). Model/dimension mismatch?`,
+      );
+    }
     const pooled = new Array(dim).fill(0);
     let maskSum = 0;
     for (let t = 0; t < seqLen; t++) {

package/server/core/memory.repository.ts

@@ -4,15 +4,17 @@ import {
   deserializeVector,
   safeParseJsonObject,
   sanitizeFtsQuery,
-  hybridRRF,
+  hybridRRFWithSignals,
   topByRRF,
   knnSearch,
-} from "./sqlite-utils.js";
+  batchedQuery,
+  SQLITE_BATCH_SIZE,
+} from "./sqlite-utils";
 import {
   type Memory,
   type HybridRow,
   DELETED_TOMBSTONE,
-} from "./memory.js";
+} from "./memory";
 
 export class MemoryRepository {
   constructor(private db: Database) {}
@@ -144,14 +146,16 @@ export class MemoryRepository {
   async findByIds(ids: string[]): Promise<Memory[]> {
     if (ids.length === 0) return [];
 
-    const placeholders = ids.map(() => "?").join(", ");
-    const rows = this.db
-      .prepare(`SELECT * FROM memories WHERE id IN (${placeholders})`)
-      .all(...ids) as Array<Record<string, unknown>>;
+    return batchedQuery(this.db, ids, (batch) => {
+      const placeholders = batch.map(() => "?").join(", ");
+      const rows = this.db
+        .prepare(`SELECT * FROM memories WHERE id IN (${placeholders})`)
+        .all(...batch) as Array<Record<string, unknown>>;
 
-    return rows.map((row) => {
-      const embedding = this.getEmbedding(row.id as string);
-      return this.rowToMemory(row, embedding);
+      return rows.map((row) => {
+        const embedding = this.getEmbedding(row.id as string);
+        return this.rowToMemory(row, embedding);
+      });
     });
   }
 
@@ -165,6 +169,28 @@ export class MemoryRepository {
     return result.changes > 0;
   }
 
+  /**
+   * Increment access_count and update last_accessed for multiple memories in batch.
+   * Uses batched IN clauses to stay within SQLite parameter limits.
+   */
+  bulkUpdateAccess(ids: string[], now: Date): void {
+    if (ids.length === 0) return;
+    const ts = now.getTime();
+
+    const runBatch = (batch: string[]) => {
+      const placeholders = batch.map(() => "?").join(", ");
+      this.db
+        .prepare(
+          `UPDATE memories SET access_count = access_count + 1, last_accessed = ? WHERE id IN (${placeholders})`
+        )
+        .run(ts, ...batch);
+    };
+
+    for (let i = 0; i < ids.length; i += SQLITE_BATCH_SIZE) {
+      runBatch(ids.slice(i, i + SQLITE_BATCH_SIZE));
+    }
+  }
+
   /**
    * Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
    */
@@ -173,7 +199,7 @@ export class MemoryRepository {
     query: string,
     limit: number,
   ): Promise<HybridRow[]> {
-    const candidateLimit = limit * 3;
+    const candidateLimit = limit * 5;
 
     // Vector KNN search (brute-force cosine similarity in JS)
     const vectorResults = knnSearch(this.db, "memories_vec", embedding, candidateLimit);
@@ -188,8 +214,10 @@ export class MemoryRepository {
           .all(ftsQuery, candidateLimit) as Array<{ id: string }>)
       : [];
 
-    // Compute RRF scores and pick top ids
-    const rrfScores = hybridRRF(vectorResults, ftsResults);
+    // Compute RRF scores with search signals for confidence scoring
+    const signalsMap = hybridRRFWithSignals(vectorResults, ftsResults);
+    const rrfScores = new Map<string, number>();
+    for (const [id, s] of signalsMap) rrfScores.set(id, s.rrfScore);
     const topIds = topByRRF(rrfScores, limit);
 
     if (topIds.length === 0) return [];
@@ -216,9 +244,16 @@ export class MemoryRepository {
 
       const memEmbedding = this.getEmbedding(id);
       const memory = this.rowToMemory(row, memEmbedding);
+      const signals = signalsMap.get(id)!;
       results.push({
         ...memory,
-        rrfScore: rrfScores.get(id) ?? 0,
+        rrfScore: signals.rrfScore,
+        signals: {
+          cosineSimilarity: signals.cosineSimilarity,
+          ftsMatch: signals.ftsMatch,
+          knnRank: signals.knnRank,
+          ftsRank: signals.ftsRank,
+        },
       });
     }
 

package/server/core/memory.service.ts

@@ -1,17 +1,19 @@
 import { randomUUID, createHash } from "crypto";
-import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory.js";
-import { isDeleted } from "./memory.js";
-import type { SearchResult, SearchOptions } from "./conversation.js";
-import type { MemoryRepository } from "./memory.repository.js";
-import type { EmbeddingsService } from "./embeddings.service.js";
-import type { ConversationHistoryService } from "./conversation.service.js";
-
+import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory";
+import { isDeleted, computeConfidence } from "./memory";
+import type { SearchResult, SearchOptions } from "./conversation";
+import type { MemoryRepository } from "./memory.repository";
+import type { EmbeddingsService } from "./embeddings.service";
+import type { ConversationHistoryService } from "./conversation.service";
+
+// Jitter values halved from original (0.02/0.05/0.15) because RRF_K=10 produces
+// ~6x more score spread than K=60, amplifying jitter's disruption effect.
 const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
-  continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.02 },
-  fact_check: { weights: { relevance: 0.6, recency: 0.1, utility: 0.3 }, jitter: 0.02 },
-  frequent: { weights: { relevance: 0.2, recency: 0.2, utility: 0.6 }, jitter: 0.02 },
-  associative: { weights: { relevance: 0.7, recency: 0.1, utility: 0.2 }, jitter: 0.05 },
-  explore: { weights: { relevance: 0.4, recency: 0.3, utility: 0.3 }, jitter: 0.15 },
+  continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.01 },
+  fact_check: { weights: { relevance: 0.6, recency: 0.1, utility: 0.3 }, jitter: 0.01 },
+  frequent: { weights: { relevance: 0.2, recency: 0.2, utility: 0.6 }, jitter: 0.01 },
+  associative: { weights: { relevance: 0.7, recency: 0.1, utility: 0.2 }, jitter: 0.025 },
+  explore: { weights: { relevance: 0.4, recency: 0.3, utility: 0.3 }, jitter: 0.08 },
 };
 
 const sigmoid = (x: number): number => 1 / (1 + Math.exp(-x));
@@ -87,19 +89,10 @@ export class MemoryService {
   async getMultiple(ids: string[]): Promise<Memory[]> {
     if (ids.length === 0) return [];
     const memories = await this.repository.findByIds(ids);
-    // Track access in bulk
     const now = new Date();
-    const live = memories.filter((m) => !isDeleted(m));
-    await Promise.all(
-      live.map((m) =>
-        this.repository.upsert({
-          ...m,
-          accessCount: m.accessCount + 1,
-          lastAccessed: now,
-        })
-      )
-    );
-    return live;
+    const liveIds = memories.filter((m) => !isDeleted(m)).map((m) => m.id);
+    this.repository.bulkUpdateAccess(liveIds, now);
+    return memories.filter((m) => !isDeleted(m));
   }
 
   async delete(id: string): Promise<boolean> {
@@ -186,10 +179,10 @@ export class MemoryService {
   async search(
     query: string,
     intent: SearchIntent,
-    limit: number = 10,
-    includeDeleted: boolean = false,
     options?: SearchOptions
   ): Promise<SearchResult[]> {
+    const limit = options?.limit ?? 10;
+    const includeDeleted = options?.includeDeleted ?? false;
     const queryEmbedding = await this.embeddings.embed(query);
     const profile = INTENT_PROFILES[intent];
     const now = new Date();
@@ -223,6 +216,7 @@ export class MemoryService {
                   updatedAt: candidate.updatedAt,
                   source: "memory" as const,
                   score: this.computeMemoryScore(candidate, profile, now),
+                  confidence: computeConfidence(candidate.signals),
                   supersededBy: candidate.supersededBy,
                   usefulness: candidate.usefulness,
                   accessCount: candidate.accessCount,
@@ -249,6 +243,7 @@ export class MemoryService {
                 updatedAt: row.createdAt,
                 source: "conversation_history" as const,
                 score: row.rrfScore * historyWeight,
+                confidence: computeConfidence(row.signals),
                 supersededBy: null,
                 sessionId: (row.metadata?.session_id as string) ?? "",
                 role: (row.metadata?.role as string) ?? "unknown",
@@ -272,19 +267,7 @@ export class MemoryService {
 
   async trackAccess(ids: string[]): Promise<void> {
     if (ids.length === 0) return;
-    const memories = await this.repository.findByIds(ids);
-    const now = new Date();
-    await Promise.all(
-      memories
-        .filter((m) => !isDeleted(m))
-        .map((m) =>
-          this.repository.upsert({
-            ...m,
-            accessCount: m.accessCount + 1,
-            lastAccessed: now,
-          })
-        )
-    );
+    this.repository.bulkUpdateAccess(ids, new Date());
   }
 
   private static readonly UUID_ZERO =
@@ -292,8 +275,16 @@ export class MemoryService {
 
   private static waypointId(project?: string): string {
     if (!project?.length) return MemoryService.UUID_ZERO;
-    const hex = createHash("sha256").update(`waypoint:${project}`).digest("hex");
-    // Format as UUID: 8-4-4-4-12
+    const normalized = project.trim().toLowerCase();
+    const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
+    return `wp:${hex.slice(0, 32)}`;
+  }
+
+  /** Legacy UUID-formatted waypoint ID for migration fallback reads. */
+  private static legacyWaypointId(project?: string): string | null {
+    if (!project?.length) return null; // UUID_ZERO is still current for no-project
+    const normalized = project.trim().toLowerCase();
+    const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
     return [
       hex.slice(0, 8),
       hex.slice(8, 12),
@@ -386,6 +377,20 @@ ${list(args.memory_ids)}`;
   }
 
   async getLatestWaypoint(project?: string): Promise<Memory | null> {
-    return await this.get(MemoryService.waypointId(project));
+    const waypoint = await this.get(MemoryService.waypointId(project));
+    if (waypoint) return waypoint;
+
+    // Fallback: try legacy UUID-formatted waypoint ID and migrate on read
+    const legacyId = MemoryService.legacyWaypointId(project);
+    if (!legacyId) return null;
+
+    const legacy = await this.repository.findById(legacyId);
+    if (!legacy) return null;
+
+    // Migrate: write under new ID, delete old
+    const newId = MemoryService.waypointId(project);
+    await this.repository.upsert({ ...legacy, id: newId });
+    await this.repository.markDeleted(legacyId);
+    return { ...legacy, id: newId };
   }
 }

package/server/core/memory.ts

@@ -38,7 +38,46 @@ export interface IntentProfile {
   jitter: number;
 }
 
+/** Signals preserved from the hybrid search pipeline for confidence scoring. */
+export interface SearchSignals {
+  cosineSimilarity: number | null;
+  ftsMatch: boolean;
+  knnRank: number | null;
+  ftsRank: number | null;
+}
+
 /** Augments any entity type with an RRF score from hybrid search. */
-export type WithRrfScore<T> = T & { rrfScore: number };
+export type WithRrfScore<T> = T & { rrfScore: number; signals: SearchSignals };
 
 export type HybridRow = WithRrfScore<Memory>;
+
+/**
+ * Compute absolute confidence (0-1) from search signals.
+ *
+ * Based primarily on cosine similarity (the strongest absolute signal)
+ * mapped through a sigmoid with an agreement bonus for dual-path matches.
+ * The midpoint and steepness are calibrated for all-MiniLM-L6-v2 embeddings.
+ */
+// Calibrated against all-MiniLM-L6-v2: noise ceiling ~0.25, weak-relevant floor ~0.30
+const CONFIDENCE_STEEPNESS = 14;
+const CONFIDENCE_MIDPOINT = 0.35;
+const CONFIDENCE_AGREEMENT_BONUS = 0.08;
+
+export function computeConfidence(signals: SearchSignals): number {
+  const sim = signals.cosineSimilarity;
+
+  if (sim === null) {
+    // FTS-only result — keyword match but no semantic confirmation
+    return signals.ftsMatch ? 0.40 : 0.0;
+  }
+
+  // Shifted sigmoid: maps cosine similarity to interpretable confidence
+  let confidence = 1 / (1 + Math.exp(-CONFIDENCE_STEEPNESS * (sim - CONFIDENCE_MIDPOINT)));
+
+  // Dual-path agreement bonus: found by both KNN and FTS
+  if (signals.ftsMatch) {
+    confidence = Math.min(1.0, confidence + CONFIDENCE_AGREEMENT_BONUS);
+  }
+
+  return confidence;
+}

package/server/core/migration.service.ts

@@ -3,9 +3,9 @@ import { createHash } from "crypto";
 import { existsSync, statSync, readdirSync } from "fs";
 import { resolve, dirname } from "path";
 import { fileURLToPath } from "url";
-import { serializeVector } from "./sqlite-utils.js";
-import type { MemoryRepository } from "./memory.repository.js";
-import type { EmbeddingsService } from "./embeddings.service.js";
+import { serializeVector } from "./sqlite-utils";
+import type { MemoryRepository } from "./memory.repository";
+import type { EmbeddingsService } from "./embeddings.service";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 

package/server/core/migrations.ts

@@ -1,6 +1,6 @@
 import type { Database } from "bun:sqlite";
-import type { EmbeddingsService } from "./embeddings.service.js";
-import { serializeVector } from "./sqlite-utils.js";
+import type { EmbeddingsService } from "./embeddings.service";
+import { serializeVector } from "./sqlite-utils";
 
 /**
  * Pre-migration step: remove vec0 virtual table entries from sqlite_master
@@ -127,41 +127,23 @@ export async function backfillVectors(
   db: Database,
   embeddings: EmbeddingsService,
 ): Promise<void> {
-  // Fast sentinel check: skip the LEFT JOIN queries entirely when backfill is done
-  const sentinel = db
-    .prepare("SELECT 1 FROM memories_vec LIMIT 1")
-    .get();
-  const memoriesExist = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
-  const convosExist = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
+  // Quick gap check: if no rows are missing vectors, skip the expensive backfill
+  const hasMemories = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
+  const hasConvos = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
 
-  // If vec tables have data and source tables have data, backfill is likely complete.
-  // Only run the expensive LEFT JOIN when there's reason to suspect gaps.
-  const convoSentinel = db
-    .prepare("SELECT 1 FROM conversation_history_vec LIMIT 1")
-    .get();
-  const mayNeedMemoryBackfill = memoriesExist && !sentinel;
-  const mayNeedConvoBackfill = convosExist && !convoSentinel;
+  if (!hasMemories && !hasConvos) return;
 
-  // If both vec tables are populated, do a quick count check to confirm
-  if (!mayNeedMemoryBackfill && !mayNeedConvoBackfill) {
-    if (memoriesExist) {
-      const gap = db.prepare(
-        `SELECT 1 FROM memories m LEFT JOIN memories_vec v ON m.id = v.id
-         WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
-      ).get();
-      if (!gap && convosExist) {
-        const convoGap = db.prepare(
-          `SELECT 1 FROM conversation_history c LEFT JOIN conversation_history_vec v ON c.id = v.id
-           WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
-        ).get();
-        if (!convoGap) return;
-      } else if (!gap && !convosExist) {
-        return;
-      }
-    } else {
-      return; // No data at all
-    }
-  }
+  const memoryGap = hasMemories && db.prepare(
+    `SELECT 1 FROM memories m LEFT JOIN memories_vec v ON m.id = v.id
+     WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
+  ).get();
+
+  const convoGap = hasConvos && db.prepare(
+    `SELECT 1 FROM conversation_history c LEFT JOIN conversation_history_vec v ON c.id = v.id
+     WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
+  ).get();
+
+  if (!memoryGap && !convoGap) return;
 
   // ── Memories ──────────────────────────────────────────────────────
   const missingMemories = db

package/server/core/parsers/claude-code.parser.ts

@@ -1,7 +1,7 @@
 import { readFile, readdir, stat } from "fs/promises";
 import { basename, dirname, join } from "path";
-import type { ParsedMessage, SessionFileInfo } from "../conversation.js";
-import type { SessionLogParser } from "./types.js";
+import type { ParsedMessage, SessionFileInfo } from "../conversation";
+import type { SessionLogParser } from "./types";
 
 // UUID pattern for session IDs
 const UUID_PATTERN =
@@ -45,7 +45,7 @@ export class ClaudeCodeSessionParser implements SessionLogParser {
     const fileName = basename(filePath, ".jsonl");
     const parentDir = basename(dirname(filePath));
     // Check if this is inside a subagents directory
-    const isSubagentFile = filePath.includes("/subagents/");
+    const isSubagentFile = /[/\\]subagents[/\\]/.test(filePath);
 
     // For subagent files, project dir is 3 levels up: <project>/<session>/subagents/<file>
     // For main files, project dir is direct parent

package/server/core/parsers/types.ts

@@ -1,4 +1,4 @@
-import type { ParsedMessage, SessionFileInfo } from "../conversation.js";
+import type { ParsedMessage, SessionFileInfo } from "../conversation";
 
 /** Interface for parsing session log files into structured messages */
 export interface SessionLogParser {

package/server/core/sqlite-utils.ts

@@ -1,7 +1,29 @@
 import type { Database } from "bun:sqlite";
 
-/** RRF constant matching the previous LanceDB reranker default */
-export const RRF_K = 60;
+/** RRF constant — lower K gives sharper top-rank discrimination in the 1/(K+rank) formula */
+export const RRF_K = 10;
+
+/**
+ * Maximum parameters per SQLite query to stay within SQLITE_MAX_VARIABLE_NUMBER.
+ */
+export const SQLITE_BATCH_SIZE = 100;
+
+/**
+ * Execute a query in batches when the number of parameters exceeds SQLITE_BATCH_SIZE.
+ * Splits the ids array and concatenates results.
+ */
+export function batchedQuery<T>(
+  db: Database,
+  ids: string[],
+  queryFn: (batch: string[]) => T[]
+): T[] {
+  if (ids.length <= SQLITE_BATCH_SIZE) return queryFn(ids);
+  const results: T[] = [];
+  for (let i = 0; i < ids.length; i += SQLITE_BATCH_SIZE) {
+    results.push(...queryFn(ids.slice(i, i + SQLITE_BATCH_SIZE)));
+  }
+  return results;
+}
 
 /**
  * Serialize a number[] embedding to raw float32 bytes for BLOB storage.
@@ -94,6 +116,49 @@ export function hybridRRF(
   return scores;
 }
 
+import type { SearchSignals } from "./memory";
+
+/**
+ * Compute hybrid RRF scores while preserving per-result search signals
+ * (cosine similarity, FTS match, rank positions) for confidence scoring.
+ */
+export function hybridRRFWithSignals(
+  vectorResults: Array<{ id: string; distance: number }>,
+  ftsResults: Array<{ id: string }>,
+  k: number = RRF_K
+): Map<string, SearchSignals & { rrfScore: number }> {
+  const knnMap = new Map<string, { similarity: number; rank: number }>();
+  vectorResults.forEach((r, i) => {
+    knnMap.set(r.id, { similarity: 1 - r.distance, rank: i + 1 });
+  });
+
+  const ftsMap = new Map<string, number>();
+  ftsResults.forEach((r, i) => {
+    ftsMap.set(r.id, i + 1);
+  });
+
+  const allIds = new Set([...knnMap.keys(), ...ftsMap.keys()]);
+  const results = new Map<string, SearchSignals & { rrfScore: number }>();
+
+  for (const id of allIds) {
+    const knn = knnMap.get(id);
+    const ftsRank = ftsMap.get(id) ?? null;
+    let rrfScore = 0;
+    if (knn) rrfScore += 1 / (k + knn.rank);
+    if (ftsRank !== null) rrfScore += 1 / (k + ftsRank);
+
+    results.set(id, {
+      rrfScore,
+      cosineSimilarity: knn?.similarity ?? null,
+      ftsMatch: ftsRank !== null,
+      knnRank: knn?.rank ?? null,
+      ftsRank,
+    });
+  }
+
+  return results;
+}
+
 /**
  * Sort ids by RRF score descending and return top N.
  */

package/server/index.ts

@@ -1,15 +1,15 @@
 #!/usr/bin/env bun
 
-import { loadConfig, parseCliArgs } from "./config/index.js";
-import { connectToDatabase } from "./core/connection.js";
-import { backfillVectors } from "./core/migrations.js";
-import { MemoryRepository } from "./core/memory.repository.js";
-import { ConversationRepository } from "./core/conversation.repository.js";
-import { EmbeddingsService } from "./core/embeddings.service.js";
-import { MemoryService } from "./core/memory.service.js";
-import { ConversationHistoryService } from "./core/conversation.service.js";
-import { startServer } from "./transports/mcp/server.js";
-import { startHttpServer } from "./transports/http/server.js";
+import { loadConfig, parseCliArgs } from "./config/index";
+import { connectToDatabase } from "./core/connection";
+import { backfillVectors } from "./core/migrations";
+import { MemoryRepository } from "./core/memory.repository";
+import { ConversationRepository } from "./core/conversation.repository";
+import { EmbeddingsService } from "./core/embeddings.service";
+import { MemoryService } from "./core/memory.service";
+import { ConversationHistoryService } from "./core/conversation.service";
+import { startServer } from "./transports/mcp/server";
+import { startHttpServer } from "./transports/http/server";
 
 async function main(): Promise<void> {
   const args = process.argv.slice(2);

package/server/transports/http/mcp-transport.ts

@@ -21,11 +21,11 @@ import {
 } from "@modelcontextprotocol/sdk/types.js";
 import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
 
-import { tools } from "../mcp/tools.js";
-import { handleToolCall } from "../mcp/handlers.js";
-import { SERVER_INSTRUCTIONS } from "../mcp/server.js";
-import { VERSION } from "../../config/index.js";
-import type { MemoryService } from "../../core/memory.service.js";
+import { tools } from "../mcp/tools";
+import { handleToolCall } from "../mcp/handlers";
+import { SERVER_INSTRUCTIONS } from "../mcp/server";
+import { VERSION } from "../../config/index";
+import type { MemoryService } from "../../core/memory.service";
 
 interface Session {
   server: Server;

package/server/transports/http/server.ts

@@ -3,11 +3,11 @@ import { cors } from "hono/cors";
 import { createServer } from "net";
 import { writeFileSync, mkdirSync, unlinkSync } from "fs";
 import { join } from "path";
-import type { MemoryService } from "../../core/memory.service.js";
-import type { Config } from "../../config/index.js";
-import { isDeleted } from "../../core/memory.js";
-import { createMcpRoutes } from "./mcp-transport.js";
-import type { Memory, SearchIntent } from "../../core/memory.js";
+import type { MemoryService } from "../../core/memory.service";
+import type { Config } from "../../config/index";
+import { isDeleted } from "../../core/memory";
+import { createMcpRoutes } from "./mcp-transport";
+import type { Memory, SearchIntent } from "../../core/memory";
 
 
 /**
@@ -139,7 +139,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
         return c.json({ error: "Missing or invalid 'query' field" }, 400);
       }
 
-      const results = await memoryService.search(query, intent, limit);
+      const results = await memoryService.search(query, intent, { limit });
 
       return c.json({
         results: results.map((r) => ({
@@ -147,6 +147,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
           content: r.content,
           metadata: r.metadata,
           source: r.source,
+          confidence: r.confidence,
           createdAt: r.createdAt.toISOString(),
         })),
         count: results.length,

package/server/transports/mcp/handlers.ts

@@ -1,9 +1,9 @@
 import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
-import type { MemoryService } from "../../core/memory.service.js";
-import type { ConversationHistoryService } from "../../core/conversation.service.js";
-import type { SearchIntent } from "../../core/memory.js";
-import type { HistoryFilters, SearchResult } from "../../core/conversation.js";
-import { DEBUG } from "../../config/index.js";
+import type { MemoryService } from "../../core/memory.service";
+import type { ConversationHistoryService } from "../../core/conversation.service";
+import type { SearchIntent } from "../../core/memory";
+import type { HistoryFilters, SearchResult } from "../../core/conversation";
+import { DEBUG } from "../../config/index";
 
 /**
  * Safely coerce a tool argument to an array. Handles the case where the MCP
@@ -51,6 +51,14 @@ function parseDate(value: unknown, fieldName: string): Date | undefined {
   return date;
 }
 
+function requireString(args: Record<string, unknown> | undefined, field: string): string {
+  const value = args?.[field];
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${field} is required`);
+  }
+  return value;
+}
+
 export async function handleStoreMemories(
   args: Record<string, unknown> | undefined,
   service: MemoryService
@@ -189,7 +197,9 @@ export async function handleSearchMemories(
     return errorResult(errorText(e));
   }
 
-  const results = await service.search(query, intent, limit, includeDeleted, {
+  const results = await service.search(query, intent, {
+    limit,
+    includeDeleted,
     includeHistory,
     historyOnly,
     historyFilters,
@@ -202,19 +212,7 @@ export async function handleSearchMemories(
     };
   }
 
-  const formatted = results.map((r: SearchResult) => {
-    let result = `[${r.source}] ID: ${r.id}\nContent: ${r.content}`;
-    if (r.metadata && Object.keys(r.metadata).length > 0) {
-      result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
-    }
-    if (r.source === "memory" && includeDeleted && r.supersededBy) {
-      result += `\n[DELETED]`;
-    }
-    if (r.source === "conversation_history" && r.sessionId) {
-      result += `\nSession: ${r.sessionId}`;
-    }
-    return result;
-  });
+  const formatted = results.map((r) => formatSearchResult(r, includeDeleted));
 
   return {
     content: [{ type: "text", text: formatted.join("\n\n---\n\n") }],
@@ -241,6 +239,20 @@ function formatMemoryDetail(
   return result;
 }
 
+function formatSearchResult(r: SearchResult, includeDeleted: boolean): string {
+  let result = `[${r.source}] ID: ${r.id}\nConfidence: ${r.confidence.toFixed(2)}\nContent: ${r.content}`;
+  if (r.metadata && Object.keys(r.metadata).length > 0) {
+    result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
+  }
+  if (r.source === "memory" && includeDeleted && r.supersededBy) {
+    result += `\n[DELETED]`;
+  }
+  if (r.source === "conversation_history" && r.sessionId) {
+    result += `\nSession: ${r.sessionId}`;
+  }
+  return result;
+}
+
 export async function handleGetMemories(
   args: Record<string, unknown> | undefined,
   service: MemoryService
@@ -267,8 +279,11 @@ export async function handleReportMemoryUsefulness(
   args: Record<string, unknown> | undefined,
   service: MemoryService
 ): Promise<CallToolResult> {
-  const memoryId = args?.memory_id as string;
-  const useful = args?.useful as boolean;
+  const memoryId = requireString(args, "memory_id");
+  const useful = args?.useful;
+  if (typeof useful !== "boolean") {
+    return errorResult("useful is required and must be a boolean");
+  }
 
   const memory = await service.vote(memoryId, useful ? 1 : -1);
 
@@ -290,10 +305,19 @@ export async function handleSetWaypoint(
   args: Record<string, unknown> | undefined,
   service: MemoryService
 ): Promise<CallToolResult> {
+  let project: string;
+  let summary: string;
+  try {
+    project = requireString(args, "project");
+    summary = requireString(args, "summary");
+  } catch (e) {
+    return errorResult(errorText(e));
+  }
+
   const memory = await service.setWaypoint({
-    project: args?.project as string,
+    project,
     branch: args?.branch as string | undefined,
-    summary: args?.summary as string,
+    summary,
     completed: (args?.completed as string[] | undefined) ?? [],
     in_progress_blocked: (args?.in_progress_blocked as string[] | undefined) ?? [],
     key_decisions: (args?.key_decisions as string[] | undefined) ?? [],

package/server/transports/mcp/server.ts

@@ -6,12 +6,12 @@ import {
   ListResourcesRequestSchema,
   ReadResourceRequestSchema,
 } from "@modelcontextprotocol/sdk/types.js";
-import { resources, readResource } from "./resources.js";
+import { resources, readResource } from "./resources";
 
-import { tools } from "./tools.js";
-import { handleToolCall } from "./handlers.js";
-import type { MemoryService } from "../../core/memory.service.js";
-import { VERSION } from "../../config/index.js";
+import { tools } from "./tools";
+import { handleToolCall } from "./handlers";
+import type { MemoryService } from "../../core/memory.service";
+import { VERSION } from "../../config/index";
 
 export const SERVER_INSTRUCTIONS = `This server is the user's canonical memory system. It provides persistent, semantic vector memory that survives across conversations and sessions.