@memrosetta/core 0.2.3 → 0.2.5

package/dist/index.d.ts

@@ -4,6 +4,7 @@ import { Embedder, ContradictionDetector } from '@memrosetta/embeddings';
 
 interface SchemaOptions {
     readonly vectorEnabled?: boolean;
+    readonly embeddingDimension?: number;
 }
 declare function ensureSchema(db: Database.Database, options?: SchemaOptions): void;
 
@@ -143,6 +144,22 @@ declare function rrfMerge(ftsResults: readonly {
     readonly memory: Memory;
     readonly rank: number;
 }[], k?: number, limit?: number): readonly SearchResult[];
+/**
+ * Weighted RRF merge: FTS results get higher weight than vector results.
+ *
+ * ftsWeight=2.0 means FTS rank contributes 2x more than vector rank.
+ * This preserves FTS precision while allowing vector to boost semantically
+ * similar results that FTS might rank lower.
+ *
+ * Items found by both FTS and vector get the combined score (strongest signal).
+ */
+declare function rrfMergeWeighted(ftsResults: readonly {
+    readonly memory: Memory;
+    readonly rank: number;
+}[], vecResults: readonly {
+    readonly memory: Memory;
+    readonly rank: number;
+}[], k?: number, limit?: number, ftsWeight?: number, vecWeight?: number): readonly SearchResult[];
 /**
  * Remove duplicate search results based on content identity.
  * Keeps the first (highest-scored) occurrence when duplicates exist.
@@ -260,4 +277,4 @@ declare function determineTier(memory: {
  */
 declare function estimateTokens(content: string): number;
 
-export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };
+export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, rrfMergeWeighted, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };

package/dist/index.js

@@ -72,11 +72,9 @@ CREATE TRIGGER memories_au AFTER UPDATE ON memories BEGIN
   INSERT INTO memories_fts(rowid, content, keywords) VALUES (new.id, new.content, new.keywords);
 END;
 `;
-var SCHEMA_V2 = `
-CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(
-  embedding float[384]
-);
-`;
+function schemaV2(dim) {
+  return `CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(embedding float[${dim}]);`;
+}
 var SCHEMA_V3 = `
 ALTER TABLE memories ADD COLUMN event_date_start TEXT;
 ALTER TABLE memories ADD COLUMN event_date_end TEXT;
@@ -99,16 +97,17 @@ function ensureSchema(db, options) {
   const hasVersionTable = db.prepare(
     "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'"
   ).get();
+  const dim = options?.embeddingDimension ?? 384;
   if (!hasVersionTable) {
-    db.exec("CREATE TABLE schema_version (version INTEGER NOT NULL)");
+    db.exec("CREATE TABLE schema_version (version INTEGER NOT NULL, embedding_dimension INTEGER DEFAULT 384)");
     db.exec(SCHEMA_V1);
     let version = 1;
     if (options?.vectorEnabled) {
-      db.exec(SCHEMA_V2);
+      db.exec(schemaV2(dim));
       version = 2;
     }
     version = 4;
-    db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(version);
+    db.prepare("INSERT INTO schema_version (version, embedding_dimension) VALUES (?, ?)").run(version, dim);
     return;
   }
   const row = db.prepare("SELECT version FROM schema_version").get();
@@ -118,7 +117,7 @@ function ensureSchema(db, options) {
     db.prepare("UPDATE schema_version SET version = ?").run(1);
   }
   if (currentVersion < 2 && options?.vectorEnabled) {
-    db.exec(SCHEMA_V2);
+    db.exec(schemaV2(dim));
     db.prepare("UPDATE schema_version SET version = ?").run(2);
   }
   if (currentVersion < 3) {
@@ -133,6 +132,39 @@ function ensureSchema(db, options) {
     }
     db.prepare("UPDATE schema_version SET version = ?").run(4);
   }
+  if (options?.vectorEnabled) {
+    const hasVecTable = db.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='vec_memories'"
+    ).get();
+    if (!hasVecTable) {
+      try {
+        db.exec(schemaV2(dim));
+      } catch {
+        process.stderr.write("[memrosetta] sqlite-vec not available, vector search disabled\n");
+      }
+    }
+    const hasDimCol = db.prepare("PRAGMA table_info(schema_version)").all().some((col) => col.name === "embedding_dimension");
+    if (!hasDimCol) {
+      db.exec("ALTER TABLE schema_version ADD COLUMN embedding_dimension INTEGER DEFAULT 384");
+    }
+    const storedDim = db.prepare("SELECT embedding_dimension FROM schema_version").get()?.embedding_dimension ?? 384;
+    if (storedDim !== dim) {
+      process.stderr.write(
+        `[memrosetta] Embedding dimension changed (${storedDim} -> ${dim}). Recreating vector index...
+`
+      );
+      try {
+        db.exec("DROP TABLE IF EXISTS vec_memories");
+      } catch {
+      }
+      try {
+        db.exec(schemaV2(dim));
+        db.prepare("UPDATE schema_version SET embedding_dimension = ?").run(dim);
+      } catch {
+        process.stderr.write("[memrosetta] sqlite-vec not available, vector search disabled\n");
+      }
+    }
+  }
 }
 
 // src/utils.ts
@@ -789,6 +821,40 @@ function rrfMerge(ftsResults, vecResults, k = 20, limit = 10) {
     matchType: "hybrid"
   }));
 }
+function rrfMergeWeighted(ftsResults, vecResults, k = 20, limit = 10, ftsWeight = 2, vecWeight = 1) {
+  const scores = /* @__PURE__ */ new Map();
+  for (let i = 0; i < ftsResults.length; i++) {
+    const item = ftsResults[i];
+    const contribution = ftsWeight / (k + i + 1);
+    const existing = scores.get(item.memory.memoryId);
+    if (existing) {
+      existing.score += contribution;
+    } else {
+      scores.set(item.memory.memoryId, {
+        score: contribution,
+        memory: item.memory
+      });
+    }
+  }
+  for (let i = 0; i < vecResults.length; i++) {
+    const item = vecResults[i];
+    const contribution = vecWeight / (k + i + 1);
+    const existing = scores.get(item.memory.memoryId);
+    if (existing) {
+      existing.score += contribution;
+    } else {
+      scores.set(item.memory.memoryId, {
+        score: contribution,
+        memory: item.memory
+      });
+    }
+  }
+  return [...scores.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, limit).map(([, { score, memory }]) => ({
+    memory,
+    score,
+    matchType: "hybrid"
+  }));
+}
 function cosineSimilarity(a, b) {
   let dot = 0;
   let normA = 0;
@@ -897,12 +963,35 @@ function searchMemories(db, query, queryVec, useVecTable = true) {
     const boosted = applyKeywordBoost(weighted, queryTokens);
     finalResults = deduplicateResults(boosted);
   } else {
-    const ftsRanked = ftsResults.map((r, i) => ({ memory: r.memory, rank: i }));
-    const vecRanked = vecResults.map((r, i) => ({ memory: r.memory, rank: i }));
-    const merged = rrfMerge(ftsRanked, vecRanked, 20, query.limit ?? DEFAULT_LIMIT);
-    const weighted = applyActivationWeighting(merged);
-    const boosted = applyKeywordBoost(weighted, queryTokens);
-    finalResults = deduplicateResults(boosted);
+    const limit = query.limit ?? DEFAULT_LIMIT;
+    const ftsHasEnough = ftsResults.length >= limit;
+    if (ftsHasEnough) {
+      const vecIds = new Set(vecResults.map((r) => r.memory.memoryId));
+      const reranked = ftsResults.map((r) => ({
+        ...r,
+        score: r.score * (vecIds.has(r.memory.memoryId) ? 1.3 : 1)
+      }));
+      reranked.sort((a, b) => b.score - a.score);
+      const weighted = applyActivationWeighting(reranked);
+      const boosted = applyKeywordBoost(weighted, queryTokens);
+      finalResults = deduplicateResults(boosted);
+    } else {
+      const ftsIds = new Set(ftsResults.map((r) => r.memory.memoryId));
+      const ftsItems = [...ftsResults];
+      for (const vr of vecResults) {
+        if (ftsItems.length >= limit) break;
+        if (ftsIds.has(vr.memory.memoryId)) continue;
+        ftsItems.push({
+          memory: vr.memory,
+          score: (1 - vr.distance) * 0.5,
+          // Lower score than FTS items
+          matchType: "hybrid"
+        });
+      }
+      const weighted = applyActivationWeighting(ftsItems);
+      const boosted = applyKeywordBoost(weighted, queryTokens);
+      finalResults = deduplicateResults(boosted);
+    }
   }
   updateAccessTracking(db, finalResults.map((r) => r.memory.memoryId));
   return {
@@ -987,7 +1076,10 @@ var SqliteMemoryEngine = class {
         this.vectorEnabled = false;
       }
     }
-    ensureSchema(this.db, { vectorEnabled: this.vectorEnabled });
+    ensureSchema(this.db, {
+      vectorEnabled: this.vectorEnabled,
+      embeddingDimension: this.options.embedder?.dimension ?? 384
+    });
     this.stmts = createPreparedStatements(this.db);
     this.relStmts = createRelationStatements(this.db);
   }
@@ -1347,6 +1439,7 @@ export {
   nowIso,
   rowToMemory,
   rrfMerge,
+  rrfMergeWeighted,
   searchMemories,
   serializeEmbedding,
   storeBatchAsync,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memrosetta/core",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -8,7 +8,8 @@
     ".": {
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
-    }
+    },
+    "./package.json": "./package.json"
   },
   "files": [
     "dist",
@@ -19,10 +20,10 @@
     "better-sqlite3": "^11.0.0",
     "nanoid": "^5.0.0",
     "sqlite-vec": "^0.1.7",
-    "@memrosetta/types": "0.2.3"
+    "@memrosetta/types": "0.2.5"
   },
   "optionalDependencies": {
-    "@memrosetta/embeddings": "0.2.3"
+    "@memrosetta/embeddings": "0.2.5"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.0",