@memrosetta/core 0.2.19 → 0.2.20

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import Database from 'better-sqlite3';
-import { Memory, MemoryInput, RelationType, MemoryRelation, SearchResult, SearchFilters, SearchQuery, SearchResponse, IMemoryEngine, CompressResult, MaintenanceResult, MemoryTier, TierConfig } from '@memrosetta/types';
+import { Memory, MemoryState, MemoryInput, RelationType, MemoryRelation, SearchResult, SearchFilters, SearchQuery, SearchResponse, IMemoryEngine, CompressResult, MaintenanceResult, MemoryTier, MemoryQuality, TierConfig } from '@memrosetta/types';
 import { Embedder, ContradictionDetector } from '@memrosetta/embeddings';
 
 interface SchemaOptions {
@@ -12,6 +12,13 @@ declare function generateMemoryId(): string;
 declare function nowIso(): string;
 declare function keywordsToString(keywords: readonly string[] | undefined): string | null;
 declare function stringToKeywords(str: string | null): readonly string[];
+/**
+ * Derive the logical state of a memory from its existing fields.
+ * - invalidated: invalidatedAt is set (takes precedence)
+ * - superseded: isLatest is false
+ * - current: isLatest is true and not invalidated
+ */
+declare function deriveMemoryState(memory: Memory): MemoryState;
 
 interface MemoryRow {
     readonly id: number;
@@ -204,7 +211,7 @@ declare function updateAccessTracking(db: Database.Database, memoryIds: readonly
  *
  * Results are weighted by activation score and access tracking is updated.
  */
-declare function searchMemories(db: Database.Database, query: SearchQuery, queryVec?: Float32Array, useVecTable?: boolean): SearchResponse;
+declare function searchMemories(db: Database.Database, query: SearchQuery, queryVec?: Float32Array, useVecTable?: boolean, skipAccessTracking?: boolean): SearchResponse;
 
 interface SqliteEngineOptions {
     readonly dbPath: string;
@@ -235,6 +242,7 @@ declare class SqliteMemoryEngine implements IMemoryEngine {
     compress(userId: string): Promise<CompressResult>;
     maintain(userId: string): Promise<MaintenanceResult>;
     setTier(memoryId: string, tier: MemoryTier): Promise<void>;
+    quality(userId: string): Promise<MemoryQuality>;
     /**
      * Check the newly stored memory against existing similar memories
      * for contradictions using the NLI model.
@@ -254,6 +262,15 @@ declare class SqliteMemoryEngine implements IMemoryEngine {
      * Only runs for single store() calls, not storeBatch() (too slow for bulk).
      */
     private checkDuplicates;
+    /**
+     * Auto-create 'extends' relations when a new memory shares keywords
+     * with existing memories. Builds graph density for future graph-based retrieval.
+     *
+     * Only runs for individual store() calls, not storeBatch().
+     * Requires at least 2 overlapping keywords to create a relation.
+     * Graceful: errors are silently swallowed.
+     */
+    private autoRelate;
     private ensureInitialized;
 }
 declare function createEngine(options: SqliteEngineOptions): SqliteMemoryEngine;
@@ -311,4 +328,4 @@ declare function determineTier(memory: {
  */
 declare function estimateTokens(content: string): number;
 
-export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, applyThreeFactorReranking, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, computeEbbinghaus, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, rrfMergeWeighted, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };
+export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, applyThreeFactorReranking, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, computeEbbinghaus, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, deriveMemoryState, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, rrfMergeWeighted, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };

package/dist/index.js

@@ -183,6 +183,11 @@ function stringToKeywords(str) {
   if (!str || str.trim() === "") return [];
   return str.split(" ").filter((s) => s.length > 0);
 }
+function deriveMemoryState(memory) {
+  if (memory.invalidatedAt) return "invalidated";
+  if (!memory.isLatest) return "superseded";
+  return "current";
+}
 
 // src/mapper.ts
 function rowToMemory(row) {
@@ -608,9 +613,29 @@ function buildSearchSql(query) {
     whereClauses.push("m.confidence >= ?");
     params.push(filters.minConfidence);
   }
-  const onlyLatest = filters?.onlyLatest ?? true;
-  if (onlyLatest) {
-    whereClauses.push("m.is_latest = 1");
+  if (filters?.states && filters.states.length > 0) {
+    const stateConditions = [];
+    if (filters.states.includes("current")) {
+      stateConditions.push("(m.is_latest = 1 AND m.invalidated_at IS NULL)");
+    }
+    if (filters.states.includes("superseded")) {
+      stateConditions.push("(m.is_latest = 0)");
+    }
+    if (filters.states.includes("invalidated")) {
+      stateConditions.push("(m.invalidated_at IS NOT NULL)");
+    }
+    if (stateConditions.length > 0) {
+      whereClauses.push(`(${stateConditions.join(" OR ")})`);
+    }
+  } else {
+    const onlyLatest = filters?.onlyLatest ?? true;
+    if (onlyLatest) {
+      whereClauses.push("m.is_latest = 1");
+    }
+    const excludeInvalidated = filters?.excludeInvalidated ?? true;
+    if (excludeInvalidated) {
+      whereClauses.push("m.invalidated_at IS NULL");
+    }
   }
   if (filters?.eventDateRange?.start) {
     whereClauses.push("m.event_date_start >= ?");
@@ -620,10 +645,6 @@ function buildSearchSql(query) {
     whereClauses.push("m.event_date_end <= ?");
     params.push(filters.eventDateRange.end);
   }
-  const excludeInvalidated = filters?.excludeInvalidated ?? true;
-  if (excludeInvalidated) {
-    whereClauses.push("m.invalidated_at IS NULL");
-  }
   const limit = query.limit ?? DEFAULT_LIMIT;
   params.push(limit);
   const sql = [
@@ -676,9 +697,29 @@ function ftsSearch(db, query) {
 function bruteForceVectorSearch(db, queryVec, userId, limit, filters) {
   const whereClauses = ["user_id = ?", "embedding IS NOT NULL"];
   const params = [userId];
-  const onlyLatest = filters?.onlyLatest ?? true;
-  if (onlyLatest) {
-    whereClauses.push("is_latest = 1");
+  if (filters?.states && filters.states.length > 0) {
+    const stateConditions = [];
+    if (filters.states.includes("current")) {
+      stateConditions.push("(is_latest = 1 AND invalidated_at IS NULL)");
+    }
+    if (filters.states.includes("superseded")) {
+      stateConditions.push("(is_latest = 0)");
+    }
+    if (filters.states.includes("invalidated")) {
+      stateConditions.push("(invalidated_at IS NOT NULL)");
+    }
+    if (stateConditions.length > 0) {
+      whereClauses.push(`(${stateConditions.join(" OR ")})`);
+    }
+  } else {
+    const onlyLatest = filters?.onlyLatest ?? true;
+    if (onlyLatest) {
+      whereClauses.push("is_latest = 1");
+    }
+    const excludeInvalidated = filters?.excludeInvalidated ?? true;
+    if (excludeInvalidated) {
+      whereClauses.push("invalidated_at IS NULL");
+    }
   }
   if (filters?.memoryTypes && filters.memoryTypes.length > 0) {
     const mtPlaceholders = filters.memoryTypes.map(() => "?").join(",");
@@ -707,10 +748,6 @@ function bruteForceVectorSearch(db, queryVec, userId, limit, filters) {
     whereClauses.push("event_date_end <= ?");
     params.push(filters.eventDateRange.end);
   }
-  const excludeInvalidated = filters?.excludeInvalidated ?? true;
-  if (excludeInvalidated) {
-    whereClauses.push("invalidated_at IS NULL");
-  }
   const sql = `SELECT * FROM memories WHERE ${whereClauses.join(" AND ")}`;
   const rows = db.prepare(sql).all(...params);
   if (rows.length === 0) {
@@ -748,9 +785,29 @@ function vectorSearch(db, queryVec, userId, limit, filters, useVecTable = true)
   const placeholders = rowids.map(() => "?").join(",");
   let sql = `SELECT * FROM memories WHERE id IN (${placeholders}) AND user_id = ?`;
   const params = [...rowids, userId];
-  const onlyLatest = filters?.onlyLatest ?? true;
-  if (onlyLatest) {
-    sql += " AND is_latest = 1";
+  if (filters?.states && filters.states.length > 0) {
+    const stateConditions = [];
+    if (filters.states.includes("current")) {
+      stateConditions.push("(is_latest = 1 AND invalidated_at IS NULL)");
+    }
+    if (filters.states.includes("superseded")) {
+      stateConditions.push("(is_latest = 0)");
+    }
+    if (filters.states.includes("invalidated")) {
+      stateConditions.push("(invalidated_at IS NOT NULL)");
+    }
+    if (stateConditions.length > 0) {
+      sql += ` AND (${stateConditions.join(" OR ")})`;
+    }
+  } else {
+    const onlyLatest = filters?.onlyLatest ?? true;
+    if (onlyLatest) {
+      sql += " AND is_latest = 1";
+    }
+    const excludeInvalidated = filters?.excludeInvalidated ?? true;
+    if (excludeInvalidated) {
+      sql += " AND invalidated_at IS NULL";
+    }
   }
   if (filters?.memoryTypes && filters.memoryTypes.length > 0) {
     const mtPlaceholders = filters.memoryTypes.map(() => "?").join(",");
@@ -779,10 +836,6 @@ function vectorSearch(db, queryVec, userId, limit, filters, useVecTable = true)
     sql += " AND event_date_end <= ?";
     params.push(filters.eventDateRange.end);
   }
-  const excludeInvalidated = filters?.excludeInvalidated ?? true;
-  if (excludeInvalidated) {
-    sql += " AND invalidated_at IS NULL";
-  }
   const rows = db.prepare(sql).all(...params);
   return rows.map((row) => ({
     memory: rowToMemory(row),
@@ -950,7 +1003,7 @@ function updateAccessTracking(db, memoryIds) {
   });
   updateAll(memoryIds);
 }
-function searchMemories(db, query, queryVec, useVecTable = true) {
+function searchMemories(db, query, queryVec, useVecTable = true, skipAccessTracking = false) {
   const startTime = performance.now();
   const queryTokens = extractQueryTokens(query.query);
   const ftsResults = ftsSearch(db, query);
@@ -959,7 +1012,9 @@ function searchMemories(db, query, queryVec, useVecTable = true) {
     const weighted = applyThreeFactorReranking(ftsResults);
     const boosted = applyKeywordBoost(weighted, queryTokens);
     finalResults = deduplicateResults(boosted);
-    updateAccessTracking(db, finalResults.map((r) => r.memory.memoryId));
+    if (!skipAccessTracking) {
+      updateAccessTracking(db, finalResults.map((r) => r.memory.memoryId));
+    }
     return {
       results: finalResults,
       totalCount: finalResults.length,
@@ -1088,19 +1143,6 @@ function estimateTokens(content) {
 }
 
 // src/engine.ts
-function cosineSim(a, b) {
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  const len = Math.min(a.length, b.length);
-  for (let i = 0; i < len; i++) {
-    dot += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}
 var SqliteMemoryEngine = class {
   db = null;
   stmts = null;
@@ -1156,6 +1198,7 @@ var SqliteMemoryEngine = class {
       await this.checkContradictions(memory);
     }
     await this.checkDuplicates(memory);
+    await this.autoRelate(memory);
     return memory;
   }
   async storeBatch(inputs) {
@@ -1424,6 +1467,41 @@ var SqliteMemoryEngine = class {
     this.ensureInitialized();
     this.db.prepare("UPDATE memories SET tier = ? WHERE memory_id = ?").run(tier, memoryId);
   }
+  async quality(userId) {
+    this.ensureInitialized();
+    const db = this.db;
+    const total = db.prepare("SELECT COUNT(*) as c FROM memories WHERE user_id = ?").get(userId).c;
+    const fresh = db.prepare(
+      "SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND is_latest = 1 AND invalidated_at IS NULL"
+    ).get(userId).c;
+    const invalidated = db.prepare(
+      "SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND invalidated_at IS NOT NULL"
+    ).get(userId).c;
+    const superseded = db.prepare(
+      "SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND is_latest = 0"
+    ).get(userId).c;
+    const withRelations = db.prepare(`
+        SELECT COUNT(DISTINCT mid) as c FROM (
+          SELECT src_memory_id as mid FROM memory_relations
+            WHERE src_memory_id IN (SELECT memory_id FROM memories WHERE user_id = ?)
+          UNION
+          SELECT dst_memory_id as mid FROM memory_relations
+            WHERE dst_memory_id IN (SELECT memory_id FROM memories WHERE user_id = ?)
+        )
+      `).get(userId, userId).c;
+    const avgRow = db.prepare(
+      "SELECT AVG(activation_score) as avg FROM memories WHERE user_id = ? AND is_latest = 1"
+    ).get(userId);
+    const avgActivation = avgRow.avg ?? 0;
+    return {
+      total,
+      fresh,
+      invalidated,
+      superseded,
+      withRelations,
+      avgActivation
+    };
+  }
   /**
    * Check the newly stored memory against existing similar memories
    * for contradictions using the NLI model.
@@ -1449,7 +1527,9 @@ var SqliteMemoryEngine = class {
           filters: { onlyLatest: true }
         },
         queryVec,
-        this.vectorEnabled
+        this.vectorEnabled,
+        true
+        // skipAccessTracking
       );
       for (const result of similar.results) {
         if (result.memory.memoryId === newMemory.memoryId) continue;
@@ -1483,27 +1563,21 @@ var SqliteMemoryEngine = class {
     if (!this.options.embedder) return;
     try {
       const queryVec = await this.options.embedder.embed(newMemory.content);
-      const similar = searchMemories(
+      const candidates = bruteForceVectorSearch(
         this.db,
-        {
-          userId: newMemory.userId,
-          query: newMemory.content,
-          limit: 5,
-          filters: { onlyLatest: true }
-        },
         queryVec,
-        this.vectorEnabled
+        newMemory.userId,
+        10,
+        { onlyLatest: true }
       );
-      for (const result of similar.results) {
-        if (result.memory.memoryId === newMemory.memoryId) continue;
-        if (!result.memory.embedding || result.memory.embedding.length === 0) continue;
-        const embB = new Float32Array(result.memory.embedding);
-        const similarity = cosineSim(queryVec, embB);
+      for (const candidate of candidates) {
+        if (candidate.memory.memoryId === newMemory.memoryId) continue;
+        const similarity = 1 - candidate.distance;
         if (similarity > 0.95) {
           try {
             await this.relate(
               newMemory.memoryId,
-              result.memory.memoryId,
+              candidate.memory.memoryId,
               "updates",
               `Auto-detected duplicate: cosine similarity ${similarity.toFixed(3)}`
             );
@@ -1514,6 +1588,55 @@ var SqliteMemoryEngine = class {
     } catch {
     }
   }
+  /**
+   * Auto-create 'extends' relations when a new memory shares keywords
+   * with existing memories. Builds graph density for future graph-based retrieval.
+   *
+   * Only runs for individual store() calls, not storeBatch().
+   * Requires at least 2 overlapping keywords to create a relation.
+   * Graceful: errors are silently swallowed.
+   */
+  async autoRelate(newMemory) {
+    if (!newMemory.keywords || newMemory.keywords.length === 0) return;
+    try {
+      const keywordList = newMemory.keywords;
+      const existing = this.db.prepare(`
+        SELECT memory_id, keywords FROM memories
+        WHERE user_id = ? AND is_latest = 1 AND memory_id != ?
+        AND invalidated_at IS NULL
+        ORDER BY learned_at DESC LIMIT 10
+      `).all(newMemory.userId, newMemory.memoryId);
+      for (const row of existing) {
+        if (!row.keywords) continue;
+        const existingKeywords = row.keywords.split(" ").map((k) => k.trim().toLowerCase());
+        const overlap = keywordList.filter((k) => existingKeywords.includes(k.toLowerCase()));
+        if (overlap.length >= 3) {
+          const existingRelation = this.db.prepare(
+            `SELECT 1 FROM memory_relations
+             WHERE (src_memory_id = ? AND dst_memory_id = ?)
+                OR (src_memory_id = ? AND dst_memory_id = ?)
+             LIMIT 1`
+          ).get(
+            newMemory.memoryId,
+            row.memory_id,
+            row.memory_id,
+            newMemory.memoryId
+          );
+          if (existingRelation) continue;
+          try {
+            await this.relate(
+              newMemory.memoryId,
+              row.memory_id,
+              "extends",
+              `Auto: ${overlap.length} shared keywords (${overlap.join(", ")})`
+            );
+          } catch {
+          }
+        }
+      }
+    } catch {
+    }
+  }
   ensureInitialized() {
     if (!this.db || !this.stmts || !this.relStmts) {
       throw new Error("Engine not initialized. Call initialize() first.");
@@ -1538,6 +1661,7 @@ export {
   createRelation,
   createRelationStatements,
   deduplicateResults,
+  deriveMemoryState,
   determineTier,
   ensureSchema,
   estimateTokens,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memrosetta/core",
-  "version": "0.2.19",
+  "version": "0.2.20",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -20,10 +20,10 @@
     "better-sqlite3": "^11.0.0",
     "nanoid": "^5.0.0",
     "sqlite-vec": "^0.1.7",
-    "@memrosetta/types": "0.2.19"
+    "@memrosetta/types": "0.2.20"
   },
   "optionalDependencies": {
-    "@memrosetta/embeddings": "0.2.19"
+    "@memrosetta/embeddings": "0.2.20"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.0",