opencode-memory-pro 1.3.9 → 1.4.0

package/README.md

@@ -40,7 +40,7 @@ Published on npm — install directly (requires OpenCode ≥ 1.x and Node.js ≥
 opencode plugin opencode-memory-pro
 ```
 
-The latest release is **v1.3.9** on [npm](https://www.npmjs.com/package/opencode-memory-pro); source and releases are on [GitHub](https://github.com/tman204-50/opencode-memory-pro).
+The latest release is **v1.4.0** on [npm](https://www.npmjs.com/package/opencode-memory-pro); source and releases are on [GitHub](https://github.com/tman204-50/opencode-memory-pro).
 
 Remove the old plugin pin at the same time:
 
@@ -489,6 +489,44 @@ so your memories and graph carry over untouched.
 
 ## Changelog
 
+### v1.4.0 (2026-09-06)
+
+Dedup correctness overhaul — the write-time duplicate check compared against
+the wrong score type, and the resulting flags were a one-way ratchet:
+
+- **Write-time dedup now compares a raw cosine similarity**: the capture path
+  went through the hybrid `search()` API, whose RRF score is algebraically
+  `>= 1.0` for `limit: 1` (and up to `1.4` with importance) — so every capture
+  in a non-empty scope compared `>= 1.0` against `dedup.writeThreshold`
+  (clamped to `[0,1]`) and got falsely flagged as a potential duplicate.
+  `storeCapturedMemory` now calls `findSimilarVectors` (the same raw cosine
+  primitive consolidation measures) and compares that to the threshold.
+  Consequence: recall scores can no longer exceed 100%, and
+  `dedup.enabled`'s write-time detection actually detects.
+- **False duplicate flags now self-correct**: `isPotentialDuplicate` was a
+  one-way ratchet — consolidation never cleared it, so `memory_stats`
+  `flaggedCount` only grew (153 flagged / 0 merged observed on a live store).
+  `consolidateDuplicates` now revalidates flags against the real cosine
+  threshold and clears (`isPotentialDuplicate`/`duplicateOf` removed) any
+  flagged row whose closest found neighbor never reaches the merge bar.
+  Returns `clearedFlags` so tools can report the correction.
+- **Auto-consolidation cooldown is per-scope**: the shared
+  `lastConsolidateAt` timestamp meant the first scope to consolidate blocked
+  all other scopes for 30 minutes. Cooldowns are now tracked per scope
+  (same for the retention sweep, which had the identical flaw).
+- **Scope cache staleness bound**: the per-process version counter can't see
+  writes from another opencode process sharing the same `dbPath`, so process A
+  could serve stale records indefinitely. Cache entries now reload after a
+  60s age bound even when the local version is unchanged (configurable via
+  `cache.staleAfterMs`; 0 restores pure version gating).
+- **Consistent truncation warnings**: `deleteByIdForce`'s 100k-row fallback
+  scan and `pruneScope`'s 100k-row read now log a warning when the cap is hit,
+  matching `getCachedScopes`.
+- **Tests**: three new integration tests — the dedup write-check primitive
+  returns cosine in `[0,1]` (plus a guard that the old RRF path still scores
+  `>= 1.0`), consolidation clears false flags, and the scope cache reloads
+  after the age bound when a second process writes behind its back.
+
 ### v1.3.8 (2026-09-06)
 
 Fixes `memory_forget(force=true)` being unable to permanently delete a memory

package/dist/index.js

@@ -11,7 +11,7 @@ import { requestLLMCapture, isOwnSession } from "./llm.js";
 import { createMemoryTools, createFeedbackTools, createEpisodicTools } from "./tools/index.js";
 import { sweepExpiredMemories } from "./tools/memory.js";
 import { createGraphStore } from "./graph.js";
-const PLUGIN_VERSION = "1.3.9";
+const PLUGIN_VERSION = "1.4.0";
 const SCHEMA_VERSION = 1;
 // Event-driven dedup: run consolidateDuplicates on session.idle (throttled to
 // this interval so chatty sessions aren't re-scanning the store every turn)
@@ -597,12 +597,16 @@ async function createRuntimeState(input) {
         activeEpisodes: new Map(),
         sessionErrors: new Map(),
         lastRecall: null,
+        // PER_SCOPE_COOLDOWN (1.4.0): cooldowns are per-scope (Map keyed by scope)
+        // instead of a single shared timestamp — one shared value meant the
+        // first scope to consolidate/sweep blocked every other scope for the
+        // whole 30-minute cooldown, even scopes that had never run.
         consolidationInProgress: new Map(),
-        lastConsolidateAt: 0,
+        lastConsolidateAt: new Map(),
         // MEMORY_RETENTION (1.0): digest-then-hide expiry sweep state — same
         // throttle pattern as consolidation (cooldown-gated, one per scope).
         sweepInProgress: new Map(),
-        lastSweepAt: 0,
+        lastSweepAt: new Map(),
         ensureInitialized: async () => {
             if (state.initialized)
                 return;
@@ -800,22 +804,19 @@ async function storeCapturedMemory(state, opts) {
     }
     let isPotentialDuplicate = false;
     let duplicateOf = null;
+    // DEDUP_COSINE_CHECK (1.4.0): the write-time dedup check used to go
+    // through the hybrid search() API, whose RRF score is algebraically >= 1.0
+    // for limit:1 (rrfScore = 1/(rrfK+1) * (rrfK+1) == 1.0, then multiplied by
+    // an importance factor in [1, 1.4]) — so every capture with any same-dim
+    // record in the scope compared >= 1.0 against writeThreshold (clamped
+    // [0,1]) and got falsely flagged as a duplicate. Now it uses
+    // findSimilarVectors, which returns a raw cosine similarity in [0,1], the
+    // same primitive consolidateDuplicates measures against.
     if (state.config.dedup.enabled) {
-        const similar = await state.store.search({
-            query: opts.text,
-            queryVector: vector,
-            scopes: [opts.scope],
-            limit: 1,
-            vectorWeight: 1.0,
-            bm25Weight: 0.0,
-            minScore: 0.0,
-            rrfK: 60,
-            recencyBoost: false,
-            globalDiscountFactor: 1.0,
-        });
+        const similar = await state.store.findSimilarVectors(vector, opts.scope, 1);
         if (similar.length > 0 && similar[0].score >= state.config.dedup.writeThreshold) {
             isPotentialDuplicate = true;
-            duplicateOf = similar[0].record.id;
+            duplicateOf = similar[0].id;
         }
     }
     const memoryId = generateId();
@@ -866,11 +867,12 @@ async function maybeConsolidateDuplicates(state, scope, force = false) {
     if (state.consolidationInProgress.get(scope))
         return;
     if (!force) {
-        const elapsed = Date.now() - state.lastConsolidateAt;
+        const last = state.lastConsolidateAt.get(scope) ?? 0;
+        const elapsed = Date.now() - last;
         if (elapsed < CONSOLIDATE_COOLDOWN_MS)
             return;
     }
-    state.lastConsolidateAt = Date.now();
+    state.lastConsolidateAt.set(scope, Date.now());
     state.consolidationInProgress.set(scope, true);
     state.store
         .consolidateDuplicates(scope, state.config.dedup.consolidateThreshold, state.config.dedup.candidateLimit)
@@ -888,11 +890,12 @@ async function maybeSweepExpiredMemories(state, scope, force = false) {
     if (state.sweepInProgress.get(scope))
         return;
     if (!force) {
-        const elapsed = Date.now() - state.lastSweepAt;
+        const last = state.lastSweepAt.get(scope) ?? 0;
+        const elapsed = Date.now() - last;
         if (elapsed < CONSOLIDATE_COOLDOWN_MS)
             return;
     }
-    state.lastSweepAt = Date.now();
+    state.lastSweepAt.set(scope, Date.now());
     state.sweepInProgress.set(scope, true);
     sweepExpiredMemories(state, { scope })
         .then((result) => {

package/dist/store.js

@@ -10,6 +10,13 @@ const DEFAULT_CACHE_CONFIG = {
     maxScopes: 10,
     maxRecordsPerScope: 1000,
     enabled: true,
+    // SCOPE_CACHE_STALENESS (1.4.0): the version counter only sees THIS
+    // process's writes, so when two opencode processes share one dbPath the
+    // scope cache could serve stale records forever. A modest age-based
+    // staleness bound forces a reload after staleAfterMs even when the local
+    // version is unchanged, bounding cross-process staleness without a schema
+    // change. 0 disables the age check (pure version gating, pre-1.4.0).
+    staleAfterMs: 60 * 1000,
 };
 // ANN_TUNABLES (1.3.0): nprobes controls IVF recall-vs-latency on filtered
 // vector searches; the consolidation query batch controls how many ANN
@@ -642,6 +649,9 @@ export class MemoryStore {
         }
         const table = this.requireTable();
         const rows = await table.query().limit(100000).toArray();
+        if (rows.length === 100000) {
+            log("warn", "[store] deleteByIdForce fallback scan hit the 100000-row cap; the target may not be found if it lives beyond the cap");
+        }
         const match = rows.find((row) => this.matchesId(row.id, id));
         if (!match)
             return false;
@@ -711,6 +721,9 @@ export class MemoryStore {
     }
     async pruneScope(scope, maxEntries) {
         const rows = await this.list(scope, 100000);
+        if (rows.length === 100000) {
+            log("warn", `[store] pruneScope scanned up to the 100000-row cap for scope=${scope}; entries older than the newest 100k are not candidates for pruning`);
+        }
         if (rows.length <= maxEntries)
             return 0;
         const flagged = rows.filter((r) => {
@@ -748,7 +761,7 @@ export class MemoryStore {
         let rows = await this.readByScopesIncludingMerged([scope]);
         rows = rows.filter((r) => r.status === undefined || r.status === null || r.status === "" || r.status === "active");
         if (rows.length === 0) {
-            return { mergedPairs: 0, updatedRecords: 0, skippedRecords: 0 };
+            return { mergedPairs: 0, updatedRecords: 0, skippedRecords: 0, clearedFlags: 0 };
         }
         const BATCH_SIZE = 100;
         const FALLBACK_THRESHOLD = 500;
@@ -763,12 +776,23 @@ export class MemoryStore {
             row,
             norm: this.scopeCache.get(scope)?.norms.get(row.id) ?? vecNorm(row.vector),
         }));
+        // DEDUP_FLAG_REVALIDATION (1.4.0): the write-time dedup check used to
+        // flag nearly every capture (RRF score >= 1.0 vs writeThreshold in
+        // [0,1]), and the flag was a one-way ratchet — nothing ever cleared it,
+        // so flaggedCount only grew. Consolidation is where a real cosine
+        // comparison happens, so flagged rows whose closest found neighbor
+        // stays below the consolidate threshold get the flag cleared; rows
+        // that DO have a near-duplicate keep it.
+        const metaById = new Map(rowsWithNorms.map(({ row }) => [row.id, parseMetadata(row.metadataJson)]));
+        const flaggedIds = new Set([...metaById].filter(([, meta]) => meta.isPotentialDuplicate === true).map(([id]) => id));
+        const bestSimByFlagged = new Map();
+        const mergedIds = new Set();
+        let clearedFlags = 0;
         log("debug", `[consolidate] scope=${scope} rows=${rows.length} threshold=${threshold} candidateLimit=${candidateLimit} batchSize=${BATCH_SIZE} fallbackThreshold=${FALLBACK_THRESHOLD}`);
         const processWithANN = async () => {
             let localMerged = 0;
             let localUpdated = 0;
             let localSkipped = 0;
-            const mergedIds = new Set();
             const totalChunks = Math.ceil(rowsWithNorms.length / BATCH_SIZE);
             for (let chunkIdx = 0; chunkIdx < totalChunks; chunkIdx++) {
                 const chunkStart = chunkIdx * BATCH_SIZE;
@@ -805,6 +829,12 @@ export class MemoryStore {
                             if (mergedIds.has(b.row.id))
                                 continue;
                             const sim = storeFastCosine(a.row.vector, b.row.vector, a.norm, b.norm);
+                            if (flaggedIds.has(a.row.id)) {
+                                bestSimByFlagged.set(a.row.id, Math.max(bestSimByFlagged.get(a.row.id) ?? -1, sim));
+                            }
+                            if (flaggedIds.has(b.row.id)) {
+                                bestSimByFlagged.set(b.row.id, Math.max(bestSimByFlagged.get(b.row.id) ?? -1, sim));
+                            }
                             if (sim < threshold)
                                 continue;
                             const aMeta = parseMetadata(a.row.metadataJson);
@@ -879,7 +909,6 @@ export class MemoryStore {
             let localMerged = 0;
             let localUpdated = 0;
             let localSkipped = 0;
-            const mergedIds = new Set();
             for (let i = 0; i < rowsWithNorms.length; i += 1) {
                 const a = rowsWithNorms[i];
                 if (mergedIds.has(a.row.id))
@@ -889,6 +918,12 @@ export class MemoryStore {
                     if (mergedIds.has(b.row.id))
                         continue;
                     const sim = storeFastCosine(a.row.vector, b.row.vector, a.norm, b.norm);
+                    if (flaggedIds.has(a.row.id)) {
+                        bestSimByFlagged.set(a.row.id, Math.max(bestSimByFlagged.get(a.row.id) ?? -1, sim));
+                    }
+                    if (flaggedIds.has(b.row.id)) {
+                        bestSimByFlagged.set(b.row.id, Math.max(bestSimByFlagged.get(b.row.id) ?? -1, sim));
+                    }
                     if (sim < threshold)
                         continue;
                     const aMeta = parseMetadata(a.row.metadataJson);
@@ -955,14 +990,33 @@ export class MemoryStore {
             }
             else {
                 log("warn", `[consolidate] Skipping fallback for large scope (${rows.length} >= ${FALLBACK_THRESHOLD})`);
-                return { mergedPairs: 0, updatedRecords: 0, skippedRecords: 0 };
+                return { mergedPairs: 0, updatedRecords: 0, skippedRecords: 0, clearedFlags: 0 };
             }
         }
-        if (mergedPairs > 0) {
+        // DEDUP_FLAG_REVALIDATION (1.4.0): clear false duplicate flags. Rows
+        // whose best found neighbor never reached the merge threshold were
+        // flagged by the pre-1.4.0 RRF write-check (or carry a flag made stale
+        // by later edits); unsetting isPotentialDuplicate lets flaggedCount
+        // self-correct instead of ratcheting up forever.
+        for (const [id, bestSim] of bestSimByFlagged) {
+            if (mergedIds.has(id) || bestSim >= threshold)
+                continue;
+            const meta = metaById.get(id);
+            if (!meta || meta.isPotentialDuplicate !== true)
+                continue;
+            delete meta.isPotentialDuplicate;
+            delete meta.duplicateOf;
+            await this.requireTable().update({
+                where: `id = '${escapeSql(id)}'`,
+                values: { metadataJson: JSON.stringify(meta) },
+            });
+            clearedFlags += 1;
+        }
+        if (mergedPairs > 0 || clearedFlags > 0) {
             this.invalidateScope(scope);
         }
         await this.maybeOptimizeAll(false);
-        return { mergedPairs, updatedRecords, skippedRecords };
+        return { mergedPairs, updatedRecords, skippedRecords, clearedFlags };
     }
     // ANN_CONSOLIDATION (1.1.7): previously this did
 // query().where(scope).limit(limit).toArray() — which returns the FIRST N
@@ -1647,7 +1701,9 @@ export class MemoryStore {
         for (const scope of scopes) {
             const currentVersion = this.scopeVersions.get(scope) ?? 0;
             let entry = this.scopeCache.get(scope);
-            if (!entry || entry.version !== currentVersion) {
+            const maxAgeMs = Number.isFinite(this.cacheConfig.staleAfterMs) ? this.cacheConfig.staleAfterMs : 0;
+            const staleByAge = maxAgeMs > 0 && (entry ? Date.now() - (entry.loadedAt ?? entry.lastAccessTimestamp) > maxAgeMs : false);
+            if (!entry || entry.version !== currentVersion || staleByAge) {
                 if (entry) {
                     this.cacheStats.evictions++;
                 }
@@ -1667,7 +1723,7 @@ export class MemoryStore {
                 for (const record of sortedRecords) {
                     norms.set(record.id, vecNorm(record.vector));
                 }
-                entry = { records: sortedRecords, tokenized, idf, norms, lastAccessTimestamp: Date.now(), version: currentVersion };
+                entry = { records: sortedRecords, tokenized, idf, norms, loadedAt: Date.now(), lastAccessTimestamp: Date.now(), version: currentVersion };
                 this.scopeCache.set(scope, entry);
                 this.cacheStats.misses++;
                 this.enforceMaxScopes();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-memory-pro",
-  "version": "1.3.9",
+  "version": "1.4.0",
   "description": "LanceDB-backed long-term memory provider for OpenCode — standalone fork of lancedb-opencode-pro with entity graph, lifecycle, and retention",
   "type": "module",
   "main": "dist/index.js",