@seanhogg/builderforce-memory 2026.9.34 → 2026.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/errors/SSMError.d.ts +1 -1
- package/dist/errors/SSMError.d.ts.map +1 -1
- package/dist/errors/SSMError.js.map +1 -1
- package/dist/handoff/index.d.ts +5 -5
- package/dist/handoff/index.js +5 -5
- package/dist/handoff/note.d.ts +1 -1
- package/dist/handoff/note.js +1 -1
- package/dist/handoff/pack.d.ts +13 -4
- package/dist/handoff/pack.d.ts.map +1 -1
- package/dist/handoff/pack.js +31 -10
- package/dist/handoff/pack.js.map +1 -1
- package/dist/memory/MemoryStore.d.ts +120 -181
- package/dist/memory/MemoryStore.d.ts.map +1 -1
- package/dist/memory/MemoryStore.js +201 -389
- package/dist/memory/MemoryStore.js.map +1 -1
- package/dist/memory/SemanticRecall.d.ts +64 -0
- package/dist/memory/SemanticRecall.d.ts.map +1 -0
- package/dist/memory/SemanticRecall.js +188 -0
- package/dist/memory/SemanticRecall.js.map +1 -0
- package/dist/memory/records.d.ts +26 -0
- package/dist/memory/records.d.ts.map +1 -0
- package/dist/memory/records.js +71 -0
- package/dist/memory/records.js.map +1 -0
- package/dist/memory/ttlSweeper.d.ts +12 -0
- package/dist/memory/ttlSweeper.d.ts.map +1 -0
- package/dist/memory/ttlSweeper.js +28 -0
- package/dist/memory/ttlSweeper.js.map +1 -0
- package/dist/memory/types.d.ts +142 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +6 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/storage/ScopedFactRepository.d.ts +156 -0
- package/dist/storage/ScopedFactRepository.d.ts.map +1 -0
- package/dist/storage/ScopedFactRepository.js +278 -0
- package/dist/storage/ScopedFactRepository.js.map +1 -0
- package/dist/storage/factImport.d.ts +21 -0
- package/dist/storage/factImport.d.ts.map +1 -0
- package/dist/storage/factImport.js +61 -0
- package/dist/storage/factImport.js.map +1 -0
- package/dist/storage/factMutations.d.ts +46 -0
- package/dist/storage/factMutations.d.ts.map +1 -0
- package/dist/storage/factMutations.js +73 -0
- package/dist/storage/factMutations.js.map +1 -0
- package/dist/storage/factQueries.d.ts +16 -0
- package/dist/storage/factQueries.d.ts.map +1 -0
- package/dist/storage/factQueries.js +36 -0
- package/dist/storage/factQueries.js.map +1 -0
- package/dist/storage/factRetention.d.ts +30 -0
- package/dist/storage/factRetention.d.ts.map +1 -0
- package/dist/storage/factRetention.js +65 -0
- package/dist/storage/factRetention.js.map +1 -0
- package/dist/storage/fileLock.d.ts +56 -0
- package/dist/storage/fileLock.d.ts.map +1 -0
- package/dist/storage/fileLock.js +138 -0
- package/dist/storage/fileLock.js.map +1 -0
- package/dist/storage/fileTransaction.d.ts +33 -0
- package/dist/storage/fileTransaction.d.ts.map +1 -0
- package/dist/storage/fileTransaction.js +108 -0
- package/dist/storage/fileTransaction.js.map +1 -0
- package/dist/storage/idbAdapter.d.ts +95 -0
- package/dist/storage/idbAdapter.d.ts.map +1 -0
- package/dist/storage/idbAdapter.js +311 -0
- package/dist/storage/idbAdapter.js.map +1 -0
- package/dist/storage/nodeFileAdapter.d.ts +127 -0
- package/dist/storage/nodeFileAdapter.d.ts.map +1 -0
- package/dist/storage/nodeFileAdapter.js +275 -0
- package/dist/storage/nodeFileAdapter.js.map +1 -0
- package/dist/storage/syncJournal.d.ts +19 -0
- package/dist/storage/syncJournal.d.ts.map +1 -0
- package/dist/storage/syncJournal.js +46 -0
- package/dist/storage/syncJournal.js.map +1 -0
- package/dist/storage/types.d.ts +207 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +90 -0
- package/dist/storage/types.js.map +1 -0
- package/package.json +3 -3
- package/src/errors/SSMError.ts +1 -0
- package/src/handoff/handoff.test.ts +52 -2
- package/src/handoff/index.ts +5 -5
- package/src/handoff/note.ts +1 -1
- package/src/handoff/pack.ts +32 -9
- package/src/memory/MemoryStore.ts +245 -525
- package/src/memory/SemanticRecall.ts +204 -0
- package/src/memory/records.ts +79 -0
- package/src/memory/ttlSweeper.ts +29 -0
- package/src/memory/types.ts +149 -0
- package/src/storage/ScopedFactRepository.ts +373 -0
- package/src/storage/factImport.ts +73 -0
- package/src/storage/factMutations.ts +106 -0
- package/src/storage/factQueries.ts +41 -0
- package/src/storage/factRetention.ts +82 -0
- package/src/storage/fileLock.ts +162 -0
- package/src/storage/fileTransaction.ts +116 -0
- package/src/storage/idbAdapter.ts +373 -0
- package/src/storage/nodeFileAdapter.ts +331 -0
- package/src/storage/syncJournal.ts +47 -0
- package/src/storage/types.ts +262 -0
|
@@ -1,184 +1,173 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* MemoryStore – persistent key-value fact store and weight checkpoint helper.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* -
|
|
4
|
+
* Persistence is delegated to a narrow {@link FactStoragePort} (M01) fronted by
|
|
5
|
+
* a {@link ScopedFactRepository} unit-of-work layer (M02–M06):
|
|
6
|
+
* - Browser default: IndexedDB via {@link IdbStorageAdapter}; every write
|
|
7
|
+
* resolves only after the transaction commits (M02).
|
|
8
|
+
* - Node.js: pass `storage: await NodeFileStorageAdapter.open(path)` for real
|
|
9
|
+
* durable storage. fake-indexeddb via `idbFactory` remains supported for
|
|
10
|
+
* TESTS, but is never durable and is not presented as such.
|
|
11
|
+
* - Facts live in an explicit {@link MemoryScope} partition (default: the
|
|
12
|
+
* unbound, never-connected scope) with durable revisions, operation
|
|
13
|
+
* identity, provenance and — when `sync: true` — a journaled tombstone
|
|
14
|
+
* trail that caps/TTL will not sacrifice (M03/M04/M06).
|
|
15
|
+
*
|
|
16
|
+
* Recall ranking and its derived embedding/index caches live in
|
|
17
|
+
* {@link SemanticRecall}; the public shapes in `./types.ts`.
|
|
7
18
|
*
|
|
8
19
|
* Weight save/load is delegated to the SSMRuntime passed to saveWeights/loadWeights.
|
|
9
20
|
*/
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Process-wide monotonic counter stamped onto each remembered entry. Strictly
|
|
18
|
-
* increasing per write regardless of store instance, so it reliably breaks
|
|
19
|
-
* `timestamp` ties (same-millisecond writes) in newest-first ordering.
|
|
20
|
-
*/
|
|
21
|
-
let _writeSeq = 0;
|
|
21
|
+
import { IdbStorageAdapter } from '../storage/idbAdapter.js';
|
|
22
|
+
import { ScopedFactRepository } from '../storage/ScopedFactRepository.js';
|
|
23
|
+
import { MemoryStorageError, } from '../storage/types.js';
|
|
24
|
+
import { SemanticRecall } from './SemanticRecall.js';
|
|
25
|
+
import { isExpired, liveEntries, rememberInput, toEntry, toImportRecord } from './records.js';
|
|
26
|
+
import { TtlSweeper } from './ttlSweeper.js';
|
|
22
27
|
/** Max content→embedding vectors retained (LRU-evicted past this). (EVM-8) */
|
|
23
28
|
const EMBED_CACHE_MAX = 2000;
|
|
24
29
|
export class MemoryStore {
|
|
25
|
-
_dbName;
|
|
26
30
|
_weightsKey;
|
|
27
|
-
_idb;
|
|
28
31
|
_defaultTtl;
|
|
29
|
-
_annThreshold;
|
|
30
32
|
_maxEntries;
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
/** Identity of the runtime whose embeddings populated the vectors. */
|
|
48
|
-
_denseRuntime = null;
|
|
33
|
+
_scope;
|
|
34
|
+
/** The unit-of-work layer every durable operation flows through. */
|
|
35
|
+
repository;
|
|
36
|
+
_storage;
|
|
37
|
+
/** Cross-context subscription, opened on the first operation (see `_ready`). */
|
|
38
|
+
_unsubscribe;
|
|
39
|
+
_subscribed = false;
|
|
40
|
+
_sweeper = new TtlSweeper();
|
|
41
|
+
/** Ranking + its embedding / dense-index caches (EVM-1, EVM-1b, EVM-8). */
|
|
42
|
+
_recall;
|
|
43
|
+
/**
|
|
44
|
+
* Last store generation observed from a FOREIGN context (another tab,
|
|
45
|
+
* window or process). Monotonic, in-memory only; lets readers notice
|
|
46
|
+
* cross-context writes without a storage round-trip per recall (M05).
|
|
47
|
+
*/
|
|
48
|
+
_foreignRevision = 0;
|
|
49
49
|
constructor(opts = {}) {
|
|
50
|
-
this._dbName = opts.dbName ?? 'ssmjs';
|
|
51
50
|
this._weightsKey = opts.weightsKey ?? 'ssmjs-weights';
|
|
52
|
-
this._idb = opts.idbFactory;
|
|
53
51
|
this._defaultTtl = opts.defaultTtlMs;
|
|
54
|
-
this._annThreshold = opts.annThreshold ?? 256;
|
|
55
52
|
this._maxEntries = opts.maxEntries;
|
|
56
|
-
this.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return;
|
|
67
|
-
}
|
|
68
|
-
const req = factory.open(this._dbName, DB_VERSION);
|
|
69
|
-
req.onupgradeneeded = (e) => {
|
|
70
|
-
const db = e.target.result;
|
|
71
|
-
if (!db.objectStoreNames.contains(FACTS_STORE)) {
|
|
72
|
-
db.createObjectStore(FACTS_STORE, { keyPath: 'key' });
|
|
73
|
-
}
|
|
74
|
-
if (!db.objectStoreNames.contains(WEIGHTS_STORE)) {
|
|
75
|
-
db.createObjectStore(WEIGHTS_STORE);
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
|
-
req.onsuccess = () => {
|
|
79
|
-
this._db = req.result;
|
|
80
|
-
resolve(req.result);
|
|
81
|
-
};
|
|
82
|
-
/* istanbul ignore next -- IDB open onerror fires only on storage faults; not reproducible with fake-indexeddb */
|
|
83
|
-
req.onerror = () => reject(new SSMError('MEMORY_UNAVAILABLE', `Failed to open IndexedDB "${this._dbName}": ${req.error?.message ?? 'unknown'}`, req.error));
|
|
53
|
+
this._recall = new SemanticRecall({
|
|
54
|
+
annThreshold: opts.annThreshold ?? 256,
|
|
55
|
+
embedCacheMax: opts.embedCacheMax ?? EMBED_CACHE_MAX,
|
|
56
|
+
// Opt-in floor: 0 keeps recall returning its best hits, as it always did.
|
|
57
|
+
minRelevance: opts.minRelevance ?? 0,
|
|
58
|
+
});
|
|
59
|
+
this._scope = opts.scope ? { ...opts.scope } : {};
|
|
60
|
+
this._storage = opts.storage ?? new IdbStorageAdapter(opts.dbName ?? 'ssmjs', {
|
|
61
|
+
idbFactory: opts.idbFactory,
|
|
62
|
+
contextId: opts.contextId,
|
|
84
63
|
});
|
|
64
|
+
this.repository = new ScopedFactRepository(this._storage, {
|
|
65
|
+
syncEnabled: opts.sync ?? false,
|
|
66
|
+
contextId: opts.contextId,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Cross-context invalidation (M05): a sibling tab/window/process that
|
|
71
|
+
* commits against the same storage invalidates this instance's derived
|
|
72
|
+
* caches. Subscribed lazily on the first operation: before it there are no
|
|
73
|
+
* caches to invalidate, and an eager subscription opened a global
|
|
74
|
+
* BroadcastChannel in the constructor that kept Node processes alive.
|
|
75
|
+
*/
|
|
76
|
+
_ready() {
|
|
77
|
+
if (this._subscribed)
|
|
78
|
+
return;
|
|
79
|
+
this._subscribed = true;
|
|
80
|
+
this._unsubscribe = this.repository.subscribe((note) => this._onForeignChange(note));
|
|
81
|
+
}
|
|
82
|
+
/** The scope partition this store is bound to (never widened implicitly). */
|
|
83
|
+
get scope() {
|
|
84
|
+
return { ...this._scope };
|
|
85
|
+
}
|
|
86
|
+
/** The durable storage backend kind in use ('indexedDB' | 'node-file'). */
|
|
87
|
+
get storageKind() {
|
|
88
|
+
return this._storage.kind;
|
|
85
89
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
90
|
+
/**
|
|
91
|
+
* Best-effort in-memory token that changes when another context writes to
|
|
92
|
+
* this store's database. Readers (e.g. EvermindCognition's recall cache)
|
|
93
|
+
* can fold it into their cache key so cross-context writes invalidate
|
|
94
|
+
* without a storage hit per recall.
|
|
95
|
+
*/
|
|
96
|
+
get foreignRevision() {
|
|
97
|
+
return this._foreignRevision;
|
|
98
|
+
}
|
|
99
|
+
_onForeignChange(note) {
|
|
100
|
+
if (note.revision <= this._foreignRevision)
|
|
101
|
+
return;
|
|
102
|
+
this._foreignRevision = note.revision;
|
|
103
|
+
// Derived caches describe a fact set another context just changed.
|
|
104
|
+
this._recall.invalidate();
|
|
91
105
|
}
|
|
92
106
|
// ── Semantic facts ────────────────────────────────────────────────────────
|
|
93
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Stores or overwrites a fact. Resolves only after the underlying
|
|
109
|
+
* transaction commits durably (M02); the fact, its provenance and its sync
|
|
110
|
+
* journal entry commit atomically.
|
|
111
|
+
*/
|
|
94
112
|
async remember(key, content, opts) {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
type: opts?.type,
|
|
103
|
-
tags: opts?.tags,
|
|
104
|
-
importance: opts?.importance,
|
|
105
|
-
};
|
|
106
|
-
const tx = db.transaction(FACTS_STORE, 'readwrite');
|
|
107
|
-
await requestToPromise(tx.objectStore(FACTS_STORE).put(entry), `Failed to store fact "${key}"`, () => undefined);
|
|
108
|
-
// EVM-7: keep the store bounded — evict lowest-value entries past the cap.
|
|
109
|
-
if (this._maxEntries != null)
|
|
110
|
-
await this._enforceCap();
|
|
113
|
+
this._ready();
|
|
114
|
+
await this.repository.remember(this._scope, key, rememberInput(content, opts, this._defaultTtl));
|
|
115
|
+
// EVM-7: keep the store bounded — evict lowest-value SYNCED entries past
|
|
116
|
+
// the cap; unsynced knowledge is never sacrificed to a cache cap (M06).
|
|
117
|
+
if (this._maxEntries != null) {
|
|
118
|
+
await this.repository.evictForCap(this._scope, this._maxEntries, isExpired);
|
|
119
|
+
}
|
|
111
120
|
}
|
|
112
121
|
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
122
|
+
* Compare-and-set write (M04/M05): commits only if the record's current
|
|
123
|
+
* revision equals `expectedRevision` (`null` asserts the key is absent).
|
|
124
|
+
* Throws a MemoryStorageError otherwise (`code` 'MEMORY_CONFLICT',
|
|
125
|
+
* `storageCode` 'STORAGE_CONFLICT'; see `isStorageConflict`) — callers reconcile
|
|
126
|
+
* (re-read, re-evaluate evidence) instead of blindly overwriting a
|
|
127
|
+
* concurrent writer. Returns the committed entry.
|
|
116
128
|
*/
|
|
117
|
-
async
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
const all = await requestToPromise(tx.objectStore(FACTS_STORE).getAll(), 'Failed to scan facts for cap', (r) => r);
|
|
123
|
-
if (all.length <= this._maxEntries)
|
|
124
|
-
return;
|
|
125
|
-
// Ascending sort = worst first: expired (0) before live (1), then lower
|
|
126
|
-
// importance, then older timestamp, then lower write sequence.
|
|
127
|
-
const rank = (e) => [this._isExpired(e) ? 0 : 1, e.importance ?? 0.5, e.timestamp, e.seq ?? 0];
|
|
128
|
-
const sorted = [...all].sort((a, b) => {
|
|
129
|
-
const ra = rank(a), rb = rank(b);
|
|
130
|
-
for (let i = 0; i < ra.length; i++) {
|
|
131
|
-
if (ra[i] !== rb[i])
|
|
132
|
-
return ra[i] - rb[i];
|
|
133
|
-
}
|
|
134
|
-
return 0;
|
|
129
|
+
async rememberCas(key, content, expectedRevision, opts) {
|
|
130
|
+
this._ready();
|
|
131
|
+
const { record } = await this.repository.remember(this._scope, key, {
|
|
132
|
+
...rememberInput(content, opts, this._defaultTtl),
|
|
133
|
+
expectedRevision,
|
|
135
134
|
});
|
|
136
|
-
|
|
137
|
-
|
|
135
|
+
return toEntry(record);
|
|
136
|
+
}
|
|
137
|
+
/** Fetches an entry WITH its durable revision, for CAS-based reconciliation. */
|
|
138
|
+
async recallVersioned(key) {
|
|
139
|
+
return this.recall(key);
|
|
138
140
|
}
|
|
139
141
|
/**
|
|
140
|
-
* Starts a background timer that periodically
|
|
142
|
+
* Starts a background timer that periodically deletes expired entries —
|
|
141
143
|
* TTL is otherwise only evaluated lazily on read, so an idle store would keep
|
|
142
144
|
* dead entries forever. Idempotent; pair with {@link stopTtlSweeper}. (EVM-7)
|
|
145
|
+
*
|
|
146
|
+
* With sync enabled each expiry is a scoped deletion (tombstone + journal),
|
|
147
|
+
* not a silent drop (M06).
|
|
143
148
|
*/
|
|
144
149
|
startTtlSweeper(intervalMs) {
|
|
145
|
-
|
|
146
|
-
return;
|
|
147
|
-
const timer = setInterval(() => { void this.purgeExpired(); }, intervalMs);
|
|
148
|
-
// Don't keep a Node process alive just for the sweeper.
|
|
149
|
-
const maybeUnref = timer;
|
|
150
|
-
/* istanbul ignore else -- unref exists in Node; absent in the browser */
|
|
151
|
-
if (typeof maybeUnref.unref === 'function') {
|
|
152
|
-
maybeUnref.unref();
|
|
153
|
-
}
|
|
154
|
-
this._sweepTimer = timer;
|
|
150
|
+
this._sweeper.start(intervalMs, () => this.purgeExpired());
|
|
155
151
|
}
|
|
156
152
|
/** Stops the background TTL sweeper started by {@link startTtlSweeper}. */
|
|
157
153
|
stopTtlSweeper() {
|
|
158
|
-
|
|
159
|
-
clearInterval(this._sweepTimer);
|
|
160
|
-
this._sweepTimer = null;
|
|
161
|
-
}
|
|
154
|
+
this._sweeper.stop();
|
|
162
155
|
}
|
|
163
156
|
/**
|
|
164
|
-
* Retrieves a fact by key.
|
|
157
|
+
* Retrieves a fact by key from this store's scope partition.
|
|
165
158
|
* Returns `undefined` if the key does not exist or the entry has expired.
|
|
166
159
|
*/
|
|
167
160
|
async recall(key) {
|
|
168
|
-
|
|
169
|
-
const
|
|
170
|
-
|
|
161
|
+
this._ready();
|
|
162
|
+
const record = await this.repository.get(this._scope, key);
|
|
163
|
+
if (!record || isExpired(record))
|
|
164
|
+
return undefined;
|
|
165
|
+
return toEntry(record);
|
|
171
166
|
}
|
|
172
|
-
/** Returns all non-expired stored facts, newest first. */
|
|
167
|
+
/** Returns all non-expired stored facts in this store's scope, newest first. */
|
|
173
168
|
async recallAll() {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
return requestToPromise(tx.objectStore(FACTS_STORE).getAll(), 'Failed to recall all facts', (entries) => entries
|
|
177
|
-
.filter(e => !this._isExpired(e))
|
|
178
|
-
// Newest first; break same-millisecond ties by write sequence so
|
|
179
|
-
// ordering is deterministic (IndexedDB getAll() returns key order,
|
|
180
|
-
// which would otherwise surface same-ms writes oldest-first).
|
|
181
|
-
.sort((a, b) => b.timestamp - a.timestamp || (b.seq ?? 0) - (a.seq ?? 0)));
|
|
169
|
+
this._ready();
|
|
170
|
+
return liveEntries(await this.repository.getAll(this._scope));
|
|
182
171
|
}
|
|
183
172
|
/**
|
|
184
173
|
* Returns the N most recently updated non-expired facts.
|
|
@@ -190,22 +179,18 @@ export class MemoryStore {
|
|
|
190
179
|
}
|
|
191
180
|
/**
|
|
192
181
|
* Returns all non-expired entries that contain the given tag.
|
|
182
|
+
* Looks the tag up on the facts `byTag` index when the storage adapter has
|
|
183
|
+
* one; a full scan is used only when that index is missing.
|
|
193
184
|
*/
|
|
194
185
|
async recallByTag(tag) {
|
|
195
|
-
|
|
196
|
-
return
|
|
186
|
+
this._ready();
|
|
187
|
+
return liveEntries(await this.repository.getByTag(this._scope, tag));
|
|
197
188
|
}
|
|
198
189
|
/**
|
|
199
|
-
* Finds the top-K semantically similar entries to `query
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
* memory layer uses the very model it is attached to, and recall quality
|
|
204
|
-
* improves automatically as that model is adapted/distilled. Embeddings are
|
|
205
|
-
* cached per content string to avoid recomputing across calls.
|
|
206
|
-
*
|
|
207
|
-
* If no embedding-capable runtime is provided, or embedding fails for any
|
|
208
|
-
* reason, it transparently falls back to Jaccard word-overlap similarity.
|
|
190
|
+
* Finds the top-K semantically similar entries to `query`: cosine over the
|
|
191
|
+
* attached SSM runtime's embeddings (so recall improves as that model is
|
|
192
|
+
* adapted), or Jaccard word overlap when no runtime can embed. See
|
|
193
|
+
* {@link SemanticRecall}.
|
|
209
194
|
*/
|
|
210
195
|
async recallSimilar(query, topK, runtime) {
|
|
211
196
|
return (await this.recallSimilarScored(query, topK, runtime)).map(s => s.entry);
|
|
@@ -217,263 +202,106 @@ export class MemoryStore {
|
|
|
217
202
|
* {@link recallSimilar} is the score-dropping convenience wrapper over this.
|
|
218
203
|
*/
|
|
219
204
|
async recallSimilarScored(query, topK, runtime) {
|
|
220
|
-
|
|
221
|
-
if (all.length === 0)
|
|
222
|
-
return [];
|
|
223
|
-
// ── Preferred path: SSM-embedding cosine similarity ───────────────────
|
|
224
|
-
if (runtime != null && typeof runtime.embed === 'function') {
|
|
225
|
-
const queryVec = await this._embedWithCache(runtime, query);
|
|
226
|
-
if (queryVec) {
|
|
227
|
-
// EVM-1b: sync the persistent vector store + index (embeds only the
|
|
228
|
-
// delta), then query it — no per-call re-embed of every fact.
|
|
229
|
-
const synced = await this._syncDense(runtime, all);
|
|
230
|
-
if (synced) {
|
|
231
|
-
const hits = synced.index
|
|
232
|
-
// EVM-1: above the threshold, query the persistent HNSW (O(log N)).
|
|
233
|
-
? synced.index.search(queryVec, topK)
|
|
234
|
-
// Below it, an exact scan over the (already-embedded) vectors.
|
|
235
|
-
: denseSearch(queryVec, synced.items, topK, this._annThreshold);
|
|
236
|
-
return hits
|
|
237
|
-
.map(h => { const entry = synced.byKey.get(h.id); return entry ? { entry, score: h.score } : null; })
|
|
238
|
-
.filter((e) => !!e);
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
// Any failure falls through to the Jaccard path below.
|
|
242
|
-
}
|
|
243
|
-
// ── Fallback: Jaccard word-overlap similarity ─────────────────────────
|
|
244
|
-
const queryTokens = new Set(tokenize(query));
|
|
245
|
-
const scored = all.map(entry => {
|
|
246
|
-
const entryTokens = new Set(tokenize(entry.content));
|
|
247
|
-
const score = jaccardSimilarity(queryTokens, entryTokens);
|
|
248
|
-
return { entry, score };
|
|
249
|
-
});
|
|
250
|
-
scored.sort((a, b) => b.score - a.score);
|
|
251
|
-
return scored.slice(0, topK);
|
|
205
|
+
return this._recall.similar(await this.recallAll(), query, topK, runtime);
|
|
252
206
|
}
|
|
253
207
|
/**
|
|
254
|
-
* Hybrid recall:
|
|
255
|
-
*
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
* (embeddings) and exact-token matches (BM25 — identifiers, codes, rare names)
|
|
259
|
-
* that cosine-only `recallSimilar` misses, and avoids returning near-duplicate
|
|
260
|
-
* facts. Degrades to BM25-only when no embedding-capable runtime is available,
|
|
261
|
-
* so it is always strictly at least as good as the lexical fallback.
|
|
208
|
+
* Hybrid recall: dense (SSM-embedding cosine) and sparse (BM25) rankings
|
|
209
|
+
* fused by Reciprocal Rank Fusion, then an MMR diversity rerank — catches
|
|
210
|
+
* exact-token matches cosine misses, and degrades to BM25 alone without a
|
|
211
|
+
* runtime. The production RAG path.
|
|
262
212
|
*/
|
|
263
213
|
async recallHybrid(query, topK, runtime, opts) {
|
|
264
214
|
return (await this.recallRanked(query, topK, runtime, opts)).hits.map(h => h.entry);
|
|
265
215
|
}
|
|
266
216
|
/**
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
*
|
|
271
|
-
* there is exactly one implementation of the blend — and it is the shared
|
|
272
|
-
* `hybridRetrieve` → `reciprocalRankFusion` primitive, never a second hand-
|
|
273
|
-
* rolled mix of cosine and word overlap.
|
|
274
|
-
*
|
|
275
|
-
* `method` is honest rather than aspirational: `'embedding'` means the query
|
|
276
|
-
* really did embed and the dense ranking really did participate in the fusion;
|
|
277
|
-
* `'lexical'` means no model was available (or embedding failed) and the
|
|
278
|
-
* ordering is BM25 alone. Callers surface it so a user can tell semantic
|
|
279
|
-
* recall from the fallback instead of guessing.
|
|
217
|
+
* {@link recallHybrid} with its evidence: each hit's fused score, and WHICH
|
|
218
|
+
* ranker produced the ordering — `'embedding'` only when the query and the
|
|
219
|
+
* candidates really embedded, `'lexical'` when the ordering is BM25 alone.
|
|
220
|
+
* One blend implementation: the shared `hybridRetrieve` primitive.
|
|
280
221
|
*/
|
|
281
222
|
async recallRanked(query, topK, runtime, opts) {
|
|
282
|
-
|
|
283
|
-
if (all.length === 0)
|
|
284
|
-
return { hits: [], method: 'lexical' };
|
|
285
|
-
// Embed candidates + query where a runtime is available; null vectors are
|
|
286
|
-
// fine — hybridRetrieve degrades that candidate to BM25-only.
|
|
287
|
-
const canEmbed = runtime != null && typeof runtime.embed === 'function';
|
|
288
|
-
const queryVec = canEmbed ? (await this._embedWithCache(runtime, query)) ?? undefined : undefined;
|
|
289
|
-
const candidates = [];
|
|
290
|
-
for (const entry of all) {
|
|
291
|
-
const vector = queryVec ? (await this._embedWithCache(runtime, entry.content)) ?? undefined : undefined;
|
|
292
|
-
candidates.push({ id: entry.key, text: entry.content, vector });
|
|
293
|
-
}
|
|
294
|
-
// The dense ranking only exists when the query embedded AND at least one
|
|
295
|
-
// candidate did — anything less is a lexical ordering wearing a label.
|
|
296
|
-
const dense = !!queryVec && candidates.some(c => !!c.vector);
|
|
297
|
-
const hits = hybridRetrieve({ text: query, vector: queryVec }, candidates, { topK, annThreshold: this._annThreshold, ...opts });
|
|
298
|
-
const byKey = new Map(all.map(e => [e.key, e]));
|
|
299
|
-
return {
|
|
300
|
-
hits: hits
|
|
301
|
-
.map(h => { const entry = byKey.get(h.id); return entry ? { entry, score: h.score } : null; })
|
|
302
|
-
.filter((h) => !!h),
|
|
303
|
-
method: dense ? 'embedding' : 'lexical',
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
/**
|
|
307
|
-
* Returns a cached embedding for `text`, computing it via `runtime.embed()`
|
|
308
|
-
* on a cache miss. Returns `null` (never throws) when embedding is
|
|
309
|
-
* unavailable so callers can fall back to lexical similarity.
|
|
310
|
-
*/
|
|
311
|
-
async _embedWithCache(runtime, text) {
|
|
312
|
-
// Embeddings are model-specific: if the runtime instance changed, the cached
|
|
313
|
-
// vectors are from a different model and must be dropped. (EVM-1b)
|
|
314
|
-
if (runtime !== this._embedRuntime) {
|
|
315
|
-
this._embedCache.clear();
|
|
316
|
-
this._embedRuntime = runtime;
|
|
317
|
-
}
|
|
318
|
-
const cached = this._embedCache.get(text);
|
|
319
|
-
if (cached) {
|
|
320
|
-
// LRU touch: re-insert so this entry becomes most-recently-used. (EVM-8)
|
|
321
|
-
this._embedCache.delete(text);
|
|
322
|
-
this._embedCache.set(text, cached);
|
|
323
|
-
return cached;
|
|
324
|
-
}
|
|
325
|
-
try {
|
|
326
|
-
const vec = await runtime.embed(text);
|
|
327
|
-
if (vec instanceof Float32Array && vec.length > 0) {
|
|
328
|
-
// EVM-8: bound cache growth by evicting the LEAST-recently-used entry
|
|
329
|
-
// (Map keeps insertion order; the first key is the LRU), instead of
|
|
330
|
-
// dumping the whole cache on every overflow.
|
|
331
|
-
if (this._embedCache.size >= this._embedCacheMax) {
|
|
332
|
-
const lru = this._embedCache.keys().next().value;
|
|
333
|
-
if (lru !== undefined)
|
|
334
|
-
this._embedCache.delete(lru);
|
|
335
|
-
}
|
|
336
|
-
this._embedCache.set(text, vec);
|
|
337
|
-
return vec;
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
catch {
|
|
341
|
-
// Embedding unavailable (no GPU, destroyed runtime, etc.) — signal fallback.
|
|
342
|
-
}
|
|
343
|
-
return null;
|
|
223
|
+
return this._recall.ranked(await this.recallAll(), query, topK, runtime, opts);
|
|
344
224
|
}
|
|
345
225
|
/**
|
|
346
|
-
*
|
|
347
|
-
*
|
|
348
|
-
*
|
|
349
|
-
* workload reuses both the embeddings and the index across recalls. Returns
|
|
350
|
-
* `null` (caller falls back to lexical) if any embedding is unavailable, or
|
|
351
|
-
* resets everything when the runtime identity changes (embeddings are
|
|
352
|
-
* model-specific).
|
|
353
|
-
*/
|
|
354
|
-
async _syncDense(runtime, all) {
|
|
355
|
-
if (runtime !== this._denseRuntime) {
|
|
356
|
-
this._denseVectors.clear();
|
|
357
|
-
this._denseIndex = null;
|
|
358
|
-
this._denseSig = '';
|
|
359
|
-
this._denseRuntime = runtime;
|
|
360
|
-
}
|
|
361
|
-
const vkeyOf = (e) => `${e.key}@${e.seq ?? 0}`;
|
|
362
|
-
const currentVkeys = new Set();
|
|
363
|
-
let sig = `${all.length}`;
|
|
364
|
-
for (const e of all) {
|
|
365
|
-
const k = vkeyOf(e);
|
|
366
|
-
currentVkeys.add(k);
|
|
367
|
-
sig += '|' + k;
|
|
368
|
-
}
|
|
369
|
-
const byKey = new Map();
|
|
370
|
-
const items = [];
|
|
371
|
-
for (const entry of all) {
|
|
372
|
-
const vk = vkeyOf(entry);
|
|
373
|
-
let vec = this._denseVectors.get(vk);
|
|
374
|
-
if (!vec) {
|
|
375
|
-
const v = await this._embedWithCache(runtime, entry.content);
|
|
376
|
-
if (!v)
|
|
377
|
-
return null; // embedding failed → lexical fallback
|
|
378
|
-
vec = v;
|
|
379
|
-
this._denseVectors.set(vk, vec);
|
|
380
|
-
}
|
|
381
|
-
items.push({ id: entry.key, vector: vec });
|
|
382
|
-
byKey.set(entry.key, entry);
|
|
383
|
-
}
|
|
384
|
-
// Drop vectors for facts that are gone or were edited (stale vkeys).
|
|
385
|
-
if (this._denseVectors.size > currentVkeys.size) {
|
|
386
|
-
for (const k of this._denseVectors.keys()) {
|
|
387
|
-
if (!currentVkeys.has(k))
|
|
388
|
-
this._denseVectors.delete(k);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
// Maintain the ANN index only above the threshold; (re)build it only when
|
|
392
|
-
// the fact set changed (no embedding involved — purely in-memory).
|
|
393
|
-
if (all.length >= this._annThreshold) {
|
|
394
|
-
if (sig !== this._denseSig || this._denseIndex === null) {
|
|
395
|
-
const index = new HnswIndex();
|
|
396
|
-
for (const it of items)
|
|
397
|
-
index.add(it.id, it.vector);
|
|
398
|
-
this._denseIndex = index;
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
else {
|
|
402
|
-
this._denseIndex = null;
|
|
403
|
-
}
|
|
404
|
-
this._denseSig = sig;
|
|
405
|
-
return { byKey, items, index: this._denseIndex };
|
|
406
|
-
}
|
|
407
|
-
/**
|
|
408
|
-
* Hard-deletes all entries whose TTL has expired.
|
|
226
|
+
* Deletes all entries whose TTL has expired. With sync enabled each expiry
|
|
227
|
+
* commits a tombstone + journal entry in the same transaction, so expiries
|
|
228
|
+
* propagate instead of resurrecting from a replica (M06).
|
|
409
229
|
* @returns The number of entries deleted.
|
|
410
230
|
*/
|
|
411
231
|
async purgeExpired() {
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
const tx = db.transaction(FACTS_STORE, 'readonly');
|
|
415
|
-
const all = await requestToPromise(tx.objectStore(FACTS_STORE).getAll(), 'Failed to scan facts for purge', (r) => r);
|
|
416
|
-
const expired = all.filter(e => this._isExpired(e));
|
|
417
|
-
if (expired.length === 0)
|
|
418
|
-
return 0;
|
|
419
|
-
await Promise.all(expired.map(e => this.forget(e.key)));
|
|
420
|
-
return expired.length;
|
|
232
|
+
this._ready();
|
|
233
|
+
return this.repository.purgeExpired(this._scope, isExpired);
|
|
421
234
|
}
|
|
422
|
-
/**
|
|
235
|
+
/**
|
|
236
|
+
* Deletes a single fact. No-op if key does not exist. With sync enabled the
|
|
237
|
+
* delete is a durable tombstone + journal entry committed atomically (M04).
|
|
238
|
+
*/
|
|
423
239
|
async forget(key) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
return requestToPromise(tx.objectStore(FACTS_STORE).delete(key), `Failed to forget fact "${key}"`, () => undefined);
|
|
240
|
+
this._ready();
|
|
241
|
+
await this.repository.forget(this._scope, key);
|
|
427
242
|
}
|
|
428
|
-
/** Deletes all facts. Does not affect saved weights. */
|
|
243
|
+
/** Deletes all facts in this store's scope. Does not affect saved weights. */
|
|
429
244
|
async clear() {
|
|
430
|
-
const db = await this._open();
|
|
431
245
|
// Drop the persistent dense index/vectors too (EVM-1b) — they describe a
|
|
432
246
|
// fact set that no longer exists.
|
|
433
|
-
this.
|
|
434
|
-
this.
|
|
435
|
-
this.
|
|
436
|
-
|
|
437
|
-
|
|
247
|
+
this._ready();
|
|
248
|
+
this._recall.resetDense();
|
|
249
|
+
await this.repository.clear(this._scope);
|
|
250
|
+
}
|
|
251
|
+
// ── Sync protocol surface (M04) ───────────────────────────────────────────
|
|
252
|
+
/** Operations not yet acknowledged by sync — the pending knowledge set. */
|
|
253
|
+
async pendingOps() {
|
|
254
|
+
return this.repository.pendingOps();
|
|
255
|
+
}
|
|
256
|
+
/** Delete tombstones in this store's scope (or all scopes when omitted). */
|
|
257
|
+
async tombstones(scope) {
|
|
258
|
+
return this.repository.tombstones(scope);
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Acknowledges that the given operations reached the sync target. Their
|
|
262
|
+
* facts/tombstones stop being `pendingSync`, so retention may touch them
|
|
263
|
+
* again. Retention NEVER touches unacknowledged knowledge (M06).
|
|
264
|
+
*/
|
|
265
|
+
async markSynced(opIds) {
|
|
266
|
+
await this.repository.markSynced(opIds);
|
|
267
|
+
}
|
|
268
|
+
/** The current durable store generation. */
|
|
269
|
+
async revision() {
|
|
270
|
+
return this.repository.revision();
|
|
271
|
+
}
|
|
272
|
+
/** Releases cross-context subscriptions and the underlying connection. */
|
|
273
|
+
close() {
|
|
274
|
+
this.stopTtlSweeper();
|
|
275
|
+
this._unsubscribe?.();
|
|
276
|
+
this._unsubscribe = undefined;
|
|
277
|
+
this._storage.close?.();
|
|
438
278
|
}
|
|
439
279
|
// ── Cross-session memory merge ────────────────────────────────────────────
|
|
440
280
|
/**
|
|
441
281
|
* Returns all non-expired facts as a plain array for export.
|
|
442
|
-
*
|
|
282
|
+
* Scope, revision and provenance travel with each entry (M03).
|
|
443
283
|
*/
|
|
444
284
|
async exportAll() {
|
|
445
285
|
return this.recallAll();
|
|
446
286
|
}
|
|
447
287
|
/**
|
|
448
|
-
* Imports entries from an external array.
|
|
288
|
+
* Imports entries from an external array INTO this store's scope partition.
|
|
289
|
+
* Source timestamps, sequences, tags, importance, TTL and provenance are
|
|
290
|
+
* preserved — import never rewrites source time (M06).
|
|
449
291
|
*
|
|
450
292
|
* - `'merge'` : only writes an entry if no existing entry with the same
|
|
451
293
|
* key exists, or if the incoming entry has a newer timestamp.
|
|
452
294
|
* - `'overwrite'` : writes all entries unconditionally.
|
|
453
295
|
*/
|
|
454
|
-
async importAll(entries, strategy) {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
}
|
|
459
|
-
else {
|
|
460
|
-
const existing = await this.recall(entry.key);
|
|
461
|
-
if (existing == null || entry.timestamp > existing.timestamp) {
|
|
462
|
-
await this._putRaw(entry);
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
/** Writes a raw MemoryEntry directly (preserves original timestamp / metadata). */
|
|
468
|
-
async _putRaw(entry) {
|
|
469
|
-
const db = await this._open();
|
|
470
|
-
const tx = db.transaction(FACTS_STORE, 'readwrite');
|
|
471
|
-
return requestToPromise(tx.objectStore(FACTS_STORE).put(entry), `Failed to import fact "${entry.key}"`, () => undefined);
|
|
296
|
+
async importAll(entries, strategy, importedFrom) {
|
|
297
|
+
this._ready();
|
|
298
|
+
const records = entries.map((e) => toImportRecord(e, this._scope));
|
|
299
|
+
await this.repository.importRecords(this._scope, records, strategy, importedFrom);
|
|
472
300
|
}
|
|
473
301
|
// ── Weight persistence ────────────────────────────────────────────────────
|
|
474
302
|
/**
|
|
475
303
|
* Saves SSM weights via `runtime.save()`.
|
|
476
|
-
* The weights are stored under `weightsKey` in
|
|
304
|
+
* The weights are stored under `weightsKey` in the session's IndexedDB,
|
|
477
305
|
* separate from MambaSession's own key.
|
|
478
306
|
*/
|
|
479
307
|
async saveWeights(runtime) {
|
|
@@ -487,22 +315,6 @@ export class MemoryStore {
|
|
|
487
315
|
return runtime.load({ key: this._weightsKey });
|
|
488
316
|
}
|
|
489
317
|
}
|
|
490
|
-
//
|
|
491
|
-
|
|
492
|
-
* Wires an IDBRequest to a promise: resolves with `map(result)` on success,
|
|
493
|
-
* rejects with a normalised MEMORY_UNAVAILABLE error on failure. Extracted so
|
|
494
|
-
* the success/error wiring lives in one place instead of being duplicated across
|
|
495
|
-
* every store method.
|
|
496
|
-
*
|
|
497
|
-
* The onerror branch fires only on IndexedDB storage faults (quota exceeded,
|
|
498
|
-
* disk failure, corruption), which the in-memory fake-indexeddb used in tests
|
|
499
|
-
* cannot reproduce — hence the istanbul ignore on that single line.
|
|
500
|
-
*/
|
|
501
|
-
function requestToPromise(req, failMsg, map) {
|
|
502
|
-
return new Promise((resolve, reject) => {
|
|
503
|
-
req.onsuccess = () => resolve(map(req.result));
|
|
504
|
-
/* istanbul ignore next -- IDB storage-fault path; not reproducible with fake-indexeddb */
|
|
505
|
-
req.onerror = () => reject(new SSMError('MEMORY_UNAVAILABLE', `${failMsg}: ${req.error?.message ?? 'unknown'}`, req.error ?? undefined));
|
|
506
|
-
});
|
|
507
|
-
}
|
|
318
|
+
// Re-export the storage seam so consumers reach it through the store's module.
|
|
319
|
+
export { IdbStorageAdapter, ScopedFactRepository, MemoryStorageError, };
|
|
508
320
|
//# sourceMappingURL=MemoryStore.js.map
|