@hviana/sema 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/dist/example/demo.js +25 -25
  2. package/dist/example/train_base.d.ts +21 -19
  3. package/dist/example/train_base.js +1952 -1684
  4. package/dist/src/alphabet.d.ts +3 -3
  5. package/dist/src/alphabet.js +27 -24
  6. package/dist/src/alu/src/alu.d.ts +194 -169
  7. package/dist/src/alu/src/alu.js +414 -374
  8. package/dist/src/alu/src/expr.d.ts +43 -36
  9. package/dist/src/alu/src/expr.js +278 -252
  10. package/dist/src/alu/src/index.d.ts +98 -9
  11. package/dist/src/alu/src/index.js +63 -9
  12. package/dist/src/alu/src/kernel-arith.d.ts +5 -1
  13. package/dist/src/alu/src/kernel-arith.js +285 -195
  14. package/dist/src/alu/src/kernel-bits.d.ts +5 -1
  15. package/dist/src/alu/src/kernel-bits.js +115 -81
  16. package/dist/src/alu/src/kernel-logic.js +63 -33
  17. package/dist/src/alu/src/kernel-nd.js +195 -155
  18. package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
  19. package/dist/src/alu/src/kernel-numeric.js +331 -259
  20. package/dist/src/alu/src/operation.d.ts +132 -112
  21. package/dist/src/alu/src/operation.js +163 -149
  22. package/dist/src/alu/src/parser.d.ts +175 -168
  23. package/dist/src/alu/src/parser.js +421 -382
  24. package/dist/src/alu/src/resonance.d.ts +25 -16
  25. package/dist/src/alu/src/resonance.js +56 -49
  26. package/dist/src/alu/src/text.d.ts +7 -3
  27. package/dist/src/alu/src/text.js +37 -31
  28. package/dist/src/alu/src/value.d.ts +14 -14
  29. package/dist/src/alu/src/value.js +160 -150
  30. package/dist/src/alu/test/alu.test.js +656 -543
  31. package/dist/src/bytes.d.ts +5 -1
  32. package/dist/src/bytes.js +40 -33
  33. package/dist/src/config.d.ts +102 -102
  34. package/dist/src/config.js +83 -83
  35. package/dist/src/derive/src/deduction.d.ts +62 -60
  36. package/dist/src/derive/src/deduction.js +109 -103
  37. package/dist/src/derive/src/index.d.ts +7 -1
  38. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  39. package/dist/src/derive/src/priority-queue.js +61 -57
  40. package/dist/src/derive/src/rewrite.d.ts +18 -15
  41. package/dist/src/derive/src/rewrite.js +67 -59
  42. package/dist/src/derive/src/trie.d.ts +56 -56
  43. package/dist/src/derive/src/trie.js +179 -175
  44. package/dist/src/derive/test/derive.test.js +100 -96
  45. package/dist/src/extension.d.ts +16 -13
  46. package/dist/src/geometry.d.ts +42 -17
  47. package/dist/src/geometry.js +266 -235
  48. package/dist/src/index.d.ts +17 -2
  49. package/dist/src/index.js +7 -1
  50. package/dist/src/ingest-cache.d.ts +30 -25
  51. package/dist/src/ingest-cache.js +127 -108
  52. package/dist/src/mind/articulation.d.ts +5 -1
  53. package/dist/src/mind/articulation.js +112 -77
  54. package/dist/src/mind/attention.d.ts +108 -40
  55. package/dist/src/mind/attention.js +871 -749
  56. package/dist/src/mind/canonical.d.ts +19 -4
  57. package/dist/src/mind/canonical.js +31 -27
  58. package/dist/src/mind/graph-search.d.ts +225 -201
  59. package/dist/src/mind/graph-search.js +821 -742
  60. package/dist/src/mind/index.d.ts +10 -2
  61. package/dist/src/mind/junction.d.ts +58 -31
  62. package/dist/src/mind/junction.js +237 -172
  63. package/dist/src/mind/learning.d.ts +52 -16
  64. package/dist/src/mind/learning.js +165 -143
  65. package/dist/src/mind/match.d.ts +69 -20
  66. package/dist/src/mind/match.js +318 -259
  67. package/dist/src/mind/mechanisms/alu.js +16 -16
  68. package/dist/src/mind/mechanisms/cast.d.ts +13 -9
  69. package/dist/src/mind/mechanisms/cast.js +456 -363
  70. package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
  71. package/dist/src/mind/mechanisms/confluence.js +183 -152
  72. package/dist/src/mind/mechanisms/cover.d.ts +8 -2
  73. package/dist/src/mind/mechanisms/cover.js +210 -148
  74. package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
  75. package/dist/src/mind/mechanisms/extraction.js +288 -234
  76. package/dist/src/mind/mechanisms/recall.d.ts +10 -6
  77. package/dist/src/mind/mechanisms/recall.js +185 -126
  78. package/dist/src/mind/mind.d.ts +154 -129
  79. package/dist/src/mind/mind.js +288 -258
  80. package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
  81. package/dist/src/mind/pipeline-mechanism.js +172 -161
  82. package/dist/src/mind/pipeline.d.ts +14 -4
  83. package/dist/src/mind/pipeline.js +189 -125
  84. package/dist/src/mind/primitives.d.ts +32 -8
  85. package/dist/src/mind/primitives.js +117 -99
  86. package/dist/src/mind/rationale.d.ts +98 -86
  87. package/dist/src/mind/rationale.js +121 -113
  88. package/dist/src/mind/reasoning.d.ts +13 -2
  89. package/dist/src/mind/reasoning.js +166 -129
  90. package/dist/src/mind/recognition.d.ts +4 -1
  91. package/dist/src/mind/recognition.js +208 -183
  92. package/dist/src/mind/resonance.d.ts +22 -5
  93. package/dist/src/mind/resonance.js +0 -0
  94. package/dist/src/mind/trace.d.ts +25 -6
  95. package/dist/src/mind/trace.js +58 -50
  96. package/dist/src/mind/traverse.d.ts +58 -16
  97. package/dist/src/mind/traverse.js +357 -304
  98. package/dist/src/mind/types.d.ts +127 -120
  99. package/dist/src/mind/types.js +69 -60
  100. package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
  101. package/dist/src/rabitq-hnsw/src/database.js +359 -342
  102. package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
  103. package/dist/src/rabitq-hnsw/src/heap.js +80 -75
  104. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
  105. package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
  106. package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
  107. package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
  108. package/dist/src/rabitq-hnsw/src/prng.js +27 -25
  109. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
  110. package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
  111. package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
  112. package/dist/src/rabitq-hnsw/src/store.js +843 -752
  113. package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
  114. package/dist/src/sema.d.ts +13 -9
  115. package/dist/src/sema.js +40 -26
  116. package/dist/src/store-sqlite.d.ts +166 -144
  117. package/dist/src/store-sqlite.js +660 -578
  118. package/dist/src/store.d.ts +630 -593
  119. package/dist/src/store.js +1581 -1440
  120. package/dist/src/vec.d.ts +9 -5
  121. package/dist/src/vec.js +73 -61
  122. package/package.json +2 -2
package/dist/src/store.js CHANGED
@@ -55,14 +55,14 @@ const CHAIN_DEPTH_CAP = 65_536;
55
55
  * D·4 bytes (max ~4096), well under the argument limit; the key is far
56
56
  * cheaper than the ANN query it deduplicates. */
57
57
  function vecKey(v) {
58
- const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
- // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
- // engine argument limits and is slower than a few bounded calls.
61
- let s = "";
62
- for (let i = 0; i < bytes.length; i += 8192) {
63
- s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
- }
65
- return s;
58
+ const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
+ // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
+ // engine argument limits and is slower than a few bounded calls.
61
+ let s = "";
62
+ for (let i = 0; i < bytes.length; i += 8192) {
63
+ s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
+ }
65
+ return s;
66
66
  }
67
67
  /** Bounded map with LRU eviction and byte accounting. On get the entry
68
68
  * moves to the most-recent end; on set the least-recently-used entries are
@@ -71,129 +71,138 @@ function vecKey(v) {
71
71
  * for uniform caches). A miss only costs a little extra work later, never
72
72
  * correctness. */
73
73
  export class BoundedMap {
74
- maxBytes;
75
- sizeOf;
76
- evict;
77
- m = new Map();
78
- _bytes = 0;
79
- // Persistent eviction cursor over the Map's insertion order. A fresh
80
- // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
- // deletions leave in V8's ordered backing store (compacted only on rehash),
82
- // making each eviction O(size) once the map first fills — the training-rate
83
- // cliff at scale. A persistent iterator passes each hole exactly once
84
- // (V8 map iterators stay valid under mutation): amortized O(1).
85
- _cursor = null;
86
- // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
- // from the cursor, so the LRU window never rescans from the front.
88
- _candidates = [];
89
- constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
- this.maxBytes = maxBytes;
91
- this.sizeOf = sizeOf;
92
- this.evict = evict;
93
- }
94
- /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
- * off; wraps to the front when exhausted. Undefined only when empty. */
96
- nextOldest() {
97
- for (let wrapped = false;;) {
98
- if (this._cursor === null)
99
- this._cursor = this.m.keys();
100
- const n = this._cursor.next();
101
- if (!n.done)
102
- return n.value;
103
- this._cursor = null;
104
- if (this.m.size === 0 || wrapped)
105
- return undefined;
106
- wrapped = true;
74
+ maxBytes;
75
+ sizeOf;
76
+ evict;
77
+ m = new Map();
78
+ _bytes = 0;
79
+ // Persistent eviction cursor over the Map's insertion order. A fresh
80
+ // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
+ // deletions leave in V8's ordered backing store (compacted only on rehash),
82
+ // making each eviction O(size) once the map first fills — the training-rate
83
+ // cliff at scale. A persistent iterator passes each hole exactly once
84
+ // (V8 map iterators stay valid under mutation): amortized O(1).
85
+ _cursor = null;
86
+ // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
+ // from the cursor, so the LRU window never rescans from the front.
88
+ _candidates = [];
89
+ constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
+ this.maxBytes = maxBytes;
91
+ this.sizeOf = sizeOf;
92
+ this.evict = evict;
93
+ }
94
+ /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
+ * off; wraps to the front when exhausted. Undefined only when empty. */
96
+ nextOldest() {
97
+ for (let wrapped = false;;) {
98
+ if (this._cursor === null) {
99
+ this._cursor = this.m.keys();
100
+ }
101
+ const n = this._cursor.next();
102
+ if (!n.done) {
103
+ return n.value;
104
+ }
105
+ this._cursor = null;
106
+ if (this.m.size === 0 || wrapped) {
107
+ return undefined;
108
+ }
109
+ wrapped = true;
110
+ }
111
+ }
112
+ get(k) {
113
+ const v = this.m.get(k);
114
+ if (v !== undefined) {
115
+ this.m.delete(k);
116
+ this.m.set(k, v);
117
+ }
118
+ return v;
119
+ }
120
+ /** Membership without touching LRU order — a pure peek, for callers that only
121
+ * need "is this key present?" and must not promote it to most-recent. */
122
+ has(k) {
123
+ return this.m.has(k);
124
+ }
125
+ set(k, v) {
126
+ const old = this.m.get(k);
127
+ if (old !== undefined) {
128
+ this._bytes -= this.sizeOf(old);
129
+ this.m.delete(k);
130
+ }
131
+ this.m.set(k, v);
132
+ this._bytes += this.sizeOf(v);
133
+ while (this._bytes > this.maxBytes && this.m.size > 0) {
134
+ if (this.evict === "smallest") {
135
+ // Among the oldest LRU candidates, evict the cheapest to rebuild.
136
+ // Window grows logarithmically with cache size — wide enough to
137
+ // find a cheap victim without scanning the whole map. Candidates
138
+ // are pulled from the persistent cursor and carried between
139
+ // evictions, so the window never rescans the map from the front.
140
+ const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
141
+ this._candidates = this._candidates.filter((k) => this.m.has(k));
142
+ while (this._candidates.length < WINDOW) {
143
+ const k = this.nextOldest();
144
+ if (k === undefined) {
145
+ break;
146
+ }
147
+ if (!this._candidates.includes(k)) {
148
+ this._candidates.push(k);
149
+ }
107
150
  }
108
- }
109
- get(k) {
110
- const v = this.m.get(k);
111
- if (v !== undefined) {
112
- this.m.delete(k);
113
- this.m.set(k, v);
151
+ let bestI = -1;
152
+ let bestSz = Infinity;
153
+ for (let i = 0; i < this._candidates.length; i++) {
154
+ const sz = this.sizeOf(this.m.get(this._candidates[i]));
155
+ if (sz < bestSz) {
156
+ bestSz = sz;
157
+ bestI = i;
158
+ }
114
159
  }
115
- return v;
116
- }
117
- /** Membership without touching LRU order — a pure peek, for callers that only
118
- * need "is this key present?" and must not promote it to most-recent. */
119
- has(k) {
120
- return this.m.has(k);
121
- }
122
- set(k, v) {
123
- const old = this.m.get(k);
124
- if (old !== undefined) {
125
- this._bytes -= this.sizeOf(old);
126
- this.m.delete(k);
160
+ if (bestI < 0) {
161
+ break;
127
162
  }
128
- this.m.set(k, v);
129
- this._bytes += this.sizeOf(v);
130
- while (this._bytes > this.maxBytes && this.m.size > 0) {
131
- if (this.evict === "smallest") {
132
- // Among the oldest LRU candidates, evict the cheapest to rebuild.
133
- // Window grows logarithmically with cache size — wide enough to
134
- // find a cheap victim without scanning the whole map. Candidates
135
- // are pulled from the persistent cursor and carried between
136
- // evictions, so the window never rescans the map from the front.
137
- const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
138
- this._candidates = this._candidates.filter((k) => this.m.has(k));
139
- while (this._candidates.length < WINDOW) {
140
- const k = this.nextOldest();
141
- if (k === undefined)
142
- break;
143
- if (!this._candidates.includes(k))
144
- this._candidates.push(k);
145
- }
146
- let bestI = -1;
147
- let bestSz = Infinity;
148
- for (let i = 0; i < this._candidates.length; i++) {
149
- const sz = this.sizeOf(this.m.get(this._candidates[i]));
150
- if (sz < bestSz) {
151
- bestSz = sz;
152
- bestI = i;
153
- }
154
- }
155
- if (bestI < 0)
156
- break;
157
- const bestK = this._candidates[bestI];
158
- this._candidates.splice(bestI, 1);
159
- this._bytes -= bestSz;
160
- this.m.delete(bestK);
161
- }
162
- else {
163
- const lru = this.nextOldest();
164
- if (lru === undefined)
165
- break;
166
- const lruv = this.m.get(lru);
167
- if (lruv === undefined)
168
- continue;
169
- this._bytes -= this.sizeOf(lruv);
170
- this.m.delete(lru);
171
- }
163
+ const bestK = this._candidates[bestI];
164
+ this._candidates.splice(bestI, 1);
165
+ this._bytes -= bestSz;
166
+ this.m.delete(bestK);
167
+ } else {
168
+ const lru = this.nextOldest();
169
+ if (lru === undefined) {
170
+ break;
172
171
  }
173
- }
174
- get size() {
175
- return this.m.size;
176
- }
177
- get bytes() {
178
- return this._bytes;
179
- }
180
- /** Remove one entry (point invalidation), with byte accounting. */
181
- delete(k) {
182
- const v = this.m.get(k);
183
- if (v === undefined)
184
- return;
185
- this._bytes -= this.sizeOf(v);
186
- this.m.delete(k);
187
- }
188
- /** Drop every entry (bulk invalidation) O(1) amortised via fresh maps. */
189
- clear() {
190
- if (this.m.size === 0)
191
- return;
192
- this.m = new Map();
193
- this._bytes = 0;
194
- this._cursor = null;
195
- this._candidates = [];
196
- }
172
+ const lruv = this.m.get(lru);
173
+ if (lruv === undefined) {
174
+ continue;
175
+ }
176
+ this._bytes -= this.sizeOf(lruv);
177
+ this.m.delete(lru);
178
+ }
179
+ }
180
+ }
181
+ get size() {
182
+ return this.m.size;
183
+ }
184
+ get bytes() {
185
+ return this._bytes;
186
+ }
187
+ /** Remove one entry (point invalidation), with byte accounting. */
188
+ delete(k) {
189
+ const v = this.m.get(k);
190
+ if (v === undefined) {
191
+ return;
192
+ }
193
+ this._bytes -= this.sizeOf(v);
194
+ this.m.delete(k);
195
+ }
196
+ /** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
197
+ clear() {
198
+ if (this.m.size === 0) {
199
+ return;
200
+ }
201
+ this.m = new Map();
202
+ this._bytes = 0;
203
+ this._cursor = null;
204
+ this._candidates = [];
205
+ }
197
206
  }
198
207
  // ── Serialisation utilities (pure functions, no DB dependency) ───────────
199
208
  const _ZERO = new Uint8Array(0);
@@ -210,26 +219,28 @@ const _ZERO = new Uint8Array(0);
210
219
  * exit mid-`ingest` whenever the event loop had nothing else alive (a bare
211
220
  * top-level deposit loop) — the await simply never resolved. */
212
221
  function yieldToEventLoop() {
213
- return new Promise((resolve) => {
214
- const g = globalThis;
215
- if (g.setImmediate)
216
- g.setImmediate(resolve);
217
- else
218
- g.setTimeout(resolve, 0);
219
- });
222
+ return new Promise((resolve) => {
223
+ const g = globalThis;
224
+ if (g.setImmediate) {
225
+ g.setImmediate(resolve);
226
+ } else {
227
+ g.setTimeout(resolve, 0);
228
+ }
229
+ });
220
230
  }
221
231
  /** Concatenate an array of byte arrays. */
222
232
  function concat(parts) {
223
- let n = 0;
224
- for (const p of parts)
225
- n += p.length;
226
- const out = new Uint8Array(n);
227
- let o = 0;
228
- for (const p of parts) {
229
- out.set(p, o);
230
- o += p.length;
231
- }
232
- return out;
233
+ let n = 0;
234
+ for (const p of parts) {
235
+ n += p.length;
236
+ }
237
+ const out = new Uint8Array(n);
238
+ let o = 0;
239
+ for (const p of parts) {
240
+ out.set(p, o);
241
+ o += p.length;
242
+ }
243
+ return out;
233
244
  }
234
245
  /** The byte string a FLAT branch spans — a branch whose kids are ALL implicit
235
246
  * single-byte leaves (ids −256…−1) is fully determined by its bytes, one per
@@ -241,55 +252,58 @@ function concat(parts) {
241
252
  * and content-addressed through the same leaf index instead of a second
242
253
  * blob-duplicating kids index. */
243
254
  function flatKidsBytes(kids) {
244
- const out = new Uint8Array(kids.length);
245
- for (let i = 0; i < kids.length; i++) {
246
- const k = kids[i];
247
- if (k < -256 || k >= 0)
248
- return null;
249
- out[i] = -(k + 1);
250
- }
251
- return out;
255
+ const out = new Uint8Array(kids.length);
256
+ for (let i = 0; i < kids.length; i++) {
257
+ const k = kids[i];
258
+ if (k < -256 || k >= 0) {
259
+ return null;
260
+ }
261
+ out[i] = -(k + 1);
262
+ }
263
+ return out;
252
264
  }
253
265
  /** The implicit kid list of a flat branch — the inverse of
254
266
  * {@link flatKidsBytes}. */
255
267
  export function flatBytesKids(bytes) {
256
- const out = new Array(bytes.length);
257
- for (let i = 0; i < bytes.length; i++)
258
- out[i] = -(bytes[i] + 1);
259
- return out;
268
+ const out = new Array(bytes.length);
269
+ for (let i = 0; i < bytes.length; i++) {
270
+ out[i] = -(bytes[i] + 1);
271
+ }
272
+ return out;
260
273
  }
261
274
  /** Pack a child-id list as little-endian int32s — 4 bytes per child, far more
262
275
  * compact than a space-joined decimal string and trivial to read back. */
263
276
  export function packKids(kids) {
264
- const out = new Int32Array(kids.length);
265
- for (let i = 0; i < kids.length; i++)
266
- out[i] = kids[i];
267
- return new Uint8Array(out.buffer);
277
+ const out = new Int32Array(kids.length);
278
+ for (let i = 0; i < kids.length; i++) {
279
+ out[i] = kids[i];
280
+ }
281
+ return new Uint8Array(out.buffer);
268
282
  }
269
283
  export function unpackKids(blob) {
270
- // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
271
- // reinterpreting as int32.
272
- const i32 = new Int32Array(blob.slice().buffer);
273
- return Array.from(i32);
284
+ // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
285
+ // reinterpreting as int32.
286
+ const i32 = new Int32Array(blob.slice().buffer);
287
+ return Array.from(i32);
274
288
  }
275
289
  /** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
276
290
  * on. Collisions are resolved by verifying the stored blob, never trusted. */
277
291
  function hashOf(bytes) {
278
- let h = 0x811c9dc5 >>> 0;
279
- for (let i = 0; i < bytes.length; i++) {
280
- h ^= bytes[i];
281
- h = Math.imul(h, 0x01000193) >>> 0;
282
- }
283
- return h >>> 0;
292
+ let h = 0x811c9dc5 >>> 0;
293
+ for (let i = 0; i < bytes.length; i++) {
294
+ h ^= bytes[i];
295
+ h = Math.imul(h, 0x01000193) >>> 0;
296
+ }
297
+ return h >>> 0;
284
298
  }
285
299
  /** Fast content key (FNV-1a + length). No array spread onto the call stack. */
286
300
  function keyOf(bytes) {
287
- return hashOf(bytes).toString(16) + ":" + bytes.length;
301
+ return hashOf(bytes).toString(16) + ":" + bytes.length;
288
302
  }
289
303
  /** Re-index a halo only when its mass is small or a power of two — O(log mass)
290
304
  * writes per node instead of O(mass). */
291
305
  function geometricMass(mass) {
292
- return mass <= 4 || (mass & (mass - 1)) === 0;
306
+ return mass <= 4 || (mass & (mass - 1)) === 0;
293
307
  }
294
308
  // Halo accumulators are stored 2-bit quantized: the exact float32 L2 norm
295
309
  // followed by D two-bit codes (sign + magnitude class) — 4 + D/4 bytes, 16×
@@ -314,41 +328,47 @@ const HALO_Q_THRESHOLD = 0.9816; // Lloyd-Max decision point, unit Gaussian
314
328
  const HALO_Q_LO = 0.4528; // reconstruction level, inner cell
315
329
  const HALO_Q_HI = 1.5104; // reconstruction level, outer cell
316
330
  function haloEncode(v) {
317
- const D = v.length;
318
- const out = new Uint8Array(4 + ((D + 3) >> 2));
319
- let n2 = 0;
320
- for (let i = 0; i < D; i++)
321
- n2 += v[i] * v[i];
322
- const norm = Math.sqrt(n2);
323
- new DataView(out.buffer).setFloat32(0, norm, true);
324
- const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
325
- for (let i = 0; i < D; i++) {
326
- const x = v[i];
327
- // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
328
- const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
329
- out[4 + (i >> 2)] |= code << ((i & 3) << 1);
330
- }
331
- return out;
331
+ const D = v.length;
332
+ const out = new Uint8Array(4 + ((D + 3) >> 2));
333
+ let n2 = 0;
334
+ for (let i = 0; i < D; i++) {
335
+ n2 += v[i] * v[i];
336
+ }
337
+ const norm = Math.sqrt(n2);
338
+ new DataView(out.buffer).setFloat32(0, norm, true);
339
+ const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
340
+ for (let i = 0; i < D; i++) {
341
+ const x = v[i];
342
+ // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
343
+ const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
344
+ out[4 + (i >> 2)] |= code << ((i & 3) << 1);
345
+ }
346
+ return out;
332
347
  }
333
348
  function haloDecode(blob, D) {
334
- const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(0, true);
335
- const out = new Float32Array(D);
336
- if (norm === 0)
337
- return out;
338
- const sigma = norm / Math.sqrt(D);
339
- let n2 = 0;
340
- for (let i = 0; i < D; i++) {
341
- const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
342
- const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
343
- const x = code & 1 ? mag : -mag;
344
- out[i] = x;
345
- n2 += x * x;
346
- }
347
- // Rescale to the exact stored norm so accumulation stays magnitude-true.
348
- const s = norm / Math.sqrt(n2);
349
- for (let i = 0; i < D; i++)
350
- out[i] *= s;
349
+ const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(
350
+ 0,
351
+ true,
352
+ );
353
+ const out = new Float32Array(D);
354
+ if (norm === 0) {
351
355
  return out;
356
+ }
357
+ const sigma = norm / Math.sqrt(D);
358
+ let n2 = 0;
359
+ for (let i = 0; i < D; i++) {
360
+ const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
361
+ const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
362
+ const x = code & 1 ? mag : -mag;
363
+ out[i] = x;
364
+ n2 += x * x;
365
+ }
366
+ // Rescale to the exact stored norm so accumulation stays magnitude-true.
367
+ const s = norm / Math.sqrt(n2);
368
+ for (let i = 0; i < D; i++) {
369
+ out[i] *= s;
370
+ }
371
+ return out;
352
372
  }
353
373
  // ── AbstractStore: template-method base with all domain logic ────────────
354
374
  /**
@@ -359,1260 +379,1381 @@ function haloDecode(blob, D) {
359
379
  * protected abstract methods that talk to the actual storage backend.
360
380
  */
361
381
  export class AbstractStore {
362
- // ── Config ─────────────────────────────────────────────────────────────
363
- _D;
364
- _maxGroup;
365
- minHaloMass;
366
- efSearch;
367
- m;
368
- efConstruction;
369
- efConstructionInterior;
370
- overfetch;
371
- batchSize;
372
- compactEveryNWrites;
373
- // ── State ──────────────────────────────────────────────────────────────
374
- /** Branch node ids are a dense, monotonically-increasing integer sequence
375
- * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
376
- * They are NEVER deleted, so the count of minted branch ids IS the next id —
377
- * which doubles as the branch-node count and lets has() be an O(1) check.
378
- * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
379
- _nextId = 0;
380
- _writtenSinceCompact = 0;
381
- closed = false;
382
- /** Lifecycle guard — resolved once `_dbOpen()` completes. */
383
- _ready = null;
384
- // ── Caches ─────────────────────────────────────────────────────────────
385
- /** Exact-content dedup: content-key → node id. Intrinsic compression. */
386
- _leafKey;
387
- _branchKey;
388
- /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
389
- _bytesCache;
390
- /** contentLen memo — content is immutable, so entries never invalidate. */
391
- _lenCache;
392
- /** Node-record cache — avoids repeated persistence queries for shared DAG
393
- * nodes. Each record is small (a few ints + short leaf buffer). */
394
- _recCache;
395
- /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
396
- * links/pours a node right after interning it, so the working set is one
397
- * deposit's nodes — a small budget captures ~all of it, and an eviction only
398
- * means that node is reached by the DAG climb instead of by direct
399
- * resonance. */
400
- _pendingGist;
401
- /** EXACT halo accumulators for the session's live pours: full-precision in
402
- * memory, 2-bit on disk, so within-session accumulate-then-compare never
403
- * round-trips through the quantizer. Regenerable — a miss reads the durable
404
- * 2-bit row. */
405
- _haloExact;
406
- /** NORMALIZED halo read cache — the decoded, normalized vector {@link halo}
407
- * returns, cached by id so repeat reads skip the per-call 2-bit decode and
408
- * normalize of a full D-element row (measured on a trained store: ~15K
409
- * halo() calls per deep query over ~50 distinct ids — all but the first
410
- * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
411
- * halo mutation site. Callers receive a COPY, so the cached vector is
412
- * never aliased. Regenerable — a miss re-decodes the durable row. */
413
- _haloNorm;
414
- /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
415
- * that bridge nothing). Remembered so subsequent visits prune the subtree
416
- * without re-checking parent count. LRU-bounded: an evicted entry is
417
- * re-checked on next visit — if it gained parents in the meantime, it will
418
- * be promoted to the index. */
419
- _coveredIds;
420
- /** Live content-index id set, LRU-bounded so a massive ingest never leaks
421
- * memory; an evicted entry is still indexed (the row is durable), so the
422
- * only cost of an eviction is a duplicate HNSW probe on next visit. */
423
- _indexedIds;
424
- // ── ANN read cache ─────────────────────────────────────────────────────
425
- // The index is read-only between writes, so the same (v,k) always returns
426
- // the same neighbours. Any index mutation (flush, delete, compact) drops
427
- // the cache; the next miss recreates it. Content-addressed (vecKey), not
428
- // identity-addressed — same principle as perceiveMemo.
429
- /** ANN read cache for {@link resonate} — keyed by vecKey(v) + ":" + k;
430
- * lazily initialised, dropped on any index mutation. */
431
- _resonateCache = null;
432
- /** ANN read cache for {@link resonateHalo} — same scheme. */
433
- _resonateHaloCache = null;
434
- // ── Write buffers ──────────────────────────────────────────────────────
435
- /** Content (gist) index write buffer. `ef` is the per-entry HNSW
436
- * construction budget: reach-only interiors carry the reduced
437
- * `efConstructionInterior`; dedup targets omit it (full budget). */
438
- _contentBuffer = [];
439
- /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
440
- _haloBuffer = new Map();
441
- /** Containment write buffer: child → new parents, merged on flush cadence. */
442
- _containBuf = new Map();
443
- /** Dedup-target candidates still in the write buffer (keyed by id). Only
444
- * roots that have gained an edge/halo are targets; a fresh intermediate
445
- * branch is never folded onto. */
446
- _nearDedupBuf = new Map();
447
- /** Ids currently in `_contentBuffer` (not yet flushed) — O(1) membership. */
448
- _bufferedIds = new Set();
449
- // ── Transparent-chain cache ────────────────────────────────────────────
450
- /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
451
- * writes: a chain is a pure function of the kid and edge tables, so any
452
- * write that could break a node's transparency (a fresh mint inserting kid
453
- * rows, a link inserting an edge) drops the whole cache — see the two
454
- * invalidation sites. Regenerable; a miss re-walks. */
455
- _chainMemo;
456
- // ── Edge-source-count cache ────────────────────────────────────────────
457
- /** Distinct edge-source count — the store's DOCUMENT COUNT (how many
458
- * learnt contexts predict a continuation), the N of every
459
- * inverse-document-frequency read. −1 until first asked for; from then
460
- * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
461
- * so a read is O(1) — never a table scan on the recall path. */
462
- _edgeSrcCount = -1;
463
- // ── Constructor ────────────────────────────────────────────────────────
464
- constructor(config, D, maxGroup) {
465
- this._D = D;
466
- this._maxGroup = maxGroup;
467
- this.minHaloMass = config.minHaloMass;
468
- this.efSearch = config.efSearch;
469
- this.m = config.m;
470
- this.efConstruction = config.efConstruction;
471
- this.efConstructionInterior = Math.max(1, Math.min(config.efConstructionInterior, config.efConstruction));
472
- this.overfetch = config.overfetch;
473
- this.batchSize = config.batchSize;
474
- this.compactEveryNWrites = config.compactEveryNWrites;
475
- this._leafKey = new BoundedMap(config.dedupCacheMax);
476
- this._branchKey = new BoundedMap(config.dedupCacheMax);
477
- this._bytesCache = new BoundedMap(config.bytesCacheMax, (v) => v.byteLength, "smallest");
478
- this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
479
- this._recCache = new BoundedMap(config.recCacheBytes, (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12);
480
- this._pendingGist = new BoundedMap(config.pendingGistBytes, (v) => v.byteLength);
481
- this._haloExact = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
482
- this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
483
- this._coveredIds = new BoundedMap(config.coveredIdsMax);
484
- this._indexedIds = new BoundedMap(config.coveredIdsMax);
485
- this._chainMemo = new BoundedMap(config.chainCacheBytes, (v) => v.length * 4 + 32);
486
- }
487
- // ── Public accessors ───────────────────────────────────────────────────
488
- get D() {
489
- return this._D;
490
- }
491
- /** Await the async initialisation performed by the concrete constructor. */
492
- async _ensureReady() {
493
- if (!this._ready)
494
- throw new Error("Store: not open");
495
- await this._ready;
496
- }
497
- // ── Id management ──────────────────────────────────────────────────────
498
- has(id) {
499
- // Byte leaves (negative ids) always exist.
500
- if (id < 0)
501
- return id >= -256;
502
- return Number.isInteger(id) && id >= 0 && id < this._nextId;
503
- }
504
- mintId() {
505
- return this._nextId++;
506
- }
507
- nodeCount() {
508
- return this._nextId;
509
- }
510
- async size() {
511
- await this._ensureReady();
512
- return this._nextId;
513
- }
514
- // ── DAG traversal ──────────────────────────────────────────────────────
515
- get(id) {
516
- // Byte leaves are implicit — fabricate from the id.
517
- if (id < 0) {
518
- return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
519
- }
520
- const hit = this._recCache.get(id);
521
- if (hit !== undefined)
522
- return hit;
523
- const rec = this._dbGetNode(id);
524
- if (rec)
525
- this._recCache.set(id, rec);
526
- return rec;
527
- }
528
- /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
529
- * Iterative post-order on an explicit stack — the call stack never sees the
530
- * tree depth, so even an adversarial chain of nodes stays safe. */
531
- /** How many reads hit a MISSING node record this session (a dangling edge
532
- * or kid id). Zero in a healthy store; a growing count means references
533
- * outlive their records — the read degrades safely to empty bytes, this
534
- * counter is what keeps that degradation observable. */
535
- danglingReads = 0;
536
- bytes(id) {
537
- // Fast path.
538
- const hit = this._bytesCache.get(id);
539
- if (hit)
540
- return hit;
541
- const stack = [id];
542
- const cache = this._bytesCache;
543
- while (stack.length > 0) {
544
- const nid = stack[stack.length - 1]; // peek
545
- // Already resolved by an earlier traversal.
546
- if (cache.get(nid)) {
547
- stack.pop();
548
- continue;
549
- }
550
- const rec = this.get(nid);
551
- if (!rec) {
552
- // A DANGLING id (an edge or kid pointing at no record) reads as
553
- // empty safe (empty-bytes guards drop it from grounding) but a
554
- // symptom of store corruption, so count it rather than stay silent.
555
- // The cache makes the empty read permanent for the session; the
556
- // counter survives as the visible trace.
557
- this.danglingReads++;
558
- cache.set(nid, _ZERO);
559
- stack.pop();
560
- continue;
561
- }
562
- if (rec.leaf) {
563
- cache.set(nid, new Uint8Array(rec.leaf));
564
- stack.pop();
565
- continue;
566
- }
567
- // Branch — push any uncached children (reverse order so they resolve
568
- // left-to-right). If every child is already cached, concatenate now.
569
- const kids = rec.kids ?? [];
570
- let ready = true;
571
- for (let i = kids.length - 1; i >= 0; i--) {
572
- if (!cache.get(kids[i])) {
573
- stack.push(kids[i]);
574
- ready = false;
575
- }
576
- }
577
- if (!ready)
578
- continue;
579
- stack.pop();
580
- const out = concat(kids.map((k) => cache.get(k)));
581
- cache.set(nid, out);
582
- }
583
- return cache.get(id) ?? _ZERO;
584
- }
585
- /** First `maxLen` bytes of a node. Walks only the leftmost branch,
586
- * stopping at `maxLen` — so a 1 MB document root costs the same as a
587
- * 4-byte leaf. Recursive, but tree depth is logarithmic.
588
- *
589
- * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
590
- * may be shared with the byte cache and with other callers — treat them
591
- * as read-only. Mutating one would corrupt every subsequent read. */
592
- bytesPrefix(id, maxLen) {
593
- if (maxLen <= 0)
594
- return _ZERO;
595
- // A FULL read (the ALL sentinel) routes through bytes(), whose
596
- // reconstruction enters the byte-budget cache. Without this, the mind's
597
- // read() — which always passes ALL — re-walked the DAG and re-concatenated
598
- // on EVERY repeated read of an uncached branch, bypassing the cache that
599
- // exists precisely for those reconstructions.
600
- if (maxLen >= 0x7fffffff)
601
- return this.bytes(id);
602
- // Full-cache hit: bytes() already reconstructed the whole node.
603
- const full = this._bytesCache.get(id);
604
- if (full)
605
- return full.length <= maxLen ? full : full.subarray(0, maxLen);
606
- const rec = this.get(id);
607
- if (!rec)
608
- return _ZERO;
609
- if (rec.leaf) {
610
- // Cache the (small) leaf bytes — cheap and reusable. COPY before
611
- // caching: rec.leaf is the node record's own buffer, and handing it
612
- // out would let one mutating caller corrupt the record AND the cache
613
- // (bytes() makes the same copy for the same reason).
614
- const leaf = new Uint8Array(rec.leaf);
615
- this._bytesCache.set(id, leaf);
616
- return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
617
- }
618
- // Branch — walk children left-to-right, stopping at maxLen.
619
- const kids = rec.kids ?? [];
620
- const parts = [];
621
- let got = 0;
622
- for (const k of kids) {
623
- if (got >= maxLen)
624
- break;
625
- const child = this.bytesPrefix(k, maxLen - got);
626
- parts.push(child);
627
- got += child.length;
628
- }
629
- return concat(parts);
630
- }
631
- contentLen(id, cap = Infinity) {
632
- if (id < 0)
633
- return 1; // implicit single-byte leaf
634
- const hit = this._lenCache.get(id);
635
- if (hit !== undefined)
636
- return hit; // exact — valid under any cap
637
- if (cap <= 0)
638
- return 0;
639
- const rec = this.get(id);
640
- let n = 0;
641
- let clamped = false;
642
- if (rec) {
643
- if (rec.leaf)
644
- n = rec.leaf.length;
645
- else if (rec.kids) {
646
- for (const k of rec.kids) {
647
- n += this.contentLen(k, cap - n);
648
- if (n >= cap) {
649
- clamped = true; // partial sum — a lower bound, not the length
650
- break;
651
- }
652
- }
653
- }
382
+ // ── Config ─────────────────────────────────────────────────────────────
383
+ _D;
384
+ _maxGroup;
385
+ minHaloMass;
386
+ efSearch;
387
+ m;
388
+ efConstruction;
389
+ efConstructionInterior;
390
+ overfetch;
391
+ batchSize;
392
+ compactEveryNWrites;
393
+ // ── State ──────────────────────────────────────────────────────────────
394
+ /** Branch node ids are a dense, monotonically-increasing integer sequence
395
+ * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
396
+ * They are NEVER deleted, so the count of minted branch ids IS the next id —
397
+ * which doubles as the branch-node count and lets has() be an O(1) check.
398
+ * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
399
+ _nextId = 0;
400
+ _writtenSinceCompact = 0;
401
+ closed = false;
402
+ /** Lifecycle guard — resolved once `_dbOpen()` completes. */
403
+ _ready = null;
404
+ // ── Caches ─────────────────────────────────────────────────────────────
405
+ /** Exact-content dedup: content-key → node id. Intrinsic compression. */
406
+ _leafKey;
407
+ _branchKey;
408
+ /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
409
+ _bytesCache;
410
+ /** contentLen memo — content is immutable, so entries never invalidate. */
411
+ _lenCache;
412
+ /** Node-record cache — avoids repeated persistence queries for shared DAG
413
+ * nodes. Each record is small (a few ints + short leaf buffer). */
414
+ _recCache;
415
+ /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
416
+ * links/pours a node right after interning it, so the working set is one
417
+ * deposit's nodes — a small budget captures ~all of it, and an eviction only
418
+ * means that node is reached by the DAG climb instead of by direct
419
+ * resonance. */
420
+ _pendingGist;
421
+ /** EXACT halo accumulators for the session's live pours: full-precision in
422
+ * memory, 2-bit on disk, so within-session accumulate-then-compare never
423
+ * round-trips through the quantizer. Regenerable — a miss reads the durable
424
+ * 2-bit row. */
425
+ _haloExact;
426
+ /** NORMALIZED halo read cache — the decoded, normalized vector {@link halo}
427
+ * returns, cached by id so repeat reads skip the per-call 2-bit decode and
428
+ * normalize of a full D-element row (measured on a trained store: ~15K
429
+ * halo() calls per deep query over ~50 distinct ids — all but the first
430
+ * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
431
+ * halo mutation site. Callers receive a COPY, so the cached vector is
432
+ * never aliased. Regenerable — a miss re-decodes the durable row. */
433
+ _haloNorm;
434
+ /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
435
+ * that bridge nothing). Remembered so subsequent visits prune the subtree
436
+ * without re-checking parent count. LRU-bounded: an evicted entry is
437
+ * re-checked on next visit — if it gained parents in the meantime, it will
438
+ * be promoted to the index. */
439
+ _coveredIds;
440
+ /** Live content-index id set, LRU-bounded so a massive ingest never leaks
441
+ * memory; an evicted entry is still indexed (the row is durable), so the
442
+ * only cost of an eviction is a duplicate HNSW probe on next visit. */
443
+ _indexedIds;
444
+ // ── ANN read cache ─────────────────────────────────────────────────────
445
+ // The index is read-only between writes, so the same (v,k) always returns
446
+ // the same neighbours. Any index mutation (flush, delete, compact) drops
447
+ // the cache; the next miss recreates it. Content-addressed (vecKey), not
448
+ // identity-addressed — same principle as perceiveMemo.
449
+ /** ANN read cache for {@link resonate} — keyed by vecKey(v) + ":" + k;
450
+ * lazily initialised, dropped on any index mutation. */
451
+ _resonateCache = null;
452
+ /** ANN read cache for {@link resonateHalo} — same scheme. */
453
+ _resonateHaloCache = null;
454
+ // ── Write buffers ──────────────────────────────────────────────────────
455
+ /** Content (gist) index write buffer. `ef` is the per-entry HNSW
456
+ * construction budget: reach-only interiors carry the reduced
457
+ * `efConstructionInterior`; dedup targets omit it (full budget). */
458
+ _contentBuffer = [];
459
+ /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
460
+ _haloBuffer = new Map();
461
+ /** Containment write buffer: child → new parents, merged on flush cadence. */
462
+ _containBuf = new Map();
463
+ /** Dedup-target candidates still in the write buffer (keyed by id). Only
464
+ * roots that have gained an edge/halo are targets; a fresh intermediate
465
+ * branch is never folded onto. */
466
+ _nearDedupBuf = new Map();
467
+ /** Ids currently in `_contentBuffer` (not yet flushed) — O(1) membership. */
468
+ _bufferedIds = new Set();
469
+ // ── Transparent-chain cache ────────────────────────────────────────────
470
+ /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
471
+ * writes: a chain is a pure function of the kid and edge tables, so any
472
+ * write that could break a node's transparency (a fresh mint inserting kid
473
+ * rows, a link inserting an edge) drops the whole cache — see the two
474
+ * invalidation sites. Regenerable; a miss re-walks. */
475
+ _chainMemo;
476
+ // ── Edge-source-count cache ────────────────────────────────────────────
477
+ /** Distinct edge-source count — the store's DOCUMENT COUNT (how many
478
+ * learnt contexts predict a continuation), the N of every
479
+ * inverse-document-frequency read. −1 until first asked for; from then
480
+ * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
481
+ * so a read is O(1) — never a table scan on the recall path. */
482
+ _edgeSrcCount = -1;
483
+ // ── Constructor ────────────────────────────────────────────────────────
484
+ constructor(config, D, maxGroup) {
485
+ this._D = D;
486
+ this._maxGroup = maxGroup;
487
+ this.minHaloMass = config.minHaloMass;
488
+ this.efSearch = config.efSearch;
489
+ this.m = config.m;
490
+ this.efConstruction = config.efConstruction;
491
+ this.efConstructionInterior = Math.max(
492
+ 1,
493
+ Math.min(config.efConstructionInterior, config.efConstruction),
494
+ );
495
+ this.overfetch = config.overfetch;
496
+ this.batchSize = config.batchSize;
497
+ this.compactEveryNWrites = config.compactEveryNWrites;
498
+ this._leafKey = new BoundedMap(config.dedupCacheMax);
499
+ this._branchKey = new BoundedMap(config.dedupCacheMax);
500
+ this._bytesCache = new BoundedMap(
501
+ config.bytesCacheMax,
502
+ (v) => v.byteLength,
503
+ "smallest",
504
+ );
505
+ this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
506
+ this._recCache = new BoundedMap(
507
+ config.recCacheBytes,
508
+ (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12,
509
+ );
510
+ this._pendingGist = new BoundedMap(
511
+ config.pendingGistBytes,
512
+ (v) => v.byteLength,
513
+ );
514
+ this._haloExact = new BoundedMap(
515
+ config.haloCacheBytes,
516
+ (v) => v.byteLength,
517
+ );
518
+ this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
519
+ this._coveredIds = new BoundedMap(config.coveredIdsMax);
520
+ this._indexedIds = new BoundedMap(config.coveredIdsMax);
521
+ this._chainMemo = new BoundedMap(
522
+ config.chainCacheBytes,
523
+ (v) => v.length * 4 + 32,
524
+ );
525
+ }
526
+ // ── Public accessors ───────────────────────────────────────────────────
527
+ get D() {
528
+ return this._D;
529
+ }
530
+ /** Await the async initialisation performed by the concrete constructor. */
531
+ async _ensureReady() {
532
+ if (!this._ready) {
533
+ throw new Error("Store: not open");
534
+ }
535
+ await this._ready;
536
+ }
537
+ // ── Id management ──────────────────────────────────────────────────────
538
+ has(id) {
539
+ // Byte leaves (negative ids) always exist.
540
+ if (id < 0) {
541
+ return id >= -256;
542
+ }
543
+ return Number.isInteger(id) && id >= 0 && id < this._nextId;
544
+ }
545
+ mintId() {
546
+ return this._nextId++;
547
+ }
548
+ nodeCount() {
549
+ return this._nextId;
550
+ }
551
+ async size() {
552
+ await this._ensureReady();
553
+ return this._nextId;
554
+ }
555
+ // ── DAG traversal ──────────────────────────────────────────────────────
556
+ get(id) {
557
+ // Byte leaves are implicit — fabricate from the id.
558
+ if (id < 0) {
559
+ return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
560
+ }
561
+ const hit = this._recCache.get(id);
562
+ if (hit !== undefined) {
563
+ return hit;
564
+ }
565
+ const rec = this._dbGetNode(id);
566
+ if (rec) {
567
+ this._recCache.set(id, rec);
568
+ }
569
+ return rec;
570
+ }
571
+ /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
572
+ * Iterative post-order on an explicit stack the call stack never sees the
573
+ * tree depth, so even an adversarial chain of nodes stays safe. */
574
+ /** How many reads hit a MISSING node record this session (a dangling edge
575
+ * or kid id). Zero in a healthy store; a growing count means references
576
+ * outlive their records the read degrades safely to empty bytes, this
577
+ * counter is what keeps that degradation observable. */
578
+ danglingReads = 0;
579
+ bytes(id) {
580
+ // Fast path.
581
+ const hit = this._bytesCache.get(id);
582
+ if (hit) {
583
+ return hit;
584
+ }
585
+ const stack = [id];
586
+ const cache = this._bytesCache;
587
+ while (stack.length > 0) {
588
+ const nid = stack[stack.length - 1]; // peek
589
+ // Already resolved by an earlier traversal.
590
+ if (cache.get(nid)) {
591
+ stack.pop();
592
+ continue;
593
+ }
594
+ const rec = this.get(nid);
595
+ if (!rec) {
596
+ // A DANGLING id (an edge or kid pointing at no record) reads as
597
+ // empty — safe (empty-bytes guards drop it from grounding) but a
598
+ // symptom of store corruption, so count it rather than stay silent.
599
+ // The cache makes the empty read permanent for the session; the
600
+ // counter survives as the visible trace.
601
+ this.danglingReads++;
602
+ cache.set(nid, _ZERO);
603
+ stack.pop();
604
+ continue;
605
+ }
606
+ if (rec.leaf) {
607
+ cache.set(nid, new Uint8Array(rec.leaf));
608
+ stack.pop();
609
+ continue;
610
+ }
611
+ // Branch push any uncached children (reverse order so they resolve
612
+ // left-to-right). If every child is already cached, concatenate now.
613
+ const kids = rec.kids ?? [];
614
+ let ready = true;
615
+ for (let i = kids.length - 1; i >= 0; i--) {
616
+ if (!cache.get(kids[i])) {
617
+ stack.push(kids[i]);
618
+ ready = false;
654
619
  }
655
- if (!clamped)
656
- this._lenCache.set(id, n);
657
- return n;
658
- }
659
- // ── Content-addressed lookup ───────────────────────────────────────────
660
- findLeaf(bytes) {
661
- if (bytes.length === 1)
662
- return -(bytes[0] + 1);
663
- const key = keyOf(bytes);
664
- const cached = this._leafKey.get(key);
665
- if (cached !== undefined)
666
- return cached;
667
- const id = this._dbFindLeaf(hashOf(bytes), bytes);
668
- if (id !== null)
669
- this._leafKey.set(key, id);
670
- return id;
671
- }
672
- findBranch(kids) {
673
- const key = kids.join(",");
674
- const cached = this._branchKey.get(key);
675
- if (cached !== undefined)
676
- return cached;
677
- const flat = flatKidsBytes(kids);
678
- let id;
679
- if (flat) {
680
- id = this._dbFindBranchByLeaf(hashOf(flat), flat);
681
- }
682
- else {
683
- const packed = packKids(kids);
684
- id = this._dbFindBranchByKids(hashOf(packed), packed);
620
+ }
621
+ if (!ready) {
622
+ continue;
623
+ }
624
+ stack.pop();
625
+ const out = concat(kids.map((k) => cache.get(k)));
626
+ cache.set(nid, out);
627
+ }
628
+ return cache.get(id) ?? _ZERO;
629
+ }
630
+ /** First `maxLen` bytes of a node. Walks only the leftmost branch,
631
+ * stopping at `maxLen` — so a 1 MB document root costs the same as a
632
+ * 4-byte leaf. Recursive, but tree depth is logarithmic.
633
+ *
634
+ * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
635
+ * may be shared with the byte cache and with other callers — treat them
636
+ * as read-only. Mutating one would corrupt every subsequent read. */
637
+ bytesPrefix(id, maxLen) {
638
+ if (maxLen <= 0) {
639
+ return _ZERO;
640
+ }
641
+ // A FULL read (the ALL sentinel) routes through bytes(), whose
642
+ // reconstruction enters the byte-budget cache. Without this, the mind's
643
+ // read() — which always passes ALL — re-walked the DAG and re-concatenated
644
+ // on EVERY repeated read of an uncached branch, bypassing the cache that
645
+ // exists precisely for those reconstructions.
646
+ if (maxLen >= 0x7fffffff) {
647
+ return this.bytes(id);
648
+ }
649
+ // Full-cache hit: bytes() already reconstructed the whole node.
650
+ const full = this._bytesCache.get(id);
651
+ if (full) {
652
+ return full.length <= maxLen ? full : full.subarray(0, maxLen);
653
+ }
654
+ const rec = this.get(id);
655
+ if (!rec) {
656
+ return _ZERO;
657
+ }
658
+ if (rec.leaf) {
659
+ // Cache the (small) leaf bytes — cheap and reusable. COPY before
660
+ // caching: rec.leaf is the node record's own buffer, and handing it
661
+ // out would let one mutating caller corrupt the record AND the cache
662
+ // (bytes() makes the same copy for the same reason).
663
+ const leaf = new Uint8Array(rec.leaf);
664
+ this._bytesCache.set(id, leaf);
665
+ return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
666
+ }
667
+ // Branch — walk children left-to-right, stopping at maxLen.
668
+ const kids = rec.kids ?? [];
669
+ const parts = [];
670
+ let got = 0;
671
+ for (const k of kids) {
672
+ if (got >= maxLen) {
673
+ break;
674
+ }
675
+ const child = this.bytesPrefix(k, maxLen - got);
676
+ parts.push(child);
677
+ got += child.length;
678
+ }
679
+ return concat(parts);
680
+ }
681
+ contentLen(id, cap = Infinity) {
682
+ if (id < 0) {
683
+ return 1; // implicit single-byte leaf
684
+ }
685
+ const hit = this._lenCache.get(id);
686
+ if (hit !== undefined) {
687
+ return hit; // exact — valid under any cap
688
+ }
689
+ if (cap <= 0) {
690
+ return 0;
691
+ }
692
+ const rec = this.get(id);
693
+ let n = 0;
694
+ let clamped = false;
695
+ if (rec) {
696
+ if (rec.leaf) {
697
+ n = rec.leaf.length;
698
+ } else if (rec.kids) {
699
+ for (const k of rec.kids) {
700
+ n += this.contentLen(k, cap - n);
701
+ if (n >= cap) {
702
+ clamped = true; // partial sum — a lower bound, not the length
703
+ break;
704
+ }
685
705
  }
686
- if (id !== null && key.length <= DEDUP_KEY_MAX) {
687
- this._branchKey.set(key, id);
706
+ }
707
+ }
708
+ if (!clamped) {
709
+ this._lenCache.set(id, n);
710
+ }
711
+ return n;
712
+ }
713
+ // ── Content-addressed lookup ───────────────────────────────────────────
714
+ findLeaf(bytes) {
715
+ if (bytes.length === 1) {
716
+ return -(bytes[0] + 1);
717
+ }
718
+ const key = keyOf(bytes);
719
+ const cached = this._leafKey.get(key);
720
+ if (cached !== undefined) {
721
+ return cached;
722
+ }
723
+ const id = this._dbFindLeaf(hashOf(bytes), bytes);
724
+ if (id !== null) {
725
+ this._leafKey.set(key, id);
726
+ }
727
+ return id;
728
+ }
729
+ findBranch(kids) {
730
+ const key = kids.join(",");
731
+ const cached = this._branchKey.get(key);
732
+ if (cached !== undefined) {
733
+ return cached;
734
+ }
735
+ const flat = flatKidsBytes(kids);
736
+ let id;
737
+ if (flat) {
738
+ id = this._dbFindBranchByLeaf(hashOf(flat), flat);
739
+ } else {
740
+ const packed = packKids(kids);
741
+ id = this._dbFindBranchByKids(hashOf(packed), packed);
742
+ }
743
+ if (id !== null && key.length <= DEDUP_KEY_MAX) {
744
+ this._branchKey.set(key, id);
745
+ }
746
+ return id;
747
+ }
748
+ // ── Structural parents ─────────────────────────────────────────────────
749
+ parents(id) {
750
+ return this._dbGetParents(id);
751
+ }
752
+ parentsFirst(id, limit) {
753
+ return this._dbGetParentsFirst(id, limit);
754
+ }
755
+ hasParents(id) {
756
+ return this._dbGetParentsFirst(id, 1).length > 0;
757
+ }
758
+ chainRun(id) {
759
+ const hit = this._chainMemo.get(id);
760
+ if (hit !== undefined) {
761
+ return hit;
762
+ }
763
+ const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
764
+ this._chainMemo.set(id, run);
765
+ return run;
766
+ }
767
+ /** {@link Store.chainRun}'s walk, node at a time through the existing
768
+ * probes. Adapters with a set-based query engine should override with a
769
+ * single server-side descent (the SQLite adapter uses a recursive CTE). */
770
+ _chainWalk(id, cap) {
771
+ const run = [id];
772
+ let n = id;
773
+ while (run.length < cap) {
774
+ if (this.hasNext(n) || this.prevCount(n) > 0) {
775
+ break;
776
+ }
777
+ const ps = this._dbGetParentsFirst(n, 2);
778
+ if (ps.length !== 1) {
779
+ break;
780
+ }
781
+ n = ps[0];
782
+ run.push(n);
783
+ }
784
+ return run;
785
+ }
786
+ // ── Containment ────────────────────────────────────────────────────────
787
+ addContainer(child, parent) {
788
+ let set = this._containBuf.get(child);
789
+ if (set === undefined) {
790
+ set = new Set();
791
+ this._containBuf.set(child, set);
792
+ }
793
+ set.add(parent);
794
+ }
795
+ hasContainers(child) {
796
+ if (this._dbContainExists(child)) {
797
+ return true;
798
+ }
799
+ const buf = this._containBuf.get(child);
800
+ return buf !== undefined && buf.size > 0;
801
+ }
802
+ containersSlice(child, offset, limit) {
803
+ const out = this._dbGetContainParentsSlice(child, offset, limit);
804
+ if (out.length >= limit) {
805
+ return out;
806
+ }
807
+ // Buffered adds page in AFTER the stored ones. A buffered parent that is
808
+ // also stored may repeat across the seam; page consumers dedup by id
809
+ // (they all carry seen-sets), so a repeat costs a skip, never an error.
810
+ const buf = this._containBuf.get(child);
811
+ if (!buf || buf.size === 0) {
812
+ return out;
813
+ }
814
+ const storedCount = this._dbGetContainCount(child);
815
+ const bufStart = Math.max(0, offset - storedCount) +
816
+ Math.max(0, out.length - Math.max(0, storedCount - offset));
817
+ let i = 0;
818
+ for (const p of buf) {
819
+ if (out.length >= limit) {
820
+ break;
821
+ }
822
+ if (i++ < bufStart) {
823
+ continue;
824
+ }
825
+ out.push(p);
826
+ }
827
+ return out;
828
+ }
829
+ containers(child) {
830
+ const stored = this._dbGetContainParents(child);
831
+ const buf = this._containBuf.get(child);
832
+ if (stored.length === 0) {
833
+ return buf ? [...buf] : [];
834
+ }
835
+ if (!buf) {
836
+ return stored;
837
+ }
838
+ const merged = new Set(stored);
839
+ for (const p of buf) {
840
+ merged.add(p);
841
+ }
842
+ return [...merged];
843
+ }
844
+ // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
845
+ flatLeafIds(kids) {
846
+ const out = [];
847
+ const stack = [...kids].reverse();
848
+ while (stack.length > 0) {
849
+ const id = stack.pop();
850
+ if (id < 0) {
851
+ out.push(id);
852
+ continue;
853
+ }
854
+ const rec = this.get(id);
855
+ if (!rec) {
856
+ return null;
857
+ }
858
+ if (rec.leaf !== null) {
859
+ for (let i = 0; i < rec.leaf.length; i++) {
860
+ out.push(-(rec.leaf[i] + 1));
688
861
  }
689
- return id;
690
- }
691
- // ── Structural parents ─────────────────────────────────────────────────
692
- parents(id) {
693
- return this._dbGetParents(id);
694
- }
695
- parentsFirst(id, limit) {
696
- return this._dbGetParentsFirst(id, limit);
697
- }
698
- hasParents(id) {
699
- return this._dbGetParentsFirst(id, 1).length > 0;
700
- }
701
- chainRun(id) {
702
- const hit = this._chainMemo.get(id);
703
- if (hit !== undefined)
704
- return hit;
705
- const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
706
- this._chainMemo.set(id, run);
707
- return run;
708
- }
709
- /** {@link Store.chainRun}'s walk, node at a time through the existing
710
- * probes. Adapters with a set-based query engine should override with a
711
- * single server-side descent (the SQLite adapter uses a recursive CTE). */
712
- _chainWalk(id, cap) {
713
- const run = [id];
714
- let n = id;
715
- while (run.length < cap) {
716
- if (this.hasNext(n) || this.prevCount(n) > 0)
717
- break;
718
- const ps = this._dbGetParentsFirst(n, 2);
719
- if (ps.length !== 1)
720
- break;
721
- n = ps[0];
722
- run.push(n);
862
+ } else if (rec.kids !== null) {
863
+ for (let i = rec.kids.length - 1; i >= 0; i--) {
864
+ stack.push(rec.kids[i]);
723
865
  }
724
- return run;
725
- }
726
- // ── Containment ────────────────────────────────────────────────────────
727
- addContainer(child, parent) {
728
- let set = this._containBuf.get(child);
729
- if (set === undefined) {
730
- set = new Set();
731
- this._containBuf.set(child, set);
866
+ }
867
+ }
868
+ return out;
869
+ }
870
+ /** On a dedup HIT, keep the node's gist available for lazy indexing —
871
+ * EXACTLY when it is not already indexed. Replaces the old id-range
872
+ * "recency" heuristic (id ≥ nextId − cacheWindow), which conflated an LRU
873
+ * entry COUNT with an id RANGE and permanently refused to index any node
874
+ * that first became a resonance target long after it was minted (an early
875
+ * interior later reused as an edge/halo-bearing deposit root was silently
876
+ * unreachable by resonance). The durable index itself is the arbiter:
877
+ * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
878
+ captureIfUnindexed(id, gist) {
879
+ if (this._indexedIds.has(id) || this._pendingGist.has(id)) {
880
+ return;
881
+ }
882
+ if (this._vecContentHas(id)) {
883
+ this._indexedIds.set(id, true);
884
+ return;
885
+ }
886
+ this._pendingGist.set(id, normalize(copy(gist)));
887
+ // A node that ALREADY bridges experiences but was never indexed (its 1→2
888
+ // transition fired while its gist was evicted, or in a pre-transition
889
+ // store) is promoted on this re-encounter — the recapture above is
890
+ // exactly what makes its gist available again. Byte leaves (negative
891
+ // ids) never have kid rows, so skip their parent probe.
892
+ if (id >= 0) {
893
+ this.promoteBridge(id);
894
+ }
895
+ }
896
+ /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
897
+ * criterion), promote its gist into the content index NOW — the exact
898
+ * moment it becomes useful for multi-experience recall. The 1→2 parent
899
+ * transition fires on {@link _dbInsertKid} during mint, and nodes that
900
+ * were already bridges but missed indexing (gist evicted, pre-transition
901
+ * store) are recaptured in {@link captureIfUnindexed}.
902
+ *
903
+ * A no-op when the node is already indexed or its gist is evicted from
904
+ * the pending cache — a future re-encounter will retry. */
905
+ promoteBridge(id) {
906
+ // A LIMITed probe: this runs once per kid insert on the MINT hot path,
907
+ // and a shared child's full parent set grows with the corpus.
908
+ if (this._dbGetParentsFirst(id, 2).length < 2) {
909
+ return;
910
+ }
911
+ this.indexGist(id, false);
912
+ }
913
+ // ── Core interning: dedup → near-dedup → mint ──────────────────────────
914
+ async intern(leaf, kids, gist) {
915
+ await this._ensureReady();
916
+ // 1. Exact dedup — equal content → one id, no vector work. Primary
917
+ // compression mechanism, intrinsic to the store.
918
+ //
919
+ // findLeaf/findBranch are the content-addressed lookups: in-memory cache
920
+ // first, then a durable probe that repopulates the cache on a hit.
921
+ // Using them here (rather than only the cache) makes dedup survive a cold
922
+ // cache — a resumed/checkpointed training run, or one whose dedup cache
923
+ // has evicted old keys, still recognises content already on disk and
924
+ // reuses its id instead of minting a duplicate.
925
+ const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
926
+ if (hit !== null) {
927
+ this.captureIfUnindexed(hit, gist);
928
+ return hit;
929
+ }
930
+ // 1b. Content-addressed lookup by leaf-id signature. When the same byte
931
+ // sequence was stored as a flat branch (via putBranch during deposit),
932
+ // a branch node spanning those bytes reuses that id even when its tree
933
+ // structure differs — pure content addressing, same bytes → same node.
934
+ if (kids !== null && kids.length >= 2) {
935
+ const leafIds = this.flatLeafIds(kids);
936
+ if (leafIds !== null) {
937
+ const flatHit = this.findBranch(leafIds);
938
+ if (flatHit !== null) {
939
+ this.captureIfUnindexed(flatHit, gist);
940
+ return flatHit;
732
941
  }
733
- set.add(parent);
734
- }
735
- hasContainers(child) {
736
- if (this._dbContainExists(child))
737
- return true;
738
- const buf = this._containBuf.get(child);
739
- return buf !== undefined && buf.size > 0;
740
- }
741
- containersSlice(child, offset, limit) {
742
- const out = this._dbGetContainParentsSlice(child, offset, limit);
743
- if (out.length >= limit)
744
- return out;
745
- // Buffered adds page in AFTER the stored ones. A buffered parent that is
746
- // also stored may repeat across the seam; page consumers dedup by id
747
- // (they all carry seen-sets), so a repeat costs a skip, never an error.
748
- const buf = this._containBuf.get(child);
749
- if (!buf || buf.size === 0)
750
- return out;
751
- const storedCount = this._dbGetContainCount(child);
752
- const bufStart = Math.max(0, offset - storedCount) +
753
- Math.max(0, out.length - Math.max(0, storedCount - offset));
754
- let i = 0;
755
- for (const p of buf) {
756
- if (out.length >= limit)
757
- break;
758
- if (i++ < bufStart)
759
- continue;
760
- out.push(p);
942
+ }
943
+ }
944
+ const cache = leaf !== null ? this._leafKey : this._branchKey;
945
+ const key = leaf !== null ? keyOf(leaf) : kids.join(",");
946
+ // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
947
+ // Leaves are single bytes: exact dedup already collapses every identical
948
+ // leaf, and near-merging distinct leaves only corrupts bytes for no real
949
+ // saving. Real near-dedup compression lives in subtree (branch) fusion.
950
+ //
951
+ // There is deliberately NO HNSW probe of the FLUSHED index here. It used
952
+ // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
953
+ // ~every interior branch, since interiors are never dedup targets —
954
+ // making one ANN query per branch the dominant training cost (it dwarfed
955
+ // perception and the index write). And it was not merely expensive but
956
+ // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
957
+ // nearest "target" of a fresh branch, so the fold collapsed two
958
+ // byte-different subtrees onto one id and corrupted exact reconstruction
959
+ // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
960
+ // are caught two cheaper, exact ways instead: identical content by the
961
+ // exact-dedup hash-cons above, and a near-gist target still in the write
962
+ // buffer by the scan below.
963
+ if (leaf === null) {
964
+ // Near-dedup PREFILTER the scale-aware identity bar
965
+ // ({@link identityBar}) for THIS branch's own length, not the fixed
966
+ // estimator floor: under the linear fold a long branch crosses
967
+ // 1 1/√D while whole windows differ, and every such crossing paid a
968
+ // full differsByOneWindow byte reconstruction. Same final semantics
969
+ // (the byte check below still decides identity); strictly fewer byte
970
+ // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
971
+ // memoized bottom-up by the interning order itself, O(kids)) is only
972
+ // computed once a nearest candidate actually exists, so the hot
973
+ // no-candidate mint pays nothing.
974
+ let best = -1;
975
+ let bestId = null;
976
+ if (this._nearDedupBuf.size > 0) {
977
+ const g = normalize(copy(gist));
978
+ // Candidates are the buffered DEDUP TARGETS only — genuine whole
979
+ // experiences (edge/halo-bearing roots) not yet flushed.
980
+ for (const [id, vector] of this._nearDedupBuf) {
981
+ const s = dot(g, vector);
982
+ if (s > best) {
983
+ best = s;
984
+ bestId = id;
985
+ }
761
986
  }
762
- return out;
763
- }
764
- containers(child) {
765
- const stored = this._dbGetContainParents(child);
766
- const buf = this._containBuf.get(child);
767
- if (stored.length === 0)
768
- return buf ? [...buf] : [];
769
- if (!buf)
770
- return stored;
771
- const merged = new Set(stored);
772
- for (const p of buf)
773
- merged.add(p);
774
- return [...merged];
775
- }
776
- // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
777
- flatLeafIds(kids) {
778
- const out = [];
779
- const stack = [...kids].reverse();
780
- while (stack.length > 0) {
781
- const id = stack.pop();
782
- if (id < 0) {
783
- out.push(id);
784
- continue;
785
- }
786
- const rec = this.get(id);
787
- if (!rec)
788
- return null;
789
- if (rec.leaf !== null) {
790
- for (let i = 0; i < rec.leaf.length; i++)
791
- out.push(-(rec.leaf[i] + 1));
792
- }
793
- else if (rec.kids !== null) {
794
- for (let i = rec.kids.length - 1; i >= 0; i--)
795
- stack.push(rec.kids[i]);
796
- }
987
+ }
988
+ let blen = 0;
989
+ if (bestId !== null) {
990
+ for (const k of kids) {
991
+ blen += this.contentLen(k);
797
992
  }
798
- return out;
799
- }
800
- /** On a dedup HIT, keep the node's gist available for lazy indexing —
801
- * EXACTLY when it is not already indexed. Replaces the old id-range
802
- * "recency" heuristic (id ≥ nextId − cacheWindow), which conflated an LRU
803
- * entry COUNT with an id RANGE and permanently refused to index any node
804
- * that first became a resonance target long after it was minted (an early
805
- * interior later reused as an edge/halo-bearing deposit root was silently
806
- * unreachable by resonance). The durable index itself is the arbiter:
807
- * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
808
- captureIfUnindexed(id, gist) {
809
- if (this._indexedIds.has(id) || this._pendingGist.has(id))
810
- return;
811
- if (this._vecContentHas(id)) {
812
- this._indexedIds.set(id, true);
813
- return;
993
+ }
994
+ if (
995
+ bestId !== null &&
996
+ best >= identityBar(this.D, this._maxGroup, blen)
997
+ ) {
998
+ // Scale-aware acceptance. The cosine bar alone is scale-BLIND
999
+ // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
1000
+ // a localized difference faster than linearly in form size, so for
1001
+ // deep forms ANY fixed bar below 1 is crossed by exactly the one
1002
+ // span that distinguishes two experiences. No inversion of the
1003
+ // deficit is trustworthy, so the bytes themselves decide: the two
1004
+ // forms must be identical except for ONE local span of at most W
1005
+ // bytes — the river window, the perception's own resolution quantum.
1006
+ const W = this._maxGroup;
1007
+ if (this.differsByOneWindow(kids, bestId, W)) {
1008
+ if (key.length <= DEDUP_KEY_MAX) {
1009
+ cache.set(key, bestId);
1010
+ }
1011
+ return bestId;
814
1012
  }
815
- this._pendingGist.set(id, normalize(copy(gist)));
816
- // A node that ALREADY bridges experiences but was never indexed (its 1→2
817
- // transition fired while its gist was evicted, or in a pre-transition
818
- // store) is promoted on this re-encounter the recapture above is
819
- // exactly what makes its gist available again. Byte leaves (negative
820
- // ids) never have kid rows, so skip their parent probe.
821
- if (id >= 0)
822
- this.promoteBridge(id);
823
- }
824
- /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
825
- * criterion), promote its gist into the content index NOW — the exact
826
- * moment it becomes useful for multi-experience recall. The 1→2 parent
827
- * transition fires on {@link _dbInsertKid} during mint, and nodes that
828
- * were already bridges but missed indexing (gist evicted, pre-transition
829
- * store) are recaptured in {@link captureIfUnindexed}.
830
- *
831
- * A no-op when the node is already indexed or its gist is evicted from
832
- * the pending cache — a future re-encounter will retry. */
833
- promoteBridge(id) {
834
- // A LIMITed probe: this runs once per kid insert on the MINT hot path,
835
- // and a shared child's full parent set grows with the corpus.
836
- if (this._dbGetParentsFirst(id, 2).length < 2)
837
- return;
838
- this.indexGist(id, false);
839
- }
840
- // ── Core interning: dedup → near-dedup → mint ──────────────────────────
841
- async intern(leaf, kids, gist) {
842
- await this._ensureReady();
843
- // 1. Exact dedup equal content one id, no vector work. Primary
844
- // compression mechanism, intrinsic to the store.
845
- //
846
- // findLeaf/findBranch are the content-addressed lookups: in-memory cache
847
- // first, then a durable probe that repopulates the cache on a hit.
848
- // Using them here (rather than only the cache) makes dedup survive a cold
849
- // cache — a resumed/checkpointed training run, or one whose dedup cache
850
- // has evicted old keys, still recognises content already on disk and
851
- // reuses its id instead of minting a duplicate.
852
- const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
853
- if (hit !== null) {
854
- this.captureIfUnindexed(hit, gist);
855
- return hit;
1013
+ }
1014
+ }
1015
+ // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
1016
+ // leaf) stores its BYTES in the leaf column with a zero-length kids blob
1017
+ // as the marker the kid list is derived on read.
1018
+ const id = this.mintId();
1019
+ this._dbBeginTx();
1020
+ const flat = kids ? flatKidsBytes(kids) : null;
1021
+ const packed = kids && !flat ? packKids(kids) : null;
1022
+ this._dbInsertNode(
1023
+ id,
1024
+ leaf ?? flat,
1025
+ packed ?? (flat ? _ZERO : null),
1026
+ hashOf(leaf ?? flat ?? packed),
1027
+ );
1028
+ // Reverse structural edge: each distinct child → this parent. Lets the graph
1029
+ // be climbed upward in index time, with no scan of the kids blobs.
1030
+ //
1031
+ // Populated NATURALLY here and only here — one write per child, in the SAME
1032
+ // mint that creates the node, inside the SAME deferred transaction as the
1033
+ // node row, so node and kid rows are always durable together.
1034
+ //
1035
+ // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
1036
+ // every branch, so its parent set is the corpus-sized hub the climb's
1037
+ // saturation guard discards unread.
1038
+ if (kids) {
1039
+ // Kid rows change parent counts — a child whose parent set grows from
1040
+ // one is no longer transparent, so every cached chain that hopped
1041
+ // through it is stale. Which chains those are is unknowable without a
1042
+ // reverse index, so the WHOLE cache drops (writes come in training
1043
+ // bursts where the cache is cold anyway; queries rebuild it lazily).
1044
+ this._chainMemo.clear();
1045
+ for (const c of kids) {
1046
+ if (c < 0 && c >= -256) {
1047
+ continue;
856
1048
  }
857
- // 1b. Content-addressed lookup by leaf-id signature. When the same byte
858
- // sequence was stored as a flat branch (via putBranch during deposit),
859
- // a branch node spanning those bytes reuses that id even when its tree
860
- // structure differs pure content addressing, same bytes same node.
861
- if (kids !== null && kids.length >= 2) {
862
- const leafIds = this.flatLeafIds(kids);
863
- if (leafIds !== null) {
864
- const flatHit = this.findBranch(leafIds);
865
- if (flatHit !== null) {
866
- this.captureIfUnindexed(flatHit, gist);
867
- return flatHit;
868
- }
869
- }
1049
+ this._dbInsertKid(c, id);
1050
+ // The 1→2 parent TRANSITION happens here and only here (hash-cons
1051
+ // means an existing branch never re-inserts kid rows, so parent sets
1052
+ // grow exclusively through fresh mints): the child just became a
1053
+ // structural bridge between experiences the exact set post-hoc
1054
+ // compaction keeps — so its gist enters the reach index NOW.
1055
+ this.promoteBridge(c);
1056
+ }
1057
+ }
1058
+ if (key.length <= DEDUP_KEY_MAX) {
1059
+ cache.set(key, id);
1060
+ }
1061
+ if (leaf) {
1062
+ this._bytesCache.set(id, new Uint8Array(leaf));
1063
+ } else if (flat) {
1064
+ this._bytesCache.set(id, flat);
1065
+ }
1066
+ // Capture the gist; it is pushed into the content index lazily, the first
1067
+ // time this node becomes a resonance target (link / pourHalo). A node that
1068
+ // never does (a pure intermediate DAG node — ~99.5% of them) is never
1069
+ // indexed: it costs one persistence row, no HNSW slot and no merge probe.
1070
+ this._pendingGist.set(id, normalize(copy(gist)));
1071
+ await this.maybeFlush();
1072
+ return id;
1073
+ }
1074
+ /** Whether the byte content under `kids` and the byte content of `targetId`
1075
+ * are identical except for ONE local span of at most `W` bytes on each side
1076
+ * — the near dedup's byte-grain definition of a near-duplicate. A
1077
+ * common-prefix / common-suffix trim: whatever remains after both trims is
1078
+ * the single differing span (substitution, insertion or deletion), and both
1079
+ * remainders must fit the budget. Scattered differences leave a wide
1080
+ * middle and are rejected. */
1081
+ differsByOneWindow(kids, targetId, W) {
1082
+ const a = concat(
1083
+ kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)),
1084
+ );
1085
+ const b = this.bytesPrefix(targetId, a.length + W + 1);
1086
+ if (Math.abs(a.length - b.length) > W) {
1087
+ return false;
1088
+ }
1089
+ const n = Math.min(a.length, b.length);
1090
+ let i = 0;
1091
+ while (i < n && a[i] === b[i]) {
1092
+ i++;
1093
+ }
1094
+ let j = 0;
1095
+ while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j]) {
1096
+ j++;
1097
+ }
1098
+ return a.length - i - j <= W && b.length - i - j <= W;
1099
+ }
1100
+ async putLeaf(bytes, gist) {
1101
+ // Single bytes are implicit — no DB row and no eager index slot. The gist
1102
+ // is captured like any other node's and promoted into the content index
1103
+ // LAZILY, the first time the byte becomes a resonance target.
1104
+ if (bytes.length === 1) {
1105
+ const id = -(bytes[0] + 1);
1106
+ this.captureIfUnindexed(id, gist);
1107
+ return id;
1108
+ }
1109
+ return this.intern(new Uint8Array(bytes), null, gist);
1110
+ }
1111
+ async putBranch(kids, gist) {
1112
+ return this.intern(null, kids, gist);
1113
+ }
1114
+ // ── Lazy content indexing ──────────────────────────────────────────────
1115
+ /** Promote a node's captured gist into the content (resonance) index, once.
1116
+ * Called the first time a node becomes a target — i.e. from `link` (it bears
1117
+ * or receives a continuation edge) or `pourHalo` (it gains distributional
1118
+ * company). Idempotent: a node already indexed, or whose gist has been evicted
1119
+ * from the bounded pending map, is a no-op.
1120
+ *
1121
+ * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1122
+ * near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
1123
+ * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1124
+ indexGist(id, dedupTarget) {
1125
+ if (dedupTarget && this._bufferedIds.has(id)) {
1126
+ // Still buffered (indexed this batch, not yet flushed) — a live dedup
1127
+ // candidate, recorded in O(1) with no scan of the content buffer.
1128
+ const v = this._pendingGist.get(id);
1129
+ if (v !== undefined) {
1130
+ this._nearDedupBuf.set(id, v);
1131
+ }
1132
+ }
1133
+ if (this._indexedIds.has(id)) {
1134
+ return;
1135
+ }
1136
+ const v = this._pendingGist.get(id);
1137
+ if (v === undefined) {
1138
+ return;
1139
+ }
1140
+ // Already durably indexed by a previous session? A node id names its
1141
+ // content, and the gist is a pure function of the content, so the stored
1142
+ // vector can only be identical — re-buffering it would spend an encode
1143
+ // and an upsert on a guaranteed no-op. One point query recognises this;
1144
+ // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1145
+ // This is what makes a RESUMED training run replay already-deposited
1146
+ // content at read speed instead of re-upserting the recent-id window.
1147
+ if (this._vecContentHas(id)) {
1148
+ this._indexedIds.set(id, true);
1149
+ return;
1150
+ }
1151
+ this._indexedIds.set(id, true);
1152
+ // Reach-only interiors build with the reduced construction budget — the
1153
+ // one deliberate speed-for-quality trade of ingestion (see
1154
+ // {@link StoreConfig.efConstructionInterior}); targets keep the full one.
1155
+ this._contentBuffer.push(
1156
+ dedupTarget
1157
+ ? { id, vector: v }
1158
+ : { id, vector: v, ef: this.efConstructionInterior },
1159
+ );
1160
+ this._bufferedIds.add(id);
1161
+ // A node indexed AS a dedup target enters the candidate set immediately.
1162
+ if (dedupTarget) {
1163
+ this._nearDedupBuf.set(id, v);
1164
+ }
1165
+ }
1166
+ /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1167
+ * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1168
+ * root is a genuine target (a whole experience), so it is a dedup target
1169
+ * too. */
1170
+ indexTarget(id) {
1171
+ this.indexGist(id, true);
1172
+ }
1173
+ /** Index a node and its interior forms as resonance targets. A node that
1174
+ * gains an edge is a learnt EXPERIENCE, and the consensus climb
1175
+ * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1176
+ * resonating its SUB-REGIONS — branch nodes within the experience — and
1177
+ * climbing their parents back to it.
1178
+ *
1179
+ * EVERY interior branch is indexed unconditionally, and this is
1180
+ * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1181
+ * the post-hoc compaction criterion) was tried and REJECTED by the test
1182
+ * suite — partial recall of an experience's interior slices, multi-topic
1183
+ * attention, and counterfactual anchoring all resonate to SINGLE-parent
1184
+ * interiors (13 tests fail without them). Post-hoc structural compaction
1185
+ * ({@link compactContentIndex}) may still remove them, but that is a
1186
+ * storage/recall trade-off for archived stores, not a free optimisation.
1187
+ * The store's hash-cons bounds the index by the number of DISTINCT byte
1188
+ * patterns in the corpus — not by the number of deposits.
1189
+ *
1190
+ * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1191
+ * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1192
+ * let a partial query resonate and climb, but a fresh branch must never
1193
+ * merge onto an interior node of another experience.
1194
+ *
1195
+ * Iterative explicit-queue walk: the call stack never sees tree depth. */
1196
+ indexSubtree(root) {
1197
+ // The root is the whole experience — always index as a merge target.
1198
+ this.indexGist(root, true);
1199
+ const seen = new Set([root]);
1200
+ const stack = [root];
1201
+ while (stack.length > 0) {
1202
+ const id = stack.pop();
1203
+ const kids = this.get(id)?.kids;
1204
+ if (!kids) {
1205
+ continue; // leaf — never a resonance anchor
1206
+ }
1207
+ const isRoot = id === root;
1208
+ // Already-indexed or already-skipped nodes PRUNE the walk — their
1209
+ // subtrees were classified in a prior call.
1210
+ if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1211
+ continue;
1212
+ }
1213
+ for (const k of kids) {
1214
+ if (!seen.has(k)) {
1215
+ seen.add(k);
1216
+ stack.push(k);
870
1217
  }
871
- const cache = leaf !== null ? this._leafKey : this._branchKey;
872
- const key = leaf !== null ? keyOf(leaf) : kids.join(",");
873
- // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
874
- // Leaves are single bytes: exact dedup already collapses every identical
875
- // leaf, and near-merging distinct leaves only corrupts bytes for no real
876
- // saving. Real near-dedup compression lives in subtree (branch) fusion.
877
- //
878
- // There is deliberately NO HNSW probe of the FLUSHED index here. It used
879
- // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
880
- // ~every interior branch, since interiors are never dedup targets —
881
- // making one ANN query per branch the dominant training cost (it dwarfed
882
- // perception and the index write). And it was not merely expensive but
883
- // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
884
- // nearest "target" of a fresh branch, so the fold collapsed two
885
- // byte-different subtrees onto one id and corrupted exact reconstruction
886
- // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
887
- // are caught two cheaper, exact ways instead: identical content by the
888
- // exact-dedup hash-cons above, and a near-gist target still in the write
889
- // buffer by the scan below.
890
- if (leaf === null) {
891
- // Near-dedup PREFILTER — the scale-aware identity bar
892
- // ({@link identityBar}) for THIS branch's own length, not the fixed
893
- // estimator floor: under the linear fold a long branch crosses
894
- // 1 1/√D while whole windows differ, and every such crossing paid a
895
- // full differsByOneWindow byte reconstruction. Same final semantics
896
- // (the byte check below still decides identity); strictly fewer byte
897
- // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
898
- // memoized bottom-up by the interning order itself, O(kids)) is only
899
- // computed once a nearest candidate actually exists, so the hot
900
- // no-candidate mint pays nothing.
901
- let best = -1;
902
- let bestId = null;
903
- if (this._nearDedupBuf.size > 0) {
904
- const g = normalize(copy(gist));
905
- // Candidates are the buffered DEDUP TARGETS only — genuine whole
906
- // experiences (edge/halo-bearing roots) not yet flushed.
907
- for (const [id, vector] of this._nearDedupBuf) {
908
- const s = dot(g, vector);
909
- if (s > best) {
910
- best = s;
911
- bestId = id;
912
- }
913
- }
914
- }
915
- let blen = 0;
916
- if (bestId !== null) {
917
- for (const k of kids)
918
- blen += this.contentLen(k);
919
- }
920
- if (bestId !== null &&
921
- best >= identityBar(this.D, this._maxGroup, blen)) {
922
- // Scale-aware acceptance. The cosine bar alone is scale-BLIND
923
- // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
924
- // a localized difference faster than linearly in form size, so for
925
- // deep forms ANY fixed bar below 1 is crossed by exactly the one
926
- // span that distinguishes two experiences. No inversion of the
927
- // deficit is trustworthy, so the bytes themselves decide: the two
928
- // forms must be identical except for ONE local span of at most W
929
- // bytes — the river window, the perception's own resolution quantum.
930
- const W = this._maxGroup;
931
- if (this.differsByOneWindow(kids, bestId, W)) {
932
- if (key.length <= DEDUP_KEY_MAX)
933
- cache.set(key, bestId);
934
- return bestId;
935
- }
936
- }
1218
+ }
1219
+ if (isRoot) {
1220
+ continue;
1221
+ }
1222
+ const g = this._pendingGist.get(id);
1223
+ if (g === undefined) {
1224
+ // Gist evicted from the bounded pending cache — the node can't be
1225
+ // indexed now. Mark as covered so subsequent visits prune here
1226
+ // instead of re-walking the subtree (the gist won't return).
1227
+ this._coveredIds.set(id, true);
1228
+ continue;
1229
+ }
1230
+ // Index unconditionally every interior node is a valid resonance
1231
+ // anchor for partial-query recall. The _indexedIds cache and the
1232
+ // durable-index check in indexGist prevent duplicate indexing across
1233
+ // multiple encounters of the same shared subtree.
1234
+ this.indexGist(id, false);
1235
+ }
1236
+ }
1237
+ // ── Soft resonance ─────────────────────────────────────────────────────
1238
+ async resonate(v, k) {
1239
+ await this._ensureReady();
1240
+ // Synchronous flush of any buffered index writes: the FIRST resonance
1241
+ // after a large ingest pays that flush here, so it shows up in respond
1242
+ // latency, not ingest latency correct behaviour, skewed attribution;
1243
+ // profile accordingly.
1244
+ this.flushContent();
1245
+ if (k <= 0) {
1246
+ return [];
1247
+ }
1248
+ // ANN read cache — content-addressed so a fresh Float32Array with the
1249
+ // same values still hits. Lazy-init: null after any index write; the
1250
+ // first miss after a flush recreates it. When voteRegions resonates
1251
+ // identical perceived sub-regions, only the first call descends the ANN.
1252
+ const rk = vecKey(v) + ":" + k;
1253
+ const cache = this._resonateCache;
1254
+ if (cache) {
1255
+ const hit = cache.get(rk);
1256
+ if (hit !== undefined) {
1257
+ return hit;
1258
+ }
1259
+ }
1260
+ const results = this._vecContentQuery(
1261
+ normalize(copy(v)),
1262
+ k * this.overfetch,
1263
+ this.efSearch,
1264
+ );
1265
+ const out = [];
1266
+ for (const r of results) {
1267
+ const id = r.id;
1268
+ out.push({ id, score: 1 - r.distance });
1269
+ if (out.length >= k) {
1270
+ break;
1271
+ }
1272
+ }
1273
+ const rc = this._resonateCache ??= new Map();
1274
+ if (rc.size >= RESONATE_CACHE_MAX) {
1275
+ rc.clear();
1276
+ }
1277
+ rc.set(rk, out);
1278
+ return out;
1279
+ }
1280
+ indexedVectorCount() {
1281
+ this.flushContent();
1282
+ return this._vecContentSize();
1283
+ }
1284
+ lastResonateReads() {
1285
+ return this._vecContentLastReads();
1286
+ }
1287
+ // ── Content index compaction ────────────────────────────────────────────
1288
+ /** How many physical compaction attempts have failed this session. Zero in
1289
+ * a healthy store; a growing count means tombstones are accumulating and
1290
+ * index query cost is drifting up (the first failure also warns once). */
1291
+ compactFailures = 0;
1292
+ /** Meta key holding the incremental scan watermark of
1293
+ * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1294
+ * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1295
+ * entry once kept can never become removable — and removed entries are
1296
+ * gone, so a pass only ever needs to examine entries indexed AFTER the
1297
+ * previous pass. Internal ids are monotone and survive the index's
1298
+ * splice compaction; the watermark is reset whenever a PHYSICAL index
1299
+ * compaction runs (id reuse after a dropped top row would otherwise hide
1300
+ * new entries behind it). */
1301
+ static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1302
+ /** {@link Store.compactContentIndex} */
1303
+ async compactContentIndex(minParents = 2) {
1304
+ await this._ensureReady();
1305
+ this.flush(); // commit any pending writes first
1306
+ // Incremental scan: resume from the last pass's watermark when its
1307
+ // minParents matches; a changed criterion forces a full rescan.
1308
+ let after = 0;
1309
+ {
1310
+ const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1311
+ if (raw !== null) {
1312
+ const sep = raw.indexOf(":");
1313
+ const mp = Number(raw.slice(0, sep));
1314
+ const wm = Number(raw.slice(sep + 1));
1315
+ if (mp === minParents && Number.isFinite(wm) && wm > 0) {
1316
+ after = wm;
937
1317
  }
938
- // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
939
- // leaf) stores its BYTES in the leaf column with a zero-length kids blob
940
- // as the marker the kid list is derived on read.
941
- const id = this.mintId();
942
- this._dbBeginTx();
943
- const flat = kids ? flatKidsBytes(kids) : null;
944
- const packed = kids && !flat ? packKids(kids) : null;
945
- this._dbInsertNode(id, leaf ?? flat, packed ?? (flat ? _ZERO : null), hashOf(leaf ?? flat ?? packed));
946
- // Reverse structural edge: each distinct child → this parent. Lets the graph
947
- // be climbed upward in index time, with no scan of the kids blobs.
948
- //
949
- // Populated NATURALLY here and only here — one write per child, in the SAME
950
- // mint that creates the node, inside the SAME deferred transaction as the
951
- // node row, so node and kid rows are always durable together.
952
- //
953
- // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
954
- // every branch, so its parent set is the corpus-sized hub the climb's
955
- // saturation guard discards unread.
956
- if (kids) {
957
- // Kid rows change parent counts — a child whose parent set grows from
958
- // one is no longer transparent, so every cached chain that hopped
959
- // through it is stale. Which chains those are is unknowable without a
960
- // reverse index, so the WHOLE cache drops (writes come in training
961
- // bursts where the cache is cold anyway; queries rebuild it lazily).
962
- this._chainMemo.clear();
963
- for (const c of kids) {
964
- if (c < 0 && c >= -256)
965
- continue;
966
- this._dbInsertKid(c, id);
967
- // The 1→2 parent TRANSITION happens here and only here (hash-cons
968
- // means an existing branch never re-inserts kid rows, so parent sets
969
- // grow exclusively through fresh mints): the child just became a
970
- // structural bridge between experiences — the exact set post-hoc
971
- // compaction keeps — so its gist enters the reach index NOW.
972
- this.promoteBridge(c);
973
- }
1318
+ }
1319
+ }
1320
+ // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1321
+ // sources edge targets ∪ halo rows), materialised by a single C-side
1322
+ // scan. A binary-search membership probe replaces the three per-entry
1323
+ // point queries that made this stage minutes long on a trained store.
1324
+ const targets = this._dbEdgeOrHaloIds();
1325
+ const isTarget = (id) => {
1326
+ let lo = 0, hi = targets.length - 1;
1327
+ while (lo <= hi) {
1328
+ const mid = (lo + hi) >> 1;
1329
+ const v = targets[mid];
1330
+ if (v === id) {
1331
+ return true;
974
1332
  }
975
- if (key.length <= DEDUP_KEY_MAX)
976
- cache.set(key, id);
977
- if (leaf)
978
- this._bytesCache.set(id, new Uint8Array(leaf));
979
- else if (flat)
980
- this._bytesCache.set(id, flat);
981
- // Capture the gist; it is pushed into the content index lazily, the first
982
- // time this node becomes a resonance target (link / pourHalo). A node that
983
- // never does (a pure intermediate DAG node — ~99.5% of them) is never
984
- // indexed: it costs one persistence row, no HNSW slot and no merge probe.
985
- this._pendingGist.set(id, normalize(copy(gist)));
986
- await this.maybeFlush();
987
- return id;
988
- }
989
- /** Whether the byte content under `kids` and the byte content of `targetId`
990
- * are identical except for ONE local span of at most `W` bytes on each side
991
- * — the near dedup's byte-grain definition of a near-duplicate. A
992
- * common-prefix / common-suffix trim: whatever remains after both trims is
993
- * the single differing span (substitution, insertion or deletion), and both
994
- * remainders must fit the budget. Scattered differences leave a wide
995
- * middle and are rejected. */
996
- differsByOneWindow(kids, targetId, W) {
997
- const a = concat(kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)));
998
- const b = this.bytesPrefix(targetId, a.length + W + 1);
999
- if (Math.abs(a.length - b.length) > W)
1000
- return false;
1001
- const n = Math.min(a.length, b.length);
1002
- let i = 0;
1003
- while (i < n && a[i] === b[i])
1004
- i++;
1005
- let j = 0;
1006
- while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j])
1007
- j++;
1008
- return a.length - i - j <= W && b.length - i - j <= W;
1009
- }
1010
- async putLeaf(bytes, gist) {
1011
- // Single bytes are implicit — no DB row and no eager index slot. The gist
1012
- // is captured like any other node's and promoted into the content index
1013
- // LAZILY, the first time the byte becomes a resonance target.
1014
- if (bytes.length === 1) {
1015
- const id = -(bytes[0] + 1);
1016
- this.captureIfUnindexed(id, gist);
1017
- return id;
1333
+ if (v < id) {
1334
+ lo = mid + 1;
1335
+ } else {
1336
+ hi = mid - 1;
1018
1337
  }
1019
- return this.intern(new Uint8Array(bytes), null, gist);
1020
- }
1021
- async putBranch(kids, gist) {
1022
- return this.intern(null, kids, gist);
1023
- }
1024
- // ── Lazy content indexing ──────────────────────────────────────────────
1025
- /** Promote a node's captured gist into the content (resonance) index, once.
1026
- * Called the first time a node becomes a target i.e. from `link` (it bears
1027
- * or receives a continuation edge) or `pourHalo` (it gains distributional
1028
- * company). Idempotent: a node already indexed, or whose gist has been evicted
1029
- * from the bounded pending map, is a no-op.
1030
- *
1031
- * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1032
- * near-gist branch ONTO. Only a genuine target an edge/halo-bearing ROOT —
1033
- * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1034
- indexGist(id, dedupTarget) {
1035
- if (dedupTarget && this._bufferedIds.has(id)) {
1036
- // Still buffered (indexed this batch, not yet flushed) — a live dedup
1037
- // candidate, recorded in O(1) with no scan of the content buffer.
1038
- const v = this._pendingGist.get(id);
1039
- if (v !== undefined)
1040
- this._nearDedupBuf.set(id, v);
1338
+ }
1339
+ return false;
1340
+ };
1341
+ // Collect ids to remove: nodes that are structurally isolated (few
1342
+ // parents), not edge-bearing, and not halo-bearing. These nodes are
1343
+ // unique to one experience tree — they bridge nothing and their index
1344
+ // slots are wasted. Multi-parent nodes are the structural bridges
1345
+ // that let a partial query climb from one experience to another.
1346
+ const toRemove = [];
1347
+ let scanned = 0;
1348
+ let watermark = after;
1349
+ for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1350
+ // Yield a real event-loop turn on a fixed cadence: this scan runs over
1351
+ // every unexamined indexed entry, and point probes alone pin the
1352
+ // thread otherwise (the "frozen" symptom).
1353
+ if (++scanned % 8192 === 0) {
1354
+ await yieldToEventLoop();
1355
+ }
1356
+ watermark = internal;
1357
+ if (isTarget(id)) {
1358
+ continue;
1359
+ }
1360
+ if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1361
+ continue;
1362
+ }
1363
+ toRemove.push(id);
1364
+ }
1365
+ // Delete in batches — each batch is ONE vector-store transaction (one WAL
1366
+ // commit), with an event-loop turn between batches.
1367
+ const BATCH = 1000;
1368
+ for (let i = 0; i < toRemove.length; i += BATCH) {
1369
+ const batch = toRemove.slice(i, i + BATCH);
1370
+ this._vecContentDeleteMany(batch);
1371
+ // Purge the "already indexed" cache for removed ids: without this, a
1372
+ // node that LATER becomes a resonance target again (gains an edge or
1373
+ // halo) hits the stale cache entry in indexGist and is silently never
1374
+ // re-indexed for the rest of the session.
1375
+ for (const id of batch) {
1376
+ this._indexedIds.delete(id);
1377
+ }
1378
+ await yieldToEventLoop();
1379
+ }
1380
+ // Persist the scan watermark so the next pass starts where this one
1381
+ // ended (kept beside the criterion that produced it).
1382
+ this._dbBeginTx();
1383
+ this._dbSetMeta(
1384
+ AbstractStore.COMPACT_WATERMARK_KEY,
1385
+ minParents + ":" + watermark,
1386
+ );
1387
+ this._dbCommitTx();
1388
+ if (toRemove.length > 0) {
1389
+ // Compact to physically reclaim space from tombstones.
1390
+ try {
1391
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1392
+ this._vecContentCompact();
1393
+ // Physical compaction may free the top internal id for reuse —
1394
+ // drop the watermark so the next pass rescans from the start.
1395
+ this._dbBeginTx();
1396
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1397
+ this._dbCommitTx();
1041
1398
  }
1042
- if (this._indexedIds.has(id))
1043
- return;
1044
- const v = this._pendingGist.get(id);
1045
- if (v === undefined)
1046
- return;
1047
- // Already durably indexed by a previous session? A node id names its
1048
- // content, and the gist is a pure function of the content, so the stored
1049
- // vector can only be identical — re-buffering it would spend an encode
1050
- // and an upsert on a guaranteed no-op. One point query recognises this;
1051
- // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1052
- // This is what makes a RESUMED training run replay already-deposited
1053
- // content at read speed instead of re-upserting the recent-id window.
1054
- if (this._vecContentHas(id)) {
1055
- this._indexedIds.set(id, true);
1056
- return;
1399
+ } catch (e) {
1400
+ // Best-effort, but never SILENT: a persistently failing compaction
1401
+ // lets tombstones accumulate and query cost grow with no signal.
1402
+ this.compactFailures++;
1403
+ if (this.compactFailures === 1) {
1404
+ console.warn(
1405
+ "sema: content-index compaction failed (will keep " +
1406
+ "counting in store.compactFailures):",
1407
+ e,
1408
+ );
1057
1409
  }
1410
+ }
1411
+ // Index mutations (delete, compact) invalidate the ANN read cache.
1412
+ this._resonateCache = null;
1413
+ }
1414
+ return toRemove.length;
1415
+ }
1416
+ /** {@link Store.repairContentIndex} */
1417
+ async repairContentIndex(regenerateGist, minParents = 2) {
1418
+ await this._ensureReady();
1419
+ this.flush(); // commit any pending writes first
1420
+ if (this._nextId === 0) {
1421
+ return 0;
1422
+ }
1423
+ let added = 0;
1424
+ // A repairable node MUST carry edges or a halo, so the candidate set IS
1425
+ // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1426
+ // thousands), not node-count-sized (tens of millions). The old walk
1427
+ // visited EVERY branch id ever minted with a parent probe each; driving
1428
+ // from the target set visits only real candidates, in the same ascending
1429
+ // id order (the set is sorted), so the result and its ordering are
1430
+ // identical. Byte leaves (negative ids) were never visited by the old
1431
+ // walk and are skipped here too.
1432
+ const targets = this._dbEdgeOrHaloIds();
1433
+ let scanned = 0;
1434
+ for (const id of targets) {
1435
+ // Yield a real event-loop turn on a fixed cadence so the scan never
1436
+ // pins the thread for its whole duration.
1437
+ if (++scanned % 8192 === 0) {
1438
+ await yieldToEventLoop();
1439
+ }
1440
+ if (id < 0) {
1441
+ continue; // byte leaves: implicit, never repaired
1442
+ }
1443
+ // Already indexed in memory — skip.
1444
+ if (this._indexedIds.has(id)) {
1445
+ continue;
1446
+ }
1447
+ // Must be a structural bridge: ≥ minParents parents in the DAG.
1448
+ // One LIMITed probe per candidate.
1449
+ if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1450
+ continue;
1451
+ }
1452
+ // Already durably indexed by a previous session — record and skip.
1453
+ // Probed AFTER the structural filters: candidates are few, so this
1454
+ // point query runs rarely instead of once per node in the store.
1455
+ if (this._vecContentHas(id)) {
1058
1456
  this._indexedIds.set(id, true);
1059
- // Reach-only interiors build with the reduced construction budget — the
1060
- // one deliberate speed-for-quality trade of ingestion (see
1061
- // {@link StoreConfig.efConstructionInterior}); targets keep the full one.
1062
- this._contentBuffer.push(dedupTarget
1063
- ? { id, vector: v }
1064
- : { id, vector: v, ef: this.efConstructionInterior });
1065
- this._bufferedIds.add(id);
1066
- // A node indexed AS a dedup target enters the candidate set immediately.
1067
- if (dedupTarget)
1068
- this._nearDedupBuf.set(id, v);
1069
- }
1070
- /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1071
- * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1072
- * root is a genuine target (a whole experience), so it is a dedup target
1073
- * too. */
1074
- indexTarget(id) {
1075
- this.indexGist(id, true);
1076
- }
1077
- /** Index a node and its interior forms as resonance targets. A node that
1078
- * gains an edge is a learnt EXPERIENCE, and the consensus climb
1079
- * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1080
- * resonating its SUB-REGIONS — branch nodes within the experience — and
1081
- * climbing their parents back to it.
1082
- *
1083
- * EVERY interior branch is indexed unconditionally, and this is
1084
- * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1085
- * the post-hoc compaction criterion) was tried and REJECTED by the test
1086
- * suite — partial recall of an experience's interior slices, multi-topic
1087
- * attention, and counterfactual anchoring all resonate to SINGLE-parent
1088
- * interiors (13 tests fail without them). Post-hoc structural compaction
1089
- * ({@link compactContentIndex}) may still remove them, but that is a
1090
- * storage/recall trade-off for archived stores, not a free optimisation.
1091
- * The store's hash-cons bounds the index by the number of DISTINCT byte
1092
- * patterns in the corpus — not by the number of deposits.
1093
- *
1094
- * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1095
- * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1096
- * let a partial query resonate and climb, but a fresh branch must never
1097
- * merge onto an interior node of another experience.
1098
- *
1099
- * Iterative explicit-queue walk: the call stack never sees tree depth. */
1100
- indexSubtree(root) {
1101
- // The root is the whole experience — always index as a merge target.
1102
- this.indexGist(root, true);
1103
- const seen = new Set([root]);
1104
- const stack = [root];
1105
- while (stack.length > 0) {
1106
- const id = stack.pop();
1107
- const kids = this.get(id)?.kids;
1108
- if (!kids)
1109
- continue; // leaf — never a resonance anchor
1110
- const isRoot = id === root;
1111
- // Already-indexed or already-skipped nodes PRUNE the walk — their
1112
- // subtrees were classified in a prior call.
1113
- if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1114
- continue;
1115
- }
1116
- for (const k of kids) {
1117
- if (!seen.has(k)) {
1118
- seen.add(k);
1119
- stack.push(k);
1120
- }
1121
- }
1122
- if (isRoot)
1123
- continue;
1124
- const g = this._pendingGist.get(id);
1125
- if (g === undefined) {
1126
- // Gist evicted from the bounded pending cache — the node can't be
1127
- // indexed now. Mark as covered so subsequent visits prune here
1128
- // instead of re-walking the subtree (the gist won't return).
1129
- this._coveredIds.set(id, true);
1130
- continue;
1131
- }
1132
- // Index unconditionally — every interior node is a valid resonance
1133
- // anchor for partial-query recall. The _indexedIds cache and the
1134
- // durable-index check in indexGist prevent duplicate indexing across
1135
- // multiple encounters of the same shared subtree.
1136
- this.indexGist(id, false);
1137
- }
1138
- }
1139
- // ── Soft resonance ─────────────────────────────────────────────────────
1140
- async resonate(v, k) {
1141
- await this._ensureReady();
1142
- // Synchronous flush of any buffered index writes: the FIRST resonance
1143
- // after a large ingest pays that flush here, so it shows up in respond
1144
- // latency, not ingest latency — correct behaviour, skewed attribution;
1145
- // profile accordingly.
1146
- this.flushContent();
1147
- if (k <= 0)
1148
- return [];
1149
- // ANN read cache — content-addressed so a fresh Float32Array with the
1150
- // same values still hits. Lazy-init: null after any index write; the
1151
- // first miss after a flush recreates it. When voteRegions resonates
1152
- // identical perceived sub-regions, only the first call descends the ANN.
1153
- const rk = vecKey(v) + ":" + k;
1154
- const cache = this._resonateCache;
1155
- if (cache) {
1156
- const hit = cache.get(rk);
1157
- if (hit !== undefined)
1158
- return hit;
1159
- }
1160
- const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efSearch);
1161
- const out = [];
1162
- for (const r of results) {
1163
- const id = r.id;
1164
- out.push({ id, score: 1 - r.distance });
1165
- if (out.length >= k)
1166
- break;
1167
- }
1168
- const rc = this._resonateCache ??= new Map();
1169
- if (rc.size >= RESONATE_CACHE_MAX)
1170
- rc.clear();
1171
- rc.set(rk, out);
1172
- return out;
1173
- }
1174
- indexedVectorCount() {
1175
- this.flushContent();
1176
- return this._vecContentSize();
1177
- }
1178
- lastResonateReads() {
1179
- return this._vecContentLastReads();
1180
- }
1181
- // ── Content index compaction ────────────────────────────────────────────
1182
- /** How many physical compaction attempts have failed this session. Zero in
1183
- * a healthy store; a growing count means tombstones are accumulating and
1184
- * index query cost is drifting up (the first failure also warns once). */
1185
- compactFailures = 0;
1186
- /** Meta key holding the incremental scan watermark of
1187
- * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1188
- * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1189
- * entry once kept can never become removable — and removed entries are
1190
- * gone, so a pass only ever needs to examine entries indexed AFTER the
1191
- * previous pass. Internal ids are monotone and survive the index's
1192
- * splice compaction; the watermark is reset whenever a PHYSICAL index
1193
- * compaction runs (id reuse after a dropped top row would otherwise hide
1194
- * new entries behind it). */
1195
- static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1196
- /** {@link Store.compactContentIndex} */
1197
- async compactContentIndex(minParents = 2) {
1198
- await this._ensureReady();
1199
- this.flush(); // commit any pending writes first
1200
- // Incremental scan: resume from the last pass's watermark when its
1201
- // minParents matches; a changed criterion forces a full rescan.
1202
- let after = 0;
1203
- {
1204
- const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1205
- if (raw !== null) {
1206
- const sep = raw.indexOf(":");
1207
- const mp = Number(raw.slice(0, sep));
1208
- const wm = Number(raw.slice(sep + 1));
1209
- if (mp === minParents && Number.isFinite(wm) && wm > 0)
1210
- after = wm;
1211
- }
1212
- }
1213
- // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1214
- // sources ∪ edge targets ∪ halo rows), materialised by a single C-side
1215
- // scan. A binary-search membership probe replaces the three per-entry
1216
- // point queries that made this stage minutes long on a trained store.
1217
- const targets = this._dbEdgeOrHaloIds();
1218
- const isTarget = (id) => {
1219
- let lo = 0, hi = targets.length - 1;
1220
- while (lo <= hi) {
1221
- const mid = (lo + hi) >> 1;
1222
- const v = targets[mid];
1223
- if (v === id)
1224
- return true;
1225
- if (v < id)
1226
- lo = mid + 1;
1227
- else
1228
- hi = mid - 1;
1229
- }
1230
- return false;
1231
- };
1232
- // Collect ids to remove: nodes that are structurally isolated (few
1233
- // parents), not edge-bearing, and not halo-bearing. These nodes are
1234
- // unique to one experience tree — they bridge nothing and their index
1235
- // slots are wasted. Multi-parent nodes are the structural bridges
1236
- // that let a partial query climb from one experience to another.
1237
- const toRemove = [];
1238
- let scanned = 0;
1239
- let watermark = after;
1240
- for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1241
- // Yield a real event-loop turn on a fixed cadence: this scan runs over
1242
- // every unexamined indexed entry, and point probes alone pin the
1243
- // thread otherwise (the "frozen" symptom).
1244
- if (++scanned % 8192 === 0)
1245
- await yieldToEventLoop();
1246
- watermark = internal;
1247
- if (isTarget(id))
1248
- continue;
1249
- if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1250
- continue;
1251
- }
1252
- toRemove.push(id);
1253
- }
1254
- // Delete in batches — each batch is ONE vector-store transaction (one WAL
1255
- // commit), with an event-loop turn between batches.
1256
- const BATCH = 1000;
1257
- for (let i = 0; i < toRemove.length; i += BATCH) {
1258
- const batch = toRemove.slice(i, i + BATCH);
1259
- this._vecContentDeleteMany(batch);
1260
- // Purge the "already indexed" cache for removed ids: without this, a
1261
- // node that LATER becomes a resonance target again (gains an edge or
1262
- // halo) hits the stale cache entry in indexGist and is silently never
1263
- // re-indexed for the rest of the session.
1264
- for (const id of batch)
1265
- this._indexedIds.delete(id);
1266
- await yieldToEventLoop();
1267
- }
1268
- // Persist the scan watermark so the next pass starts where this one
1269
- // ended (kept beside the criterion that produced it).
1270
- this._dbBeginTx();
1271
- this._dbSetMeta(AbstractStore.COMPACT_WATERMARK_KEY, minParents + ":" + watermark);
1272
- this._dbCommitTx();
1273
- if (toRemove.length > 0) {
1274
- // Compact to physically reclaim space from tombstones.
1275
- try {
1276
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1277
- this._vecContentCompact();
1278
- // Physical compaction may free the top internal id for reuse —
1279
- // drop the watermark so the next pass rescans from the start.
1280
- this._dbBeginTx();
1281
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1282
- this._dbCommitTx();
1283
- }
1284
- }
1285
- catch (e) {
1286
- // Best-effort, but never SILENT: a persistently failing compaction
1287
- // lets tombstones accumulate and query cost grow with no signal.
1288
- this.compactFailures++;
1289
- if (this.compactFailures === 1) {
1290
- console.warn("sema: content-index compaction failed (will keep " +
1291
- "counting in store.compactFailures):", e);
1292
- }
1293
- }
1294
- // Index mutations (delete, compact) invalidate the ANN read cache.
1295
- this._resonateCache = null;
1296
- }
1297
- return toRemove.length;
1298
- }
1299
- /** {@link Store.repairContentIndex} */
1300
- async repairContentIndex(regenerateGist, minParents = 2) {
1301
- await this._ensureReady();
1302
- this.flush(); // commit any pending writes first
1303
- if (this._nextId === 0)
1304
- return 0;
1305
- let added = 0;
1306
- // A repairable node MUST carry edges or a halo, so the candidate set IS
1307
- // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1308
- // thousands), not node-count-sized (tens of millions). The old walk
1309
- // visited EVERY branch id ever minted with a parent probe each; driving
1310
- // from the target set visits only real candidates, in the same ascending
1311
- // id order (the set is sorted), so the result and its ordering are
1312
- // identical. Byte leaves (negative ids) were never visited by the old
1313
- // walk and are skipped here too.
1314
- const targets = this._dbEdgeOrHaloIds();
1315
- let scanned = 0;
1316
- for (const id of targets) {
1317
- // Yield a real event-loop turn on a fixed cadence so the scan never
1318
- // pins the thread for its whole duration.
1319
- if (++scanned % 8192 === 0)
1320
- await yieldToEventLoop();
1321
- if (id < 0)
1322
- continue; // byte leaves: implicit, never repaired
1323
- // Already indexed in memory — skip.
1324
- if (this._indexedIds.has(id))
1325
- continue;
1326
- // Must be a structural bridge: ≥ minParents parents in the DAG.
1327
- // One LIMITed probe per candidate.
1328
- if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1329
- continue;
1330
- }
1331
- // Already durably indexed by a previous session — record and skip.
1332
- // Probed AFTER the structural filters: candidates are few, so this
1333
- // point query runs rarely instead of once per node in the store.
1334
- if (this._vecContentHas(id)) {
1335
- this._indexedIds.set(id, true);
1336
- continue;
1337
- }
1338
- // Regenerate the gist from bytes. The callback is async (the Mind's
1339
- // perception is synchronous, but the interface allows a disk-backed
1340
- // regenerator that yields).
1341
- const gist = await regenerateGist(id);
1342
- if (!gist)
1343
- continue;
1344
- // Index it — same code path as indexGist, but the vector is injected
1345
- // directly rather than read from the (empty) pending-gist cache.
1346
- this._indexedIds.set(id, true);
1347
- this._contentBuffer.push({
1348
- id,
1349
- vector: gist,
1350
- ef: this.efConstructionInterior,
1351
- });
1352
- this._bufferedIds.add(id);
1353
- // Repaired nodes are reach-indexed, never dedup targets: their gist is
1354
- // regenerated (may differ numerically from the original) and they are
1355
- // interiors, not deposit roots.
1356
- added++;
1357
- // Flush periodically to bound the write-transaction size and yield to
1358
- // the event loop, same cadence compactContentIndex uses.
1359
- if (added % 1000 === 0)
1360
- await this.maybeFlush();
1361
- }
1362
- // Final flush for the last partial batch.
1363
- if (added > 0)
1364
- await this.maybeFlush();
1365
- return added;
1366
- }
1367
- // ── Continuation edges ─────────────────────────────────────────────────
1368
- async link(from, to) {
1369
- await this._ensureReady();
1370
- // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1371
- // indexed, because the seat is symmetric — a query may name only a PORTION of
1372
- // either side and must still resonate to its interior and climb to the whole.
1373
- // Only each ROOT is a MERGE target; the interiors are reach-only.
1374
- this.indexSubtree(from);
1375
- this.indexSubtree(to);
1376
- // Flush the vectors indexSubtree just added to the buffer.
1377
- // This keeps the buffer bounded and yields to the event loop.
1457
+ continue;
1458
+ }
1459
+ // Regenerate the gist from bytes. The callback is async (the Mind's
1460
+ // perception is synchronous, but the interface allows a disk-backed
1461
+ // regenerator that yields).
1462
+ const gist = await regenerateGist(id);
1463
+ if (!gist) {
1464
+ continue;
1465
+ }
1466
+ // Index it — same code path as indexGist, but the vector is injected
1467
+ // directly rather than read from the (empty) pending-gist cache.
1468
+ this._indexedIds.set(id, true);
1469
+ this._contentBuffer.push({
1470
+ id,
1471
+ vector: gist,
1472
+ ef: this.efConstructionInterior,
1473
+ });
1474
+ this._bufferedIds.add(id);
1475
+ // Repaired nodes are reach-indexed, never dedup targets: their gist is
1476
+ // regenerated (may differ numerically from the original) and they are
1477
+ // interiors, not deposit roots.
1478
+ added++;
1479
+ // Flush periodically to bound the write-transaction size and yield to
1480
+ // the event loop, same cadence compactContentIndex uses.
1481
+ if (added % 1000 === 0) {
1378
1482
  await this.maybeFlush();
1379
- this._dbBeginTx();
1380
- // Keep the document count exact as it grows: a source gaining its FIRST
1381
- // edge is one new learnt context. One indexed point probe, and only once
1382
- // the count has been materialised — before that, the lazy full count in
1383
- // {@link edgeSourceCount} will see this edge anyway.
1384
- if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1385
- this._edgeSrcCount++;
1386
- }
1387
- // An edge breaks the transparency of both endpoints — drop cached chains
1388
- // (same reasoning as the kid-row invalidation in put).
1389
- this._chainMemo.clear();
1390
- this._dbInsertEdge(from, to);
1391
- }
1392
- next(id) {
1393
- return this._dbGetNextEdges(id);
1394
- }
1395
- /** {@link Store.hasNext} — one indexed point probe, never a range read. */
1396
- hasNext(id) {
1397
- return this._dbEdgeSrcExists(id);
1398
- }
1399
- prev(id) {
1400
- return this._dbGetPrevEdges(id);
1401
- }
1402
- nextFirst(id, limit) {
1403
- return this._dbGetNextEdgesFirst(id, limit);
1404
- }
1405
- prevFirst(id, limit) {
1406
- return this._dbGetPrevEdgesFirst(id, limit);
1407
- }
1408
- /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1409
- * should override; this default materialises (correct, not optimal). */
1410
- prevCount(id) {
1411
- return this._dbGetPrevEdges(id).length;
1412
- }
1413
- edgeSourceCount() {
1414
- // Materialised once per session (edges written before this moment are
1415
- // covered by the full count), then kept exact incrementally by link().
1416
- // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1417
- // detect staleness — O(edges) on the recall hot path, at every IDF read.
1418
- if (this._edgeSrcCount < 0) {
1419
- this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1420
- }
1421
- return this._edgeSrcCount;
1422
- }
1423
- // ── Halos ──────────────────────────────────────────────────────────────
1424
- haloMass(id) {
1425
- const r = this._dbGetHalo(id);
1426
- return r ? r.mass : 0;
1427
- }
1428
- halo(id) {
1429
- const cached = this._haloNorm.get(id);
1430
- if (cached !== undefined)
1431
- return copy(cached);
1432
- const r = this._dbGetHalo(id);
1433
- if (!r || r.mass < this.minHaloMass)
1434
- return null;
1435
- const exact = this._haloExact.get(id);
1436
- const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1437
- this._haloNorm.set(id, v);
1438
- return copy(v);
1439
- }
1440
- /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1441
- * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1442
- hasHalo(id) {
1443
- const r = this._dbGetHalo(id);
1444
- return r !== null && r.mass >= this.minHaloMass;
1445
- }
1446
- async pourHalo(id, add) {
1447
- await this._ensureReady();
1448
- // A node with a halo is a genuine resonance target — the consensus climb
1449
- // resonates a query region to it, and articulation reads its halo.
1450
- this.indexGist(id, true);
1451
- const r = this._dbGetHalo(id);
1452
- const acc = this._haloExact.get(id) ??
1453
- (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1454
- addInto(acc, add);
1455
- this._haloExact.set(id, acc);
1456
- this._haloNorm.delete(id); // the normalized read cache is now stale
1457
- const mass = (r?.mass ?? 0) + 1;
1458
- this._dbBeginTx();
1459
- this._dbUpsertHalo(id, haloEncode(acc), mass);
1460
- // Re-index on a geometric schedule only (the exact halo is persisted).
1461
- if (mass >= this.minHaloMass && geometricMass(mass)) {
1462
- this._haloBuffer.set(id, normalize(copy(acc)));
1463
- await this.maybeFlush();
1464
- }
1465
- }
1466
- async resonateHalo(v, k) {
1467
- await this._ensureReady();
1468
- this.flushHalos();
1469
- if (k <= 0)
1470
- return [];
1471
- // ANN read cache same scheme as resonate's, but for the halo index.
1472
- const rk = vecKey(v) + ":" + k;
1473
- const cache = this._resonateHaloCache;
1474
- if (cache) {
1475
- const hit = cache.get(rk);
1476
- if (hit !== undefined)
1477
- return hit;
1478
- }
1479
- const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.efSearch);
1480
- const out = [];
1481
- for (const r of results) {
1482
- const id = r.id;
1483
- out.push({ id, score: 1 - r.distance });
1484
- if (out.length >= k)
1485
- break;
1486
- }
1487
- const rhc = this._resonateHaloCache ??= new Map();
1488
- if (rhc.size >= RESONATE_CACHE_MAX)
1489
- rhc.clear();
1490
- rhc.set(rk, out);
1491
- return out;
1492
- }
1493
- // ── Buffering, flushing, compaction ────────────────────────────────────
1494
- pending() {
1495
- return this._contentBuffer.length + this._haloBuffer.size;
1496
- }
1497
- flushContent() {
1498
- if (this._contentBuffer.length === 0)
1499
- return 0;
1500
- const batch = this._contentBuffer.splice(0);
1501
- // The merge scan only consults UNFLUSHED candidates, so clear them in
1502
- // lockstep with the content buffer they mirror.
1503
- this._nearDedupBuf.clear();
1504
- this._bufferedIds.clear();
1505
- this._vecContentUpsert(batch);
1506
- this._resonateCache = null;
1507
- return batch.length;
1508
- }
1509
- flushHalos() {
1510
- if (this._haloBuffer.size === 0)
1511
- return 0;
1512
- const batch = [...this._haloBuffer.entries()];
1513
- this._haloBuffer.clear();
1514
- this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1515
- this._resonateHaloCache = null;
1516
- return batch.length;
1517
- }
1518
- /** Append the buffered containment pairs, inside the deferred transaction.
1519
- * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1520
- * flush never reads a child's stored list back — the old packed-blob
1521
- * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1522
- * over a long training run on a hot window. */
1523
- flushContain() {
1524
- if (this._containBuf.size === 0)
1525
- return;
1526
- this._dbBeginTx();
1527
- for (const [child, set] of this._containBuf) {
1528
- this._dbAppendContain(child, [...set]);
1483
+ }
1484
+ }
1485
+ // Final flush for the last partial batch.
1486
+ if (added > 0) {
1487
+ await this.maybeFlush();
1488
+ }
1489
+ return added;
1490
+ }
1491
+ // ── Continuation edges ─────────────────────────────────────────────────
1492
+ async link(from, to) {
1493
+ await this._ensureReady();
1494
+ // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1495
+ // indexed, because the seat is symmetric — a query may name only a PORTION of
1496
+ // either side and must still resonate to its interior and climb to the whole.
1497
+ // Only each ROOT is a MERGE target; the interiors are reach-only.
1498
+ this.indexSubtree(from);
1499
+ this.indexSubtree(to);
1500
+ // Flush the vectors indexSubtree just added to the buffer.
1501
+ // This keeps the buffer bounded and yields to the event loop.
1502
+ await this.maybeFlush();
1503
+ this._dbBeginTx();
1504
+ // Keep the document count exact as it grows: a source gaining its FIRST
1505
+ // edge is one new learnt context. One indexed point probe, and only once
1506
+ // the count has been materialised — before that, the lazy full count in
1507
+ // {@link edgeSourceCount} will see this edge anyway.
1508
+ if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1509
+ this._edgeSrcCount++;
1510
+ }
1511
+ // An edge breaks the transparency of both endpoints — drop cached chains
1512
+ // (same reasoning as the kid-row invalidation in put).
1513
+ this._chainMemo.clear();
1514
+ this._dbInsertEdge(from, to);
1515
+ }
1516
+ next(id) {
1517
+ return this._dbGetNextEdges(id);
1518
+ }
1519
+ /** {@link Store.hasNext} one indexed point probe, never a range read. */
1520
+ hasNext(id) {
1521
+ return this._dbEdgeSrcExists(id);
1522
+ }
1523
+ prev(id) {
1524
+ return this._dbGetPrevEdges(id);
1525
+ }
1526
+ nextFirst(id, limit) {
1527
+ return this._dbGetNextEdgesFirst(id, limit);
1528
+ }
1529
+ prevFirst(id, limit) {
1530
+ return this._dbGetPrevEdgesFirst(id, limit);
1531
+ }
1532
+ /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1533
+ * should override; this default materialises (correct, not optimal). */
1534
+ prevCount(id) {
1535
+ return this._dbGetPrevEdges(id).length;
1536
+ }
1537
+ edgeSourceCount() {
1538
+ // Materialised once per session (edges written before this moment are
1539
+ // covered by the full count), then kept exact incrementally by link().
1540
+ // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1541
+ // detect staleness — O(edges) on the recall hot path, at every IDF read.
1542
+ if (this._edgeSrcCount < 0) {
1543
+ this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1544
+ }
1545
+ return this._edgeSrcCount;
1546
+ }
1547
+ // ── Halos ──────────────────────────────────────────────────────────────
1548
+ haloMass(id) {
1549
+ const r = this._dbGetHalo(id);
1550
+ return r ? r.mass : 0;
1551
+ }
1552
+ halo(id) {
1553
+ const cached = this._haloNorm.get(id);
1554
+ if (cached !== undefined) {
1555
+ return copy(cached);
1556
+ }
1557
+ const r = this._dbGetHalo(id);
1558
+ if (!r || r.mass < this.minHaloMass) {
1559
+ return null;
1560
+ }
1561
+ const exact = this._haloExact.get(id);
1562
+ const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1563
+ this._haloNorm.set(id, v);
1564
+ return copy(v);
1565
+ }
1566
+ /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1567
+ * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1568
+ hasHalo(id) {
1569
+ const r = this._dbGetHalo(id);
1570
+ return r !== null && r.mass >= this.minHaloMass;
1571
+ }
1572
+ async pourHalo(id, add) {
1573
+ await this._ensureReady();
1574
+ // A node with a halo is a genuine resonance target — the consensus climb
1575
+ // resonates a query region to it, and articulation reads its halo.
1576
+ this.indexGist(id, true);
1577
+ const r = this._dbGetHalo(id);
1578
+ const acc = this._haloExact.get(id) ??
1579
+ (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1580
+ addInto(acc, add);
1581
+ this._haloExact.set(id, acc);
1582
+ this._haloNorm.delete(id); // the normalized read cache is now stale
1583
+ const mass = (r?.mass ?? 0) + 1;
1584
+ this._dbBeginTx();
1585
+ this._dbUpsertHalo(id, haloEncode(acc), mass);
1586
+ // Re-index on a geometric schedule only (the exact halo is persisted).
1587
+ if (mass >= this.minHaloMass && geometricMass(mass)) {
1588
+ this._haloBuffer.set(id, normalize(copy(acc)));
1589
+ await this.maybeFlush();
1590
+ }
1591
+ }
1592
+ async resonateHalo(v, k) {
1593
+ await this._ensureReady();
1594
+ this.flushHalos();
1595
+ if (k <= 0) {
1596
+ return [];
1597
+ }
1598
+ // ANN read cache — same scheme as resonate's, but for the halo index.
1599
+ const rk = vecKey(v) + ":" + k;
1600
+ const cache = this._resonateHaloCache;
1601
+ if (cache) {
1602
+ const hit = cache.get(rk);
1603
+ if (hit !== undefined) {
1604
+ return hit;
1605
+ }
1606
+ }
1607
+ const results = this._vecHaloQuery(
1608
+ normalize(copy(v)),
1609
+ k * this.overfetch,
1610
+ this.efSearch,
1611
+ );
1612
+ const out = [];
1613
+ for (const r of results) {
1614
+ const id = r.id;
1615
+ out.push({ id, score: 1 - r.distance });
1616
+ if (out.length >= k) {
1617
+ break;
1618
+ }
1619
+ }
1620
+ const rhc = this._resonateHaloCache ??= new Map();
1621
+ if (rhc.size >= RESONATE_CACHE_MAX) {
1622
+ rhc.clear();
1623
+ }
1624
+ rhc.set(rk, out);
1625
+ return out;
1626
+ }
1627
+ // ── Buffering, flushing, compaction ────────────────────────────────────
1628
+ pending() {
1629
+ return this._contentBuffer.length + this._haloBuffer.size;
1630
+ }
1631
+ flushContent() {
1632
+ if (this._contentBuffer.length === 0) {
1633
+ return 0;
1634
+ }
1635
+ const batch = this._contentBuffer.splice(0);
1636
+ // The merge scan only consults UNFLUSHED candidates, so clear them in
1637
+ // lockstep with the content buffer they mirror.
1638
+ this._nearDedupBuf.clear();
1639
+ this._bufferedIds.clear();
1640
+ this._vecContentUpsert(batch);
1641
+ this._resonateCache = null;
1642
+ return batch.length;
1643
+ }
1644
+ flushHalos() {
1645
+ if (this._haloBuffer.size === 0) {
1646
+ return 0;
1647
+ }
1648
+ const batch = [...this._haloBuffer.entries()];
1649
+ this._haloBuffer.clear();
1650
+ this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1651
+ this._resonateHaloCache = null;
1652
+ return batch.length;
1653
+ }
1654
+ /** Append the buffered containment pairs, inside the deferred transaction.
1655
+ * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1656
+ * flush never reads a child's stored list back — the old packed-blob
1657
+ * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1658
+ * over a long training run on a hot window. */
1659
+ flushContain() {
1660
+ if (this._containBuf.size === 0) {
1661
+ return;
1662
+ }
1663
+ this._dbBeginTx();
1664
+ for (const [child, set] of this._containBuf) {
1665
+ this._dbAppendContain(child, [...set]);
1666
+ }
1667
+ this._containBuf.clear();
1668
+ }
1669
+ /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1670
+ flush() {
1671
+ this.flushContain();
1672
+ const written = this.flushContent() + this.flushHalos();
1673
+ // Commit the deferred write transaction on the same cadence as the vector
1674
+ // buffers, so node rows / edges / halos become durable in coalesced batches.
1675
+ this._dbCommitTx();
1676
+ if (written === 0) {
1677
+ return;
1678
+ }
1679
+ this._writtenSinceCompact += written;
1680
+ if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1681
+ this._writtenSinceCompact = 0;
1682
+ try {
1683
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1684
+ this._vecContentCompact();
1685
+ // Physical compaction may free the top internal id for reuse —
1686
+ // invalidate the incremental maintenance watermark.
1687
+ this._dbBeginTx();
1688
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1689
+ this._dbCommitTx();
1529
1690
  }
1530
- this._containBuf.clear();
1531
- }
1532
- /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1533
- flush() {
1534
- this.flushContain();
1535
- const written = this.flushContent() + this.flushHalos();
1536
- // Commit the deferred write transaction on the same cadence as the vector
1537
- // buffers, so node rows / edges / halos become durable in coalesced batches.
1538
- this._dbCommitTx();
1539
- if (written === 0)
1540
- return;
1541
- this._writtenSinceCompact += written;
1542
- if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1543
- this._writtenSinceCompact = 0;
1544
- try {
1545
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1546
- this._vecContentCompact();
1547
- // Physical compaction may free the top internal id for reuse —
1548
- // invalidate the incremental maintenance watermark.
1549
- this._dbBeginTx();
1550
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1551
- this._dbCommitTx();
1552
- }
1553
- if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1554
- this._vecHaloCompact();
1555
- }
1556
- }
1557
- catch (e) {
1558
- // Best-effort, but never SILENT (same contract as the prune-path
1559
- // compaction above): a persistently failing compaction lets
1560
- // tombstones accumulate and query cost grow with no signal.
1561
- this.compactFailures++;
1562
- if (this.compactFailures === 1) {
1563
- console.warn("sema: vector-index compaction failed (will keep " +
1564
- "counting in store.compactFailures):", e);
1565
- }
1566
- }
1691
+ if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1692
+ this._vecHaloCompact();
1567
1693
  }
1568
- }
1569
- async maybeFlush() {
1570
- if (this.pending() >= this.batchSize) {
1571
- this.flush();
1572
- // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1573
- // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1574
- // engine drains to empty before it ever services a MACROTASK — so a deposit
1575
- // burst that only micro-awaits pins the single thread for its whole
1576
- // duration, starving timers, pending I/O and any overlapped work. Here —
1577
- // buffers empty, transaction committed, nothing mid-write — is the one
1578
- // safe point to hand the event loop a real turn.
1579
- await yieldToEventLoop();
1694
+ } catch (e) {
1695
+ // Best-effort, but never SILENT (same contract as the prune-path
1696
+ // compaction above): a persistently failing compaction lets
1697
+ // tombstones accumulate and query cost grow with no signal.
1698
+ this.compactFailures++;
1699
+ if (this.compactFailures === 1) {
1700
+ console.warn(
1701
+ "sema: vector-index compaction failed (will keep " +
1702
+ "counting in store.compactFailures):",
1703
+ e,
1704
+ );
1580
1705
  }
1581
- }
1582
- // ── Meta ───────────────────────────────────────────────────────────────
1583
- async setMeta(key, val) {
1584
- await this._ensureReady();
1585
- this._dbBeginTx();
1586
- this._dbSetMeta(key, val);
1587
- }
1588
- async getMeta(key) {
1589
- await this._ensureReady();
1590
- return this._dbGetMeta(key);
1591
- }
1592
- async deleteMeta(key) {
1593
- await this._ensureReady();
1594
- this._dbBeginTx();
1595
- this._dbDeleteMeta(key);
1596
- }
1597
- // ── Snapshot ───────────────────────────────────────────────────────────
1598
- async saveSnapshot(bytes) {
1599
- await this._ensureReady();
1600
- this.flush(); // commits any open write transaction + vector buffers
1601
- this._dbSaveSnapshot(bytes);
1602
- }
1603
- async loadSnapshot() {
1604
- await this._ensureReady();
1605
- return this._dbLoadSnapshot();
1606
- }
1607
- // ── Lifecycle ──────────────────────────────────────────────────────────
1608
- commit() {
1609
- this.flush();
1610
- }
1611
- async close() {
1612
- if (this.closed)
1613
- return;
1614
- this.closed = true;
1615
- this.flush(); // commits any open write transaction + vector buffers
1616
- this._dbClose();
1617
- }
1706
+ }
1707
+ }
1708
+ }
1709
+ async maybeFlush() {
1710
+ if (this.pending() >= this.batchSize) {
1711
+ this.flush();
1712
+ // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1713
+ // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1714
+ // engine drains to empty before it ever services a MACROTASK — so a deposit
1715
+ // burst that only micro-awaits pins the single thread for its whole
1716
+ // duration, starving timers, pending I/O and any overlapped work. Here —
1717
+ // buffers empty, transaction committed, nothing mid-write — is the one
1718
+ // safe point to hand the event loop a real turn.
1719
+ await yieldToEventLoop();
1720
+ }
1721
+ }
1722
+ // ── Meta ───────────────────────────────────────────────────────────────
1723
+ async setMeta(key, val) {
1724
+ await this._ensureReady();
1725
+ this._dbBeginTx();
1726
+ this._dbSetMeta(key, val);
1727
+ }
1728
+ async getMeta(key) {
1729
+ await this._ensureReady();
1730
+ return this._dbGetMeta(key);
1731
+ }
1732
+ async deleteMeta(key) {
1733
+ await this._ensureReady();
1734
+ this._dbBeginTx();
1735
+ this._dbDeleteMeta(key);
1736
+ }
1737
+ // ── Snapshot ───────────────────────────────────────────────────────────
1738
+ async saveSnapshot(bytes) {
1739
+ await this._ensureReady();
1740
+ this.flush(); // commits any open write transaction + vector buffers
1741
+ this._dbSaveSnapshot(bytes);
1742
+ }
1743
+ async loadSnapshot() {
1744
+ await this._ensureReady();
1745
+ return this._dbLoadSnapshot();
1746
+ }
1747
+ // ── Lifecycle ──────────────────────────────────────────────────────────
1748
+ commit() {
1749
+ this.flush();
1750
+ }
1751
+ async close() {
1752
+ if (this.closed) {
1753
+ return;
1754
+ }
1755
+ this.closed = true;
1756
+ this.flush(); // commits any open write transaction + vector buffers
1757
+ this._dbClose();
1758
+ }
1618
1759
  }