@hviana/sema 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1700 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +29 -43
  51. package/dist/src/geometry.js +324 -268
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +754 -872
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +141 -185
  82. package/dist/src/mind/mind.d.ts +194 -155
  83. package/dist/src/mind/mind.js +392 -289
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +20 -33
  89. package/dist/src/mind/primitives.js +124 -120
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +198 -210
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +29 -58
  101. package/dist/src/mind/traverse.js +344 -356
  102. package/dist/src/mind/types.d.ts +122 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +171 -166
  117. package/dist/src/store-sqlite.js +768 -662
  118. package/dist/src/store.d.ts +597 -630
  119. package/dist/src/store.js +1434 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +33 -15
  123. package/index.html +65 -0
  124. package/package.json +1 -1
  125. package/src/alu/README.md +1 -1
  126. package/src/alu/src/index.ts +1 -1
  127. package/src/config.ts +19 -27
  128. package/src/geometry.ts +93 -1
  129. package/src/index.ts +6 -11
  130. package/src/mind/attention.ts +11 -6
  131. package/src/mind/mechanisms/recall.ts +29 -13
  132. package/src/mind/mind.ts +250 -19
  133. package/src/mind/primitives.ts +43 -6
  134. package/src/mind/recognition.ts +26 -8
  135. package/src/mind/traverse.ts +47 -0
  136. package/src/mind/types.ts +20 -21
  137. package/src/rabitq-ivf/README.md +56 -0
  138. package/src/rabitq-ivf/src/database.ts +276 -0
  139. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  140. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  141. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  142. package/src/store-sqlite.ts +204 -9
  143. package/src/store.ts +27 -34
  144. package/test/08-storage.test.mjs +3 -3
  145. package/test/13-conversation.test.mjs +190 -20
  146. package/test/14-scaling.test.mjs +2 -2
  147. package/test/35-ivf.test.mjs +263 -0
  148. package/test/36-bloom.test.mjs +123 -0
  149. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  150. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  151. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  152. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  153. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  154. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  155. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  156. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  157. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  158. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  159. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  160. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  161. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  162. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  163. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  164. package/src/rabitq-hnsw/README.md +0 -303
  165. package/src/rabitq-hnsw/src/database.ts +0 -492
  166. package/src/rabitq-hnsw/src/heap.ts +0 -90
  167. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  168. package/src/rabitq-hnsw/src/store.ts +0 -994
  169. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  170. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
package/dist/src/store.js CHANGED
@@ -55,14 +55,14 @@ const CHAIN_DEPTH_CAP = 65_536;
55
55
  * D·4 bytes (max ~4096), well under the argument limit; the key is far
56
56
  * cheaper than the ANN query it deduplicates. */
57
57
  function vecKey(v) {
58
- const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
- // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
- // engine argument limits and is slower than a few bounded calls.
61
- let s = "";
62
- for (let i = 0; i < bytes.length; i += 8192) {
63
- s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
- }
65
- return s;
58
+ const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
+ // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
+ // engine argument limits and is slower than a few bounded calls.
61
+ let s = "";
62
+ for (let i = 0; i < bytes.length; i += 8192) {
63
+ s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
+ }
65
+ return s;
66
66
  }
67
67
  /** Bounded map with LRU eviction and byte accounting. On get the entry
68
68
  * moves to the most-recent end; on set the least-recently-used entries are
@@ -71,138 +71,129 @@ function vecKey(v) {
71
71
  * for uniform caches). A miss only costs a little extra work later, never
72
72
  * correctness. */
73
73
  export class BoundedMap {
74
- maxBytes;
75
- sizeOf;
76
- evict;
77
- m = new Map();
78
- _bytes = 0;
79
- // Persistent eviction cursor over the Map's insertion order. A fresh
80
- // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
- // deletions leave in V8's ordered backing store (compacted only on rehash),
82
- // making each eviction O(size) once the map first fills — the training-rate
83
- // cliff at scale. A persistent iterator passes each hole exactly once
84
- // (V8 map iterators stay valid under mutation): amortized O(1).
85
- _cursor = null;
86
- // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
- // from the cursor, so the LRU window never rescans from the front.
88
- _candidates = [];
89
- constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
- this.maxBytes = maxBytes;
91
- this.sizeOf = sizeOf;
92
- this.evict = evict;
93
- }
94
- /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
- * off; wraps to the front when exhausted. Undefined only when empty. */
96
- nextOldest() {
97
- for (let wrapped = false;;) {
98
- if (this._cursor === null) {
99
- this._cursor = this.m.keys();
100
- }
101
- const n = this._cursor.next();
102
- if (!n.done) {
103
- return n.value;
104
- }
105
- this._cursor = null;
106
- if (this.m.size === 0 || wrapped) {
107
- return undefined;
108
- }
109
- wrapped = true;
110
- }
111
- }
112
- get(k) {
113
- const v = this.m.get(k);
114
- if (v !== undefined) {
115
- this.m.delete(k);
116
- this.m.set(k, v);
117
- }
118
- return v;
119
- }
120
- /** Membership without touching LRU order — a pure peek, for callers that only
121
- * need "is this key present?" and must not promote it to most-recent. */
122
- has(k) {
123
- return this.m.has(k);
124
- }
125
- set(k, v) {
126
- const old = this.m.get(k);
127
- if (old !== undefined) {
128
- this._bytes -= this.sizeOf(old);
129
- this.m.delete(k);
130
- }
131
- this.m.set(k, v);
132
- this._bytes += this.sizeOf(v);
133
- while (this._bytes > this.maxBytes && this.m.size > 0) {
134
- if (this.evict === "smallest") {
135
- // Among the oldest LRU candidates, evict the cheapest to rebuild.
136
- // Window grows logarithmically with cache size — wide enough to
137
- // find a cheap victim without scanning the whole map. Candidates
138
- // are pulled from the persistent cursor and carried between
139
- // evictions, so the window never rescans the map from the front.
140
- const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
141
- this._candidates = this._candidates.filter((k) => this.m.has(k));
142
- while (this._candidates.length < WINDOW) {
143
- const k = this.nextOldest();
144
- if (k === undefined) {
145
- break;
146
- }
147
- if (!this._candidates.includes(k)) {
148
- this._candidates.push(k);
149
- }
74
+ maxBytes;
75
+ sizeOf;
76
+ evict;
77
+ m = new Map();
78
+ _bytes = 0;
79
+ // Persistent eviction cursor over the Map's insertion order. A fresh
80
+ // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
+ // deletions leave in V8's ordered backing store (compacted only on rehash),
82
+ // making each eviction O(size) once the map first fills — the training-rate
83
+ // cliff at scale. A persistent iterator passes each hole exactly once
84
+ // (V8 map iterators stay valid under mutation): amortized O(1).
85
+ _cursor = null;
86
+ // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
+ // from the cursor, so the LRU window never rescans from the front.
88
+ _candidates = [];
89
+ constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
+ this.maxBytes = maxBytes;
91
+ this.sizeOf = sizeOf;
92
+ this.evict = evict;
93
+ }
94
+ /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
+ * off; wraps to the front when exhausted. Undefined only when empty. */
96
+ nextOldest() {
97
+ for (let wrapped = false;;) {
98
+ if (this._cursor === null)
99
+ this._cursor = this.m.keys();
100
+ const n = this._cursor.next();
101
+ if (!n.done)
102
+ return n.value;
103
+ this._cursor = null;
104
+ if (this.m.size === 0 || wrapped)
105
+ return undefined;
106
+ wrapped = true;
150
107
  }
151
- let bestI = -1;
152
- let bestSz = Infinity;
153
- for (let i = 0; i < this._candidates.length; i++) {
154
- const sz = this.sizeOf(this.m.get(this._candidates[i]));
155
- if (sz < bestSz) {
156
- bestSz = sz;
157
- bestI = i;
158
- }
159
- }
160
- if (bestI < 0) {
161
- break;
108
+ }
109
+ get(k) {
110
+ const v = this.m.get(k);
111
+ if (v !== undefined) {
112
+ this.m.delete(k);
113
+ this.m.set(k, v);
162
114
  }
163
- const bestK = this._candidates[bestI];
164
- this._candidates.splice(bestI, 1);
165
- this._bytes -= bestSz;
166
- this.m.delete(bestK);
167
- } else {
168
- const lru = this.nextOldest();
169
- if (lru === undefined) {
170
- break;
115
+ return v;
116
+ }
117
+ /** Membership without touching LRU order — a pure peek, for callers that only
118
+ * need "is this key present?" and must not promote it to most-recent. */
119
+ has(k) {
120
+ return this.m.has(k);
121
+ }
122
+ set(k, v) {
123
+ const old = this.m.get(k);
124
+ if (old !== undefined) {
125
+ this._bytes -= this.sizeOf(old);
126
+ this.m.delete(k);
171
127
  }
172
- const lruv = this.m.get(lru);
173
- if (lruv === undefined) {
174
- continue;
128
+ this.m.set(k, v);
129
+ this._bytes += this.sizeOf(v);
130
+ while (this._bytes > this.maxBytes && this.m.size > 0) {
131
+ if (this.evict === "smallest") {
132
+ // Among the oldest LRU candidates, evict the cheapest to rebuild.
133
+ // Window grows logarithmically with cache size — wide enough to
134
+ // find a cheap victim without scanning the whole map. Candidates
135
+ // are pulled from the persistent cursor and carried between
136
+ // evictions, so the window never rescans the map from the front.
137
+ const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
138
+ this._candidates = this._candidates.filter((k) => this.m.has(k));
139
+ while (this._candidates.length < WINDOW) {
140
+ const k = this.nextOldest();
141
+ if (k === undefined)
142
+ break;
143
+ if (!this._candidates.includes(k))
144
+ this._candidates.push(k);
145
+ }
146
+ let bestI = -1;
147
+ let bestSz = Infinity;
148
+ for (let i = 0; i < this._candidates.length; i++) {
149
+ const sz = this.sizeOf(this.m.get(this._candidates[i]));
150
+ if (sz < bestSz) {
151
+ bestSz = sz;
152
+ bestI = i;
153
+ }
154
+ }
155
+ if (bestI < 0)
156
+ break;
157
+ const bestK = this._candidates[bestI];
158
+ this._candidates.splice(bestI, 1);
159
+ this._bytes -= bestSz;
160
+ this.m.delete(bestK);
161
+ }
162
+ else {
163
+ const lru = this.nextOldest();
164
+ if (lru === undefined)
165
+ break;
166
+ const lruv = this.m.get(lru);
167
+ if (lruv === undefined)
168
+ continue;
169
+ this._bytes -= this.sizeOf(lruv);
170
+ this.m.delete(lru);
171
+ }
175
172
  }
176
- this._bytes -= this.sizeOf(lruv);
177
- this.m.delete(lru);
178
- }
179
- }
180
- }
181
- get size() {
182
- return this.m.size;
183
- }
184
- get bytes() {
185
- return this._bytes;
186
- }
187
- /** Remove one entry (point invalidation), with byte accounting. */
188
- delete(k) {
189
- const v = this.m.get(k);
190
- if (v === undefined) {
191
- return;
192
- }
193
- this._bytes -= this.sizeOf(v);
194
- this.m.delete(k);
195
- }
196
- /** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
197
- clear() {
198
- if (this.m.size === 0) {
199
- return;
200
- }
201
- this.m = new Map();
202
- this._bytes = 0;
203
- this._cursor = null;
204
- this._candidates = [];
205
- }
173
+ }
174
+ get size() {
175
+ return this.m.size;
176
+ }
177
+ get bytes() {
178
+ return this._bytes;
179
+ }
180
+ /** Remove one entry (point invalidation), with byte accounting. */
181
+ delete(k) {
182
+ const v = this.m.get(k);
183
+ if (v === undefined)
184
+ return;
185
+ this._bytes -= this.sizeOf(v);
186
+ this.m.delete(k);
187
+ }
188
+ /** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
189
+ clear() {
190
+ if (this.m.size === 0)
191
+ return;
192
+ this.m = new Map();
193
+ this._bytes = 0;
194
+ this._cursor = null;
195
+ this._candidates = [];
196
+ }
206
197
  }
207
198
  // ── Serialisation utilities (pure functions, no DB dependency) ───────────
208
199
  const _ZERO = new Uint8Array(0);
@@ -219,28 +210,26 @@ const _ZERO = new Uint8Array(0);
219
210
  * exit mid-`ingest` whenever the event loop had nothing else alive (a bare
220
211
  * top-level deposit loop) — the await simply never resolved. */
221
212
  function yieldToEventLoop() {
222
- return new Promise((resolve) => {
223
- const g = globalThis;
224
- if (g.setImmediate) {
225
- g.setImmediate(resolve);
226
- } else {
227
- g.setTimeout(resolve, 0);
228
- }
229
- });
213
+ return new Promise((resolve) => {
214
+ const g = globalThis;
215
+ if (g.setImmediate)
216
+ g.setImmediate(resolve);
217
+ else
218
+ g.setTimeout(resolve, 0);
219
+ });
230
220
  }
231
221
  /** Concatenate an array of byte arrays. */
232
222
  function concat(parts) {
233
- let n = 0;
234
- for (const p of parts) {
235
- n += p.length;
236
- }
237
- const out = new Uint8Array(n);
238
- let o = 0;
239
- for (const p of parts) {
240
- out.set(p, o);
241
- o += p.length;
242
- }
243
- return out;
223
+ let n = 0;
224
+ for (const p of parts)
225
+ n += p.length;
226
+ const out = new Uint8Array(n);
227
+ let o = 0;
228
+ for (const p of parts) {
229
+ out.set(p, o);
230
+ o += p.length;
231
+ }
232
+ return out;
244
233
  }
245
234
  /** The byte string a FLAT branch spans — a branch whose kids are ALL implicit
246
235
  * single-byte leaves (ids −256…−1) is fully determined by its bytes, one per
@@ -252,58 +241,55 @@ function concat(parts) {
252
241
  * and content-addressed through the same leaf index instead of a second
253
242
  * blob-duplicating kids index. */
254
243
  function flatKidsBytes(kids) {
255
- const out = new Uint8Array(kids.length);
256
- for (let i = 0; i < kids.length; i++) {
257
- const k = kids[i];
258
- if (k < -256 || k >= 0) {
259
- return null;
260
- }
261
- out[i] = -(k + 1);
262
- }
263
- return out;
244
+ const out = new Uint8Array(kids.length);
245
+ for (let i = 0; i < kids.length; i++) {
246
+ const k = kids[i];
247
+ if (k < -256 || k >= 0)
248
+ return null;
249
+ out[i] = -(k + 1);
250
+ }
251
+ return out;
264
252
  }
265
253
  /** The implicit kid list of a flat branch — the inverse of
266
254
  * {@link flatKidsBytes}. */
267
255
  export function flatBytesKids(bytes) {
268
- const out = new Array(bytes.length);
269
- for (let i = 0; i < bytes.length; i++) {
270
- out[i] = -(bytes[i] + 1);
271
- }
272
- return out;
256
+ const out = new Array(bytes.length);
257
+ for (let i = 0; i < bytes.length; i++)
258
+ out[i] = -(bytes[i] + 1);
259
+ return out;
273
260
  }
274
261
  /** Pack a child-id list as little-endian int32s — 4 bytes per child, far more
275
262
  * compact than a space-joined decimal string and trivial to read back. */
276
263
  export function packKids(kids) {
277
- const out = new Int32Array(kids.length);
278
- for (let i = 0; i < kids.length; i++) {
279
- out[i] = kids[i];
280
- }
281
- return new Uint8Array(out.buffer);
264
+ const out = new Int32Array(kids.length);
265
+ for (let i = 0; i < kids.length; i++)
266
+ out[i] = kids[i];
267
+ return new Uint8Array(out.buffer);
282
268
  }
283
269
  export function unpackKids(blob) {
284
- // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
285
- // reinterpreting as int32.
286
- const i32 = new Int32Array(blob.slice().buffer);
287
- return Array.from(i32);
270
+ // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
271
+ // reinterpreting as int32.
272
+ const i32 = new Int32Array(blob.slice().buffer);
273
+ return Array.from(i32);
288
274
  }
289
275
  /** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
290
276
  * on. Collisions are resolved by verifying the stored blob, never trusted. */
291
277
  function hashOf(bytes) {
292
- let h = 0x811c9dc5 >>> 0;
293
- for (let i = 0; i < bytes.length; i++) {
294
- h ^= bytes[i];
295
- h = Math.imul(h, 0x01000193) >>> 0;
296
- }
297
- return h >>> 0;
278
+ let h = 0x811c9dc5 >>> 0;
279
+ for (let i = 0; i < bytes.length; i++) {
280
+ h ^= bytes[i];
281
+ h = Math.imul(h, 0x01000193) >>> 0;
282
+ }
283
+ return h >>> 0;
298
284
  }
299
285
  /** Fast content key (FNV-1a + length). No array spread onto the call stack. */
300
286
  function keyOf(bytes) {
301
- return hashOf(bytes).toString(16) + ":" + bytes.length;
287
+ return hashOf(bytes).toString(16) + ":" + bytes.length;
302
288
  }
303
289
  /** Re-index a halo only when its mass is small or a power of two — O(log mass)
304
290
  * writes per node instead of O(mass). */
305
291
  function geometricMass(mass) {
306
- return mass <= 4 || (mass & (mass - 1)) === 0;
292
+ return mass <= 4 || (mass & (mass - 1)) === 0;
307
293
  }
308
294
  // Halo accumulators are stored 2-bit quantized: the exact float32 L2 norm
309
295
  // followed by D two-bit codes (sign + magnitude class) — 4 + D/4 bytes, 16×
@@ -328,47 +314,41 @@ const HALO_Q_THRESHOLD = 0.9816; // Lloyd-Max decision point, unit Gaussian
328
314
  const HALO_Q_LO = 0.4528; // reconstruction level, inner cell
329
315
  const HALO_Q_HI = 1.5104; // reconstruction level, outer cell
330
316
  function haloEncode(v) {
331
- const D = v.length;
332
- const out = new Uint8Array(4 + ((D + 3) >> 2));
333
- let n2 = 0;
334
- for (let i = 0; i < D; i++) {
335
- n2 += v[i] * v[i];
336
- }
337
- const norm = Math.sqrt(n2);
338
- new DataView(out.buffer).setFloat32(0, norm, true);
339
- const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
340
- for (let i = 0; i < D; i++) {
341
- const x = v[i];
342
- // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
343
- const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
344
- out[4 + (i >> 2)] |= code << ((i & 3) << 1);
345
- }
346
- return out;
317
+ const D = v.length;
318
+ const out = new Uint8Array(4 + ((D + 3) >> 2));
319
+ let n2 = 0;
320
+ for (let i = 0; i < D; i++)
321
+ n2 += v[i] * v[i];
322
+ const norm = Math.sqrt(n2);
323
+ new DataView(out.buffer).setFloat32(0, norm, true);
324
+ const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
325
+ for (let i = 0; i < D; i++) {
326
+ const x = v[i];
327
+ // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
328
+ const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
329
+ out[4 + (i >> 2)] |= code << ((i & 3) << 1);
330
+ }
331
+ return out;
347
332
  }
348
333
  function haloDecode(blob, D) {
349
- const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(
350
- 0,
351
- true,
352
- );
353
- const out = new Float32Array(D);
354
- if (norm === 0) {
334
+ const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(0, true);
335
+ const out = new Float32Array(D);
336
+ if (norm === 0)
337
+ return out;
338
+ const sigma = norm / Math.sqrt(D);
339
+ let n2 = 0;
340
+ for (let i = 0; i < D; i++) {
341
+ const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
342
+ const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
343
+ const x = code & 1 ? mag : -mag;
344
+ out[i] = x;
345
+ n2 += x * x;
346
+ }
347
+ // Rescale to the exact stored norm so accumulation stays magnitude-true.
348
+ const s = norm / Math.sqrt(n2);
349
+ for (let i = 0; i < D; i++)
350
+ out[i] *= s;
355
351
  return out;
356
- }
357
- const sigma = norm / Math.sqrt(D);
358
- let n2 = 0;
359
- for (let i = 0; i < D; i++) {
360
- const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
361
- const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
362
- const x = code & 1 ? mag : -mag;
363
- out[i] = x;
364
- n2 += x * x;
365
- }
366
- // Rescale to the exact stored norm so accumulation stays magnitude-true.
367
- const s = norm / Math.sqrt(n2);
368
- for (let i = 0; i < D; i++) {
369
- out[i] *= s;
370
- }
371
- return out;
372
352
  }
373
353
  // ── AbstractStore: template-method base with all domain logic ────────────
374
354
  /**
@@ -379,1381 +359,1254 @@ function haloDecode(blob, D) {
379
359
  * protected abstract methods that talk to the actual storage backend.
380
360
  */
381
361
  export class AbstractStore {
382
- // ── Config ─────────────────────────────────────────────────────────────
383
- _D;
384
- _maxGroup;
385
- minHaloMass;
386
- efSearch;
387
- m;
388
- efConstruction;
389
- efConstructionInterior;
390
- overfetch;
391
- batchSize;
392
- compactEveryNWrites;
393
- // ── State ──────────────────────────────────────────────────────────────
394
- /** Branch node ids are a dense, monotonically-increasing integer sequence
395
- * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
396
- * They are NEVER deleted, so the count of minted branch ids IS the next id —
397
- * which doubles as the branch-node count and lets has() be an O(1) check.
398
- * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
399
- _nextId = 0;
400
- _writtenSinceCompact = 0;
401
- closed = false;
402
- /** Lifecycle guard resolved once `_dbOpen()` completes. */
403
- _ready = null;
404
- // ── Caches ─────────────────────────────────────────────────────────────
405
- /** Exact-content dedup: content-key node id. Intrinsic compression. */
406
- _leafKey;
407
- _branchKey;
408
- /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
409
- _bytesCache;
410
- /** contentLen memocontent is immutable, so entries never invalidate. */
411
- _lenCache;
412
- /** Node-record cache — avoids repeated persistence queries for shared DAG
413
- * nodes. Each record is small (a few ints + short leaf buffer). */
414
- _recCache;
415
- /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
416
- * links/pours a node right after interning it, so the working set is one
417
- * deposit's nodes — a small budget captures ~all of it, and an eviction only
418
- * means that node is reached by the DAG climb instead of by direct
419
- * resonance. */
420
- _pendingGist;
421
- /** EXACT halo accumulators for the session's live pours: full-precision in
422
- * memory, 2-bit on disk, so within-session accumulate-then-compare never
423
- * round-trips through the quantizer. Regenerable a miss reads the durable
424
- * 2-bit row. */
425
- _haloExact;
426
- /** NORMALIZED halo read cache the decoded, normalized vector {@link halo}
427
- * returns, cached by id so repeat reads skip the per-call 2-bit decode and
428
- * normalize of a full D-element row (measured on a trained store: ~15K
429
- * halo() calls per deep query over ~50 distinct ids all but the first
430
- * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
431
- * halo mutation site. Callers receive a COPY, so the cached vector is
432
- * never aliased. Regenerable — a miss re-decodes the durable row. */
433
- _haloNorm;
434
- /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
435
- * that bridge nothing). Remembered so subsequent visits prune the subtree
436
- * without re-checking parent count. LRU-bounded: an evicted entry is
437
- * re-checked on next visit if it gained parents in the meantime, it will
438
- * be promoted to the index. */
439
- _coveredIds;
440
- /** Live content-index id set, LRU-bounded so a massive ingest never leaks
441
- * memory; an evicted entry is still indexed (the row is durable), so the
442
- * only cost of an eviction is a duplicate HNSW probe on next visit. */
443
- _indexedIds;
444
- // ── ANN read cache ─────────────────────────────────────────────────────
445
- // The index is read-only between writes, so the same (v,k) always returns
446
- // the same neighbours. Any index mutation (flush, delete, compact) drops
447
- // the cache; the next miss recreates it. Content-addressed (vecKey), not
448
- // identity-addressed same principle as perceiveMemo.
449
- /** ANN read cache for {@link resonate} keyed by vecKey(v) + ":" + k;
450
- * lazily initialised, dropped on any index mutation. */
451
- _resonateCache = null;
452
- /** ANN read cache for {@link resonateHalo} — same scheme. */
453
- _resonateHaloCache = null;
454
- // ── Write buffers ──────────────────────────────────────────────────────
455
- /** Content (gist) index write buffer. `ef` is the per-entry HNSW
456
- * construction budget: reach-only interiors carry the reduced
457
- * `efConstructionInterior`; dedup targets omit it (full budget). */
458
- _contentBuffer = [];
459
- /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
460
- _haloBuffer = new Map();
461
- /** Containment write buffer: child → new parents, merged on flush cadence. */
462
- _containBuf = new Map();
463
- /** Dedup-target candidates still in the write buffer (keyed by id). Only
464
- * roots that have gained an edge/halo are targets; a fresh intermediate
465
- * branch is never folded onto. */
466
- _nearDedupBuf = new Map();
467
- /** Ids currently in `_contentBuffer` (not yet flushed) O(1) membership. */
468
- _bufferedIds = new Set();
469
- // ── Transparent-chain cache ────────────────────────────────────────────
470
- /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
471
- * writes: a chain is a pure function of the kid and edge tables, so any
472
- * write that could break a node's transparency (a fresh mint inserting kid
473
- * rows, a link inserting an edge) drops the whole cache see the two
474
- * invalidation sites. Regenerable; a miss re-walks. */
475
- _chainMemo;
476
- // ── Edge-source-count cache ────────────────────────────────────────────
477
- /** Distinct edge-source count the store's DOCUMENT COUNT (how many
478
- * learnt contexts predict a continuation), the N of every
479
- * inverse-document-frequency read. −1 until first asked for; from then
480
- * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
481
- * so a read is O(1) — never a table scan on the recall path. */
482
- _edgeSrcCount = -1;
483
- // ── Constructor ────────────────────────────────────────────────────────
484
- constructor(config, D, maxGroup) {
485
- this._D = D;
486
- this._maxGroup = maxGroup;
487
- this.minHaloMass = config.minHaloMass;
488
- this.efSearch = config.efSearch;
489
- this.m = config.m;
490
- this.efConstruction = config.efConstruction;
491
- this.efConstructionInterior = Math.max(
492
- 1,
493
- Math.min(config.efConstructionInterior, config.efConstruction),
494
- );
495
- this.overfetch = config.overfetch;
496
- this.batchSize = config.batchSize;
497
- this.compactEveryNWrites = config.compactEveryNWrites;
498
- this._leafKey = new BoundedMap(config.dedupCacheMax);
499
- this._branchKey = new BoundedMap(config.dedupCacheMax);
500
- this._bytesCache = new BoundedMap(
501
- config.bytesCacheMax,
502
- (v) => v.byteLength,
503
- "smallest",
504
- );
505
- this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
506
- this._recCache = new BoundedMap(
507
- config.recCacheBytes,
508
- (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12,
509
- );
510
- this._pendingGist = new BoundedMap(
511
- config.pendingGistBytes,
512
- (v) => v.byteLength,
513
- );
514
- this._haloExact = new BoundedMap(
515
- config.haloCacheBytes,
516
- (v) => v.byteLength,
517
- );
518
- this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
519
- this._coveredIds = new BoundedMap(config.coveredIdsMax);
520
- this._indexedIds = new BoundedMap(config.coveredIdsMax);
521
- this._chainMemo = new BoundedMap(
522
- config.chainCacheBytes,
523
- (v) => v.length * 4 + 32,
524
- );
525
- }
526
- // ── Public accessors ───────────────────────────────────────────────────
527
- get D() {
528
- return this._D;
529
- }
530
- /** Await the async initialisation performed by the concrete constructor. */
531
- async _ensureReady() {
532
- if (!this._ready) {
533
- throw new Error("Store: not open");
534
- }
535
- await this._ready;
536
- }
537
- // ── Id management ──────────────────────────────────────────────────────
538
- has(id) {
539
- // Byte leaves (negative ids) always exist.
540
- if (id < 0) {
541
- return id >= -256;
542
- }
543
- return Number.isInteger(id) && id >= 0 && id < this._nextId;
544
- }
545
- mintId() {
546
- return this._nextId++;
547
- }
548
- nodeCount() {
549
- return this._nextId;
550
- }
551
- async size() {
552
- await this._ensureReady();
553
- return this._nextId;
554
- }
555
- // ── DAG traversal ──────────────────────────────────────────────────────
556
- get(id) {
557
- // Byte leaves are implicit — fabricate from the id.
558
- if (id < 0) {
559
- return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
560
- }
561
- const hit = this._recCache.get(id);
562
- if (hit !== undefined) {
563
- return hit;
564
- }
565
- const rec = this._dbGetNode(id);
566
- if (rec) {
567
- this._recCache.set(id, rec);
568
- }
569
- return rec;
570
- }
571
- /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
572
- * Iterative post-order on an explicit stack — the call stack never sees the
573
- * tree depth, so even an adversarial chain of nodes stays safe. */
574
- /** How many reads hit a MISSING node record this session (a dangling edge
575
- * or kid id). Zero in a healthy store; a growing count means references
576
- * outlive their records — the read degrades safely to empty bytes, this
577
- * counter is what keeps that degradation observable. */
578
- danglingReads = 0;
579
- bytes(id) {
580
- // Fast path.
581
- const hit = this._bytesCache.get(id);
582
- if (hit) {
583
- return hit;
584
- }
585
- const stack = [id];
586
- const cache = this._bytesCache;
587
- while (stack.length > 0) {
588
- const nid = stack[stack.length - 1]; // peek
589
- // Already resolved by an earlier traversal.
590
- if (cache.get(nid)) {
591
- stack.pop();
592
- continue;
593
- }
594
- const rec = this.get(nid);
595
- if (!rec) {
596
- // A DANGLING id (an edge or kid pointing at no record) reads as
597
- // empty — safe (empty-bytes guards drop it from grounding) but a
598
- // symptom of store corruption, so count it rather than stay silent.
599
- // The cache makes the empty read permanent for the session; the
600
- // counter survives as the visible trace.
601
- this.danglingReads++;
602
- cache.set(nid, _ZERO);
603
- stack.pop();
604
- continue;
605
- }
606
- if (rec.leaf) {
607
- cache.set(nid, new Uint8Array(rec.leaf));
608
- stack.pop();
609
- continue;
610
- }
611
- // Branch — push any uncached children (reverse order so they resolve
612
- // left-to-right). If every child is already cached, concatenate now.
613
- const kids = rec.kids ?? [];
614
- let ready = true;
615
- for (let i = kids.length - 1; i >= 0; i--) {
616
- if (!cache.get(kids[i])) {
617
- stack.push(kids[i]);
618
- ready = false;
362
+ /** Derived query breadth for a partitioned index of C clusters: probe √C
363
+ * of them (the same √-of-the-population convention as the hub bound √N).
364
+ * The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
365
+ * ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
366
+ * collection outgrows it: at 4,270 clusters the default 64 probed 16
367
+ * clusters (0.4%), and an exact stored match of a query routinely sat in
368
+ * an unprobed cluster — recall silently degraded as the store grew. The
369
+ * configured efSearch remains the floor for small collections. */
370
+ efFor(clusterCount) {
371
+ return Math.max(this.efSearch, 4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))));
372
+ }
373
+ // ── Config ─────────────────────────────────────────────────────────────
374
+ _D;
375
+ _maxGroup;
376
+ minHaloMass;
377
+ efSearch;
378
+ overfetch;
379
+ batchSize;
380
+ compactEveryNWrites;
381
+ // ── State ──────────────────────────────────────────────────────────────
382
+ /** Branch node ids are a dense, monotonically-increasing integer sequence
383
+ * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
384
+ * They are NEVER deleted, so the count of minted branch ids IS the next id —
385
+ * which doubles as the branch-node count and lets has() be an O(1) check.
386
+ * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
387
+ _nextId = 0;
388
+ _writtenSinceCompact = 0;
389
+ closed = false;
390
+ /** Lifecycle guardresolved once `_dbOpen()` completes. */
391
+ _ready = null;
392
+ // ── Caches ─────────────────────────────────────────────────────────────
393
+ /** Exact-content dedup: content-key node id. Intrinsic compression. */
394
+ _leafKey;
395
+ _branchKey;
396
+ /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
397
+ _bytesCache;
398
+ /** contentLen memo content is immutable, so entries never invalidate. */
399
+ _lenCache;
400
+ /** Node-record cache — avoids repeated persistence queries for shared DAG
401
+ * nodes. Each record is small (a few ints + short leaf buffer). */
402
+ _recCache;
403
+ /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
404
+ * links/pours a node right after interning it, so the working set is one
405
+ * deposit's nodes — a small budget captures ~all of it, and an eviction only
406
+ * means that node is reached by the DAG climb instead of by direct
407
+ * resonance. */
408
+ _pendingGist;
409
+ /** EXACT halo accumulators for the session's live pours: full-precision in
410
+ * memory, 2-bit on disk, so within-session accumulate-then-compare never
411
+ * round-trips through the quantizer. Regenerable a miss reads the durable
412
+ * 2-bit row. */
413
+ _haloExact;
414
+ /** NORMALIZED halo read cache the decoded, normalized vector {@link halo}
415
+ * returns, cached by id so repeat reads skip the per-call 2-bit decode and
416
+ * normalize of a full D-element row (measured on a trained store: ~15K
417
+ * halo() calls per deep query over ~50 distinct ids all but the first
418
+ * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
419
+ * halo mutation site. Callers receive a COPY, so the cached vector is
420
+ * never aliased. Regenerable a miss re-decodes the durable row. */
421
+ _haloNorm;
422
+ /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
423
+ * that bridge nothing). Remembered so subsequent visits prune the subtree
424
+ * without re-checking parent count. LRU-bounded: an evicted entry is
425
+ * re-checked on next visit if it gained parents in the meantime, it will
426
+ * be promoted to the index. */
427
+ _coveredIds;
428
+ /** Live content-index id set, LRU-bounded so a massive ingest never leaks
429
+ * memory; an evicted entry is still indexed (the row is durable), so the
430
+ * only cost of an eviction is a duplicate index probe on next visit. */
431
+ _indexedIds;
432
+ // ── ANN read cache ─────────────────────────────────────────────────────
433
+ // The index is read-only between writes, so the same (v,k) always returns
434
+ // the same neighbours. Any index mutation (flush, delete, compact) drops
435
+ // the cache; the next miss recreates it. Content-addressed (vecKey), not
436
+ // identity-addressed same principle as perceiveMemo.
437
+ /** ANN read cache for {@link resonate} — keyed by vecKey(v) + ":" + k;
438
+ * lazily initialised, dropped on any index mutation. */
439
+ _resonateCache = null;
440
+ /** ANN read cache for {@link resonateHalo} — same scheme. */
441
+ _resonateHaloCache = null;
442
+ // ── Write buffers ──────────────────────────────────────────────────────
443
+ /** Content (gist) index write buffer. */
444
+ _contentBuffer = [];
445
+ /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
446
+ _haloBuffer = new Map();
447
+ /** Containment write buffer: child new parents, merged on flush cadence. */
448
+ _containBuf = new Map();
449
+ /** Dedup-target candidates still in the write buffer (keyed by id). Only
450
+ * roots that have gained an edge/halo are targets; a fresh intermediate
451
+ * branch is never folded onto. */
452
+ _nearDedupBuf = new Map();
453
+ /** Ids currently in `_contentBuffer` (not yet flushed)O(1) membership. */
454
+ _bufferedIds = new Set();
455
+ // ── Transparent-chain cache ────────────────────────────────────────────
456
+ /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
457
+ * writes: a chain is a pure function of the kid and edge tables, so any
458
+ * write that could break a node's transparency (a fresh mint inserting kid
459
+ * rows, a link inserting an edge) drops the whole cache — see the two
460
+ * invalidation sites. Regenerable; a miss re-walks. */
461
+ _chainMemo;
462
+ // ── Edge-source-count cache ────────────────────────────────────────────
463
+ /** Distinct edge-source count — the store's DOCUMENT COUNT (how many
464
+ * learnt contexts predict a continuation), the N of every
465
+ * inverse-document-frequency read. −1 until first asked for; from then
466
+ * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
467
+ * so a read is O(1) — never a table scan on the recall path. */
468
+ _edgeSrcCount = -1;
469
+ // ── Constructor ────────────────────────────────────────────────────────
470
+ constructor(config, D, maxGroup) {
471
+ this._D = D;
472
+ this._maxGroup = maxGroup;
473
+ this.minHaloMass = config.minHaloMass;
474
+ this.efSearch = config.efSearch;
475
+ this.overfetch = config.overfetch;
476
+ this.batchSize = config.batchSize;
477
+ this.compactEveryNWrites = config.compactEveryNWrites;
478
+ this._leafKey = new BoundedMap(config.dedupCacheMax);
479
+ this._branchKey = new BoundedMap(config.dedupCacheMax);
480
+ this._bytesCache = new BoundedMap(config.bytesCacheMax, (v) => v.byteLength, "smallest");
481
+ this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
482
+ this._recCache = new BoundedMap(config.recCacheBytes, (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12);
483
+ this._pendingGist = new BoundedMap(config.pendingGistBytes, (v) => v.byteLength);
484
+ this._haloExact = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
485
+ this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
486
+ this._coveredIds = new BoundedMap(config.coveredIdsMax);
487
+ this._indexedIds = new BoundedMap(config.coveredIdsMax);
488
+ this._chainMemo = new BoundedMap(config.chainCacheBytes, (v) => v.length * 4 + 32);
489
+ }
490
+ // ── Public accessors ───────────────────────────────────────────────────
491
+ get D() {
492
+ return this._D;
493
+ }
494
+ /** Await the async initialisation performed by the concrete constructor. */
495
+ async _ensureReady() {
496
+ if (!this._ready)
497
+ throw new Error("Store: not open");
498
+ await this._ready;
499
+ }
500
+ // ── Id management ──────────────────────────────────────────────────────
501
+ has(id) {
502
+ // Byte leaves (negative ids) always exist.
503
+ if (id < 0)
504
+ return id >= -256;
505
+ return Number.isInteger(id) && id >= 0 && id < this._nextId;
506
+ }
507
+ mintId() {
508
+ return this._nextId++;
509
+ }
510
+ nodeCount() {
511
+ return this._nextId;
512
+ }
513
+ async size() {
514
+ await this._ensureReady();
515
+ return this._nextId;
516
+ }
517
+ // ── DAG traversal ──────────────────────────────────────────────────────
518
+ get(id) {
519
+ // Byte leaves are implicit fabricate from the id.
520
+ if (id < 0) {
521
+ return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
619
522
  }
620
- }
621
- if (!ready) {
622
- continue;
623
- }
624
- stack.pop();
625
- const out = concat(kids.map((k) => cache.get(k)));
626
- cache.set(nid, out);
627
- }
628
- return cache.get(id) ?? _ZERO;
629
- }
630
- /** First `maxLen` bytes of a node. Walks only the leftmost branch,
631
- * stopping at `maxLen` so a 1 MB document root costs the same as a
632
- * 4-byte leaf. Recursive, but tree depth is logarithmic.
633
- *
634
- * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
635
- * may be shared with the byte cache and with other callers — treat them
636
- * as read-only. Mutating one would corrupt every subsequent read. */
637
- bytesPrefix(id, maxLen) {
638
- if (maxLen <= 0) {
639
- return _ZERO;
640
- }
641
- // A FULL read (the ALL sentinel) routes through bytes(), whose
642
- // reconstruction enters the byte-budget cache. Without this, the mind's
643
- // read() which always passes ALL — re-walked the DAG and re-concatenated
644
- // on EVERY repeated read of an uncached branch, bypassing the cache that
645
- // exists precisely for those reconstructions.
646
- if (maxLen >= 0x7fffffff) {
647
- return this.bytes(id);
648
- }
649
- // Full-cache hit: bytes() already reconstructed the whole node.
650
- const full = this._bytesCache.get(id);
651
- if (full) {
652
- return full.length <= maxLen ? full : full.subarray(0, maxLen);
653
- }
654
- const rec = this.get(id);
655
- if (!rec) {
656
- return _ZERO;
657
- }
658
- if (rec.leaf) {
659
- // Cache the (small) leaf bytes — cheap and reusable. COPY before
660
- // caching: rec.leaf is the node record's own buffer, and handing it
661
- // out would let one mutating caller corrupt the record AND the cache
662
- // (bytes() makes the same copy for the same reason).
663
- const leaf = new Uint8Array(rec.leaf);
664
- this._bytesCache.set(id, leaf);
665
- return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
666
- }
667
- // Branch — walk children left-to-right, stopping at maxLen.
668
- const kids = rec.kids ?? [];
669
- const parts = [];
670
- let got = 0;
671
- for (const k of kids) {
672
- if (got >= maxLen) {
673
- break;
674
- }
675
- const child = this.bytesPrefix(k, maxLen - got);
676
- parts.push(child);
677
- got += child.length;
678
- }
679
- return concat(parts);
680
- }
681
- contentLen(id, cap = Infinity) {
682
- if (id < 0) {
683
- return 1; // implicit single-byte leaf
684
- }
685
- const hit = this._lenCache.get(id);
686
- if (hit !== undefined) {
687
- return hit; // exact — valid under any cap
688
- }
689
- if (cap <= 0) {
690
- return 0;
691
- }
692
- const rec = this.get(id);
693
- let n = 0;
694
- let clamped = false;
695
- if (rec) {
696
- if (rec.leaf) {
697
- n = rec.leaf.length;
698
- } else if (rec.kids) {
699
- for (const k of rec.kids) {
700
- n += this.contentLen(k, cap - n);
701
- if (n >= cap) {
702
- clamped = true; // partial sum — a lower bound, not the length
703
- break;
704
- }
523
+ const hit = this._recCache.get(id);
524
+ if (hit !== undefined)
525
+ return hit;
526
+ const rec = this._dbGetNode(id);
527
+ if (rec)
528
+ this._recCache.set(id, rec);
529
+ return rec;
530
+ }
531
+ /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
532
+ * Iterative post-order on an explicit stack — the call stack never sees the
533
+ * tree depth, so even an adversarial chain of nodes stays safe. */
534
+ /** How many reads hit a MISSING node record this session (a dangling edge
535
+ * or kid id). Zero in a healthy store; a growing count means references
536
+ * outlive their records — the read degrades safely to empty bytes, this
537
+ * counter is what keeps that degradation observable. */
538
+ danglingReads = 0;
539
+ bytes(id) {
540
+ // Fast path.
541
+ const hit = this._bytesCache.get(id);
542
+ if (hit)
543
+ return hit;
544
+ const stack = [id];
545
+ const cache = this._bytesCache;
546
+ while (stack.length > 0) {
547
+ const nid = stack[stack.length - 1]; // peek
548
+ // Already resolved by an earlier traversal.
549
+ if (cache.get(nid)) {
550
+ stack.pop();
551
+ continue;
552
+ }
553
+ const rec = this.get(nid);
554
+ if (!rec) {
555
+ // A DANGLING id (an edge or kid pointing at no record) reads as
556
+ // empty — safe (empty-bytes guards drop it from grounding) but a
557
+ // symptom of store corruption, so count it rather than stay silent.
558
+ // The cache makes the empty read permanent for the session; the
559
+ // counter survives as the visible trace.
560
+ this.danglingReads++;
561
+ cache.set(nid, _ZERO);
562
+ stack.pop();
563
+ continue;
564
+ }
565
+ if (rec.leaf) {
566
+ cache.set(nid, new Uint8Array(rec.leaf));
567
+ stack.pop();
568
+ continue;
569
+ }
570
+ // Branch — push any uncached children (reverse order so they resolve
571
+ // left-to-right). If every child is already cached, concatenate now.
572
+ const kids = rec.kids ?? [];
573
+ let ready = true;
574
+ for (let i = kids.length - 1; i >= 0; i--) {
575
+ if (!cache.get(kids[i])) {
576
+ stack.push(kids[i]);
577
+ ready = false;
578
+ }
579
+ }
580
+ if (!ready)
581
+ continue;
582
+ stack.pop();
583
+ const out = concat(kids.map((k) => cache.get(k)));
584
+ cache.set(nid, out);
705
585
  }
706
- }
707
- }
708
- if (!clamped) {
709
- this._lenCache.set(id, n);
710
- }
711
- return n;
712
- }
713
- // ── Content-addressed lookup ───────────────────────────────────────────
714
- findLeaf(bytes) {
715
- if (bytes.length === 1) {
716
- return -(bytes[0] + 1);
717
- }
718
- const key = keyOf(bytes);
719
- const cached = this._leafKey.get(key);
720
- if (cached !== undefined) {
721
- return cached;
722
- }
723
- const id = this._dbFindLeaf(hashOf(bytes), bytes);
724
- if (id !== null) {
725
- this._leafKey.set(key, id);
726
- }
727
- return id;
728
- }
729
- findBranch(kids) {
730
- const key = kids.join(",");
731
- const cached = this._branchKey.get(key);
732
- if (cached !== undefined) {
733
- return cached;
734
- }
735
- const flat = flatKidsBytes(kids);
736
- let id;
737
- if (flat) {
738
- id = this._dbFindBranchByLeaf(hashOf(flat), flat);
739
- } else {
740
- const packed = packKids(kids);
741
- id = this._dbFindBranchByKids(hashOf(packed), packed);
742
- }
743
- if (id !== null && key.length <= DEDUP_KEY_MAX) {
744
- this._branchKey.set(key, id);
745
- }
746
- return id;
747
- }
748
- // ── Structural parents ─────────────────────────────────────────────────
749
- parents(id) {
750
- return this._dbGetParents(id);
751
- }
752
- parentsFirst(id, limit) {
753
- return this._dbGetParentsFirst(id, limit);
754
- }
755
- hasParents(id) {
756
- return this._dbGetParentsFirst(id, 1).length > 0;
757
- }
758
- chainRun(id) {
759
- const hit = this._chainMemo.get(id);
760
- if (hit !== undefined) {
761
- return hit;
762
- }
763
- const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
764
- this._chainMemo.set(id, run);
765
- return run;
766
- }
767
- /** {@link Store.chainRun}'s walk, node at a time through the existing
768
- * probes. Adapters with a set-based query engine should override with a
769
- * single server-side descent (the SQLite adapter uses a recursive CTE). */
770
- _chainWalk(id, cap) {
771
- const run = [id];
772
- let n = id;
773
- while (run.length < cap) {
774
- if (this.hasNext(n) || this.prevCount(n) > 0) {
775
- break;
776
- }
777
- const ps = this._dbGetParentsFirst(n, 2);
778
- if (ps.length !== 1) {
779
- break;
780
- }
781
- n = ps[0];
782
- run.push(n);
783
- }
784
- return run;
785
- }
786
- // ── Containment ────────────────────────────────────────────────────────
787
- addContainer(child, parent) {
788
- let set = this._containBuf.get(child);
789
- if (set === undefined) {
790
- set = new Set();
791
- this._containBuf.set(child, set);
792
- }
793
- set.add(parent);
794
- }
795
- hasContainers(child) {
796
- if (this._dbContainExists(child)) {
797
- return true;
798
- }
799
- const buf = this._containBuf.get(child);
800
- return buf !== undefined && buf.size > 0;
801
- }
802
- containersSlice(child, offset, limit) {
803
- const out = this._dbGetContainParentsSlice(child, offset, limit);
804
- if (out.length >= limit) {
805
- return out;
806
- }
807
- // Buffered adds page in AFTER the stored ones. A buffered parent that is
808
- // also stored may repeat across the seam; page consumers dedup by id
809
- // (they all carry seen-sets), so a repeat costs a skip, never an error.
810
- const buf = this._containBuf.get(child);
811
- if (!buf || buf.size === 0) {
812
- return out;
813
- }
814
- const storedCount = this._dbGetContainCount(child);
815
- const bufStart = Math.max(0, offset - storedCount) +
816
- Math.max(0, out.length - Math.max(0, storedCount - offset));
817
- let i = 0;
818
- for (const p of buf) {
819
- if (out.length >= limit) {
820
- break;
821
- }
822
- if (i++ < bufStart) {
823
- continue;
824
- }
825
- out.push(p);
826
- }
827
- return out;
828
- }
829
- containers(child) {
830
- const stored = this._dbGetContainParents(child);
831
- const buf = this._containBuf.get(child);
832
- if (stored.length === 0) {
833
- return buf ? [...buf] : [];
834
- }
835
- if (!buf) {
836
- return stored;
837
- }
838
- const merged = new Set(stored);
839
- for (const p of buf) {
840
- merged.add(p);
841
- }
842
- return [...merged];
843
- }
844
- // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
845
- flatLeafIds(kids) {
846
- const out = [];
847
- const stack = [...kids].reverse();
848
- while (stack.length > 0) {
849
- const id = stack.pop();
850
- if (id < 0) {
851
- out.push(id);
852
- continue;
853
- }
854
- const rec = this.get(id);
855
- if (!rec) {
856
- return null;
857
- }
858
- if (rec.leaf !== null) {
859
- for (let i = 0; i < rec.leaf.length; i++) {
860
- out.push(-(rec.leaf[i] + 1));
586
+ return cache.get(id) ?? _ZERO;
587
+ }
588
+ /** First `maxLen` bytes of a node. Walks only the leftmost branch,
589
+ * stopping at `maxLen` — so a 1 MB document root costs the same as a
590
+ * 4-byte leaf. Recursive, but tree depth is logarithmic.
591
+ *
592
+ * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
593
+ * may be shared with the byte cache and with other callers — treat them
594
+ * as read-only. Mutating one would corrupt every subsequent read. */
595
+ bytesPrefix(id, maxLen) {
596
+ if (maxLen <= 0)
597
+ return _ZERO;
598
+ // A FULL read (the ALL sentinel) routes through bytes(), whose
599
+ // reconstruction enters the byte-budget cache. Without this, the mind's
600
+ // read() which always passes ALL — re-walked the DAG and re-concatenated
601
+ // on EVERY repeated read of an uncached branch, bypassing the cache that
602
+ // exists precisely for those reconstructions.
603
+ if (maxLen >= 0x7fffffff)
604
+ return this.bytes(id);
605
+ // Full-cache hit: bytes() already reconstructed the whole node.
606
+ const full = this._bytesCache.get(id);
607
+ if (full)
608
+ return full.length <= maxLen ? full : full.subarray(0, maxLen);
609
+ const rec = this.get(id);
610
+ if (!rec)
611
+ return _ZERO;
612
+ if (rec.leaf) {
613
+ // Cache the (small) leaf bytes — cheap and reusable. COPY before
614
+ // caching: rec.leaf is the node record's own buffer, and handing it
615
+ // out would let one mutating caller corrupt the record AND the cache
616
+ // (bytes() makes the same copy for the same reason).
617
+ const leaf = new Uint8Array(rec.leaf);
618
+ this._bytesCache.set(id, leaf);
619
+ return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
861
620
  }
862
- } else if (rec.kids !== null) {
863
- for (let i = rec.kids.length - 1; i >= 0; i--) {
864
- stack.push(rec.kids[i]);
621
+ // Branch walk children left-to-right, stopping at maxLen.
622
+ const kids = rec.kids ?? [];
623
+ const parts = [];
624
+ let got = 0;
625
+ for (const k of kids) {
626
+ if (got >= maxLen)
627
+ break;
628
+ const child = this.bytesPrefix(k, maxLen - got);
629
+ parts.push(child);
630
+ got += child.length;
865
631
  }
866
- }
867
- }
868
- return out;
869
- }
870
- /** On a dedup HIT, keep the node's gist available for lazy indexing —
871
- * EXACTLY when it is not already indexed. Replaces the old id-range
872
- * "recency" heuristic (id nextId − cacheWindow), which conflated an LRU
873
- * entry COUNT with an id RANGE and permanently refused to index any node
874
- * that first became a resonance target long after it was minted (an early
875
- * interior later reused as an edge/halo-bearing deposit root was silently
876
- * unreachable by resonance). The durable index itself is the arbiter:
877
- * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
878
- captureIfUnindexed(id, gist) {
879
- if (this._indexedIds.has(id) || this._pendingGist.has(id)) {
880
- return;
881
- }
882
- if (this._vecContentHas(id)) {
883
- this._indexedIds.set(id, true);
884
- return;
885
- }
886
- this._pendingGist.set(id, normalize(copy(gist)));
887
- // A node that ALREADY bridges experiences but was never indexed (its 1→2
888
- // transition fired while its gist was evicted, or in a pre-transition
889
- // store) is promoted on this re-encounter — the recapture above is
890
- // exactly what makes its gist available again. Byte leaves (negative
891
- // ids) never have kid rows, so skip their parent probe.
892
- if (id >= 0) {
893
- this.promoteBridge(id);
894
- }
895
- }
896
- /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
897
- * criterion), promote its gist into the content index NOW — the exact
898
- * moment it becomes useful for multi-experience recall. The 1→2 parent
899
- * transition fires on {@link _dbInsertKid} during mint, and nodes that
900
- * were already bridges but missed indexing (gist evicted, pre-transition
901
- * store) are recaptured in {@link captureIfUnindexed}.
902
- *
903
- * A no-op when the node is already indexed or its gist is evicted from
904
- * the pending cache — a future re-encounter will retry. */
905
- promoteBridge(id) {
906
- // A LIMITed probe: this runs once per kid insert on the MINT hot path,
907
- // and a shared child's full parent set grows with the corpus.
908
- if (this._dbGetParentsFirst(id, 2).length < 2) {
909
- return;
910
- }
911
- this.indexGist(id, false);
912
- }
913
- // ── Core interning: dedup → near-dedup → mint ──────────────────────────
914
- async intern(leaf, kids, gist) {
915
- await this._ensureReady();
916
- // 1. Exact dedup — equal content → one id, no vector work. Primary
917
- // compression mechanism, intrinsic to the store.
918
- //
919
- // findLeaf/findBranch are the content-addressed lookups: in-memory cache
920
- // first, then a durable probe that repopulates the cache on a hit.
921
- // Using them here (rather than only the cache) makes dedup survive a cold
922
- // cache — a resumed/checkpointed training run, or one whose dedup cache
923
- // has evicted old keys, still recognises content already on disk and
924
- // reuses its id instead of minting a duplicate.
925
- const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
926
- if (hit !== null) {
927
- this.captureIfUnindexed(hit, gist);
928
- return hit;
929
- }
930
- // 1b. Content-addressed lookup by leaf-id signature. When the same byte
931
- // sequence was stored as a flat branch (via putBranch during deposit),
932
- // a branch node spanning those bytes reuses that id even when its tree
933
- // structure differs — pure content addressing, same bytes → same node.
934
- if (kids !== null && kids.length >= 2) {
935
- const leafIds = this.flatLeafIds(kids);
936
- if (leafIds !== null) {
937
- const flatHit = this.findBranch(leafIds);
938
- if (flatHit !== null) {
939
- this.captureIfUnindexed(flatHit, gist);
940
- return flatHit;
632
+ return concat(parts);
633
+ }
634
+ contentLen(id, cap = Infinity) {
635
+ if (id < 0)
636
+ return 1; // implicit single-byte leaf
637
+ const hit = this._lenCache.get(id);
638
+ if (hit !== undefined)
639
+ return hit; // exact valid under any cap
640
+ if (cap <= 0)
641
+ return 0;
642
+ const rec = this.get(id);
643
+ let n = 0;
644
+ let clamped = false;
645
+ if (rec) {
646
+ if (rec.leaf)
647
+ n = rec.leaf.length;
648
+ else if (rec.kids) {
649
+ for (const k of rec.kids) {
650
+ n += this.contentLen(k, cap - n);
651
+ if (n >= cap) {
652
+ clamped = true; // partial sum — a lower bound, not the length
653
+ break;
654
+ }
655
+ }
656
+ }
941
657
  }
942
- }
943
- }
944
- const cache = leaf !== null ? this._leafKey : this._branchKey;
945
- const key = leaf !== null ? keyOf(leaf) : kids.join(",");
946
- // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
947
- // Leaves are single bytes: exact dedup already collapses every identical
948
- // leaf, and near-merging distinct leaves only corrupts bytes for no real
949
- // saving. Real near-dedup compression lives in subtree (branch) fusion.
950
- //
951
- // There is deliberately NO HNSW probe of the FLUSHED index here. It used
952
- // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
953
- // ~every interior branch, since interiors are never dedup targets —
954
- // making one ANN query per branch the dominant training cost (it dwarfed
955
- // perception and the index write). And it was not merely expensive but
956
- // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
957
- // nearest "target" of a fresh branch, so the fold collapsed two
958
- // byte-different subtrees onto one id and corrupted exact reconstruction
959
- // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
960
- // are caught two cheaper, exact ways instead: identical content by the
961
- // exact-dedup hash-cons above, and a near-gist target still in the write
962
- // buffer by the scan below.
963
- if (leaf === null) {
964
- // Near-dedup PREFILTER — the scale-aware identity bar
965
- // ({@link identityBar}) for THIS branch's own length, not the fixed
966
- // estimator floor: under the linear fold a long branch crosses
967
- // 1 − 1/√D while whole windows differ, and every such crossing paid a
968
- // full differsByOneWindow byte reconstruction. Same final semantics
969
- // (the byte check below still decides identity); strictly fewer byte
970
- // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
971
- // memoized bottom-up by the interning order itself, O(kids)) is only
972
- // computed once a nearest candidate actually exists, so the hot
973
- // no-candidate mint pays nothing.
974
- let best = -1;
975
- let bestId = null;
976
- if (this._nearDedupBuf.size > 0) {
977
- const g = normalize(copy(gist));
978
- // Candidates are the buffered DEDUP TARGETS only — genuine whole
979
- // experiences (edge/halo-bearing roots) not yet flushed.
980
- for (const [id, vector] of this._nearDedupBuf) {
981
- const s = dot(g, vector);
982
- if (s > best) {
983
- best = s;
984
- bestId = id;
985
- }
658
+ if (!clamped)
659
+ this._lenCache.set(id, n);
660
+ return n;
661
+ }
662
+ // ── Content-addressed lookup ───────────────────────────────────────────
663
+ findLeaf(bytes) {
664
+ if (bytes.length === 1)
665
+ return -(bytes[0] + 1);
666
+ const key = keyOf(bytes);
667
+ const cached = this._leafKey.get(key);
668
+ if (cached !== undefined)
669
+ return cached;
670
+ const id = this._dbFindLeaf(hashOf(bytes), bytes);
671
+ if (id !== null)
672
+ this._leafKey.set(key, id);
673
+ return id;
674
+ }
675
+ findBranch(kids) {
676
+ const key = kids.join(",");
677
+ const cached = this._branchKey.get(key);
678
+ if (cached !== undefined)
679
+ return cached;
680
+ const flat = flatKidsBytes(kids);
681
+ let id;
682
+ if (flat) {
683
+ id = this._dbFindBranchByLeaf(hashOf(flat), flat);
986
684
  }
987
- }
988
- let blen = 0;
989
- if (bestId !== null) {
990
- for (const k of kids) {
991
- blen += this.contentLen(k);
685
+ else {
686
+ const packed = packKids(kids);
687
+ id = this._dbFindBranchByKids(hashOf(packed), packed);
992
688
  }
993
- }
994
- if (
995
- bestId !== null &&
996
- best >= identityBar(this.D, this._maxGroup, blen)
997
- ) {
998
- // Scale-aware acceptance. The cosine bar alone is scale-BLIND
999
- // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
1000
- // a localized difference faster than linearly in form size, so for
1001
- // deep forms ANY fixed bar below 1 is crossed by exactly the one
1002
- // span that distinguishes two experiences. No inversion of the
1003
- // deficit is trustworthy, so the bytes themselves decide: the two
1004
- // forms must be identical except for ONE local span of at most W
1005
- // bytes — the river window, the perception's own resolution quantum.
1006
- const W = this._maxGroup;
1007
- if (this.differsByOneWindow(kids, bestId, W)) {
1008
- if (key.length <= DEDUP_KEY_MAX) {
1009
- cache.set(key, bestId);
1010
- }
1011
- return bestId;
689
+ if (id !== null && key.length <= DEDUP_KEY_MAX) {
690
+ this._branchKey.set(key, id);
1012
691
  }
1013
- }
1014
- }
1015
- // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
1016
- // leaf) stores its BYTES in the leaf column with a zero-length kids blob
1017
- // as the marker — the kid list is derived on read.
1018
- const id = this.mintId();
1019
- this._dbBeginTx();
1020
- const flat = kids ? flatKidsBytes(kids) : null;
1021
- const packed = kids && !flat ? packKids(kids) : null;
1022
- this._dbInsertNode(
1023
- id,
1024
- leaf ?? flat,
1025
- packed ?? (flat ? _ZERO : null),
1026
- hashOf(leaf ?? flat ?? packed),
1027
- );
1028
- // Reverse structural edge: each distinct child → this parent. Lets the graph
1029
- // be climbed upward in index time, with no scan of the kids blobs.
1030
- //
1031
- // Populated NATURALLY here and only here — one write per child, in the SAME
1032
- // mint that creates the node, inside the SAME deferred transaction as the
1033
- // node row, so node and kid rows are always durable together.
1034
- //
1035
- // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
1036
- // every branch, so its parent set is the corpus-sized hub the climb's
1037
- // saturation guard discards unread.
1038
- if (kids) {
1039
- // Kid rows change parent counts — a child whose parent set grows from
1040
- // one is no longer transparent, so every cached chain that hopped
1041
- // through it is stale. Which chains those are is unknowable without a
1042
- // reverse index, so the WHOLE cache drops (writes come in training
1043
- // bursts where the cache is cold anyway; queries rebuild it lazily).
1044
- this._chainMemo.clear();
1045
- for (const c of kids) {
1046
- if (c < 0 && c >= -256) {
1047
- continue;
692
+ return id;
693
+ }
694
+ // ── Structural parents ─────────────────────────────────────────────────
695
+ parents(id) {
696
+ return this._dbGetParents(id);
697
+ }
698
+ parentsFirst(id, limit) {
699
+ return this._dbGetParentsFirst(id, limit);
700
+ }
701
+ hasParents(id) {
702
+ return this._dbGetParentsFirst(id, 1).length > 0;
703
+ }
704
+ chainRun(id) {
705
+ const hit = this._chainMemo.get(id);
706
+ if (hit !== undefined)
707
+ return hit;
708
+ const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
709
+ this._chainMemo.set(id, run);
710
+ return run;
711
+ }
712
+ /** {@link Store.chainRun}'s walk, node at a time through the existing
713
+ * probes. Adapters with a set-based query engine should override with a
714
+ * single server-side descent (the SQLite adapter uses a recursive CTE). */
715
+ _chainWalk(id, cap) {
716
+ const run = [id];
717
+ let n = id;
718
+ while (run.length < cap) {
719
+ if (this.hasNext(n) || this.prevCount(n) > 0)
720
+ break;
721
+ const ps = this._dbGetParentsFirst(n, 2);
722
+ if (ps.length !== 1)
723
+ break;
724
+ n = ps[0];
725
+ run.push(n);
1048
726
  }
1049
- this._dbInsertKid(c, id);
1050
- // The 1→2 parent TRANSITION happens here and only here (hash-cons
1051
- // means an existing branch never re-inserts kid rows, so parent sets
1052
- // grow exclusively through fresh mints): the child just became a
1053
- // structural bridge between experiences — the exact set post-hoc
1054
- // compaction keeps so its gist enters the reach index NOW.
1055
- this.promoteBridge(c);
1056
- }
1057
- }
1058
- if (key.length <= DEDUP_KEY_MAX) {
1059
- cache.set(key, id);
1060
- }
1061
- if (leaf) {
1062
- this._bytesCache.set(id, new Uint8Array(leaf));
1063
- } else if (flat) {
1064
- this._bytesCache.set(id, flat);
1065
- }
1066
- // Capture the gist; it is pushed into the content index lazily, the first
1067
- // time this node becomes a resonance target (link / pourHalo). A node that
1068
- // never does (a pure intermediate DAG node — ~99.5% of them) is never
1069
- // indexed: it costs one persistence row, no HNSW slot and no merge probe.
1070
- this._pendingGist.set(id, normalize(copy(gist)));
1071
- await this.maybeFlush();
1072
- return id;
1073
- }
1074
- /** Whether the byte content under `kids` and the byte content of `targetId`
1075
- * are identical except for ONE local span of at most `W` bytes on each side
1076
- * — the near dedup's byte-grain definition of a near-duplicate. A
1077
- * common-prefix / common-suffix trim: whatever remains after both trims is
1078
- * the single differing span (substitution, insertion or deletion), and both
1079
- * remainders must fit the budget. Scattered differences leave a wide
1080
- * middle and are rejected. */
1081
- differsByOneWindow(kids, targetId, W) {
1082
- const a = concat(
1083
- kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)),
1084
- );
1085
- const b = this.bytesPrefix(targetId, a.length + W + 1);
1086
- if (Math.abs(a.length - b.length) > W) {
1087
- return false;
1088
- }
1089
- const n = Math.min(a.length, b.length);
1090
- let i = 0;
1091
- while (i < n && a[i] === b[i]) {
1092
- i++;
1093
- }
1094
- let j = 0;
1095
- while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j]) {
1096
- j++;
1097
- }
1098
- return a.length - i - j <= W && b.length - i - j <= W;
1099
- }
1100
- async putLeaf(bytes, gist) {
1101
- // Single bytes are implicit — no DB row and no eager index slot. The gist
1102
- // is captured like any other node's and promoted into the content index
1103
- // LAZILY, the first time the byte becomes a resonance target.
1104
- if (bytes.length === 1) {
1105
- const id = -(bytes[0] + 1);
1106
- this.captureIfUnindexed(id, gist);
1107
- return id;
1108
- }
1109
- return this.intern(new Uint8Array(bytes), null, gist);
1110
- }
1111
- async putBranch(kids, gist) {
1112
- return this.intern(null, kids, gist);
1113
- }
1114
- // ── Lazy content indexing ──────────────────────────────────────────────
1115
- /** Promote a node's captured gist into the content (resonance) index, once.
1116
- * Called the first time a node becomes a target — i.e. from `link` (it bears
1117
- * or receives a continuation edge) or `pourHalo` (it gains distributional
1118
- * company). Idempotent: a node already indexed, or whose gist has been evicted
1119
- * from the bounded pending map, is a no-op.
1120
- *
1121
- * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1122
- * near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
1123
- * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1124
- indexGist(id, dedupTarget) {
1125
- if (dedupTarget && this._bufferedIds.has(id)) {
1126
- // Still buffered (indexed this batch, not yet flushed) — a live dedup
1127
- // candidate, recorded in O(1) with no scan of the content buffer.
1128
- const v = this._pendingGist.get(id);
1129
- if (v !== undefined) {
1130
- this._nearDedupBuf.set(id, v);
1131
- }
1132
- }
1133
- if (this._indexedIds.has(id)) {
1134
- return;
1135
- }
1136
- const v = this._pendingGist.get(id);
1137
- if (v === undefined) {
1138
- return;
1139
- }
1140
- // Already durably indexed by a previous session? A node id names its
1141
- // content, and the gist is a pure function of the content, so the stored
1142
- // vector can only be identical — re-buffering it would spend an encode
1143
- // and an upsert on a guaranteed no-op. One point query recognises this;
1144
- // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1145
- // This is what makes a RESUMED training run replay already-deposited
1146
- // content at read speed instead of re-upserting the recent-id window.
1147
- if (this._vecContentHas(id)) {
1148
- this._indexedIds.set(id, true);
1149
- return;
1150
- }
1151
- this._indexedIds.set(id, true);
1152
- // Reach-only interiors build with the reduced construction budget — the
1153
- // one deliberate speed-for-quality trade of ingestion (see
1154
- // {@link StoreConfig.efConstructionInterior}); targets keep the full one.
1155
- this._contentBuffer.push(
1156
- dedupTarget
1157
- ? { id, vector: v }
1158
- : { id, vector: v, ef: this.efConstructionInterior },
1159
- );
1160
- this._bufferedIds.add(id);
1161
- // A node indexed AS a dedup target enters the candidate set immediately.
1162
- if (dedupTarget) {
1163
- this._nearDedupBuf.set(id, v);
1164
- }
1165
- }
1166
- /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1167
- * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1168
- * root is a genuine target (a whole experience), so it is a dedup target
1169
- * too. */
1170
- indexTarget(id) {
1171
- this.indexGist(id, true);
1172
- }
1173
- /** Index a node and its interior forms as resonance targets. A node that
1174
- * gains an edge is a learnt EXPERIENCE, and the consensus climb
1175
- * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1176
- * resonating its SUB-REGIONS — branch nodes within the experience — and
1177
- * climbing their parents back to it.
1178
- *
1179
- * EVERY interior branch is indexed unconditionally, and this is
1180
- * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1181
- * the post-hoc compaction criterion) was tried and REJECTED by the test
1182
- * suite — partial recall of an experience's interior slices, multi-topic
1183
- * attention, and counterfactual anchoring all resonate to SINGLE-parent
1184
- * interiors (13 tests fail without them). Post-hoc structural compaction
1185
- * ({@link compactContentIndex}) may still remove them, but that is a
1186
- * storage/recall trade-off for archived stores, not a free optimisation.
1187
- * The store's hash-cons bounds the index by the number of DISTINCT byte
1188
- * patterns in the corpus — not by the number of deposits.
1189
- *
1190
- * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1191
- * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1192
- * let a partial query resonate and climb, but a fresh branch must never
1193
- * merge onto an interior node of another experience.
1194
- *
1195
- * Iterative explicit-queue walk: the call stack never sees tree depth. */
1196
- indexSubtree(root) {
1197
- // The root is the whole experience — always index as a merge target.
1198
- this.indexGist(root, true);
1199
- const seen = new Set([root]);
1200
- const stack = [root];
1201
- while (stack.length > 0) {
1202
- const id = stack.pop();
1203
- const kids = this.get(id)?.kids;
1204
- if (!kids) {
1205
- continue; // leaf — never a resonance anchor
1206
- }
1207
- const isRoot = id === root;
1208
- // Already-indexed or already-skipped nodes PRUNE the walk — their
1209
- // subtrees were classified in a prior call.
1210
- if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1211
- continue;
1212
- }
1213
- for (const k of kids) {
1214
- if (!seen.has(k)) {
1215
- seen.add(k);
1216
- stack.push(k);
727
+ return run;
728
+ }
729
+ // ── Containment ────────────────────────────────────────────────────────
730
+ addContainer(child, parent) {
731
+ let set = this._containBuf.get(child);
732
+ if (set === undefined) {
733
+ set = new Set();
734
+ this._containBuf.set(child, set);
1217
735
  }
1218
- }
1219
- if (isRoot) {
1220
- continue;
1221
- }
1222
- const g = this._pendingGist.get(id);
1223
- if (g === undefined) {
1224
- // Gist evicted from the bounded pending cache — the node can't be
1225
- // indexed now. Mark as covered so subsequent visits prune here
1226
- // instead of re-walking the subtree (the gist won't return).
1227
- this._coveredIds.set(id, true);
1228
- continue;
1229
- }
1230
- // Index unconditionally every interior node is a valid resonance
1231
- // anchor for partial-query recall. The _indexedIds cache and the
1232
- // durable-index check in indexGist prevent duplicate indexing across
1233
- // multiple encounters of the same shared subtree.
1234
- this.indexGist(id, false);
1235
- }
1236
- }
1237
- // ── Soft resonance ─────────────────────────────────────────────────────
1238
- async resonate(v, k) {
1239
- await this._ensureReady();
1240
- // Synchronous flush of any buffered index writes: the FIRST resonance
1241
- // after a large ingest pays that flush here, so it shows up in respond
1242
- // latency, not ingest latency — correct behaviour, skewed attribution;
1243
- // profile accordingly.
1244
- this.flushContent();
1245
- if (k <= 0) {
1246
- return [];
1247
- }
1248
- // ANN read cache — content-addressed so a fresh Float32Array with the
1249
- // same values still hits. Lazy-init: null after any index write; the
1250
- // first miss after a flush recreates it. When voteRegions resonates
1251
- // identical perceived sub-regions, only the first call descends the ANN.
1252
- const rk = vecKey(v) + ":" + k;
1253
- const cache = this._resonateCache;
1254
- if (cache) {
1255
- const hit = cache.get(rk);
1256
- if (hit !== undefined) {
1257
- return hit;
1258
- }
1259
- }
1260
- const results = this._vecContentQuery(
1261
- normalize(copy(v)),
1262
- k * this.overfetch,
1263
- this.efSearch,
1264
- );
1265
- const out = [];
1266
- for (const r of results) {
1267
- const id = r.id;
1268
- out.push({ id, score: 1 - r.distance });
1269
- if (out.length >= k) {
1270
- break;
1271
- }
1272
- }
1273
- const rc = this._resonateCache ??= new Map();
1274
- if (rc.size >= RESONATE_CACHE_MAX) {
1275
- rc.clear();
1276
- }
1277
- rc.set(rk, out);
1278
- return out;
1279
- }
1280
- indexedVectorCount() {
1281
- this.flushContent();
1282
- return this._vecContentSize();
1283
- }
1284
- lastResonateReads() {
1285
- return this._vecContentLastReads();
1286
- }
1287
- // ── Content index compaction ────────────────────────────────────────────
1288
- /** How many physical compaction attempts have failed this session. Zero in
1289
- * a healthy store; a growing count means tombstones are accumulating and
1290
- * index query cost is drifting up (the first failure also warns once). */
1291
- compactFailures = 0;
1292
- /** Meta key holding the incremental scan watermark of
1293
- * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1294
- * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1295
- * entry once kept can never become removable — and removed entries are
1296
- * gone, so a pass only ever needs to examine entries indexed AFTER the
1297
- * previous pass. Internal ids are monotone and survive the index's
1298
- * splice compaction; the watermark is reset whenever a PHYSICAL index
1299
- * compaction runs (id reuse after a dropped top row would otherwise hide
1300
- * new entries behind it). */
1301
- static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1302
- /** {@link Store.compactContentIndex} */
1303
- async compactContentIndex(minParents = 2) {
1304
- await this._ensureReady();
1305
- this.flush(); // commit any pending writes first
1306
- // Incremental scan: resume from the last pass's watermark when its
1307
- // minParents matches; a changed criterion forces a full rescan.
1308
- let after = 0;
1309
- {
1310
- const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1311
- if (raw !== null) {
1312
- const sep = raw.indexOf(":");
1313
- const mp = Number(raw.slice(0, sep));
1314
- const wm = Number(raw.slice(sep + 1));
1315
- if (mp === minParents && Number.isFinite(wm) && wm > 0) {
1316
- after = wm;
736
+ set.add(parent);
737
+ }
738
+ hasContainers(child) {
739
+ if (this._dbContainExists(child))
740
+ return true;
741
+ const buf = this._containBuf.get(child);
742
+ return buf !== undefined && buf.size > 0;
743
+ }
744
+ containersSlice(child, offset, limit) {
745
+ const out = this._dbGetContainParentsSlice(child, offset, limit);
746
+ if (out.length >= limit)
747
+ return out;
748
+ // Buffered adds page in AFTER the stored ones. A buffered parent that is
749
+ // also stored may repeat across the seam; page consumers dedup by id
750
+ // (they all carry seen-sets), so a repeat costs a skip, never an error.
751
+ const buf = this._containBuf.get(child);
752
+ if (!buf || buf.size === 0)
753
+ return out;
754
+ const storedCount = this._dbGetContainCount(child);
755
+ const bufStart = Math.max(0, offset - storedCount) +
756
+ Math.max(0, out.length - Math.max(0, storedCount - offset));
757
+ let i = 0;
758
+ for (const p of buf) {
759
+ if (out.length >= limit)
760
+ break;
761
+ if (i++ < bufStart)
762
+ continue;
763
+ out.push(p);
764
+ }
765
+ return out;
766
+ }
767
+ containers(child) {
768
+ const stored = this._dbGetContainParents(child);
769
+ const buf = this._containBuf.get(child);
770
+ if (stored.length === 0)
771
+ return buf ? [...buf] : [];
772
+ if (!buf)
773
+ return stored;
774
+ const merged = new Set(stored);
775
+ for (const p of buf)
776
+ merged.add(p);
777
+ return [...merged];
778
+ }
779
+ // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
780
+ flatLeafIds(kids) {
781
+ const out = [];
782
+ const stack = [...kids].reverse();
783
+ while (stack.length > 0) {
784
+ const id = stack.pop();
785
+ if (id < 0) {
786
+ out.push(id);
787
+ continue;
788
+ }
789
+ const rec = this.get(id);
790
+ if (!rec)
791
+ return null;
792
+ if (rec.leaf !== null) {
793
+ for (let i = 0; i < rec.leaf.length; i++)
794
+ out.push(-(rec.leaf[i] + 1));
795
+ }
796
+ else if (rec.kids !== null) {
797
+ for (let i = rec.kids.length - 1; i >= 0; i--)
798
+ stack.push(rec.kids[i]);
799
+ }
800
+ }
801
+ return out;
802
+ }
803
+ /** On a dedup HIT, keep the node's gist available for lazy indexing —
804
+ * EXACTLY when it is not already indexed. Replaces the old id-range
805
+ * "recency" heuristic (id nextId − cacheWindow), which conflated an LRU
806
+ * entry COUNT with an id RANGE and permanently refused to index any node
807
+ * that first became a resonance target long after it was minted (an early
808
+ * interior later reused as an edge/halo-bearing deposit root was silently
809
+ * unreachable by resonance). The durable index itself is the arbiter:
810
+ * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
811
+ captureIfUnindexed(id, gist) {
812
+ if (this._indexedIds.has(id) || this._pendingGist.has(id))
813
+ return;
814
+ if (this._vecContentHas(id)) {
815
+ this._indexedIds.set(id, true);
816
+ return;
1317
817
  }
1318
- }
1319
- }
1320
- // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1321
- // sources edge targets halo rows), materialised by a single C-side
1322
- // scan. A binary-search membership probe replaces the three per-entry
1323
- // point queries that made this stage minutes long on a trained store.
1324
- const targets = this._dbEdgeOrHaloIds();
1325
- const isTarget = (id) => {
1326
- let lo = 0, hi = targets.length - 1;
1327
- while (lo <= hi) {
1328
- const mid = (lo + hi) >> 1;
1329
- const v = targets[mid];
1330
- if (v === id) {
1331
- return true;
818
+ this._pendingGist.set(id, normalize(copy(gist)));
819
+ // A node that ALREADY bridges experiences but was never indexed (its 1→2
820
+ // transition fired while its gist was evicted, or in a pre-transition
821
+ // store) is promoted on this re-encounter the recapture above is
822
+ // exactly what makes its gist available again. Byte leaves (negative
823
+ // ids) never have kid rows, so skip their parent probe.
824
+ if (id >= 0)
825
+ this.promoteBridge(id);
826
+ }
827
+ /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
828
+ * criterion), promote its gist into the content index NOW — the exact
829
+ * moment it becomes useful for multi-experience recall. The 1→2 parent
830
+ * transition fires on {@link _dbInsertKid} during mint, and nodes that
831
+ * were already bridges but missed indexing (gist evicted, pre-transition
832
+ * store) are recaptured in {@link captureIfUnindexed}.
833
+ *
834
+ * A no-op when the node is already indexed or its gist is evicted from
835
+ * the pending cache — a future re-encounter will retry. */
836
+ promoteBridge(id) {
837
+ // A LIMITed probe: this runs once per kid insert on the MINT hot path,
838
+ // and a shared child's full parent set grows with the corpus.
839
+ if (this._dbGetParentsFirst(id, 2).length < 2)
840
+ return;
841
+ this.indexGist(id, false);
842
+ }
843
+ // ── Core interning: dedup → near-dedup → mint ──────────────────────────
844
+ async intern(leaf, kids, gist) {
845
+ await this._ensureReady();
846
+ // 1. Exact dedup — equal content → one id, no vector work. Primary
847
+ // compression mechanism, intrinsic to the store.
848
+ //
849
+ // findLeaf/findBranch are the content-addressed lookups: in-memory cache
850
+ // first, then a durable probe that repopulates the cache on a hit.
851
+ // Using them here (rather than only the cache) makes dedup survive a cold
852
+ // cache — a resumed/checkpointed training run, or one whose dedup cache
853
+ // has evicted old keys, still recognises content already on disk and
854
+ // reuses its id instead of minting a duplicate.
855
+ const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
856
+ if (hit !== null) {
857
+ this.captureIfUnindexed(hit, gist);
858
+ return hit;
1332
859
  }
1333
- if (v < id) {
1334
- lo = mid + 1;
1335
- } else {
1336
- hi = mid - 1;
860
+ // 1b. Content-addressed lookup by leaf-id signature. When the same byte
861
+ // sequence was stored as a flat branch (via putBranch during deposit),
862
+ // a branch node spanning those bytes reuses that id even when its tree
863
+ // structure differs pure content addressing, same bytes → same node.
864
+ if (kids !== null && kids.length >= 2) {
865
+ const leafIds = this.flatLeafIds(kids);
866
+ if (leafIds !== null) {
867
+ const flatHit = this.findBranch(leafIds);
868
+ if (flatHit !== null) {
869
+ this.captureIfUnindexed(flatHit, gist);
870
+ return flatHit;
871
+ }
872
+ }
1337
873
  }
1338
- }
1339
- return false;
1340
- };
1341
- // Collect ids to remove: nodes that are structurally isolated (few
1342
- // parents), not edge-bearing, and not halo-bearing. These nodes are
1343
- // unique to one experience tree they bridge nothing and their index
1344
- // slots are wasted. Multi-parent nodes are the structural bridges
1345
- // that let a partial query climb from one experience to another.
1346
- const toRemove = [];
1347
- let scanned = 0;
1348
- let watermark = after;
1349
- for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1350
- // Yield a real event-loop turn on a fixed cadence: this scan runs over
1351
- // every unexamined indexed entry, and point probes alone pin the
1352
- // thread otherwise (the "frozen" symptom).
1353
- if (++scanned % 8192 === 0) {
1354
- await yieldToEventLoop();
1355
- }
1356
- watermark = internal;
1357
- if (isTarget(id)) {
1358
- continue;
1359
- }
1360
- if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1361
- continue;
1362
- }
1363
- toRemove.push(id);
1364
- }
1365
- // Delete in batches each batch is ONE vector-store transaction (one WAL
1366
- // commit), with an event-loop turn between batches.
1367
- const BATCH = 1000;
1368
- for (let i = 0; i < toRemove.length; i += BATCH) {
1369
- const batch = toRemove.slice(i, i + BATCH);
1370
- this._vecContentDeleteMany(batch);
1371
- // Purge the "already indexed" cache for removed ids: without this, a
1372
- // node that LATER becomes a resonance target again (gains an edge or
1373
- // halo) hits the stale cache entry in indexGist and is silently never
1374
- // re-indexed for the rest of the session.
1375
- for (const id of batch) {
1376
- this._indexedIds.delete(id);
1377
- }
1378
- await yieldToEventLoop();
1379
- }
1380
- // Persist the scan watermark so the next pass starts where this one
1381
- // ended (kept beside the criterion that produced it).
1382
- this._dbBeginTx();
1383
- this._dbSetMeta(
1384
- AbstractStore.COMPACT_WATERMARK_KEY,
1385
- minParents + ":" + watermark,
1386
- );
1387
- this._dbCommitTx();
1388
- if (toRemove.length > 0) {
1389
- // Compact to physically reclaim space from tombstones.
1390
- try {
1391
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1392
- this._vecContentCompact();
1393
- // Physical compaction may free the top internal id for reuse —
1394
- // drop the watermark so the next pass rescans from the start.
1395
- this._dbBeginTx();
1396
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1397
- this._dbCommitTx();
874
+ const cache = leaf !== null ? this._leafKey : this._branchKey;
875
+ const key = leaf !== null ? keyOf(leaf) : kids.join(",");
876
+ // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
877
+ // Leaves are single bytes: exact dedup already collapses every identical
878
+ // leaf, and near-merging distinct leaves only corrupts bytes for no real
879
+ // saving. Real near-dedup compression lives in subtree (branch) fusion.
880
+ //
881
+ // There is deliberately NO ANN probe of the FLUSHED index here. It used
882
+ // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
883
+ // ~every interior branch, since interiors are never dedup targets —
884
+ // making one ANN query per branch the dominant training cost (it dwarfed
885
+ // perception and the index write). And it was not merely expensive but
886
+ // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
887
+ // nearest "target" of a fresh branch, so the fold collapsed two
888
+ // byte-different subtrees onto one id and corrupted exact reconstruction
889
+ // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
890
+ // are caught two cheaper, exact ways instead: identical content by the
891
+ // exact-dedup hash-cons above, and a near-gist target still in the write
892
+ // buffer by the scan below.
893
+ if (leaf === null) {
894
+ // Near-dedup PREFILTER — the scale-aware identity bar
895
+ // ({@link identityBar}) for THIS branch's own length, not the fixed
896
+ // estimator floor: under the linear fold a long branch crosses
897
+ // 1 − 1/√D while whole windows differ, and every such crossing paid a
898
+ // full differsByOneWindow byte reconstruction. Same final semantics
899
+ // (the byte check below still decides identity); strictly fewer byte
900
+ // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
901
+ // memoized bottom-up by the interning order itself, O(kids)) is only
902
+ // computed once a nearest candidate actually exists, so the hot
903
+ // no-candidate mint pays nothing.
904
+ let best = -1;
905
+ let bestId = null;
906
+ if (this._nearDedupBuf.size > 0) {
907
+ const g = normalize(copy(gist));
908
+ // Candidates are the buffered DEDUP TARGETS only genuine whole
909
+ // experiences (edge/halo-bearing roots) not yet flushed.
910
+ for (const [id, vector] of this._nearDedupBuf) {
911
+ const s = dot(g, vector);
912
+ if (s > best) {
913
+ best = s;
914
+ bestId = id;
915
+ }
916
+ }
917
+ }
918
+ let blen = 0;
919
+ if (bestId !== null) {
920
+ for (const k of kids)
921
+ blen += this.contentLen(k);
922
+ }
923
+ if (bestId !== null &&
924
+ best >= identityBar(this.D, this._maxGroup, blen)) {
925
+ // Scale-aware acceptance. The cosine bar alone is scale-BLIND
926
+ // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
927
+ // a localized difference faster than linearly in form size, so for
928
+ // deep forms ANY fixed bar below 1 is crossed by exactly the one
929
+ // span that distinguishes two experiences. No inversion of the
930
+ // deficit is trustworthy, so the bytes themselves decide: the two
931
+ // forms must be identical except for ONE local span of at most W
932
+ // bytes — the river window, the perception's own resolution quantum.
933
+ const W = this._maxGroup;
934
+ if (this.differsByOneWindow(kids, bestId, W)) {
935
+ if (key.length <= DEDUP_KEY_MAX)
936
+ cache.set(key, bestId);
937
+ return bestId;
938
+ }
939
+ }
1398
940
  }
1399
- } catch (e) {
1400
- // Best-effort, but never SILENT: a persistently failing compaction
1401
- // lets tombstones accumulate and query cost grow with no signal.
1402
- this.compactFailures++;
1403
- if (this.compactFailures === 1) {
1404
- console.warn(
1405
- "sema: content-index compaction failed (will keep " +
1406
- "counting in store.compactFailures):",
1407
- e,
1408
- );
941
+ // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
942
+ // leaf) stores its BYTES in the leaf column with a zero-length kids blob
943
+ // as the marker the kid list is derived on read.
944
+ const id = this.mintId();
945
+ this._dbBeginTx();
946
+ const flat = kids ? flatKidsBytes(kids) : null;
947
+ const packed = kids && !flat ? packKids(kids) : null;
948
+ this._dbInsertNode(id, leaf ?? flat, packed ?? (flat ? _ZERO : null), hashOf(leaf ?? flat ?? packed));
949
+ // Reverse structural edge: each distinct child → this parent. Lets the graph
950
+ // be climbed upward in index time, with no scan of the kids blobs.
951
+ //
952
+ // Populated NATURALLY here and only here — one write per child, in the SAME
953
+ // mint that creates the node, inside the SAME deferred transaction as the
954
+ // node row, so node and kid rows are always durable together.
955
+ //
956
+ // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
957
+ // every branch, so its parent set is the corpus-sized hub the climb's
958
+ // saturation guard discards unread.
959
+ if (kids) {
960
+ // Kid rows change parent counts — a child whose parent set grows from
961
+ // one is no longer transparent, so every cached chain that hopped
962
+ // through it is stale. Which chains those are is unknowable without a
963
+ // reverse index, so the WHOLE cache drops (writes come in training
964
+ // bursts where the cache is cold anyway; queries rebuild it lazily).
965
+ this._chainMemo.clear();
966
+ for (const c of kids) {
967
+ if (c < 0 && c >= -256)
968
+ continue;
969
+ this._dbInsertKid(c, id);
970
+ // The 1→2 parent TRANSITION happens here and only here (hash-cons
971
+ // means an existing branch never re-inserts kid rows, so parent sets
972
+ // grow exclusively through fresh mints): the child just became a
973
+ // structural bridge between experiences — the exact set post-hoc
974
+ // compaction keeps — so its gist enters the reach index NOW.
975
+ this.promoteBridge(c);
976
+ }
977
+ }
978
+ if (key.length <= DEDUP_KEY_MAX)
979
+ cache.set(key, id);
980
+ if (leaf)
981
+ this._bytesCache.set(id, new Uint8Array(leaf));
982
+ else if (flat)
983
+ this._bytesCache.set(id, flat);
984
+ // Capture the gist; it is pushed into the content index lazily, the first
985
+ // time this node becomes a resonance target (link / pourHalo). A node that
986
+ // never does (a pure intermediate DAG node — ~99.5% of them) is never
987
+ // indexed: it costs one persistence row, no vector-index slot and no merge probe.
988
+ this._pendingGist.set(id, normalize(copy(gist)));
989
+ await this.maybeFlush();
990
+ return id;
991
+ }
992
+ /** Whether the byte content under `kids` and the byte content of `targetId`
993
+ * are identical except for ONE local span of at most `W` bytes on each side
994
+ * — the near dedup's byte-grain definition of a near-duplicate. A
995
+ * common-prefix / common-suffix trim: whatever remains after both trims is
996
+ * the single differing span (substitution, insertion or deletion), and both
997
+ * remainders must fit the budget. Scattered differences leave a wide
998
+ * middle and are rejected. */
999
+ differsByOneWindow(kids, targetId, W) {
1000
+ const a = concat(kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)));
1001
+ const b = this.bytesPrefix(targetId, a.length + W + 1);
1002
+ if (Math.abs(a.length - b.length) > W)
1003
+ return false;
1004
+ const n = Math.min(a.length, b.length);
1005
+ let i = 0;
1006
+ while (i < n && a[i] === b[i])
1007
+ i++;
1008
+ let j = 0;
1009
+ while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j])
1010
+ j++;
1011
+ return a.length - i - j <= W && b.length - i - j <= W;
1012
+ }
1013
+ async putLeaf(bytes, gist) {
1014
+ // Single bytes are implicit — no DB row and no eager index slot. The gist
1015
+ // is captured like any other node's and promoted into the content index
1016
+ // LAZILY, the first time the byte becomes a resonance target.
1017
+ if (bytes.length === 1) {
1018
+ const id = -(bytes[0] + 1);
1019
+ this.captureIfUnindexed(id, gist);
1020
+ return id;
1021
+ }
1022
+ return this.intern(new Uint8Array(bytes), null, gist);
1023
+ }
1024
+ async putBranch(kids, gist) {
1025
+ return this.intern(null, kids, gist);
1026
+ }
1027
+ // ── Lazy content indexing ──────────────────────────────────────────────
1028
+ /** Promote a node's captured gist into the content (resonance) index, once.
1029
+ * Called the first time a node becomes a target — i.e. from `link` (it bears
1030
+ * or receives a continuation edge) or `pourHalo` (it gains distributional
1031
+ * company). Idempotent: a node already indexed, or whose gist has been evicted
1032
+ * from the bounded pending map, is a no-op.
1033
+ *
1034
+ * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1035
+ * near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
1036
+ * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1037
+ indexGist(id, dedupTarget) {
1038
+ if (dedupTarget && this._bufferedIds.has(id)) {
1039
+ // Still buffered (indexed this batch, not yet flushed) — a live dedup
1040
+ // candidate, recorded in O(1) with no scan of the content buffer.
1041
+ const v = this._pendingGist.get(id);
1042
+ if (v !== undefined)
1043
+ this._nearDedupBuf.set(id, v);
1044
+ }
1045
+ if (this._indexedIds.has(id))
1046
+ return;
1047
+ const v = this._pendingGist.get(id);
1048
+ if (v === undefined)
1049
+ return;
1050
+ // Already durably indexed by a previous session? A node id names its
1051
+ // content, and the gist is a pure function of the content, so the stored
1052
+ // vector can only be identical — re-buffering it would spend an encode
1053
+ // and an upsert on a guaranteed no-op. One point query recognises this;
1054
+ // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1055
+ // This is what makes a RESUMED training run replay already-deposited
1056
+ // content at read speed instead of re-upserting the recent-id window.
1057
+ if (this._vecContentHas(id)) {
1058
+ this._indexedIds.set(id, true);
1059
+ return;
1409
1060
  }
1410
- }
1411
- // Index mutations (delete, compact) invalidate the ANN read cache.
1412
- this._resonateCache = null;
1413
- }
1414
- return toRemove.length;
1415
- }
1416
- /** {@link Store.repairContentIndex} */
1417
- async repairContentIndex(regenerateGist, minParents = 2) {
1418
- await this._ensureReady();
1419
- this.flush(); // commit any pending writes first
1420
- if (this._nextId === 0) {
1421
- return 0;
1422
- }
1423
- let added = 0;
1424
- // A repairable node MUST carry edges or a halo, so the candidate set IS
1425
- // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1426
- // thousands), not node-count-sized (tens of millions). The old walk
1427
- // visited EVERY branch id ever minted with a parent probe each; driving
1428
- // from the target set visits only real candidates, in the same ascending
1429
- // id order (the set is sorted), so the result and its ordering are
1430
- // identical. Byte leaves (negative ids) were never visited by the old
1431
- // walk and are skipped here too.
1432
- const targets = this._dbEdgeOrHaloIds();
1433
- let scanned = 0;
1434
- for (const id of targets) {
1435
- // Yield a real event-loop turn on a fixed cadence so the scan never
1436
- // pins the thread for its whole duration.
1437
- if (++scanned % 8192 === 0) {
1438
- await yieldToEventLoop();
1439
- }
1440
- if (id < 0) {
1441
- continue; // byte leaves: implicit, never repaired
1442
- }
1443
- // Already indexed in memory — skip.
1444
- if (this._indexedIds.has(id)) {
1445
- continue;
1446
- }
1447
- // Must be a structural bridge: ≥ minParents parents in the DAG.
1448
- // One LIMITed probe per candidate.
1449
- if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1450
- continue;
1451
- }
1452
- // Already durably indexed by a previous session — record and skip.
1453
- // Probed AFTER the structural filters: candidates are few, so this
1454
- // point query runs rarely instead of once per node in the store.
1455
- if (this._vecContentHas(id)) {
1456
1061
  this._indexedIds.set(id, true);
1457
- continue;
1458
- }
1459
- // Regenerate the gist from bytes. The callback is async (the Mind's
1460
- // perception is synchronous, but the interface allows a disk-backed
1461
- // regenerator that yields).
1462
- const gist = await regenerateGist(id);
1463
- if (!gist) {
1464
- continue;
1465
- }
1466
- // Index it — same code path as indexGist, but the vector is injected
1467
- // directly rather than read from the (empty) pending-gist cache.
1468
- this._indexedIds.set(id, true);
1469
- this._contentBuffer.push({
1470
- id,
1471
- vector: gist,
1472
- ef: this.efConstructionInterior,
1473
- });
1474
- this._bufferedIds.add(id);
1475
- // Repaired nodes are reach-indexed, never dedup targets: their gist is
1476
- // regenerated (may differ numerically from the original) and they are
1477
- // interiors, not deposit roots.
1478
- added++;
1479
- // Flush periodically to bound the write-transaction size and yield to
1480
- // the event loop, same cadence compactContentIndex uses.
1481
- if (added % 1000 === 0) {
1062
+ this._contentBuffer.push({ id, vector: v });
1063
+ this._bufferedIds.add(id);
1064
+ // A node indexed AS a dedup target enters the candidate set immediately.
1065
+ if (dedupTarget)
1066
+ this._nearDedupBuf.set(id, v);
1067
+ }
1068
+ /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1069
+ * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1070
+ * root is a genuine target (a whole experience), so it is a dedup target
1071
+ * too. */
1072
+ indexTarget(id) {
1073
+ this.indexGist(id, true);
1074
+ }
1075
+ /** Index a node and its interior forms as resonance targets. A node that
1076
+ * gains an edge is a learnt EXPERIENCE, and the consensus climb
1077
+ * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1078
+ * resonating its SUB-REGIONS — branch nodes within the experience — and
1079
+ * climbing their parents back to it.
1080
+ *
1081
+ * EVERY interior branch is indexed unconditionally, and this is
1082
+ * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1083
+ * the post-hoc compaction criterion) was tried and REJECTED by the test
1084
+ * suite partial recall of an experience's interior slices, multi-topic
1085
+ * attention, and counterfactual anchoring all resonate to SINGLE-parent
1086
+ * interiors (13 tests fail without them). Post-hoc structural compaction
1087
+ * ({@link compactContentIndex}) may still remove them, but that is a
1088
+ * storage/recall trade-off for archived stores, not a free optimisation.
1089
+ * The store's hash-cons bounds the index by the number of DISTINCT byte
1090
+ * patterns in the corpus — not by the number of deposits.
1091
+ *
1092
+ * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1093
+ * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1094
+ * let a partial query resonate and climb, but a fresh branch must never
1095
+ * merge onto an interior node of another experience.
1096
+ *
1097
+ * Iterative explicit-queue walk: the call stack never sees tree depth. */
1098
+ indexSubtree(root) {
1099
+ // The root is the whole experience — always index as a merge target.
1100
+ this.indexGist(root, true);
1101
+ const seen = new Set([root]);
1102
+ const stack = [root];
1103
+ while (stack.length > 0) {
1104
+ const id = stack.pop();
1105
+ const kids = this.get(id)?.kids;
1106
+ if (!kids)
1107
+ continue; // leaf — never a resonance anchor
1108
+ const isRoot = id === root;
1109
+ // Already-indexed or already-skipped nodes PRUNE the walk — their
1110
+ // subtrees were classified in a prior call.
1111
+ if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1112
+ continue;
1113
+ }
1114
+ for (const k of kids) {
1115
+ if (!seen.has(k)) {
1116
+ seen.add(k);
1117
+ stack.push(k);
1118
+ }
1119
+ }
1120
+ if (isRoot)
1121
+ continue;
1122
+ const g = this._pendingGist.get(id);
1123
+ if (g === undefined) {
1124
+ // Gist evicted from the bounded pending cache — the node can't be
1125
+ // indexed now. Mark as covered so subsequent visits prune here
1126
+ // instead of re-walking the subtree (the gist won't return).
1127
+ this._coveredIds.set(id, true);
1128
+ continue;
1129
+ }
1130
+ // Index unconditionally — every interior node is a valid resonance
1131
+ // anchor for partial-query recall. The _indexedIds cache and the
1132
+ // durable-index check in indexGist prevent duplicate indexing across
1133
+ // multiple encounters of the same shared subtree.
1134
+ this.indexGist(id, false);
1135
+ }
1136
+ }
1137
+ // ── Soft resonance ─────────────────────────────────────────────────────
1138
+ async resonate(v, k) {
1139
+ await this._ensureReady();
1140
+ // Synchronous flush of any buffered index writes: the FIRST resonance
1141
+ // after a large ingest pays that flush here, so it shows up in respond
1142
+ // latency, not ingest latency — correct behaviour, skewed attribution;
1143
+ // profile accordingly.
1144
+ this.flushContent();
1145
+ if (k <= 0)
1146
+ return [];
1147
+ // ANN read cache — content-addressed so a fresh Float32Array with the
1148
+ // same values still hits. Lazy-init: null after any index write; the
1149
+ // first miss after a flush recreates it. When voteRegions resonates
1150
+ // identical perceived sub-regions, only the first call descends the ANN.
1151
+ const rk = vecKey(v) + ":" + k;
1152
+ const cache = this._resonateCache;
1153
+ if (cache) {
1154
+ const hit = cache.get(rk);
1155
+ if (hit !== undefined)
1156
+ return hit;
1157
+ }
1158
+ const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecContentClusterCount()));
1159
+ const out = [];
1160
+ for (const r of results) {
1161
+ const id = r.id;
1162
+ out.push({ id, score: 1 - r.distance });
1163
+ if (out.length >= k)
1164
+ break;
1165
+ }
1166
+ const rc = this._resonateCache ??= new Map();
1167
+ if (rc.size >= RESONATE_CACHE_MAX)
1168
+ rc.clear();
1169
+ rc.set(rk, out);
1170
+ return out;
1171
+ }
1172
+ indexedVectorCount() {
1173
+ this.flushContent();
1174
+ return this._vecContentSize();
1175
+ }
1176
+ lastResonateReads() {
1177
+ return this._vecContentLastReads();
1178
+ }
1179
+ // ── Content index compaction ────────────────────────────────────────────
1180
+ /** How many physical compaction attempts have failed this session. Zero in
1181
+ * a healthy store; a growing count means tombstones are accumulating and
1182
+ * index query cost is drifting up (the first failure also warns once). */
1183
+ compactFailures = 0;
1184
+ /** Meta key holding the incremental scan watermark of
1185
+ * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1186
+ * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1187
+ * entry once kept can never become removable — and removed entries are
1188
+ * gone, so a pass only ever needs to examine entries indexed AFTER the
1189
+ * previous pass. Internal ids are monotone and survive the index's
1190
+ * splice compaction; the watermark is reset whenever a PHYSICAL index
1191
+ * compaction runs (id reuse after a dropped top row would otherwise hide
1192
+ * new entries behind it). */
1193
+ static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1194
+ /** {@link Store.compactContentIndex} */
1195
+ async compactContentIndex(minParents = 2) {
1196
+ await this._ensureReady();
1197
+ this.flush(); // commit any pending writes first
1198
+ // Incremental scan: resume from the last pass's watermark when its
1199
+ // minParents matches; a changed criterion forces a full rescan.
1200
+ let after = 0;
1201
+ {
1202
+ const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1203
+ if (raw !== null) {
1204
+ const sep = raw.indexOf(":");
1205
+ const mp = Number(raw.slice(0, sep));
1206
+ const wm = Number(raw.slice(sep + 1));
1207
+ if (mp === minParents && Number.isFinite(wm) && wm > 0)
1208
+ after = wm;
1209
+ }
1210
+ }
1211
+ // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1212
+ // sources ∪ edge targets ∪ halo rows), materialised by a single C-side
1213
+ // scan. A binary-search membership probe replaces the three per-entry
1214
+ // point queries that made this stage minutes long on a trained store.
1215
+ const targets = this._dbEdgeOrHaloIds();
1216
+ const isTarget = (id) => {
1217
+ let lo = 0, hi = targets.length - 1;
1218
+ while (lo <= hi) {
1219
+ const mid = (lo + hi) >> 1;
1220
+ const v = targets[mid];
1221
+ if (v === id)
1222
+ return true;
1223
+ if (v < id)
1224
+ lo = mid + 1;
1225
+ else
1226
+ hi = mid - 1;
1227
+ }
1228
+ return false;
1229
+ };
1230
+ // Collect ids to remove: nodes that are structurally isolated (few
1231
+ // parents), not edge-bearing, and not halo-bearing. These nodes are
1232
+ // unique to one experience tree — they bridge nothing and their index
1233
+ // slots are wasted. Multi-parent nodes are the structural bridges
1234
+ // that let a partial query climb from one experience to another.
1235
+ const toRemove = [];
1236
+ let scanned = 0;
1237
+ let watermark = after;
1238
+ for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1239
+ // Yield a real event-loop turn on a fixed cadence: this scan runs over
1240
+ // every unexamined indexed entry, and point probes alone pin the
1241
+ // thread otherwise (the "frozen" symptom).
1242
+ if (++scanned % 8192 === 0)
1243
+ await yieldToEventLoop();
1244
+ watermark = internal;
1245
+ if (isTarget(id))
1246
+ continue;
1247
+ if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1248
+ continue;
1249
+ }
1250
+ toRemove.push(id);
1251
+ }
1252
+ // Delete in batches — each batch is ONE vector-store transaction (one WAL
1253
+ // commit), with an event-loop turn between batches.
1254
+ const BATCH = 1000;
1255
+ for (let i = 0; i < toRemove.length; i += BATCH) {
1256
+ const batch = toRemove.slice(i, i + BATCH);
1257
+ this._vecContentDeleteMany(batch);
1258
+ // Purge the "already indexed" cache for removed ids: without this, a
1259
+ // node that LATER becomes a resonance target again (gains an edge or
1260
+ // halo) hits the stale cache entry in indexGist and is silently never
1261
+ // re-indexed for the rest of the session.
1262
+ for (const id of batch)
1263
+ this._indexedIds.delete(id);
1264
+ await yieldToEventLoop();
1265
+ }
1266
+ // Persist the scan watermark so the next pass starts where this one
1267
+ // ended (kept beside the criterion that produced it).
1268
+ this._dbBeginTx();
1269
+ this._dbSetMeta(AbstractStore.COMPACT_WATERMARK_KEY, minParents + ":" + watermark);
1270
+ this._dbCommitTx();
1271
+ if (toRemove.length > 0) {
1272
+ // Compact to physically reclaim space from tombstones.
1273
+ try {
1274
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1275
+ this._vecContentCompact();
1276
+ // Physical compaction may free the top internal id for reuse —
1277
+ // drop the watermark so the next pass rescans from the start.
1278
+ this._dbBeginTx();
1279
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1280
+ this._dbCommitTx();
1281
+ }
1282
+ }
1283
+ catch (e) {
1284
+ // Best-effort, but never SILENT: a persistently failing compaction
1285
+ // lets tombstones accumulate and query cost grow with no signal.
1286
+ this.compactFailures++;
1287
+ if (this.compactFailures === 1) {
1288
+ console.warn("sema: content-index compaction failed (will keep " +
1289
+ "counting in store.compactFailures):", e);
1290
+ }
1291
+ }
1292
+ // Index mutations (delete, compact) invalidate the ANN read cache.
1293
+ this._resonateCache = null;
1294
+ }
1295
+ return toRemove.length;
1296
+ }
1297
+ /** {@link Store.repairContentIndex} */
1298
+ async repairContentIndex(regenerateGist, minParents = 2) {
1299
+ await this._ensureReady();
1300
+ this.flush(); // commit any pending writes first
1301
+ if (this._nextId === 0)
1302
+ return 0;
1303
+ let added = 0;
1304
+ // A repairable node MUST carry edges or a halo, so the candidate set IS
1305
+ // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1306
+ // thousands), not node-count-sized (tens of millions). The old walk
1307
+ // visited EVERY branch id ever minted with a parent probe each; driving
1308
+ // from the target set visits only real candidates, in the same ascending
1309
+ // id order (the set is sorted), so the result and its ordering are
1310
+ // identical. Byte leaves (negative ids) were never visited by the old
1311
+ // walk and are skipped here too.
1312
+ const targets = this._dbEdgeOrHaloIds();
1313
+ let scanned = 0;
1314
+ for (const id of targets) {
1315
+ // Yield a real event-loop turn on a fixed cadence so the scan never
1316
+ // pins the thread for its whole duration.
1317
+ if (++scanned % 8192 === 0)
1318
+ await yieldToEventLoop();
1319
+ if (id < 0)
1320
+ continue; // byte leaves: implicit, never repaired
1321
+ // Already indexed in memory — skip.
1322
+ if (this._indexedIds.has(id))
1323
+ continue;
1324
+ // Must be a structural bridge: ≥ minParents parents in the DAG.
1325
+ // One LIMITed probe per candidate.
1326
+ if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1327
+ continue;
1328
+ }
1329
+ // Already durably indexed by a previous session — record and skip.
1330
+ // Probed AFTER the structural filters: candidates are few, so this
1331
+ // point query runs rarely instead of once per node in the store.
1332
+ if (this._vecContentHas(id)) {
1333
+ this._indexedIds.set(id, true);
1334
+ continue;
1335
+ }
1336
+ // Regenerate the gist from bytes. The callback is async (the Mind's
1337
+ // perception is synchronous, but the interface allows a disk-backed
1338
+ // regenerator that yields).
1339
+ const gist = await regenerateGist(id);
1340
+ if (!gist)
1341
+ continue;
1342
+ // Index it — same code path as indexGist, but the vector is injected
1343
+ // directly rather than read from the (empty) pending-gist cache.
1344
+ this._indexedIds.set(id, true);
1345
+ this._contentBuffer.push({ id, vector: gist });
1346
+ this._bufferedIds.add(id);
1347
+ // Repaired nodes are reach-indexed, never dedup targets: their gist is
1348
+ // regenerated (may differ numerically from the original) and they are
1349
+ // interiors, not deposit roots.
1350
+ added++;
1351
+ // Flush periodically to bound the write-transaction size and yield to
1352
+ // the event loop, same cadence compactContentIndex uses.
1353
+ if (added % 1000 === 0)
1354
+ await this.maybeFlush();
1355
+ }
1356
+ // Final flush for the last partial batch.
1357
+ if (added > 0)
1358
+ await this.maybeFlush();
1359
+ return added;
1360
+ }
1361
+ // ── Continuation edges ─────────────────────────────────────────────────
1362
+ async link(from, to) {
1363
+ await this._ensureReady();
1364
+ // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1365
+ // indexed, because the seat is symmetric — a query may name only a PORTION of
1366
+ // either side and must still resonate to its interior and climb to the whole.
1367
+ // Only each ROOT is a MERGE target; the interiors are reach-only.
1368
+ this.indexSubtree(from);
1369
+ this.indexSubtree(to);
1370
+ // Flush the vectors indexSubtree just added to the buffer.
1371
+ // This keeps the buffer bounded and yields to the event loop.
1482
1372
  await this.maybeFlush();
1483
- }
1484
- }
1485
- // Final flush for the last partial batch.
1486
- if (added > 0) {
1487
- await this.maybeFlush();
1488
- }
1489
- return added;
1490
- }
1491
- // ── Continuation edges ─────────────────────────────────────────────────
1492
- async link(from, to) {
1493
- await this._ensureReady();
1494
- // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1495
- // indexed, because the seat is symmetric — a query may name only a PORTION of
1496
- // either side and must still resonate to its interior and climb to the whole.
1497
- // Only each ROOT is a MERGE target; the interiors are reach-only.
1498
- this.indexSubtree(from);
1499
- this.indexSubtree(to);
1500
- // Flush the vectors indexSubtree just added to the buffer.
1501
- // This keeps the buffer bounded and yields to the event loop.
1502
- await this.maybeFlush();
1503
- this._dbBeginTx();
1504
- // Keep the document count exact as it grows: a source gaining its FIRST
1505
- // edge is one new learnt context. One indexed point probe, and only once
1506
- // the count has been materialised — before that, the lazy full count in
1507
- // {@link edgeSourceCount} will see this edge anyway.
1508
- if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1509
- this._edgeSrcCount++;
1510
- }
1511
- // An edge breaks the transparency of both endpoints — drop cached chains
1512
- // (same reasoning as the kid-row invalidation in put).
1513
- this._chainMemo.clear();
1514
- this._dbInsertEdge(from, to);
1515
- }
1516
- next(id) {
1517
- return this._dbGetNextEdges(id);
1518
- }
1519
- /** {@link Store.hasNext} one indexed point probe, never a range read. */
1520
- hasNext(id) {
1521
- return this._dbEdgeSrcExists(id);
1522
- }
1523
- prev(id) {
1524
- return this._dbGetPrevEdges(id);
1525
- }
1526
- nextFirst(id, limit) {
1527
- return this._dbGetNextEdgesFirst(id, limit);
1528
- }
1529
- prevFirst(id, limit) {
1530
- return this._dbGetPrevEdgesFirst(id, limit);
1531
- }
1532
- /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1533
- * should override; this default materialises (correct, not optimal). */
1534
- prevCount(id) {
1535
- return this._dbGetPrevEdges(id).length;
1536
- }
1537
- edgeSourceCount() {
1538
- // Materialised once per session (edges written before this moment are
1539
- // covered by the full count), then kept exact incrementally by link().
1540
- // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1541
- // detect staleness — O(edges) on the recall hot path, at every IDF read.
1542
- if (this._edgeSrcCount < 0) {
1543
- this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1544
- }
1545
- return this._edgeSrcCount;
1546
- }
1547
- // ── Halos ──────────────────────────────────────────────────────────────
1548
- haloMass(id) {
1549
- const r = this._dbGetHalo(id);
1550
- return r ? r.mass : 0;
1551
- }
1552
- halo(id) {
1553
- const cached = this._haloNorm.get(id);
1554
- if (cached !== undefined) {
1555
- return copy(cached);
1556
- }
1557
- const r = this._dbGetHalo(id);
1558
- if (!r || r.mass < this.minHaloMass) {
1559
- return null;
1560
- }
1561
- const exact = this._haloExact.get(id);
1562
- const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1563
- this._haloNorm.set(id, v);
1564
- return copy(v);
1565
- }
1566
- /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1567
- * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1568
- hasHalo(id) {
1569
- const r = this._dbGetHalo(id);
1570
- return r !== null && r.mass >= this.minHaloMass;
1571
- }
1572
- async pourHalo(id, add) {
1573
- await this._ensureReady();
1574
- // A node with a halo is a genuine resonance target — the consensus climb
1575
- // resonates a query region to it, and articulation reads its halo.
1576
- this.indexGist(id, true);
1577
- const r = this._dbGetHalo(id);
1578
- const acc = this._haloExact.get(id) ??
1579
- (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1580
- addInto(acc, add);
1581
- this._haloExact.set(id, acc);
1582
- this._haloNorm.delete(id); // the normalized read cache is now stale
1583
- const mass = (r?.mass ?? 0) + 1;
1584
- this._dbBeginTx();
1585
- this._dbUpsertHalo(id, haloEncode(acc), mass);
1586
- // Re-index on a geometric schedule only (the exact halo is persisted).
1587
- if (mass >= this.minHaloMass && geometricMass(mass)) {
1588
- this._haloBuffer.set(id, normalize(copy(acc)));
1589
- await this.maybeFlush();
1590
- }
1591
- }
1592
- async resonateHalo(v, k) {
1593
- await this._ensureReady();
1594
- this.flushHalos();
1595
- if (k <= 0) {
1596
- return [];
1597
- }
1598
- // ANN read cache — same scheme as resonate's, but for the halo index.
1599
- const rk = vecKey(v) + ":" + k;
1600
- const cache = this._resonateHaloCache;
1601
- if (cache) {
1602
- const hit = cache.get(rk);
1603
- if (hit !== undefined) {
1604
- return hit;
1605
- }
1606
- }
1607
- const results = this._vecHaloQuery(
1608
- normalize(copy(v)),
1609
- k * this.overfetch,
1610
- this.efSearch,
1611
- );
1612
- const out = [];
1613
- for (const r of results) {
1614
- const id = r.id;
1615
- out.push({ id, score: 1 - r.distance });
1616
- if (out.length >= k) {
1617
- break;
1618
- }
1619
- }
1620
- const rhc = this._resonateHaloCache ??= new Map();
1621
- if (rhc.size >= RESONATE_CACHE_MAX) {
1622
- rhc.clear();
1623
- }
1624
- rhc.set(rk, out);
1625
- return out;
1626
- }
1627
- // ── Buffering, flushing, compaction ────────────────────────────────────
1628
- pending() {
1629
- return this._contentBuffer.length + this._haloBuffer.size;
1630
- }
1631
- flushContent() {
1632
- if (this._contentBuffer.length === 0) {
1633
- return 0;
1634
- }
1635
- const batch = this._contentBuffer.splice(0);
1636
- // The merge scan only consults UNFLUSHED candidates, so clear them in
1637
- // lockstep with the content buffer they mirror.
1638
- this._nearDedupBuf.clear();
1639
- this._bufferedIds.clear();
1640
- this._vecContentUpsert(batch);
1641
- this._resonateCache = null;
1642
- return batch.length;
1643
- }
1644
- flushHalos() {
1645
- if (this._haloBuffer.size === 0) {
1646
- return 0;
1647
- }
1648
- const batch = [...this._haloBuffer.entries()];
1649
- this._haloBuffer.clear();
1650
- this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1651
- this._resonateHaloCache = null;
1652
- return batch.length;
1653
- }
1654
- /** Append the buffered containment pairs, inside the deferred transaction.
1655
- * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1656
- * flush never reads a child's stored list back — the old packed-blob
1657
- * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1658
- * over a long training run on a hot window. */
1659
- flushContain() {
1660
- if (this._containBuf.size === 0) {
1661
- return;
1662
- }
1663
- this._dbBeginTx();
1664
- for (const [child, set] of this._containBuf) {
1665
- this._dbAppendContain(child, [...set]);
1666
- }
1667
- this._containBuf.clear();
1668
- }
1669
- /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1670
- flush() {
1671
- this.flushContain();
1672
- const written = this.flushContent() + this.flushHalos();
1673
- // Commit the deferred write transaction on the same cadence as the vector
1674
- // buffers, so node rows / edges / halos become durable in coalesced batches.
1675
- this._dbCommitTx();
1676
- if (written === 0) {
1677
- return;
1678
- }
1679
- this._writtenSinceCompact += written;
1680
- if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1681
- this._writtenSinceCompact = 0;
1682
- try {
1683
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1684
- this._vecContentCompact();
1685
- // Physical compaction may free the top internal id for reuse —
1686
- // invalidate the incremental maintenance watermark.
1687
- this._dbBeginTx();
1688
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1689
- this._dbCommitTx();
1373
+ this._dbBeginTx();
1374
+ // Keep the document count exact as it grows: a source gaining its FIRST
1375
+ // edge is one new learnt context. One indexed point probe, and only once
1376
+ // the count has been materialised — before that, the lazy full count in
1377
+ // {@link edgeSourceCount} will see this edge anyway.
1378
+ if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1379
+ this._edgeSrcCount++;
1380
+ }
1381
+ // An edge breaks the transparency of both endpoints — drop cached chains
1382
+ // (same reasoning as the kid-row invalidation in put).
1383
+ this._chainMemo.clear();
1384
+ this._dbInsertEdge(from, to);
1385
+ }
1386
+ next(id) {
1387
+ return this._dbGetNextEdges(id);
1388
+ }
1389
+ /** {@link Store.hasNext} — one indexed point probe, never a range read. */
1390
+ hasNext(id) {
1391
+ return this._dbEdgeSrcExists(id);
1392
+ }
1393
+ prev(id) {
1394
+ return this._dbGetPrevEdges(id);
1395
+ }
1396
+ nextFirst(id, limit) {
1397
+ return this._dbGetNextEdgesFirst(id, limit);
1398
+ }
1399
+ prevFirst(id, limit) {
1400
+ return this._dbGetPrevEdgesFirst(id, limit);
1401
+ }
1402
+ /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1403
+ * should override; this default materialises (correct, not optimal). */
1404
+ prevCount(id) {
1405
+ return this._dbGetPrevEdges(id).length;
1406
+ }
1407
+ edgeSourceCount() {
1408
+ // Materialised once per session (edges written before this moment are
1409
+ // covered by the full count), then kept exact incrementally by link().
1410
+ // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1411
+ // detect staleness — O(edges) on the recall hot path, at every IDF read.
1412
+ if (this._edgeSrcCount < 0) {
1413
+ this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1414
+ }
1415
+ return this._edgeSrcCount;
1416
+ }
1417
+ // ── Halos ──────────────────────────────────────────────────────────────
1418
+ haloMass(id) {
1419
+ const r = this._dbGetHalo(id);
1420
+ return r ? r.mass : 0;
1421
+ }
1422
+ halo(id) {
1423
+ const cached = this._haloNorm.get(id);
1424
+ if (cached !== undefined)
1425
+ return copy(cached);
1426
+ const r = this._dbGetHalo(id);
1427
+ if (!r || r.mass < this.minHaloMass)
1428
+ return null;
1429
+ const exact = this._haloExact.get(id);
1430
+ const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1431
+ this._haloNorm.set(id, v);
1432
+ return copy(v);
1433
+ }
1434
+ /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1435
+ * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1436
+ hasHalo(id) {
1437
+ const r = this._dbGetHalo(id);
1438
+ return r !== null && r.mass >= this.minHaloMass;
1439
+ }
1440
+ async pourHalo(id, add) {
1441
+ await this._ensureReady();
1442
+ // A node with a halo is a genuine resonance target — the consensus climb
1443
+ // resonates a query region to it, and articulation reads its halo.
1444
+ this.indexGist(id, true);
1445
+ const r = this._dbGetHalo(id);
1446
+ const acc = this._haloExact.get(id) ??
1447
+ (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1448
+ addInto(acc, add);
1449
+ this._haloExact.set(id, acc);
1450
+ this._haloNorm.delete(id); // the normalized read cache is now stale
1451
+ const mass = (r?.mass ?? 0) + 1;
1452
+ this._dbBeginTx();
1453
+ this._dbUpsertHalo(id, haloEncode(acc), mass);
1454
+ // Re-index on a geometric schedule only (the exact halo is persisted).
1455
+ if (mass >= this.minHaloMass && geometricMass(mass)) {
1456
+ this._haloBuffer.set(id, normalize(copy(acc)));
1457
+ await this.maybeFlush();
1458
+ }
1459
+ }
1460
+ async resonateHalo(v, k) {
1461
+ await this._ensureReady();
1462
+ this.flushHalos();
1463
+ if (k <= 0)
1464
+ return [];
1465
+ // ANN read cache same scheme as resonate's, but for the halo index.
1466
+ const rk = vecKey(v) + ":" + k;
1467
+ const cache = this._resonateHaloCache;
1468
+ if (cache) {
1469
+ const hit = cache.get(rk);
1470
+ if (hit !== undefined)
1471
+ return hit;
1690
1472
  }
1691
- if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1692
- this._vecHaloCompact();
1473
+ const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecHaloClusterCount()));
1474
+ const out = [];
1475
+ for (const r of results) {
1476
+ const id = r.id;
1477
+ out.push({ id, score: 1 - r.distance });
1478
+ if (out.length >= k)
1479
+ break;
1693
1480
  }
1694
- } catch (e) {
1695
- // Best-effort, but never SILENT (same contract as the prune-path
1696
- // compaction above): a persistently failing compaction lets
1697
- // tombstones accumulate and query cost grow with no signal.
1698
- this.compactFailures++;
1699
- if (this.compactFailures === 1) {
1700
- console.warn(
1701
- "sema: vector-index compaction failed (will keep " +
1702
- "counting in store.compactFailures):",
1703
- e,
1704
- );
1481
+ const rhc = this._resonateHaloCache ??= new Map();
1482
+ if (rhc.size >= RESONATE_CACHE_MAX)
1483
+ rhc.clear();
1484
+ rhc.set(rk, out);
1485
+ return out;
1486
+ }
1487
+ // ── Buffering, flushing, compaction ────────────────────────────────────
1488
+ pending() {
1489
+ return this._contentBuffer.length + this._haloBuffer.size;
1490
+ }
1491
+ flushContent() {
1492
+ if (this._contentBuffer.length === 0)
1493
+ return 0;
1494
+ const batch = this._contentBuffer.splice(0);
1495
+ // The merge scan only consults UNFLUSHED candidates, so clear them in
1496
+ // lockstep with the content buffer they mirror.
1497
+ this._nearDedupBuf.clear();
1498
+ this._bufferedIds.clear();
1499
+ this._vecContentUpsert(batch);
1500
+ this._resonateCache = null;
1501
+ return batch.length;
1502
+ }
1503
+ flushHalos() {
1504
+ if (this._haloBuffer.size === 0)
1505
+ return 0;
1506
+ const batch = [...this._haloBuffer.entries()];
1507
+ this._haloBuffer.clear();
1508
+ this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1509
+ this._resonateHaloCache = null;
1510
+ return batch.length;
1511
+ }
1512
+ /** Append the buffered containment pairs, inside the deferred transaction.
1513
+ * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1514
+ * flush never reads a child's stored list back — the old packed-blob
1515
+ * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1516
+ * over a long training run on a hot window. */
1517
+ flushContain() {
1518
+ if (this._containBuf.size === 0)
1519
+ return;
1520
+ this._dbBeginTx();
1521
+ for (const [child, set] of this._containBuf) {
1522
+ this._dbAppendContain(child, [...set]);
1705
1523
  }
1706
- }
1707
- }
1708
- }
1709
- async maybeFlush() {
1710
- if (this.pending() >= this.batchSize) {
1711
- this.flush();
1712
- // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1713
- // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1714
- // engine drains to empty before it ever services a MACROTASK — so a deposit
1715
- // burst that only micro-awaits pins the single thread for its whole
1716
- // duration, starving timers, pending I/O and any overlapped work. Here —
1717
- // buffers empty, transaction committed, nothing mid-write — is the one
1718
- // safe point to hand the event loop a real turn.
1719
- await yieldToEventLoop();
1720
- }
1721
- }
1722
- // ── Meta ───────────────────────────────────────────────────────────────
1723
- async setMeta(key, val) {
1724
- await this._ensureReady();
1725
- this._dbBeginTx();
1726
- this._dbSetMeta(key, val);
1727
- }
1728
- async getMeta(key) {
1729
- await this._ensureReady();
1730
- return this._dbGetMeta(key);
1731
- }
1732
- async deleteMeta(key) {
1733
- await this._ensureReady();
1734
- this._dbBeginTx();
1735
- this._dbDeleteMeta(key);
1736
- }
1737
- // ── Snapshot ───────────────────────────────────────────────────────────
1738
- async saveSnapshot(bytes) {
1739
- await this._ensureReady();
1740
- this.flush(); // commits any open write transaction + vector buffers
1741
- this._dbSaveSnapshot(bytes);
1742
- }
1743
- async loadSnapshot() {
1744
- await this._ensureReady();
1745
- return this._dbLoadSnapshot();
1746
- }
1747
- // ── Lifecycle ──────────────────────────────────────────────────────────
1748
- commit() {
1749
- this.flush();
1750
- }
1751
- async close() {
1752
- if (this.closed) {
1753
- return;
1754
- }
1755
- this.closed = true;
1756
- this.flush(); // commits any open write transaction + vector buffers
1757
- this._dbClose();
1758
- }
1524
+ this._containBuf.clear();
1525
+ }
1526
+ /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1527
+ flush() {
1528
+ this.flushContain();
1529
+ const written = this.flushContent() + this.flushHalos();
1530
+ // Commit the deferred write transaction on the same cadence as the vector
1531
+ // buffers, so node rows / edges / halos become durable in coalesced batches.
1532
+ this._dbCommitTx();
1533
+ if (written === 0)
1534
+ return;
1535
+ this._writtenSinceCompact += written;
1536
+ if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1537
+ this._writtenSinceCompact = 0;
1538
+ try {
1539
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1540
+ this._vecContentCompact();
1541
+ // Physical compaction may free the top internal id for reuse —
1542
+ // invalidate the incremental maintenance watermark.
1543
+ this._dbBeginTx();
1544
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1545
+ this._dbCommitTx();
1546
+ }
1547
+ if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1548
+ this._vecHaloCompact();
1549
+ }
1550
+ }
1551
+ catch (e) {
1552
+ // Best-effort, but never SILENT (same contract as the prune-path
1553
+ // compaction above): a persistently failing compaction lets
1554
+ // tombstones accumulate and query cost grow with no signal.
1555
+ this.compactFailures++;
1556
+ if (this.compactFailures === 1) {
1557
+ console.warn("sema: vector-index compaction failed (will keep " +
1558
+ "counting in store.compactFailures):", e);
1559
+ }
1560
+ }
1561
+ }
1562
+ }
1563
+ async maybeFlush() {
1564
+ if (this.pending() >= this.batchSize) {
1565
+ this.flush();
1566
+ // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1567
+ // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1568
+ // engine drains to empty before it ever services a MACROTASK — so a deposit
1569
+ // burst that only micro-awaits pins the single thread for its whole
1570
+ // duration, starving timers, pending I/O and any overlapped work. Here —
1571
+ // buffers empty, transaction committed, nothing mid-write — is the one
1572
+ // safe point to hand the event loop a real turn.
1573
+ await yieldToEventLoop();
1574
+ }
1575
+ }
1576
+ // ── Meta ───────────────────────────────────────────────────────────────
1577
+ async setMeta(key, val) {
1578
+ await this._ensureReady();
1579
+ this._dbBeginTx();
1580
+ this._dbSetMeta(key, val);
1581
+ }
1582
+ async getMeta(key) {
1583
+ await this._ensureReady();
1584
+ return this._dbGetMeta(key);
1585
+ }
1586
+ async deleteMeta(key) {
1587
+ await this._ensureReady();
1588
+ this._dbBeginTx();
1589
+ this._dbDeleteMeta(key);
1590
+ }
1591
+ // ── Snapshot ───────────────────────────────────────────────────────────
1592
+ async saveSnapshot(bytes) {
1593
+ await this._ensureReady();
1594
+ this.flush(); // commits any open write transaction + vector buffers
1595
+ this._dbSaveSnapshot(bytes);
1596
+ }
1597
+ async loadSnapshot() {
1598
+ await this._ensureReady();
1599
+ return this._dbLoadSnapshot();
1600
+ }
1601
+ // ── Lifecycle ──────────────────────────────────────────────────────────
1602
+ commit() {
1603
+ this.flush();
1604
+ }
1605
+ async close() {
1606
+ if (this.closed)
1607
+ return;
1608
+ this.closed = true;
1609
+ this.flush(); // commits any open write transaction + vector buffers
1610
+ this._dbClose();
1611
+ }
1759
1612
  }