@hviana/sema 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1687 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +17 -42
  51. package/dist/src/geometry.js +235 -266
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +749 -871
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +126 -185
  82. package/dist/src/mind/mind.d.ts +129 -154
  83. package/dist/src/mind/mind.js +258 -288
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +8 -32
  89. package/dist/src/mind/primitives.js +99 -117
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +183 -208
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +16 -58
  101. package/dist/src/mind/traverse.js +304 -357
  102. package/dist/src/mind/types.d.ts +120 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +169 -166
  117. package/dist/src/store-sqlite.js +762 -662
  118. package/dist/src/store.d.ts +586 -630
  119. package/dist/src/store.js +1423 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +13 -10
  123. package/package.json +1 -1
  124. package/src/alu/README.md +1 -1
  125. package/src/alu/src/index.ts +1 -1
  126. package/src/config.ts +19 -27
  127. package/src/index.ts +6 -11
  128. package/src/rabitq-ivf/README.md +56 -0
  129. package/src/rabitq-ivf/src/database.ts +276 -0
  130. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  131. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  132. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  133. package/src/store-sqlite.ts +196 -9
  134. package/src/store.ts +8 -32
  135. package/test/08-storage.test.mjs +3 -3
  136. package/test/14-scaling.test.mjs +2 -2
  137. package/test/35-ivf.test.mjs +263 -0
  138. package/test/36-bloom.test.mjs +123 -0
  139. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  140. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  141. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  142. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  143. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  144. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  145. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  146. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  147. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  148. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  149. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  150. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  151. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  152. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  153. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  154. package/src/rabitq-hnsw/README.md +0 -303
  155. package/src/rabitq-hnsw/src/database.ts +0 -492
  156. package/src/rabitq-hnsw/src/heap.ts +0 -90
  157. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  158. package/src/rabitq-hnsw/src/store.ts +0 -994
  159. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  160. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
package/dist/src/store.js CHANGED
@@ -55,14 +55,14 @@ const CHAIN_DEPTH_CAP = 65_536;
55
55
  * D·4 bytes (max ~4096), well under the argument limit; the key is far
56
56
  * cheaper than the ANN query it deduplicates. */
57
57
  function vecKey(v) {
58
- const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
- // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
- // engine argument limits and is slower than a few bounded calls.
61
- let s = "";
62
- for (let i = 0; i < bytes.length; i += 8192) {
63
- s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
- }
65
- return s;
58
+ const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
59
+ // Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
60
+ // engine argument limits and is slower than a few bounded calls.
61
+ let s = "";
62
+ for (let i = 0; i < bytes.length; i += 8192) {
63
+ s += String.fromCharCode(...bytes.subarray(i, i + 8192));
64
+ }
65
+ return s;
66
66
  }
67
67
  /** Bounded map with LRU eviction and byte accounting. On get the entry
68
68
  * moves to the most-recent end; on set the least-recently-used entries are
@@ -71,138 +71,129 @@ function vecKey(v) {
71
71
  * for uniform caches). A miss only costs a little extra work later, never
72
72
  * correctness. */
73
73
  export class BoundedMap {
74
- maxBytes;
75
- sizeOf;
76
- evict;
77
- m = new Map();
78
- _bytes = 0;
79
- // Persistent eviction cursor over the Map's insertion order. A fresh
80
- // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
- // deletions leave in V8's ordered backing store (compacted only on rehash),
82
- // making each eviction O(size) once the map first fills — the training-rate
83
- // cliff at scale. A persistent iterator passes each hole exactly once
84
- // (V8 map iterators stay valid under mutation): amortized O(1).
85
- _cursor = null;
86
- // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
- // from the cursor, so the LRU window never rescans from the front.
88
- _candidates = [];
89
- constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
- this.maxBytes = maxBytes;
91
- this.sizeOf = sizeOf;
92
- this.evict = evict;
93
- }
94
- /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
- * off; wraps to the front when exhausted. Undefined only when empty. */
96
- nextOldest() {
97
- for (let wrapped = false;;) {
98
- if (this._cursor === null) {
99
- this._cursor = this.m.keys();
100
- }
101
- const n = this._cursor.next();
102
- if (!n.done) {
103
- return n.value;
104
- }
105
- this._cursor = null;
106
- if (this.m.size === 0 || wrapped) {
107
- return undefined;
108
- }
109
- wrapped = true;
110
- }
111
- }
112
- get(k) {
113
- const v = this.m.get(k);
114
- if (v !== undefined) {
115
- this.m.delete(k);
116
- this.m.set(k, v);
117
- }
118
- return v;
119
- }
120
- /** Membership without touching LRU order — a pure peek, for callers that only
121
- * need "is this key present?" and must not promote it to most-recent. */
122
- has(k) {
123
- return this.m.has(k);
124
- }
125
- set(k, v) {
126
- const old = this.m.get(k);
127
- if (old !== undefined) {
128
- this._bytes -= this.sizeOf(old);
129
- this.m.delete(k);
130
- }
131
- this.m.set(k, v);
132
- this._bytes += this.sizeOf(v);
133
- while (this._bytes > this.maxBytes && this.m.size > 0) {
134
- if (this.evict === "smallest") {
135
- // Among the oldest LRU candidates, evict the cheapest to rebuild.
136
- // Window grows logarithmically with cache size — wide enough to
137
- // find a cheap victim without scanning the whole map. Candidates
138
- // are pulled from the persistent cursor and carried between
139
- // evictions, so the window never rescans the map from the front.
140
- const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
141
- this._candidates = this._candidates.filter((k) => this.m.has(k));
142
- while (this._candidates.length < WINDOW) {
143
- const k = this.nextOldest();
144
- if (k === undefined) {
145
- break;
146
- }
147
- if (!this._candidates.includes(k)) {
148
- this._candidates.push(k);
149
- }
74
+ maxBytes;
75
+ sizeOf;
76
+ evict;
77
+ m = new Map();
78
+ _bytes = 0;
79
+ // Persistent eviction cursor over the Map's insertion order. A fresh
80
+ // `keys()` iterator per eviction re-skips the growing prefix of holes that
81
+ // deletions leave in V8's ordered backing store (compacted only on rehash),
82
+ // making each eviction O(size) once the map first fills — the training-rate
83
+ // cliff at scale. A persistent iterator passes each hole exactly once
84
+ // (V8 map iterators stay valid under mutation): amortized O(1).
85
+ _cursor = null;
86
+ // "smallest" mode: oldest-entry candidates carried between evictions, fed
87
+ // from the cursor, so the LRU window never rescans from the front.
88
+ _candidates = [];
89
+ constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
90
+ this.maxBytes = maxBytes;
91
+ this.sizeOf = sizeOf;
92
+ this.evict = evict;
93
+ }
94
+ /** Next key in insertion (≈ LRU) order, resuming where the last call left
95
+ * off; wraps to the front when exhausted. Undefined only when empty. */
96
+ nextOldest() {
97
+ for (let wrapped = false;;) {
98
+ if (this._cursor === null)
99
+ this._cursor = this.m.keys();
100
+ const n = this._cursor.next();
101
+ if (!n.done)
102
+ return n.value;
103
+ this._cursor = null;
104
+ if (this.m.size === 0 || wrapped)
105
+ return undefined;
106
+ wrapped = true;
150
107
  }
151
- let bestI = -1;
152
- let bestSz = Infinity;
153
- for (let i = 0; i < this._candidates.length; i++) {
154
- const sz = this.sizeOf(this.m.get(this._candidates[i]));
155
- if (sz < bestSz) {
156
- bestSz = sz;
157
- bestI = i;
158
- }
159
- }
160
- if (bestI < 0) {
161
- break;
108
+ }
109
+ get(k) {
110
+ const v = this.m.get(k);
111
+ if (v !== undefined) {
112
+ this.m.delete(k);
113
+ this.m.set(k, v);
162
114
  }
163
- const bestK = this._candidates[bestI];
164
- this._candidates.splice(bestI, 1);
165
- this._bytes -= bestSz;
166
- this.m.delete(bestK);
167
- } else {
168
- const lru = this.nextOldest();
169
- if (lru === undefined) {
170
- break;
115
+ return v;
116
+ }
117
+ /** Membership without touching LRU order — a pure peek, for callers that only
118
+ * need "is this key present?" and must not promote it to most-recent. */
119
+ has(k) {
120
+ return this.m.has(k);
121
+ }
122
+ set(k, v) {
123
+ const old = this.m.get(k);
124
+ if (old !== undefined) {
125
+ this._bytes -= this.sizeOf(old);
126
+ this.m.delete(k);
171
127
  }
172
- const lruv = this.m.get(lru);
173
- if (lruv === undefined) {
174
- continue;
128
+ this.m.set(k, v);
129
+ this._bytes += this.sizeOf(v);
130
+ while (this._bytes > this.maxBytes && this.m.size > 0) {
131
+ if (this.evict === "smallest") {
132
+ // Among the oldest LRU candidates, evict the cheapest to rebuild.
133
+ // Window grows logarithmically with cache size — wide enough to
134
+ // find a cheap victim without scanning the whole map. Candidates
135
+ // are pulled from the persistent cursor and carried between
136
+ // evictions, so the window never rescans the map from the front.
137
+ const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
138
+ this._candidates = this._candidates.filter((k) => this.m.has(k));
139
+ while (this._candidates.length < WINDOW) {
140
+ const k = this.nextOldest();
141
+ if (k === undefined)
142
+ break;
143
+ if (!this._candidates.includes(k))
144
+ this._candidates.push(k);
145
+ }
146
+ let bestI = -1;
147
+ let bestSz = Infinity;
148
+ for (let i = 0; i < this._candidates.length; i++) {
149
+ const sz = this.sizeOf(this.m.get(this._candidates[i]));
150
+ if (sz < bestSz) {
151
+ bestSz = sz;
152
+ bestI = i;
153
+ }
154
+ }
155
+ if (bestI < 0)
156
+ break;
157
+ const bestK = this._candidates[bestI];
158
+ this._candidates.splice(bestI, 1);
159
+ this._bytes -= bestSz;
160
+ this.m.delete(bestK);
161
+ }
162
+ else {
163
+ const lru = this.nextOldest();
164
+ if (lru === undefined)
165
+ break;
166
+ const lruv = this.m.get(lru);
167
+ if (lruv === undefined)
168
+ continue;
169
+ this._bytes -= this.sizeOf(lruv);
170
+ this.m.delete(lru);
171
+ }
175
172
  }
176
- this._bytes -= this.sizeOf(lruv);
177
- this.m.delete(lru);
178
- }
179
- }
180
- }
181
- get size() {
182
- return this.m.size;
183
- }
184
- get bytes() {
185
- return this._bytes;
186
- }
187
- /** Remove one entry (point invalidation), with byte accounting. */
188
- delete(k) {
189
- const v = this.m.get(k);
190
- if (v === undefined) {
191
- return;
192
- }
193
- this._bytes -= this.sizeOf(v);
194
- this.m.delete(k);
195
- }
196
- /** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
197
- clear() {
198
- if (this.m.size === 0) {
199
- return;
200
- }
201
- this.m = new Map();
202
- this._bytes = 0;
203
- this._cursor = null;
204
- this._candidates = [];
205
- }
173
+ }
174
+ get size() {
175
+ return this.m.size;
176
+ }
177
+ get bytes() {
178
+ return this._bytes;
179
+ }
180
+ /** Remove one entry (point invalidation), with byte accounting. */
181
+ delete(k) {
182
+ const v = this.m.get(k);
183
+ if (v === undefined)
184
+ return;
185
+ this._bytes -= this.sizeOf(v);
186
+ this.m.delete(k);
187
+ }
188
+ /** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
189
+ clear() {
190
+ if (this.m.size === 0)
191
+ return;
192
+ this.m = new Map();
193
+ this._bytes = 0;
194
+ this._cursor = null;
195
+ this._candidates = [];
196
+ }
206
197
  }
207
198
  // ── Serialisation utilities (pure functions, no DB dependency) ───────────
208
199
  const _ZERO = new Uint8Array(0);
@@ -219,28 +210,26 @@ const _ZERO = new Uint8Array(0);
219
210
  * exit mid-`ingest` whenever the event loop had nothing else alive (a bare
220
211
  * top-level deposit loop) — the await simply never resolved. */
221
212
  function yieldToEventLoop() {
222
- return new Promise((resolve) => {
223
- const g = globalThis;
224
- if (g.setImmediate) {
225
- g.setImmediate(resolve);
226
- } else {
227
- g.setTimeout(resolve, 0);
228
- }
229
- });
213
+ return new Promise((resolve) => {
214
+ const g = globalThis;
215
+ if (g.setImmediate)
216
+ g.setImmediate(resolve);
217
+ else
218
+ g.setTimeout(resolve, 0);
219
+ });
230
220
  }
231
221
  /** Concatenate an array of byte arrays. */
232
222
  function concat(parts) {
233
- let n = 0;
234
- for (const p of parts) {
235
- n += p.length;
236
- }
237
- const out = new Uint8Array(n);
238
- let o = 0;
239
- for (const p of parts) {
240
- out.set(p, o);
241
- o += p.length;
242
- }
243
- return out;
223
+ let n = 0;
224
+ for (const p of parts)
225
+ n += p.length;
226
+ const out = new Uint8Array(n);
227
+ let o = 0;
228
+ for (const p of parts) {
229
+ out.set(p, o);
230
+ o += p.length;
231
+ }
232
+ return out;
244
233
  }
245
234
  /** The byte string a FLAT branch spans — a branch whose kids are ALL implicit
246
235
  * single-byte leaves (ids −256…−1) is fully determined by its bytes, one per
@@ -252,58 +241,55 @@ function concat(parts) {
252
241
  * and content-addressed through the same leaf index instead of a second
253
242
  * blob-duplicating kids index. */
254
243
  function flatKidsBytes(kids) {
255
- const out = new Uint8Array(kids.length);
256
- for (let i = 0; i < kids.length; i++) {
257
- const k = kids[i];
258
- if (k < -256 || k >= 0) {
259
- return null;
260
- }
261
- out[i] = -(k + 1);
262
- }
263
- return out;
244
+ const out = new Uint8Array(kids.length);
245
+ for (let i = 0; i < kids.length; i++) {
246
+ const k = kids[i];
247
+ if (k < -256 || k >= 0)
248
+ return null;
249
+ out[i] = -(k + 1);
250
+ }
251
+ return out;
264
252
  }
265
253
  /** The implicit kid list of a flat branch — the inverse of
266
254
  * {@link flatKidsBytes}. */
267
255
  export function flatBytesKids(bytes) {
268
- const out = new Array(bytes.length);
269
- for (let i = 0; i < bytes.length; i++) {
270
- out[i] = -(bytes[i] + 1);
271
- }
272
- return out;
256
+ const out = new Array(bytes.length);
257
+ for (let i = 0; i < bytes.length; i++)
258
+ out[i] = -(bytes[i] + 1);
259
+ return out;
273
260
  }
274
261
  /** Pack a child-id list as little-endian int32s — 4 bytes per child, far more
275
262
  * compact than a space-joined decimal string and trivial to read back. */
276
263
  export function packKids(kids) {
277
- const out = new Int32Array(kids.length);
278
- for (let i = 0; i < kids.length; i++) {
279
- out[i] = kids[i];
280
- }
281
- return new Uint8Array(out.buffer);
264
+ const out = new Int32Array(kids.length);
265
+ for (let i = 0; i < kids.length; i++)
266
+ out[i] = kids[i];
267
+ return new Uint8Array(out.buffer);
282
268
  }
283
269
  export function unpackKids(blob) {
284
- // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
285
- // reinterpreting as int32.
286
- const i32 = new Int32Array(blob.slice().buffer);
287
- return Array.from(i32);
270
+ // The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
271
+ // reinterpreting as int32.
272
+ const i32 = new Int32Array(blob.slice().buffer);
273
+ return Array.from(i32);
288
274
  }
289
275
  /** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
290
276
  * on. Collisions are resolved by verifying the stored blob, never trusted. */
291
277
  function hashOf(bytes) {
292
- let h = 0x811c9dc5 >>> 0;
293
- for (let i = 0; i < bytes.length; i++) {
294
- h ^= bytes[i];
295
- h = Math.imul(h, 0x01000193) >>> 0;
296
- }
297
- return h >>> 0;
278
+ let h = 0x811c9dc5 >>> 0;
279
+ for (let i = 0; i < bytes.length; i++) {
280
+ h ^= bytes[i];
281
+ h = Math.imul(h, 0x01000193) >>> 0;
282
+ }
283
+ return h >>> 0;
298
284
  }
299
285
  /** Fast content key (FNV-1a + length). No array spread onto the call stack. */
300
286
  function keyOf(bytes) {
301
- return hashOf(bytes).toString(16) + ":" + bytes.length;
287
+ return hashOf(bytes).toString(16) + ":" + bytes.length;
302
288
  }
303
289
  /** Re-index a halo only when its mass is small or a power of two — O(log mass)
304
290
  * writes per node instead of O(mass). */
305
291
  function geometricMass(mass) {
306
- return mass <= 4 || (mass & (mass - 1)) === 0;
292
+ return mass <= 4 || (mass & (mass - 1)) === 0;
307
293
  }
308
294
  // Halo accumulators are stored 2-bit quantized: the exact float32 L2 norm
309
295
  // followed by D two-bit codes (sign + magnitude class) — 4 + D/4 bytes, 16×
@@ -328,47 +314,41 @@ const HALO_Q_THRESHOLD = 0.9816; // Lloyd-Max decision point, unit Gaussian
328
314
  const HALO_Q_LO = 0.4528; // reconstruction level, inner cell
329
315
  const HALO_Q_HI = 1.5104; // reconstruction level, outer cell
330
316
  function haloEncode(v) {
331
- const D = v.length;
332
- const out = new Uint8Array(4 + ((D + 3) >> 2));
333
- let n2 = 0;
334
- for (let i = 0; i < D; i++) {
335
- n2 += v[i] * v[i];
336
- }
337
- const norm = Math.sqrt(n2);
338
- new DataView(out.buffer).setFloat32(0, norm, true);
339
- const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
340
- for (let i = 0; i < D; i++) {
341
- const x = v[i];
342
- // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
343
- const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
344
- out[4 + (i >> 2)] |= code << ((i & 3) << 1);
345
- }
346
- return out;
317
+ const D = v.length;
318
+ const out = new Uint8Array(4 + ((D + 3) >> 2));
319
+ let n2 = 0;
320
+ for (let i = 0; i < D; i++)
321
+ n2 += v[i] * v[i];
322
+ const norm = Math.sqrt(n2);
323
+ new DataView(out.buffer).setFloat32(0, norm, true);
324
+ const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
325
+ for (let i = 0; i < D; i++) {
326
+ const x = v[i];
327
+ // bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
328
+ const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
329
+ out[4 + (i >> 2)] |= code << ((i & 3) << 1);
330
+ }
331
+ return out;
347
332
  }
348
333
  function haloDecode(blob, D) {
349
- const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(
350
- 0,
351
- true,
352
- );
353
- const out = new Float32Array(D);
354
- if (norm === 0) {
334
+ const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(0, true);
335
+ const out = new Float32Array(D);
336
+ if (norm === 0)
337
+ return out;
338
+ const sigma = norm / Math.sqrt(D);
339
+ let n2 = 0;
340
+ for (let i = 0; i < D; i++) {
341
+ const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
342
+ const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
343
+ const x = code & 1 ? mag : -mag;
344
+ out[i] = x;
345
+ n2 += x * x;
346
+ }
347
+ // Rescale to the exact stored norm so accumulation stays magnitude-true.
348
+ const s = norm / Math.sqrt(n2);
349
+ for (let i = 0; i < D; i++)
350
+ out[i] *= s;
355
351
  return out;
356
- }
357
- const sigma = norm / Math.sqrt(D);
358
- let n2 = 0;
359
- for (let i = 0; i < D; i++) {
360
- const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
361
- const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
362
- const x = code & 1 ? mag : -mag;
363
- out[i] = x;
364
- n2 += x * x;
365
- }
366
- // Rescale to the exact stored norm so accumulation stays magnitude-true.
367
- const s = norm / Math.sqrt(n2);
368
- for (let i = 0; i < D; i++) {
369
- out[i] *= s;
370
- }
371
- return out;
372
352
  }
373
353
  // ── AbstractStore: template-method base with all domain logic ────────────
374
354
  /**
@@ -379,1381 +359,1243 @@ function haloDecode(blob, D) {
379
359
  * protected abstract methods that talk to the actual storage backend.
380
360
  */
381
361
  export class AbstractStore {
382
- // ── Config ─────────────────────────────────────────────────────────────
383
- _D;
384
- _maxGroup;
385
- minHaloMass;
386
- efSearch;
387
- m;
388
- efConstruction;
389
- efConstructionInterior;
390
- overfetch;
391
- batchSize;
392
- compactEveryNWrites;
393
- // ── State ──────────────────────────────────────────────────────────────
394
- /** Branch node ids are a dense, monotonically-increasing integer sequence
395
- * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
396
- * They are NEVER deleted, so the count of minted branch ids IS the next id —
397
- * which doubles as the branch-node count and lets has() be an O(1) check.
398
- * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
399
- _nextId = 0;
400
- _writtenSinceCompact = 0;
401
- closed = false;
402
- /** Lifecycle guard resolved once `_dbOpen()` completes. */
403
- _ready = null;
404
- // ── Caches ─────────────────────────────────────────────────────────────
405
- /** Exact-content dedup: content-key node id. Intrinsic compression. */
406
- _leafKey;
407
- _branchKey;
408
- /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
409
- _bytesCache;
410
- /** contentLen memo content is immutable, so entries never invalidate. */
411
- _lenCache;
412
- /** Node-record cache avoids repeated persistence queries for shared DAG
413
- * nodes. Each record is small (a few ints + short leaf buffer). */
414
- _recCache;
415
- /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
416
- * links/pours a node right after interning it, so the working set is one
417
- * deposit's nodes — a small budget captures ~all of it, and an eviction only
418
- * means that node is reached by the DAG climb instead of by direct
419
- * resonance. */
420
- _pendingGist;
421
- /** EXACT halo accumulators for the session's live pours: full-precision in
422
- * memory, 2-bit on disk, so within-session accumulate-then-compare never
423
- * round-trips through the quantizer. Regenerablea miss reads the durable
424
- * 2-bit row. */
425
- _haloExact;
426
- /** NORMALIZED halo read cache the decoded, normalized vector {@link halo}
427
- * returns, cached by id so repeat reads skip the per-call 2-bit decode and
428
- * normalize of a full D-element row (measured on a trained store: ~15K
429
- * halo() calls per deep query over ~50 distinct ids — all but the first
430
- * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
431
- * halo mutation site. Callers receive a COPY, so the cached vector is
432
- * never aliased. Regenerable a miss re-decodes the durable row. */
433
- _haloNorm;
434
- /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
435
- * that bridge nothing). Remembered so subsequent visits prune the subtree
436
- * without re-checking parent count. LRU-bounded: an evicted entry is
437
- * re-checked on next visit if it gained parents in the meantime, it will
438
- * be promoted to the index. */
439
- _coveredIds;
440
- /** Live content-index id set, LRU-bounded so a massive ingest never leaks
441
- * memory; an evicted entry is still indexed (the row is durable), so the
442
- * only cost of an eviction is a duplicate HNSW probe on next visit. */
443
- _indexedIds;
444
- // ── ANN read cache ─────────────────────────────────────────────────────
445
- // The index is read-only between writes, so the same (v,k) always returns
446
- // the same neighbours. Any index mutation (flush, delete, compact) drops
447
- // the cache; the next miss recreates it. Content-addressed (vecKey), not
448
- // identity-addressed — same principle as perceiveMemo.
449
- /** ANN read cache for {@link resonate} — keyed by vecKey(v) + ":" + k;
450
- * lazily initialised, dropped on any index mutation. */
451
- _resonateCache = null;
452
- /** ANN read cache for {@link resonateHalo} — same scheme. */
453
- _resonateHaloCache = null;
454
- // ── Write buffers ──────────────────────────────────────────────────────
455
- /** Content (gist) index write buffer. `ef` is the per-entry HNSW
456
- * construction budget: reach-only interiors carry the reduced
457
- * `efConstructionInterior`; dedup targets omit it (full budget). */
458
- _contentBuffer = [];
459
- /** Halo index write buffer keyed by id so repeats within a batch coalesce. */
460
- _haloBuffer = new Map();
461
- /** Containment write buffer: child → new parents, merged on flush cadence. */
462
- _containBuf = new Map();
463
- /** Dedup-target candidates still in the write buffer (keyed by id). Only
464
- * roots that have gained an edge/halo are targets; a fresh intermediate
465
- * branch is never folded onto. */
466
- _nearDedupBuf = new Map();
467
- /** Ids currently in `_contentBuffer` (not yet flushed) O(1) membership. */
468
- _bufferedIds = new Set();
469
- // ── Transparent-chain cache ────────────────────────────────────────────
470
- /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
471
- * writes: a chain is a pure function of the kid and edge tables, so any
472
- * write that could break a node's transparency (a fresh mint inserting kid
473
- * rows, a link inserting an edge) drops the whole cache — see the two
474
- * invalidation sites. Regenerable; a miss re-walks. */
475
- _chainMemo;
476
- // ── Edge-source-count cache ────────────────────────────────────────────
477
- /** Distinct edge-source count — the store's DOCUMENT COUNT (how many
478
- * learnt contexts predict a continuation), the N of every
479
- * inverse-document-frequency read. −1 until first asked for; from then
480
- * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
481
- * so a read is O(1) — never a table scan on the recall path. */
482
- _edgeSrcCount = -1;
483
- // ── Constructor ────────────────────────────────────────────────────────
484
- constructor(config, D, maxGroup) {
485
- this._D = D;
486
- this._maxGroup = maxGroup;
487
- this.minHaloMass = config.minHaloMass;
488
- this.efSearch = config.efSearch;
489
- this.m = config.m;
490
- this.efConstruction = config.efConstruction;
491
- this.efConstructionInterior = Math.max(
492
- 1,
493
- Math.min(config.efConstructionInterior, config.efConstruction),
494
- );
495
- this.overfetch = config.overfetch;
496
- this.batchSize = config.batchSize;
497
- this.compactEveryNWrites = config.compactEveryNWrites;
498
- this._leafKey = new BoundedMap(config.dedupCacheMax);
499
- this._branchKey = new BoundedMap(config.dedupCacheMax);
500
- this._bytesCache = new BoundedMap(
501
- config.bytesCacheMax,
502
- (v) => v.byteLength,
503
- "smallest",
504
- );
505
- this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
506
- this._recCache = new BoundedMap(
507
- config.recCacheBytes,
508
- (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12,
509
- );
510
- this._pendingGist = new BoundedMap(
511
- config.pendingGistBytes,
512
- (v) => v.byteLength,
513
- );
514
- this._haloExact = new BoundedMap(
515
- config.haloCacheBytes,
516
- (v) => v.byteLength,
517
- );
518
- this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
519
- this._coveredIds = new BoundedMap(config.coveredIdsMax);
520
- this._indexedIds = new BoundedMap(config.coveredIdsMax);
521
- this._chainMemo = new BoundedMap(
522
- config.chainCacheBytes,
523
- (v) => v.length * 4 + 32,
524
- );
525
- }
526
- // ── Public accessors ───────────────────────────────────────────────────
527
- get D() {
528
- return this._D;
529
- }
530
- /** Await the async initialisation performed by the concrete constructor. */
531
- async _ensureReady() {
532
- if (!this._ready) {
533
- throw new Error("Store: not open");
534
- }
535
- await this._ready;
536
- }
537
- // ── Id management ──────────────────────────────────────────────────────
538
- has(id) {
539
- // Byte leaves (negative ids) always exist.
540
- if (id < 0) {
541
- return id >= -256;
542
- }
543
- return Number.isInteger(id) && id >= 0 && id < this._nextId;
544
- }
545
- mintId() {
546
- return this._nextId++;
547
- }
548
- nodeCount() {
549
- return this._nextId;
550
- }
551
- async size() {
552
- await this._ensureReady();
553
- return this._nextId;
554
- }
555
- // ── DAG traversal ──────────────────────────────────────────────────────
556
- get(id) {
557
- // Byte leaves are implicit — fabricate from the id.
558
- if (id < 0) {
559
- return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
560
- }
561
- const hit = this._recCache.get(id);
562
- if (hit !== undefined) {
563
- return hit;
564
- }
565
- const rec = this._dbGetNode(id);
566
- if (rec) {
567
- this._recCache.set(id, rec);
568
- }
569
- return rec;
570
- }
571
- /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
572
- * Iterative post-order on an explicit stack — the call stack never sees the
573
- * tree depth, so even an adversarial chain of nodes stays safe. */
574
- /** How many reads hit a MISSING node record this session (a dangling edge
575
- * or kid id). Zero in a healthy store; a growing count means references
576
- * outlive their records — the read degrades safely to empty bytes, this
577
- * counter is what keeps that degradation observable. */
578
- danglingReads = 0;
579
- bytes(id) {
580
- // Fast path.
581
- const hit = this._bytesCache.get(id);
582
- if (hit) {
583
- return hit;
584
- }
585
- const stack = [id];
586
- const cache = this._bytesCache;
587
- while (stack.length > 0) {
588
- const nid = stack[stack.length - 1]; // peek
589
- // Already resolved by an earlier traversal.
590
- if (cache.get(nid)) {
591
- stack.pop();
592
- continue;
593
- }
594
- const rec = this.get(nid);
595
- if (!rec) {
596
- // A DANGLING id (an edge or kid pointing at no record) reads as
597
- // empty — safe (empty-bytes guards drop it from grounding) but a
598
- // symptom of store corruption, so count it rather than stay silent.
599
- // The cache makes the empty read permanent for the session; the
600
- // counter survives as the visible trace.
601
- this.danglingReads++;
602
- cache.set(nid, _ZERO);
603
- stack.pop();
604
- continue;
605
- }
606
- if (rec.leaf) {
607
- cache.set(nid, new Uint8Array(rec.leaf));
608
- stack.pop();
609
- continue;
610
- }
611
- // Branch — push any uncached children (reverse order so they resolve
612
- // left-to-right). If every child is already cached, concatenate now.
613
- const kids = rec.kids ?? [];
614
- let ready = true;
615
- for (let i = kids.length - 1; i >= 0; i--) {
616
- if (!cache.get(kids[i])) {
617
- stack.push(kids[i]);
618
- ready = false;
362
+ // ── Config ─────────────────────────────────────────────────────────────
363
+ _D;
364
+ _maxGroup;
365
+ minHaloMass;
366
+ efSearch;
367
+ overfetch;
368
+ batchSize;
369
+ compactEveryNWrites;
370
+ // ── State ──────────────────────────────────────────────────────────────
371
+ /** Branch node ids are a dense, monotonically-increasing integer sequence
372
+ * (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
373
+ * They are NEVER deleted, so the count of minted branch ids IS the next id —
374
+ * which doubles as the branch-node count and lets has() be an O(1) check.
375
+ * Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
376
+ _nextId = 0;
377
+ _writtenSinceCompact = 0;
378
+ closed = false;
379
+ /** Lifecycle guard — resolved once `_dbOpen()` completes. */
380
+ _ready = null;
381
+ // ── Caches ─────────────────────────────────────────────────────────────
382
+ /** Exact-content dedup: content-key node id. Intrinsic compression. */
383
+ _leafKey;
384
+ _branchKey;
385
+ /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
386
+ _bytesCache;
387
+ /** contentLen memo — content is immutable, so entries never invalidate. */
388
+ _lenCache;
389
+ /** Node-record cache — avoids repeated persistence queries for shared DAG
390
+ * nodes. Each record is small (a few ints + short leaf buffer). */
391
+ _recCache;
392
+ /** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
393
+ * links/pours a node right after interning it, so the working set is one
394
+ * deposit's nodes — a small budget captures ~all of it, and an eviction only
395
+ * means that node is reached by the DAG climb instead of by direct
396
+ * resonance. */
397
+ _pendingGist;
398
+ /** EXACT halo accumulators for the session's live pours: full-precision in
399
+ * memory, 2-bit on disk, so within-session accumulate-then-compare never
400
+ * round-trips through the quantizer. Regenerable — a miss reads the durable
401
+ * 2-bit row. */
402
+ _haloExact;
403
+ /** NORMALIZED halo read cachethe decoded, normalized vector {@link halo}
404
+ * returns, cached by id so repeat reads skip the per-call 2-bit decode and
405
+ * normalize of a full D-element row (measured on a trained store: ~15K
406
+ * halo() calls per deep query over ~50 distinct ids all but the first
407
+ * per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
408
+ * halo mutation site. Callers receive a COPY, so the cached vector is
409
+ * never aliased. Regenerable a miss re-decodes the durable row. */
410
+ _haloNorm;
411
+ /** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
412
+ * that bridge nothing). Remembered so subsequent visits prune the subtree
413
+ * without re-checking parent count. LRU-bounded: an evicted entry is
414
+ * re-checked on next visit if it gained parents in the meantime, it will
415
+ * be promoted to the index. */
416
+ _coveredIds;
417
+ /** Live content-index id set, LRU-bounded so a massive ingest never leaks
418
+ * memory; an evicted entry is still indexed (the row is durable), so the
419
+ * only cost of an eviction is a duplicate index probe on next visit. */
420
+ _indexedIds;
421
+ // ── ANN read cache ─────────────────────────────────────────────────────
422
+ // The index is read-only between writes, so the same (v,k) always returns
423
+ // the same neighbours. Any index mutation (flush, delete, compact) drops
424
+ // the cache; the next miss recreates it. Content-addressed (vecKey), not
425
+ // identity-addressed same principle as perceiveMemo.
426
+ /** ANN read cache for {@link resonate} keyed by vecKey(v) + ":" + k;
427
+ * lazily initialised, dropped on any index mutation. */
428
+ _resonateCache = null;
429
+ /** ANN read cache for {@link resonateHalo} — same scheme. */
430
+ _resonateHaloCache = null;
431
+ // ── Write buffers ──────────────────────────────────────────────────────
432
+ /** Content (gist) index write buffer. */
433
+ _contentBuffer = [];
434
+ /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
435
+ _haloBuffer = new Map();
436
+ /** Containment write buffer: child new parents, merged on flush cadence. */
437
+ _containBuf = new Map();
438
+ /** Dedup-target candidates still in the write buffer (keyed by id). Only
439
+ * roots that have gained an edge/halo are targets; a fresh intermediate
440
+ * branch is never folded onto. */
441
+ _nearDedupBuf = new Map();
442
+ /** Ids currently in `_contentBuffer` (not yet flushed) — O(1) membership. */
443
+ _bufferedIds = new Set();
444
+ // ── Transparent-chain cache ────────────────────────────────────────────
445
+ /** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
446
+ * writes: a chain is a pure function of the kid and edge tables, so any
447
+ * write that could break a node's transparency (a fresh mint inserting kid
448
+ * rows, a link inserting an edge) drops the whole cache — see the two
449
+ * invalidation sites. Regenerable; a miss re-walks. */
450
+ _chainMemo;
451
+ // ── Edge-source-count cache ────────────────────────────────────────────
452
+ /** Distinct edge-source count the store's DOCUMENT COUNT (how many
453
+ * learnt contexts predict a continuation), the N of every
454
+ * inverse-document-frequency read. −1 until first asked for; from then
455
+ * on maintained INCREMENTALLY by {@link link} (edges are never deleted),
456
+ * so a read is O(1) — never a table scan on the recall path. */
457
+ _edgeSrcCount = -1;
458
+ // ── Constructor ────────────────────────────────────────────────────────
459
+ constructor(config, D, maxGroup) {
460
+ this._D = D;
461
+ this._maxGroup = maxGroup;
462
+ this.minHaloMass = config.minHaloMass;
463
+ this.efSearch = config.efSearch;
464
+ this.overfetch = config.overfetch;
465
+ this.batchSize = config.batchSize;
466
+ this.compactEveryNWrites = config.compactEveryNWrites;
467
+ this._leafKey = new BoundedMap(config.dedupCacheMax);
468
+ this._branchKey = new BoundedMap(config.dedupCacheMax);
469
+ this._bytesCache = new BoundedMap(config.bytesCacheMax, (v) => v.byteLength, "smallest");
470
+ this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
471
+ this._recCache = new BoundedMap(config.recCacheBytes, (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12);
472
+ this._pendingGist = new BoundedMap(config.pendingGistBytes, (v) => v.byteLength);
473
+ this._haloExact = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
474
+ this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
475
+ this._coveredIds = new BoundedMap(config.coveredIdsMax);
476
+ this._indexedIds = new BoundedMap(config.coveredIdsMax);
477
+ this._chainMemo = new BoundedMap(config.chainCacheBytes, (v) => v.length * 4 + 32);
478
+ }
479
+ // ── Public accessors ───────────────────────────────────────────────────
480
+ get D() {
481
+ return this._D;
482
+ }
483
+ /** Await the async initialisation performed by the concrete constructor. */
484
+ async _ensureReady() {
485
+ if (!this._ready)
486
+ throw new Error("Store: not open");
487
+ await this._ready;
488
+ }
489
+ // ── Id management ──────────────────────────────────────────────────────
490
+ has(id) {
491
+ // Byte leaves (negative ids) always exist.
492
+ if (id < 0)
493
+ return id >= -256;
494
+ return Number.isInteger(id) && id >= 0 && id < this._nextId;
495
+ }
496
+ mintId() {
497
+ return this._nextId++;
498
+ }
499
+ nodeCount() {
500
+ return this._nextId;
501
+ }
502
+ async size() {
503
+ await this._ensureReady();
504
+ return this._nextId;
505
+ }
506
+ // ── DAG traversal ──────────────────────────────────────────────────────
507
+ get(id) {
508
+ // Byte leaves are implicit — fabricate from the id.
509
+ if (id < 0) {
510
+ return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
619
511
  }
620
- }
621
- if (!ready) {
622
- continue;
623
- }
624
- stack.pop();
625
- const out = concat(kids.map((k) => cache.get(k)));
626
- cache.set(nid, out);
627
- }
628
- return cache.get(id) ?? _ZERO;
629
- }
630
- /** First `maxLen` bytes of a node. Walks only the leftmost branch,
631
- * stopping at `maxLen` so a 1 MB document root costs the same as a
632
- * 4-byte leaf. Recursive, but tree depth is logarithmic.
633
- *
634
- * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
635
- * may be shared with the byte cache and with other callers — treat them
636
- * as read-only. Mutating one would corrupt every subsequent read. */
637
- bytesPrefix(id, maxLen) {
638
- if (maxLen <= 0) {
639
- return _ZERO;
640
- }
641
- // A FULL read (the ALL sentinel) routes through bytes(), whose
642
- // reconstruction enters the byte-budget cache. Without this, the mind's
643
- // read() which always passes ALL — re-walked the DAG and re-concatenated
644
- // on EVERY repeated read of an uncached branch, bypassing the cache that
645
- // exists precisely for those reconstructions.
646
- if (maxLen >= 0x7fffffff) {
647
- return this.bytes(id);
648
- }
649
- // Full-cache hit: bytes() already reconstructed the whole node.
650
- const full = this._bytesCache.get(id);
651
- if (full) {
652
- return full.length <= maxLen ? full : full.subarray(0, maxLen);
653
- }
654
- const rec = this.get(id);
655
- if (!rec) {
656
- return _ZERO;
657
- }
658
- if (rec.leaf) {
659
- // Cache the (small) leaf bytes — cheap and reusable. COPY before
660
- // caching: rec.leaf is the node record's own buffer, and handing it
661
- // out would let one mutating caller corrupt the record AND the cache
662
- // (bytes() makes the same copy for the same reason).
663
- const leaf = new Uint8Array(rec.leaf);
664
- this._bytesCache.set(id, leaf);
665
- return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
666
- }
667
- // Branch — walk children left-to-right, stopping at maxLen.
668
- const kids = rec.kids ?? [];
669
- const parts = [];
670
- let got = 0;
671
- for (const k of kids) {
672
- if (got >= maxLen) {
673
- break;
674
- }
675
- const child = this.bytesPrefix(k, maxLen - got);
676
- parts.push(child);
677
- got += child.length;
678
- }
679
- return concat(parts);
680
- }
681
- contentLen(id, cap = Infinity) {
682
- if (id < 0) {
683
- return 1; // implicit single-byte leaf
684
- }
685
- const hit = this._lenCache.get(id);
686
- if (hit !== undefined) {
687
- return hit; // exact — valid under any cap
688
- }
689
- if (cap <= 0) {
690
- return 0;
691
- }
692
- const rec = this.get(id);
693
- let n = 0;
694
- let clamped = false;
695
- if (rec) {
696
- if (rec.leaf) {
697
- n = rec.leaf.length;
698
- } else if (rec.kids) {
699
- for (const k of rec.kids) {
700
- n += this.contentLen(k, cap - n);
701
- if (n >= cap) {
702
- clamped = true; // partial sum — a lower bound, not the length
703
- break;
704
- }
512
+ const hit = this._recCache.get(id);
513
+ if (hit !== undefined)
514
+ return hit;
515
+ const rec = this._dbGetNode(id);
516
+ if (rec)
517
+ this._recCache.set(id, rec);
518
+ return rec;
519
+ }
520
+ /** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
521
+ * Iterative post-order on an explicit stack — the call stack never sees the
522
+ * tree depth, so even an adversarial chain of nodes stays safe. */
523
+ /** How many reads hit a MISSING node record this session (a dangling edge
524
+ * or kid id). Zero in a healthy store; a growing count means references
525
+ * outlive their records — the read degrades safely to empty bytes, this
526
+ * counter is what keeps that degradation observable. */
527
+ danglingReads = 0;
528
+ bytes(id) {
529
+ // Fast path.
530
+ const hit = this._bytesCache.get(id);
531
+ if (hit)
532
+ return hit;
533
+ const stack = [id];
534
+ const cache = this._bytesCache;
535
+ while (stack.length > 0) {
536
+ const nid = stack[stack.length - 1]; // peek
537
+ // Already resolved by an earlier traversal.
538
+ if (cache.get(nid)) {
539
+ stack.pop();
540
+ continue;
541
+ }
542
+ const rec = this.get(nid);
543
+ if (!rec) {
544
+ // A DANGLING id (an edge or kid pointing at no record) reads as
545
+ // empty — safe (empty-bytes guards drop it from grounding) but a
546
+ // symptom of store corruption, so count it rather than stay silent.
547
+ // The cache makes the empty read permanent for the session; the
548
+ // counter survives as the visible trace.
549
+ this.danglingReads++;
550
+ cache.set(nid, _ZERO);
551
+ stack.pop();
552
+ continue;
553
+ }
554
+ if (rec.leaf) {
555
+ cache.set(nid, new Uint8Array(rec.leaf));
556
+ stack.pop();
557
+ continue;
558
+ }
559
+ // Branch — push any uncached children (reverse order so they resolve
560
+ // left-to-right). If every child is already cached, concatenate now.
561
+ const kids = rec.kids ?? [];
562
+ let ready = true;
563
+ for (let i = kids.length - 1; i >= 0; i--) {
564
+ if (!cache.get(kids[i])) {
565
+ stack.push(kids[i]);
566
+ ready = false;
567
+ }
568
+ }
569
+ if (!ready)
570
+ continue;
571
+ stack.pop();
572
+ const out = concat(kids.map((k) => cache.get(k)));
573
+ cache.set(nid, out);
705
574
  }
706
- }
707
- }
708
- if (!clamped) {
709
- this._lenCache.set(id, n);
710
- }
711
- return n;
712
- }
713
- // ── Content-addressed lookup ───────────────────────────────────────────
714
- findLeaf(bytes) {
715
- if (bytes.length === 1) {
716
- return -(bytes[0] + 1);
717
- }
718
- const key = keyOf(bytes);
719
- const cached = this._leafKey.get(key);
720
- if (cached !== undefined) {
721
- return cached;
722
- }
723
- const id = this._dbFindLeaf(hashOf(bytes), bytes);
724
- if (id !== null) {
725
- this._leafKey.set(key, id);
726
- }
727
- return id;
728
- }
729
- findBranch(kids) {
730
- const key = kids.join(",");
731
- const cached = this._branchKey.get(key);
732
- if (cached !== undefined) {
733
- return cached;
734
- }
735
- const flat = flatKidsBytes(kids);
736
- let id;
737
- if (flat) {
738
- id = this._dbFindBranchByLeaf(hashOf(flat), flat);
739
- } else {
740
- const packed = packKids(kids);
741
- id = this._dbFindBranchByKids(hashOf(packed), packed);
742
- }
743
- if (id !== null && key.length <= DEDUP_KEY_MAX) {
744
- this._branchKey.set(key, id);
745
- }
746
- return id;
747
- }
748
- // ── Structural parents ─────────────────────────────────────────────────
749
- parents(id) {
750
- return this._dbGetParents(id);
751
- }
752
- parentsFirst(id, limit) {
753
- return this._dbGetParentsFirst(id, limit);
754
- }
755
- hasParents(id) {
756
- return this._dbGetParentsFirst(id, 1).length > 0;
757
- }
758
- chainRun(id) {
759
- const hit = this._chainMemo.get(id);
760
- if (hit !== undefined) {
761
- return hit;
762
- }
763
- const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
764
- this._chainMemo.set(id, run);
765
- return run;
766
- }
767
- /** {@link Store.chainRun}'s walk, node at a time through the existing
768
- * probes. Adapters with a set-based query engine should override with a
769
- * single server-side descent (the SQLite adapter uses a recursive CTE). */
770
- _chainWalk(id, cap) {
771
- const run = [id];
772
- let n = id;
773
- while (run.length < cap) {
774
- if (this.hasNext(n) || this.prevCount(n) > 0) {
775
- break;
776
- }
777
- const ps = this._dbGetParentsFirst(n, 2);
778
- if (ps.length !== 1) {
779
- break;
780
- }
781
- n = ps[0];
782
- run.push(n);
783
- }
784
- return run;
785
- }
786
- // ── Containment ────────────────────────────────────────────────────────
787
- addContainer(child, parent) {
788
- let set = this._containBuf.get(child);
789
- if (set === undefined) {
790
- set = new Set();
791
- this._containBuf.set(child, set);
792
- }
793
- set.add(parent);
794
- }
795
- hasContainers(child) {
796
- if (this._dbContainExists(child)) {
797
- return true;
798
- }
799
- const buf = this._containBuf.get(child);
800
- return buf !== undefined && buf.size > 0;
801
- }
802
- containersSlice(child, offset, limit) {
803
- const out = this._dbGetContainParentsSlice(child, offset, limit);
804
- if (out.length >= limit) {
805
- return out;
806
- }
807
- // Buffered adds page in AFTER the stored ones. A buffered parent that is
808
- // also stored may repeat across the seam; page consumers dedup by id
809
- // (they all carry seen-sets), so a repeat costs a skip, never an error.
810
- const buf = this._containBuf.get(child);
811
- if (!buf || buf.size === 0) {
812
- return out;
813
- }
814
- const storedCount = this._dbGetContainCount(child);
815
- const bufStart = Math.max(0, offset - storedCount) +
816
- Math.max(0, out.length - Math.max(0, storedCount - offset));
817
- let i = 0;
818
- for (const p of buf) {
819
- if (out.length >= limit) {
820
- break;
821
- }
822
- if (i++ < bufStart) {
823
- continue;
824
- }
825
- out.push(p);
826
- }
827
- return out;
828
- }
829
- containers(child) {
830
- const stored = this._dbGetContainParents(child);
831
- const buf = this._containBuf.get(child);
832
- if (stored.length === 0) {
833
- return buf ? [...buf] : [];
834
- }
835
- if (!buf) {
836
- return stored;
837
- }
838
- const merged = new Set(stored);
839
- for (const p of buf) {
840
- merged.add(p);
841
- }
842
- return [...merged];
843
- }
844
- // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
845
- flatLeafIds(kids) {
846
- const out = [];
847
- const stack = [...kids].reverse();
848
- while (stack.length > 0) {
849
- const id = stack.pop();
850
- if (id < 0) {
851
- out.push(id);
852
- continue;
853
- }
854
- const rec = this.get(id);
855
- if (!rec) {
856
- return null;
857
- }
858
- if (rec.leaf !== null) {
859
- for (let i = 0; i < rec.leaf.length; i++) {
860
- out.push(-(rec.leaf[i] + 1));
575
+ return cache.get(id) ?? _ZERO;
576
+ }
577
+ /** First `maxLen` bytes of a node. Walks only the leftmost branch,
578
+ * stopping at `maxLen` — so a 1 MB document root costs the same as a
579
+ * 4-byte leaf. Recursive, but tree depth is logarithmic.
580
+ *
581
+ * IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
582
+ * may be shared with the byte cache and with other callers — treat them
583
+ * as read-only. Mutating one would corrupt every subsequent read. */
584
+ bytesPrefix(id, maxLen) {
585
+ if (maxLen <= 0)
586
+ return _ZERO;
587
+ // A FULL read (the ALL sentinel) routes through bytes(), whose
588
+ // reconstruction enters the byte-budget cache. Without this, the mind's
589
+ // read() which always passes ALL — re-walked the DAG and re-concatenated
590
+ // on EVERY repeated read of an uncached branch, bypassing the cache that
591
+ // exists precisely for those reconstructions.
592
+ if (maxLen >= 0x7fffffff)
593
+ return this.bytes(id);
594
+ // Full-cache hit: bytes() already reconstructed the whole node.
595
+ const full = this._bytesCache.get(id);
596
+ if (full)
597
+ return full.length <= maxLen ? full : full.subarray(0, maxLen);
598
+ const rec = this.get(id);
599
+ if (!rec)
600
+ return _ZERO;
601
+ if (rec.leaf) {
602
+ // Cache the (small) leaf bytes — cheap and reusable. COPY before
603
+ // caching: rec.leaf is the node record's own buffer, and handing it
604
+ // out would let one mutating caller corrupt the record AND the cache
605
+ // (bytes() makes the same copy for the same reason).
606
+ const leaf = new Uint8Array(rec.leaf);
607
+ this._bytesCache.set(id, leaf);
608
+ return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
861
609
  }
862
- } else if (rec.kids !== null) {
863
- for (let i = rec.kids.length - 1; i >= 0; i--) {
864
- stack.push(rec.kids[i]);
610
+ // Branch walk children left-to-right, stopping at maxLen.
611
+ const kids = rec.kids ?? [];
612
+ const parts = [];
613
+ let got = 0;
614
+ for (const k of kids) {
615
+ if (got >= maxLen)
616
+ break;
617
+ const child = this.bytesPrefix(k, maxLen - got);
618
+ parts.push(child);
619
+ got += child.length;
865
620
  }
866
- }
867
- }
868
- return out;
869
- }
870
- /** On a dedup HIT, keep the node's gist available for lazy indexing —
871
- * EXACTLY when it is not already indexed. Replaces the old id-range
872
- * "recency" heuristic (id nextId − cacheWindow), which conflated an LRU
873
- * entry COUNT with an id RANGE and permanently refused to index any node
874
- * that first became a resonance target long after it was minted (an early
875
- * interior later reused as an edge/halo-bearing deposit root was silently
876
- * unreachable by resonance). The durable index itself is the arbiter:
877
- * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
878
- captureIfUnindexed(id, gist) {
879
- if (this._indexedIds.has(id) || this._pendingGist.has(id)) {
880
- return;
881
- }
882
- if (this._vecContentHas(id)) {
883
- this._indexedIds.set(id, true);
884
- return;
885
- }
886
- this._pendingGist.set(id, normalize(copy(gist)));
887
- // A node that ALREADY bridges experiences but was never indexed (its 1→2
888
- // transition fired while its gist was evicted, or in a pre-transition
889
- // store) is promoted on this re-encounter — the recapture above is
890
- // exactly what makes its gist available again. Byte leaves (negative
891
- // ids) never have kid rows, so skip their parent probe.
892
- if (id >= 0) {
893
- this.promoteBridge(id);
894
- }
895
- }
896
- /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
897
- * criterion), promote its gist into the content index NOW — the exact
898
- * moment it becomes useful for multi-experience recall. The 1→2 parent
899
- * transition fires on {@link _dbInsertKid} during mint, and nodes that
900
- * were already bridges but missed indexing (gist evicted, pre-transition
901
- * store) are recaptured in {@link captureIfUnindexed}.
902
- *
903
- * A no-op when the node is already indexed or its gist is evicted from
904
- * the pending cache — a future re-encounter will retry. */
905
- promoteBridge(id) {
906
- // A LIMITed probe: this runs once per kid insert on the MINT hot path,
907
- // and a shared child's full parent set grows with the corpus.
908
- if (this._dbGetParentsFirst(id, 2).length < 2) {
909
- return;
910
- }
911
- this.indexGist(id, false);
912
- }
913
- // ── Core interning: dedup → near-dedup → mint ──────────────────────────
914
- async intern(leaf, kids, gist) {
915
- await this._ensureReady();
916
- // 1. Exact dedup — equal content → one id, no vector work. Primary
917
- // compression mechanism, intrinsic to the store.
918
- //
919
- // findLeaf/findBranch are the content-addressed lookups: in-memory cache
920
- // first, then a durable probe that repopulates the cache on a hit.
921
- // Using them here (rather than only the cache) makes dedup survive a cold
922
- // cache — a resumed/checkpointed training run, or one whose dedup cache
923
- // has evicted old keys, still recognises content already on disk and
924
- // reuses its id instead of minting a duplicate.
925
- const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
926
- if (hit !== null) {
927
- this.captureIfUnindexed(hit, gist);
928
- return hit;
929
- }
930
- // 1b. Content-addressed lookup by leaf-id signature. When the same byte
931
- // sequence was stored as a flat branch (via putBranch during deposit),
932
- // a branch node spanning those bytes reuses that id even when its tree
933
- // structure differs — pure content addressing, same bytes → same node.
934
- if (kids !== null && kids.length >= 2) {
935
- const leafIds = this.flatLeafIds(kids);
936
- if (leafIds !== null) {
937
- const flatHit = this.findBranch(leafIds);
938
- if (flatHit !== null) {
939
- this.captureIfUnindexed(flatHit, gist);
940
- return flatHit;
621
+ return concat(parts);
622
+ }
623
+ contentLen(id, cap = Infinity) {
624
+ if (id < 0)
625
+ return 1; // implicit single-byte leaf
626
+ const hit = this._lenCache.get(id);
627
+ if (hit !== undefined)
628
+ return hit; // exact valid under any cap
629
+ if (cap <= 0)
630
+ return 0;
631
+ const rec = this.get(id);
632
+ let n = 0;
633
+ let clamped = false;
634
+ if (rec) {
635
+ if (rec.leaf)
636
+ n = rec.leaf.length;
637
+ else if (rec.kids) {
638
+ for (const k of rec.kids) {
639
+ n += this.contentLen(k, cap - n);
640
+ if (n >= cap) {
641
+ clamped = true; // partial sum — a lower bound, not the length
642
+ break;
643
+ }
644
+ }
645
+ }
941
646
  }
942
- }
943
- }
944
- const cache = leaf !== null ? this._leafKey : this._branchKey;
945
- const key = leaf !== null ? keyOf(leaf) : kids.join(",");
946
- // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
947
- // Leaves are single bytes: exact dedup already collapses every identical
948
- // leaf, and near-merging distinct leaves only corrupts bytes for no real
949
- // saving. Real near-dedup compression lives in subtree (branch) fusion.
950
- //
951
- // There is deliberately NO HNSW probe of the FLUSHED index here. It used
952
- // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
953
- // ~every interior branch, since interiors are never dedup targets —
954
- // making one ANN query per branch the dominant training cost (it dwarfed
955
- // perception and the index write). And it was not merely expensive but
956
- // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
957
- // nearest "target" of a fresh branch, so the fold collapsed two
958
- // byte-different subtrees onto one id and corrupted exact reconstruction
959
- // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
960
- // are caught two cheaper, exact ways instead: identical content by the
961
- // exact-dedup hash-cons above, and a near-gist target still in the write
962
- // buffer by the scan below.
963
- if (leaf === null) {
964
- // Near-dedup PREFILTER — the scale-aware identity bar
965
- // ({@link identityBar}) for THIS branch's own length, not the fixed
966
- // estimator floor: under the linear fold a long branch crosses
967
- // 1 − 1/√D while whole windows differ, and every such crossing paid a
968
- // full differsByOneWindow byte reconstruction. Same final semantics
969
- // (the byte check below still decides identity); strictly fewer byte
970
- // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
971
- // memoized bottom-up by the interning order itself, O(kids)) is only
972
- // computed once a nearest candidate actually exists, so the hot
973
- // no-candidate mint pays nothing.
974
- let best = -1;
975
- let bestId = null;
976
- if (this._nearDedupBuf.size > 0) {
977
- const g = normalize(copy(gist));
978
- // Candidates are the buffered DEDUP TARGETS only — genuine whole
979
- // experiences (edge/halo-bearing roots) not yet flushed.
980
- for (const [id, vector] of this._nearDedupBuf) {
981
- const s = dot(g, vector);
982
- if (s > best) {
983
- best = s;
984
- bestId = id;
985
- }
647
+ if (!clamped)
648
+ this._lenCache.set(id, n);
649
+ return n;
650
+ }
651
+ // ── Content-addressed lookup ───────────────────────────────────────────
652
+ findLeaf(bytes) {
653
+ if (bytes.length === 1)
654
+ return -(bytes[0] + 1);
655
+ const key = keyOf(bytes);
656
+ const cached = this._leafKey.get(key);
657
+ if (cached !== undefined)
658
+ return cached;
659
+ const id = this._dbFindLeaf(hashOf(bytes), bytes);
660
+ if (id !== null)
661
+ this._leafKey.set(key, id);
662
+ return id;
663
+ }
664
+ findBranch(kids) {
665
+ const key = kids.join(",");
666
+ const cached = this._branchKey.get(key);
667
+ if (cached !== undefined)
668
+ return cached;
669
+ const flat = flatKidsBytes(kids);
670
+ let id;
671
+ if (flat) {
672
+ id = this._dbFindBranchByLeaf(hashOf(flat), flat);
986
673
  }
987
- }
988
- let blen = 0;
989
- if (bestId !== null) {
990
- for (const k of kids) {
991
- blen += this.contentLen(k);
674
+ else {
675
+ const packed = packKids(kids);
676
+ id = this._dbFindBranchByKids(hashOf(packed), packed);
992
677
  }
993
- }
994
- if (
995
- bestId !== null &&
996
- best >= identityBar(this.D, this._maxGroup, blen)
997
- ) {
998
- // Scale-aware acceptance. The cosine bar alone is scale-BLIND
999
- // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
1000
- // a localized difference faster than linearly in form size, so for
1001
- // deep forms ANY fixed bar below 1 is crossed by exactly the one
1002
- // span that distinguishes two experiences. No inversion of the
1003
- // deficit is trustworthy, so the bytes themselves decide: the two
1004
- // forms must be identical except for ONE local span of at most W
1005
- // bytes — the river window, the perception's own resolution quantum.
1006
- const W = this._maxGroup;
1007
- if (this.differsByOneWindow(kids, bestId, W)) {
1008
- if (key.length <= DEDUP_KEY_MAX) {
1009
- cache.set(key, bestId);
1010
- }
1011
- return bestId;
678
+ if (id !== null && key.length <= DEDUP_KEY_MAX) {
679
+ this._branchKey.set(key, id);
1012
680
  }
1013
- }
1014
- }
1015
- // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
1016
- // leaf) stores its BYTES in the leaf column with a zero-length kids blob
1017
- // as the marker — the kid list is derived on read.
1018
- const id = this.mintId();
1019
- this._dbBeginTx();
1020
- const flat = kids ? flatKidsBytes(kids) : null;
1021
- const packed = kids && !flat ? packKids(kids) : null;
1022
- this._dbInsertNode(
1023
- id,
1024
- leaf ?? flat,
1025
- packed ?? (flat ? _ZERO : null),
1026
- hashOf(leaf ?? flat ?? packed),
1027
- );
1028
- // Reverse structural edge: each distinct child → this parent. Lets the graph
1029
- // be climbed upward in index time, with no scan of the kids blobs.
1030
- //
1031
- // Populated NATURALLY here and only here — one write per child, in the SAME
1032
- // mint that creates the node, inside the SAME deferred transaction as the
1033
- // node row, so node and kid rows are always durable together.
1034
- //
1035
- // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
1036
- // every branch, so its parent set is the corpus-sized hub the climb's
1037
- // saturation guard discards unread.
1038
- if (kids) {
1039
- // Kid rows change parent counts — a child whose parent set grows from
1040
- // one is no longer transparent, so every cached chain that hopped
1041
- // through it is stale. Which chains those are is unknowable without a
1042
- // reverse index, so the WHOLE cache drops (writes come in training
1043
- // bursts where the cache is cold anyway; queries rebuild it lazily).
1044
- this._chainMemo.clear();
1045
- for (const c of kids) {
1046
- if (c < 0 && c >= -256) {
1047
- continue;
681
+ return id;
682
+ }
683
+ // ── Structural parents ─────────────────────────────────────────────────
684
+ parents(id) {
685
+ return this._dbGetParents(id);
686
+ }
687
+ parentsFirst(id, limit) {
688
+ return this._dbGetParentsFirst(id, limit);
689
+ }
690
+ hasParents(id) {
691
+ return this._dbGetParentsFirst(id, 1).length > 0;
692
+ }
693
+ chainRun(id) {
694
+ const hit = this._chainMemo.get(id);
695
+ if (hit !== undefined)
696
+ return hit;
697
+ const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
698
+ this._chainMemo.set(id, run);
699
+ return run;
700
+ }
701
+ /** {@link Store.chainRun}'s walk, node at a time through the existing
702
+ * probes. Adapters with a set-based query engine should override with a
703
+ * single server-side descent (the SQLite adapter uses a recursive CTE). */
704
+ _chainWalk(id, cap) {
705
+ const run = [id];
706
+ let n = id;
707
+ while (run.length < cap) {
708
+ if (this.hasNext(n) || this.prevCount(n) > 0)
709
+ break;
710
+ const ps = this._dbGetParentsFirst(n, 2);
711
+ if (ps.length !== 1)
712
+ break;
713
+ n = ps[0];
714
+ run.push(n);
1048
715
  }
1049
- this._dbInsertKid(c, id);
1050
- // The 1→2 parent TRANSITION happens here and only here (hash-cons
1051
- // means an existing branch never re-inserts kid rows, so parent sets
1052
- // grow exclusively through fresh mints): the child just became a
1053
- // structural bridge between experiences — the exact set post-hoc
1054
- // compaction keeps so its gist enters the reach index NOW.
1055
- this.promoteBridge(c);
1056
- }
1057
- }
1058
- if (key.length <= DEDUP_KEY_MAX) {
1059
- cache.set(key, id);
1060
- }
1061
- if (leaf) {
1062
- this._bytesCache.set(id, new Uint8Array(leaf));
1063
- } else if (flat) {
1064
- this._bytesCache.set(id, flat);
1065
- }
1066
- // Capture the gist; it is pushed into the content index lazily, the first
1067
- // time this node becomes a resonance target (link / pourHalo). A node that
1068
- // never does (a pure intermediate DAG node — ~99.5% of them) is never
1069
- // indexed: it costs one persistence row, no HNSW slot and no merge probe.
1070
- this._pendingGist.set(id, normalize(copy(gist)));
1071
- await this.maybeFlush();
1072
- return id;
1073
- }
1074
- /** Whether the byte content under `kids` and the byte content of `targetId`
1075
- * are identical except for ONE local span of at most `W` bytes on each side
1076
- * — the near dedup's byte-grain definition of a near-duplicate. A
1077
- * common-prefix / common-suffix trim: whatever remains after both trims is
1078
- * the single differing span (substitution, insertion or deletion), and both
1079
- * remainders must fit the budget. Scattered differences leave a wide
1080
- * middle and are rejected. */
1081
- differsByOneWindow(kids, targetId, W) {
1082
- const a = concat(
1083
- kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)),
1084
- );
1085
- const b = this.bytesPrefix(targetId, a.length + W + 1);
1086
- if (Math.abs(a.length - b.length) > W) {
1087
- return false;
1088
- }
1089
- const n = Math.min(a.length, b.length);
1090
- let i = 0;
1091
- while (i < n && a[i] === b[i]) {
1092
- i++;
1093
- }
1094
- let j = 0;
1095
- while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j]) {
1096
- j++;
1097
- }
1098
- return a.length - i - j <= W && b.length - i - j <= W;
1099
- }
1100
- async putLeaf(bytes, gist) {
1101
- // Single bytes are implicit — no DB row and no eager index slot. The gist
1102
- // is captured like any other node's and promoted into the content index
1103
- // LAZILY, the first time the byte becomes a resonance target.
1104
- if (bytes.length === 1) {
1105
- const id = -(bytes[0] + 1);
1106
- this.captureIfUnindexed(id, gist);
1107
- return id;
1108
- }
1109
- return this.intern(new Uint8Array(bytes), null, gist);
1110
- }
1111
- async putBranch(kids, gist) {
1112
- return this.intern(null, kids, gist);
1113
- }
1114
- // ── Lazy content indexing ──────────────────────────────────────────────
1115
- /** Promote a node's captured gist into the content (resonance) index, once.
1116
- * Called the first time a node becomes a target — i.e. from `link` (it bears
1117
- * or receives a continuation edge) or `pourHalo` (it gains distributional
1118
- * company). Idempotent: a node already indexed, or whose gist has been evicted
1119
- * from the bounded pending map, is a no-op.
1120
- *
1121
- * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1122
- * near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
1123
- * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1124
- indexGist(id, dedupTarget) {
1125
- if (dedupTarget && this._bufferedIds.has(id)) {
1126
- // Still buffered (indexed this batch, not yet flushed) — a live dedup
1127
- // candidate, recorded in O(1) with no scan of the content buffer.
1128
- const v = this._pendingGist.get(id);
1129
- if (v !== undefined) {
1130
- this._nearDedupBuf.set(id, v);
1131
- }
1132
- }
1133
- if (this._indexedIds.has(id)) {
1134
- return;
1135
- }
1136
- const v = this._pendingGist.get(id);
1137
- if (v === undefined) {
1138
- return;
1139
- }
1140
- // Already durably indexed by a previous session? A node id names its
1141
- // content, and the gist is a pure function of the content, so the stored
1142
- // vector can only be identical — re-buffering it would spend an encode
1143
- // and an upsert on a guaranteed no-op. One point query recognises this;
1144
- // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1145
- // This is what makes a RESUMED training run replay already-deposited
1146
- // content at read speed instead of re-upserting the recent-id window.
1147
- if (this._vecContentHas(id)) {
1148
- this._indexedIds.set(id, true);
1149
- return;
1150
- }
1151
- this._indexedIds.set(id, true);
1152
- // Reach-only interiors build with the reduced construction budget — the
1153
- // one deliberate speed-for-quality trade of ingestion (see
1154
- // {@link StoreConfig.efConstructionInterior}); targets keep the full one.
1155
- this._contentBuffer.push(
1156
- dedupTarget
1157
- ? { id, vector: v }
1158
- : { id, vector: v, ef: this.efConstructionInterior },
1159
- );
1160
- this._bufferedIds.add(id);
1161
- // A node indexed AS a dedup target enters the candidate set immediately.
1162
- if (dedupTarget) {
1163
- this._nearDedupBuf.set(id, v);
1164
- }
1165
- }
1166
- /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1167
- * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1168
- * root is a genuine target (a whole experience), so it is a dedup target
1169
- * too. */
1170
- indexTarget(id) {
1171
- this.indexGist(id, true);
1172
- }
1173
- /** Index a node and its interior forms as resonance targets. A node that
1174
- * gains an edge is a learnt EXPERIENCE, and the consensus climb
1175
- * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1176
- * resonating its SUB-REGIONS — branch nodes within the experience — and
1177
- * climbing their parents back to it.
1178
- *
1179
- * EVERY interior branch is indexed unconditionally, and this is
1180
- * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1181
- * the post-hoc compaction criterion) was tried and REJECTED by the test
1182
- * suite — partial recall of an experience's interior slices, multi-topic
1183
- * attention, and counterfactual anchoring all resonate to SINGLE-parent
1184
- * interiors (13 tests fail without them). Post-hoc structural compaction
1185
- * ({@link compactContentIndex}) may still remove them, but that is a
1186
- * storage/recall trade-off for archived stores, not a free optimisation.
1187
- * The store's hash-cons bounds the index by the number of DISTINCT byte
1188
- * patterns in the corpus — not by the number of deposits.
1189
- *
1190
- * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1191
- * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1192
- * let a partial query resonate and climb, but a fresh branch must never
1193
- * merge onto an interior node of another experience.
1194
- *
1195
- * Iterative explicit-queue walk: the call stack never sees tree depth. */
1196
- indexSubtree(root) {
1197
- // The root is the whole experience — always index as a merge target.
1198
- this.indexGist(root, true);
1199
- const seen = new Set([root]);
1200
- const stack = [root];
1201
- while (stack.length > 0) {
1202
- const id = stack.pop();
1203
- const kids = this.get(id)?.kids;
1204
- if (!kids) {
1205
- continue; // leaf — never a resonance anchor
1206
- }
1207
- const isRoot = id === root;
1208
- // Already-indexed or already-skipped nodes PRUNE the walk — their
1209
- // subtrees were classified in a prior call.
1210
- if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1211
- continue;
1212
- }
1213
- for (const k of kids) {
1214
- if (!seen.has(k)) {
1215
- seen.add(k);
1216
- stack.push(k);
716
+ return run;
717
+ }
718
+ // ── Containment ────────────────────────────────────────────────────────
719
+ addContainer(child, parent) {
720
+ let set = this._containBuf.get(child);
721
+ if (set === undefined) {
722
+ set = new Set();
723
+ this._containBuf.set(child, set);
1217
724
  }
1218
- }
1219
- if (isRoot) {
1220
- continue;
1221
- }
1222
- const g = this._pendingGist.get(id);
1223
- if (g === undefined) {
1224
- // Gist evicted from the bounded pending cache — the node can't be
1225
- // indexed now. Mark as covered so subsequent visits prune here
1226
- // instead of re-walking the subtree (the gist won't return).
1227
- this._coveredIds.set(id, true);
1228
- continue;
1229
- }
1230
- // Index unconditionally every interior node is a valid resonance
1231
- // anchor for partial-query recall. The _indexedIds cache and the
1232
- // durable-index check in indexGist prevent duplicate indexing across
1233
- // multiple encounters of the same shared subtree.
1234
- this.indexGist(id, false);
1235
- }
1236
- }
1237
- // ── Soft resonance ─────────────────────────────────────────────────────
1238
- async resonate(v, k) {
1239
- await this._ensureReady();
1240
- // Synchronous flush of any buffered index writes: the FIRST resonance
1241
- // after a large ingest pays that flush here, so it shows up in respond
1242
- // latency, not ingest latency — correct behaviour, skewed attribution;
1243
- // profile accordingly.
1244
- this.flushContent();
1245
- if (k <= 0) {
1246
- return [];
1247
- }
1248
- // ANN read cache — content-addressed so a fresh Float32Array with the
1249
- // same values still hits. Lazy-init: null after any index write; the
1250
- // first miss after a flush recreates it. When voteRegions resonates
1251
- // identical perceived sub-regions, only the first call descends the ANN.
1252
- const rk = vecKey(v) + ":" + k;
1253
- const cache = this._resonateCache;
1254
- if (cache) {
1255
- const hit = cache.get(rk);
1256
- if (hit !== undefined) {
1257
- return hit;
1258
- }
1259
- }
1260
- const results = this._vecContentQuery(
1261
- normalize(copy(v)),
1262
- k * this.overfetch,
1263
- this.efSearch,
1264
- );
1265
- const out = [];
1266
- for (const r of results) {
1267
- const id = r.id;
1268
- out.push({ id, score: 1 - r.distance });
1269
- if (out.length >= k) {
1270
- break;
1271
- }
1272
- }
1273
- const rc = this._resonateCache ??= new Map();
1274
- if (rc.size >= RESONATE_CACHE_MAX) {
1275
- rc.clear();
1276
- }
1277
- rc.set(rk, out);
1278
- return out;
1279
- }
1280
- indexedVectorCount() {
1281
- this.flushContent();
1282
- return this._vecContentSize();
1283
- }
1284
- lastResonateReads() {
1285
- return this._vecContentLastReads();
1286
- }
1287
- // ── Content index compaction ────────────────────────────────────────────
1288
- /** How many physical compaction attempts have failed this session. Zero in
1289
- * a healthy store; a growing count means tombstones are accumulating and
1290
- * index query cost is drifting up (the first failure also warns once). */
1291
- compactFailures = 0;
1292
- /** Meta key holding the incremental scan watermark of
1293
- * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1294
- * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1295
- * entry once kept can never become removable — and removed entries are
1296
- * gone, so a pass only ever needs to examine entries indexed AFTER the
1297
- * previous pass. Internal ids are monotone and survive the index's
1298
- * splice compaction; the watermark is reset whenever a PHYSICAL index
1299
- * compaction runs (id reuse after a dropped top row would otherwise hide
1300
- * new entries behind it). */
1301
- static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1302
- /** {@link Store.compactContentIndex} */
1303
- async compactContentIndex(minParents = 2) {
1304
- await this._ensureReady();
1305
- this.flush(); // commit any pending writes first
1306
- // Incremental scan: resume from the last pass's watermark when its
1307
- // minParents matches; a changed criterion forces a full rescan.
1308
- let after = 0;
1309
- {
1310
- const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1311
- if (raw !== null) {
1312
- const sep = raw.indexOf(":");
1313
- const mp = Number(raw.slice(0, sep));
1314
- const wm = Number(raw.slice(sep + 1));
1315
- if (mp === minParents && Number.isFinite(wm) && wm > 0) {
1316
- after = wm;
725
+ set.add(parent);
726
+ }
727
+ hasContainers(child) {
728
+ if (this._dbContainExists(child))
729
+ return true;
730
+ const buf = this._containBuf.get(child);
731
+ return buf !== undefined && buf.size > 0;
732
+ }
733
+ containersSlice(child, offset, limit) {
734
+ const out = this._dbGetContainParentsSlice(child, offset, limit);
735
+ if (out.length >= limit)
736
+ return out;
737
+ // Buffered adds page in AFTER the stored ones. A buffered parent that is
738
+ // also stored may repeat across the seam; page consumers dedup by id
739
+ // (they all carry seen-sets), so a repeat costs a skip, never an error.
740
+ const buf = this._containBuf.get(child);
741
+ if (!buf || buf.size === 0)
742
+ return out;
743
+ const storedCount = this._dbGetContainCount(child);
744
+ const bufStart = Math.max(0, offset - storedCount) +
745
+ Math.max(0, out.length - Math.max(0, storedCount - offset));
746
+ let i = 0;
747
+ for (const p of buf) {
748
+ if (out.length >= limit)
749
+ break;
750
+ if (i++ < bufStart)
751
+ continue;
752
+ out.push(p);
753
+ }
754
+ return out;
755
+ }
756
+ containers(child) {
757
+ const stored = this._dbGetContainParents(child);
758
+ const buf = this._containBuf.get(child);
759
+ if (stored.length === 0)
760
+ return buf ? [...buf] : [];
761
+ if (!buf)
762
+ return stored;
763
+ const merged = new Set(stored);
764
+ for (const p of buf)
765
+ merged.add(p);
766
+ return [...merged];
767
+ }
768
+ // ── Walk kids to collect implicit per-byte leaf ids ────────────────────
769
+ flatLeafIds(kids) {
770
+ const out = [];
771
+ const stack = [...kids].reverse();
772
+ while (stack.length > 0) {
773
+ const id = stack.pop();
774
+ if (id < 0) {
775
+ out.push(id);
776
+ continue;
777
+ }
778
+ const rec = this.get(id);
779
+ if (!rec)
780
+ return null;
781
+ if (rec.leaf !== null) {
782
+ for (let i = 0; i < rec.leaf.length; i++)
783
+ out.push(-(rec.leaf[i] + 1));
784
+ }
785
+ else if (rec.kids !== null) {
786
+ for (let i = rec.kids.length - 1; i >= 0; i--)
787
+ stack.push(rec.kids[i]);
788
+ }
789
+ }
790
+ return out;
791
+ }
792
+ /** On a dedup HIT, keep the node's gist available for lazy indexing —
793
+ * EXACTLY when it is not already indexed. Replaces the old id-range
794
+ * "recency" heuristic (id nextId − cacheWindow), which conflated an LRU
795
+ * entry COUNT with an id RANGE and permanently refused to index any node
796
+ * that first became a resonance target long after it was minted (an early
797
+ * interior later reused as an edge/halo-bearing deposit root was silently
798
+ * unreachable by resonance). The durable index itself is the arbiter:
799
+ * one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
800
+ captureIfUnindexed(id, gist) {
801
+ if (this._indexedIds.has(id) || this._pendingGist.has(id))
802
+ return;
803
+ if (this._vecContentHas(id)) {
804
+ this._indexedIds.set(id, true);
805
+ return;
1317
806
  }
1318
- }
1319
- }
1320
- // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1321
- // sources edge targets halo rows), materialised by a single C-side
1322
- // scan. A binary-search membership probe replaces the three per-entry
1323
- // point queries that made this stage minutes long on a trained store.
1324
- const targets = this._dbEdgeOrHaloIds();
1325
- const isTarget = (id) => {
1326
- let lo = 0, hi = targets.length - 1;
1327
- while (lo <= hi) {
1328
- const mid = (lo + hi) >> 1;
1329
- const v = targets[mid];
1330
- if (v === id) {
1331
- return true;
807
+ this._pendingGist.set(id, normalize(copy(gist)));
808
+ // A node that ALREADY bridges experiences but was never indexed (its 1→2
809
+ // transition fired while its gist was evicted, or in a pre-transition
810
+ // store) is promoted on this re-encounter the recapture above is
811
+ // exactly what makes its gist available again. Byte leaves (negative
812
+ // ids) never have kid rows, so skip their parent probe.
813
+ if (id >= 0)
814
+ this.promoteBridge(id);
815
+ }
816
+ /** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
817
+ * criterion), promote its gist into the content index NOW — the exact
818
+ * moment it becomes useful for multi-experience recall. The 1→2 parent
819
+ * transition fires on {@link _dbInsertKid} during mint, and nodes that
820
+ * were already bridges but missed indexing (gist evicted, pre-transition
821
+ * store) are recaptured in {@link captureIfUnindexed}.
822
+ *
823
+ * A no-op when the node is already indexed or its gist is evicted from
824
+ * the pending cache — a future re-encounter will retry. */
825
+ promoteBridge(id) {
826
+ // A LIMITed probe: this runs once per kid insert on the MINT hot path,
827
+ // and a shared child's full parent set grows with the corpus.
828
+ if (this._dbGetParentsFirst(id, 2).length < 2)
829
+ return;
830
+ this.indexGist(id, false);
831
+ }
832
+ // ── Core interning: dedup → near-dedup → mint ──────────────────────────
833
+ async intern(leaf, kids, gist) {
834
+ await this._ensureReady();
835
+ // 1. Exact dedup — equal content → one id, no vector work. Primary
836
+ // compression mechanism, intrinsic to the store.
837
+ //
838
+ // findLeaf/findBranch are the content-addressed lookups: in-memory cache
839
+ // first, then a durable probe that repopulates the cache on a hit.
840
+ // Using them here (rather than only the cache) makes dedup survive a cold
841
+ // cache — a resumed/checkpointed training run, or one whose dedup cache
842
+ // has evicted old keys, still recognises content already on disk and
843
+ // reuses its id instead of minting a duplicate.
844
+ const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
845
+ if (hit !== null) {
846
+ this.captureIfUnindexed(hit, gist);
847
+ return hit;
1332
848
  }
1333
- if (v < id) {
1334
- lo = mid + 1;
1335
- } else {
1336
- hi = mid - 1;
849
+ // 1b. Content-addressed lookup by leaf-id signature. When the same byte
850
+ // sequence was stored as a flat branch (via putBranch during deposit),
851
+ // a branch node spanning those bytes reuses that id even when its tree
852
+ // structure differs pure content addressing, same bytes → same node.
853
+ if (kids !== null && kids.length >= 2) {
854
+ const leafIds = this.flatLeafIds(kids);
855
+ if (leafIds !== null) {
856
+ const flatHit = this.findBranch(leafIds);
857
+ if (flatHit !== null) {
858
+ this.captureIfUnindexed(flatHit, gist);
859
+ return flatHit;
860
+ }
861
+ }
1337
862
  }
1338
- }
1339
- return false;
1340
- };
1341
- // Collect ids to remove: nodes that are structurally isolated (few
1342
- // parents), not edge-bearing, and not halo-bearing. These nodes are
1343
- // unique to one experience tree they bridge nothing and their index
1344
- // slots are wasted. Multi-parent nodes are the structural bridges
1345
- // that let a partial query climb from one experience to another.
1346
- const toRemove = [];
1347
- let scanned = 0;
1348
- let watermark = after;
1349
- for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1350
- // Yield a real event-loop turn on a fixed cadence: this scan runs over
1351
- // every unexamined indexed entry, and point probes alone pin the
1352
- // thread otherwise (the "frozen" symptom).
1353
- if (++scanned % 8192 === 0) {
1354
- await yieldToEventLoop();
1355
- }
1356
- watermark = internal;
1357
- if (isTarget(id)) {
1358
- continue;
1359
- }
1360
- if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1361
- continue;
1362
- }
1363
- toRemove.push(id);
1364
- }
1365
- // Delete in batches each batch is ONE vector-store transaction (one WAL
1366
- // commit), with an event-loop turn between batches.
1367
- const BATCH = 1000;
1368
- for (let i = 0; i < toRemove.length; i += BATCH) {
1369
- const batch = toRemove.slice(i, i + BATCH);
1370
- this._vecContentDeleteMany(batch);
1371
- // Purge the "already indexed" cache for removed ids: without this, a
1372
- // node that LATER becomes a resonance target again (gains an edge or
1373
- // halo) hits the stale cache entry in indexGist and is silently never
1374
- // re-indexed for the rest of the session.
1375
- for (const id of batch) {
1376
- this._indexedIds.delete(id);
1377
- }
1378
- await yieldToEventLoop();
1379
- }
1380
- // Persist the scan watermark so the next pass starts where this one
1381
- // ended (kept beside the criterion that produced it).
1382
- this._dbBeginTx();
1383
- this._dbSetMeta(
1384
- AbstractStore.COMPACT_WATERMARK_KEY,
1385
- minParents + ":" + watermark,
1386
- );
1387
- this._dbCommitTx();
1388
- if (toRemove.length > 0) {
1389
- // Compact to physically reclaim space from tombstones.
1390
- try {
1391
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1392
- this._vecContentCompact();
1393
- // Physical compaction may free the top internal id for reuse —
1394
- // drop the watermark so the next pass rescans from the start.
1395
- this._dbBeginTx();
1396
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1397
- this._dbCommitTx();
863
+ const cache = leaf !== null ? this._leafKey : this._branchKey;
864
+ const key = leaf !== null ? keyOf(leaf) : kids.join(",");
865
+ // 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
866
+ // Leaves are single bytes: exact dedup already collapses every identical
867
+ // leaf, and near-merging distinct leaves only corrupts bytes for no real
868
+ // saving. Real near-dedup compression lives in subtree (branch) fusion.
869
+ //
870
+ // There is deliberately NO ANN probe of the FLUSHED index here. It used
871
+ // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
872
+ // ~every interior branch, since interiors are never dedup targets —
873
+ // making one ANN query per branch the dominant training cost (it dwarfed
874
+ // perception and the index write). And it was not merely expensive but
875
+ // WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
876
+ // nearest "target" of a fresh branch, so the fold collapsed two
877
+ // byte-different subtrees onto one id and corrupted exact reconstruction
878
+ // (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
879
+ // are caught two cheaper, exact ways instead: identical content by the
880
+ // exact-dedup hash-cons above, and a near-gist target still in the write
881
+ // buffer by the scan below.
882
+ if (leaf === null) {
883
+ // Near-dedup PREFILTER — the scale-aware identity bar
884
+ // ({@link identityBar}) for THIS branch's own length, not the fixed
885
+ // estimator floor: under the linear fold a long branch crosses
886
+ // 1 − 1/√D while whole windows differ, and every such crossing paid a
887
+ // full differsByOneWindow byte reconstruction. Same final semantics
888
+ // (the byte check below still decides identity); strictly fewer byte
889
+ // reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
890
+ // memoized bottom-up by the interning order itself, O(kids)) is only
891
+ // computed once a nearest candidate actually exists, so the hot
892
+ // no-candidate mint pays nothing.
893
+ let best = -1;
894
+ let bestId = null;
895
+ if (this._nearDedupBuf.size > 0) {
896
+ const g = normalize(copy(gist));
897
+ // Candidates are the buffered DEDUP TARGETS only genuine whole
898
+ // experiences (edge/halo-bearing roots) not yet flushed.
899
+ for (const [id, vector] of this._nearDedupBuf) {
900
+ const s = dot(g, vector);
901
+ if (s > best) {
902
+ best = s;
903
+ bestId = id;
904
+ }
905
+ }
906
+ }
907
+ let blen = 0;
908
+ if (bestId !== null) {
909
+ for (const k of kids)
910
+ blen += this.contentLen(k);
911
+ }
912
+ if (bestId !== null &&
913
+ best >= identityBar(this.D, this._maxGroup, blen)) {
914
+ // Scale-aware acceptance. The cosine bar alone is scale-BLIND
915
+ // against a scale-DEPENDENT quantity: the hierarchical fold dilutes
916
+ // a localized difference faster than linearly in form size, so for
917
+ // deep forms ANY fixed bar below 1 is crossed by exactly the one
918
+ // span that distinguishes two experiences. No inversion of the
919
+ // deficit is trustworthy, so the bytes themselves decide: the two
920
+ // forms must be identical except for ONE local span of at most W
921
+ // bytes — the river window, the perception's own resolution quantum.
922
+ const W = this._maxGroup;
923
+ if (this.differsByOneWindow(kids, bestId, W)) {
924
+ if (key.length <= DEDUP_KEY_MAX)
925
+ cache.set(key, bestId);
926
+ return bestId;
927
+ }
928
+ }
1398
929
  }
1399
- } catch (e) {
1400
- // Best-effort, but never SILENT: a persistently failing compaction
1401
- // lets tombstones accumulate and query cost grow with no signal.
1402
- this.compactFailures++;
1403
- if (this.compactFailures === 1) {
1404
- console.warn(
1405
- "sema: content-index compaction failed (will keep " +
1406
- "counting in store.compactFailures):",
1407
- e,
1408
- );
930
+ // 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
931
+ // leaf) stores its BYTES in the leaf column with a zero-length kids blob
932
+ // as the marker the kid list is derived on read.
933
+ const id = this.mintId();
934
+ this._dbBeginTx();
935
+ const flat = kids ? flatKidsBytes(kids) : null;
936
+ const packed = kids && !flat ? packKids(kids) : null;
937
+ this._dbInsertNode(id, leaf ?? flat, packed ?? (flat ? _ZERO : null), hashOf(leaf ?? flat ?? packed));
938
+ // Reverse structural edge: each distinct child → this parent. Lets the graph
939
+ // be climbed upward in index time, with no scan of the kids blobs.
940
+ //
941
+ // Populated NATURALLY here and only here — one write per child, in the SAME
942
+ // mint that creates the node, inside the SAME deferred transaction as the
943
+ // node row, so node and kid rows are always durable together.
944
+ //
945
+ // Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
946
+ // every branch, so its parent set is the corpus-sized hub the climb's
947
+ // saturation guard discards unread.
948
+ if (kids) {
949
+ // Kid rows change parent counts — a child whose parent set grows from
950
+ // one is no longer transparent, so every cached chain that hopped
951
+ // through it is stale. Which chains those are is unknowable without a
952
+ // reverse index, so the WHOLE cache drops (writes come in training
953
+ // bursts where the cache is cold anyway; queries rebuild it lazily).
954
+ this._chainMemo.clear();
955
+ for (const c of kids) {
956
+ if (c < 0 && c >= -256)
957
+ continue;
958
+ this._dbInsertKid(c, id);
959
+ // The 1→2 parent TRANSITION happens here and only here (hash-cons
960
+ // means an existing branch never re-inserts kid rows, so parent sets
961
+ // grow exclusively through fresh mints): the child just became a
962
+ // structural bridge between experiences — the exact set post-hoc
963
+ // compaction keeps — so its gist enters the reach index NOW.
964
+ this.promoteBridge(c);
965
+ }
966
+ }
967
+ if (key.length <= DEDUP_KEY_MAX)
968
+ cache.set(key, id);
969
+ if (leaf)
970
+ this._bytesCache.set(id, new Uint8Array(leaf));
971
+ else if (flat)
972
+ this._bytesCache.set(id, flat);
973
+ // Capture the gist; it is pushed into the content index lazily, the first
974
+ // time this node becomes a resonance target (link / pourHalo). A node that
975
+ // never does (a pure intermediate DAG node — ~99.5% of them) is never
976
+ // indexed: it costs one persistence row, no vector-index slot and no merge probe.
977
+ this._pendingGist.set(id, normalize(copy(gist)));
978
+ await this.maybeFlush();
979
+ return id;
980
+ }
981
+ /** Whether the byte content under `kids` and the byte content of `targetId`
982
+ * are identical except for ONE local span of at most `W` bytes on each side
983
+ * — the near dedup's byte-grain definition of a near-duplicate. A
984
+ * common-prefix / common-suffix trim: whatever remains after both trims is
985
+ * the single differing span (substitution, insertion or deletion), and both
986
+ * remainders must fit the budget. Scattered differences leave a wide
987
+ * middle and are rejected. */
988
+ differsByOneWindow(kids, targetId, W) {
989
+ const a = concat(kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)));
990
+ const b = this.bytesPrefix(targetId, a.length + W + 1);
991
+ if (Math.abs(a.length - b.length) > W)
992
+ return false;
993
+ const n = Math.min(a.length, b.length);
994
+ let i = 0;
995
+ while (i < n && a[i] === b[i])
996
+ i++;
997
+ let j = 0;
998
+ while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j])
999
+ j++;
1000
+ return a.length - i - j <= W && b.length - i - j <= W;
1001
+ }
1002
+ async putLeaf(bytes, gist) {
1003
+ // Single bytes are implicit — no DB row and no eager index slot. The gist
1004
+ // is captured like any other node's and promoted into the content index
1005
+ // LAZILY, the first time the byte becomes a resonance target.
1006
+ if (bytes.length === 1) {
1007
+ const id = -(bytes[0] + 1);
1008
+ this.captureIfUnindexed(id, gist);
1009
+ return id;
1010
+ }
1011
+ return this.intern(new Uint8Array(bytes), null, gist);
1012
+ }
1013
+ async putBranch(kids, gist) {
1014
+ return this.intern(null, kids, gist);
1015
+ }
1016
+ // ── Lazy content indexing ──────────────────────────────────────────────
1017
+ /** Promote a node's captured gist into the content (resonance) index, once.
1018
+ * Called the first time a node becomes a target — i.e. from `link` (it bears
1019
+ * or receives a continuation edge) or `pourHalo` (it gains distributional
1020
+ * company). Idempotent: a node already indexed, or whose gist has been evicted
1021
+ * from the bounded pending map, is a no-op.
1022
+ *
1023
+ * `dedupTarget` marks the node a candidate the near dedup may fold a fresh
1024
+ * near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
1025
+ * is one; a climb-only interior is reach-indexed but never a dedup sink. */
1026
+ indexGist(id, dedupTarget) {
1027
+ if (dedupTarget && this._bufferedIds.has(id)) {
1028
+ // Still buffered (indexed this batch, not yet flushed) — a live dedup
1029
+ // candidate, recorded in O(1) with no scan of the content buffer.
1030
+ const v = this._pendingGist.get(id);
1031
+ if (v !== undefined)
1032
+ this._nearDedupBuf.set(id, v);
1033
+ }
1034
+ if (this._indexedIds.has(id))
1035
+ return;
1036
+ const v = this._pendingGist.get(id);
1037
+ if (v === undefined)
1038
+ return;
1039
+ // Already durably indexed by a previous session? A node id names its
1040
+ // content, and the gist is a pure function of the content, so the stored
1041
+ // vector can only be identical — re-buffering it would spend an encode
1042
+ // and an upsert on a guaranteed no-op. One point query recognises this;
1043
+ // it costs ~nothing for genuinely new nodes (a miss on a covering index).
1044
+ // This is what makes a RESUMED training run replay already-deposited
1045
+ // content at read speed instead of re-upserting the recent-id window.
1046
+ if (this._vecContentHas(id)) {
1047
+ this._indexedIds.set(id, true);
1048
+ return;
1409
1049
  }
1410
- }
1411
- // Index mutations (delete, compact) invalidate the ANN read cache.
1412
- this._resonateCache = null;
1413
- }
1414
- return toRemove.length;
1415
- }
1416
- /** {@link Store.repairContentIndex} */
1417
- async repairContentIndex(regenerateGist, minParents = 2) {
1418
- await this._ensureReady();
1419
- this.flush(); // commit any pending writes first
1420
- if (this._nextId === 0) {
1421
- return 0;
1422
- }
1423
- let added = 0;
1424
- // A repairable node MUST carry edges or a halo, so the candidate set IS
1425
- // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1426
- // thousands), not node-count-sized (tens of millions). The old walk
1427
- // visited EVERY branch id ever minted with a parent probe each; driving
1428
- // from the target set visits only real candidates, in the same ascending
1429
- // id order (the set is sorted), so the result and its ordering are
1430
- // identical. Byte leaves (negative ids) were never visited by the old
1431
- // walk and are skipped here too.
1432
- const targets = this._dbEdgeOrHaloIds();
1433
- let scanned = 0;
1434
- for (const id of targets) {
1435
- // Yield a real event-loop turn on a fixed cadence so the scan never
1436
- // pins the thread for its whole duration.
1437
- if (++scanned % 8192 === 0) {
1438
- await yieldToEventLoop();
1439
- }
1440
- if (id < 0) {
1441
- continue; // byte leaves: implicit, never repaired
1442
- }
1443
- // Already indexed in memory — skip.
1444
- if (this._indexedIds.has(id)) {
1445
- continue;
1446
- }
1447
- // Must be a structural bridge: ≥ minParents parents in the DAG.
1448
- // One LIMITed probe per candidate.
1449
- if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1450
- continue;
1451
- }
1452
- // Already durably indexed by a previous session — record and skip.
1453
- // Probed AFTER the structural filters: candidates are few, so this
1454
- // point query runs rarely instead of once per node in the store.
1455
- if (this._vecContentHas(id)) {
1456
1050
  this._indexedIds.set(id, true);
1457
- continue;
1458
- }
1459
- // Regenerate the gist from bytes. The callback is async (the Mind's
1460
- // perception is synchronous, but the interface allows a disk-backed
1461
- // regenerator that yields).
1462
- const gist = await regenerateGist(id);
1463
- if (!gist) {
1464
- continue;
1465
- }
1466
- // Index it — same code path as indexGist, but the vector is injected
1467
- // directly rather than read from the (empty) pending-gist cache.
1468
- this._indexedIds.set(id, true);
1469
- this._contentBuffer.push({
1470
- id,
1471
- vector: gist,
1472
- ef: this.efConstructionInterior,
1473
- });
1474
- this._bufferedIds.add(id);
1475
- // Repaired nodes are reach-indexed, never dedup targets: their gist is
1476
- // regenerated (may differ numerically from the original) and they are
1477
- // interiors, not deposit roots.
1478
- added++;
1479
- // Flush periodically to bound the write-transaction size and yield to
1480
- // the event loop, same cadence compactContentIndex uses.
1481
- if (added % 1000 === 0) {
1051
+ this._contentBuffer.push({ id, vector: v });
1052
+ this._bufferedIds.add(id);
1053
+ // A node indexed AS a dedup target enters the candidate set immediately.
1054
+ if (dedupTarget)
1055
+ this._nearDedupBuf.set(id, v);
1056
+ }
1057
+ /** {@link Store.indexTarget} — the public hook for marking a deposit root a
1058
+ * resonance target, the one target `link`/`pourHalo` do not cover. A deposit
1059
+ * root is a genuine target (a whole experience), so it is a dedup target
1060
+ * too. */
1061
+ indexTarget(id) {
1062
+ this.indexGist(id, true);
1063
+ }
1064
+ /** Index a node and its interior forms as resonance targets. A node that
1065
+ * gains an edge is a learnt EXPERIENCE, and the consensus climb
1066
+ * ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
1067
+ * resonating its SUB-REGIONS — branch nodes within the experience — and
1068
+ * climbing their parents back to it.
1069
+ *
1070
+ * EVERY interior branch is indexed unconditionally, and this is
1071
+ * LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
1072
+ * the post-hoc compaction criterion) was tried and REJECTED by the test
1073
+ * suite partial recall of an experience's interior slices, multi-topic
1074
+ * attention, and counterfactual anchoring all resonate to SINGLE-parent
1075
+ * interiors (13 tests fail without them). Post-hoc structural compaction
1076
+ * ({@link compactContentIndex}) may still remove them, but that is a
1077
+ * storage/recall trade-off for archived stores, not a free optimisation.
1078
+ * The store's hash-cons bounds the index by the number of DISTINCT byte
1079
+ * patterns in the corpus — not by the number of deposits.
1080
+ *
1081
+ * Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
1082
+ * branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
1083
+ * let a partial query resonate and climb, but a fresh branch must never
1084
+ * merge onto an interior node of another experience.
1085
+ *
1086
+ * Iterative explicit-queue walk: the call stack never sees tree depth. */
1087
+ indexSubtree(root) {
1088
+ // The root is the whole experience — always index as a merge target.
1089
+ this.indexGist(root, true);
1090
+ const seen = new Set([root]);
1091
+ const stack = [root];
1092
+ while (stack.length > 0) {
1093
+ const id = stack.pop();
1094
+ const kids = this.get(id)?.kids;
1095
+ if (!kids)
1096
+ continue; // leaf — never a resonance anchor
1097
+ const isRoot = id === root;
1098
+ // Already-indexed or already-skipped nodes PRUNE the walk — their
1099
+ // subtrees were classified in a prior call.
1100
+ if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
1101
+ continue;
1102
+ }
1103
+ for (const k of kids) {
1104
+ if (!seen.has(k)) {
1105
+ seen.add(k);
1106
+ stack.push(k);
1107
+ }
1108
+ }
1109
+ if (isRoot)
1110
+ continue;
1111
+ const g = this._pendingGist.get(id);
1112
+ if (g === undefined) {
1113
+ // Gist evicted from the bounded pending cache — the node can't be
1114
+ // indexed now. Mark as covered so subsequent visits prune here
1115
+ // instead of re-walking the subtree (the gist won't return).
1116
+ this._coveredIds.set(id, true);
1117
+ continue;
1118
+ }
1119
+ // Index unconditionally — every interior node is a valid resonance
1120
+ // anchor for partial-query recall. The _indexedIds cache and the
1121
+ // durable-index check in indexGist prevent duplicate indexing across
1122
+ // multiple encounters of the same shared subtree.
1123
+ this.indexGist(id, false);
1124
+ }
1125
+ }
1126
+ // ── Soft resonance ─────────────────────────────────────────────────────
1127
+ async resonate(v, k) {
1128
+ await this._ensureReady();
1129
+ // Synchronous flush of any buffered index writes: the FIRST resonance
1130
+ // after a large ingest pays that flush here, so it shows up in respond
1131
+ // latency, not ingest latency — correct behaviour, skewed attribution;
1132
+ // profile accordingly.
1133
+ this.flushContent();
1134
+ if (k <= 0)
1135
+ return [];
1136
+ // ANN read cache — content-addressed so a fresh Float32Array with the
1137
+ // same values still hits. Lazy-init: null after any index write; the
1138
+ // first miss after a flush recreates it. When voteRegions resonates
1139
+ // identical perceived sub-regions, only the first call descends the ANN.
1140
+ const rk = vecKey(v) + ":" + k;
1141
+ const cache = this._resonateCache;
1142
+ if (cache) {
1143
+ const hit = cache.get(rk);
1144
+ if (hit !== undefined)
1145
+ return hit;
1146
+ }
1147
+ const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efSearch);
1148
+ const out = [];
1149
+ for (const r of results) {
1150
+ const id = r.id;
1151
+ out.push({ id, score: 1 - r.distance });
1152
+ if (out.length >= k)
1153
+ break;
1154
+ }
1155
+ const rc = this._resonateCache ??= new Map();
1156
+ if (rc.size >= RESONATE_CACHE_MAX)
1157
+ rc.clear();
1158
+ rc.set(rk, out);
1159
+ return out;
1160
+ }
1161
+ indexedVectorCount() {
1162
+ this.flushContent();
1163
+ return this._vecContentSize();
1164
+ }
1165
+ lastResonateReads() {
1166
+ return this._vecContentLastReads();
1167
+ }
1168
+ // ── Content index compaction ────────────────────────────────────────────
1169
+ /** How many physical compaction attempts have failed this session. Zero in
1170
+ * a healthy store; a growing count means tombstones are accumulating and
1171
+ * index query cost is drifting up (the first failure also warns once). */
1172
+ compactFailures = 0;
1173
+ /** Meta key holding the incremental scan watermark of
1174
+ * {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
1175
+ * decisions are MONOTONE — parents, edges and halos only ever grow, so an
1176
+ * entry once kept can never become removable — and removed entries are
1177
+ * gone, so a pass only ever needs to examine entries indexed AFTER the
1178
+ * previous pass. Internal ids are monotone and survive the index's
1179
+ * splice compaction; the watermark is reset whenever a PHYSICAL index
1180
+ * compaction runs (id reuse after a dropped top row would otherwise hide
1181
+ * new entries behind it). */
1182
+ static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
1183
+ /** {@link Store.compactContentIndex} */
1184
+ async compactContentIndex(minParents = 2) {
1185
+ await this._ensureReady();
1186
+ this.flush(); // commit any pending writes first
1187
+ // Incremental scan: resume from the last pass's watermark when its
1188
+ // minParents matches; a changed criterion forces a full rescan.
1189
+ let after = 0;
1190
+ {
1191
+ const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1192
+ if (raw !== null) {
1193
+ const sep = raw.indexOf(":");
1194
+ const mp = Number(raw.slice(0, sep));
1195
+ const wm = Number(raw.slice(sep + 1));
1196
+ if (mp === minParents && Number.isFinite(wm) && wm > 0)
1197
+ after = wm;
1198
+ }
1199
+ }
1200
+ // The keep criterion "has edges or a halo" as ONE sorted id set (edge
1201
+ // sources ∪ edge targets ∪ halo rows), materialised by a single C-side
1202
+ // scan. A binary-search membership probe replaces the three per-entry
1203
+ // point queries that made this stage minutes long on a trained store.
1204
+ const targets = this._dbEdgeOrHaloIds();
1205
+ const isTarget = (id) => {
1206
+ let lo = 0, hi = targets.length - 1;
1207
+ while (lo <= hi) {
1208
+ const mid = (lo + hi) >> 1;
1209
+ const v = targets[mid];
1210
+ if (v === id)
1211
+ return true;
1212
+ if (v < id)
1213
+ lo = mid + 1;
1214
+ else
1215
+ hi = mid - 1;
1216
+ }
1217
+ return false;
1218
+ };
1219
+ // Collect ids to remove: nodes that are structurally isolated (few
1220
+ // parents), not edge-bearing, and not halo-bearing. These nodes are
1221
+ // unique to one experience tree — they bridge nothing and their index
1222
+ // slots are wasted. Multi-parent nodes are the structural bridges
1223
+ // that let a partial query climb from one experience to another.
1224
+ const toRemove = [];
1225
+ let scanned = 0;
1226
+ let watermark = after;
1227
+ for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
1228
+ // Yield a real event-loop turn on a fixed cadence: this scan runs over
1229
+ // every unexamined indexed entry, and point probes alone pin the
1230
+ // thread otherwise (the "frozen" symptom).
1231
+ if (++scanned % 8192 === 0)
1232
+ await yieldToEventLoop();
1233
+ watermark = internal;
1234
+ if (isTarget(id))
1235
+ continue;
1236
+ if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
1237
+ continue;
1238
+ }
1239
+ toRemove.push(id);
1240
+ }
1241
+ // Delete in batches — each batch is ONE vector-store transaction (one WAL
1242
+ // commit), with an event-loop turn between batches.
1243
+ const BATCH = 1000;
1244
+ for (let i = 0; i < toRemove.length; i += BATCH) {
1245
+ const batch = toRemove.slice(i, i + BATCH);
1246
+ this._vecContentDeleteMany(batch);
1247
+ // Purge the "already indexed" cache for removed ids: without this, a
1248
+ // node that LATER becomes a resonance target again (gains an edge or
1249
+ // halo) hits the stale cache entry in indexGist and is silently never
1250
+ // re-indexed for the rest of the session.
1251
+ for (const id of batch)
1252
+ this._indexedIds.delete(id);
1253
+ await yieldToEventLoop();
1254
+ }
1255
+ // Persist the scan watermark so the next pass starts where this one
1256
+ // ended (kept beside the criterion that produced it).
1257
+ this._dbBeginTx();
1258
+ this._dbSetMeta(AbstractStore.COMPACT_WATERMARK_KEY, minParents + ":" + watermark);
1259
+ this._dbCommitTx();
1260
+ if (toRemove.length > 0) {
1261
+ // Compact to physically reclaim space from tombstones.
1262
+ try {
1263
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
1264
+ this._vecContentCompact();
1265
+ // Physical compaction may free the top internal id for reuse —
1266
+ // drop the watermark so the next pass rescans from the start.
1267
+ this._dbBeginTx();
1268
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1269
+ this._dbCommitTx();
1270
+ }
1271
+ }
1272
+ catch (e) {
1273
+ // Best-effort, but never SILENT: a persistently failing compaction
1274
+ // lets tombstones accumulate and query cost grow with no signal.
1275
+ this.compactFailures++;
1276
+ if (this.compactFailures === 1) {
1277
+ console.warn("sema: content-index compaction failed (will keep " +
1278
+ "counting in store.compactFailures):", e);
1279
+ }
1280
+ }
1281
+ // Index mutations (delete, compact) invalidate the ANN read cache.
1282
+ this._resonateCache = null;
1283
+ }
1284
+ return toRemove.length;
1285
+ }
1286
+ /** {@link Store.repairContentIndex} */
1287
+ async repairContentIndex(regenerateGist, minParents = 2) {
1288
+ await this._ensureReady();
1289
+ this.flush(); // commit any pending writes first
1290
+ if (this._nextId === 0)
1291
+ return 0;
1292
+ let added = 0;
1293
+ // A repairable node MUST carry edges or a halo, so the candidate set IS
1294
+ // the edge/halo id set — corpus-of-experiences-sized (hundreds of
1295
+ // thousands), not node-count-sized (tens of millions). The old walk
1296
+ // visited EVERY branch id ever minted with a parent probe each; driving
1297
+ // from the target set visits only real candidates, in the same ascending
1298
+ // id order (the set is sorted), so the result and its ordering are
1299
+ // identical. Byte leaves (negative ids) were never visited by the old
1300
+ // walk and are skipped here too.
1301
+ const targets = this._dbEdgeOrHaloIds();
1302
+ let scanned = 0;
1303
+ for (const id of targets) {
1304
+ // Yield a real event-loop turn on a fixed cadence so the scan never
1305
+ // pins the thread for its whole duration.
1306
+ if (++scanned % 8192 === 0)
1307
+ await yieldToEventLoop();
1308
+ if (id < 0)
1309
+ continue; // byte leaves: implicit, never repaired
1310
+ // Already indexed in memory — skip.
1311
+ if (this._indexedIds.has(id))
1312
+ continue;
1313
+ // Must be a structural bridge: ≥ minParents parents in the DAG.
1314
+ // One LIMITed probe per candidate.
1315
+ if (this._dbGetParentsFirst(id, minParents).length < minParents) {
1316
+ continue;
1317
+ }
1318
+ // Already durably indexed by a previous session — record and skip.
1319
+ // Probed AFTER the structural filters: candidates are few, so this
1320
+ // point query runs rarely instead of once per node in the store.
1321
+ if (this._vecContentHas(id)) {
1322
+ this._indexedIds.set(id, true);
1323
+ continue;
1324
+ }
1325
+ // Regenerate the gist from bytes. The callback is async (the Mind's
1326
+ // perception is synchronous, but the interface allows a disk-backed
1327
+ // regenerator that yields).
1328
+ const gist = await regenerateGist(id);
1329
+ if (!gist)
1330
+ continue;
1331
+ // Index it — same code path as indexGist, but the vector is injected
1332
+ // directly rather than read from the (empty) pending-gist cache.
1333
+ this._indexedIds.set(id, true);
1334
+ this._contentBuffer.push({ id, vector: gist });
1335
+ this._bufferedIds.add(id);
1336
+ // Repaired nodes are reach-indexed, never dedup targets: their gist is
1337
+ // regenerated (may differ numerically from the original) and they are
1338
+ // interiors, not deposit roots.
1339
+ added++;
1340
+ // Flush periodically to bound the write-transaction size and yield to
1341
+ // the event loop, same cadence compactContentIndex uses.
1342
+ if (added % 1000 === 0)
1343
+ await this.maybeFlush();
1344
+ }
1345
+ // Final flush for the last partial batch.
1346
+ if (added > 0)
1347
+ await this.maybeFlush();
1348
+ return added;
1349
+ }
1350
+ // ── Continuation edges ─────────────────────────────────────────────────
1351
+ async link(from, to) {
1352
+ await this._ensureReady();
1353
+ // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1354
+ // indexed, because the seat is symmetric — a query may name only a PORTION of
1355
+ // either side and must still resonate to its interior and climb to the whole.
1356
+ // Only each ROOT is a MERGE target; the interiors are reach-only.
1357
+ this.indexSubtree(from);
1358
+ this.indexSubtree(to);
1359
+ // Flush the vectors indexSubtree just added to the buffer.
1360
+ // This keeps the buffer bounded and yields to the event loop.
1482
1361
  await this.maybeFlush();
1483
- }
1484
- }
1485
- // Final flush for the last partial batch.
1486
- if (added > 0) {
1487
- await this.maybeFlush();
1488
- }
1489
- return added;
1490
- }
1491
- // ── Continuation edges ─────────────────────────────────────────────────
1492
- async link(from, to) {
1493
- await this._ensureReady();
1494
- // Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
1495
- // indexed, because the seat is symmetric — a query may name only a PORTION of
1496
- // either side and must still resonate to its interior and climb to the whole.
1497
- // Only each ROOT is a MERGE target; the interiors are reach-only.
1498
- this.indexSubtree(from);
1499
- this.indexSubtree(to);
1500
- // Flush the vectors indexSubtree just added to the buffer.
1501
- // This keeps the buffer bounded and yields to the event loop.
1502
- await this.maybeFlush();
1503
- this._dbBeginTx();
1504
- // Keep the document count exact as it grows: a source gaining its FIRST
1505
- // edge is one new learnt context. One indexed point probe, and only once
1506
- // the count has been materialised — before that, the lazy full count in
1507
- // {@link edgeSourceCount} will see this edge anyway.
1508
- if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1509
- this._edgeSrcCount++;
1510
- }
1511
- // An edge breaks the transparency of both endpoints — drop cached chains
1512
- // (same reasoning as the kid-row invalidation in put).
1513
- this._chainMemo.clear();
1514
- this._dbInsertEdge(from, to);
1515
- }
1516
- next(id) {
1517
- return this._dbGetNextEdges(id);
1518
- }
1519
- /** {@link Store.hasNext} one indexed point probe, never a range read. */
1520
- hasNext(id) {
1521
- return this._dbEdgeSrcExists(id);
1522
- }
1523
- prev(id) {
1524
- return this._dbGetPrevEdges(id);
1525
- }
1526
- nextFirst(id, limit) {
1527
- return this._dbGetNextEdgesFirst(id, limit);
1528
- }
1529
- prevFirst(id, limit) {
1530
- return this._dbGetPrevEdgesFirst(id, limit);
1531
- }
1532
- /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1533
- * should override; this default materialises (correct, not optimal). */
1534
- prevCount(id) {
1535
- return this._dbGetPrevEdges(id).length;
1536
- }
1537
- edgeSourceCount() {
1538
- // Materialised once per session (edges written before this moment are
1539
- // covered by the full count), then kept exact incrementally by link().
1540
- // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1541
- // detect staleness — O(edges) on the recall hot path, at every IDF read.
1542
- if (this._edgeSrcCount < 0) {
1543
- this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1544
- }
1545
- return this._edgeSrcCount;
1546
- }
1547
- // ── Halos ──────────────────────────────────────────────────────────────
1548
- haloMass(id) {
1549
- const r = this._dbGetHalo(id);
1550
- return r ? r.mass : 0;
1551
- }
1552
- halo(id) {
1553
- const cached = this._haloNorm.get(id);
1554
- if (cached !== undefined) {
1555
- return copy(cached);
1556
- }
1557
- const r = this._dbGetHalo(id);
1558
- if (!r || r.mass < this.minHaloMass) {
1559
- return null;
1560
- }
1561
- const exact = this._haloExact.get(id);
1562
- const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1563
- this._haloNorm.set(id, v);
1564
- return copy(v);
1565
- }
1566
- /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1567
- * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1568
- hasHalo(id) {
1569
- const r = this._dbGetHalo(id);
1570
- return r !== null && r.mass >= this.minHaloMass;
1571
- }
1572
- async pourHalo(id, add) {
1573
- await this._ensureReady();
1574
- // A node with a halo is a genuine resonance target — the consensus climb
1575
- // resonates a query region to it, and articulation reads its halo.
1576
- this.indexGist(id, true);
1577
- const r = this._dbGetHalo(id);
1578
- const acc = this._haloExact.get(id) ??
1579
- (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1580
- addInto(acc, add);
1581
- this._haloExact.set(id, acc);
1582
- this._haloNorm.delete(id); // the normalized read cache is now stale
1583
- const mass = (r?.mass ?? 0) + 1;
1584
- this._dbBeginTx();
1585
- this._dbUpsertHalo(id, haloEncode(acc), mass);
1586
- // Re-index on a geometric schedule only (the exact halo is persisted).
1587
- if (mass >= this.minHaloMass && geometricMass(mass)) {
1588
- this._haloBuffer.set(id, normalize(copy(acc)));
1589
- await this.maybeFlush();
1590
- }
1591
- }
1592
- async resonateHalo(v, k) {
1593
- await this._ensureReady();
1594
- this.flushHalos();
1595
- if (k <= 0) {
1596
- return [];
1597
- }
1598
- // ANN read cache — same scheme as resonate's, but for the halo index.
1599
- const rk = vecKey(v) + ":" + k;
1600
- const cache = this._resonateHaloCache;
1601
- if (cache) {
1602
- const hit = cache.get(rk);
1603
- if (hit !== undefined) {
1604
- return hit;
1605
- }
1606
- }
1607
- const results = this._vecHaloQuery(
1608
- normalize(copy(v)),
1609
- k * this.overfetch,
1610
- this.efSearch,
1611
- );
1612
- const out = [];
1613
- for (const r of results) {
1614
- const id = r.id;
1615
- out.push({ id, score: 1 - r.distance });
1616
- if (out.length >= k) {
1617
- break;
1618
- }
1619
- }
1620
- const rhc = this._resonateHaloCache ??= new Map();
1621
- if (rhc.size >= RESONATE_CACHE_MAX) {
1622
- rhc.clear();
1623
- }
1624
- rhc.set(rk, out);
1625
- return out;
1626
- }
1627
- // ── Buffering, flushing, compaction ────────────────────────────────────
1628
- pending() {
1629
- return this._contentBuffer.length + this._haloBuffer.size;
1630
- }
1631
- flushContent() {
1632
- if (this._contentBuffer.length === 0) {
1633
- return 0;
1634
- }
1635
- const batch = this._contentBuffer.splice(0);
1636
- // The merge scan only consults UNFLUSHED candidates, so clear them in
1637
- // lockstep with the content buffer they mirror.
1638
- this._nearDedupBuf.clear();
1639
- this._bufferedIds.clear();
1640
- this._vecContentUpsert(batch);
1641
- this._resonateCache = null;
1642
- return batch.length;
1643
- }
1644
- flushHalos() {
1645
- if (this._haloBuffer.size === 0) {
1646
- return 0;
1647
- }
1648
- const batch = [...this._haloBuffer.entries()];
1649
- this._haloBuffer.clear();
1650
- this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1651
- this._resonateHaloCache = null;
1652
- return batch.length;
1653
- }
1654
- /** Append the buffered containment pairs, inside the deferred transaction.
1655
- * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1656
- * flush never reads a child's stored list back — the old packed-blob
1657
- * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1658
- * over a long training run on a hot window. */
1659
- flushContain() {
1660
- if (this._containBuf.size === 0) {
1661
- return;
1662
- }
1663
- this._dbBeginTx();
1664
- for (const [child, set] of this._containBuf) {
1665
- this._dbAppendContain(child, [...set]);
1666
- }
1667
- this._containBuf.clear();
1668
- }
1669
- /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1670
- flush() {
1671
- this.flushContain();
1672
- const written = this.flushContent() + this.flushHalos();
1673
- // Commit the deferred write transaction on the same cadence as the vector
1674
- // buffers, so node rows / edges / halos become durable in coalesced batches.
1675
- this._dbCommitTx();
1676
- if (written === 0) {
1677
- return;
1678
- }
1679
- this._writtenSinceCompact += written;
1680
- if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1681
- this._writtenSinceCompact = 0;
1682
- try {
1683
- if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1684
- this._vecContentCompact();
1685
- // Physical compaction may free the top internal id for reuse —
1686
- // invalidate the incremental maintenance watermark.
1687
- this._dbBeginTx();
1688
- this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1689
- this._dbCommitTx();
1362
+ this._dbBeginTx();
1363
+ // Keep the document count exact as it grows: a source gaining its FIRST
1364
+ // edge is one new learnt context. One indexed point probe, and only once
1365
+ // the count has been materialised — before that, the lazy full count in
1366
+ // {@link edgeSourceCount} will see this edge anyway.
1367
+ if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
1368
+ this._edgeSrcCount++;
1369
+ }
1370
+ // An edge breaks the transparency of both endpoints — drop cached chains
1371
+ // (same reasoning as the kid-row invalidation in put).
1372
+ this._chainMemo.clear();
1373
+ this._dbInsertEdge(from, to);
1374
+ }
1375
+ next(id) {
1376
+ return this._dbGetNextEdges(id);
1377
+ }
1378
+ /** {@link Store.hasNext} — one indexed point probe, never a range read. */
1379
+ hasNext(id) {
1380
+ return this._dbEdgeSrcExists(id);
1381
+ }
1382
+ prev(id) {
1383
+ return this._dbGetPrevEdges(id);
1384
+ }
1385
+ nextFirst(id, limit) {
1386
+ return this._dbGetNextEdgesFirst(id, limit);
1387
+ }
1388
+ prevFirst(id, limit) {
1389
+ return this._dbGetPrevEdgesFirst(id, limit);
1390
+ }
1391
+ /** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
1392
+ * should override; this default materialises (correct, not optimal). */
1393
+ prevCount(id) {
1394
+ return this._dbGetPrevEdges(id).length;
1395
+ }
1396
+ edgeSourceCount() {
1397
+ // Materialised once per session (edges written before this moment are
1398
+ // covered by the full count), then kept exact incrementally by link().
1399
+ // The old form re-ran a full COUNT(*) table scan on EVERY call just to
1400
+ // detect staleness — O(edges) on the recall hot path, at every IDF read.
1401
+ if (this._edgeSrcCount < 0) {
1402
+ this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
1403
+ }
1404
+ return this._edgeSrcCount;
1405
+ }
1406
+ // ── Halos ──────────────────────────────────────────────────────────────
1407
+ haloMass(id) {
1408
+ const r = this._dbGetHalo(id);
1409
+ return r ? r.mass : 0;
1410
+ }
1411
+ halo(id) {
1412
+ const cached = this._haloNorm.get(id);
1413
+ if (cached !== undefined)
1414
+ return copy(cached);
1415
+ const r = this._dbGetHalo(id);
1416
+ if (!r || r.mass < this.minHaloMass)
1417
+ return null;
1418
+ const exact = this._haloExact.get(id);
1419
+ const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
1420
+ this._haloNorm.set(id, v);
1421
+ return copy(v);
1422
+ }
1423
+ /** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
1424
+ * exactly (row present AND mass ≥ minHaloMass), minus the decode. */
1425
+ hasHalo(id) {
1426
+ const r = this._dbGetHalo(id);
1427
+ return r !== null && r.mass >= this.minHaloMass;
1428
+ }
1429
+ async pourHalo(id, add) {
1430
+ await this._ensureReady();
1431
+ // A node with a halo is a genuine resonance target — the consensus climb
1432
+ // resonates a query region to it, and articulation reads its halo.
1433
+ this.indexGist(id, true);
1434
+ const r = this._dbGetHalo(id);
1435
+ const acc = this._haloExact.get(id) ??
1436
+ (r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
1437
+ addInto(acc, add);
1438
+ this._haloExact.set(id, acc);
1439
+ this._haloNorm.delete(id); // the normalized read cache is now stale
1440
+ const mass = (r?.mass ?? 0) + 1;
1441
+ this._dbBeginTx();
1442
+ this._dbUpsertHalo(id, haloEncode(acc), mass);
1443
+ // Re-index on a geometric schedule only (the exact halo is persisted).
1444
+ if (mass >= this.minHaloMass && geometricMass(mass)) {
1445
+ this._haloBuffer.set(id, normalize(copy(acc)));
1446
+ await this.maybeFlush();
1447
+ }
1448
+ }
1449
+ async resonateHalo(v, k) {
1450
+ await this._ensureReady();
1451
+ this.flushHalos();
1452
+ if (k <= 0)
1453
+ return [];
1454
+ // ANN read cache same scheme as resonate's, but for the halo index.
1455
+ const rk = vecKey(v) + ":" + k;
1456
+ const cache = this._resonateHaloCache;
1457
+ if (cache) {
1458
+ const hit = cache.get(rk);
1459
+ if (hit !== undefined)
1460
+ return hit;
1690
1461
  }
1691
- if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1692
- this._vecHaloCompact();
1462
+ const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.efSearch);
1463
+ const out = [];
1464
+ for (const r of results) {
1465
+ const id = r.id;
1466
+ out.push({ id, score: 1 - r.distance });
1467
+ if (out.length >= k)
1468
+ break;
1693
1469
  }
1694
- } catch (e) {
1695
- // Best-effort, but never SILENT (same contract as the prune-path
1696
- // compaction above): a persistently failing compaction lets
1697
- // tombstones accumulate and query cost grow with no signal.
1698
- this.compactFailures++;
1699
- if (this.compactFailures === 1) {
1700
- console.warn(
1701
- "sema: vector-index compaction failed (will keep " +
1702
- "counting in store.compactFailures):",
1703
- e,
1704
- );
1470
+ const rhc = this._resonateHaloCache ??= new Map();
1471
+ if (rhc.size >= RESONATE_CACHE_MAX)
1472
+ rhc.clear();
1473
+ rhc.set(rk, out);
1474
+ return out;
1475
+ }
1476
+ // ── Buffering, flushing, compaction ────────────────────────────────────
1477
+ pending() {
1478
+ return this._contentBuffer.length + this._haloBuffer.size;
1479
+ }
1480
+ flushContent() {
1481
+ if (this._contentBuffer.length === 0)
1482
+ return 0;
1483
+ const batch = this._contentBuffer.splice(0);
1484
+ // The merge scan only consults UNFLUSHED candidates, so clear them in
1485
+ // lockstep with the content buffer they mirror.
1486
+ this._nearDedupBuf.clear();
1487
+ this._bufferedIds.clear();
1488
+ this._vecContentUpsert(batch);
1489
+ this._resonateCache = null;
1490
+ return batch.length;
1491
+ }
1492
+ flushHalos() {
1493
+ if (this._haloBuffer.size === 0)
1494
+ return 0;
1495
+ const batch = [...this._haloBuffer.entries()];
1496
+ this._haloBuffer.clear();
1497
+ this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
1498
+ this._resonateHaloCache = null;
1499
+ return batch.length;
1500
+ }
1501
+ /** Append the buffered containment pairs, inside the deferred transaction.
1502
+ * Pure appends: durable dedup lives in the adapter (the pair PK), so a
1503
+ * flush never reads a child's stored list back — the old packed-blob
1504
+ * read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
1505
+ * over a long training run on a hot window. */
1506
+ flushContain() {
1507
+ if (this._containBuf.size === 0)
1508
+ return;
1509
+ this._dbBeginTx();
1510
+ for (const [child, set] of this._containBuf) {
1511
+ this._dbAppendContain(child, [...set]);
1705
1512
  }
1706
- }
1707
- }
1708
- }
1709
- async maybeFlush() {
1710
- if (this.pending() >= this.batchSize) {
1711
- this.flush();
1712
- // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1713
- // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1714
- // engine drains to empty before it ever services a MACROTASK — so a deposit
1715
- // burst that only micro-awaits pins the single thread for its whole
1716
- // duration, starving timers, pending I/O and any overlapped work. Here —
1717
- // buffers empty, transaction committed, nothing mid-write — is the one
1718
- // safe point to hand the event loop a real turn.
1719
- await yieldToEventLoop();
1720
- }
1721
- }
1722
- // ── Meta ───────────────────────────────────────────────────────────────
1723
- async setMeta(key, val) {
1724
- await this._ensureReady();
1725
- this._dbBeginTx();
1726
- this._dbSetMeta(key, val);
1727
- }
1728
- async getMeta(key) {
1729
- await this._ensureReady();
1730
- return this._dbGetMeta(key);
1731
- }
1732
- async deleteMeta(key) {
1733
- await this._ensureReady();
1734
- this._dbBeginTx();
1735
- this._dbDeleteMeta(key);
1736
- }
1737
- // ── Snapshot ───────────────────────────────────────────────────────────
1738
- async saveSnapshot(bytes) {
1739
- await this._ensureReady();
1740
- this.flush(); // commits any open write transaction + vector buffers
1741
- this._dbSaveSnapshot(bytes);
1742
- }
1743
- async loadSnapshot() {
1744
- await this._ensureReady();
1745
- return this._dbLoadSnapshot();
1746
- }
1747
- // ── Lifecycle ──────────────────────────────────────────────────────────
1748
- commit() {
1749
- this.flush();
1750
- }
1751
- async close() {
1752
- if (this.closed) {
1753
- return;
1754
- }
1755
- this.closed = true;
1756
- this.flush(); // commits any open write transaction + vector buffers
1757
- this._dbClose();
1758
- }
1513
+ this._containBuf.clear();
1514
+ }
1515
+ /** Flush all three buffers; compact vector indices on a write-volume cadence. */
1516
+ flush() {
1517
+ this.flushContain();
1518
+ const written = this.flushContent() + this.flushHalos();
1519
+ // Commit the deferred write transaction on the same cadence as the vector
1520
+ // buffers, so node rows / edges / halos become durable in coalesced batches.
1521
+ this._dbCommitTx();
1522
+ if (written === 0)
1523
+ return;
1524
+ this._writtenSinceCompact += written;
1525
+ if (this._writtenSinceCompact >= this.compactEveryNWrites) {
1526
+ this._writtenSinceCompact = 0;
1527
+ try {
1528
+ if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
1529
+ this._vecContentCompact();
1530
+ // Physical compaction may free the top internal id for reuse —
1531
+ // invalidate the incremental maintenance watermark.
1532
+ this._dbBeginTx();
1533
+ this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
1534
+ this._dbCommitTx();
1535
+ }
1536
+ if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
1537
+ this._vecHaloCompact();
1538
+ }
1539
+ }
1540
+ catch (e) {
1541
+ // Best-effort, but never SILENT (same contract as the prune-path
1542
+ // compaction above): a persistently failing compaction lets
1543
+ // tombstones accumulate and query cost grow with no signal.
1544
+ this.compactFailures++;
1545
+ if (this.compactFailures === 1) {
1546
+ console.warn("sema: vector-index compaction failed (will keep " +
1547
+ "counting in store.compactFailures):", e);
1548
+ }
1549
+ }
1550
+ }
1551
+ }
1552
+ async maybeFlush() {
1553
+ if (this.pending() >= this.batchSize) {
1554
+ this.flush();
1555
+ // A flush is the one HEAVY, fully-synchronous unit of the ingest path.
1556
+ // Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
1557
+ // engine drains to empty before it ever services a MACROTASK — so a deposit
1558
+ // burst that only micro-awaits pins the single thread for its whole
1559
+ // duration, starving timers, pending I/O and any overlapped work. Here —
1560
+ // buffers empty, transaction committed, nothing mid-write — is the one
1561
+ // safe point to hand the event loop a real turn.
1562
+ await yieldToEventLoop();
1563
+ }
1564
+ }
1565
+ // ── Meta ───────────────────────────────────────────────────────────────
1566
+ async setMeta(key, val) {
1567
+ await this._ensureReady();
1568
+ this._dbBeginTx();
1569
+ this._dbSetMeta(key, val);
1570
+ }
1571
+ async getMeta(key) {
1572
+ await this._ensureReady();
1573
+ return this._dbGetMeta(key);
1574
+ }
1575
+ async deleteMeta(key) {
1576
+ await this._ensureReady();
1577
+ this._dbBeginTx();
1578
+ this._dbDeleteMeta(key);
1579
+ }
1580
+ // ── Snapshot ───────────────────────────────────────────────────────────
1581
+ async saveSnapshot(bytes) {
1582
+ await this._ensureReady();
1583
+ this.flush(); // commits any open write transaction + vector buffers
1584
+ this._dbSaveSnapshot(bytes);
1585
+ }
1586
+ async loadSnapshot() {
1587
+ await this._ensureReady();
1588
+ return this._dbLoadSnapshot();
1589
+ }
1590
+ // ── Lifecycle ──────────────────────────────────────────────────────────
1591
+ commit() {
1592
+ this.flush();
1593
+ }
1594
+ async close() {
1595
+ if (this.closed)
1596
+ return;
1597
+ this.closed = true;
1598
+ this.flush(); // commits any open write transaction + vector buffers
1599
+ this._dbClose();
1600
+ }
1759
1601
  }