@hviana/sema 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1687 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +17 -42
  51. package/dist/src/geometry.js +235 -266
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +749 -871
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +126 -185
  82. package/dist/src/mind/mind.d.ts +129 -154
  83. package/dist/src/mind/mind.js +258 -288
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +8 -32
  89. package/dist/src/mind/primitives.js +99 -117
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +183 -208
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +16 -58
  101. package/dist/src/mind/traverse.js +304 -357
  102. package/dist/src/mind/types.d.ts +120 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +169 -166
  117. package/dist/src/store-sqlite.js +762 -662
  118. package/dist/src/store.d.ts +586 -630
  119. package/dist/src/store.js +1423 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +13 -10
  123. package/package.json +1 -1
  124. package/src/alu/README.md +1 -1
  125. package/src/alu/src/index.ts +1 -1
  126. package/src/config.ts +19 -27
  127. package/src/index.ts +6 -11
  128. package/src/rabitq-ivf/README.md +56 -0
  129. package/src/rabitq-ivf/src/database.ts +276 -0
  130. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  131. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  132. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  133. package/src/store-sqlite.ts +196 -9
  134. package/src/store.ts +8 -32
  135. package/test/08-storage.test.mjs +3 -3
  136. package/test/14-scaling.test.mjs +2 -2
  137. package/test/35-ivf.test.mjs +263 -0
  138. package/test/36-bloom.test.mjs +123 -0
  139. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  140. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  141. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  142. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  143. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  144. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  145. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  146. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  147. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  148. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  149. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  150. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  151. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  152. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  153. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  154. package/src/rabitq-hnsw/README.md +0 -303
  155. package/src/rabitq-hnsw/src/database.ts +0 -492
  156. package/src/rabitq-hnsw/src/heap.ts +0 -90
  157. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  158. package/src/rabitq-hnsw/src/store.ts +0 -994
  159. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  160. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -30,7 +30,7 @@ import { sema } from "./sema.js";
30
30
  * longer than the perception quantum must use the scale-aware
31
31
  * {@link identityBar}, which converts the tolerated fraction into bytes. */
32
32
  export function mergeThreshold(D) {
33
- return 1 - 1 / Math.sqrt(D);
33
+ return 1 - 1 / Math.sqrt(D);
34
34
  }
35
35
  /** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
36
36
  * of `len` bytes. Under the linear fold a cosine reads "fraction of aligned
@@ -46,7 +46,7 @@ export function mergeThreshold(D) {
46
46
  * the tolerated fraction into tolerated bytes. Derived from W, D and the
47
47
  * span; never tuned. */
48
48
  export function identityBar(D, maxGroup, len) {
49
- return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
49
+ return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
50
50
  }
51
51
  /** The reach bar: half a river quantum, derived from the fold's own geometry.
52
52
  * A branch folds up to `maxGroup` children, so two forms that differ in ONE
@@ -59,7 +59,7 @@ export function identityBar(D, maxGroup, len) {
59
59
  * — further than any single-child variant — and the system returns null rather
60
60
  * than fabricate an answer from an unrelated form. Derived, never tuned. */
61
61
  export function reachThreshold(maxGroup) {
62
- return 1 - 1 / (2 * maxGroup);
62
+ return 1 - 1 / (2 * maxGroup);
63
63
  }
64
64
  /** The estimator's own noise floor: 1/√D — ONE standard deviation of the
65
65
  * cosine between two independent random vectors in D dimensions (the same σ
@@ -71,7 +71,7 @@ export function reachThreshold(maxGroup) {
71
71
  * minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
72
72
  * Derived, never tuned. */
73
73
  export function estimatorNoise(D) {
74
- return 1 / Math.sqrt(D);
74
+ return 1 / Math.sqrt(D);
75
75
  }
76
76
  /** The statistical-significance bar for whole-query resonance: 3/√D.
77
77
  * In D dimensions the expected cosine of two independent random vectors is 0
@@ -81,7 +81,7 @@ export function estimatorNoise(D) {
81
81
  * sub-region resonance) is skipped: there is no evidence the query belongs to
82
82
  * the same distribution as the stored content. Derived, never tuned. */
83
83
  export function significanceBar(D) {
84
- return 3 / Math.sqrt(D);
84
+ return 3 / Math.sqrt(D);
85
85
  }
86
86
  /** The concept (halo) threshold: the cosine above which two nodes share a
87
87
  * distributional concept. A halo is a superposition of episode signatures in
@@ -93,7 +93,7 @@ export function significanceBar(D) {
93
93
  * 3% of 0.5 — existing behavior is preserved while threshold and D move
94
94
  * together. Derived, never tuned. */
95
95
  export function conceptThreshold(D) {
96
- return 0.5 + 0.5 / Math.sqrt(D);
96
+ return 0.5 + 0.5 / Math.sqrt(D);
97
97
  }
98
98
  /** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
99
99
  * whole of `wholeLen` covers STRICTLY more than half of it. A span that
@@ -104,7 +104,7 @@ export function conceptThreshold(D) {
104
104
  * already unusable there) and stays inline where it is documented.
105
105
  * Derived from the structural midpoint, never tuned. */
106
106
  export function dominates(partLen, wholeLen) {
107
- return partLen * 2 > wholeLen;
107
+ return partLen * 2 > wholeLen;
108
108
  }
109
109
  /** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
110
110
  * of learnt contexts (edge sources). A single region's IDF-weighted vote for
@@ -116,7 +116,7 @@ export function dominates(partLen, wholeLen) {
116
116
  * further point of attention. Defined once here so the two can never
117
117
  * drift apart. Derived from N, never tuned. */
118
118
  export function consensusFloor(N) {
119
- return Math.log(N) + 1 / 2;
119
+ return Math.log(N) + 1 / 2;
120
120
  }
121
121
  /** The coverage bar for the reach (interior) index, when vector-similarity
122
122
  * gating is used. Returns the concept threshold — the structural midpoint
@@ -128,7 +128,7 @@ export function consensusFloor(N) {
128
128
  * replaces runtime coverage gating with a batch pass that removes
129
129
  * structurally-isolated entries. Derived, never tuned. */
130
130
  export function coverageBar(_maxGroup, D) {
131
- return conceptThreshold(D);
131
+ return conceptThreshold(D);
132
132
  }
133
133
  // ---- folding ----
134
134
  //
@@ -160,302 +160,271 @@ export function coverageBar(_maxGroup, D) {
160
160
  * well — only a lone singleton passes through — guaranteeing progress when
161
161
  * the caller detected a stall. */
162
162
  function foldSlice(space, items, start, count, out, force) {
163
- const mg = space.maxGroup;
164
- const D = space.D;
165
- const complete = count - (count % mg);
166
- const foldAt = (at, size) => {
167
- const gist = new Float32Array(D);
168
- const kids = new Array(size);
169
- let len = 0;
170
- for (let k = 0; k < size; k++) {
171
- const f = items[at + k];
172
- const seat = space.seats[k].fwd;
173
- const v = f.tree.v;
174
- // Fused permute-and-accumulate — same FP ops, same order as the old
175
- // permuteInto + addInto pair, with no scratch buffer.
176
- for (let d = 0; d < D; d++) {
177
- gist[d] += v[seat[d]];
178
- }
179
- kids[k] = f.tree;
180
- len += f.len;
181
- }
182
- out.push({ tree: sema(gist, null, kids), len });
183
- };
184
- for (let i = 0; i < complete; i += mg) {
185
- foldAt(start + i, mg);
186
- }
187
- const leftover = count - complete;
188
- if (leftover === 0) {
189
- return;
190
- }
191
- if (force && leftover >= 2) {
192
- foldAt(start + complete, leftover);
193
- } else {
194
- for (let i = complete; i < count; i++) {
195
- out.push(items[start + i]);
196
- }
197
- }
163
+ const mg = space.maxGroup;
164
+ const D = space.D;
165
+ const complete = count - (count % mg);
166
+ const foldAt = (at, size) => {
167
+ const gist = new Float32Array(D);
168
+ const kids = new Array(size);
169
+ let len = 0;
170
+ for (let k = 0; k < size; k++) {
171
+ const f = items[at + k];
172
+ const seat = space.seats[k].fwd;
173
+ const v = f.tree.v;
174
+ // Fused permute-and-accumulate — same FP ops, same order as the old
175
+ // permuteInto + addInto pair, with no scratch buffer.
176
+ for (let d = 0; d < D; d++)
177
+ gist[d] += v[seat[d]];
178
+ kids[k] = f.tree;
179
+ len += f.len;
180
+ }
181
+ out.push({ tree: sema(gist, null, kids), len });
182
+ };
183
+ for (let i = 0; i < complete; i += mg)
184
+ foldAt(start + i, mg);
185
+ const leftover = count - complete;
186
+ if (leftover === 0)
187
+ return;
188
+ if (force && leftover >= 2)
189
+ foldAt(start + complete, leftover);
190
+ else
191
+ for (let i = complete; i < count; i++)
192
+ out.push(items[start + i]);
198
193
  }
199
194
  function riverFold(space, row, stableBytes) {
200
- if (row.length === 0) {
201
- const z = new Float32Array(space.D);
202
- return { tree: sema(z, new Uint8Array(0), null), len: 0 };
203
- }
204
- let level = row;
205
- while (level.length > 1) {
206
- // Find the item index where accumulated bytes reaches stableBytes.
207
- let boundary = level.length;
208
- if (stableBytes > 0) {
209
- let acc = 0;
210
- for (let i = 0; i < level.length; i++) {
211
- acc += level[i].len;
212
- if (acc >= stableBytes) {
213
- boundary = i + 1;
214
- break;
215
- }
216
- }
217
- }
218
- const next = [];
219
- if (boundary < level.length) {
220
- // Prefix folds independently of the suffix — structural stability.
221
- foldSlice(space, level, 0, boundary, next, false);
222
- foldSlice(space, level, boundary, level.length - boundary, next, false);
223
- } else {
224
- foldSlice(space, level, 0, level.length, next, false);
195
+ if (row.length === 0) {
196
+ const z = new Float32Array(space.D);
197
+ return { tree: sema(z, new Uint8Array(0), null), len: 0 };
225
198
  }
226
- if (next.length === level.length) {
227
- // Stuck — every group was incomplete. Force-fold to break the stall.
228
- const forced = [];
229
- foldSlice(space, next, 0, next.length, forced, true);
230
- level = forced;
231
- } else {
232
- level = next;
199
+ let level = row;
200
+ while (level.length > 1) {
201
+ // Find the item index where accumulated bytes reaches stableBytes.
202
+ let boundary = level.length;
203
+ if (stableBytes > 0) {
204
+ let acc = 0;
205
+ for (let i = 0; i < level.length; i++) {
206
+ acc += level[i].len;
207
+ if (acc >= stableBytes) {
208
+ boundary = i + 1;
209
+ break;
210
+ }
211
+ }
212
+ }
213
+ const next = [];
214
+ if (boundary < level.length) {
215
+ // Prefix folds independently of the suffix — structural stability.
216
+ foldSlice(space, level, 0, boundary, next, false);
217
+ foldSlice(space, level, boundary, level.length - boundary, next, false);
218
+ }
219
+ else {
220
+ foldSlice(space, level, 0, level.length, next, false);
221
+ }
222
+ if (next.length === level.length) {
223
+ // Stuck — every group was incomplete. Force-fold to break the stall.
224
+ const forced = [];
225
+ foldSlice(space, next, 0, next.length, forced, true);
226
+ level = forced;
227
+ }
228
+ else {
229
+ level = next;
230
+ }
233
231
  }
234
- }
235
- // LINEAR fold this root normalize is the ONLY normalize of the entire
236
- // fold; every intermediate gist stays unnormalized (see the folding
237
- // header). Skipped for a single-leaf input: that root IS the shared
238
- // alphabet vector (already unit), and normalizing in place would mutate the
239
- // alphabet itself.
240
- if (row.length > 1) {
241
- normalize(level[0].tree.v);
242
- }
243
- return level[0];
232
+ // LINEAR fold — this root normalize is the ONLY normalize of the entire
233
+ // fold; every intermediate gist stays unnormalized (see the folding
234
+ // header). Skipped for a single-leaf input: that root IS the shared
235
+ // alphabet vector (already unit), and normalizing in place would mutate the
236
+ // alphabet itself.
237
+ if (row.length > 1)
238
+ normalize(level[0].tree.v);
239
+ return level[0];
244
240
  }
245
241
  // ---- public API ----
246
242
  function bytesToLeaves(alphabet, bytes) {
247
- return Array.from(bytes, (b, i) => {
248
- const v = alphabet.vecs[b];
249
- return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
250
- });
243
+ return Array.from(bytes, (b, i) => {
244
+ const v = alphabet.vecs[b];
245
+ return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
246
+ });
251
247
  }
252
248
  /** Find the longest prefix of `bytes` whose leaf-id signature matches a
253
249
  * known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
254
250
  export function knownPrefixLength(bytes, leafAt, lookup) {
255
- const leafIds = [];
256
- for (let i = 0; i < bytes.length; i++) {
257
- const lid = leafAt(i);
258
- if (lid === null) {
259
- break;
251
+ const leafIds = [];
252
+ for (let i = 0; i < bytes.length; i++) {
253
+ const lid = leafAt(i);
254
+ if (lid === null)
255
+ break;
256
+ leafIds.push(lid);
260
257
  }
261
- leafIds.push(lid);
262
- }
263
- // Match the longest PROPER prefix a full-length match means the entire
264
- // input already exists as a stored form (e.g. the flat leaf-id branch
265
- // stored alongside the structural root). That would hide the true split
266
- // point and prevent the river from producing the same tree it folded
267
- // during training, so the structural recognition cannot find the right
268
- // forms. A proper prefix guarantees at least two regions.
269
- for (let len = bytes.length - 1; len >= 2; len--) {
270
- if (lookup(leafIds.slice(0, len)) !== null) {
271
- return len;
258
+ // Match the longest PROPER prefix — a full-length match means the entire
259
+ // input already exists as a stored form (e.g. the flat leaf-id branch
260
+ // stored alongside the structural root). That would hide the true split
261
+ // point and prevent the river from producing the same tree it folded
262
+ // during training, so the structural recognition cannot find the right
263
+ // forms. A proper prefix guarantees at least two regions.
264
+ for (let len = bytes.length - 1; len >= 2; len--) {
265
+ if (lookup(leafIds.slice(0, len)) !== null)
266
+ return len;
272
267
  }
273
- }
274
- return 0;
268
+ return 0;
275
269
  }
276
270
  /** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
277
271
  * detecting previously-stored prefixes so the river can split at the
278
272
  * correct boundary. Pass them through from `perceive`; the geometry
279
273
  * computes the stable prefix internally. */
280
274
  export function bytesToTree(space, alphabet, bytes, leafAt, lookup) {
281
- if (bytes.length === 0) {
282
- return sema(alphabet.vecs[0], new Uint8Array(0), null);
283
- }
284
- const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
285
- return riverFold(
286
- space,
287
- bytesToLeaves(alphabet, bytes),
288
- sb > 0 ? sb : bytes.length,
289
- ).tree;
275
+ if (bytes.length === 0) {
276
+ return sema(alphabet.vecs[0], new Uint8Array(0), null);
277
+ }
278
+ const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
279
+ return riverFold(space, bytesToLeaves(alphabet, bytes), sb > 0 ? sb : bytes.length).tree;
290
280
  }
291
281
  /** Plain bytes→tree (identical to capability-less {@link bytesToTree}) that
292
282
  * also RETURNS its pyramid, reusing `prev` — the pyramid of a PROPER
293
283
  * prefix of `bytes` (caller guarantees content match and
294
284
  * prev.bytes < bytes.length). */
295
285
  export function bytesToTreePyramid(space, alphabet, bytes, prev) {
296
- if (bytes.length === 0) {
297
- return {
298
- tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
299
- pyramid: { levels: [], bytes: 0 },
300
- };
301
- }
302
- const mg = space.maxGroup;
303
- const reusable = (L) => {
304
- // prev's TOPMOST level holds its normalized ROOT — reusable blocks must
305
- // be raw interiors, so the top level is always excluded.
306
- if (!prev || L > prev.levels.length - 2) {
307
- return null;
286
+ if (bytes.length === 0) {
287
+ return {
288
+ tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
289
+ pyramid: { levels: [], bytes: 0 },
290
+ };
308
291
  }
309
- return prev.levels[L];
310
- };
311
- // Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
312
- const lv0 = reusable(0);
313
- const row = lv0
314
- ? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
315
- : bytesToLeaves(alphabet, bytes);
316
- const levels = [row];
317
- let level = row;
318
- let L = 0;
319
- while (level.length > 1) {
320
- let next = [];
321
- // Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
322
- // wholly inside the prefix); the rule below folds the rest from this
323
- // level's items exactly as the from-scratch fold would.
324
- const blockLen = mg ** (L + 1);
325
- let reused = 0;
326
- const cand = reusable(L + 1);
327
- if (cand) {
328
- const maxFull = Math.floor(prev.bytes / blockLen);
329
- while (
330
- reused < maxFull && reused < cand.length &&
331
- cand[reused].len === blockLen
332
- ) {
333
- reused++;
334
- }
335
- for (let i = 0; i < reused; i++) {
336
- next.push(cand[i]);
337
- }
338
- }
339
- foldSlice(
340
- space,
341
- level,
342
- reused * mg,
343
- level.length - reused * mg,
344
- next,
345
- false,
346
- );
347
- if (next.length === level.length) {
348
- // Stuck — same force-fold as riverFold (reuse cannot cause it: any
349
- // reused block replaces a whole folded group, so reused > 0 implies
350
- // progress).
351
- const forced = [];
352
- foldSlice(space, next, 0, next.length, forced, true);
353
- next = forced;
292
+ const mg = space.maxGroup;
293
+ const reusable = (L) => {
294
+ // prev's TOPMOST level holds its normalized ROOT reusable blocks must
295
+ // be raw interiors, so the top level is always excluded.
296
+ if (!prev || L > prev.levels.length - 2)
297
+ return null;
298
+ return prev.levels[L];
299
+ };
300
+ // Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
301
+ const lv0 = reusable(0);
302
+ const row = lv0
303
+ ? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
304
+ : bytesToLeaves(alphabet, bytes);
305
+ const levels = [row];
306
+ let level = row;
307
+ let L = 0;
308
+ while (level.length > 1) {
309
+ let next = [];
310
+ // Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
311
+ // wholly inside the prefix); the rule below folds the rest from this
312
+ // level's items exactly as the from-scratch fold would.
313
+ const blockLen = mg ** (L + 1);
314
+ let reused = 0;
315
+ const cand = reusable(L + 1);
316
+ if (cand) {
317
+ const maxFull = Math.floor(prev.bytes / blockLen);
318
+ while (reused < maxFull && reused < cand.length &&
319
+ cand[reused].len === blockLen)
320
+ reused++;
321
+ for (let i = 0; i < reused; i++)
322
+ next.push(cand[i]);
323
+ }
324
+ foldSlice(space, level, reused * mg, level.length - reused * mg, next, false);
325
+ if (next.length === level.length) {
326
+ // Stuck — same force-fold as riverFold (reuse cannot cause it: any
327
+ // reused block replaces a whole folded group, so reused > 0 implies
328
+ // progress).
329
+ const forced = [];
330
+ foldSlice(space, next, 0, next.length, forced, true);
331
+ next = forced;
332
+ }
333
+ levels.push(next);
334
+ level = next;
335
+ L++;
354
336
  }
355
- levels.push(next);
356
- level = next;
357
- L++;
358
- }
359
- if (row.length > 1) {
360
- normalize(level[0].tree.v);
361
- }
362
- return {
363
- tree: level[0].tree,
364
- pyramid: { levels, bytes: bytes.length },
365
- };
337
+ if (row.length > 1)
338
+ normalize(level[0].tree.v);
339
+ return {
340
+ tree: level[0].tree,
341
+ pyramid: { levels, bytes: bytes.length },
342
+ };
366
343
  }
367
344
  // ---- n-D Hilbert curve ----
368
345
  function gridDims(grid) {
369
- if (grid.dims && grid.dims.length > 0) {
370
- return grid.dims.slice();
371
- }
372
- const dims = [grid.height, grid.width];
373
- if (grid.channels > 1) {
374
- dims.push(grid.channels);
375
- }
376
- return dims;
346
+ if (grid.dims && grid.dims.length > 0)
347
+ return grid.dims.slice();
348
+ const dims = [grid.height, grid.width];
349
+ if (grid.channels > 1)
350
+ dims.push(grid.channels);
351
+ return dims;
377
352
  }
378
353
  function hilbertPoint(index, n, bits) {
379
- const x = new Array(n).fill(0);
380
- for (let b = 0; b < bits; b++) {
381
- for (let d = 0; d < n; d++) {
382
- const bit = (index >>> (b * n + (n - 1 - d))) & 1;
383
- x[d] |= bit << b;
354
+ const x = new Array(n).fill(0);
355
+ for (let b = 0; b < bits; b++) {
356
+ for (let d = 0; d < n; d++) {
357
+ const bit = (index >>> (b * n + (n - 1 - d))) & 1;
358
+ x[d] |= bit << b;
359
+ }
384
360
  }
385
- }
386
- const N = 1 << bits;
387
- let t = x[n - 1] >> 1;
388
- for (let i = n - 1; i > 0; i--) {
389
- x[i] ^= x[i - 1];
390
- }
391
- x[0] ^= t;
392
- for (let q = 2; q !== N; q <<= 1) {
393
- const p = q - 1;
394
- for (let i = n - 1; i >= 0; i--) {
395
- if (x[i] & q) {
396
- x[0] ^= p;
397
- } else {
398
- t = (x[0] ^ x[i]) & p;
399
- x[0] ^= t;
400
- x[i] ^= t;
401
- }
361
+ const N = 1 << bits;
362
+ let t = x[n - 1] >> 1;
363
+ for (let i = n - 1; i > 0; i--)
364
+ x[i] ^= x[i - 1];
365
+ x[0] ^= t;
366
+ for (let q = 2; q !== N; q <<= 1) {
367
+ const p = q - 1;
368
+ for (let i = n - 1; i >= 0; i--) {
369
+ if (x[i] & q)
370
+ x[0] ^= p;
371
+ else {
372
+ t = (x[0] ^ x[i]) & p;
373
+ x[0] ^= t;
374
+ x[i] ^= t;
375
+ }
376
+ }
402
377
  }
403
- }
404
- return x;
378
+ return x;
405
379
  }
406
380
  export function hilbertBytes(grid) {
407
- const dims = gridDims(grid);
408
- const n = dims.length;
409
- if (n === 0 || grid.data.length === 0) {
410
- return new Uint8Array(0);
411
- }
412
- if (n === 1) {
413
- return grid.data.slice(0, dims[0]);
414
- }
415
- const maxAxis = Math.max(...dims);
416
- const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
417
- const side = 1 << bits;
418
- const total = Math.pow(side, n);
419
- const stride = new Array(n);
420
- stride[n - 1] = 1;
421
- for (let d = n - 2; d >= 0; d--) {
422
- stride[d] = stride[d + 1] * dims[d + 1];
423
- }
424
- const out = [];
425
- for (let h = 0; h < total; h++) {
426
- const pt = hilbertPoint(h, n, bits);
427
- let inside = true, flat = 0;
428
- for (let d = 0; d < n; d++) {
429
- if (pt[d] >= dims[d]) {
430
- inside = false;
431
- break;
432
- }
433
- flat += pt[d] * stride[d];
434
- }
435
- if (inside) {
436
- out.push(grid.data[flat]);
381
+ const dims = gridDims(grid);
382
+ const n = dims.length;
383
+ if (n === 0 || grid.data.length === 0)
384
+ return new Uint8Array(0);
385
+ if (n === 1)
386
+ return grid.data.slice(0, dims[0]);
387
+ const maxAxis = Math.max(...dims);
388
+ const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
389
+ const side = 1 << bits;
390
+ const total = Math.pow(side, n);
391
+ const stride = new Array(n);
392
+ stride[n - 1] = 1;
393
+ for (let d = n - 2; d >= 0; d--)
394
+ stride[d] = stride[d + 1] * dims[d + 1];
395
+ const out = [];
396
+ for (let h = 0; h < total; h++) {
397
+ const pt = hilbertPoint(h, n, bits);
398
+ let inside = true, flat = 0;
399
+ for (let d = 0; d < n; d++) {
400
+ if (pt[d] >= dims[d]) {
401
+ inside = false;
402
+ break;
403
+ }
404
+ flat += pt[d] * stride[d];
405
+ }
406
+ if (inside)
407
+ out.push(grid.data[flat]);
437
408
  }
438
- }
439
- return Uint8Array.from(out);
409
+ return Uint8Array.from(out);
440
410
  }
441
411
  export function gridToTree(space, alphabet, grid) {
442
- return bytesToTree(space, alphabet, hilbertBytes(grid));
412
+ return bytesToTree(space, alphabet, hilbertBytes(grid));
443
413
  }
444
414
  export function stackGrids(frames) {
445
- if (frames.length === 0) {
446
- return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
447
- }
448
- const frameDims = gridDims(frames[0]);
449
- const per = frames[0].data.length;
450
- const data = new Uint8Array(per * frames.length);
451
- for (let i = 0; i < frames.length; i++) {
452
- data.set(frames[i].data, i * per);
453
- }
454
- return {
455
- width: 0,
456
- height: 0,
457
- channels: 0,
458
- dims: [frames.length, ...frameDims],
459
- data,
460
- };
415
+ if (frames.length === 0) {
416
+ return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
417
+ }
418
+ const frameDims = gridDims(frames[0]);
419
+ const per = frames[0].data.length;
420
+ const data = new Uint8Array(per * frames.length);
421
+ for (let i = 0; i < frames.length; i++)
422
+ data.set(frames[i].data, i * per);
423
+ return {
424
+ width: 0,
425
+ height: 0,
426
+ channels: 0,
427
+ dims: [frames.length, ...frameDims],
428
+ data,
429
+ };
461
430
  }
@@ -10,21 +10,6 @@ export * from "./store-sqlite.js";
10
10
  export * from "./config.js";
11
11
  export * from "./extension.js";
12
12
  export * from "./ingest-cache.js";
13
- export {
14
- Heap,
15
- HnswIndex,
16
- Prng,
17
- RaBitQuantizer,
18
- VectorDatabase,
19
- } from "./rabitq-hnsw/src/index.js";
20
- export type {
21
- DatabaseOptions,
22
- ExternalId,
23
- HnswParams,
24
- KnnHit,
25
- QueryContext,
26
- QueryResult,
27
- RaBitQOptions,
28
- StorageStats,
29
- } from "./rabitq-hnsw/src/index.js";
13
+ export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
14
+ export type { DatabaseOptions, ExternalId, IvfConfig, IvfHit, QueryContext, QueryResult, RaBitQOptions, StorageStats, } from "./rabitq-ivf/src/index.js";
30
15
  export * from "./derive/src/index.js";
package/dist/src/index.js CHANGED
@@ -12,16 +12,6 @@ export * from "./store-sqlite.js";
12
12
  export * from "./config.js";
13
13
  export * from "./extension.js";
14
14
  export * from "./ingest-cache.js";
15
- // rabitq-hnsw is re-exported selectively: its `Store`, `NodeRec`, and
16
- // `StoreConfig` are internal names that collide with sema's own top-level
17
- // `store.js` / `config.js` exports, and sema code that needs the rabitq ones
18
- // imports them from the subpath directly. Re-export the public vector-DB
19
- // surface under the sema root, omitting the three colliding names.
20
- export {
21
- Heap,
22
- HnswIndex,
23
- Prng,
24
- RaBitQuantizer,
25
- VectorDatabase,
26
- } from "./rabitq-hnsw/src/index.js";
15
+ // rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
16
+ export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
27
17
  export * from "./derive/src/index.js";