@hviana/sema 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CITATION.cff +49 -0
- package/HOW_IT_WORKS.md +11 -12
- package/README.md +7 -5
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +19 -21
- package/dist/example/train_base.js +1687 -1952
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +24 -27
- package/dist/src/alu/src/alu.d.ts +169 -194
- package/dist/src/alu/src/alu.js +374 -414
- package/dist/src/alu/src/expr.d.ts +36 -43
- package/dist/src/alu/src/expr.js +252 -278
- package/dist/src/alu/src/index.d.ts +9 -98
- package/dist/src/alu/src/index.js +10 -64
- package/dist/src/alu/src/kernel-arith.d.ts +1 -5
- package/dist/src/alu/src/kernel-arith.js +195 -285
- package/dist/src/alu/src/kernel-bits.d.ts +1 -5
- package/dist/src/alu/src/kernel-bits.js +81 -115
- package/dist/src/alu/src/kernel-logic.js +33 -63
- package/dist/src/alu/src/kernel-nd.js +155 -195
- package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
- package/dist/src/alu/src/kernel-numeric.js +259 -331
- package/dist/src/alu/src/operation.d.ts +112 -132
- package/dist/src/alu/src/operation.js +149 -163
- package/dist/src/alu/src/parser.d.ts +168 -175
- package/dist/src/alu/src/parser.js +382 -421
- package/dist/src/alu/src/resonance.d.ts +16 -25
- package/dist/src/alu/src/resonance.js +49 -56
- package/dist/src/alu/src/text.d.ts +3 -7
- package/dist/src/alu/src/text.js +31 -37
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +150 -160
- package/dist/src/alu/test/alu.test.js +543 -656
- package/dist/src/bytes.d.ts +1 -5
- package/dist/src/bytes.js +33 -40
- package/dist/src/config.d.ts +99 -102
- package/dist/src/config.js +78 -83
- package/dist/src/derive/src/deduction.d.ts +60 -62
- package/dist/src/derive/src/deduction.js +103 -109
- package/dist/src/derive/src/index.d.ts +1 -7
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +57 -61
- package/dist/src/derive/src/rewrite.d.ts +15 -18
- package/dist/src/derive/src/rewrite.js +59 -67
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +175 -179
- package/dist/src/derive/test/derive.test.js +96 -100
- package/dist/src/extension.d.ts +13 -16
- package/dist/src/geometry.d.ts +17 -42
- package/dist/src/geometry.js +235 -266
- package/dist/src/index.d.ts +2 -17
- package/dist/src/index.js +2 -12
- package/dist/src/ingest-cache.d.ts +25 -30
- package/dist/src/ingest-cache.js +108 -127
- package/dist/src/mind/articulation.d.ts +1 -5
- package/dist/src/mind/articulation.js +77 -112
- package/dist/src/mind/attention.d.ts +40 -108
- package/dist/src/mind/attention.js +749 -871
- package/dist/src/mind/canonical.d.ts +4 -19
- package/dist/src/mind/canonical.js +27 -31
- package/dist/src/mind/graph-search.d.ts +201 -225
- package/dist/src/mind/graph-search.js +742 -821
- package/dist/src/mind/index.d.ts +2 -10
- package/dist/src/mind/junction.d.ts +31 -58
- package/dist/src/mind/junction.js +172 -237
- package/dist/src/mind/learning.d.ts +16 -52
- package/dist/src/mind/learning.js +143 -165
- package/dist/src/mind/match.d.ts +20 -69
- package/dist/src/mind/match.js +259 -318
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +9 -13
- package/dist/src/mind/mechanisms/cast.js +363 -456
- package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
- package/dist/src/mind/mechanisms/confluence.js +152 -183
- package/dist/src/mind/mechanisms/cover.d.ts +2 -8
- package/dist/src/mind/mechanisms/cover.js +148 -210
- package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
- package/dist/src/mind/mechanisms/extraction.js +234 -288
- package/dist/src/mind/mechanisms/recall.d.ts +6 -10
- package/dist/src/mind/mechanisms/recall.js +126 -185
- package/dist/src/mind/mind.d.ts +129 -154
- package/dist/src/mind/mind.js +258 -288
- package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
- package/dist/src/mind/pipeline-mechanism.js +161 -172
- package/dist/src/mind/pipeline.d.ts +4 -14
- package/dist/src/mind/pipeline.js +125 -189
- package/dist/src/mind/primitives.d.ts +8 -32
- package/dist/src/mind/primitives.js +99 -117
- package/dist/src/mind/rationale.d.ts +86 -98
- package/dist/src/mind/rationale.js +113 -121
- package/dist/src/mind/reasoning.d.ts +2 -13
- package/dist/src/mind/reasoning.js +129 -166
- package/dist/src/mind/recognition.d.ts +1 -4
- package/dist/src/mind/recognition.js +183 -208
- package/dist/src/mind/resonance.d.ts +5 -22
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +6 -25
- package/dist/src/mind/trace.js +50 -58
- package/dist/src/mind/traverse.d.ts +16 -58
- package/dist/src/mind/traverse.js +304 -357
- package/dist/src/mind/types.d.ts +120 -127
- package/dist/src/mind/types.js +60 -69
- package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
- package/dist/src/rabitq-ivf/src/database.js +201 -0
- package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
- package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
- package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
- package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
- package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
- package/dist/src/rabitq-ivf/src/prng.js +36 -0
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
- package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
- package/dist/src/sema.d.ts +9 -13
- package/dist/src/sema.js +26 -40
- package/dist/src/store-sqlite.d.ts +169 -166
- package/dist/src/store-sqlite.js +762 -662
- package/dist/src/store.d.ts +586 -630
- package/dist/src/store.js +1423 -1581
- package/dist/src/vec.d.ts +5 -9
- package/dist/src/vec.js +61 -73
- package/example/train_base.ts +13 -10
- package/package.json +1 -1
- package/src/alu/README.md +1 -1
- package/src/alu/src/index.ts +1 -1
- package/src/config.ts +19 -27
- package/src/index.ts +6 -11
- package/src/rabitq-ivf/README.md +56 -0
- package/src/rabitq-ivf/src/database.ts +276 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
- package/src/rabitq-ivf/src/ivf.ts +1330 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
- package/src/store-sqlite.ts +196 -9
- package/src/store.ts +8 -32
- package/test/08-storage.test.mjs +3 -3
- package/test/14-scaling.test.mjs +2 -2
- package/test/35-ivf.test.mjs +263 -0
- package/test/36-bloom.test.mjs +123 -0
- package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
- package/dist/src/rabitq-hnsw/src/database.js +0 -405
- package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
- package/dist/src/rabitq-hnsw/src/heap.js +0 -94
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
- package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
- package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
- package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
- package/dist/src/rabitq-hnsw/src/prng.js +0 -38
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
- package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
- package/dist/src/rabitq-hnsw/src/store.js +0 -916
- package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
- package/src/rabitq-hnsw/README.md +0 -303
- package/src/rabitq-hnsw/src/database.ts +0 -492
- package/src/rabitq-hnsw/src/heap.ts +0 -90
- package/src/rabitq-hnsw/src/hnsw.ts +0 -514
- package/src/rabitq-hnsw/src/store.ts +0 -994
- package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
- /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
package/dist/src/geometry.js
CHANGED
|
@@ -30,7 +30,7 @@ import { sema } from "./sema.js";
|
|
|
30
30
|
* longer than the perception quantum must use the scale-aware
|
|
31
31
|
* {@link identityBar}, which converts the tolerated fraction into bytes. */
|
|
32
32
|
export function mergeThreshold(D) {
|
|
33
|
-
|
|
33
|
+
return 1 - 1 / Math.sqrt(D);
|
|
34
34
|
}
|
|
35
35
|
/** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
|
|
36
36
|
* of `len` bytes. Under the linear fold a cosine reads "fraction of aligned
|
|
@@ -46,7 +46,7 @@ export function mergeThreshold(D) {
|
|
|
46
46
|
* the tolerated fraction into tolerated bytes. Derived from W, D and the
|
|
47
47
|
* span; never tuned. */
|
|
48
48
|
export function identityBar(D, maxGroup, len) {
|
|
49
|
-
|
|
49
|
+
return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
|
|
50
50
|
}
|
|
51
51
|
/** The reach bar: half a river quantum, derived from the fold's own geometry.
|
|
52
52
|
* A branch folds up to `maxGroup` children, so two forms that differ in ONE
|
|
@@ -59,7 +59,7 @@ export function identityBar(D, maxGroup, len) {
|
|
|
59
59
|
* — further than any single-child variant — and the system returns null rather
|
|
60
60
|
* than fabricate an answer from an unrelated form. Derived, never tuned. */
|
|
61
61
|
export function reachThreshold(maxGroup) {
|
|
62
|
-
|
|
62
|
+
return 1 - 1 / (2 * maxGroup);
|
|
63
63
|
}
|
|
64
64
|
/** The estimator's own noise floor: 1/√D — ONE standard deviation of the
|
|
65
65
|
* cosine between two independent random vectors in D dimensions (the same σ
|
|
@@ -71,7 +71,7 @@ export function reachThreshold(maxGroup) {
|
|
|
71
71
|
* minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
|
|
72
72
|
* Derived, never tuned. */
|
|
73
73
|
export function estimatorNoise(D) {
|
|
74
|
-
|
|
74
|
+
return 1 / Math.sqrt(D);
|
|
75
75
|
}
|
|
76
76
|
/** The statistical-significance bar for whole-query resonance: 3/√D.
|
|
77
77
|
* In D dimensions the expected cosine of two independent random vectors is 0
|
|
@@ -81,7 +81,7 @@ export function estimatorNoise(D) {
|
|
|
81
81
|
* sub-region resonance) is skipped: there is no evidence the query belongs to
|
|
82
82
|
* the same distribution as the stored content. Derived, never tuned. */
|
|
83
83
|
export function significanceBar(D) {
|
|
84
|
-
|
|
84
|
+
return 3 / Math.sqrt(D);
|
|
85
85
|
}
|
|
86
86
|
/** The concept (halo) threshold: the cosine above which two nodes share a
|
|
87
87
|
* distributional concept. A halo is a superposition of episode signatures in
|
|
@@ -93,7 +93,7 @@ export function significanceBar(D) {
|
|
|
93
93
|
* 3% of 0.5 — existing behavior is preserved while threshold and D move
|
|
94
94
|
* together. Derived, never tuned. */
|
|
95
95
|
export function conceptThreshold(D) {
|
|
96
|
-
|
|
96
|
+
return 0.5 + 0.5 / Math.sqrt(D);
|
|
97
97
|
}
|
|
98
98
|
/** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
|
|
99
99
|
* whole of `wholeLen` covers STRICTLY more than half of it. A span that
|
|
@@ -104,7 +104,7 @@ export function conceptThreshold(D) {
|
|
|
104
104
|
* already unusable there) and stays inline where it is documented.
|
|
105
105
|
* Derived from the structural midpoint, never tuned. */
|
|
106
106
|
export function dominates(partLen, wholeLen) {
|
|
107
|
-
|
|
107
|
+
return partLen * 2 > wholeLen;
|
|
108
108
|
}
|
|
109
109
|
/** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
|
|
110
110
|
* of learnt contexts (edge sources). A single region's IDF-weighted vote for
|
|
@@ -116,7 +116,7 @@ export function dominates(partLen, wholeLen) {
|
|
|
116
116
|
* further point of attention. Defined once here so the two can never
|
|
117
117
|
* drift apart. Derived from N, never tuned. */
|
|
118
118
|
export function consensusFloor(N) {
|
|
119
|
-
|
|
119
|
+
return Math.log(N) + 1 / 2;
|
|
120
120
|
}
|
|
121
121
|
/** The coverage bar for the reach (interior) index, when vector-similarity
|
|
122
122
|
* gating is used. Returns the concept threshold — the structural midpoint
|
|
@@ -128,7 +128,7 @@ export function consensusFloor(N) {
|
|
|
128
128
|
* replaces runtime coverage gating with a batch pass that removes
|
|
129
129
|
* structurally-isolated entries. Derived, never tuned. */
|
|
130
130
|
export function coverageBar(_maxGroup, D) {
|
|
131
|
-
|
|
131
|
+
return conceptThreshold(D);
|
|
132
132
|
}
|
|
133
133
|
// ---- folding ----
|
|
134
134
|
//
|
|
@@ -160,302 +160,271 @@ export function coverageBar(_maxGroup, D) {
|
|
|
160
160
|
* well — only a lone singleton passes through — guaranteeing progress when
|
|
161
161
|
* the caller detected a stall. */
|
|
162
162
|
function foldSlice(space, items, start, count, out, force) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
} else {
|
|
194
|
-
for (let i = complete; i < count; i++) {
|
|
195
|
-
out.push(items[start + i]);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
163
|
+
const mg = space.maxGroup;
|
|
164
|
+
const D = space.D;
|
|
165
|
+
const complete = count - (count % mg);
|
|
166
|
+
const foldAt = (at, size) => {
|
|
167
|
+
const gist = new Float32Array(D);
|
|
168
|
+
const kids = new Array(size);
|
|
169
|
+
let len = 0;
|
|
170
|
+
for (let k = 0; k < size; k++) {
|
|
171
|
+
const f = items[at + k];
|
|
172
|
+
const seat = space.seats[k].fwd;
|
|
173
|
+
const v = f.tree.v;
|
|
174
|
+
// Fused permute-and-accumulate — same FP ops, same order as the old
|
|
175
|
+
// permuteInto + addInto pair, with no scratch buffer.
|
|
176
|
+
for (let d = 0; d < D; d++)
|
|
177
|
+
gist[d] += v[seat[d]];
|
|
178
|
+
kids[k] = f.tree;
|
|
179
|
+
len += f.len;
|
|
180
|
+
}
|
|
181
|
+
out.push({ tree: sema(gist, null, kids), len });
|
|
182
|
+
};
|
|
183
|
+
for (let i = 0; i < complete; i += mg)
|
|
184
|
+
foldAt(start + i, mg);
|
|
185
|
+
const leftover = count - complete;
|
|
186
|
+
if (leftover === 0)
|
|
187
|
+
return;
|
|
188
|
+
if (force && leftover >= 2)
|
|
189
|
+
foldAt(start + complete, leftover);
|
|
190
|
+
else
|
|
191
|
+
for (let i = complete; i < count; i++)
|
|
192
|
+
out.push(items[start + i]);
|
|
198
193
|
}
|
|
199
194
|
function riverFold(space, row, stableBytes) {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
}
|
|
204
|
-
let level = row;
|
|
205
|
-
while (level.length > 1) {
|
|
206
|
-
// Find the item index where accumulated bytes reaches stableBytes.
|
|
207
|
-
let boundary = level.length;
|
|
208
|
-
if (stableBytes > 0) {
|
|
209
|
-
let acc = 0;
|
|
210
|
-
for (let i = 0; i < level.length; i++) {
|
|
211
|
-
acc += level[i].len;
|
|
212
|
-
if (acc >= stableBytes) {
|
|
213
|
-
boundary = i + 1;
|
|
214
|
-
break;
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
const next = [];
|
|
219
|
-
if (boundary < level.length) {
|
|
220
|
-
// Prefix folds independently of the suffix — structural stability.
|
|
221
|
-
foldSlice(space, level, 0, boundary, next, false);
|
|
222
|
-
foldSlice(space, level, boundary, level.length - boundary, next, false);
|
|
223
|
-
} else {
|
|
224
|
-
foldSlice(space, level, 0, level.length, next, false);
|
|
195
|
+
if (row.length === 0) {
|
|
196
|
+
const z = new Float32Array(space.D);
|
|
197
|
+
return { tree: sema(z, new Uint8Array(0), null), len: 0 };
|
|
225
198
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
199
|
+
let level = row;
|
|
200
|
+
while (level.length > 1) {
|
|
201
|
+
// Find the item index where accumulated bytes reaches stableBytes.
|
|
202
|
+
let boundary = level.length;
|
|
203
|
+
if (stableBytes > 0) {
|
|
204
|
+
let acc = 0;
|
|
205
|
+
for (let i = 0; i < level.length; i++) {
|
|
206
|
+
acc += level[i].len;
|
|
207
|
+
if (acc >= stableBytes) {
|
|
208
|
+
boundary = i + 1;
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
const next = [];
|
|
214
|
+
if (boundary < level.length) {
|
|
215
|
+
// Prefix folds independently of the suffix — structural stability.
|
|
216
|
+
foldSlice(space, level, 0, boundary, next, false);
|
|
217
|
+
foldSlice(space, level, boundary, level.length - boundary, next, false);
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
foldSlice(space, level, 0, level.length, next, false);
|
|
221
|
+
}
|
|
222
|
+
if (next.length === level.length) {
|
|
223
|
+
// Stuck — every group was incomplete. Force-fold to break the stall.
|
|
224
|
+
const forced = [];
|
|
225
|
+
foldSlice(space, next, 0, next.length, forced, true);
|
|
226
|
+
level = forced;
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
level = next;
|
|
230
|
+
}
|
|
233
231
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
}
|
|
243
|
-
return level[0];
|
|
232
|
+
// LINEAR fold — this root normalize is the ONLY normalize of the entire
|
|
233
|
+
// fold; every intermediate gist stays unnormalized (see the folding
|
|
234
|
+
// header). Skipped for a single-leaf input: that root IS the shared
|
|
235
|
+
// alphabet vector (already unit), and normalizing in place would mutate the
|
|
236
|
+
// alphabet itself.
|
|
237
|
+
if (row.length > 1)
|
|
238
|
+
normalize(level[0].tree.v);
|
|
239
|
+
return level[0];
|
|
244
240
|
}
|
|
245
241
|
// ---- public API ----
|
|
246
242
|
function bytesToLeaves(alphabet, bytes) {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
243
|
+
return Array.from(bytes, (b, i) => {
|
|
244
|
+
const v = alphabet.vecs[b];
|
|
245
|
+
return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
|
|
246
|
+
});
|
|
251
247
|
}
|
|
252
248
|
/** Find the longest prefix of `bytes` whose leaf-id signature matches a
|
|
253
249
|
* known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
|
|
254
250
|
export function knownPrefixLength(bytes, leafAt, lookup) {
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
251
|
+
const leafIds = [];
|
|
252
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
253
|
+
const lid = leafAt(i);
|
|
254
|
+
if (lid === null)
|
|
255
|
+
break;
|
|
256
|
+
leafIds.push(lid);
|
|
260
257
|
}
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (lookup(leafIds.slice(0, len)) !== null) {
|
|
271
|
-
return len;
|
|
258
|
+
// Match the longest PROPER prefix — a full-length match means the entire
|
|
259
|
+
// input already exists as a stored form (e.g. the flat leaf-id branch
|
|
260
|
+
// stored alongside the structural root). That would hide the true split
|
|
261
|
+
// point and prevent the river from producing the same tree it folded
|
|
262
|
+
// during training, so the structural recognition cannot find the right
|
|
263
|
+
// forms. A proper prefix guarantees at least two regions.
|
|
264
|
+
for (let len = bytes.length - 1; len >= 2; len--) {
|
|
265
|
+
if (lookup(leafIds.slice(0, len)) !== null)
|
|
266
|
+
return len;
|
|
272
267
|
}
|
|
273
|
-
|
|
274
|
-
return 0;
|
|
268
|
+
return 0;
|
|
275
269
|
}
|
|
276
270
|
/** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
|
|
277
271
|
* detecting previously-stored prefixes so the river can split at the
|
|
278
272
|
* correct boundary. Pass them through from `perceive`; the geometry
|
|
279
273
|
* computes the stable prefix internally. */
|
|
280
274
|
export function bytesToTree(space, alphabet, bytes, leafAt, lookup) {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
space,
|
|
287
|
-
bytesToLeaves(alphabet, bytes),
|
|
288
|
-
sb > 0 ? sb : bytes.length,
|
|
289
|
-
).tree;
|
|
275
|
+
if (bytes.length === 0) {
|
|
276
|
+
return sema(alphabet.vecs[0], new Uint8Array(0), null);
|
|
277
|
+
}
|
|
278
|
+
const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
|
|
279
|
+
return riverFold(space, bytesToLeaves(alphabet, bytes), sb > 0 ? sb : bytes.length).tree;
|
|
290
280
|
}
|
|
291
281
|
/** Plain bytes→tree (identical to capability-less {@link bytesToTree}) that
|
|
292
282
|
* also RETURNS its pyramid, reusing `prev` — the pyramid of a PROPER
|
|
293
283
|
* prefix of `bytes` (caller guarantees content match and
|
|
294
284
|
* prev.bytes < bytes.length). */
|
|
295
285
|
export function bytesToTreePyramid(space, alphabet, bytes, prev) {
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}
|
|
302
|
-
const mg = space.maxGroup;
|
|
303
|
-
const reusable = (L) => {
|
|
304
|
-
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
305
|
-
// be raw interiors, so the top level is always excluded.
|
|
306
|
-
if (!prev || L > prev.levels.length - 2) {
|
|
307
|
-
return null;
|
|
286
|
+
if (bytes.length === 0) {
|
|
287
|
+
return {
|
|
288
|
+
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
289
|
+
pyramid: { levels: [], bytes: 0 },
|
|
290
|
+
};
|
|
308
291
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
next = forced;
|
|
292
|
+
const mg = space.maxGroup;
|
|
293
|
+
const reusable = (L) => {
|
|
294
|
+
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
295
|
+
// be raw interiors, so the top level is always excluded.
|
|
296
|
+
if (!prev || L > prev.levels.length - 2)
|
|
297
|
+
return null;
|
|
298
|
+
return prev.levels[L];
|
|
299
|
+
};
|
|
300
|
+
// Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
|
|
301
|
+
const lv0 = reusable(0);
|
|
302
|
+
const row = lv0
|
|
303
|
+
? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
|
|
304
|
+
: bytesToLeaves(alphabet, bytes);
|
|
305
|
+
const levels = [row];
|
|
306
|
+
let level = row;
|
|
307
|
+
let L = 0;
|
|
308
|
+
while (level.length > 1) {
|
|
309
|
+
let next = [];
|
|
310
|
+
// Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
|
|
311
|
+
// wholly inside the prefix); the rule below folds the rest from this
|
|
312
|
+
// level's items exactly as the from-scratch fold would.
|
|
313
|
+
const blockLen = mg ** (L + 1);
|
|
314
|
+
let reused = 0;
|
|
315
|
+
const cand = reusable(L + 1);
|
|
316
|
+
if (cand) {
|
|
317
|
+
const maxFull = Math.floor(prev.bytes / blockLen);
|
|
318
|
+
while (reused < maxFull && reused < cand.length &&
|
|
319
|
+
cand[reused].len === blockLen)
|
|
320
|
+
reused++;
|
|
321
|
+
for (let i = 0; i < reused; i++)
|
|
322
|
+
next.push(cand[i]);
|
|
323
|
+
}
|
|
324
|
+
foldSlice(space, level, reused * mg, level.length - reused * mg, next, false);
|
|
325
|
+
if (next.length === level.length) {
|
|
326
|
+
// Stuck — same force-fold as riverFold (reuse cannot cause it: any
|
|
327
|
+
// reused block replaces a whole folded group, so reused > 0 implies
|
|
328
|
+
// progress).
|
|
329
|
+
const forced = [];
|
|
330
|
+
foldSlice(space, next, 0, next.length, forced, true);
|
|
331
|
+
next = forced;
|
|
332
|
+
}
|
|
333
|
+
levels.push(next);
|
|
334
|
+
level = next;
|
|
335
|
+
L++;
|
|
354
336
|
}
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
}
|
|
362
|
-
return {
|
|
363
|
-
tree: level[0].tree,
|
|
364
|
-
pyramid: { levels, bytes: bytes.length },
|
|
365
|
-
};
|
|
337
|
+
if (row.length > 1)
|
|
338
|
+
normalize(level[0].tree.v);
|
|
339
|
+
return {
|
|
340
|
+
tree: level[0].tree,
|
|
341
|
+
pyramid: { levels, bytes: bytes.length },
|
|
342
|
+
};
|
|
366
343
|
}
|
|
367
344
|
// ---- n-D Hilbert curve ----
|
|
368
345
|
function gridDims(grid) {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
dims
|
|
375
|
-
}
|
|
376
|
-
return dims;
|
|
346
|
+
if (grid.dims && grid.dims.length > 0)
|
|
347
|
+
return grid.dims.slice();
|
|
348
|
+
const dims = [grid.height, grid.width];
|
|
349
|
+
if (grid.channels > 1)
|
|
350
|
+
dims.push(grid.channels);
|
|
351
|
+
return dims;
|
|
377
352
|
}
|
|
378
353
|
function hilbertPoint(index, n, bits) {
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
354
|
+
const x = new Array(n).fill(0);
|
|
355
|
+
for (let b = 0; b < bits; b++) {
|
|
356
|
+
for (let d = 0; d < n; d++) {
|
|
357
|
+
const bit = (index >>> (b * n + (n - 1 - d))) & 1;
|
|
358
|
+
x[d] |= bit << b;
|
|
359
|
+
}
|
|
384
360
|
}
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
x[
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
}
|
|
361
|
+
const N = 1 << bits;
|
|
362
|
+
let t = x[n - 1] >> 1;
|
|
363
|
+
for (let i = n - 1; i > 0; i--)
|
|
364
|
+
x[i] ^= x[i - 1];
|
|
365
|
+
x[0] ^= t;
|
|
366
|
+
for (let q = 2; q !== N; q <<= 1) {
|
|
367
|
+
const p = q - 1;
|
|
368
|
+
for (let i = n - 1; i >= 0; i--) {
|
|
369
|
+
if (x[i] & q)
|
|
370
|
+
x[0] ^= p;
|
|
371
|
+
else {
|
|
372
|
+
t = (x[0] ^ x[i]) & p;
|
|
373
|
+
x[0] ^= t;
|
|
374
|
+
x[i] ^= t;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
402
377
|
}
|
|
403
|
-
|
|
404
|
-
return x;
|
|
378
|
+
return x;
|
|
405
379
|
}
|
|
406
380
|
export function hilbertBytes(grid) {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
}
|
|
435
|
-
if (inside) {
|
|
436
|
-
out.push(grid.data[flat]);
|
|
381
|
+
const dims = gridDims(grid);
|
|
382
|
+
const n = dims.length;
|
|
383
|
+
if (n === 0 || grid.data.length === 0)
|
|
384
|
+
return new Uint8Array(0);
|
|
385
|
+
if (n === 1)
|
|
386
|
+
return grid.data.slice(0, dims[0]);
|
|
387
|
+
const maxAxis = Math.max(...dims);
|
|
388
|
+
const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
|
|
389
|
+
const side = 1 << bits;
|
|
390
|
+
const total = Math.pow(side, n);
|
|
391
|
+
const stride = new Array(n);
|
|
392
|
+
stride[n - 1] = 1;
|
|
393
|
+
for (let d = n - 2; d >= 0; d--)
|
|
394
|
+
stride[d] = stride[d + 1] * dims[d + 1];
|
|
395
|
+
const out = [];
|
|
396
|
+
for (let h = 0; h < total; h++) {
|
|
397
|
+
const pt = hilbertPoint(h, n, bits);
|
|
398
|
+
let inside = true, flat = 0;
|
|
399
|
+
for (let d = 0; d < n; d++) {
|
|
400
|
+
if (pt[d] >= dims[d]) {
|
|
401
|
+
inside = false;
|
|
402
|
+
break;
|
|
403
|
+
}
|
|
404
|
+
flat += pt[d] * stride[d];
|
|
405
|
+
}
|
|
406
|
+
if (inside)
|
|
407
|
+
out.push(grid.data[flat]);
|
|
437
408
|
}
|
|
438
|
-
|
|
439
|
-
return Uint8Array.from(out);
|
|
409
|
+
return Uint8Array.from(out);
|
|
440
410
|
}
|
|
441
411
|
export function gridToTree(space, alphabet, grid) {
|
|
442
|
-
|
|
412
|
+
return bytesToTree(space, alphabet, hilbertBytes(grid));
|
|
443
413
|
}
|
|
444
414
|
export function stackGrids(frames) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
};
|
|
415
|
+
if (frames.length === 0) {
|
|
416
|
+
return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
|
|
417
|
+
}
|
|
418
|
+
const frameDims = gridDims(frames[0]);
|
|
419
|
+
const per = frames[0].data.length;
|
|
420
|
+
const data = new Uint8Array(per * frames.length);
|
|
421
|
+
for (let i = 0; i < frames.length; i++)
|
|
422
|
+
data.set(frames[i].data, i * per);
|
|
423
|
+
return {
|
|
424
|
+
width: 0,
|
|
425
|
+
height: 0,
|
|
426
|
+
channels: 0,
|
|
427
|
+
dims: [frames.length, ...frameDims],
|
|
428
|
+
data,
|
|
429
|
+
};
|
|
461
430
|
}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -10,21 +10,6 @@ export * from "./store-sqlite.js";
|
|
|
10
10
|
export * from "./config.js";
|
|
11
11
|
export * from "./extension.js";
|
|
12
12
|
export * from "./ingest-cache.js";
|
|
13
|
-
export {
|
|
14
|
-
|
|
15
|
-
HnswIndex,
|
|
16
|
-
Prng,
|
|
17
|
-
RaBitQuantizer,
|
|
18
|
-
VectorDatabase,
|
|
19
|
-
} from "./rabitq-hnsw/src/index.js";
|
|
20
|
-
export type {
|
|
21
|
-
DatabaseOptions,
|
|
22
|
-
ExternalId,
|
|
23
|
-
HnswParams,
|
|
24
|
-
KnnHit,
|
|
25
|
-
QueryContext,
|
|
26
|
-
QueryResult,
|
|
27
|
-
RaBitQOptions,
|
|
28
|
-
StorageStats,
|
|
29
|
-
} from "./rabitq-hnsw/src/index.js";
|
|
13
|
+
export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
|
|
14
|
+
export type { DatabaseOptions, ExternalId, IvfConfig, IvfHit, QueryContext, QueryResult, RaBitQOptions, StorageStats, } from "./rabitq-ivf/src/index.js";
|
|
30
15
|
export * from "./derive/src/index.js";
|
package/dist/src/index.js
CHANGED
|
@@ -12,16 +12,6 @@ export * from "./store-sqlite.js";
|
|
|
12
12
|
export * from "./config.js";
|
|
13
13
|
export * from "./extension.js";
|
|
14
14
|
export * from "./ingest-cache.js";
|
|
15
|
-
// rabitq-
|
|
16
|
-
|
|
17
|
-
// `store.js` / `config.js` exports, and sema code that needs the rabitq ones
|
|
18
|
-
// imports them from the subpath directly. Re-export the public vector-DB
|
|
19
|
-
// surface under the sema root, omitting the three colliding names.
|
|
20
|
-
export {
|
|
21
|
-
Heap,
|
|
22
|
-
HnswIndex,
|
|
23
|
-
Prng,
|
|
24
|
-
RaBitQuantizer,
|
|
25
|
-
VectorDatabase,
|
|
26
|
-
} from "./rabitq-hnsw/src/index.js";
|
|
15
|
+
// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
|
|
16
|
+
export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
|
|
27
17
|
export * from "./derive/src/index.js";
|