@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
package/dist/src/geometry.js
CHANGED
|
@@ -30,7 +30,7 @@ import { sema } from "./sema.js";
|
|
|
30
30
|
* longer than the perception quantum must use the scale-aware
|
|
31
31
|
* {@link identityBar}, which converts the tolerated fraction into bytes. */
|
|
32
32
|
export function mergeThreshold(D) {
|
|
33
|
-
|
|
33
|
+
return 1 - 1 / Math.sqrt(D);
|
|
34
34
|
}
|
|
35
35
|
/** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
|
|
36
36
|
* of `len` bytes. Under the linear fold a cosine reads "fraction of aligned
|
|
@@ -46,7 +46,7 @@ export function mergeThreshold(D) {
|
|
|
46
46
|
* the tolerated fraction into tolerated bytes. Derived from W, D and the
|
|
47
47
|
* span; never tuned. */
|
|
48
48
|
export function identityBar(D, maxGroup, len) {
|
|
49
|
-
|
|
49
|
+
return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
|
|
50
50
|
}
|
|
51
51
|
/** The reach bar: half a river quantum, derived from the fold's own geometry.
|
|
52
52
|
* A branch folds up to `maxGroup` children, so two forms that differ in ONE
|
|
@@ -59,7 +59,7 @@ export function identityBar(D, maxGroup, len) {
|
|
|
59
59
|
* — further than any single-child variant — and the system returns null rather
|
|
60
60
|
* than fabricate an answer from an unrelated form. Derived, never tuned. */
|
|
61
61
|
export function reachThreshold(maxGroup) {
|
|
62
|
-
|
|
62
|
+
return 1 - 1 / (2 * maxGroup);
|
|
63
63
|
}
|
|
64
64
|
/** The estimator's own noise floor: 1/√D — ONE standard deviation of the
|
|
65
65
|
* cosine between two independent random vectors in D dimensions (the same σ
|
|
@@ -71,7 +71,7 @@ export function reachThreshold(maxGroup) {
|
|
|
71
71
|
* minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
|
|
72
72
|
* Derived, never tuned. */
|
|
73
73
|
export function estimatorNoise(D) {
|
|
74
|
-
|
|
74
|
+
return 1 / Math.sqrt(D);
|
|
75
75
|
}
|
|
76
76
|
/** The statistical-significance bar for whole-query resonance: 3/√D.
|
|
77
77
|
* In D dimensions the expected cosine of two independent random vectors is 0
|
|
@@ -81,7 +81,7 @@ export function estimatorNoise(D) {
|
|
|
81
81
|
* sub-region resonance) is skipped: there is no evidence the query belongs to
|
|
82
82
|
* the same distribution as the stored content. Derived, never tuned. */
|
|
83
83
|
export function significanceBar(D) {
|
|
84
|
-
|
|
84
|
+
return 3 / Math.sqrt(D);
|
|
85
85
|
}
|
|
86
86
|
/** The concept (halo) threshold: the cosine above which two nodes share a
|
|
87
87
|
* distributional concept. A halo is a superposition of episode signatures in
|
|
@@ -93,7 +93,7 @@ export function significanceBar(D) {
|
|
|
93
93
|
* 3% of 0.5 — existing behavior is preserved while threshold and D move
|
|
94
94
|
* together. Derived, never tuned. */
|
|
95
95
|
export function conceptThreshold(D) {
|
|
96
|
-
|
|
96
|
+
return 0.5 + 0.5 / Math.sqrt(D);
|
|
97
97
|
}
|
|
98
98
|
/** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
|
|
99
99
|
* whole of `wholeLen` covers STRICTLY more than half of it. A span that
|
|
@@ -104,7 +104,7 @@ export function conceptThreshold(D) {
|
|
|
104
104
|
* already unusable there) and stays inline where it is documented.
|
|
105
105
|
* Derived from the structural midpoint, never tuned. */
|
|
106
106
|
export function dominates(partLen, wholeLen) {
|
|
107
|
-
|
|
107
|
+
return partLen * 2 > wholeLen;
|
|
108
108
|
}
|
|
109
109
|
/** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
|
|
110
110
|
* of learnt contexts (edge sources). A single region's IDF-weighted vote for
|
|
@@ -116,7 +116,7 @@ export function dominates(partLen, wholeLen) {
|
|
|
116
116
|
* further point of attention. Defined once here so the two can never
|
|
117
117
|
* drift apart. Derived from N, never tuned. */
|
|
118
118
|
export function consensusFloor(N) {
|
|
119
|
-
|
|
119
|
+
return Math.log(N) + 1 / 2;
|
|
120
120
|
}
|
|
121
121
|
/** The coverage bar for the reach (interior) index, when vector-similarity
|
|
122
122
|
* gating is used. Returns the concept threshold — the structural midpoint
|
|
@@ -128,7 +128,7 @@ export function consensusFloor(N) {
|
|
|
128
128
|
* replaces runtime coverage gating with a batch pass that removes
|
|
129
129
|
* structurally-isolated entries. Derived, never tuned. */
|
|
130
130
|
export function coverageBar(_maxGroup, D) {
|
|
131
|
-
|
|
131
|
+
return conceptThreshold(D);
|
|
132
132
|
}
|
|
133
133
|
// ---- folding ----
|
|
134
134
|
//
|
|
@@ -160,271 +160,302 @@ export function coverageBar(_maxGroup, D) {
|
|
|
160
160
|
* well — only a lone singleton passes through — guaranteeing progress when
|
|
161
161
|
* the caller detected a stall. */
|
|
162
162
|
function foldSlice(space, items, start, count, out, force) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
163
|
+
const mg = space.maxGroup;
|
|
164
|
+
const D = space.D;
|
|
165
|
+
const complete = count - (count % mg);
|
|
166
|
+
const foldAt = (at, size) => {
|
|
167
|
+
const gist = new Float32Array(D);
|
|
168
|
+
const kids = new Array(size);
|
|
169
|
+
let len = 0;
|
|
170
|
+
for (let k = 0; k < size; k++) {
|
|
171
|
+
const f = items[at + k];
|
|
172
|
+
const seat = space.seats[k].fwd;
|
|
173
|
+
const v = f.tree.v;
|
|
174
|
+
// Fused permute-and-accumulate — same FP ops, same order as the old
|
|
175
|
+
// permuteInto + addInto pair, with no scratch buffer.
|
|
176
|
+
for (let d = 0; d < D; d++) {
|
|
177
|
+
gist[d] += v[seat[d]];
|
|
178
|
+
}
|
|
179
|
+
kids[k] = f.tree;
|
|
180
|
+
len += f.len;
|
|
181
|
+
}
|
|
182
|
+
out.push({ tree: sema(gist, null, kids), len });
|
|
183
|
+
};
|
|
184
|
+
for (let i = 0; i < complete; i += mg) {
|
|
185
|
+
foldAt(start + i, mg);
|
|
186
|
+
}
|
|
187
|
+
const leftover = count - complete;
|
|
188
|
+
if (leftover === 0) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
if (force && leftover >= 2) {
|
|
192
|
+
foldAt(start + complete, leftover);
|
|
193
|
+
} else {
|
|
194
|
+
for (let i = complete; i < count; i++) {
|
|
195
|
+
out.push(items[start + i]);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
193
198
|
}
|
|
194
199
|
function riverFold(space, row, stableBytes) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
const next = [];
|
|
214
|
-
if (boundary < level.length) {
|
|
215
|
-
// Prefix folds independently of the suffix — structural stability.
|
|
216
|
-
foldSlice(space, level, 0, boundary, next, false);
|
|
217
|
-
foldSlice(space, level, boundary, level.length - boundary, next, false);
|
|
218
|
-
}
|
|
219
|
-
else {
|
|
220
|
-
foldSlice(space, level, 0, level.length, next, false);
|
|
221
|
-
}
|
|
222
|
-
if (next.length === level.length) {
|
|
223
|
-
// Stuck — every group was incomplete. Force-fold to break the stall.
|
|
224
|
-
const forced = [];
|
|
225
|
-
foldSlice(space, next, 0, next.length, forced, true);
|
|
226
|
-
level = forced;
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
level = next;
|
|
200
|
+
if (row.length === 0) {
|
|
201
|
+
const z = new Float32Array(space.D);
|
|
202
|
+
return { tree: sema(z, new Uint8Array(0), null), len: 0 };
|
|
203
|
+
}
|
|
204
|
+
let level = row;
|
|
205
|
+
while (level.length > 1) {
|
|
206
|
+
// Find the item index where accumulated bytes reaches stableBytes.
|
|
207
|
+
let boundary = level.length;
|
|
208
|
+
if (stableBytes > 0) {
|
|
209
|
+
let acc = 0;
|
|
210
|
+
for (let i = 0; i < level.length; i++) {
|
|
211
|
+
acc += level[i].len;
|
|
212
|
+
if (acc >= stableBytes) {
|
|
213
|
+
boundary = i + 1;
|
|
214
|
+
break;
|
|
230
215
|
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const next = [];
|
|
219
|
+
if (boundary < level.length) {
|
|
220
|
+
// Prefix folds independently of the suffix — structural stability.
|
|
221
|
+
foldSlice(space, level, 0, boundary, next, false);
|
|
222
|
+
foldSlice(space, level, boundary, level.length - boundary, next, false);
|
|
223
|
+
} else {
|
|
224
|
+
foldSlice(space, level, 0, level.length, next, false);
|
|
225
|
+
}
|
|
226
|
+
if (next.length === level.length) {
|
|
227
|
+
// Stuck — every group was incomplete. Force-fold to break the stall.
|
|
228
|
+
const forced = [];
|
|
229
|
+
foldSlice(space, next, 0, next.length, forced, true);
|
|
230
|
+
level = forced;
|
|
231
|
+
} else {
|
|
232
|
+
level = next;
|
|
231
233
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
234
|
+
}
|
|
235
|
+
// LINEAR fold — this root normalize is the ONLY normalize of the entire
|
|
236
|
+
// fold; every intermediate gist stays unnormalized (see the folding
|
|
237
|
+
// header). Skipped for a single-leaf input: that root IS the shared
|
|
238
|
+
// alphabet vector (already unit), and normalizing in place would mutate the
|
|
239
|
+
// alphabet itself.
|
|
240
|
+
if (row.length > 1) {
|
|
241
|
+
normalize(level[0].tree.v);
|
|
242
|
+
}
|
|
243
|
+
return level[0];
|
|
240
244
|
}
|
|
241
245
|
// ---- public API ----
|
|
242
246
|
function bytesToLeaves(alphabet, bytes) {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
+
return Array.from(bytes, (b, i) => {
|
|
248
|
+
const v = alphabet.vecs[b];
|
|
249
|
+
return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
|
|
250
|
+
});
|
|
247
251
|
}
|
|
248
252
|
/** Find the longest prefix of `bytes` whose leaf-id signature matches a
|
|
249
253
|
* known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
|
|
250
254
|
export function knownPrefixLength(bytes, leafAt, lookup) {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
leafIds.push(lid);
|
|
255
|
+
const leafIds = [];
|
|
256
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
257
|
+
const lid = leafAt(i);
|
|
258
|
+
if (lid === null) {
|
|
259
|
+
break;
|
|
257
260
|
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
261
|
+
leafIds.push(lid);
|
|
262
|
+
}
|
|
263
|
+
// Match the longest PROPER prefix — a full-length match means the entire
|
|
264
|
+
// input already exists as a stored form (e.g. the flat leaf-id branch
|
|
265
|
+
// stored alongside the structural root). That would hide the true split
|
|
266
|
+
// point and prevent the river from producing the same tree it folded
|
|
267
|
+
// during training, so the structural recognition cannot find the right
|
|
268
|
+
// forms. A proper prefix guarantees at least two regions.
|
|
269
|
+
for (let len = bytes.length - 1; len >= 2; len--) {
|
|
270
|
+
if (lookup(leafIds.slice(0, len)) !== null) {
|
|
271
|
+
return len;
|
|
267
272
|
}
|
|
268
|
-
|
|
273
|
+
}
|
|
274
|
+
return 0;
|
|
269
275
|
}
|
|
270
276
|
/** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
|
|
271
277
|
* detecting previously-stored prefixes so the river can split at the
|
|
272
278
|
* correct boundary. Pass them through from `perceive`; the geometry
|
|
273
279
|
* computes the stable prefix internally. */
|
|
274
280
|
export function bytesToTree(space, alphabet, bytes, leafAt, lookup) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
281
|
+
if (bytes.length === 0) {
|
|
282
|
+
return sema(alphabet.vecs[0], new Uint8Array(0), null);
|
|
283
|
+
}
|
|
284
|
+
const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
|
|
285
|
+
return riverFold(
|
|
286
|
+
space,
|
|
287
|
+
bytesToLeaves(alphabet, bytes),
|
|
288
|
+
sb > 0 ? sb : bytes.length,
|
|
289
|
+
).tree;
|
|
280
290
|
}
|
|
281
291
|
/** Plain bytes→tree (identical to capability-less {@link bytesToTree}) that
|
|
282
292
|
* also RETURNS its pyramid, reusing `prev` — the pyramid of a PROPER
|
|
283
293
|
* prefix of `bytes` (caller guarantees content match and
|
|
284
294
|
* prev.bytes < bytes.length). */
|
|
285
295
|
export function bytesToTreePyramid(space, alphabet, bytes, prev) {
|
|
286
|
-
|
|
287
|
-
return {
|
|
288
|
-
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
289
|
-
pyramid: { levels: [], bytes: 0 },
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
|
-
const mg = space.maxGroup;
|
|
293
|
-
const reusable = (L) => {
|
|
294
|
-
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
295
|
-
// be raw interiors, so the top level is always excluded.
|
|
296
|
-
if (!prev || L > prev.levels.length - 2)
|
|
297
|
-
return null;
|
|
298
|
-
return prev.levels[L];
|
|
299
|
-
};
|
|
300
|
-
// Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
|
|
301
|
-
const lv0 = reusable(0);
|
|
302
|
-
const row = lv0
|
|
303
|
-
? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
|
|
304
|
-
: bytesToLeaves(alphabet, bytes);
|
|
305
|
-
const levels = [row];
|
|
306
|
-
let level = row;
|
|
307
|
-
let L = 0;
|
|
308
|
-
while (level.length > 1) {
|
|
309
|
-
let next = [];
|
|
310
|
-
// Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
|
|
311
|
-
// wholly inside the prefix); the rule below folds the rest from this
|
|
312
|
-
// level's items exactly as the from-scratch fold would.
|
|
313
|
-
const blockLen = mg ** (L + 1);
|
|
314
|
-
let reused = 0;
|
|
315
|
-
const cand = reusable(L + 1);
|
|
316
|
-
if (cand) {
|
|
317
|
-
const maxFull = Math.floor(prev.bytes / blockLen);
|
|
318
|
-
while (reused < maxFull && reused < cand.length &&
|
|
319
|
-
cand[reused].len === blockLen)
|
|
320
|
-
reused++;
|
|
321
|
-
for (let i = 0; i < reused; i++)
|
|
322
|
-
next.push(cand[i]);
|
|
323
|
-
}
|
|
324
|
-
foldSlice(space, level, reused * mg, level.length - reused * mg, next, false);
|
|
325
|
-
if (next.length === level.length) {
|
|
326
|
-
// Stuck — same force-fold as riverFold (reuse cannot cause it: any
|
|
327
|
-
// reused block replaces a whole folded group, so reused > 0 implies
|
|
328
|
-
// progress).
|
|
329
|
-
const forced = [];
|
|
330
|
-
foldSlice(space, next, 0, next.length, forced, true);
|
|
331
|
-
next = forced;
|
|
332
|
-
}
|
|
333
|
-
levels.push(next);
|
|
334
|
-
level = next;
|
|
335
|
-
L++;
|
|
336
|
-
}
|
|
337
|
-
if (row.length > 1)
|
|
338
|
-
normalize(level[0].tree.v);
|
|
296
|
+
if (bytes.length === 0) {
|
|
339
297
|
return {
|
|
340
|
-
|
|
341
|
-
|
|
298
|
+
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
299
|
+
pyramid: { levels: [], bytes: 0 },
|
|
342
300
|
};
|
|
301
|
+
}
|
|
302
|
+
const mg = space.maxGroup;
|
|
303
|
+
const reusable = (L) => {
|
|
304
|
+
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
305
|
+
// be raw interiors, so the top level is always excluded.
|
|
306
|
+
if (!prev || L > prev.levels.length - 2) {
|
|
307
|
+
return null;
|
|
308
|
+
}
|
|
309
|
+
return prev.levels[L];
|
|
310
|
+
};
|
|
311
|
+
// Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
|
|
312
|
+
const lv0 = reusable(0);
|
|
313
|
+
const row = lv0
|
|
314
|
+
? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
|
|
315
|
+
: bytesToLeaves(alphabet, bytes);
|
|
316
|
+
const levels = [row];
|
|
317
|
+
let level = row;
|
|
318
|
+
let L = 0;
|
|
319
|
+
while (level.length > 1) {
|
|
320
|
+
let next = [];
|
|
321
|
+
// Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
|
|
322
|
+
// wholly inside the prefix); the rule below folds the rest from this
|
|
323
|
+
// level's items exactly as the from-scratch fold would.
|
|
324
|
+
const blockLen = mg ** (L + 1);
|
|
325
|
+
let reused = 0;
|
|
326
|
+
const cand = reusable(L + 1);
|
|
327
|
+
if (cand) {
|
|
328
|
+
const maxFull = Math.floor(prev.bytes / blockLen);
|
|
329
|
+
while (
|
|
330
|
+
reused < maxFull && reused < cand.length &&
|
|
331
|
+
cand[reused].len === blockLen
|
|
332
|
+
) {
|
|
333
|
+
reused++;
|
|
334
|
+
}
|
|
335
|
+
for (let i = 0; i < reused; i++) {
|
|
336
|
+
next.push(cand[i]);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
foldSlice(
|
|
340
|
+
space,
|
|
341
|
+
level,
|
|
342
|
+
reused * mg,
|
|
343
|
+
level.length - reused * mg,
|
|
344
|
+
next,
|
|
345
|
+
false,
|
|
346
|
+
);
|
|
347
|
+
if (next.length === level.length) {
|
|
348
|
+
// Stuck — same force-fold as riverFold (reuse cannot cause it: any
|
|
349
|
+
// reused block replaces a whole folded group, so reused > 0 implies
|
|
350
|
+
// progress).
|
|
351
|
+
const forced = [];
|
|
352
|
+
foldSlice(space, next, 0, next.length, forced, true);
|
|
353
|
+
next = forced;
|
|
354
|
+
}
|
|
355
|
+
levels.push(next);
|
|
356
|
+
level = next;
|
|
357
|
+
L++;
|
|
358
|
+
}
|
|
359
|
+
if (row.length > 1) {
|
|
360
|
+
normalize(level[0].tree.v);
|
|
361
|
+
}
|
|
362
|
+
return {
|
|
363
|
+
tree: level[0].tree,
|
|
364
|
+
pyramid: { levels, bytes: bytes.length },
|
|
365
|
+
};
|
|
343
366
|
}
|
|
344
367
|
// ---- n-D Hilbert curve ----
|
|
345
368
|
function gridDims(grid) {
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
369
|
+
if (grid.dims && grid.dims.length > 0) {
|
|
370
|
+
return grid.dims.slice();
|
|
371
|
+
}
|
|
372
|
+
const dims = [grid.height, grid.width];
|
|
373
|
+
if (grid.channels > 1) {
|
|
374
|
+
dims.push(grid.channels);
|
|
375
|
+
}
|
|
376
|
+
return dims;
|
|
352
377
|
}
|
|
353
378
|
function hilbertPoint(index, n, bits) {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
}
|
|
379
|
+
const x = new Array(n).fill(0);
|
|
380
|
+
for (let b = 0; b < bits; b++) {
|
|
381
|
+
for (let d = 0; d < n; d++) {
|
|
382
|
+
const bit = (index >>> (b * n + (n - 1 - d))) & 1;
|
|
383
|
+
x[d] |= bit << b;
|
|
360
384
|
}
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
x[
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
385
|
+
}
|
|
386
|
+
const N = 1 << bits;
|
|
387
|
+
let t = x[n - 1] >> 1;
|
|
388
|
+
for (let i = n - 1; i > 0; i--) {
|
|
389
|
+
x[i] ^= x[i - 1];
|
|
390
|
+
}
|
|
391
|
+
x[0] ^= t;
|
|
392
|
+
for (let q = 2; q !== N; q <<= 1) {
|
|
393
|
+
const p = q - 1;
|
|
394
|
+
for (let i = n - 1; i >= 0; i--) {
|
|
395
|
+
if (x[i] & q) {
|
|
396
|
+
x[0] ^= p;
|
|
397
|
+
} else {
|
|
398
|
+
t = (x[0] ^ x[i]) & p;
|
|
399
|
+
x[0] ^= t;
|
|
400
|
+
x[i] ^= t;
|
|
401
|
+
}
|
|
377
402
|
}
|
|
378
|
-
|
|
403
|
+
}
|
|
404
|
+
return x;
|
|
379
405
|
}
|
|
380
406
|
export function hilbertBytes(grid) {
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
407
|
+
const dims = gridDims(grid);
|
|
408
|
+
const n = dims.length;
|
|
409
|
+
if (n === 0 || grid.data.length === 0) {
|
|
410
|
+
return new Uint8Array(0);
|
|
411
|
+
}
|
|
412
|
+
if (n === 1) {
|
|
413
|
+
return grid.data.slice(0, dims[0]);
|
|
414
|
+
}
|
|
415
|
+
const maxAxis = Math.max(...dims);
|
|
416
|
+
const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
|
|
417
|
+
const side = 1 << bits;
|
|
418
|
+
const total = Math.pow(side, n);
|
|
419
|
+
const stride = new Array(n);
|
|
420
|
+
stride[n - 1] = 1;
|
|
421
|
+
for (let d = n - 2; d >= 0; d--) {
|
|
422
|
+
stride[d] = stride[d + 1] * dims[d + 1];
|
|
423
|
+
}
|
|
424
|
+
const out = [];
|
|
425
|
+
for (let h = 0; h < total; h++) {
|
|
426
|
+
const pt = hilbertPoint(h, n, bits);
|
|
427
|
+
let inside = true, flat = 0;
|
|
428
|
+
for (let d = 0; d < n; d++) {
|
|
429
|
+
if (pt[d] >= dims[d]) {
|
|
430
|
+
inside = false;
|
|
431
|
+
break;
|
|
432
|
+
}
|
|
433
|
+
flat += pt[d] * stride[d];
|
|
434
|
+
}
|
|
435
|
+
if (inside) {
|
|
436
|
+
out.push(grid.data[flat]);
|
|
408
437
|
}
|
|
409
|
-
|
|
438
|
+
}
|
|
439
|
+
return Uint8Array.from(out);
|
|
410
440
|
}
|
|
411
441
|
export function gridToTree(space, alphabet, grid) {
|
|
412
|
-
|
|
442
|
+
return bytesToTree(space, alphabet, hilbertBytes(grid));
|
|
413
443
|
}
|
|
414
444
|
export function stackGrids(frames) {
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
445
|
+
if (frames.length === 0) {
|
|
446
|
+
return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
|
|
447
|
+
}
|
|
448
|
+
const frameDims = gridDims(frames[0]);
|
|
449
|
+
const per = frames[0].data.length;
|
|
450
|
+
const data = new Uint8Array(per * frames.length);
|
|
451
|
+
for (let i = 0; i < frames.length; i++) {
|
|
452
|
+
data.set(frames[i].data, i * per);
|
|
453
|
+
}
|
|
454
|
+
return {
|
|
455
|
+
width: 0,
|
|
456
|
+
height: 0,
|
|
457
|
+
channels: 0,
|
|
458
|
+
dims: [frames.length, ...frameDims],
|
|
459
|
+
data,
|
|
460
|
+
};
|
|
430
461
|
}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -10,6 +10,21 @@ export * from "./store-sqlite.js";
|
|
|
10
10
|
export * from "./config.js";
|
|
11
11
|
export * from "./extension.js";
|
|
12
12
|
export * from "./ingest-cache.js";
|
|
13
|
-
export {
|
|
14
|
-
|
|
13
|
+
export {
|
|
14
|
+
Heap,
|
|
15
|
+
HnswIndex,
|
|
16
|
+
Prng,
|
|
17
|
+
RaBitQuantizer,
|
|
18
|
+
VectorDatabase,
|
|
19
|
+
} from "./rabitq-hnsw/src/index.js";
|
|
20
|
+
export type {
|
|
21
|
+
DatabaseOptions,
|
|
22
|
+
ExternalId,
|
|
23
|
+
HnswParams,
|
|
24
|
+
KnnHit,
|
|
25
|
+
QueryContext,
|
|
26
|
+
QueryResult,
|
|
27
|
+
RaBitQOptions,
|
|
28
|
+
StorageStats,
|
|
29
|
+
} from "./rabitq-hnsw/src/index.js";
|
|
15
30
|
export * from "./derive/src/index.js";
|
package/dist/src/index.js
CHANGED
|
@@ -17,5 +17,11 @@ export * from "./ingest-cache.js";
|
|
|
17
17
|
// `store.js` / `config.js` exports, and sema code that needs the rabitq ones
|
|
18
18
|
// imports them from the subpath directly. Re-export the public vector-DB
|
|
19
19
|
// surface under the sema root, omitting the three colliding names.
|
|
20
|
-
export {
|
|
20
|
+
export {
|
|
21
|
+
Heap,
|
|
22
|
+
HnswIndex,
|
|
23
|
+
Prng,
|
|
24
|
+
RaBitQuantizer,
|
|
25
|
+
VectorDatabase,
|
|
26
|
+
} from "./rabitq-hnsw/src/index.js";
|
|
21
27
|
export * from "./derive/src/index.js";
|