@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
|
@@ -7,16 +7,16 @@
|
|
|
7
7
|
* allocation in the search hot path.
|
|
8
8
|
*/
|
|
9
9
|
export declare class Heap {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
10
|
+
readonly keys: number[];
|
|
11
|
+
readonly vals: number[];
|
|
12
|
+
private readonly minHeap;
|
|
13
|
+
constructor(minHeap: boolean);
|
|
14
|
+
get size(): number;
|
|
15
|
+
clear(): void;
|
|
16
|
+
topKey(): number;
|
|
17
|
+
topVal(): number;
|
|
18
|
+
/** true if `a` belongs closer to the root than `b`. */
|
|
19
|
+
private higher;
|
|
20
|
+
push(key: number, val: number): void;
|
|
21
|
+
pop(): void;
|
|
22
22
|
}
|
|
@@ -7,83 +7,88 @@
|
|
|
7
7
|
* allocation in the search hot path.
|
|
8
8
|
*/
|
|
9
9
|
export class Heap {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
keys = [];
|
|
11
|
+
vals = [];
|
|
12
|
+
minHeap;
|
|
13
|
+
constructor(minHeap) {
|
|
14
|
+
this.minHeap = minHeap;
|
|
15
|
+
}
|
|
16
|
+
get size() {
|
|
17
|
+
return this.keys.length;
|
|
18
|
+
}
|
|
19
|
+
clear() {
|
|
20
|
+
this.keys.length = 0;
|
|
21
|
+
this.vals.length = 0;
|
|
22
|
+
}
|
|
23
|
+
topKey() {
|
|
24
|
+
return this.keys[0];
|
|
25
|
+
}
|
|
26
|
+
topVal() {
|
|
27
|
+
return this.vals[0];
|
|
28
|
+
}
|
|
29
|
+
/** true if `a` belongs closer to the root than `b`. */
|
|
30
|
+
higher(a, b) {
|
|
31
|
+
return this.minHeap ? a < b : a > b;
|
|
32
|
+
}
|
|
33
|
+
push(key, val) {
|
|
34
|
+
const keys = this.keys;
|
|
35
|
+
const vals = this.vals;
|
|
36
|
+
let i = keys.length;
|
|
37
|
+
keys.push(key);
|
|
38
|
+
vals.push(val);
|
|
39
|
+
while (i > 0) {
|
|
40
|
+
const parent = (i - 1) >> 1;
|
|
41
|
+
if (this.higher(keys[i], keys[parent])) {
|
|
42
|
+
const tk = keys[i];
|
|
43
|
+
keys[i] = keys[parent];
|
|
44
|
+
keys[parent] = tk;
|
|
45
|
+
const tv = vals[i];
|
|
46
|
+
vals[i] = vals[parent];
|
|
47
|
+
vals[parent] = tv;
|
|
48
|
+
i = parent;
|
|
49
|
+
} else {
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
15
52
|
}
|
|
16
|
-
|
|
17
|
-
|
|
53
|
+
}
|
|
54
|
+
pop() {
|
|
55
|
+
const keys = this.keys;
|
|
56
|
+
const vals = this.vals;
|
|
57
|
+
const n = keys.length;
|
|
58
|
+
if (n === 0) {
|
|
59
|
+
return;
|
|
18
60
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
61
|
+
const lastKey = keys[n - 1];
|
|
62
|
+
const lastVal = vals[n - 1];
|
|
63
|
+
keys.pop();
|
|
64
|
+
vals.pop();
|
|
65
|
+
const m = keys.length;
|
|
66
|
+
if (m === 0) {
|
|
67
|
+
return;
|
|
22
68
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
vals[i] = vals[parent];
|
|
47
|
-
vals[parent] = tv;
|
|
48
|
-
i = parent;
|
|
49
|
-
}
|
|
50
|
-
else
|
|
51
|
-
break;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
pop() {
|
|
55
|
-
const keys = this.keys;
|
|
56
|
-
const vals = this.vals;
|
|
57
|
-
const n = keys.length;
|
|
58
|
-
if (n === 0)
|
|
59
|
-
return;
|
|
60
|
-
const lastKey = keys[n - 1];
|
|
61
|
-
const lastVal = vals[n - 1];
|
|
62
|
-
keys.pop();
|
|
63
|
-
vals.pop();
|
|
64
|
-
const m = keys.length;
|
|
65
|
-
if (m === 0)
|
|
66
|
-
return;
|
|
67
|
-
keys[0] = lastKey;
|
|
68
|
-
vals[0] = lastVal;
|
|
69
|
-
let i = 0;
|
|
70
|
-
while (true) {
|
|
71
|
-
const left = 2 * i + 1;
|
|
72
|
-
const right = left + 1;
|
|
73
|
-
let best = i;
|
|
74
|
-
if (left < m && this.higher(keys[left], keys[best]))
|
|
75
|
-
best = left;
|
|
76
|
-
if (right < m && this.higher(keys[right], keys[best]))
|
|
77
|
-
best = right;
|
|
78
|
-
if (best === i)
|
|
79
|
-
break;
|
|
80
|
-
const tk = keys[i];
|
|
81
|
-
keys[i] = keys[best];
|
|
82
|
-
keys[best] = tk;
|
|
83
|
-
const tv = vals[i];
|
|
84
|
-
vals[i] = vals[best];
|
|
85
|
-
vals[best] = tv;
|
|
86
|
-
i = best;
|
|
87
|
-
}
|
|
69
|
+
keys[0] = lastKey;
|
|
70
|
+
vals[0] = lastVal;
|
|
71
|
+
let i = 0;
|
|
72
|
+
while (true) {
|
|
73
|
+
const left = 2 * i + 1;
|
|
74
|
+
const right = left + 1;
|
|
75
|
+
let best = i;
|
|
76
|
+
if (left < m && this.higher(keys[left], keys[best])) {
|
|
77
|
+
best = left;
|
|
78
|
+
}
|
|
79
|
+
if (right < m && this.higher(keys[right], keys[best])) {
|
|
80
|
+
best = right;
|
|
81
|
+
}
|
|
82
|
+
if (best === i) {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
const tk = keys[i];
|
|
86
|
+
keys[i] = keys[best];
|
|
87
|
+
keys[best] = tk;
|
|
88
|
+
const tv = vals[i];
|
|
89
|
+
vals[i] = vals[best];
|
|
90
|
+
vals[best] = tv;
|
|
91
|
+
i = best;
|
|
88
92
|
}
|
|
93
|
+
}
|
|
89
94
|
}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { RaBitQuantizer } from "./rabitq.js";
|
|
2
2
|
import { Store } from "./store.js";
|
|
3
3
|
export interface HnswParams {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
M: number;
|
|
5
|
+
efConstruction: number;
|
|
6
|
+
efSearch: number;
|
|
7
|
+
seed: number;
|
|
8
8
|
}
|
|
9
9
|
export interface KnnHit {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
/** external (user) id */
|
|
11
|
+
id: number;
|
|
12
|
+
/** estimated cosine distance (1 - cosine) */
|
|
13
|
+
distance: number;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
|
|
@@ -27,99 +27,99 @@ export interface KnnHit {
|
|
|
27
27
|
* access to the original vectors.
|
|
28
28
|
*/
|
|
29
29
|
export declare class HnswIndex {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
30
|
+
readonly M: number;
|
|
31
|
+
readonly Mmax0: number;
|
|
32
|
+
readonly efConstruction: number;
|
|
33
|
+
private readonly mL;
|
|
34
|
+
private readonly quantizer;
|
|
35
|
+
private readonly store;
|
|
36
|
+
private readonly seed;
|
|
37
|
+
private readonly levelRng;
|
|
38
|
+
private entry;
|
|
39
|
+
private maxLevel;
|
|
40
|
+
private live;
|
|
41
|
+
private total;
|
|
42
|
+
private _efSearch;
|
|
43
|
+
private readonly working;
|
|
44
|
+
private readonly visited;
|
|
45
|
+
private visitedTag;
|
|
46
|
+
private visitedEpoch;
|
|
47
|
+
private seenTag;
|
|
48
|
+
private markVisited;
|
|
49
|
+
private readonly candHeap;
|
|
50
|
+
private readonly resHeap;
|
|
51
|
+
private readonly singleEp;
|
|
52
|
+
private readonly idxScratch;
|
|
53
|
+
private readonly distScratch;
|
|
54
|
+
private readonly freshScratch;
|
|
55
|
+
private qCtx;
|
|
56
|
+
private refBytes;
|
|
57
|
+
private building;
|
|
58
|
+
lastQueryDistComps: number;
|
|
59
|
+
/** Storage row reads issued by the most recent query (cache-independent). */
|
|
60
|
+
lastQueryStorageReads: number;
|
|
61
|
+
constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
|
|
62
|
+
get size(): number;
|
|
63
|
+
get physicalSize(): number;
|
|
64
|
+
get bytesPerVector(): number;
|
|
65
|
+
get efSearch(): number;
|
|
66
|
+
set efSearch(v: number);
|
|
67
|
+
private persistState;
|
|
68
|
+
private randomLevel;
|
|
69
|
+
/**
|
|
70
|
+
* Fetch a node record into the operation's working set. The first touch in an
|
|
71
|
+
* op is a storage read; later touches in the same op reuse it. The set is
|
|
72
|
+
* cleared per op and bounded by efConstruction / efSearch, so storage reads
|
|
73
|
+
* per op equal the number of *distinct* nodes the op visits -- minimal, and
|
|
74
|
+
* independent of any cache.
|
|
75
|
+
*/
|
|
76
|
+
private node;
|
|
77
|
+
/** Prefetch several nodes into the working set with ONE batched storage
|
|
78
|
+
* read for the misses. Point queries per neighbour made statement
|
|
79
|
+
* dispatch — not distance arithmetic — the dominant cost of a large
|
|
80
|
+
* build; the batch keeps `reads` accounting identical per row. */
|
|
81
|
+
private fetchNodes;
|
|
82
|
+
/** Distance from the current source (query vector or reference code) to a node. */
|
|
83
|
+
private distOf;
|
|
84
|
+
/** Code-to-code distance between two stored nodes (graph wiring only). */
|
|
85
|
+
private codeDist;
|
|
86
|
+
/**
|
|
87
|
+
* SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
|
|
88
|
+
* `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
|
|
89
|
+
* never returned. `visited` is a per-call set sized by the nodes seen here.
|
|
90
|
+
*/
|
|
91
|
+
private searchLayer;
|
|
92
|
+
/** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
|
|
93
|
+
private greedyDescend;
|
|
94
|
+
/**
|
|
95
|
+
* SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
|
|
96
|
+
* neighbours from `candIds`, preferring those closer to `base` than to any
|
|
97
|
+
* already-chosen neighbour, then fills remaining slots with the closest
|
|
98
|
+
* leftovers (keep-pruned connections). Appends to `out`.
|
|
99
|
+
*/
|
|
100
|
+
private selectNeighbors;
|
|
101
|
+
/**
|
|
102
|
+
* Insert a vector's code under external id `ext`; returns the internal node id.
|
|
103
|
+
* All graph reads/writes go through the store; the caller owns the transaction.
|
|
104
|
+
*/
|
|
105
|
+
insert(ext: number, code: Uint8Array, efC?: number): number;
|
|
106
|
+
/** k-NN with a full-precision query vector (accurate estimator). */
|
|
107
|
+
searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
|
|
108
|
+
/** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
|
|
109
|
+
searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
|
|
110
|
+
private collectKnn;
|
|
111
|
+
/** Tombstone a live node by internal id. Caller owns the transaction. */
|
|
112
|
+
remove(id: number): boolean;
|
|
113
|
+
/**
|
|
114
|
+
* Reclaim tombstones with a graph-preserving splice (see
|
|
115
|
+
* {@link Store.spliceCompact}): live nodes keep their internal ids and their
|
|
116
|
+
* wiring; each dead neighbour is replaced by its own live neighbours. Codes
|
|
117
|
+
* are untouched, so the index loses nothing beyond what deletion itself
|
|
118
|
+
* removes. The previous implementation replayed every live code through a
|
|
119
|
+
* full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
|
|
120
|
+
* multi-million-node store, inside one WAL-bloating transaction. The splice
|
|
121
|
+
* is a streaming pass with batched commits, then a VACUUM to return the
|
|
122
|
+
* freed pages.
|
|
123
|
+
*/
|
|
124
|
+
compact(): void;
|
|
125
125
|
}
|