@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
|
@@ -15,460 +15,486 @@ import { Prng } from "./prng.js";
|
|
|
15
15
|
* access to the original vectors.
|
|
16
16
|
*/
|
|
17
17
|
export class HnswIndex {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
18
|
+
M;
|
|
19
|
+
Mmax0;
|
|
20
|
+
efConstruction;
|
|
21
|
+
mL;
|
|
22
|
+
quantizer;
|
|
23
|
+
store;
|
|
24
|
+
seed;
|
|
25
|
+
levelRng;
|
|
26
|
+
// tiny global scalars, cached from meta and written through on change
|
|
27
|
+
entry;
|
|
28
|
+
maxLevel;
|
|
29
|
+
live;
|
|
30
|
+
total;
|
|
31
|
+
_efSearch;
|
|
32
|
+
// Per-operation working set: the records the current insert/query is actively
|
|
33
|
+
// comparing. Cleared at the start of every op and bounded by efConstruction /
|
|
34
|
+
// efSearch (the nodes one operation can touch), never by the collection size.
|
|
35
|
+
// This is the algorithm's working memory -- it makes each touched node cost one
|
|
36
|
+
// storage read per op -- not a tunable cache. The only cache is SQLite's page
|
|
37
|
+
// cache (see Store), sized in MB.
|
|
38
|
+
working = new Map();
|
|
39
|
+
visited = new Set();
|
|
40
|
+
// Epoch-tagged visited marks — the Set above without per-candidate hashing:
|
|
41
|
+
// visitedTag[id] === visitedEpoch means "seen this operation", and bumping
|
|
42
|
+
// the epoch clears the whole set in O(1). Semantically IDENTICAL to the
|
|
43
|
+
// Set (results never differ); used only when the store has a cache budget,
|
|
44
|
+
// because the array is 4 B per internal id — amortized working memory that
|
|
45
|
+
// grows with the collection, which the cacheSizeMb:0 flat-memory mode must
|
|
46
|
+
// not pay. The epoch wrap (2³²−1 ops) refills the array once.
|
|
47
|
+
visitedTag = new Uint32Array(1024);
|
|
48
|
+
visitedEpoch = 0;
|
|
49
|
+
seenTag(id, epoch) {
|
|
50
|
+
return id < this.visitedTag.length && this.visitedTag[id] === epoch;
|
|
51
|
+
}
|
|
52
|
+
markVisited(id, epoch) {
|
|
53
|
+
if (id >= this.visitedTag.length) {
|
|
54
|
+
const grown = new Uint32Array(
|
|
55
|
+
Math.max(id + 1, this.visitedTag.length * 2),
|
|
56
|
+
);
|
|
57
|
+
grown.set(this.visitedTag);
|
|
58
|
+
this.visitedTag = grown;
|
|
51
59
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
60
|
+
this.visitedTag[id] = epoch;
|
|
61
|
+
}
|
|
62
|
+
candHeap = new Heap(true); // min-heap: nearest first
|
|
63
|
+
resHeap = new Heap(false); // max-heap: farthest first
|
|
64
|
+
singleEp = [0];
|
|
65
|
+
idxScratch = [];
|
|
66
|
+
distScratch = [];
|
|
67
|
+
freshScratch = [];
|
|
68
|
+
// distance dispatch: a full-precision query sets qCtx; building or a code
|
|
69
|
+
// query sets refBytes (code<->code). `building` only suppresses the counter.
|
|
70
|
+
qCtx = null;
|
|
71
|
+
refBytes = null;
|
|
72
|
+
building = false;
|
|
73
|
+
lastQueryDistComps = 0;
|
|
74
|
+
/** Storage row reads issued by the most recent query (cache-independent). */
|
|
75
|
+
lastQueryStorageReads = 0;
|
|
76
|
+
constructor(quantizer, store, params) {
|
|
77
|
+
this.quantizer = quantizer;
|
|
78
|
+
this.store = store;
|
|
79
|
+
this.M = params.M;
|
|
80
|
+
this.Mmax0 = this.M * 2;
|
|
81
|
+
this.efConstruction = params.efConstruction;
|
|
82
|
+
this._efSearch = params.efSearch;
|
|
83
|
+
this.mL = 1 / Math.log(this.M);
|
|
84
|
+
this.seed = params.seed >>> 0;
|
|
85
|
+
const s = store.loadState();
|
|
86
|
+
this.entry = s.entryPoint;
|
|
87
|
+
this.maxLevel = s.maxLevel;
|
|
88
|
+
this.live = s.live;
|
|
89
|
+
this.total = s.total;
|
|
90
|
+
this.levelRng = new Prng(this.seed);
|
|
91
|
+
this.levelRng.restore(s.rng);
|
|
92
|
+
}
|
|
93
|
+
get size() {
|
|
94
|
+
return this.live;
|
|
95
|
+
}
|
|
96
|
+
get physicalSize() {
|
|
97
|
+
return this.total;
|
|
98
|
+
}
|
|
99
|
+
get bytesPerVector() {
|
|
100
|
+
return this.quantizer.codeWords * 4;
|
|
101
|
+
}
|
|
102
|
+
get efSearch() {
|
|
103
|
+
return this._efSearch;
|
|
104
|
+
}
|
|
105
|
+
set efSearch(v) {
|
|
106
|
+
this._efSearch = Math.max(1, v | 0);
|
|
107
|
+
this.store.setEfSearch(this._efSearch);
|
|
108
|
+
}
|
|
109
|
+
persistState() {
|
|
110
|
+
this.store.saveState({
|
|
111
|
+
entryPoint: this.entry,
|
|
112
|
+
maxLevel: this.maxLevel,
|
|
113
|
+
live: this.live,
|
|
114
|
+
total: this.total,
|
|
115
|
+
rng: this.levelRng.snapshot(),
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
randomLevel() {
|
|
119
|
+
let u = this.levelRng.next();
|
|
120
|
+
if (u < 1e-12) {
|
|
121
|
+
u = 1e-12;
|
|
96
122
|
}
|
|
97
|
-
|
|
98
|
-
|
|
123
|
+
return Math.floor(-Math.log(u) * this.mL);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Fetch a node record into the operation's working set. The first touch in an
|
|
127
|
+
* op is a storage read; later touches in the same op reuse it. The set is
|
|
128
|
+
* cleared per op and bounded by efConstruction / efSearch, so storage reads
|
|
129
|
+
* per op equal the number of *distinct* nodes the op visits -- minimal, and
|
|
130
|
+
* independent of any cache.
|
|
131
|
+
*/
|
|
132
|
+
node(id) {
|
|
133
|
+
const hit = this.working.get(id);
|
|
134
|
+
if (hit !== undefined) {
|
|
135
|
+
return hit;
|
|
99
136
|
}
|
|
100
|
-
|
|
101
|
-
|
|
137
|
+
const rec = this.store.getNode(id);
|
|
138
|
+
if (rec === null) {
|
|
139
|
+
throw new Error(`node ${id} not found`);
|
|
102
140
|
}
|
|
103
|
-
set
|
|
104
|
-
|
|
105
|
-
|
|
141
|
+
this.working.set(id, rec);
|
|
142
|
+
return rec;
|
|
143
|
+
}
|
|
144
|
+
/** Prefetch several nodes into the working set with ONE batched storage
|
|
145
|
+
* read for the misses. Point queries per neighbour made statement
|
|
146
|
+
* dispatch — not distance arithmetic — the dominant cost of a large
|
|
147
|
+
* build; the batch keeps `reads` accounting identical per row. */
|
|
148
|
+
fetchNodes(ids, count = ids.length) {
|
|
149
|
+
// getNodesInto skips ids already present in `working` itself — no
|
|
150
|
+
// second pre-filter pass over the same map here.
|
|
151
|
+
this.store.getNodesInto(ids, this.working, count);
|
|
152
|
+
}
|
|
153
|
+
/** Distance from the current source (query vector or reference code) to a node. */
|
|
154
|
+
distOf(rec) {
|
|
155
|
+
if (this.qCtx !== null) {
|
|
156
|
+
this.lastQueryDistComps++;
|
|
157
|
+
return this.quantizer.estimate(rec.code, 0, this.qCtx);
|
|
106
158
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
entryPoint: this.entry,
|
|
110
|
-
maxLevel: this.maxLevel,
|
|
111
|
-
live: this.live,
|
|
112
|
-
total: this.total,
|
|
113
|
-
rng: this.levelRng.snapshot(),
|
|
114
|
-
});
|
|
159
|
+
if (!this.building) {
|
|
160
|
+
this.lastQueryDistComps++;
|
|
115
161
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
162
|
+
return this.quantizer.codeDistanceBytes(this.refBytes, rec.code);
|
|
163
|
+
}
|
|
164
|
+
/** Code-to-code distance between two stored nodes (graph wiring only). */
|
|
165
|
+
codeDist(a, b) {
|
|
166
|
+
return this.quantizer.codeDistanceBytes(
|
|
167
|
+
this.node(a).code,
|
|
168
|
+
this.node(b).code,
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
|
|
173
|
+
* `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
|
|
174
|
+
* never returned. `visited` is a per-call set sized by the nodes seen here.
|
|
175
|
+
*/
|
|
176
|
+
searchLayer(entryPoints, ef, layer) {
|
|
177
|
+
const cand = this.candHeap;
|
|
178
|
+
const res = this.resHeap;
|
|
179
|
+
cand.clear();
|
|
180
|
+
res.clear();
|
|
181
|
+
// Visited marks: epoch tags when RAM-for-speed is allowed, the plain Set
|
|
182
|
+
// in flat-memory mode. Identical semantics either way.
|
|
183
|
+
const useTags = this.store.cacheEnabled;
|
|
184
|
+
let epoch = 0;
|
|
185
|
+
const visited = this.visited;
|
|
186
|
+
if (useTags) {
|
|
187
|
+
epoch = ++this.visitedEpoch;
|
|
188
|
+
if (epoch === 0xffffffff) {
|
|
189
|
+
this.visitedTag.fill(0);
|
|
190
|
+
epoch = this.visitedEpoch = 1;
|
|
191
|
+
}
|
|
192
|
+
} else {
|
|
193
|
+
visited.clear();
|
|
121
194
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
* independent of any cache.
|
|
128
|
-
*/
|
|
129
|
-
node(id) {
|
|
130
|
-
const hit = this.working.get(id);
|
|
131
|
-
if (hit !== undefined)
|
|
132
|
-
return hit;
|
|
133
|
-
const rec = this.store.getNode(id);
|
|
134
|
-
if (rec === null)
|
|
135
|
-
throw new Error(`node ${id} not found`);
|
|
136
|
-
this.working.set(id, rec);
|
|
137
|
-
return rec;
|
|
138
|
-
}
|
|
139
|
-
/** Prefetch several nodes into the working set with ONE batched storage
|
|
140
|
-
* read for the misses. Point queries per neighbour made statement
|
|
141
|
-
* dispatch — not distance arithmetic — the dominant cost of a large
|
|
142
|
-
* build; the batch keeps `reads` accounting identical per row. */
|
|
143
|
-
fetchNodes(ids, count = ids.length) {
|
|
144
|
-
// getNodesInto skips ids already present in `working` itself — no
|
|
145
|
-
// second pre-filter pass over the same map here.
|
|
146
|
-
this.store.getNodesInto(ids, this.working, count);
|
|
147
|
-
}
|
|
148
|
-
/** Distance from the current source (query vector or reference code) to a node. */
|
|
149
|
-
distOf(rec) {
|
|
150
|
-
if (this.qCtx !== null) {
|
|
151
|
-
this.lastQueryDistComps++;
|
|
152
|
-
return this.quantizer.estimate(rec.code, 0, this.qCtx);
|
|
195
|
+
for (let k = 0; k < entryPoints.length; k++) {
|
|
196
|
+
const ep = entryPoints[k];
|
|
197
|
+
if (useTags) {
|
|
198
|
+
if (this.seenTag(ep, epoch)) {
|
|
199
|
+
continue;
|
|
153
200
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
/** Code-to-code distance between two stored nodes (graph wiring only). */
|
|
159
|
-
codeDist(a, b) {
|
|
160
|
-
return this.quantizer.codeDistanceBytes(this.node(a).code, this.node(b).code);
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
|
|
164
|
-
* `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
|
|
165
|
-
* never returned. `visited` is a per-call set sized by the nodes seen here.
|
|
166
|
-
*/
|
|
167
|
-
searchLayer(entryPoints, ef, layer) {
|
|
168
|
-
const cand = this.candHeap;
|
|
169
|
-
const res = this.resHeap;
|
|
170
|
-
cand.clear();
|
|
171
|
-
res.clear();
|
|
172
|
-
// Visited marks: epoch tags when RAM-for-speed is allowed, the plain Set
|
|
173
|
-
// in flat-memory mode. Identical semantics either way.
|
|
174
|
-
const useTags = this.store.cacheEnabled;
|
|
175
|
-
let epoch = 0;
|
|
176
|
-
const visited = this.visited;
|
|
177
|
-
if (useTags) {
|
|
178
|
-
epoch = ++this.visitedEpoch;
|
|
179
|
-
if (epoch === 0xffffffff) {
|
|
180
|
-
this.visitedTag.fill(0);
|
|
181
|
-
epoch = this.visitedEpoch = 1;
|
|
182
|
-
}
|
|
201
|
+
this.markVisited(ep, epoch);
|
|
202
|
+
} else {
|
|
203
|
+
if (visited.has(ep)) {
|
|
204
|
+
continue;
|
|
183
205
|
}
|
|
184
|
-
|
|
185
|
-
|
|
206
|
+
visited.add(ep);
|
|
207
|
+
}
|
|
208
|
+
const rec = this.node(ep);
|
|
209
|
+
const d = this.distOf(rec);
|
|
210
|
+
cand.push(d, ep);
|
|
211
|
+
if (rec.deleted === 0) {
|
|
212
|
+
res.push(d, ep);
|
|
213
|
+
if (res.size > ef) {
|
|
214
|
+
res.pop();
|
|
186
215
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
while (cand.size > 0) {
|
|
219
|
+
const cd = cand.topKey();
|
|
220
|
+
if (res.size >= ef && cd > res.topKey()) {
|
|
221
|
+
break;
|
|
222
|
+
}
|
|
223
|
+
const c = cand.topVal();
|
|
224
|
+
cand.pop();
|
|
225
|
+
const nbrs = this.store.getNeighbors(c, layer);
|
|
226
|
+
if (nbrs === null) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
// Batch: mark the unvisited neighbours, fetch their codes in one
|
|
230
|
+
// storage read, then score them in the original order.
|
|
231
|
+
const fresh = this.freshScratch;
|
|
232
|
+
fresh.length = 0;
|
|
233
|
+
for (let i = 0; i < nbrs.length; i++) {
|
|
234
|
+
const e = nbrs[i];
|
|
235
|
+
if (useTags) {
|
|
236
|
+
if (this.seenTag(e, epoch)) {
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
this.markVisited(e, epoch);
|
|
240
|
+
} else {
|
|
241
|
+
if (visited.has(e)) {
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
visited.add(e);
|
|
207
245
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
if (this.seenTag(e, epoch))
|
|
225
|
-
continue;
|
|
226
|
-
this.markVisited(e, epoch);
|
|
227
|
-
}
|
|
228
|
-
else {
|
|
229
|
-
if (visited.has(e))
|
|
230
|
-
continue;
|
|
231
|
-
visited.add(e);
|
|
232
|
-
}
|
|
233
|
-
fresh.push(e);
|
|
234
|
-
}
|
|
235
|
-
if (fresh.length > 1)
|
|
236
|
-
this.fetchNodes(fresh);
|
|
237
|
-
for (let i = 0; i < fresh.length; i++) {
|
|
238
|
-
const e = fresh[i];
|
|
239
|
-
const rec = this.node(e);
|
|
240
|
-
const d = this.distOf(rec);
|
|
241
|
-
const worst = res.size > 0 ? res.topKey() : Infinity;
|
|
242
|
-
if (res.size < ef || d < worst) {
|
|
243
|
-
cand.push(d, e);
|
|
244
|
-
if (rec.deleted === 0) {
|
|
245
|
-
res.push(d, e);
|
|
246
|
-
if (res.size > ef)
|
|
247
|
-
res.pop();
|
|
248
|
-
}
|
|
249
|
-
}
|
|
246
|
+
fresh.push(e);
|
|
247
|
+
}
|
|
248
|
+
if (fresh.length > 1) {
|
|
249
|
+
this.fetchNodes(fresh);
|
|
250
|
+
}
|
|
251
|
+
for (let i = 0; i < fresh.length; i++) {
|
|
252
|
+
const e = fresh[i];
|
|
253
|
+
const rec = this.node(e);
|
|
254
|
+
const d = this.distOf(rec);
|
|
255
|
+
const worst = res.size > 0 ? res.topKey() : Infinity;
|
|
256
|
+
if (res.size < ef || d < worst) {
|
|
257
|
+
cand.push(d, e);
|
|
258
|
+
if (rec.deleted === 0) {
|
|
259
|
+
res.push(d, e);
|
|
260
|
+
if (res.size > ef) {
|
|
261
|
+
res.pop();
|
|
250
262
|
}
|
|
263
|
+
}
|
|
251
264
|
}
|
|
265
|
+
}
|
|
252
266
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
267
|
+
}
|
|
268
|
+
/** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
|
|
269
|
+
greedyDescend(entry, fromLayer, toLayer) {
|
|
270
|
+
let cur = entry;
|
|
271
|
+
const ep = this.singleEp;
|
|
272
|
+
for (let layer = fromLayer; layer > toLayer; layer--) {
|
|
273
|
+
ep[0] = cur;
|
|
274
|
+
this.searchLayer(ep, 1, layer);
|
|
275
|
+
if (this.resHeap.size > 0) {
|
|
276
|
+
cur = this.resHeap.vals[0];
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return cur;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
|
|
283
|
+
* neighbours from `candIds`, preferring those closer to `base` than to any
|
|
284
|
+
* already-chosen neighbour, then fills remaining slots with the closest
|
|
285
|
+
* leftovers (keep-pruned connections). Appends to `out`.
|
|
286
|
+
*/
|
|
287
|
+
selectNeighbors(base, candIds, count, M, out) {
|
|
288
|
+
const idx = this.idxScratch;
|
|
289
|
+
const ds = this.distScratch;
|
|
290
|
+
this.fetchNodes(candIds, count); // one batched read for the misses
|
|
291
|
+
idx.length = count;
|
|
292
|
+
ds.length = count;
|
|
293
|
+
for (let i = 0; i < count; i++) {
|
|
294
|
+
idx[i] = i;
|
|
295
|
+
ds[i] = this.codeDist(base, candIds[i]);
|
|
296
|
+
}
|
|
297
|
+
idx.sort((a, b) => ds[a] - ds[b]);
|
|
298
|
+
for (let s = 0; s < count; s++) {
|
|
299
|
+
if (out.length >= M) {
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
const i = idx[s];
|
|
303
|
+
const cid = candIds[i];
|
|
304
|
+
if (cid === base) {
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
const cd = ds[i];
|
|
308
|
+
let keep = true;
|
|
309
|
+
for (let j = 0; j < out.length; j++) {
|
|
310
|
+
if (this.codeDist(cid, out[j]) < cd) {
|
|
311
|
+
keep = false;
|
|
312
|
+
break;
|
|
262
313
|
}
|
|
263
|
-
|
|
314
|
+
}
|
|
315
|
+
if (keep) {
|
|
316
|
+
out.push(cid);
|
|
317
|
+
}
|
|
264
318
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
*/
|
|
271
|
-
selectNeighbors(base, candIds, count, M, out) {
|
|
272
|
-
const idx = this.idxScratch;
|
|
273
|
-
const ds = this.distScratch;
|
|
274
|
-
this.fetchNodes(candIds, count); // one batched read for the misses
|
|
275
|
-
idx.length = count;
|
|
276
|
-
ds.length = count;
|
|
277
|
-
for (let i = 0; i < count; i++) {
|
|
278
|
-
idx[i] = i;
|
|
279
|
-
ds[i] = this.codeDist(base, candIds[i]);
|
|
319
|
+
if (out.length < M) {
|
|
320
|
+
for (let s = 0; s < count && out.length < M; s++) {
|
|
321
|
+
const cid = candIds[idx[s]];
|
|
322
|
+
if (cid === base) {
|
|
323
|
+
continue;
|
|
280
324
|
}
|
|
281
|
-
|
|
282
|
-
for (let
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
if (cid === base)
|
|
288
|
-
continue;
|
|
289
|
-
const cd = ds[i];
|
|
290
|
-
let keep = true;
|
|
291
|
-
for (let j = 0; j < out.length; j++) {
|
|
292
|
-
if (this.codeDist(cid, out[j]) < cd) {
|
|
293
|
-
keep = false;
|
|
294
|
-
break;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
if (keep)
|
|
298
|
-
out.push(cid);
|
|
325
|
+
let dup = false;
|
|
326
|
+
for (let j = 0; j < out.length; j++) {
|
|
327
|
+
if (out[j] === cid) {
|
|
328
|
+
dup = true;
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
299
331
|
}
|
|
300
|
-
if (
|
|
301
|
-
|
|
302
|
-
const cid = candIds[idx[s]];
|
|
303
|
-
if (cid === base)
|
|
304
|
-
continue;
|
|
305
|
-
let dup = false;
|
|
306
|
-
for (let j = 0; j < out.length; j++) {
|
|
307
|
-
if (out[j] === cid) {
|
|
308
|
-
dup = true;
|
|
309
|
-
break;
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
if (!dup)
|
|
313
|
-
out.push(cid);
|
|
314
|
-
}
|
|
332
|
+
if (!dup) {
|
|
333
|
+
out.push(cid);
|
|
315
334
|
}
|
|
335
|
+
}
|
|
316
336
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
if (wCount > 0) {
|
|
370
|
-
const next = new Array(wCount);
|
|
371
|
-
for (let i = 0; i < wCount; i++)
|
|
372
|
-
next[i] = res.vals[i];
|
|
373
|
-
entryPoints = next;
|
|
374
|
-
}
|
|
375
|
-
else {
|
|
376
|
-
entryPoints = [entry];
|
|
377
|
-
}
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Insert a vector's code under external id `ext`; returns the internal node id.
|
|
340
|
+
* All graph reads/writes go through the store; the caller owns the transaction.
|
|
341
|
+
*/
|
|
342
|
+
insert(ext, code, efC) {
|
|
343
|
+
const ef = efC !== undefined && efC >= 1
|
|
344
|
+
? Math.min(efC, this.efConstruction)
|
|
345
|
+
: this.efConstruction;
|
|
346
|
+
this.working.clear();
|
|
347
|
+
const level = this.randomLevel();
|
|
348
|
+
const id = this.store.addNode(ext, level, code);
|
|
349
|
+
this.total++;
|
|
350
|
+
this.working.set(id, { code, deleted: 0, ext });
|
|
351
|
+
if (this.entry === -1) {
|
|
352
|
+
this.entry = id;
|
|
353
|
+
this.maxLevel = level;
|
|
354
|
+
this.live++;
|
|
355
|
+
this.persistState();
|
|
356
|
+
return id;
|
|
357
|
+
}
|
|
358
|
+
this.qCtx = null;
|
|
359
|
+
this.refBytes = code;
|
|
360
|
+
this.building = true;
|
|
361
|
+
let entry = this.entry;
|
|
362
|
+
const topLayer = this.maxLevel;
|
|
363
|
+
if (topLayer > level) {
|
|
364
|
+
entry = this.greedyDescend(entry, topLayer, level);
|
|
365
|
+
}
|
|
366
|
+
let entryPoints = [entry];
|
|
367
|
+
const startLayer = Math.min(topLayer, level);
|
|
368
|
+
for (let layer = startLayer; layer >= 0; layer--) {
|
|
369
|
+
this.searchLayer(entryPoints, ef, layer);
|
|
370
|
+
const res = this.resHeap;
|
|
371
|
+
const wCount = res.size;
|
|
372
|
+
const selected = [];
|
|
373
|
+
this.selectNeighbors(id, res.vals, wCount, this.M, selected);
|
|
374
|
+
if (selected.length > 0) {
|
|
375
|
+
this.store.setNeighbors(id, layer, selected);
|
|
376
|
+
}
|
|
377
|
+
const cap = layer === 0 ? this.Mmax0 : this.M;
|
|
378
|
+
for (let s = 0; s < selected.length; s++) {
|
|
379
|
+
const nb = selected[s];
|
|
380
|
+
const cur = this.store.getNeighbors(nb, layer);
|
|
381
|
+
const list = cur ? Array.from(cur) : [];
|
|
382
|
+
list.push(id);
|
|
383
|
+
if (list.length > cap) {
|
|
384
|
+
const pruned = [];
|
|
385
|
+
this.selectNeighbors(nb, list, list.length, cap, pruned);
|
|
386
|
+
this.store.setNeighbors(nb, layer, pruned);
|
|
387
|
+
} else {
|
|
388
|
+
this.store.setNeighbors(nb, layer, list);
|
|
378
389
|
}
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
this.entry = id;
|
|
390
|
+
}
|
|
391
|
+
if (wCount > 0) {
|
|
392
|
+
const next = new Array(wCount);
|
|
393
|
+
for (let i = 0; i < wCount; i++) {
|
|
394
|
+
next[i] = res.vals[i];
|
|
385
395
|
}
|
|
386
|
-
|
|
387
|
-
|
|
396
|
+
entryPoints = next;
|
|
397
|
+
} else {
|
|
398
|
+
entryPoints = [entry];
|
|
399
|
+
}
|
|
388
400
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
this.refBytes = null;
|
|
396
|
-
if (this.entry === -1 || k <= 0) {
|
|
397
|
-
this.lastQueryStorageReads = 0;
|
|
398
|
-
return [];
|
|
399
|
-
}
|
|
400
|
-
this.qCtx = this.quantizer.prepareQuery(vec);
|
|
401
|
-
const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
|
|
402
|
-
this.qCtx = null;
|
|
403
|
-
this.lastQueryStorageReads = this.store.reads;
|
|
404
|
-
return hits;
|
|
401
|
+
this.building = false;
|
|
402
|
+
this.refBytes = null;
|
|
403
|
+
this.live++;
|
|
404
|
+
if (level > this.maxLevel) {
|
|
405
|
+
this.maxLevel = level;
|
|
406
|
+
this.entry = id;
|
|
405
407
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
this.refBytes = null;
|
|
420
|
-
this.lastQueryStorageReads = this.store.reads;
|
|
421
|
-
return hits;
|
|
408
|
+
this.persistState();
|
|
409
|
+
return id;
|
|
410
|
+
}
|
|
411
|
+
/** k-NN with a full-precision query vector (accurate estimator). */
|
|
412
|
+
searchKnn(vec, k, ef) {
|
|
413
|
+
this.lastQueryDistComps = 0;
|
|
414
|
+
this.store.resetReads();
|
|
415
|
+
this.working.clear();
|
|
416
|
+
this.building = false;
|
|
417
|
+
this.refBytes = null;
|
|
418
|
+
if (this.entry === -1 || k <= 0) {
|
|
419
|
+
this.lastQueryStorageReads = 0;
|
|
420
|
+
return [];
|
|
422
421
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
422
|
+
this.qCtx = this.quantizer.prepareQuery(vec);
|
|
423
|
+
const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
|
|
424
|
+
this.qCtx = null;
|
|
425
|
+
this.lastQueryStorageReads = this.store.reads;
|
|
426
|
+
return hits;
|
|
427
|
+
}
|
|
428
|
+
/** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
|
|
429
|
+
searchKnnByCode(codeBytes, k, ef) {
|
|
430
|
+
this.lastQueryDistComps = 0;
|
|
431
|
+
this.store.resetReads();
|
|
432
|
+
this.working.clear();
|
|
433
|
+
this.building = false;
|
|
434
|
+
this.qCtx = null;
|
|
435
|
+
if (this.entry === -1 || k <= 0) {
|
|
436
|
+
this.lastQueryStorageReads = 0;
|
|
437
|
+
return [];
|
|
438
|
+
}
|
|
439
|
+
this.refBytes = codeBytes;
|
|
440
|
+
const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
|
|
441
|
+
this.refBytes = null;
|
|
442
|
+
this.lastQueryStorageReads = this.store.reads;
|
|
443
|
+
return hits;
|
|
444
|
+
}
|
|
445
|
+
collectKnn(k, efs) {
|
|
446
|
+
let entry = this.entry;
|
|
447
|
+
if (this.maxLevel > 0) {
|
|
448
|
+
entry = this.greedyDescend(entry, this.maxLevel, 0);
|
|
449
|
+
}
|
|
450
|
+
this.singleEp[0] = entry;
|
|
451
|
+
this.searchLayer(this.singleEp, efs, 0);
|
|
452
|
+
const res = this.resHeap;
|
|
453
|
+
const n = res.size;
|
|
454
|
+
const hits = new Array(n);
|
|
455
|
+
for (let i = 0; i < n; i++) {
|
|
456
|
+
let d = res.keys[i];
|
|
457
|
+
if (d < 0) {
|
|
458
|
+
d = 0;
|
|
459
|
+
}
|
|
460
|
+
hits[i] = { id: this.node(res.vals[i]).ext, distance: d };
|
|
442
461
|
}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
if (rec === null || rec.deleted === 1)
|
|
447
|
-
return false;
|
|
448
|
-
this.store.tombstone(id);
|
|
449
|
-
this.live--;
|
|
450
|
-
this.persistState();
|
|
451
|
-
return true;
|
|
462
|
+
hits.sort((a, b) => a.distance - b.distance);
|
|
463
|
+
if (hits.length > k) {
|
|
464
|
+
hits.length = k;
|
|
452
465
|
}
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
* multi-million-node store, inside one WAL-bloating transaction. The splice
|
|
461
|
-
* is a streaming pass with batched commits, then a VACUUM to return the
|
|
462
|
-
* freed pages.
|
|
463
|
-
*/
|
|
464
|
-
compact() {
|
|
465
|
-
const r = this.store.spliceCompact(this.M, this.Mmax0, this.entry);
|
|
466
|
-
this.entry = r.entry;
|
|
467
|
-
this.maxLevel = r.maxLevel;
|
|
468
|
-
this.live = r.live;
|
|
469
|
-
this.total = r.live;
|
|
470
|
-
this.persistState();
|
|
471
|
-
this.working.clear();
|
|
472
|
-
this.store.vacuum();
|
|
466
|
+
return hits;
|
|
467
|
+
}
|
|
468
|
+
/** Tombstone a live node by internal id. Caller owns the transaction. */
|
|
469
|
+
remove(id) {
|
|
470
|
+
const rec = this.store.getNode(id);
|
|
471
|
+
if (rec === null || rec.deleted === 1) {
|
|
472
|
+
return false;
|
|
473
473
|
}
|
|
474
|
+
this.store.tombstone(id);
|
|
475
|
+
this.live--;
|
|
476
|
+
this.persistState();
|
|
477
|
+
return true;
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Reclaim tombstones with a graph-preserving splice (see
|
|
481
|
+
* {@link Store.spliceCompact}): live nodes keep their internal ids and their
|
|
482
|
+
* wiring; each dead neighbour is replaced by its own live neighbours. Codes
|
|
483
|
+
* are untouched, so the index loses nothing beyond what deletion itself
|
|
484
|
+
* removes. The previous implementation replayed every live code through a
|
|
485
|
+
* full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
|
|
486
|
+
* multi-million-node store, inside one WAL-bloating transaction. The splice
|
|
487
|
+
* is a streaming pass with batched commits, then a VACUUM to return the
|
|
488
|
+
* freed pages.
|
|
489
|
+
*/
|
|
490
|
+
compact() {
|
|
491
|
+
const r = this.store.spliceCompact(this.M, this.Mmax0, this.entry);
|
|
492
|
+
this.entry = r.entry;
|
|
493
|
+
this.maxLevel = r.maxLevel;
|
|
494
|
+
this.live = r.live;
|
|
495
|
+
this.total = r.live;
|
|
496
|
+
this.persistState();
|
|
497
|
+
this.working.clear();
|
|
498
|
+
this.store.vacuum();
|
|
499
|
+
}
|
|
474
500
|
}
|