@hviana/sema 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/dist/example/demo.js +25 -25
  2. package/dist/example/train_base.d.ts +21 -19
  3. package/dist/example/train_base.js +1952 -1684
  4. package/dist/src/alphabet.d.ts +3 -3
  5. package/dist/src/alphabet.js +27 -24
  6. package/dist/src/alu/src/alu.d.ts +194 -169
  7. package/dist/src/alu/src/alu.js +414 -374
  8. package/dist/src/alu/src/expr.d.ts +43 -36
  9. package/dist/src/alu/src/expr.js +278 -252
  10. package/dist/src/alu/src/index.d.ts +98 -9
  11. package/dist/src/alu/src/index.js +63 -9
  12. package/dist/src/alu/src/kernel-arith.d.ts +5 -1
  13. package/dist/src/alu/src/kernel-arith.js +285 -195
  14. package/dist/src/alu/src/kernel-bits.d.ts +5 -1
  15. package/dist/src/alu/src/kernel-bits.js +115 -81
  16. package/dist/src/alu/src/kernel-logic.js +63 -33
  17. package/dist/src/alu/src/kernel-nd.js +195 -155
  18. package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
  19. package/dist/src/alu/src/kernel-numeric.js +331 -259
  20. package/dist/src/alu/src/operation.d.ts +132 -112
  21. package/dist/src/alu/src/operation.js +163 -149
  22. package/dist/src/alu/src/parser.d.ts +175 -168
  23. package/dist/src/alu/src/parser.js +421 -382
  24. package/dist/src/alu/src/resonance.d.ts +25 -16
  25. package/dist/src/alu/src/resonance.js +56 -49
  26. package/dist/src/alu/src/text.d.ts +7 -3
  27. package/dist/src/alu/src/text.js +37 -31
  28. package/dist/src/alu/src/value.d.ts +14 -14
  29. package/dist/src/alu/src/value.js +160 -150
  30. package/dist/src/alu/test/alu.test.js +656 -543
  31. package/dist/src/bytes.d.ts +5 -1
  32. package/dist/src/bytes.js +40 -33
  33. package/dist/src/config.d.ts +102 -102
  34. package/dist/src/config.js +83 -83
  35. package/dist/src/derive/src/deduction.d.ts +62 -60
  36. package/dist/src/derive/src/deduction.js +109 -103
  37. package/dist/src/derive/src/index.d.ts +7 -1
  38. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  39. package/dist/src/derive/src/priority-queue.js +61 -57
  40. package/dist/src/derive/src/rewrite.d.ts +18 -15
  41. package/dist/src/derive/src/rewrite.js +67 -59
  42. package/dist/src/derive/src/trie.d.ts +56 -56
  43. package/dist/src/derive/src/trie.js +179 -175
  44. package/dist/src/derive/test/derive.test.js +100 -96
  45. package/dist/src/extension.d.ts +16 -13
  46. package/dist/src/geometry.d.ts +42 -17
  47. package/dist/src/geometry.js +266 -235
  48. package/dist/src/index.d.ts +17 -2
  49. package/dist/src/index.js +7 -1
  50. package/dist/src/ingest-cache.d.ts +30 -25
  51. package/dist/src/ingest-cache.js +127 -108
  52. package/dist/src/mind/articulation.d.ts +5 -1
  53. package/dist/src/mind/articulation.js +112 -77
  54. package/dist/src/mind/attention.d.ts +108 -40
  55. package/dist/src/mind/attention.js +871 -749
  56. package/dist/src/mind/canonical.d.ts +19 -4
  57. package/dist/src/mind/canonical.js +31 -27
  58. package/dist/src/mind/graph-search.d.ts +225 -201
  59. package/dist/src/mind/graph-search.js +821 -742
  60. package/dist/src/mind/index.d.ts +10 -2
  61. package/dist/src/mind/junction.d.ts +58 -31
  62. package/dist/src/mind/junction.js +237 -172
  63. package/dist/src/mind/learning.d.ts +52 -16
  64. package/dist/src/mind/learning.js +165 -143
  65. package/dist/src/mind/match.d.ts +69 -20
  66. package/dist/src/mind/match.js +318 -259
  67. package/dist/src/mind/mechanisms/alu.js +16 -16
  68. package/dist/src/mind/mechanisms/cast.d.ts +13 -9
  69. package/dist/src/mind/mechanisms/cast.js +456 -363
  70. package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
  71. package/dist/src/mind/mechanisms/confluence.js +183 -152
  72. package/dist/src/mind/mechanisms/cover.d.ts +8 -2
  73. package/dist/src/mind/mechanisms/cover.js +210 -148
  74. package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
  75. package/dist/src/mind/mechanisms/extraction.js +288 -234
  76. package/dist/src/mind/mechanisms/recall.d.ts +10 -6
  77. package/dist/src/mind/mechanisms/recall.js +185 -126
  78. package/dist/src/mind/mind.d.ts +154 -129
  79. package/dist/src/mind/mind.js +288 -258
  80. package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
  81. package/dist/src/mind/pipeline-mechanism.js +172 -161
  82. package/dist/src/mind/pipeline.d.ts +14 -4
  83. package/dist/src/mind/pipeline.js +189 -125
  84. package/dist/src/mind/primitives.d.ts +32 -8
  85. package/dist/src/mind/primitives.js +117 -99
  86. package/dist/src/mind/rationale.d.ts +98 -86
  87. package/dist/src/mind/rationale.js +121 -113
  88. package/dist/src/mind/reasoning.d.ts +13 -2
  89. package/dist/src/mind/reasoning.js +166 -129
  90. package/dist/src/mind/recognition.d.ts +4 -1
  91. package/dist/src/mind/recognition.js +208 -183
  92. package/dist/src/mind/resonance.d.ts +22 -5
  93. package/dist/src/mind/resonance.js +0 -0
  94. package/dist/src/mind/trace.d.ts +25 -6
  95. package/dist/src/mind/trace.js +58 -50
  96. package/dist/src/mind/traverse.d.ts +58 -16
  97. package/dist/src/mind/traverse.js +357 -304
  98. package/dist/src/mind/types.d.ts +127 -120
  99. package/dist/src/mind/types.js +69 -60
  100. package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
  101. package/dist/src/rabitq-hnsw/src/database.js +359 -342
  102. package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
  103. package/dist/src/rabitq-hnsw/src/heap.js +80 -75
  104. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
  105. package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
  106. package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
  107. package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
  108. package/dist/src/rabitq-hnsw/src/prng.js +27 -25
  109. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
  110. package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
  111. package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
  112. package/dist/src/rabitq-hnsw/src/store.js +843 -752
  113. package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
  114. package/dist/src/sema.d.ts +13 -9
  115. package/dist/src/sema.js +40 -26
  116. package/dist/src/store-sqlite.d.ts +166 -144
  117. package/dist/src/store-sqlite.js +660 -578
  118. package/dist/src/store.d.ts +630 -593
  119. package/dist/src/store.js +1581 -1440
  120. package/dist/src/vec.d.ts +9 -5
  121. package/dist/src/vec.js +73 -61
  122. package/package.json +2 -2
@@ -15,460 +15,486 @@ import { Prng } from "./prng.js";
15
15
  * access to the original vectors.
16
16
  */
17
17
  export class HnswIndex {
18
- M;
19
- Mmax0;
20
- efConstruction;
21
- mL;
22
- quantizer;
23
- store;
24
- seed;
25
- levelRng;
26
- // tiny global scalars, cached from meta and written through on change
27
- entry;
28
- maxLevel;
29
- live;
30
- total;
31
- _efSearch;
32
- // Per-operation working set: the records the current insert/query is actively
33
- // comparing. Cleared at the start of every op and bounded by efConstruction /
34
- // efSearch (the nodes one operation can touch), never by the collection size.
35
- // This is the algorithm's working memory -- it makes each touched node cost one
36
- // storage read per op -- not a tunable cache. The only cache is SQLite's page
37
- // cache (see Store), sized in MB.
38
- working = new Map();
39
- visited = new Set();
40
- // Epoch-tagged visited marks — the Set above without per-candidate hashing:
41
- // visitedTag[id] === visitedEpoch means "seen this operation", and bumping
42
- // the epoch clears the whole set in O(1). Semantically IDENTICAL to the
43
- // Set (results never differ); used only when the store has a cache budget,
44
- // because the array is 4 B per internal id — amortized working memory that
45
- // grows with the collection, which the cacheSizeMb:0 flat-memory mode must
46
- // not pay. The epoch wrap (2³²−1 ops) refills the array once.
47
- visitedTag = new Uint32Array(1024);
48
- visitedEpoch = 0;
49
- seenTag(id, epoch) {
50
- return id < this.visitedTag.length && this.visitedTag[id] === epoch;
18
+ M;
19
+ Mmax0;
20
+ efConstruction;
21
+ mL;
22
+ quantizer;
23
+ store;
24
+ seed;
25
+ levelRng;
26
+ // tiny global scalars, cached from meta and written through on change
27
+ entry;
28
+ maxLevel;
29
+ live;
30
+ total;
31
+ _efSearch;
32
+ // Per-operation working set: the records the current insert/query is actively
33
+ // comparing. Cleared at the start of every op and bounded by efConstruction /
34
+ // efSearch (the nodes one operation can touch), never by the collection size.
35
+ // This is the algorithm's working memory -- it makes each touched node cost one
36
+ // storage read per op -- not a tunable cache. The only cache is SQLite's page
37
+ // cache (see Store), sized in MB.
38
+ working = new Map();
39
+ visited = new Set();
40
+ // Epoch-tagged visited marks — the Set above without per-candidate hashing:
41
+ // visitedTag[id] === visitedEpoch means "seen this operation", and bumping
42
+ // the epoch clears the whole set in O(1). Semantically IDENTICAL to the
43
+ // Set (results never differ); used only when the store has a cache budget,
44
+ // because the array is 4 B per internal id — amortized working memory that
45
+ // grows with the collection, which the cacheSizeMb:0 flat-memory mode must
46
+ // not pay. The epoch wrap (2³²−1 ops) refills the array once.
47
+ visitedTag = new Uint32Array(1024);
48
+ visitedEpoch = 0;
49
+ seenTag(id, epoch) {
50
+ return id < this.visitedTag.length && this.visitedTag[id] === epoch;
51
+ }
52
+ markVisited(id, epoch) {
53
+ if (id >= this.visitedTag.length) {
54
+ const grown = new Uint32Array(
55
+ Math.max(id + 1, this.visitedTag.length * 2),
56
+ );
57
+ grown.set(this.visitedTag);
58
+ this.visitedTag = grown;
51
59
  }
52
- markVisited(id, epoch) {
53
- if (id >= this.visitedTag.length) {
54
- const grown = new Uint32Array(Math.max(id + 1, this.visitedTag.length * 2));
55
- grown.set(this.visitedTag);
56
- this.visitedTag = grown;
57
- }
58
- this.visitedTag[id] = epoch;
59
- }
60
- candHeap = new Heap(true); // min-heap: nearest first
61
- resHeap = new Heap(false); // max-heap: farthest first
62
- singleEp = [0];
63
- idxScratch = [];
64
- distScratch = [];
65
- freshScratch = [];
66
- // distance dispatch: a full-precision query sets qCtx; building or a code
67
- // query sets refBytes (code<->code). `building` only suppresses the counter.
68
- qCtx = null;
69
- refBytes = null;
70
- building = false;
71
- lastQueryDistComps = 0;
72
- /** Storage row reads issued by the most recent query (cache-independent). */
73
- lastQueryStorageReads = 0;
74
- constructor(quantizer, store, params) {
75
- this.quantizer = quantizer;
76
- this.store = store;
77
- this.M = params.M;
78
- this.Mmax0 = this.M * 2;
79
- this.efConstruction = params.efConstruction;
80
- this._efSearch = params.efSearch;
81
- this.mL = 1 / Math.log(this.M);
82
- this.seed = params.seed >>> 0;
83
- const s = store.loadState();
84
- this.entry = s.entryPoint;
85
- this.maxLevel = s.maxLevel;
86
- this.live = s.live;
87
- this.total = s.total;
88
- this.levelRng = new Prng(this.seed);
89
- this.levelRng.restore(s.rng);
90
- }
91
- get size() {
92
- return this.live;
93
- }
94
- get physicalSize() {
95
- return this.total;
60
+ this.visitedTag[id] = epoch;
61
+ }
62
+ candHeap = new Heap(true); // min-heap: nearest first
63
+ resHeap = new Heap(false); // max-heap: farthest first
64
+ singleEp = [0];
65
+ idxScratch = [];
66
+ distScratch = [];
67
+ freshScratch = [];
68
+ // distance dispatch: a full-precision query sets qCtx; building or a code
69
+ // query sets refBytes (code<->code). `building` only suppresses the counter.
70
+ qCtx = null;
71
+ refBytes = null;
72
+ building = false;
73
+ lastQueryDistComps = 0;
74
+ /** Storage row reads issued by the most recent query (cache-independent). */
75
+ lastQueryStorageReads = 0;
76
+ constructor(quantizer, store, params) {
77
+ this.quantizer = quantizer;
78
+ this.store = store;
79
+ this.M = params.M;
80
+ this.Mmax0 = this.M * 2;
81
+ this.efConstruction = params.efConstruction;
82
+ this._efSearch = params.efSearch;
83
+ this.mL = 1 / Math.log(this.M);
84
+ this.seed = params.seed >>> 0;
85
+ const s = store.loadState();
86
+ this.entry = s.entryPoint;
87
+ this.maxLevel = s.maxLevel;
88
+ this.live = s.live;
89
+ this.total = s.total;
90
+ this.levelRng = new Prng(this.seed);
91
+ this.levelRng.restore(s.rng);
92
+ }
93
+ get size() {
94
+ return this.live;
95
+ }
96
+ get physicalSize() {
97
+ return this.total;
98
+ }
99
+ get bytesPerVector() {
100
+ return this.quantizer.codeWords * 4;
101
+ }
102
+ get efSearch() {
103
+ return this._efSearch;
104
+ }
105
+ set efSearch(v) {
106
+ this._efSearch = Math.max(1, v | 0);
107
+ this.store.setEfSearch(this._efSearch);
108
+ }
109
+ persistState() {
110
+ this.store.saveState({
111
+ entryPoint: this.entry,
112
+ maxLevel: this.maxLevel,
113
+ live: this.live,
114
+ total: this.total,
115
+ rng: this.levelRng.snapshot(),
116
+ });
117
+ }
118
+ randomLevel() {
119
+ let u = this.levelRng.next();
120
+ if (u < 1e-12) {
121
+ u = 1e-12;
96
122
  }
97
- get bytesPerVector() {
98
- return this.quantizer.codeWords * 4;
123
+ return Math.floor(-Math.log(u) * this.mL);
124
+ }
125
+ /**
126
+ * Fetch a node record into the operation's working set. The first touch in an
127
+ * op is a storage read; later touches in the same op reuse it. The set is
128
+ * cleared per op and bounded by efConstruction / efSearch, so storage reads
129
+ * per op equal the number of *distinct* nodes the op visits -- minimal, and
130
+ * independent of any cache.
131
+ */
132
+ node(id) {
133
+ const hit = this.working.get(id);
134
+ if (hit !== undefined) {
135
+ return hit;
99
136
  }
100
- get efSearch() {
101
- return this._efSearch;
137
+ const rec = this.store.getNode(id);
138
+ if (rec === null) {
139
+ throw new Error(`node ${id} not found`);
102
140
  }
103
- set efSearch(v) {
104
- this._efSearch = Math.max(1, v | 0);
105
- this.store.setEfSearch(this._efSearch);
141
+ this.working.set(id, rec);
142
+ return rec;
143
+ }
144
+ /** Prefetch several nodes into the working set with ONE batched storage
145
+ * read for the misses. Point queries per neighbour made statement
146
+ * dispatch — not distance arithmetic — the dominant cost of a large
147
+ * build; the batch keeps `reads` accounting identical per row. */
148
+ fetchNodes(ids, count = ids.length) {
149
+ // getNodesInto skips ids already present in `working` itself — no
150
+ // second pre-filter pass over the same map here.
151
+ this.store.getNodesInto(ids, this.working, count);
152
+ }
153
+ /** Distance from the current source (query vector or reference code) to a node. */
154
+ distOf(rec) {
155
+ if (this.qCtx !== null) {
156
+ this.lastQueryDistComps++;
157
+ return this.quantizer.estimate(rec.code, 0, this.qCtx);
106
158
  }
107
- persistState() {
108
- this.store.saveState({
109
- entryPoint: this.entry,
110
- maxLevel: this.maxLevel,
111
- live: this.live,
112
- total: this.total,
113
- rng: this.levelRng.snapshot(),
114
- });
159
+ if (!this.building) {
160
+ this.lastQueryDistComps++;
115
161
  }
116
- randomLevel() {
117
- let u = this.levelRng.next();
118
- if (u < 1e-12)
119
- u = 1e-12;
120
- return Math.floor(-Math.log(u) * this.mL);
162
+ return this.quantizer.codeDistanceBytes(this.refBytes, rec.code);
163
+ }
164
+ /** Code-to-code distance between two stored nodes (graph wiring only). */
165
+ codeDist(a, b) {
166
+ return this.quantizer.codeDistanceBytes(
167
+ this.node(a).code,
168
+ this.node(b).code,
169
+ );
170
+ }
171
+ /**
172
+ * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
173
+ * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
174
+ * never returned. `visited` is a per-call set sized by the nodes seen here.
175
+ */
176
+ searchLayer(entryPoints, ef, layer) {
177
+ const cand = this.candHeap;
178
+ const res = this.resHeap;
179
+ cand.clear();
180
+ res.clear();
181
+ // Visited marks: epoch tags when RAM-for-speed is allowed, the plain Set
182
+ // in flat-memory mode. Identical semantics either way.
183
+ const useTags = this.store.cacheEnabled;
184
+ let epoch = 0;
185
+ const visited = this.visited;
186
+ if (useTags) {
187
+ epoch = ++this.visitedEpoch;
188
+ if (epoch === 0xffffffff) {
189
+ this.visitedTag.fill(0);
190
+ epoch = this.visitedEpoch = 1;
191
+ }
192
+ } else {
193
+ visited.clear();
121
194
  }
122
- /**
123
- * Fetch a node record into the operation's working set. The first touch in an
124
- * op is a storage read; later touches in the same op reuse it. The set is
125
- * cleared per op and bounded by efConstruction / efSearch, so storage reads
126
- * per op equal the number of *distinct* nodes the op visits -- minimal, and
127
- * independent of any cache.
128
- */
129
- node(id) {
130
- const hit = this.working.get(id);
131
- if (hit !== undefined)
132
- return hit;
133
- const rec = this.store.getNode(id);
134
- if (rec === null)
135
- throw new Error(`node ${id} not found`);
136
- this.working.set(id, rec);
137
- return rec;
138
- }
139
- /** Prefetch several nodes into the working set with ONE batched storage
140
- * read for the misses. Point queries per neighbour made statement
141
- * dispatch — not distance arithmetic — the dominant cost of a large
142
- * build; the batch keeps `reads` accounting identical per row. */
143
- fetchNodes(ids, count = ids.length) {
144
- // getNodesInto skips ids already present in `working` itself — no
145
- // second pre-filter pass over the same map here.
146
- this.store.getNodesInto(ids, this.working, count);
147
- }
148
- /** Distance from the current source (query vector or reference code) to a node. */
149
- distOf(rec) {
150
- if (this.qCtx !== null) {
151
- this.lastQueryDistComps++;
152
- return this.quantizer.estimate(rec.code, 0, this.qCtx);
195
+ for (let k = 0; k < entryPoints.length; k++) {
196
+ const ep = entryPoints[k];
197
+ if (useTags) {
198
+ if (this.seenTag(ep, epoch)) {
199
+ continue;
153
200
  }
154
- if (!this.building)
155
- this.lastQueryDistComps++;
156
- return this.quantizer.codeDistanceBytes(this.refBytes, rec.code);
157
- }
158
- /** Code-to-code distance between two stored nodes (graph wiring only). */
159
- codeDist(a, b) {
160
- return this.quantizer.codeDistanceBytes(this.node(a).code, this.node(b).code);
161
- }
162
- /**
163
- * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
164
- * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
165
- * never returned. `visited` is a per-call set sized by the nodes seen here.
166
- */
167
- searchLayer(entryPoints, ef, layer) {
168
- const cand = this.candHeap;
169
- const res = this.resHeap;
170
- cand.clear();
171
- res.clear();
172
- // Visited marks: epoch tags when RAM-for-speed is allowed, the plain Set
173
- // in flat-memory mode. Identical semantics either way.
174
- const useTags = this.store.cacheEnabled;
175
- let epoch = 0;
176
- const visited = this.visited;
177
- if (useTags) {
178
- epoch = ++this.visitedEpoch;
179
- if (epoch === 0xffffffff) {
180
- this.visitedTag.fill(0);
181
- epoch = this.visitedEpoch = 1;
182
- }
201
+ this.markVisited(ep, epoch);
202
+ } else {
203
+ if (visited.has(ep)) {
204
+ continue;
183
205
  }
184
- else {
185
- visited.clear();
206
+ visited.add(ep);
207
+ }
208
+ const rec = this.node(ep);
209
+ const d = this.distOf(rec);
210
+ cand.push(d, ep);
211
+ if (rec.deleted === 0) {
212
+ res.push(d, ep);
213
+ if (res.size > ef) {
214
+ res.pop();
186
215
  }
187
- for (let k = 0; k < entryPoints.length; k++) {
188
- const ep = entryPoints[k];
189
- if (useTags) {
190
- if (this.seenTag(ep, epoch))
191
- continue;
192
- this.markVisited(ep, epoch);
193
- }
194
- else {
195
- if (visited.has(ep))
196
- continue;
197
- visited.add(ep);
198
- }
199
- const rec = this.node(ep);
200
- const d = this.distOf(rec);
201
- cand.push(d, ep);
202
- if (rec.deleted === 0) {
203
- res.push(d, ep);
204
- if (res.size > ef)
205
- res.pop();
206
- }
216
+ }
217
+ }
218
+ while (cand.size > 0) {
219
+ const cd = cand.topKey();
220
+ if (res.size >= ef && cd > res.topKey()) {
221
+ break;
222
+ }
223
+ const c = cand.topVal();
224
+ cand.pop();
225
+ const nbrs = this.store.getNeighbors(c, layer);
226
+ if (nbrs === null) {
227
+ continue;
228
+ }
229
+ // Batch: mark the unvisited neighbours, fetch their codes in one
230
+ // storage read, then score them in the original order.
231
+ const fresh = this.freshScratch;
232
+ fresh.length = 0;
233
+ for (let i = 0; i < nbrs.length; i++) {
234
+ const e = nbrs[i];
235
+ if (useTags) {
236
+ if (this.seenTag(e, epoch)) {
237
+ continue;
238
+ }
239
+ this.markVisited(e, epoch);
240
+ } else {
241
+ if (visited.has(e)) {
242
+ continue;
243
+ }
244
+ visited.add(e);
207
245
  }
208
- while (cand.size > 0) {
209
- const cd = cand.topKey();
210
- if (res.size >= ef && cd > res.topKey())
211
- break;
212
- const c = cand.topVal();
213
- cand.pop();
214
- const nbrs = this.store.getNeighbors(c, layer);
215
- if (nbrs === null)
216
- continue;
217
- // Batch: mark the unvisited neighbours, fetch their codes in one
218
- // storage read, then score them in the original order.
219
- const fresh = this.freshScratch;
220
- fresh.length = 0;
221
- for (let i = 0; i < nbrs.length; i++) {
222
- const e = nbrs[i];
223
- if (useTags) {
224
- if (this.seenTag(e, epoch))
225
- continue;
226
- this.markVisited(e, epoch);
227
- }
228
- else {
229
- if (visited.has(e))
230
- continue;
231
- visited.add(e);
232
- }
233
- fresh.push(e);
234
- }
235
- if (fresh.length > 1)
236
- this.fetchNodes(fresh);
237
- for (let i = 0; i < fresh.length; i++) {
238
- const e = fresh[i];
239
- const rec = this.node(e);
240
- const d = this.distOf(rec);
241
- const worst = res.size > 0 ? res.topKey() : Infinity;
242
- if (res.size < ef || d < worst) {
243
- cand.push(d, e);
244
- if (rec.deleted === 0) {
245
- res.push(d, e);
246
- if (res.size > ef)
247
- res.pop();
248
- }
249
- }
246
+ fresh.push(e);
247
+ }
248
+ if (fresh.length > 1) {
249
+ this.fetchNodes(fresh);
250
+ }
251
+ for (let i = 0; i < fresh.length; i++) {
252
+ const e = fresh[i];
253
+ const rec = this.node(e);
254
+ const d = this.distOf(rec);
255
+ const worst = res.size > 0 ? res.topKey() : Infinity;
256
+ if (res.size < ef || d < worst) {
257
+ cand.push(d, e);
258
+ if (rec.deleted === 0) {
259
+ res.push(d, e);
260
+ if (res.size > ef) {
261
+ res.pop();
250
262
  }
263
+ }
251
264
  }
265
+ }
252
266
  }
253
- /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
254
- greedyDescend(entry, fromLayer, toLayer) {
255
- let cur = entry;
256
- const ep = this.singleEp;
257
- for (let layer = fromLayer; layer > toLayer; layer--) {
258
- ep[0] = cur;
259
- this.searchLayer(ep, 1, layer);
260
- if (this.resHeap.size > 0)
261
- cur = this.resHeap.vals[0];
267
+ }
268
+ /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
269
+ greedyDescend(entry, fromLayer, toLayer) {
270
+ let cur = entry;
271
+ const ep = this.singleEp;
272
+ for (let layer = fromLayer; layer > toLayer; layer--) {
273
+ ep[0] = cur;
274
+ this.searchLayer(ep, 1, layer);
275
+ if (this.resHeap.size > 0) {
276
+ cur = this.resHeap.vals[0];
277
+ }
278
+ }
279
+ return cur;
280
+ }
281
+ /**
282
+ * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
283
+ * neighbours from `candIds`, preferring those closer to `base` than to any
284
+ * already-chosen neighbour, then fills remaining slots with the closest
285
+ * leftovers (keep-pruned connections). Appends to `out`.
286
+ */
287
+ selectNeighbors(base, candIds, count, M, out) {
288
+ const idx = this.idxScratch;
289
+ const ds = this.distScratch;
290
+ this.fetchNodes(candIds, count); // one batched read for the misses
291
+ idx.length = count;
292
+ ds.length = count;
293
+ for (let i = 0; i < count; i++) {
294
+ idx[i] = i;
295
+ ds[i] = this.codeDist(base, candIds[i]);
296
+ }
297
+ idx.sort((a, b) => ds[a] - ds[b]);
298
+ for (let s = 0; s < count; s++) {
299
+ if (out.length >= M) {
300
+ break;
301
+ }
302
+ const i = idx[s];
303
+ const cid = candIds[i];
304
+ if (cid === base) {
305
+ continue;
306
+ }
307
+ const cd = ds[i];
308
+ let keep = true;
309
+ for (let j = 0; j < out.length; j++) {
310
+ if (this.codeDist(cid, out[j]) < cd) {
311
+ keep = false;
312
+ break;
262
313
  }
263
- return cur;
314
+ }
315
+ if (keep) {
316
+ out.push(cid);
317
+ }
264
318
  }
265
- /**
266
- * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
267
- * neighbours from `candIds`, preferring those closer to `base` than to any
268
- * already-chosen neighbour, then fills remaining slots with the closest
269
- * leftovers (keep-pruned connections). Appends to `out`.
270
- */
271
- selectNeighbors(base, candIds, count, M, out) {
272
- const idx = this.idxScratch;
273
- const ds = this.distScratch;
274
- this.fetchNodes(candIds, count); // one batched read for the misses
275
- idx.length = count;
276
- ds.length = count;
277
- for (let i = 0; i < count; i++) {
278
- idx[i] = i;
279
- ds[i] = this.codeDist(base, candIds[i]);
319
+ if (out.length < M) {
320
+ for (let s = 0; s < count && out.length < M; s++) {
321
+ const cid = candIds[idx[s]];
322
+ if (cid === base) {
323
+ continue;
280
324
  }
281
- idx.sort((a, b) => ds[a] - ds[b]);
282
- for (let s = 0; s < count; s++) {
283
- if (out.length >= M)
284
- break;
285
- const i = idx[s];
286
- const cid = candIds[i];
287
- if (cid === base)
288
- continue;
289
- const cd = ds[i];
290
- let keep = true;
291
- for (let j = 0; j < out.length; j++) {
292
- if (this.codeDist(cid, out[j]) < cd) {
293
- keep = false;
294
- break;
295
- }
296
- }
297
- if (keep)
298
- out.push(cid);
325
+ let dup = false;
326
+ for (let j = 0; j < out.length; j++) {
327
+ if (out[j] === cid) {
328
+ dup = true;
329
+ break;
330
+ }
299
331
  }
300
- if (out.length < M) {
301
- for (let s = 0; s < count && out.length < M; s++) {
302
- const cid = candIds[idx[s]];
303
- if (cid === base)
304
- continue;
305
- let dup = false;
306
- for (let j = 0; j < out.length; j++) {
307
- if (out[j] === cid) {
308
- dup = true;
309
- break;
310
- }
311
- }
312
- if (!dup)
313
- out.push(cid);
314
- }
332
+ if (!dup) {
333
+ out.push(cid);
315
334
  }
335
+ }
316
336
  }
317
- /**
318
- * Insert a vector's code under external id `ext`; returns the internal node id.
319
- * All graph reads/writes go through the store; the caller owns the transaction.
320
- */
321
- insert(ext, code, efC) {
322
- const ef = efC !== undefined && efC >= 1
323
- ? Math.min(efC, this.efConstruction)
324
- : this.efConstruction;
325
- this.working.clear();
326
- const level = this.randomLevel();
327
- const id = this.store.addNode(ext, level, code);
328
- this.total++;
329
- this.working.set(id, { code, deleted: 0, ext });
330
- if (this.entry === -1) {
331
- this.entry = id;
332
- this.maxLevel = level;
333
- this.live++;
334
- this.persistState();
335
- return id;
336
- }
337
- this.qCtx = null;
338
- this.refBytes = code;
339
- this.building = true;
340
- let entry = this.entry;
341
- const topLayer = this.maxLevel;
342
- if (topLayer > level)
343
- entry = this.greedyDescend(entry, topLayer, level);
344
- let entryPoints = [entry];
345
- const startLayer = Math.min(topLayer, level);
346
- for (let layer = startLayer; layer >= 0; layer--) {
347
- this.searchLayer(entryPoints, ef, layer);
348
- const res = this.resHeap;
349
- const wCount = res.size;
350
- const selected = [];
351
- this.selectNeighbors(id, res.vals, wCount, this.M, selected);
352
- if (selected.length > 0)
353
- this.store.setNeighbors(id, layer, selected);
354
- const cap = layer === 0 ? this.Mmax0 : this.M;
355
- for (let s = 0; s < selected.length; s++) {
356
- const nb = selected[s];
357
- const cur = this.store.getNeighbors(nb, layer);
358
- const list = cur ? Array.from(cur) : [];
359
- list.push(id);
360
- if (list.length > cap) {
361
- const pruned = [];
362
- this.selectNeighbors(nb, list, list.length, cap, pruned);
363
- this.store.setNeighbors(nb, layer, pruned);
364
- }
365
- else {
366
- this.store.setNeighbors(nb, layer, list);
367
- }
368
- }
369
- if (wCount > 0) {
370
- const next = new Array(wCount);
371
- for (let i = 0; i < wCount; i++)
372
- next[i] = res.vals[i];
373
- entryPoints = next;
374
- }
375
- else {
376
- entryPoints = [entry];
377
- }
337
+ }
338
+ /**
339
+ * Insert a vector's code under external id `ext`; returns the internal node id.
340
+ * All graph reads/writes go through the store; the caller owns the transaction.
341
+ */
342
+ insert(ext, code, efC) {
343
+ const ef = efC !== undefined && efC >= 1
344
+ ? Math.min(efC, this.efConstruction)
345
+ : this.efConstruction;
346
+ this.working.clear();
347
+ const level = this.randomLevel();
348
+ const id = this.store.addNode(ext, level, code);
349
+ this.total++;
350
+ this.working.set(id, { code, deleted: 0, ext });
351
+ if (this.entry === -1) {
352
+ this.entry = id;
353
+ this.maxLevel = level;
354
+ this.live++;
355
+ this.persistState();
356
+ return id;
357
+ }
358
+ this.qCtx = null;
359
+ this.refBytes = code;
360
+ this.building = true;
361
+ let entry = this.entry;
362
+ const topLayer = this.maxLevel;
363
+ if (topLayer > level) {
364
+ entry = this.greedyDescend(entry, topLayer, level);
365
+ }
366
+ let entryPoints = [entry];
367
+ const startLayer = Math.min(topLayer, level);
368
+ for (let layer = startLayer; layer >= 0; layer--) {
369
+ this.searchLayer(entryPoints, ef, layer);
370
+ const res = this.resHeap;
371
+ const wCount = res.size;
372
+ const selected = [];
373
+ this.selectNeighbors(id, res.vals, wCount, this.M, selected);
374
+ if (selected.length > 0) {
375
+ this.store.setNeighbors(id, layer, selected);
376
+ }
377
+ const cap = layer === 0 ? this.Mmax0 : this.M;
378
+ for (let s = 0; s < selected.length; s++) {
379
+ const nb = selected[s];
380
+ const cur = this.store.getNeighbors(nb, layer);
381
+ const list = cur ? Array.from(cur) : [];
382
+ list.push(id);
383
+ if (list.length > cap) {
384
+ const pruned = [];
385
+ this.selectNeighbors(nb, list, list.length, cap, pruned);
386
+ this.store.setNeighbors(nb, layer, pruned);
387
+ } else {
388
+ this.store.setNeighbors(nb, layer, list);
378
389
  }
379
- this.building = false;
380
- this.refBytes = null;
381
- this.live++;
382
- if (level > this.maxLevel) {
383
- this.maxLevel = level;
384
- this.entry = id;
390
+ }
391
+ if (wCount > 0) {
392
+ const next = new Array(wCount);
393
+ for (let i = 0; i < wCount; i++) {
394
+ next[i] = res.vals[i];
385
395
  }
386
- this.persistState();
387
- return id;
396
+ entryPoints = next;
397
+ } else {
398
+ entryPoints = [entry];
399
+ }
388
400
  }
389
- /** k-NN with a full-precision query vector (accurate estimator). */
390
- searchKnn(vec, k, ef) {
391
- this.lastQueryDistComps = 0;
392
- this.store.resetReads();
393
- this.working.clear();
394
- this.building = false;
395
- this.refBytes = null;
396
- if (this.entry === -1 || k <= 0) {
397
- this.lastQueryStorageReads = 0;
398
- return [];
399
- }
400
- this.qCtx = this.quantizer.prepareQuery(vec);
401
- const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
402
- this.qCtx = null;
403
- this.lastQueryStorageReads = this.store.reads;
404
- return hits;
401
+ this.building = false;
402
+ this.refBytes = null;
403
+ this.live++;
404
+ if (level > this.maxLevel) {
405
+ this.maxLevel = level;
406
+ this.entry = id;
405
407
  }
406
- /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
407
- searchKnnByCode(codeBytes, k, ef) {
408
- this.lastQueryDistComps = 0;
409
- this.store.resetReads();
410
- this.working.clear();
411
- this.building = false;
412
- this.qCtx = null;
413
- if (this.entry === -1 || k <= 0) {
414
- this.lastQueryStorageReads = 0;
415
- return [];
416
- }
417
- this.refBytes = codeBytes;
418
- const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
419
- this.refBytes = null;
420
- this.lastQueryStorageReads = this.store.reads;
421
- return hits;
408
+ this.persistState();
409
+ return id;
410
+ }
411
+ /** k-NN with a full-precision query vector (accurate estimator). */
412
+ searchKnn(vec, k, ef) {
413
+ this.lastQueryDistComps = 0;
414
+ this.store.resetReads();
415
+ this.working.clear();
416
+ this.building = false;
417
+ this.refBytes = null;
418
+ if (this.entry === -1 || k <= 0) {
419
+ this.lastQueryStorageReads = 0;
420
+ return [];
422
421
  }
423
- collectKnn(k, efs) {
424
- let entry = this.entry;
425
- if (this.maxLevel > 0)
426
- entry = this.greedyDescend(entry, this.maxLevel, 0);
427
- this.singleEp[0] = entry;
428
- this.searchLayer(this.singleEp, efs, 0);
429
- const res = this.resHeap;
430
- const n = res.size;
431
- const hits = new Array(n);
432
- for (let i = 0; i < n; i++) {
433
- let d = res.keys[i];
434
- if (d < 0)
435
- d = 0;
436
- hits[i] = { id: this.node(res.vals[i]).ext, distance: d };
437
- }
438
- hits.sort((a, b) => a.distance - b.distance);
439
- if (hits.length > k)
440
- hits.length = k;
441
- return hits;
422
+ this.qCtx = this.quantizer.prepareQuery(vec);
423
+ const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
424
+ this.qCtx = null;
425
+ this.lastQueryStorageReads = this.store.reads;
426
+ return hits;
427
+ }
428
+ /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
429
+ searchKnnByCode(codeBytes, k, ef) {
430
+ this.lastQueryDistComps = 0;
431
+ this.store.resetReads();
432
+ this.working.clear();
433
+ this.building = false;
434
+ this.qCtx = null;
435
+ if (this.entry === -1 || k <= 0) {
436
+ this.lastQueryStorageReads = 0;
437
+ return [];
438
+ }
439
+ this.refBytes = codeBytes;
440
+ const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
441
+ this.refBytes = null;
442
+ this.lastQueryStorageReads = this.store.reads;
443
+ return hits;
444
+ }
445
+ collectKnn(k, efs) {
446
+ let entry = this.entry;
447
+ if (this.maxLevel > 0) {
448
+ entry = this.greedyDescend(entry, this.maxLevel, 0);
449
+ }
450
+ this.singleEp[0] = entry;
451
+ this.searchLayer(this.singleEp, efs, 0);
452
+ const res = this.resHeap;
453
+ const n = res.size;
454
+ const hits = new Array(n);
455
+ for (let i = 0; i < n; i++) {
456
+ let d = res.keys[i];
457
+ if (d < 0) {
458
+ d = 0;
459
+ }
460
+ hits[i] = { id: this.node(res.vals[i]).ext, distance: d };
442
461
  }
443
- /** Tombstone a live node by internal id. Caller owns the transaction. */
444
- remove(id) {
445
- const rec = this.store.getNode(id);
446
- if (rec === null || rec.deleted === 1)
447
- return false;
448
- this.store.tombstone(id);
449
- this.live--;
450
- this.persistState();
451
- return true;
462
+ hits.sort((a, b) => a.distance - b.distance);
463
+ if (hits.length > k) {
464
+ hits.length = k;
452
465
  }
453
- /**
454
- * Reclaim tombstones with a graph-preserving splice (see
455
- * {@link Store.spliceCompact}): live nodes keep their internal ids and their
456
- * wiring; each dead neighbour is replaced by its own live neighbours. Codes
457
- * are untouched, so the index loses nothing beyond what deletion itself
458
- * removes. The previous implementation replayed every live code through a
459
- * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
460
- * multi-million-node store, inside one WAL-bloating transaction. The splice
461
- * is a streaming pass with batched commits, then a VACUUM to return the
462
- * freed pages.
463
- */
464
- compact() {
465
- const r = this.store.spliceCompact(this.M, this.Mmax0, this.entry);
466
- this.entry = r.entry;
467
- this.maxLevel = r.maxLevel;
468
- this.live = r.live;
469
- this.total = r.live;
470
- this.persistState();
471
- this.working.clear();
472
- this.store.vacuum();
466
+ return hits;
467
+ }
468
+ /** Tombstone a live node by internal id. Caller owns the transaction. */
469
+ remove(id) {
470
+ const rec = this.store.getNode(id);
471
+ if (rec === null || rec.deleted === 1) {
472
+ return false;
473
473
  }
474
+ this.store.tombstone(id);
475
+ this.live--;
476
+ this.persistState();
477
+ return true;
478
+ }
479
+ /**
480
+ * Reclaim tombstones with a graph-preserving splice (see
481
+ * {@link Store.spliceCompact}): live nodes keep their internal ids and their
482
+ * wiring; each dead neighbour is replaced by its own live neighbours. Codes
483
+ * are untouched, so the index loses nothing beyond what deletion itself
484
+ * removes. The previous implementation replayed every live code through a
485
+ * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
486
+ * multi-million-node store, inside one WAL-bloating transaction. The splice
487
+ * is a streaming pass with batched commits, then a VACUUM to return the
488
+ * freed pages.
489
+ */
490
+ compact() {
491
+ const r = this.store.spliceCompact(this.M, this.Mmax0, this.entry);
492
+ this.entry = r.entry;
493
+ this.maxLevel = r.maxLevel;
494
+ this.live = r.live;
495
+ this.total = r.live;
496
+ this.persistState();
497
+ this.working.clear();
498
+ this.store.vacuum();
499
+ }
474
500
  }