@hviana/sema 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +13 -9
- package/dist/src/sema.js +40 -26
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
|
@@ -1,233 +1,287 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as os from "node:os";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
-
import { RaBitQuantizer, VectorDatabase
|
|
4
|
+
import { RaBitQuantizer, VectorDatabase } from "../src/index.js";
|
|
5
5
|
// ----------------------------- tiny harness ----------------------------------
|
|
6
6
|
let passed = 0;
|
|
7
7
|
function assert(cond, msg) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
if (!cond) {
|
|
9
|
+
throw new Error("ASSERTION FAILED: " + msg);
|
|
10
|
+
}
|
|
11
|
+
passed++;
|
|
11
12
|
}
|
|
12
13
|
function approxEqual(a, b, eps = 1e-4) {
|
|
13
|
-
|
|
14
|
+
return Math.abs(a - b) <= eps * (1 + Math.abs(a) + Math.abs(b));
|
|
14
15
|
}
|
|
15
16
|
function section(title) {
|
|
16
|
-
|
|
17
|
+
console.log("\n" + "=".repeat(72) + "\n" + title + "\n" + "=".repeat(72));
|
|
17
18
|
}
|
|
18
19
|
function now() {
|
|
19
|
-
|
|
20
|
+
return typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
20
21
|
}
|
|
21
22
|
// ------------------------------- seeded RNG ----------------------------------
|
|
22
23
|
class RNG {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
while (v === 0)
|
|
50
|
-
v = this.next();
|
|
51
|
-
const r = Math.sqrt(-2 * Math.log(u));
|
|
52
|
-
this._gaussSpare = r * Math.sin(2 * Math.PI * v);
|
|
53
|
-
this._gaussHasSpare = true;
|
|
54
|
-
return r * Math.cos(2 * Math.PI * v);
|
|
24
|
+
s;
|
|
25
|
+
constructor(seed) {
|
|
26
|
+
this.s = seed >>> 0 || 1;
|
|
27
|
+
}
|
|
28
|
+
next() {
|
|
29
|
+
let t = (this.s = (this.s + 0x6d2b79f5) | 0);
|
|
30
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
31
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
32
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
33
|
+
}
|
|
34
|
+
int(n) {
|
|
35
|
+
return (this.next() * n) | 0;
|
|
36
|
+
}
|
|
37
|
+
_gaussSpare = 0;
|
|
38
|
+
_gaussHasSpare = false;
|
|
39
|
+
/** Box-Muller — caches the second value so every call produces one Gaussian
|
|
40
|
+
* without wasting the log/sqrt pair. */
|
|
41
|
+
gauss() {
|
|
42
|
+
if (this._gaussHasSpare) {
|
|
43
|
+
this._gaussHasSpare = false;
|
|
44
|
+
return this._gaussSpare;
|
|
45
|
+
}
|
|
46
|
+
let u = 0;
|
|
47
|
+
let v = 0;
|
|
48
|
+
while (u === 0) {
|
|
49
|
+
u = this.next();
|
|
55
50
|
}
|
|
51
|
+
while (v === 0) {
|
|
52
|
+
v = this.next();
|
|
53
|
+
}
|
|
54
|
+
const r = Math.sqrt(-2 * Math.log(u));
|
|
55
|
+
this._gaussSpare = r * Math.sin(2 * Math.PI * v);
|
|
56
|
+
this._gaussHasSpare = true;
|
|
57
|
+
return r * Math.cos(2 * Math.PI * v);
|
|
58
|
+
}
|
|
56
59
|
}
|
|
57
60
|
function unit(v) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
let s = 0;
|
|
62
|
+
for (const x of v) {
|
|
63
|
+
s += x * x;
|
|
64
|
+
}
|
|
65
|
+
const n = Math.sqrt(s) || 1;
|
|
66
|
+
return v.map((x) => x / n);
|
|
63
67
|
}
|
|
64
68
|
// =============================================================================
|
|
65
69
|
// 1) Vector CRUD + persistence (SQLite file round-trip)
|
|
66
70
|
// =============================================================================
|
|
67
71
|
function testCrudAndPersistence() {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
72
|
+
section("1) Vector CRUD + persistence (get returns the 1-bit code)");
|
|
73
|
+
const DIM = 48; // paddedDim 64 -> codeWords 2, distinctive codes
|
|
74
|
+
const dbPath = path.join(
|
|
75
|
+
os.tmpdir(),
|
|
76
|
+
`rabitq-test-crud-${Date.now()}.sqlite`,
|
|
77
|
+
);
|
|
78
|
+
let db = new VectorDatabase({
|
|
79
|
+
dbPath,
|
|
80
|
+
dim: DIM,
|
|
81
|
+
M: 8,
|
|
82
|
+
efConstruction: 64,
|
|
83
|
+
seed: 7,
|
|
84
|
+
});
|
|
85
|
+
// distinct directions so the 1-bit codes differ
|
|
86
|
+
const mk = (i, scale = 1) => {
|
|
87
|
+
const v = new Array(DIM).fill(0);
|
|
88
|
+
v[i] = scale;
|
|
89
|
+
return v;
|
|
90
|
+
};
|
|
91
|
+
// Integer IDs only
|
|
92
|
+
const ID = { alpha: 1, bravo: 2, charlie: 3, delta: 4, echo: 5 };
|
|
93
|
+
const vectors = {
|
|
94
|
+
alpha: mk(0),
|
|
95
|
+
bravo: mk(10),
|
|
96
|
+
charlie: mk(20),
|
|
97
|
+
delta: mk(30),
|
|
98
|
+
echo: mk(45),
|
|
99
|
+
};
|
|
100
|
+
// CREATE
|
|
101
|
+
for (const [name, v] of Object.entries(vectors)) {
|
|
102
|
+
db.insert(ID[name], v);
|
|
103
|
+
}
|
|
104
|
+
assert(db.size === 5, "size should be 5 after inserting 5 vectors");
|
|
105
|
+
let threw = false;
|
|
106
|
+
try {
|
|
107
|
+
db.insert(ID.alpha, vectors.alpha);
|
|
108
|
+
} catch {
|
|
109
|
+
threw = true;
|
|
110
|
+
}
|
|
111
|
+
assert(threw, "inserting a duplicate external id must throw");
|
|
112
|
+
// READ — returns the bare 1-bit code (Uint32Array)
|
|
113
|
+
const code = db.get(ID.alpha);
|
|
114
|
+
assert(code !== null, "get(alpha) returns a stored code");
|
|
115
|
+
assert(
|
|
116
|
+
code instanceof Uint32Array && code.length === 2,
|
|
117
|
+
"code is 2 packed 32-bit words",
|
|
118
|
+
);
|
|
119
|
+
assert(code.length * 32 === 64, "code covers paddedDim = 64 sign bits");
|
|
120
|
+
assert(db.get(9999) === null, "get of unknown id returns null");
|
|
121
|
+
assert(db.has(ID.bravo) && !db.has(9999), "has() reflects membership");
|
|
122
|
+
// SEARCH — querying with an exact copy returns that vector first
|
|
123
|
+
assert(
|
|
124
|
+
db.query(vectors.alpha, 3)[0].id === ID.alpha,
|
|
125
|
+
"exact-copy query returns alpha first",
|
|
126
|
+
);
|
|
127
|
+
assert(
|
|
128
|
+
db.query(vectors.bravo, 1)[0].id === ID.bravo,
|
|
129
|
+
"exact-copy query returns bravo first",
|
|
130
|
+
);
|
|
131
|
+
// UPDATE
|
|
132
|
+
db.update(ID.charlie, mk(40));
|
|
133
|
+
assert(
|
|
134
|
+
db.query(mk(40), 1)[0].id === ID.charlie,
|
|
135
|
+
"updated charlie is found at its new location",
|
|
136
|
+
);
|
|
137
|
+
assert(db.size === 5, "size unchanged after update");
|
|
138
|
+
// DELETE
|
|
139
|
+
assert(db.delete(ID.delta) === true, "delete returns true for existing id");
|
|
140
|
+
assert(
|
|
141
|
+
db.delete(ID.delta) === false,
|
|
142
|
+
"delete returns false for already removed id",
|
|
143
|
+
);
|
|
144
|
+
assert(!db.has(ID.delta) && db.size === 4, "delta removed, size now 4");
|
|
145
|
+
assert(
|
|
146
|
+
!db.query(vectors.delta, 4).some((r) => r.id === ID.delta),
|
|
147
|
+
"deleted vector never appears in results",
|
|
148
|
+
);
|
|
149
|
+
// PERSISTENCE round-trip — capture state, then close and reopen the same file
|
|
150
|
+
const liveKeys = Array.from(db.keys());
|
|
151
|
+
const probe = mk(0);
|
|
152
|
+
const before = db.query(probe, 4);
|
|
153
|
+
const codesBefore = new Map();
|
|
154
|
+
for (const id of liveKeys) {
|
|
155
|
+
codesBefore.set(id, db.get(id));
|
|
156
|
+
}
|
|
157
|
+
db.close();
|
|
158
|
+
const reloaded = new VectorDatabase({ dbPath });
|
|
159
|
+
assert(reloaded.size === db.size, "reloaded size matches");
|
|
160
|
+
for (const id of liveKeys) {
|
|
161
|
+
const a = codesBefore.get(id);
|
|
162
|
+
const b = reloaded.get(id);
|
|
163
|
+
assert(b !== null, `reloaded code for ${id} is not null`);
|
|
164
|
+
assert(a.length === b.length, `reloaded code length for ${id} matches`);
|
|
165
|
+
for (let i = 0; i < a.length; i++) {
|
|
166
|
+
assert(a[i] === b[i], `reloaded code word for ${id} matches`);
|
|
143
167
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
168
|
+
}
|
|
169
|
+
const after = reloaded.query(probe, 4);
|
|
170
|
+
assert(
|
|
171
|
+
before.length === after.length,
|
|
172
|
+
"reloaded query result length matches",
|
|
173
|
+
);
|
|
174
|
+
for (let i = 0; i < before.length; i++) {
|
|
175
|
+
assert(before[i].id === after[i].id, "reloaded query ordering matches");
|
|
176
|
+
assert(
|
|
177
|
+
approxEqual(before[i].distance, after[i].distance, 1e-6),
|
|
178
|
+
"reloaded query distance matches",
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
reloaded.close();
|
|
182
|
+
// Cleanup
|
|
183
|
+
fs.unlinkSync(dbPath);
|
|
184
|
+
console.log(
|
|
185
|
+
`CRUD + persistence OK (get returns ${code.length}-word codes; reload identical)`,
|
|
186
|
+
);
|
|
154
187
|
}
|
|
155
188
|
// =============================================================================
|
|
156
189
|
// 1b) insert / update / query by an already-quantized 1-bit code
|
|
157
190
|
// =============================================================================
|
|
158
191
|
function testCodeIO() {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
192
|
+
section("1b) insert / update / query by a 1-bit code (length-detected)");
|
|
193
|
+
const DIM = 128; // paddedDim 128 -> codeWords 4
|
|
194
|
+
const rng = new RNG(909);
|
|
195
|
+
const vec = () => Array.from({ length: DIM }, () => rng.gauss());
|
|
196
|
+
const db = new VectorDatabase({
|
|
197
|
+
dbPath: ":memory:",
|
|
198
|
+
dim: DIM,
|
|
199
|
+
M: 16,
|
|
200
|
+
efConstruction: 100,
|
|
201
|
+
seed: 5,
|
|
202
|
+
});
|
|
203
|
+
const N = 800;
|
|
204
|
+
for (let i = 0; i < N; i++) {
|
|
205
|
+
db.insert(i, vec());
|
|
206
|
+
}
|
|
207
|
+
const src = db.get(123);
|
|
208
|
+
assert(
|
|
209
|
+
src instanceof Uint32Array && src.length === 4,
|
|
210
|
+
"get() yields a 4-word code",
|
|
211
|
+
);
|
|
212
|
+
// QUERY by code: a stored code finds itself at distance 0
|
|
213
|
+
const byCode = db.query(src, 5);
|
|
214
|
+
assert(byCode[0].id === 123, "query(code) returns the owning vector first");
|
|
215
|
+
assert(
|
|
216
|
+
approxEqual(byCode[0].distance, 0, 1e-9),
|
|
217
|
+
"query(code) distance to the same code is 0",
|
|
218
|
+
);
|
|
219
|
+
// INSERT by code (length-detected): the copy is a real, findable node
|
|
220
|
+
const copyId = N + 1;
|
|
221
|
+
db.insert(copyId, src);
|
|
222
|
+
assert(db.size === N + 1, "insert(code) adds a vector");
|
|
223
|
+
const copyCode = db.get(copyId);
|
|
224
|
+
for (let i = 0; i < src.length; i++) {
|
|
225
|
+
assert(copyCode[i] === src[i], "insert(code) stored the code verbatim");
|
|
226
|
+
}
|
|
227
|
+
const hits = db.query(src, 5).map((h) => h.id);
|
|
228
|
+
assert(
|
|
229
|
+
hits.includes(copyId) && hits.includes(123),
|
|
230
|
+
"both the original and its code-copy are retrievable",
|
|
231
|
+
);
|
|
232
|
+
// number[] code (e.g. after conversion) is detected the same way
|
|
233
|
+
const asArray = Array.from(src);
|
|
234
|
+
const arrId = N + 2;
|
|
235
|
+
db.insert(arrId, asArray);
|
|
236
|
+
const arrCode = db.get(arrId);
|
|
237
|
+
for (let i = 0; i < src.length; i++) {
|
|
238
|
+
assert(arrCode[i] === src[i], "insert(number[] code) works identically");
|
|
239
|
+
}
|
|
240
|
+
// UPDATE by code: move id 7 onto id 200's code
|
|
241
|
+
const target = db.get(200);
|
|
242
|
+
db.update(7, target);
|
|
243
|
+
const moved = db.get(7);
|
|
244
|
+
for (let i = 0; i < target.length; i++) {
|
|
245
|
+
assert(moved[i] === target[i], "update(code) replaced the code");
|
|
246
|
+
}
|
|
247
|
+
assert(db.size === N + 2, "update(code) keeps size (replace, not add)");
|
|
248
|
+
const near200 = db.query(target, 5).map((h) => h.id);
|
|
249
|
+
assert(
|
|
250
|
+
near200.includes(7) && near200.includes(200),
|
|
251
|
+
"updated-by-code id now sits with its target",
|
|
252
|
+
);
|
|
253
|
+
db.close();
|
|
254
|
+
console.log(
|
|
255
|
+
"insert / update / query by 1-bit code OK (Uint32Array and number[])",
|
|
256
|
+
);
|
|
209
257
|
}
|
|
210
258
|
// =============================================================================
|
|
211
259
|
// 2) Cosine metric sanity
|
|
212
260
|
// =============================================================================
|
|
213
261
|
function testCosine() {
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
262
|
+
section("2) Cosine sanity");
|
|
263
|
+
const DIM = 64;
|
|
264
|
+
const db = new VectorDatabase({ dbPath: ":memory:", dim: DIM, seed: 3 });
|
|
265
|
+
const axis = (i) => {
|
|
266
|
+
const v = new Array(DIM).fill(0);
|
|
267
|
+
v[i] = 1;
|
|
268
|
+
return v;
|
|
269
|
+
};
|
|
270
|
+
db.insert(1, axis(0)); // "x"
|
|
271
|
+
db.insert(2, axis(20)); // "y"
|
|
272
|
+
db.insert(3, axis(40)); // "z"
|
|
273
|
+
const r = db.query(axis(0), 3); // query along x's direction
|
|
274
|
+
assert(r[0].id === 1, "cosine: nearest direction to x's axis is x");
|
|
275
|
+
assert(
|
|
276
|
+
r[0].distance <= 0.1,
|
|
277
|
+
"cosine distance to an identical direction is ~ 0",
|
|
278
|
+
);
|
|
279
|
+
assert(
|
|
280
|
+
r[r.length - 1].distance > r[0].distance,
|
|
281
|
+
"orthogonal directions are farther than the aligned one",
|
|
282
|
+
);
|
|
283
|
+
db.close();
|
|
284
|
+
console.log("Cosine metric OK");
|
|
231
285
|
}
|
|
232
286
|
// =============================================================================
|
|
233
287
|
// 2a) Near-duplicate self-recall (tight cluster)
|
|
@@ -251,68 +305,86 @@ function testCosine() {
|
|
|
251
305
|
// assertions pin both ends: an explicit 4 bits is measurably degraded, the
|
|
252
306
|
// default recovers.
|
|
253
307
|
function testNearDuplicateRecall() {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
};
|
|
269
|
-
const base = new Array(DIM);
|
|
270
|
-
for (let d = 0; d < DIM; d++)
|
|
271
|
-
base[d] = next() * 2 - 1;
|
|
272
|
-
const vecs = [];
|
|
273
|
-
const db = new VectorDatabase({
|
|
274
|
-
dbPath: ":memory:",
|
|
275
|
-
dim: DIM,
|
|
276
|
-
M: 16,
|
|
277
|
-
efConstruction: 64,
|
|
278
|
-
efSearch: 64,
|
|
279
|
-
seed: 123,
|
|
280
|
-
queryBits,
|
|
281
|
-
});
|
|
282
|
-
for (let i = 0; i < N; i++) {
|
|
283
|
-
const v = new Array(DIM);
|
|
284
|
-
let n = 0;
|
|
285
|
-
for (let d = 0; d < DIM; d++) {
|
|
286
|
-
v[d] = base[d] + SPREAD * (next() * 2 - 1);
|
|
287
|
-
n += v[d] * v[d];
|
|
288
|
-
}
|
|
289
|
-
n = Math.sqrt(n) || 1;
|
|
290
|
-
for (let d = 0; d < DIM; d++)
|
|
291
|
-
v[d] /= n;
|
|
292
|
-
vecs.push(v);
|
|
293
|
-
db.insert(i, v);
|
|
294
|
-
}
|
|
295
|
-
let self = 0;
|
|
296
|
-
for (let i = 0; i < N; i++) {
|
|
297
|
-
const h = db.query(vecs[i], 1);
|
|
298
|
-
if (h[0] && h[0].id === i)
|
|
299
|
-
self++;
|
|
300
|
-
}
|
|
301
|
-
db.close();
|
|
302
|
-
return self / N;
|
|
308
|
+
section("2a) Near-duplicate self-recall (tight cluster)");
|
|
309
|
+
const DIM = 256;
|
|
310
|
+
const N = 150;
|
|
311
|
+
const SPREAD = 0.15;
|
|
312
|
+
// A fixed base direction; each vector is the base plus a small per-coordinate
|
|
313
|
+
// perturbation, then normalised — a dense cluster of near-duplicates. A plain
|
|
314
|
+
// LCG drives the perturbation: it produces a uniformly spread cloud around the
|
|
315
|
+
// base (the regime where the queryBits effect is sharpest); the Box-Muller
|
|
316
|
+
// RNG above clusters too tightly to separate the two settings.
|
|
317
|
+
const build = (queryBits) => {
|
|
318
|
+
let s = 7 >>> 0;
|
|
319
|
+
const next = () => {
|
|
320
|
+
s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
|
|
321
|
+
return s / 4294967296;
|
|
303
322
|
};
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
323
|
+
const base = new Array(DIM);
|
|
324
|
+
for (let d = 0; d < DIM; d++) {
|
|
325
|
+
base[d] = next() * 2 - 1;
|
|
326
|
+
}
|
|
327
|
+
const vecs = [];
|
|
328
|
+
const db = new VectorDatabase({
|
|
329
|
+
dbPath: ":memory:",
|
|
330
|
+
dim: DIM,
|
|
331
|
+
M: 16,
|
|
332
|
+
efConstruction: 64,
|
|
333
|
+
efSearch: 64,
|
|
334
|
+
seed: 123,
|
|
335
|
+
queryBits,
|
|
336
|
+
});
|
|
337
|
+
for (let i = 0; i < N; i++) {
|
|
338
|
+
const v = new Array(DIM);
|
|
339
|
+
let n = 0;
|
|
340
|
+
for (let d = 0; d < DIM; d++) {
|
|
341
|
+
v[d] = base[d] + SPREAD * (next() * 2 - 1);
|
|
342
|
+
n += v[d] * v[d];
|
|
343
|
+
}
|
|
344
|
+
n = Math.sqrt(n) || 1;
|
|
345
|
+
for (let d = 0; d < DIM; d++) {
|
|
346
|
+
v[d] /= n;
|
|
347
|
+
}
|
|
348
|
+
vecs.push(v);
|
|
349
|
+
db.insert(i, v);
|
|
350
|
+
}
|
|
351
|
+
let self = 0;
|
|
352
|
+
for (let i = 0; i < N; i++) {
|
|
353
|
+
const h = db.query(vecs[i], 1);
|
|
354
|
+
if (h[0] && h[0].id === i) {
|
|
355
|
+
self++;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
db.close();
|
|
359
|
+
return self / N;
|
|
360
|
+
};
|
|
361
|
+
const r4 = build(4); // explicitly coarse — the old default
|
|
362
|
+
const rDefault = build(8); // the library default after the fix
|
|
363
|
+
console.log(
|
|
364
|
+
`near-duplicate self-recall: queryBits=4 ${(r4 * 100).toFixed(1)}% ` +
|
|
365
|
+
`default(8) ${(rDefault * 100).toFixed(1)}%`,
|
|
366
|
+
);
|
|
367
|
+
// The bug: 4-bit query quantisation cannot resolve a tight cluster, so a
|
|
368
|
+
// sizeable fraction of vectors fail to retrieve themselves.
|
|
369
|
+
assert(
|
|
370
|
+
r4 < 0.9,
|
|
371
|
+
`queryBits=4 is degraded on a tight cluster (got ${
|
|
372
|
+
(r4 * 100).toFixed(1)
|
|
373
|
+
}%)`,
|
|
374
|
+
);
|
|
375
|
+
// The fix: the default B_q restores near-perfect self-recall — same stored
|
|
376
|
+
// codes, sharper query estimate.
|
|
377
|
+
assert(
|
|
378
|
+
rDefault >= 0.95,
|
|
379
|
+
`default queryBits recovers self-recall on a tight cluster (got ${
|
|
380
|
+
(rDefault * 100).toFixed(1)
|
|
381
|
+
}%)`,
|
|
382
|
+
);
|
|
383
|
+
assert(
|
|
384
|
+
rDefault > r4,
|
|
385
|
+
"the default queryBits must beat the coarse 4-bit setting",
|
|
386
|
+
);
|
|
387
|
+
console.log("Near-duplicate self-recall OK (queryBits resolves the cluster)");
|
|
316
388
|
}
|
|
317
389
|
// =============================================================================
|
|
318
390
|
// 2a-bis) The cosFactor bias — a known, deliberate approximation in estimate()
|
|
@@ -345,485 +417,643 @@ function testNearDuplicateRecall() {
|
|
|
345
417
|
// here would tighten to ~0 and the negatives would vanish — at which point this
|
|
346
418
|
// test is the tripwire that flags the behavioural change.
|
|
347
419
|
function testCosFactorBias() {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
420
|
+
section(
|
|
421
|
+
"2a-bis) cosFactor self-distance bias (fixed mean vs per-vector norm)",
|
|
422
|
+
);
|
|
423
|
+
const DIM = 256;
|
|
424
|
+
const N = 300;
|
|
425
|
+
const SPREAD = 0.15;
|
|
426
|
+
let s = 7 >>> 0;
|
|
427
|
+
const next = () => {
|
|
428
|
+
s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
|
|
429
|
+
return s / 4294967296;
|
|
430
|
+
};
|
|
431
|
+
const base = new Array(DIM);
|
|
432
|
+
for (let d = 0; d < DIM; d++) {
|
|
433
|
+
base[d] = next() * 2 - 1;
|
|
434
|
+
}
|
|
435
|
+
const q = new RaBitQuantizer(DIM, { seed: 123 });
|
|
436
|
+
const vecs = [];
|
|
437
|
+
const codes = [];
|
|
438
|
+
for (let i = 0; i < N; i++) {
|
|
439
|
+
const v = new Array(DIM);
|
|
440
|
+
let n = 0;
|
|
441
|
+
for (let d = 0; d < DIM; d++) {
|
|
442
|
+
v[d] = base[d] + SPREAD * (next() * 2 - 1);
|
|
443
|
+
n += v[d] * v[d];
|
|
444
|
+
}
|
|
445
|
+
n = Math.sqrt(n) || 1;
|
|
446
|
+
for (let d = 0; d < DIM; d++) {
|
|
447
|
+
v[d] /= n;
|
|
448
|
+
}
|
|
449
|
+
vecs.push(v);
|
|
450
|
+
codes.push(q.codeToBytes(q.encode(v)));
|
|
451
|
+
}
|
|
452
|
+
// Self-distance: each vector scored against its OWN code. An unbiased cosine
|
|
453
|
+
// estimator returns 0 here; the fixed cosFactor does not.
|
|
454
|
+
let sum = 0;
|
|
455
|
+
let max = 0;
|
|
456
|
+
let negatives = 0;
|
|
457
|
+
for (let i = 0; i < N; i++) {
|
|
458
|
+
const ctx = q.prepareQuery(vecs[i]);
|
|
459
|
+
const d = q.estimate(codes[i], 0, ctx);
|
|
460
|
+
sum += d;
|
|
461
|
+
if (d > max) {
|
|
462
|
+
max = d;
|
|
463
|
+
}
|
|
464
|
+
if (d < 0) {
|
|
465
|
+
negatives++; // distance < 0 <=> estimated cosine > 1 (impossible)
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
const mean = sum / N;
|
|
469
|
+
console.log(
|
|
470
|
+
`self-distance under fixed cosFactor: mean ${mean.toFixed(5)}, ` +
|
|
471
|
+
`max ${max.toFixed(5)}, negatives ${negatives}/${N} ` +
|
|
472
|
+
`(an unbiased estimator would give 0 with 0 negatives)`,
|
|
473
|
+
);
|
|
474
|
+
// The estimate is genuinely non-zero on a self-match: the fixed mean factor
|
|
475
|
+
// does not reproduce the per-vector normalisation.
|
|
476
|
+
assert(
|
|
477
|
+
max > 0.005,
|
|
478
|
+
`self-distance is biased away from 0 (max ${max.toFixed(5)})`,
|
|
479
|
+
);
|
|
480
|
+
// The fingerprint: a large fraction of self-matches yield an IMPOSSIBLE
|
|
481
|
+
// cosine > 1 (negative distance) — only a mean-vs-per-vector substitution
|
|
482
|
+
// produces this. If this ever drops to 0, estimate() changed.
|
|
483
|
+
assert(
|
|
484
|
+
negatives > N / 10,
|
|
485
|
+
`cosFactor produces impossible cosine>1 on self-matches ` +
|
|
486
|
+
`(${negatives}/${N} negative distances)`,
|
|
487
|
+
);
|
|
488
|
+
console.log(
|
|
489
|
+
"cosFactor bias documented (fixed mean L1 norm, not the per-vector factor)",
|
|
490
|
+
);
|
|
403
491
|
}
|
|
404
492
|
// =============================================================================
|
|
405
493
|
// 2b) SQLite persistence round-trip
|
|
406
494
|
// =============================================================================
|
|
407
495
|
function testPersistence() {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
496
|
+
section("2b) SQLite persistence round-trip");
|
|
497
|
+
const DIM = 96;
|
|
498
|
+
const rng = new RNG(2024);
|
|
499
|
+
const vec = () => Array.from({ length: DIM }, () => rng.gauss());
|
|
500
|
+
{
|
|
501
|
+
const dbPath = path.join(
|
|
502
|
+
os.tmpdir(),
|
|
503
|
+
`rabitq-test-persist-${Date.now()}.sqlite`,
|
|
504
|
+
);
|
|
505
|
+
const db = new VectorDatabase({
|
|
506
|
+
dbPath,
|
|
507
|
+
dim: DIM,
|
|
508
|
+
M: 12,
|
|
509
|
+
efConstruction: 80,
|
|
510
|
+
seed: 17,
|
|
511
|
+
});
|
|
512
|
+
for (let i = 0; i < 1500; i++) {
|
|
513
|
+
db.insert(i, vec());
|
|
514
|
+
}
|
|
515
|
+
for (let i = 0; i < 150; i++) {
|
|
516
|
+
db.delete(i * 10); // tombstones
|
|
517
|
+
}
|
|
518
|
+
for (let i = 0; i < 120; i++) {
|
|
519
|
+
db.update(3 + i * 10, vec()); // re-inserts
|
|
520
|
+
}
|
|
521
|
+
// Capture state before closing
|
|
522
|
+
const liveKeys = Array.from(db.keys());
|
|
523
|
+
const codesBefore = new Map();
|
|
524
|
+
for (const id of liveKeys) {
|
|
525
|
+
codesBefore.set(id, db.get(id));
|
|
526
|
+
}
|
|
527
|
+
const queriesBefore = [];
|
|
528
|
+
for (let q = 0; q < 40; q++) {
|
|
529
|
+
const query = vec();
|
|
530
|
+
queriesBefore.push({ query, results: db.query(query, 10) });
|
|
531
|
+
}
|
|
532
|
+
const sizeBefore = db.size;
|
|
533
|
+
db.close();
|
|
534
|
+
// Reopen from file
|
|
535
|
+
const db2 = new VectorDatabase({ dbPath });
|
|
536
|
+
assert(
|
|
537
|
+
db2.size === sizeBefore,
|
|
538
|
+
"size preserved across persistence round-trip",
|
|
539
|
+
);
|
|
540
|
+
// get() parity: identical codes for every live id
|
|
541
|
+
for (const id of liveKeys) {
|
|
542
|
+
const a = codesBefore.get(id);
|
|
543
|
+
const b = db2.get(id);
|
|
544
|
+
assert(b !== null, `id ${id} present after reload`);
|
|
545
|
+
assert(a.length === b.length, "code length identical after reload");
|
|
546
|
+
for (let i = 0; i < a.length; i++) {
|
|
547
|
+
assert(a[i] === b[i], "code identical after reload");
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
// query parity: identical ids and (bit-exact) distances
|
|
551
|
+
let maxDiff = 0;
|
|
552
|
+
for (const { query, results: ra } of queriesBefore) {
|
|
553
|
+
const rb = db2.query(query, 10);
|
|
554
|
+
assert(ra.length === rb.length, "result length identical after reload");
|
|
555
|
+
for (let i = 0; i < ra.length; i++) {
|
|
556
|
+
assert(ra[i].id === rb[i].id, "result id/order identical after reload");
|
|
557
|
+
maxDiff = Math.max(maxDiff, Math.abs(ra[i].distance - rb[i].distance));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
assert(
|
|
561
|
+
maxDiff === 0,
|
|
562
|
+
`distances are bit-exact after reload (max diff ${maxDiff})`,
|
|
563
|
+
);
|
|
564
|
+
// Mutation still works after a reload
|
|
565
|
+
db2.insert(99999, vec());
|
|
566
|
+
assert(db2.has(99999), "insert works after persistence load");
|
|
567
|
+
assert(db2.get(99999) !== null, "get works after persistence load");
|
|
568
|
+
db2.close();
|
|
569
|
+
fs.unlinkSync(dbPath);
|
|
570
|
+
console.log(`${sizeBefore} vecs, round-trip via SQLite file — exact match`);
|
|
571
|
+
}
|
|
572
|
+
// Non-existent file: node:sqlite may create the file or may error — this is
|
|
573
|
+
// implementation-specific. We just verify no crash.
|
|
574
|
+
try {
|
|
575
|
+
new VectorDatabase({ dbPath: "/nonexistent/path/db.sqlite" });
|
|
576
|
+
} catch {
|
|
577
|
+
// tolerated — either outcome is acceptable
|
|
578
|
+
}
|
|
579
|
+
console.log("Non-existent path handled gracefully");
|
|
480
580
|
}
|
|
481
581
|
// =============================================================================
|
|
482
582
|
// 2c) compact() — lossless rebuild that reclaims tombstones
|
|
483
583
|
// =============================================================================
|
|
484
584
|
function testCompact() {
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
dbFull.
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
dbPath: ":memory:",
|
|
526
|
-
dim: DIM,
|
|
527
|
-
M: 16,
|
|
528
|
-
efConstruction: 100,
|
|
529
|
-
seed: 88,
|
|
530
|
-
});
|
|
531
|
-
for (const i of survivors)
|
|
532
|
-
dbFresh.insert(i, data[i]);
|
|
533
|
-
// HNSW is an approximate index — two builds with the same data+seed may
|
|
534
|
-
// produce slightly different graphs (the heuristic is not order-invariant
|
|
535
|
-
// under all edge cases). What matters is that recall between the two is
|
|
536
|
-
// very high, and that distances are consistent.
|
|
537
|
-
let compactHits = 0;
|
|
538
|
-
let maxDistDiff = 0;
|
|
539
|
-
let totalCmp = 0;
|
|
540
|
-
for (let q = 0; q < 60; q++) {
|
|
541
|
-
const query = vec();
|
|
542
|
-
const a = dbFull.query(query, 10);
|
|
543
|
-
const b = dbFresh.query(query, 10);
|
|
544
|
-
const bIds = new Set(b.map((r) => r.id));
|
|
545
|
-
for (const r of a)
|
|
546
|
-
if (bIds.has(r.id))
|
|
547
|
-
compactHits++;
|
|
548
|
-
totalCmp += a.length;
|
|
549
|
-
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
550
|
-
maxDistDiff = Math.max(maxDistDiff, Math.abs(a[i].distance - b[i].distance));
|
|
551
|
-
}
|
|
585
|
+
section("2c) compact() — lossless rebuild reclaiming tombstones");
|
|
586
|
+
const DIM = 80;
|
|
587
|
+
const rng = new RNG(555);
|
|
588
|
+
const vec = () => Array.from({ length: DIM }, () => rng.gauss());
|
|
589
|
+
// ---- losslessness: build-then-delete-then-compact == fresh build of survivors ----
|
|
590
|
+
const N = 2500;
|
|
591
|
+
const data = [];
|
|
592
|
+
for (let i = 0; i < N; i++) {
|
|
593
|
+
data.push(vec());
|
|
594
|
+
}
|
|
595
|
+
const deleted = new Set();
|
|
596
|
+
for (let i = 0; i < N; i += 3) {
|
|
597
|
+
deleted.add(i); // delete ~1/3
|
|
598
|
+
}
|
|
599
|
+
const dbFull = new VectorDatabase({
|
|
600
|
+
dbPath: ":memory:",
|
|
601
|
+
dim: DIM,
|
|
602
|
+
M: 16,
|
|
603
|
+
efConstruction: 100,
|
|
604
|
+
seed: 88,
|
|
605
|
+
});
|
|
606
|
+
for (let i = 0; i < N; i++) {
|
|
607
|
+
dbFull.insert(i, data[i]);
|
|
608
|
+
}
|
|
609
|
+
for (const i of deleted) {
|
|
610
|
+
dbFull.delete(i);
|
|
611
|
+
}
|
|
612
|
+
assert(
|
|
613
|
+
dbFull.physicalSize === N,
|
|
614
|
+
"before compact: tombstones still occupy physical slots",
|
|
615
|
+
);
|
|
616
|
+
assert(
|
|
617
|
+
dbFull.size === N - deleted.size,
|
|
618
|
+
"before compact: live size already excludes deleted",
|
|
619
|
+
);
|
|
620
|
+
dbFull.compact();
|
|
621
|
+
const survivors = [];
|
|
622
|
+
for (let i = 0; i < N; i++) {
|
|
623
|
+
if (!deleted.has(i)) {
|
|
624
|
+
survivors.push(i);
|
|
552
625
|
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
626
|
+
}
|
|
627
|
+
assert(
|
|
628
|
+
dbFull.size === survivors.length,
|
|
629
|
+
"after compact: size is the survivor count",
|
|
630
|
+
);
|
|
631
|
+
assert(
|
|
632
|
+
dbFull.physicalSize === survivors.length,
|
|
633
|
+
"after compact: no tombstones remain (physical == live)",
|
|
634
|
+
);
|
|
635
|
+
for (const i of survivors) {
|
|
636
|
+
assert(dbFull.has(i), `survivor ${i} still present after compact`);
|
|
637
|
+
}
|
|
638
|
+
for (const i of deleted) {
|
|
639
|
+
assert(!dbFull.has(i), `deleted ${i} absent after compact`);
|
|
640
|
+
}
|
|
641
|
+
// Fresh index built from the survivors, same order + seed.
|
|
642
|
+
const dbFresh = new VectorDatabase({
|
|
643
|
+
dbPath: ":memory:",
|
|
644
|
+
dim: DIM,
|
|
645
|
+
M: 16,
|
|
646
|
+
efConstruction: 100,
|
|
647
|
+
seed: 88,
|
|
648
|
+
});
|
|
649
|
+
for (const i of survivors) {
|
|
650
|
+
dbFresh.insert(i, data[i]);
|
|
651
|
+
}
|
|
652
|
+
// HNSW is an approximate index — two builds with the same data+seed may
|
|
653
|
+
// produce slightly different graphs (the heuristic is not order-invariant
|
|
654
|
+
// under all edge cases). What matters is that recall between the two is
|
|
655
|
+
// very high, and that distances are consistent.
|
|
656
|
+
let compactHits = 0;
|
|
657
|
+
let maxDistDiff = 0;
|
|
658
|
+
let totalCmp = 0;
|
|
659
|
+
for (let q = 0; q < 60; q++) {
|
|
660
|
+
const query = vec();
|
|
661
|
+
const a = dbFull.query(query, 10);
|
|
662
|
+
const b = dbFresh.query(query, 10);
|
|
663
|
+
const bIds = new Set(b.map((r) => r.id));
|
|
664
|
+
for (const r of a) {
|
|
665
|
+
if (bIds.has(r.id)) {
|
|
666
|
+
compactHits++;
|
|
667
|
+
}
|
|
579
668
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
db2.update(k, v); // re-place within the same group
|
|
587
|
-
live.set(k, v);
|
|
588
|
-
}
|
|
589
|
-
assert(db2.physicalSize > db2.size, "updates+deletes created tombstones");
|
|
590
|
-
db2.compact();
|
|
591
|
-
assert(db2.physicalSize === db2.size && db2.size === live.size, "compact reclaimed all tombstones");
|
|
592
|
-
// recall@10 over the surviving set after compaction
|
|
593
|
-
const liveIds = Array.from(live.keys());
|
|
594
|
-
const liveVecs = Array.from(live.values());
|
|
595
|
-
// Precompute data norms once (they don't change per query).
|
|
596
|
-
const liveNorms = new Float64Array(liveVecs.length);
|
|
597
|
-
for (let n = 0; n < liveVecs.length; n++) {
|
|
598
|
-
let s = 0;
|
|
599
|
-
const v = liveVecs[n];
|
|
600
|
-
for (let d = 0; d < DIM; d++)
|
|
601
|
-
s += v[d] * v[d];
|
|
602
|
-
liveNorms[n] = Math.sqrt(s) || 1;
|
|
669
|
+
totalCmp += a.length;
|
|
670
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
671
|
+
maxDistDiff = Math.max(
|
|
672
|
+
maxDistDiff,
|
|
673
|
+
Math.abs(a[i].distance - b[i].distance),
|
|
674
|
+
);
|
|
603
675
|
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
676
|
+
}
|
|
677
|
+
const recallCompact = compactHits / totalCmp;
|
|
678
|
+
assert(
|
|
679
|
+
recallCompact >= 0.97,
|
|
680
|
+
`compacted recall vs fresh build must be high (got ${
|
|
681
|
+
(recallCompact * 100).toFixed(1)
|
|
682
|
+
}%)`,
|
|
683
|
+
);
|
|
684
|
+
assert(
|
|
685
|
+
maxDistDiff < 0.05,
|
|
686
|
+
`distance agreement across the two builds (max diff ${maxDistDiff})`,
|
|
687
|
+
);
|
|
688
|
+
console.log(
|
|
689
|
+
`losslessness: ${N} built, ${deleted.size} deleted, compacted to ${survivors.length}; recall vs fresh=${
|
|
690
|
+
(recallCompact * 100).toFixed(1)
|
|
691
|
+
}%`,
|
|
692
|
+
);
|
|
693
|
+
// ---- compact also handles updates, and recall stays high on separable data ----
|
|
694
|
+
const G = 10;
|
|
695
|
+
const groups = 400;
|
|
696
|
+
const centers = [];
|
|
697
|
+
for (let g = 0; g < groups; g++) {
|
|
698
|
+
centers.push(vec());
|
|
699
|
+
}
|
|
700
|
+
const db2 = new VectorDatabase({
|
|
701
|
+
dbPath: ":memory:",
|
|
702
|
+
dim: DIM,
|
|
703
|
+
M: 16,
|
|
704
|
+
efConstruction: 100,
|
|
705
|
+
seed: 7,
|
|
706
|
+
});
|
|
707
|
+
const live = new Map();
|
|
708
|
+
let id = 0;
|
|
709
|
+
for (let g = 0; g < groups; g++) {
|
|
710
|
+
for (let j = 0; j < G; j++) {
|
|
711
|
+
const v = centers[g].map((c) => c + 0.1 * rng.gauss());
|
|
712
|
+
db2.insert(id, v);
|
|
713
|
+
live.set(id, v);
|
|
714
|
+
id++;
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
for (let k = 0; k < 400; k++) {
|
|
718
|
+
db2.delete(k); // drop 40 whole groups
|
|
719
|
+
}
|
|
720
|
+
for (let k = 0; k < 400; k++) {
|
|
721
|
+
live.delete(k);
|
|
722
|
+
}
|
|
723
|
+
for (let k = 600; k < 800; k++) {
|
|
724
|
+
const v = centers[Math.floor(k / G)].map((c) => c + 0.1 * rng.gauss());
|
|
725
|
+
db2.update(k, v); // re-place within the same group
|
|
726
|
+
live.set(k, v);
|
|
727
|
+
}
|
|
728
|
+
assert(db2.physicalSize > db2.size, "updates+deletes created tombstones");
|
|
729
|
+
db2.compact();
|
|
730
|
+
assert(
|
|
731
|
+
db2.physicalSize === db2.size && db2.size === live.size,
|
|
732
|
+
"compact reclaimed all tombstones",
|
|
733
|
+
);
|
|
734
|
+
// recall@10 over the surviving set after compaction
|
|
735
|
+
const liveIds = Array.from(live.keys());
|
|
736
|
+
const liveVecs = Array.from(live.values());
|
|
737
|
+
// Precompute data norms once (they don't change per query).
|
|
738
|
+
const liveNorms = new Float64Array(liveVecs.length);
|
|
739
|
+
for (let n = 0; n < liveVecs.length; n++) {
|
|
740
|
+
let s = 0;
|
|
741
|
+
const v = liveVecs[n];
|
|
742
|
+
for (let d = 0; d < DIM; d++) {
|
|
743
|
+
s += v[d] * v[d];
|
|
744
|
+
}
|
|
745
|
+
liveNorms[n] = Math.sqrt(s) || 1;
|
|
746
|
+
}
|
|
747
|
+
let hits = 0;
|
|
748
|
+
const Q = 100;
|
|
749
|
+
for (let q = 0; q < Q; q++) {
|
|
750
|
+
const center = centers[Math.floor((600 + (q % 200)) / G)];
|
|
751
|
+
const query = center.map((c) => c + 0.1 * rng.gauss());
|
|
752
|
+
// Query norm computed once, not N times.
|
|
753
|
+
let qn = 0;
|
|
754
|
+
for (let d = 0; d < DIM; d++) {
|
|
755
|
+
qn += query[d] * query[d];
|
|
756
|
+
}
|
|
757
|
+
qn = Math.sqrt(qn) || 1;
|
|
758
|
+
const bd = new Float64Array(10).fill(Infinity);
|
|
759
|
+
const bi = new Int32Array(10).fill(-1);
|
|
760
|
+
for (let n = 0; n < liveVecs.length; n++) {
|
|
761
|
+
const v = liveVecs[n];
|
|
762
|
+
let dot = 0;
|
|
763
|
+
for (let d = 0; d < DIM; d++) {
|
|
764
|
+
dot += v[d] * query[d];
|
|
765
|
+
}
|
|
766
|
+
const s = 1 - dot / (liveNorms[n] * qn);
|
|
767
|
+
if (s < bd[9]) {
|
|
768
|
+
let p = 9;
|
|
769
|
+
while (p > 0 && bd[p - 1] > s) {
|
|
770
|
+
bd[p] = bd[p - 1];
|
|
771
|
+
bi[p] = bi[p - 1];
|
|
772
|
+
p--;
|
|
632
773
|
}
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
const
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
774
|
+
bd[p] = s;
|
|
775
|
+
bi[p] = liveIds[n];
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
const truth = new Set(Array.from(bi).filter((x) => x >= 0));
|
|
779
|
+
for (const r of db2.query(query, 10)) {
|
|
780
|
+
if (truth.has(r.id)) {
|
|
781
|
+
hits++;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
const recall = hits / (Q * 10);
|
|
786
|
+
assert(
|
|
787
|
+
recall >= 0.9,
|
|
788
|
+
`recall after compact stays high (got ${(recall * 100).toFixed(1)}%)`,
|
|
789
|
+
);
|
|
790
|
+
console.log(
|
|
791
|
+
`updates+deletes: compacted to ${db2.size} live; recall@10 after compact ${
|
|
792
|
+
(recall * 100).toFixed(1)
|
|
793
|
+
}%`,
|
|
794
|
+
);
|
|
795
|
+
dbFull.close();
|
|
796
|
+
dbFresh.close();
|
|
797
|
+
db2.close();
|
|
644
798
|
}
|
|
645
799
|
// =============================================================================
|
|
646
800
|
// 3) Memory + recall + sub-linearity benchmark
|
|
647
801
|
// =============================================================================
|
|
648
802
|
function benchmark() {
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
for (let
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
803
|
+
section(
|
|
804
|
+
"3) Memory + recall + sub-linearity benchmark (HNSW over 1-bit codes)",
|
|
805
|
+
);
|
|
806
|
+
// Tweak these for a faster run or a wider sweep.
|
|
807
|
+
const DIM = 256; // 256-d -> 32-byte codes (the headline example)
|
|
808
|
+
const checkpoints = [1000, 2000, 4000, 8000, 16000, 32000];
|
|
809
|
+
const GROUP = 10; // well-separated groups of near-duplicates
|
|
810
|
+
const K = 10; // recall@K against the exact K nearest
|
|
811
|
+
const SPREAD = 0.1;
|
|
812
|
+
const Q = 200;
|
|
813
|
+
const EF = 150;
|
|
814
|
+
const rng = new RNG(12345);
|
|
815
|
+
const MAX_N = checkpoints[checkpoints.length - 1];
|
|
816
|
+
const numGroups = Math.ceil(MAX_N / GROUP);
|
|
817
|
+
console.log(
|
|
818
|
+
`dim=${DIM}, ${GROUP}-point groups, queries=${Q}, k=${K}, efSearch=${EF}`,
|
|
819
|
+
);
|
|
820
|
+
console.log("generating data ...");
|
|
821
|
+
const centers = new Float32Array(numGroups * DIM);
|
|
822
|
+
for (let g = 0; g < numGroups; g++) {
|
|
823
|
+
for (let d = 0; d < DIM; d++) {
|
|
824
|
+
centers[g * DIM + d] = rng.gauss();
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
const data = new Float32Array(MAX_N * DIM);
|
|
828
|
+
for (let i = 0; i < MAX_N; i++) {
|
|
829
|
+
const base = Math.floor(i / GROUP) * DIM;
|
|
830
|
+
const off = i * DIM;
|
|
831
|
+
for (let d = 0; d < DIM; d++) {
|
|
832
|
+
data[off + d] = centers[base + d] + SPREAD * rng.gauss();
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
// queries land in groups that exist at every checkpoint
|
|
836
|
+
const firstGroups = checkpoints[0] / GROUP;
|
|
837
|
+
const queries = new Float32Array(Q * DIM);
|
|
838
|
+
for (let q = 0; q < Q; q++) {
|
|
839
|
+
const base = rng.int(firstGroups) * DIM;
|
|
840
|
+
const off = q * DIM;
|
|
841
|
+
for (let d = 0; d < DIM; d++) {
|
|
842
|
+
queries[off + d] = centers[base + d] + SPREAD * rng.gauss();
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
const db = new VectorDatabase({
|
|
846
|
+
dbPath: ":memory:",
|
|
847
|
+
dim: DIM,
|
|
848
|
+
M: 16,
|
|
849
|
+
efConstruction: 100,
|
|
850
|
+
efSearch: EF,
|
|
851
|
+
seed: 999,
|
|
852
|
+
});
|
|
853
|
+
// Precompute data vector norms once — O(N·D) instead of O(Q·N·D).
|
|
854
|
+
const dataNorms = new Float64Array(MAX_N);
|
|
855
|
+
for (let i = 0; i < MAX_N; i++) {
|
|
856
|
+
let s = 0;
|
|
857
|
+
const off = i * DIM;
|
|
858
|
+
for (let d = 0; d < DIM; d++) {
|
|
859
|
+
s += data[off + d] * data[off + d];
|
|
860
|
+
}
|
|
861
|
+
dataNorms[i] = Math.sqrt(s) || 1;
|
|
862
|
+
}
|
|
863
|
+
function bruteForceTopK(n, qOff) {
|
|
864
|
+
const bd = new Float64Array(K).fill(Infinity);
|
|
865
|
+
const bi = new Int32Array(K).fill(-1);
|
|
866
|
+
let qn = 0;
|
|
867
|
+
for (let d = 0; d < DIM; d++) {
|
|
868
|
+
qn += queries[qOff + d] * queries[qOff + d];
|
|
869
|
+
}
|
|
870
|
+
qn = Math.sqrt(qn) || 1;
|
|
871
|
+
for (let i = 0; i < n; i++) {
|
|
872
|
+
const off = i * DIM;
|
|
873
|
+
let dot = 0;
|
|
874
|
+
for (let d = 0; d < DIM; d++) {
|
|
875
|
+
dot += data[off + d] * queries[qOff + d];
|
|
876
|
+
}
|
|
877
|
+
const s = 1 - dot / (dataNorms[i] * qn);
|
|
878
|
+
if (s < bd[K - 1]) {
|
|
879
|
+
let p = K - 1;
|
|
880
|
+
while (p > 0 && bd[p - 1] > s) {
|
|
881
|
+
bd[p] = bd[p - 1];
|
|
882
|
+
bi[p] = bi[p - 1];
|
|
883
|
+
p--;
|
|
674
884
|
}
|
|
885
|
+
bd[p] = s;
|
|
886
|
+
bi[p] = i;
|
|
887
|
+
}
|
|
675
888
|
}
|
|
676
|
-
//
|
|
677
|
-
const
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
889
|
+
// Direct set from typed array filtered values.
|
|
890
|
+
const truth = new Set();
|
|
891
|
+
for (let i = 0; i < K; i++) {
|
|
892
|
+
if (bi[i] >= 0) {
|
|
893
|
+
truth.add(bi[i]);
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
return truth;
|
|
897
|
+
}
|
|
898
|
+
const rows = [];
|
|
899
|
+
let inserted = 0;
|
|
900
|
+
for (const cp of checkpoints) {
|
|
901
|
+
const tb = now();
|
|
902
|
+
for (; inserted < cp; inserted++) {
|
|
903
|
+
db.insert(inserted, data.subarray(inserted * DIM, inserted * DIM + DIM));
|
|
904
|
+
}
|
|
905
|
+
console.log(` built N=${cp} (+${(now() - tb).toFixed(0)}ms)`);
|
|
906
|
+
for (let w = 0; w < 25; w++) {
|
|
907
|
+
const o = (w % Q) * DIM;
|
|
908
|
+
db.query(queries.subarray(o, o + DIM), K);
|
|
909
|
+
}
|
|
910
|
+
const hnswResults = new Array(Q);
|
|
911
|
+
let comps = 0;
|
|
912
|
+
const th = now();
|
|
913
|
+
for (let qi = 0; qi < Q; qi++) {
|
|
914
|
+
const o = qi * DIM;
|
|
915
|
+
hnswResults[qi] = db.query(queries.subarray(o, o + DIM), K).map((x) =>
|
|
916
|
+
x.id
|
|
917
|
+
);
|
|
918
|
+
comps += db.lastQueryDistanceComputations;
|
|
919
|
+
}
|
|
920
|
+
const hnswMs = now() - th;
|
|
921
|
+
const truths = new Array(Q);
|
|
922
|
+
const tbf = now();
|
|
923
|
+
for (let qi = 0; qi < Q; qi++) {
|
|
924
|
+
truths[qi] = bruteForceTopK(cp, qi * DIM);
|
|
925
|
+
}
|
|
926
|
+
const bruteMs = now() - tbf;
|
|
927
|
+
let hits = 0;
|
|
928
|
+
for (let qi = 0; qi < Q; qi++) {
|
|
929
|
+
for (const id of hnswResults[qi]) {
|
|
930
|
+
if (truths[qi].has(id)) {
|
|
931
|
+
hits++;
|
|
684
932
|
}
|
|
933
|
+
}
|
|
685
934
|
}
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
935
|
+
rows.push({
|
|
936
|
+
n: cp,
|
|
937
|
+
hnswUs: (hnswMs / Q) * 1000,
|
|
938
|
+
comps: comps / Q,
|
|
939
|
+
bruteUs: (bruteMs / Q) * 1000,
|
|
940
|
+
recall: hits / (Q * K),
|
|
941
|
+
speedup: bruteMs / hnswMs,
|
|
693
942
|
});
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
const ns = rows.map((r) => r.n);
|
|
809
|
-
const expComps = exponent(ns, rows.map((r) => r.comps));
|
|
810
|
-
const expBrute = exponent(ns, rows.map((r) => r.bruteUs));
|
|
811
|
-
const avgRecall = mean(rows.map((r) => r.recall));
|
|
812
|
-
const range = ns[ns.length - 1] / ns[0];
|
|
813
|
-
console.log("");
|
|
814
|
-
console.log(`Empirical scaling over a ${range}x increase in N (work ~ N^exponent):`);
|
|
815
|
-
console.log(` HNSW distance computations : N^${expComps.toFixed(3)} <- sub-linear (target < 1)`);
|
|
816
|
-
console.log(` Brute-force query : N^${expBrute.toFixed(3)} (linear reference ~ 1.0)`);
|
|
817
|
-
console.log(` Mean recall@${K} : ${(avgRecall * 100).toFixed(2)}%`);
|
|
818
|
-
const last = rows[rows.length - 1];
|
|
819
|
-
assert(st.codeBytesPerVector === Math.ceil(DIM / 32) * 4, "code size is ceil(dim/32) 32-bit words");
|
|
820
|
-
assert(st.compressionRatio > 10, `index must be far smaller than Float32 (got ${st.compressionRatio.toFixed(1)}x)`);
|
|
821
|
-
assert(expComps < 0.8, `HNSW distance-computation growth must be sub-linear (got N^${expComps.toFixed(3)})`);
|
|
822
|
-
assert(avgRecall >= 0.9, `mean recall must be >= 90% (got ${(avgRecall * 100).toFixed(1)}%)`);
|
|
823
|
-
assert(last.hnswUs < last.bruteUs, "HNSW must be faster than brute force at the largest N");
|
|
824
|
-
assert(last.speedup > 1, `HNSW must be faster than brute force at the largest N (got ${last.speedup.toFixed(1)}x)`);
|
|
825
|
-
console.log("\nMemory, sub-linearity, recall and speed-up checks OK");
|
|
826
|
-
db.close();
|
|
943
|
+
}
|
|
944
|
+
// ---- memory report ----
|
|
945
|
+
const st = db.storage;
|
|
946
|
+
const mb = (bytes) => (bytes / (1024 * 1024)).toFixed(2) + " MB";
|
|
947
|
+
console.log("");
|
|
948
|
+
console.log(
|
|
949
|
+
"Per-vector storage (vector payload only, excludes the HNSW graph):",
|
|
950
|
+
);
|
|
951
|
+
console.log(` Float32 baseline : ${st.float32BytesPerVector} B`);
|
|
952
|
+
console.log(
|
|
953
|
+
` 1-bit RaBitQ code : ${st.codeBytesPerVector} B (${
|
|
954
|
+
(st.float32BytesPerVector / st.codeBytesPerVector).toFixed(1)
|
|
955
|
+
}x smaller)`,
|
|
956
|
+
);
|
|
957
|
+
console.log(
|
|
958
|
+
` code (kept) : ${st.bytesPerVector} B (${
|
|
959
|
+
st.compressionRatio.toFixed(1)
|
|
960
|
+
}x smaller overall)`,
|
|
961
|
+
);
|
|
962
|
+
console.log(
|
|
963
|
+
` payload at N=${MAX_N} : Float32 ${
|
|
964
|
+
mb(st.float32BytesPerVector * MAX_N)
|
|
965
|
+
} vs index ${mb(st.bytesPerVector * MAX_N)}`,
|
|
966
|
+
);
|
|
967
|
+
// ---- results table ----
|
|
968
|
+
console.log("");
|
|
969
|
+
console.log(
|
|
970
|
+
" N | HNSW q (us) | dist comps | Brute q (us) | recall@10 | speedup",
|
|
971
|
+
);
|
|
972
|
+
console.log(
|
|
973
|
+
"-------+-------------+------------+--------------+-----------+--------",
|
|
974
|
+
);
|
|
975
|
+
for (const r of rows) {
|
|
976
|
+
console.log(
|
|
977
|
+
`${String(r.n).padStart(6)} | ${r.hnswUs.toFixed(1).padStart(11)} | ${
|
|
978
|
+
r.comps.toFixed(0).padStart(10)
|
|
979
|
+
} | ${
|
|
980
|
+
r.bruteUs
|
|
981
|
+
.toFixed(1)
|
|
982
|
+
.padStart(12)
|
|
983
|
+
} | ${(r.recall * 100).toFixed(1).padStart(8)}% | ${
|
|
984
|
+
r.speedup.toFixed(1).padStart(6)
|
|
985
|
+
}x`,
|
|
986
|
+
);
|
|
987
|
+
}
|
|
988
|
+
const mean = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
989
|
+
const exponent = (xs, ys) => {
|
|
990
|
+
const lx = xs.map(Math.log);
|
|
991
|
+
const ly = ys.map((y) => Math.log(Math.max(y, 1e-12)));
|
|
992
|
+
const mx = mean(lx);
|
|
993
|
+
const my = mean(ly);
|
|
994
|
+
let num = 0;
|
|
995
|
+
let den = 0;
|
|
996
|
+
for (let i = 0; i < lx.length; i++) {
|
|
997
|
+
num += (lx[i] - mx) * (ly[i] - my);
|
|
998
|
+
den += (lx[i] - mx) * (lx[i] - mx);
|
|
999
|
+
}
|
|
1000
|
+
return num / den;
|
|
1001
|
+
};
|
|
1002
|
+
const ns = rows.map((r) => r.n);
|
|
1003
|
+
const expComps = exponent(ns, rows.map((r) => r.comps));
|
|
1004
|
+
const expBrute = exponent(ns, rows.map((r) => r.bruteUs));
|
|
1005
|
+
const avgRecall = mean(rows.map((r) => r.recall));
|
|
1006
|
+
const range = ns[ns.length - 1] / ns[0];
|
|
1007
|
+
console.log("");
|
|
1008
|
+
console.log(
|
|
1009
|
+
`Empirical scaling over a ${range}x increase in N (work ~ N^exponent):`,
|
|
1010
|
+
);
|
|
1011
|
+
console.log(
|
|
1012
|
+
` HNSW distance computations : N^${
|
|
1013
|
+
expComps.toFixed(3)
|
|
1014
|
+
} <- sub-linear (target < 1)`,
|
|
1015
|
+
);
|
|
1016
|
+
console.log(
|
|
1017
|
+
` Brute-force query : N^${
|
|
1018
|
+
expBrute.toFixed(3)
|
|
1019
|
+
} (linear reference ~ 1.0)`,
|
|
1020
|
+
);
|
|
1021
|
+
console.log(
|
|
1022
|
+
` Mean recall@${K} : ${(avgRecall * 100).toFixed(2)}%`,
|
|
1023
|
+
);
|
|
1024
|
+
const last = rows[rows.length - 1];
|
|
1025
|
+
assert(
|
|
1026
|
+
st.codeBytesPerVector === Math.ceil(DIM / 32) * 4,
|
|
1027
|
+
"code size is ceil(dim/32) 32-bit words",
|
|
1028
|
+
);
|
|
1029
|
+
assert(
|
|
1030
|
+
st.compressionRatio > 10,
|
|
1031
|
+
`index must be far smaller than Float32 (got ${
|
|
1032
|
+
st.compressionRatio.toFixed(1)
|
|
1033
|
+
}x)`,
|
|
1034
|
+
);
|
|
1035
|
+
assert(
|
|
1036
|
+
expComps < 0.8,
|
|
1037
|
+
`HNSW distance-computation growth must be sub-linear (got N^${
|
|
1038
|
+
expComps.toFixed(3)
|
|
1039
|
+
})`,
|
|
1040
|
+
);
|
|
1041
|
+
assert(
|
|
1042
|
+
avgRecall >= 0.9,
|
|
1043
|
+
`mean recall must be >= 90% (got ${(avgRecall * 100).toFixed(1)}%)`,
|
|
1044
|
+
);
|
|
1045
|
+
assert(
|
|
1046
|
+
last.hnswUs < last.bruteUs,
|
|
1047
|
+
"HNSW must be faster than brute force at the largest N",
|
|
1048
|
+
);
|
|
1049
|
+
assert(
|
|
1050
|
+
last.speedup > 1,
|
|
1051
|
+
`HNSW must be faster than brute force at the largest N (got ${
|
|
1052
|
+
last.speedup.toFixed(1)
|
|
1053
|
+
}x)`,
|
|
1054
|
+
);
|
|
1055
|
+
console.log("\nMemory, sub-linearity, recall and speed-up checks OK");
|
|
1056
|
+
db.close();
|
|
827
1057
|
}
|
|
828
1058
|
// =============================================================================
|
|
829
1059
|
// 4) Scalability is INTRINSIC, not cache-borne
|
|
@@ -847,102 +1077,121 @@ function benchmark() {
|
|
|
847
1077
|
// flat band; the log–log growth exponent makes "does not grow with N" precise.
|
|
848
1078
|
// =============================================================================
|
|
849
1079
|
function testScalabilityWithoutCache() {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1080
|
+
section("4) Scalability is intrinsic, not cache-borne (cacheSizeMb = 0)");
|
|
1081
|
+
// A small dimension and modest graph degree keep the build cheap — this test
|
|
1082
|
+
// is about the SHAPE of storage-reads vs N, not throughput, so the constants
|
|
1083
|
+
// only need to be large enough for HNSW's structure to hold.
|
|
1084
|
+
const DIM = 64;
|
|
1085
|
+
const db = new VectorDatabase({
|
|
1086
|
+
dbPath: ":memory:",
|
|
1087
|
+
dim: DIM,
|
|
1088
|
+
M: 8,
|
|
1089
|
+
efConstruction: 40,
|
|
1090
|
+
efSearch: 40,
|
|
1091
|
+
seed: 4242,
|
|
1092
|
+
cacheSizeMb: 0, // ALL caches off (page cache + code LRU) — honest reads
|
|
1093
|
+
});
|
|
1094
|
+
const rng = new RNG(20260628);
|
|
1095
|
+
const vec = () => {
|
|
1096
|
+
const v = new Array(DIM);
|
|
1097
|
+
for (let d = 0; d < DIM; d++) {
|
|
1098
|
+
v[d] = rng.gauss();
|
|
1099
|
+
}
|
|
1100
|
+
return unit(v);
|
|
1101
|
+
};
|
|
1102
|
+
// A fixed set of probes, reused at every checkpoint so the only variable is N.
|
|
1103
|
+
const Q = 40;
|
|
1104
|
+
const probes = [];
|
|
1105
|
+
for (let q = 0; q < Q; q++) {
|
|
1106
|
+
probes.push(vec());
|
|
1107
|
+
}
|
|
1108
|
+
const checkpoints = [1000, 2000, 4000, 8000]; // 8x range
|
|
1109
|
+
const readsPerQuery = [];
|
|
1110
|
+
let id = 0;
|
|
1111
|
+
for (const cp of checkpoints) {
|
|
1112
|
+
const tb = now();
|
|
1113
|
+
for (; id < cp; id++) {
|
|
1114
|
+
db.insert(id, vec());
|
|
1115
|
+
}
|
|
1116
|
+
// Mean storage reads over the fixed probe set — the cache-independent cost
|
|
1117
|
+
// of ONE query at this N.
|
|
1118
|
+
let reads = 0;
|
|
1119
|
+
for (const p of probes) {
|
|
1120
|
+
db.query(p, 10);
|
|
1121
|
+
reads += db.lastQueryStorageReads;
|
|
1122
|
+
}
|
|
1123
|
+
readsPerQuery.push(reads / Q);
|
|
1124
|
+
console.log(
|
|
1125
|
+
` N=${String(cp).padStart(6)} +${(now() - tb).toFixed(0)}ms ` +
|
|
1126
|
+
`storage-reads/query ${(reads / Q).toFixed(1)}`,
|
|
1127
|
+
);
|
|
1128
|
+
}
|
|
1129
|
+
// Compare the per-query read count at the LARGEST store against the smallest:
|
|
1130
|
+
// if scaling leaned on the cache, the cache-disabled reads would rise with N
|
|
1131
|
+
// (a deeper graph re-read from disk). The contract is that it stays within a
|
|
1132
|
+
// small constant band — work set by the query, not by the corpus. Log–log
|
|
1133
|
+
// growth exponent makes it precise.
|
|
1134
|
+
const lx = checkpoints.map(Math.log);
|
|
1135
|
+
const ly = readsPerQuery.map((y) => Math.log(Math.max(y, 1e-9)));
|
|
1136
|
+
const mx = lx.reduce((a, c) => a + c, 0) / lx.length;
|
|
1137
|
+
const my = ly.reduce((a, c) => a + c, 0) / ly.length;
|
|
1138
|
+
let num = 0, den = 0;
|
|
1139
|
+
for (let i = 0; i < lx.length; i++) {
|
|
1140
|
+
num += (lx[i] - mx) * (ly[i] - my);
|
|
1141
|
+
den += (lx[i] - mx) * (lx[i] - mx);
|
|
1142
|
+
}
|
|
1143
|
+
const exponent = num / den;
|
|
1144
|
+
const first = readsPerQuery[0];
|
|
1145
|
+
const last = readsPerQuery[readsPerQuery.length - 1];
|
|
1146
|
+
console.log(
|
|
1147
|
+
` reads/query ${first.toFixed(1)} → ${last.toFixed(1)} over ${
|
|
1148
|
+
checkpoints[checkpoints.length - 1] / checkpoints[0]
|
|
1149
|
+
}x N · growth N^${exponent.toFixed(3)}`,
|
|
1150
|
+
);
|
|
1151
|
+
// PRIMARY guard — sub-linear in N: an O(1)-per-touch, log-N-touched query has
|
|
1152
|
+
// storage-reads growing like log N (exponent ≈ 0); an implementation forced to
|
|
1153
|
+
// re-read its working set from disk as N grows pushes this toward and past 1.
|
|
1154
|
+
// Well under 0.5 proves the index does not lean on the cache to scale.
|
|
1155
|
+
assert(
|
|
1156
|
+
exponent < 0.5,
|
|
1157
|
+
`cache-disabled storage-reads/query grew as N^${
|
|
1158
|
+
exponent.toFixed(3)
|
|
1159
|
+
} — the index must scale WITHOUT the page cache (reads set by the query's work, not the corpus size)`,
|
|
1160
|
+
);
|
|
1161
|
+
// SECONDARY — a flat band: the largest store costs no more than ~2.5x the
|
|
1162
|
+
// smallest per query. Generous against HNSW's genuine log-N growth, strict
|
|
1163
|
+
// enough to fail a regression whose reads track the corpus.
|
|
1164
|
+
assert(
|
|
1165
|
+
last < first * 2.5 + 8,
|
|
1166
|
+
`cache-disabled reads/query climbed ${first.toFixed(1)} → ${
|
|
1167
|
+
last.toFixed(1)
|
|
1168
|
+
} as the store grew — expected a flat band (work set by the query, not the corpus)`,
|
|
1169
|
+
);
|
|
1170
|
+
db.close();
|
|
1171
|
+
console.log("\nCache-independent scalability OK");
|
|
923
1172
|
}
|
|
924
1173
|
// ------------------------------------ main -----------------------------------
|
|
925
1174
|
function main() {
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
1175
|
+
const start = now();
|
|
1176
|
+
testCrudAndPersistence();
|
|
1177
|
+
testCodeIO();
|
|
1178
|
+
testCosine();
|
|
1179
|
+
testNearDuplicateRecall();
|
|
1180
|
+
testCosFactorBias();
|
|
1181
|
+
testPersistence();
|
|
1182
|
+
testCompact();
|
|
1183
|
+
testScalabilityWithoutCache();
|
|
1184
|
+
benchmark();
|
|
1185
|
+
const dt = ((now() - start) / 1000).toFixed(1);
|
|
1186
|
+
console.log("\n" + "=".repeat(72));
|
|
1187
|
+
console.log(`ALL TESTS PASSED (${passed} assertions, ${dt}s)`);
|
|
1188
|
+
console.log("=".repeat(72));
|
|
940
1189
|
}
|
|
941
1190
|
try {
|
|
942
|
-
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
1191
|
+
main();
|
|
1192
|
+
} catch (err) {
|
|
1193
|
+
console.error("\n" + String(err && err.stack ? err.stack : err));
|
|
1194
|
+
if (typeof process !== "undefined") {
|
|
1195
|
+
process.exitCode = 1;
|
|
1196
|
+
}
|
|
948
1197
|
}
|