@statewalker/db-sqlite-index 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +289 -2
  3. package/dist/ngram.d.mts +93 -0
  4. package/dist/ngram.d.mts.map +1 -0
  5. package/dist/ngram.mjs +615 -0
  6. package/dist/ngram.mjs.map +1 -0
  7. package/dist/vector.d.mts +103 -0
  8. package/dist/vector.d.mts.map +1 -0
  9. package/dist/vector.mjs +1061 -0
  10. package/dist/vector.mjs.map +1 -0
  11. package/package.json +63 -3
  12. package/src/ngram/do-smoke.ts +46 -0
  13. package/src/ngram/fold.test.ts +50 -0
  14. package/src/ngram/fold.ts +21 -0
  15. package/src/ngram/fts5-backend.ts +55 -0
  16. package/src/ngram/index.ts +12 -0
  17. package/src/ngram/json-chunks.test.ts +23 -0
  18. package/src/ngram/json-chunks.ts +23 -0
  19. package/src/ngram/ngram-store.test.ts +291 -0
  20. package/src/ngram/ngram-store.ts +183 -0
  21. package/src/ngram/plan-query.test.ts +141 -0
  22. package/src/ngram/plan-query.ts +42 -0
  23. package/src/ngram/postings-backend.ts +53 -0
  24. package/src/ngram/regex-literals.ts +359 -0
  25. package/src/ngram/trigrams.test.ts +28 -0
  26. package/src/ngram/trigrams.ts +25 -0
  27. package/src/vector/auto-compact.test.ts +28 -0
  28. package/src/vector/do-smoke.ts +39 -0
  29. package/src/vector/index.ts +9 -0
  30. package/src/vector/kmeans.ts +131 -0
  31. package/src/vector/math.ts +185 -0
  32. package/src/vector/segment.test.ts +61 -0
  33. package/src/vector/segment.ts +256 -0
  34. package/src/vector/sql-segment-store.ts +218 -0
  35. package/src/vector/test-fixtures.ts +64 -0
  36. package/src/vector/types.ts +30 -0
  37. package/src/vector/vector-index.test.ts +105 -0
  38. package/src/vector/vector-index.ts +586 -0
  39. package/src/vector/vector-lifecycle.test.ts +171 -0
@@ -0,0 +1,185 @@
1
+ import type { Metric } from "./types.js";
2
+
3
+ export function normalize(v: Float32Array): Float32Array {
4
+ let n = 0;
5
+ for (let i = 0; i < v.length; i++) n += v[i] * v[i];
6
+ n = Math.sqrt(n);
7
+ const out = new Float32Array(v.length);
8
+ if (n === 0) return out;
9
+ for (let i = 0; i < v.length; i++) out[i] = v[i] / n;
10
+ return out;
11
+ }
12
+
13
+ /** Vectors are stored pre-normalized for cosine, so cosine reduces to dot. */
14
+ export function prepare(v: ArrayLike<number>, dims: number, metric: Metric): Float32Array {
15
+ if (v.length !== dims) throw new Error(`vector-index: expected ${dims} dims, got ${v.length}`);
16
+ const f = v instanceof Float32Array ? v : Float32Array.from(v);
17
+ for (let i = 0; i < f.length; i++) {
18
+ if (!Number.isFinite(f[i])) throw new Error("vector-index: vector contains a non-finite value");
19
+ }
20
+ return metric === "cosine" ? normalize(f) : f.slice();
21
+ }
22
+
23
+ /** Distance between two float vectors at offsets; lower is closer. */
24
+ export function distance(
25
+ metric: Metric,
26
+ a: Float32Array,
27
+ ao: number,
28
+ b: Float32Array,
29
+ bo: number,
30
+ d: number,
31
+ ): number {
32
+ if (metric === "l2") {
33
+ let s = 0;
34
+ for (let i = 0; i < d; i++) {
35
+ const x = a[ao + i] - b[bo + i];
36
+ s += x * x;
37
+ }
38
+ return s;
39
+ }
40
+ let s = 0;
41
+ for (let i = 0; i < d; i++) s += a[ao + i] * b[bo + i];
42
+ return metric === "cosine" ? 1 - s : -s;
43
+ }
44
+
45
+ // ------------------------------------------------------------ int8
46
+
47
+ /** Symmetric per-vector int8: x ≈ q * scale / 127. Returns the scale. */
48
+ export function quantizeInt8(v: Float32Array, vo: number, d: number, out: Int8Array, oo: number) {
49
+ let max = 0;
50
+ for (let i = 0; i < d; i++) max = Math.max(max, Math.abs(v[vo + i]));
51
+ const inv = max === 0 ? 0 : 127 / max;
52
+ for (let i = 0; i < d; i++) out[oo + i] = Math.round(v[vo + i] * inv);
53
+ return max;
54
+ }
55
+
56
+ /** Distance between a float query and an int8-coded vector. */
57
+ export function distanceInt8(
58
+ metric: Metric,
59
+ q: Float32Array,
60
+ codes: Int8Array,
61
+ co: number,
62
+ scale: number,
63
+ d: number,
64
+ ): number {
65
+ const f = scale / 127;
66
+ if (metric === "l2") {
67
+ let s = 0;
68
+ for (let i = 0; i < d; i++) {
69
+ const x = q[i] - codes[co + i] * f;
70
+ s += x * x;
71
+ }
72
+ return s;
73
+ }
74
+ let s = 0;
75
+ for (let i = 0; i < d; i++) s += q[i] * codes[co + i];
76
+ s *= f;
77
+ return metric === "cosine" ? 1 - s : -s;
78
+ }
79
+
80
+ export function dequantizeInt8(codes: Int8Array, co: number, scale: number, d: number) {
81
+ const out = new Float32Array(d);
82
+ const f = scale / 127;
83
+ for (let i = 0; i < d; i++) out[i] = codes[co + i] * f;
84
+ return out;
85
+ }
86
+
87
+ // ------------------------------------------------------------ 1-bit
88
+
89
+ export const wordsFor = (d: number) => (d + 31) >>> 5;
90
+
91
+ /** Sign bits of (v - centroid): residual binarization, meaningful for every metric. */
92
+ export function residualBits(
93
+ v: Float32Array,
94
+ vo: number,
95
+ c: Float32Array,
96
+ co: number,
97
+ d: number,
98
+ out: Uint32Array,
99
+ oo: number,
100
+ ) {
101
+ const w = wordsFor(d);
102
+ out.fill(0, oo, oo + w);
103
+ for (let i = 0; i < d; i++) {
104
+ if (v[vo + i] > c[co + i]) out[oo + (i >>> 5)] |= 1 << (i & 31);
105
+ }
106
+ }
107
+
108
+ export function hamming(a: Uint32Array, b: Uint32Array, bo: number, w: number): number {
109
+ let n = 0;
110
+ for (let i = 0; i < w; i++) {
111
+ let x = a[i] ^ b[bo + i];
112
+ x -= (x >>> 1) & 0x55555555;
113
+ x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
114
+ n += Math.imul((x + (x >>> 4)) & 0x0f0f0f0f, 0x01010101) >>> 24;
115
+ }
116
+ return n;
117
+ }
118
+
119
+ // ------------------------------------------------------------ top-k
120
+
121
+ /** Keeps the `k` smallest scores. A binary max-heap over parallel arrays. */
122
+ export class TopK<T> {
123
+ readonly #k: number;
124
+ readonly #scores: number[] = [];
125
+ readonly #items: T[] = [];
126
+
127
+ constructor(k: number) {
128
+ this.#k = Math.max(0, k);
129
+ }
130
+
131
+ get size() {
132
+ return this.#scores.length;
133
+ }
134
+
135
+ /** The score a new item must beat, or Infinity while not full. */
136
+ get bound() {
137
+ return this.#scores.length < this.#k ? Number.POSITIVE_INFINITY : this.#scores[0];
138
+ }
139
+
140
+ push(score: number, item: T) {
141
+ if (this.#k === 0) return;
142
+ const s = this.#scores;
143
+ const it = this.#items;
144
+ if (s.length < this.#k) {
145
+ s.push(score);
146
+ it.push(item);
147
+ let i = s.length - 1;
148
+ while (i > 0) {
149
+ const p = (i - 1) >> 1;
150
+ if (s[p] >= s[i]) break;
151
+ this.#swap(i, p);
152
+ i = p;
153
+ }
154
+ return;
155
+ }
156
+ if (score >= s[0]) return;
157
+ s[0] = score;
158
+ it[0] = item;
159
+ let i = 0;
160
+ for (;;) {
161
+ const l = 2 * i + 1;
162
+ const r = l + 1;
163
+ let m = i;
164
+ if (l < s.length && s[l] > s[m]) m = l;
165
+ if (r < s.length && s[r] > s[m]) m = r;
166
+ if (m === i) break;
167
+ this.#swap(i, m);
168
+ i = m;
169
+ }
170
+ }
171
+
172
+ /** Ascending by score. */
173
+ sorted(): { score: number; item: T }[] {
174
+ return this.#scores
175
+ .map((score, i) => ({ score, item: this.#items[i] }))
176
+ .sort((a, b) => a.score - b.score);
177
+ }
178
+
179
+ #swap(i: number, j: number) {
180
+ const s = this.#scores;
181
+ const it = this.#items;
182
+ [s[i], s[j]] = [s[j], s[i]];
183
+ [it[i], it[j]] = [it[j], it[i]];
184
+ }
185
+ }
@@ -0,0 +1,61 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { mulberry32 } from "./kmeans.js";
3
+ import { prepare } from "./math.js";
4
+ import { buildSegment, MIN_TRAIN, parseList, SegmentIndex } from "./segment.js";
5
+
6
+ const D = 32;
7
+
8
+ function clustered(n: number, seed: number): { ids: Float64Array; vectors: Float32Array } {
9
+ const r = mulberry32(seed);
10
+ const centers = Array.from({ length: 20 }, () =>
11
+ Float32Array.from({ length: D }, () => r() * 2 - 1),
12
+ );
13
+ const vectors = new Float32Array(n * D);
14
+ for (let i = 0; i < n; i++) {
15
+ const c = centers[Math.floor(r() * centers.length)] as Float32Array;
16
+ vectors.set(
17
+ prepare(
18
+ c.map((x) => x + (r() - 0.5) * 0.6),
19
+ D,
20
+ "cosine",
21
+ ),
22
+ i * D,
23
+ );
24
+ }
25
+ return { ids: Float64Array.from({ length: n }, (_, i) => i + 1), vectors };
26
+ }
27
+
28
+ describe("segment codec", () => {
29
+ it("is one flat list below MIN_TRAIN and several lists from MIN_TRAIN on", () => {
30
+ const small = clustered(MIN_TRAIN - 1, 1);
31
+ const large = clustered(MIN_TRAIN, 1);
32
+ expect(buildSegment({ dims: D, metric: "cosine", ...small }).nlist).toBe(1);
33
+ expect(buildSegment({ dims: D, metric: "cosine", ...large }).nlist).toBeGreaterThan(1);
34
+ });
35
+
36
+ it("keeps every row exactly once, and a stored vector ranks itself first", () => {
37
+ const { ids, vectors } = clustered(3000, 2);
38
+ const built = buildSegment({ dims: D, metric: "cosine", ids, vectors });
39
+ const seg = new SegmentIndex(1, D, "cosine", built.centroids, built.counts);
40
+ const lists = built.lists.map((bytes, c) => parseList(bytes, built.counts[c] as number, D));
41
+
42
+ const seen = lists.flatMap((l) => Array.from(l.ids)).sort((a, b) => a - b);
43
+ expect(seen).toEqual(Array.from(ids));
44
+
45
+ const q = vectors.subarray(0, D);
46
+ let best = { id: -1, d: Number.POSITIVE_INFINITY };
47
+ for (const c of seg.probe(q, seg.nlist)) {
48
+ const data = lists[c];
49
+ if (!data) continue;
50
+ for (const cand of seg.filterList(q, c, data, 50)) {
51
+ const d = seg.scoreInt8(q, data, cand.row);
52
+ if (d < best.d) best = { id: cand.id, d };
53
+ }
54
+ }
55
+ expect(best.id).toBe(1);
56
+ });
57
+
58
+ it("rejects a list block of the wrong size", () => {
59
+ expect(() => parseList(new Uint8Array(10), 3, D)).toThrow(/list block has 10 bytes/);
60
+ });
61
+ });
@@ -0,0 +1,256 @@
1
+ import { kmeans, nearest } from "./kmeans.js";
2
+ import {
3
+ distance,
4
+ distanceInt8,
5
+ hamming,
6
+ quantizeInt8,
7
+ residualBits,
8
+ TopK,
9
+ wordsFor,
10
+ } from "./math.js";
11
+ import type { Metric } from "./types.js";
12
+
13
+ /**
14
+ * An IVF segment as storage-neutral parts. This module does no I/O; a store
15
+ * decides where the parts live.
16
+ *
17
+ * per list: a centroid f32[dims] and a row count
18
+ * per list, the searchable block (little-endian, each section padded to 8 bytes):
19
+ * ids f64[count] | bits u32[count × words] | rnorm f32[count] | cdot f32[count]
20
+ * scales f32[count] | codes i8[count × dims]
21
+ * per list, the float block f32[count × dims] (exact re-rank, lossless merge, read-back)
22
+ *
23
+ * bits = signs of the residual r = x - c (c = the list centroid)
24
+ * rnorm = |r|, cdot = c·r. With them the 1-bit stage estimates the real
25
+ * metric, RaBitQ-style: r_q·r ≈ |r_q| |r| cos(π · hamming / dims).
26
+ */
27
+ const align8 = (n: number) => (n + 7) & ~7;
28
+
29
+ export interface ListLayout {
30
+ count: number;
31
+ idsAt: number;
32
+ bitsAt: number;
33
+ rnormAt: number;
34
+ cdotAt: number;
35
+ scalesAt: number;
36
+ codesAt: number;
37
+ /** Bytes of the searchable block. */
38
+ length: number;
39
+ }
40
+
41
+ export function listLayout(count: number, dims: number): ListLayout {
42
+ const w = wordsFor(dims);
43
+ const idsAt = 0;
44
+ const bitsAt = align8(idsAt + 8 * count);
45
+ const rnormAt = align8(bitsAt + 4 * w * count);
46
+ const cdotAt = align8(rnormAt + 4 * count);
47
+ const scalesAt = align8(cdotAt + 4 * count);
48
+ const codesAt = align8(scalesAt + 4 * count);
49
+ const length = align8(codesAt + dims * count);
50
+ return { count, idsAt, bitsAt, rnormAt, cdotAt, scalesAt, codesAt, length };
51
+ }
52
+
53
+ // ------------------------------------------------------------------ build
54
+
55
+ export interface BuildInput {
56
+ dims: number;
57
+ metric: Metric;
58
+ ids: Float64Array;
59
+ /** n × dims, already prepared (normalized for cosine). */
60
+ vectors: Float32Array;
61
+ nlist?: number;
62
+ seed?: number;
63
+ }
64
+
65
+ export interface BuiltSegment {
66
+ nlist: number;
67
+ counts: Uint32Array;
68
+ /** nlist × dims */
69
+ centroids: Float32Array;
70
+ /** Searchable block per list. */
71
+ lists: Uint8Array[];
72
+ /** Float32 block per list. */
73
+ floats: Uint8Array[];
74
+ }
75
+
76
+ /** Below this size a segment is a single flat list: training would not pay off. */
77
+ export const MIN_TRAIN = 2048;
78
+
79
+ export function defaultNlist(n: number) {
80
+ if (n < MIN_TRAIN) return 1;
81
+ return Math.max(1, Math.min(1024, Math.round(Math.sqrt(n))));
82
+ }
83
+
84
+ export function buildSegment(input: BuildInput): BuiltSegment {
85
+ const { dims: d, metric, ids, vectors } = input;
86
+ const n = ids.length;
87
+ const nlist = Math.max(1, Math.min(input.nlist ?? defaultNlist(n), Math.max(1, n)));
88
+
89
+ let centroids: Float32Array;
90
+ if (nlist === 1) {
91
+ centroids = new Float32Array(d);
92
+ for (let i = 0; i < n; i++) for (let j = 0; j < d; j++) centroids[j] += vectors[i * d + j];
93
+ if (n > 0) for (let j = 0; j < d; j++) centroids[j] /= n;
94
+ } else {
95
+ centroids = kmeans(vectors, n, d, nlist, metric, { seed: input.seed });
96
+ }
97
+
98
+ const assign = new Uint32Array(n);
99
+ const counts = new Uint32Array(nlist);
100
+ for (let i = 0; i < n; i++) {
101
+ const c = nlist === 1 ? 0 : nearest(metric, vectors, i * d, centroids, nlist, d);
102
+ assign[i] = c;
103
+ counts[c]++;
104
+ }
105
+
106
+ const w = wordsFor(d);
107
+ const layouts = Array.from(counts, (count) => listLayout(count, d));
108
+ const lists = layouts.map((l) => new Uint8Array(l.length));
109
+ const floats = Array.from(counts, (count) => new Uint8Array(4 * count * d));
110
+ const views = layouts.map((l, c) => {
111
+ const buf = lists[c].buffer;
112
+ return {
113
+ ids: new Float64Array(buf, l.idsAt, l.count),
114
+ bits: new Uint32Array(buf, l.bitsAt, l.count * w),
115
+ rnorm: new Float32Array(buf, l.rnormAt, l.count),
116
+ cdot: new Float32Array(buf, l.cdotAt, l.count),
117
+ scales: new Float32Array(buf, l.scalesAt, l.count),
118
+ codes: new Int8Array(buf, l.codesAt, l.count * d),
119
+ floats: new Float32Array(floats[c].buffer, 0, l.count * d),
120
+ };
121
+ });
122
+
123
+ const fill = new Uint32Array(nlist);
124
+ for (let i = 0; i < n; i++) {
125
+ const c = assign[i];
126
+ const j = fill[c]++;
127
+ const v = views[c];
128
+ v.ids[j] = ids[i];
129
+ residualBits(vectors, i * d, centroids, c * d, d, v.bits, j * w);
130
+ const [rn, cd] = residualStats(vectors, i * d, centroids, c * d, d);
131
+ v.rnorm[j] = rn;
132
+ v.cdot[j] = cd;
133
+ v.scales[j] = quantizeInt8(vectors, i * d, d, v.codes, j * d);
134
+ v.floats.set(vectors.subarray(i * d, i * d + d), j * d);
135
+ }
136
+ return { nlist, counts, centroids, lists, floats };
137
+ }
138
+
139
+ /** [|x - c|, c·(x - c)] */
140
+ function residualStats(x: Float32Array, xo: number, c: Float32Array, co: number, d: number) {
141
+ let nn = 0;
142
+ let cd = 0;
143
+ for (let i = 0; i < d; i++) {
144
+ const r = x[xo + i] - c[co + i];
145
+ nn += r * r;
146
+ cd += c[co + i] * r;
147
+ }
148
+ return [Math.sqrt(nn), cd];
149
+ }
150
+
151
+ // ------------------------------------------------------------------ read
152
+
153
+ export interface ListData {
154
+ ids: Float64Array;
155
+ bits: Uint32Array;
156
+ rnorm: Float32Array;
157
+ cdot: Float32Array;
158
+ scales: Float32Array;
159
+ codes: Int8Array;
160
+ }
161
+
162
+ /** Typed views over a searchable block (copied only when it does not start at offset 0). */
163
+ export function parseList(bytes: Uint8Array, count: number, dims: number): ListData {
164
+ const l = listLayout(count, dims);
165
+ if (bytes.byteLength !== l.length) {
166
+ throw new Error(`vector-index: list block has ${bytes.byteLength} bytes, expected ${l.length}`);
167
+ }
168
+ const buf = bytes.byteOffset === 0 ? bytes.buffer : bytes.slice().buffer;
169
+ const w = wordsFor(dims);
170
+ return {
171
+ ids: new Float64Array(buf, l.idsAt, count),
172
+ bits: new Uint32Array(buf, l.bitsAt, count * w),
173
+ rnorm: new Float32Array(buf, l.rnormAt, count),
174
+ cdot: new Float32Array(buf, l.cdotAt, count),
175
+ scales: new Float32Array(buf, l.scalesAt, count),
176
+ codes: new Int8Array(buf, l.codesAt, count * dims),
177
+ };
178
+ }
179
+
180
+ export interface Candidate {
181
+ list: number;
182
+ row: number;
183
+ id: number;
184
+ }
185
+
186
+ /** The in-memory part of a segment: centroids and list sizes. List blocks are fetched by the caller. */
187
+ export class SegmentIndex {
188
+ constructor(
189
+ readonly gen: number,
190
+ readonly dims: number,
191
+ readonly metric: Metric,
192
+ readonly centroids: Float32Array,
193
+ readonly counts: Uint32Array,
194
+ ) {}
195
+
196
+ get nlist() {
197
+ return this.counts.length;
198
+ }
199
+
200
+ /** The `nprobe` non-empty lists whose centroids are closest to the query. */
201
+ probe(query: Float32Array, nprobe: number): number[] {
202
+ const m: Metric = this.metric === "l2" ? "l2" : "dot";
203
+ const top = new TopK<number>(Math.min(nprobe, this.nlist));
204
+ for (let c = 0; c < this.nlist; c++) {
205
+ if (this.counts[c] === 0) continue;
206
+ top.push(distance(m, query, 0, this.centroids, c * this.dims, this.dims), c);
207
+ }
208
+ return top.sorted().map((e) => e.item);
209
+ }
210
+
211
+ /**
212
+ * Stage 1: the `limit` rows of a list with the best estimated distance, from
213
+ * residual sign bits plus the stored residual norm and centroid projection.
214
+ */
215
+ filterList(
216
+ query: Float32Array,
217
+ c: number,
218
+ data: ListData,
219
+ limit: number,
220
+ keep?: (id: number) => boolean,
221
+ ): Candidate[] {
222
+ const d = this.dims;
223
+ const w = wordsFor(d);
224
+ const co = c * d;
225
+ const qb = new Uint32Array(w);
226
+ residualBits(query, 0, this.centroids, co, d, qb, 0);
227
+ const [qn, qcd] = residualStats(query, 0, this.centroids, co, d);
228
+ let cc = 0;
229
+ for (let i = 0; i < d; i++) cc += this.centroids[co + i] * this.centroids[co + i];
230
+ const qc = cc + qcd; // q·c
231
+ const angle = Math.PI / d;
232
+ const l2 = this.metric === "l2";
233
+ const top = new TopK<Candidate>(limit);
234
+ for (let r = 0; r < data.ids.length; r++) {
235
+ const id = data.ids[r];
236
+ if (keep && !keep(id)) continue;
237
+ const rr = qn * data.rnorm[r] * Math.cos(angle * hamming(qb, data.bits, r * w, w));
238
+ // q·x = q·c + c·r_x + r_q·r_x ; |q-x|² = |r_q|² + |r_x|² - 2 r_q·r_x
239
+ const est = l2 ? qn * qn + data.rnorm[r] * data.rnorm[r] - 2 * rr : -(qc + data.cdot[r] + rr);
240
+ if (est < top.bound) top.push(est, { list: c, row: r, id });
241
+ }
242
+ return top.sorted().map((e) => e.item);
243
+ }
244
+
245
+ /** Stage 2: approximate distance from the int8 codes. */
246
+ scoreInt8(query: Float32Array, data: ListData, row: number): number {
247
+ return distanceInt8(
248
+ this.metric,
249
+ query,
250
+ data.codes,
251
+ row * this.dims,
252
+ data.scales[row],
253
+ this.dims,
254
+ );
255
+ }
256
+ }