@hviana/sema 0.1.9 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/example/demo.d.ts +1 -0
  2. package/dist/example/demo.js +39 -0
  3. package/dist/example/train_base.d.ts +87 -0
  4. package/dist/example/train_base.js +2252 -0
  5. package/dist/src/alphabet.d.ts +7 -0
  6. package/dist/src/alphabet.js +33 -0
  7. package/dist/src/alu/src/alu.d.ts +185 -0
  8. package/dist/src/alu/src/alu.js +440 -0
  9. package/dist/src/alu/src/expr.d.ts +61 -0
  10. package/dist/src/alu/src/expr.js +318 -0
  11. package/dist/src/alu/src/index.d.ts +11 -0
  12. package/dist/src/alu/src/index.js +19 -0
  13. package/dist/src/alu/src/kernel-arith.d.ts +16 -0
  14. package/dist/src/alu/src/kernel-arith.js +264 -0
  15. package/dist/src/alu/src/kernel-bits.d.ts +19 -0
  16. package/dist/src/alu/src/kernel-bits.js +152 -0
  17. package/dist/src/alu/src/kernel-logic.d.ts +4 -0
  18. package/dist/src/alu/src/kernel-logic.js +60 -0
  19. package/dist/src/alu/src/kernel-nd.d.ts +3 -0
  20. package/dist/src/alu/src/kernel-nd.js +208 -0
  21. package/dist/src/alu/src/kernel-numeric.d.ts +54 -0
  22. package/dist/src/alu/src/kernel-numeric.js +366 -0
  23. package/dist/src/alu/src/operation.d.ts +168 -0
  24. package/dist/src/alu/src/operation.js +189 -0
  25. package/dist/src/alu/src/parser.d.ts +221 -0
  26. package/dist/src/alu/src/parser.js +577 -0
  27. package/dist/src/alu/src/resonance.d.ts +55 -0
  28. package/dist/src/alu/src/resonance.js +126 -0
  29. package/dist/src/alu/src/text.d.ts +31 -0
  30. package/dist/src/alu/src/text.js +73 -0
  31. package/dist/src/alu/src/value.d.ts +109 -0
  32. package/dist/src/alu/src/value.js +300 -0
  33. package/dist/src/alu/test/alu.test.d.ts +1 -0
  34. package/dist/src/alu/test/alu.test.js +764 -0
  35. package/dist/src/bytes.d.ts +14 -0
  36. package/dist/src/bytes.js +59 -0
  37. package/dist/src/canon.d.ts +26 -0
  38. package/dist/src/canon.js +57 -0
  39. package/dist/src/config.d.ts +111 -0
  40. package/dist/src/config.js +91 -0
  41. package/dist/src/derive/src/deduction.d.ts +125 -0
  42. package/dist/src/derive/src/deduction.js +155 -0
  43. package/dist/src/derive/src/index.d.ts +7 -0
  44. package/dist/src/derive/src/index.js +11 -0
  45. package/dist/src/derive/src/priority-queue.d.ts +20 -0
  46. package/dist/src/derive/src/priority-queue.js +73 -0
  47. package/dist/src/derive/src/rewrite.d.ts +56 -0
  48. package/dist/src/derive/src/rewrite.js +100 -0
  49. package/dist/src/derive/src/trie.d.ts +90 -0
  50. package/dist/src/derive/src/trie.js +217 -0
  51. package/dist/src/derive/test/derive.test.d.ts +1 -0
  52. package/dist/src/derive/test/derive.test.js +122 -0
  53. package/dist/src/extension.d.ts +37 -0
  54. package/dist/src/extension.js +7 -0
  55. package/dist/src/geometry.d.ts +148 -0
  56. package/dist/src/geometry.js +511 -0
  57. package/dist/src/index.d.ts +16 -0
  58. package/dist/src/index.js +18 -0
  59. package/dist/src/ingest-cache.d.ts +41 -0
  60. package/dist/src/ingest-cache.js +161 -0
  61. package/dist/src/mind/articulation.d.ts +6 -0
  62. package/dist/src/mind/articulation.js +99 -0
  63. package/dist/src/mind/attention.d.ts +72 -0
  64. package/dist/src/mind/attention.js +898 -0
  65. package/dist/src/mind/canonical.d.ts +29 -0
  66. package/dist/src/mind/canonical.js +88 -0
  67. package/dist/src/mind/graph-search.d.ts +270 -0
  68. package/dist/src/mind/graph-search.js +927 -0
  69. package/dist/src/mind/index.d.ts +5 -0
  70. package/dist/src/mind/index.js +5 -0
  71. package/dist/src/mind/junction.d.ts +95 -0
  72. package/dist/src/mind/junction.js +262 -0
  73. package/dist/src/mind/learning.d.ts +47 -0
  74. package/dist/src/mind/learning.js +233 -0
  75. package/dist/src/mind/match.d.ts +113 -0
  76. package/dist/src/mind/match.js +446 -0
  77. package/dist/src/mind/mechanisms/alu.d.ts +4 -0
  78. package/dist/src/mind/mechanisms/alu.js +29 -0
  79. package/dist/src/mind/mechanisms/cast.d.ts +35 -0
  80. package/dist/src/mind/mechanisms/cast.js +461 -0
  81. package/dist/src/mind/mechanisms/confluence.d.ts +24 -0
  82. package/dist/src/mind/mechanisms/confluence.js +225 -0
  83. package/dist/src/mind/mechanisms/cover.d.ts +6 -0
  84. package/dist/src/mind/mechanisms/cover.js +179 -0
  85. package/dist/src/mind/mechanisms/extraction.d.ts +67 -0
  86. package/dist/src/mind/mechanisms/extraction.js +342 -0
  87. package/dist/src/mind/mechanisms/recall.d.ts +13 -0
  88. package/dist/src/mind/mechanisms/recall.js +235 -0
  89. package/dist/src/mind/mind.d.ts +273 -0
  90. package/dist/src/mind/mind.js +598 -0
  91. package/dist/src/mind/pipeline-mechanism.d.ts +142 -0
  92. package/dist/src/mind/pipeline-mechanism.js +213 -0
  93. package/dist/src/mind/pipeline.d.ts +20 -0
  94. package/dist/src/mind/pipeline.js +185 -0
  95. package/dist/src/mind/primitives.d.ts +63 -0
  96. package/dist/src/mind/primitives.js +251 -0
  97. package/dist/src/mind/rationale.d.ts +134 -0
  98. package/dist/src/mind/rationale.js +162 -0
  99. package/dist/src/mind/reasoning.d.ts +15 -0
  100. package/dist/src/mind/reasoning.js +162 -0
  101. package/dist/src/mind/recognition.d.ts +20 -0
  102. package/dist/src/mind/recognition.js +246 -0
  103. package/dist/src/mind/resonance.d.ts +23 -0
  104. package/dist/src/mind/resonance.js +0 -0
  105. package/dist/src/mind/trace.d.ts +15 -0
  106. package/dist/src/mind/trace.js +73 -0
  107. package/dist/src/mind/traverse.d.ts +113 -0
  108. package/dist/src/mind/traverse.js +488 -0
  109. package/dist/src/mind/types.d.ts +186 -0
  110. package/dist/src/mind/types.js +84 -0
  111. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  112. package/dist/src/rabitq-ivf/src/database.js +201 -0
  113. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  114. package/dist/src/rabitq-ivf/src/index.js +4 -0
  115. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  116. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  117. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  118. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  119. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  120. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  121. package/dist/src/sema.d.ts +31 -0
  122. package/dist/src/sema.js +63 -0
  123. package/dist/src/store-sqlite.d.ts +184 -0
  124. package/dist/src/store-sqlite.js +942 -0
  125. package/dist/src/store.d.ts +654 -0
  126. package/dist/src/store.js +1612 -0
  127. package/dist/src/vec.d.ts +31 -0
  128. package/dist/src/vec.js +109 -0
  129. package/package.json +1 -1
  130. package/src/mind/graph-search.ts +128 -46
  131. package/src/mind/mind.ts +37 -0
@@ -0,0 +1,84 @@
1
+ // types.ts — all interfaces, types, and free functions for the mind.
2
+ //
3
+ // GraphSearchHost is defined first (minimal imports) so GraphSearch can import
4
+ // it without pulling in the full MindContext.
5
+ import { concatBytes } from "../bytes.js";
6
+ import { dominates } from "../geometry.js";
7
+ // ═══════════════════════════════════════════════════════════════════════════
8
+ // FREE FUNCTIONS (pure, no state)
9
+ // ═══════════════════════════════════════════════════════════════════════════
10
+ /** Read a whole node's bytes. */
11
+ export const ALL = 0x7fffffff;
12
+ /** Splice every chosen span in order — the whole cover as one byte string. */
13
+ export function spliceAll(segs) {
14
+ if (!segs.some((s) => s.rec))
15
+ return null;
16
+ return concatBytes(segs.map((s) => s.bytes));
17
+ }
18
+ /** Lift the answer out of the cover for think: the recognised region, free of
19
+ * the asker's surrounding (unrecognised) framing. */
20
+ export function liftAnswer(segs, queryLen) {
21
+ const recognised = [];
22
+ for (let k = 0; k < segs.length; k++)
23
+ if (segs[k].rec)
24
+ recognised.push(k);
25
+ if (recognised.length === 0)
26
+ return null;
27
+ if (recognised.length === 1) {
28
+ const s = segs[recognised[0]];
29
+ if (dominates(s.j - s.i, queryLen)) {
30
+ return concatBytes(segs.map((x) => x.bytes));
31
+ }
32
+ return s.bytes;
33
+ }
34
+ const lo = recognised[0];
35
+ const hi = recognised[recognised.length - 1];
36
+ return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
37
+ }
38
+ /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
39
+ * tracked deposit interned (`prevSeen`). */
40
+ export function changedNodes(tree, ids, prevSeen) {
41
+ const newCount = new Map();
42
+ const count = (n) => {
43
+ const memo = newCount.get(n);
44
+ if (memo !== undefined)
45
+ return memo;
46
+ const id = ids.get(n);
47
+ // PRUNE: a node whose id the previous deposit already interned is old,
48
+ // and content addressing makes that transitive — the same id names the
49
+ // same content, so every descendant was interned then too. The whole
50
+ // subtree counts 0 without walking it; with the pyramid fold sharing a
51
+ // conversation's prefix subtree, this is what keeps the changed-nodes
52
+ // read O(new nodes) instead of O(context). (A node internTreeIds
53
+ // memo-skipped has an id here exactly when it is such a shared root.)
54
+ if (id !== undefined && prevSeen.has(id)) {
55
+ newCount.set(n, 0);
56
+ return 0;
57
+ }
58
+ let c = 1; // reachable only when NOT pruned above ⇒ this node is new
59
+ if (n.kids) {
60
+ for (const k of n.kids)
61
+ c += count(k);
62
+ }
63
+ newCount.set(n, c);
64
+ return c;
65
+ };
66
+ const total = count(tree);
67
+ if (total === 0)
68
+ return [tree];
69
+ let n = tree;
70
+ for (;;) {
71
+ if (n.kids === null)
72
+ return [n];
73
+ let holder = null;
74
+ for (const k of n.kids) {
75
+ if (newCount.get(k) === total) {
76
+ holder = k;
77
+ break;
78
+ }
79
+ }
80
+ if (holder === null)
81
+ return [n];
82
+ n = holder;
83
+ }
84
+ }
@@ -0,0 +1,113 @@
1
+ /** External ids are integers only. */
2
+ export type ExternalId = number;
3
+ export interface DatabaseOptions {
4
+ /** Path to the SQLite database file (use ":memory:" for a transient store). */
5
+ dbPath: string;
6
+ /** Vector dimensionality. Required for a new database; ignored when reopening. */
7
+ dim?: number;
8
+ /** Query breadth: clusters probed per query = ceil(efSearch / 4). Default
9
+ * 64 (16 probes). Tunable at runtime and at reopen. */
10
+ efSearch?: number;
11
+ /** Bits used to scalar-quantise the query for the RaBitQ estimator.
12
+ * Query-side only (stored codes are independent of it), so it is tunable
13
+ * at reopen, like efSearch. Default 8. */
14
+ queryBits?: number;
15
+ /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
16
+ rotationRounds?: number;
17
+ /** Seed for the rotation. Default fixed. */
18
+ seed?: number;
19
+ /** Optional centroid the vectors are centered by before quantisation. */
20
+ centroid?: ArrayLike<number>;
21
+ /** SQLite page-cache budget in MiB. Purely a latency knob: correctness and
22
+ * the per-operation storage-read count are identical with it at 0. */
23
+ cacheSizeMb?: number;
24
+ }
25
+ export interface QueryResult {
26
+ id: ExternalId;
27
+ /** Estimated cosine distance (1 - cosine). */
28
+ distance: number;
29
+ }
30
+ export interface StorageStats {
31
+ float32BytesPerVector: number;
32
+ codeBytesPerVector: number;
33
+ bytesPerVector: number;
34
+ compressionRatio: number;
35
+ }
36
+ /**
37
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
38
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`. See
39
+ * ivf.ts for the index design. The original float vectors are never
40
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
41
+ *
42
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
43
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
44
+ * walk). Query cost is bounded by nprobe × cluster size — constant in N
45
+ * once the collection is past its first splits.
46
+ */
47
+ export declare class VectorDatabase {
48
+ private readonly index;
49
+ constructor(options: DatabaseOptions);
50
+ get dim(): number;
51
+ /** Number of live (non-deleted) vectors. */
52
+ get size(): number;
53
+ /** Physical slot count including tombstones from deletes/updates. */
54
+ get physicalSize(): number;
55
+ /** Number of clusters (partitions) currently in the index. */
56
+ get clusterCount(): number;
57
+ get efSearch(): number;
58
+ set efSearch(value: number);
59
+ /** Distance computations performed during the most recent query. */
60
+ get lastQueryDistanceComputations(): number;
61
+ /** Storage row reads issued by the most recent query — the honest,
62
+ * cache-independent scalability metric. */
63
+ get lastQueryStorageReads(): number;
64
+ get storage(): StorageStats;
65
+ private checkId;
66
+ /** Convert a value to code bytes, selecting by length: `codeWords`
67
+ * elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
68
+ private toCodeBytes;
69
+ has(id: ExternalId): boolean;
70
+ /** Stream every live external id (bounded memory). */
71
+ keys(): IterableIterator<ExternalId>;
72
+ /** Stream live entries whose INTERNAL id is > `after` — a durable
73
+ * incremental watermark (internal ids are monotone at insert and preserved
74
+ * by update and compact). */
75
+ keysSince(after: number): IterableIterator<{
76
+ ext: ExternalId;
77
+ internal: number;
78
+ }>;
79
+ /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
80
+ get(id: ExternalId): Uint32Array | null;
81
+ insert(id: ExternalId, value: ArrayLike<number>): void;
82
+ update(id: ExternalId, value: ArrayLike<number>): void;
83
+ upsert(id: ExternalId, value: ArrayLike<number>): void;
84
+ /**
85
+ * Upsert many vectors under ONE transaction — one WAL commit for the whole
86
+ * batch instead of one per vector, with chunk appends to the same cluster
87
+ * coalesced in the index's write-back buffer.
88
+ */
89
+ upsertMany(entries: Array<{
90
+ id: ExternalId;
91
+ vector: ArrayLike<number>;
92
+ }>): void;
93
+ /** Delete many ids under ONE transaction. Absent ids are skipped.
94
+ * Returns the number of vectors actually removed. */
95
+ deleteMany(ids: ExternalId[]): number;
96
+ /** Delete the vector bound to an id. Returns false if absent. */
97
+ delete(id: ExternalId): boolean;
98
+ /** Heat the SQLite page cache with sequential scans (latency only). */
99
+ warmCache(): number;
100
+ /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
101
+ * VACUUM. Internal ids and cluster assignment are preserved. */
102
+ compact(): void;
103
+ /**
104
+ * k-NN search. The argument's length selects the mode:
105
+ * - `dim` elements → a raw vector (accurate estimator)
106
+ * - `codeWords` elements → a 1-bit code, by sign-bit Hamming
107
+ */
108
+ query(query: ArrayLike<number>, k?: number, opts?: {
109
+ ef?: number;
110
+ }): QueryResult[];
111
+ /** Close the underlying database. The instance must not be used afterwards. */
112
+ close(): void;
113
+ }
@@ -0,0 +1,201 @@
1
+ import { IvfIndex } from "./ivf.js";
2
+ /**
3
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
4
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`. See
5
+ * ivf.ts for the index design. The original float vectors are never
6
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
7
+ *
8
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
9
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
10
+ * walk). Query cost is bounded by nprobe × cluster size — constant in N
11
+ * once the collection is past its first splits.
12
+ */
13
+ export class VectorDatabase {
14
+ index;
15
+ constructor(options) {
16
+ if (!options || typeof options.dbPath !== "string" ||
17
+ options.dbPath.length === 0) {
18
+ throw new Error("DatabaseOptions.dbPath (string) is required");
19
+ }
20
+ this.index = new IvfIndex(options.dbPath, options);
21
+ }
22
+ get dim() {
23
+ return this.index.dim;
24
+ }
25
+ /** Number of live (non-deleted) vectors. */
26
+ get size() {
27
+ return this.index.size;
28
+ }
29
+ /** Physical slot count including tombstones from deletes/updates. */
30
+ get physicalSize() {
31
+ return this.index.physicalSize;
32
+ }
33
+ /** Number of clusters (partitions) currently in the index. */
34
+ get clusterCount() {
35
+ return this.index.clusterCount;
36
+ }
37
+ get efSearch() {
38
+ return this.index.efSearch;
39
+ }
40
+ set efSearch(value) {
41
+ this.index.efSearch = value;
42
+ }
43
+ /** Distance computations performed during the most recent query. */
44
+ get lastQueryDistanceComputations() {
45
+ return this.index.lastQueryDistComps;
46
+ }
47
+ /** Storage row reads issued by the most recent query — the honest,
48
+ * cache-independent scalability metric. */
49
+ get lastQueryStorageReads() {
50
+ return this.index.lastQueryStorageReads;
51
+ }
52
+ get storage() {
53
+ const f32 = this.dim * 4;
54
+ const bpv = this.index.bytesPerVector;
55
+ return {
56
+ float32BytesPerVector: f32,
57
+ codeBytesPerVector: bpv,
58
+ bytesPerVector: bpv,
59
+ compressionRatio: f32 / bpv,
60
+ };
61
+ }
62
+ checkId(id) {
63
+ // 32-bit signed range: chunk blobs store exts as int32 — a wider id
64
+ // would silently truncate, so it is rejected at the door instead.
65
+ if (!Number.isInteger(id) || id > 0x7fffffff || id < -0x80000000) {
66
+ throw new Error(`External id must be a 32-bit integer, got ${id}`);
67
+ }
68
+ return id;
69
+ }
70
+ /** Convert a value to code bytes, selecting by length: `codeWords`
71
+ * elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
72
+ toCodeBytes(value) {
73
+ if (value.length === this.index.codeWords) {
74
+ return this.index.codeToBytes(value);
75
+ }
76
+ if (value.length !== this.dim) {
77
+ throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
78
+ }
79
+ return this.index.encodeToBytes(value);
80
+ }
81
+ has(id) {
82
+ return this.index.has(this.checkId(id));
83
+ }
84
+ /** Stream every live external id (bounded memory). */
85
+ *keys() {
86
+ yield* this.index.keys();
87
+ }
88
+ /** Stream live entries whose INTERNAL id is > `after` — a durable
89
+ * incremental watermark (internal ids are monotone at insert and preserved
90
+ * by update and compact). */
91
+ *keysSince(after) {
92
+ yield* this.index.keysSince(after);
93
+ }
94
+ /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
95
+ get(id) {
96
+ const bytes = this.index.codeOf(this.checkId(id));
97
+ return bytes ? this.index.bytesToCode(bytes) : null;
98
+ }
99
+ insert(id, value) {
100
+ this.index.begin();
101
+ try {
102
+ this.index.insert(this.checkId(id), this.toCodeBytes(value));
103
+ this.index.commit();
104
+ }
105
+ catch (e) {
106
+ this.index.rollback();
107
+ throw e;
108
+ }
109
+ }
110
+ update(id, value) {
111
+ this.index.begin();
112
+ try {
113
+ this.index.update(this.checkId(id), this.toCodeBytes(value));
114
+ this.index.commit();
115
+ }
116
+ catch (e) {
117
+ this.index.rollback();
118
+ throw e;
119
+ }
120
+ }
121
+ upsert(id, value) {
122
+ this.upsertMany([{ id, vector: value }]);
123
+ }
124
+ /**
125
+ * Upsert many vectors under ONE transaction — one WAL commit for the whole
126
+ * batch instead of one per vector, with chunk appends to the same cluster
127
+ * coalesced in the index's write-back buffer.
128
+ */
129
+ upsertMany(entries) {
130
+ if (entries.length === 0)
131
+ return;
132
+ this.index.begin();
133
+ try {
134
+ for (const e of entries) {
135
+ // One shared vmap probe decides insert vs update — the per-row point
136
+ // query is the dominant bulk-load cost, so it is never duplicated.
137
+ this.index.upsert(this.checkId(e.id), this.toCodeBytes(e.vector));
138
+ }
139
+ this.index.commit();
140
+ }
141
+ catch (e) {
142
+ this.index.rollback();
143
+ throw e;
144
+ }
145
+ }
146
+ /** Delete many ids under ONE transaction. Absent ids are skipped.
147
+ * Returns the number of vectors actually removed. */
148
+ deleteMany(ids) {
149
+ if (ids.length === 0)
150
+ return 0;
151
+ let removed = 0;
152
+ this.index.begin();
153
+ try {
154
+ for (const id of ids) {
155
+ if (this.index.remove(this.checkId(id)))
156
+ removed++;
157
+ }
158
+ this.index.commit();
159
+ }
160
+ catch (e) {
161
+ this.index.rollback();
162
+ throw e;
163
+ }
164
+ return removed;
165
+ }
166
+ /** Delete the vector bound to an id. Returns false if absent. */
167
+ delete(id) {
168
+ return this.deleteMany([id]) === 1;
169
+ }
170
+ /** Heat the SQLite page cache with sequential scans (latency only). */
171
+ warmCache() {
172
+ return this.index.warmCache();
173
+ }
174
+ /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
175
+ * VACUUM. Internal ids and cluster assignment are preserved. */
176
+ compact() {
177
+ this.index.compact();
178
+ }
179
+ /**
180
+ * k-NN search. The argument's length selects the mode:
181
+ * - `dim` elements → a raw vector (accurate estimator)
182
+ * - `codeWords` elements → a 1-bit code, by sign-bit Hamming
183
+ */
184
+ query(query, k = 10, opts) {
185
+ let hits;
186
+ if (query.length === this.index.codeWords) {
187
+ hits = this.index.searchKnnByCode(this.index.codeToBytes(query), k, opts?.ef);
188
+ }
189
+ else {
190
+ if (query.length !== this.dim) {
191
+ throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`);
192
+ }
193
+ hits = this.index.searchKnn(query, k, opts?.ef);
194
+ }
195
+ return hits;
196
+ }
197
+ /** Close the underlying database. The instance must not be used afterwards. */
198
+ close() {
199
+ this.index.close();
200
+ }
201
+ }
@@ -0,0 +1,7 @@
1
+ export { VectorDatabase } from "./database.js";
2
+ export type { DatabaseOptions, ExternalId, QueryResult, StorageStats, } from "./database.js";
3
+ export { IvfIndex } from "./ivf.js";
4
+ export type { IvfConfig, IvfHit } from "./ivf.js";
5
+ export { RaBitQuantizer } from "./rabitq.js";
6
+ export type { QueryContext, RaBitQOptions } from "./rabitq.js";
7
+ export { Prng } from "./prng.js";
@@ -0,0 +1,4 @@
1
+ export { VectorDatabase } from "./database.js";
2
+ export { IvfIndex } from "./ivf.js";
3
+ export { RaBitQuantizer } from "./rabitq.js";
4
+ export { Prng } from "./prng.js";
@@ -0,0 +1,200 @@
1
+ import { DatabaseSync } from "node:sqlite";
2
+ export interface IvfConfig {
3
+ dim: number;
4
+ efSearch: number;
5
+ queryBits: number;
6
+ rotationRounds: number;
7
+ seed: number;
8
+ centroid: Float64Array | null;
9
+ codeWords: number;
10
+ paddedDim: number;
11
+ }
12
+ export interface IvfHit {
13
+ /** external (user) id */
14
+ id: number;
15
+ /** estimated cosine distance (1 - cosine) */
16
+ distance: number;
17
+ }
18
+ export declare class IvfIndex {
19
+ readonly db: DatabaseSync;
20
+ private readonly quantizer;
21
+ private readonly codeBytes;
22
+ private readonly chunkBytes;
23
+ private _efSearch;
24
+ /** Pivot codes, K × codeBytes, contiguous — the insert/query routing scan
25
+ * is one linear pass over this array. */
26
+ private centCodes;
27
+ /** Aligned 32-bit word view of {@link centCodes} — the routing scan's
28
+ * operand (4 bytes per XOR+popcount instead of byte re-composition). */
29
+ private centWords;
30
+ /** Per-cluster entry count (live + dead) — the split trigger. */
31
+ private centEntries;
32
+ /** Per-cluster chunk count (tail seq = count − 1). */
33
+ private centChunks;
34
+ private K;
35
+ private live;
36
+ private totalSlots;
37
+ private nextId;
38
+ /** Storage row reads (chunk blob fetches) — the cache-independent
39
+ * scalability witness, same discipline as the graph index it replaces. */
40
+ reads: number;
41
+ lastQueryDistComps: number;
42
+ lastQueryStorageReads: number;
43
+ private readonly chunkCache;
44
+ private readonly dirtyChunks;
45
+ private readonly dirtyCents;
46
+ private metaDirty;
47
+ private readonly cacheEnabled;
48
+ private readonly chunkCacheMax;
49
+ private sSelChunk;
50
+ private sUpsChunk;
51
+ private sDelChunk;
52
+ private sSelVmap;
53
+ private sInsVmap;
54
+ private sUpdVmap;
55
+ private sDelVmap;
56
+ private sUpsCent;
57
+ private sMeta;
58
+ constructor(dbPath: string, options: {
59
+ dim?: number;
60
+ efSearch?: number;
61
+ queryBits?: number;
62
+ rotationRounds?: number;
63
+ seed?: number;
64
+ centroid?: ArrayLike<number>;
65
+ cacheSizeMb?: number;
66
+ });
67
+ readonly dim: number;
68
+ private prepareAll;
69
+ /** Load the pivot table into RAM — K rows, ~(codeBytes + 8) bytes each. */
70
+ private loadCents;
71
+ private growCents;
72
+ get size(): number;
73
+ get physicalSize(): number;
74
+ get clusterCount(): number;
75
+ get bytesPerVector(): number;
76
+ get efSearch(): number;
77
+ set efSearch(v: number);
78
+ resetReads(): void;
79
+ /** Fetch a chunk, serving dirty/cached copies first. `retain` controls
80
+ * whether a clean fetch enters the cache — read-only consumers pass the
81
+ * cacheEnabled flag so the `cacheSizeMb: 0` mode stays honestly uncached;
82
+ * writers always retain (the copy is about to become dirty). */
83
+ private chunkAt;
84
+ private cacheChunk;
85
+ private markDirty;
86
+ /** Write every dirty chunk row; drop the clean cache only when retention
87
+ * is off (no budget) — under a budget the warm set survives the flush.
88
+ * Runs inside the caller's transaction. */
89
+ private flushChunks;
90
+ private flushCents;
91
+ /** Persist all buffered state. MUST be called before the enclosing
92
+ * transaction commits (upsertMany/deleteMany do; single-op paths too). */
93
+ private flushAll;
94
+ private static readonly SUPER_G;
95
+ private static readonly SUPER_TOP;
96
+ /** Majority super-pivot codes, one per group of SUPER_G clusters. */
97
+ private superWords;
98
+ /** Group ids whose super-pivot is stale (member pivot changed). */
99
+ private readonly superStale;
100
+ /** Scratch word view of the code being routed (avoids per-call copies). */
101
+ private routeScratch;
102
+ private markSuperStale;
103
+ /** Recompute stale super-pivots (majority bit over member pivots). */
104
+ private freshenSupers;
105
+ /** Word-popcount Hamming between the routing scratch and a words-row. */
106
+ private hamAt;
107
+ /** Nearest cluster to a code — exact over member pivots of the SUPER_TOP
108
+ * nearest super-groups (or over everything while K is small). Pure RAM,
109
+ * no allocation, no storage reads. */
110
+ private nearestCid;
111
+ /** The nprobe nearest clusters to a QUERY (accurate estimator on pivots),
112
+ * as cids ordered nearest-first. */
113
+ private probeOrder;
114
+ private probeOrderByCode;
115
+ /** Whether an external id is present (live). */
116
+ has(ext: number): boolean;
117
+ /** The stored code for an ext (a copy), or null. */
118
+ codeOf(ext: number): Uint8Array | null;
119
+ /** Insert a code under `ext`. Caller owns the transaction; buffered rows
120
+ * are flushed by {@link commitFlush}. Throws if ext is already live. */
121
+ insert(ext: number, code: Uint8Array): void;
122
+ /** Insert-or-update with ONE vmap probe. The point probe is a real cost
123
+ * at bulk-load rates (statement dispatch + a B-tree descent per row —
124
+ * profiled as the single largest insert-path term), so the presence check
125
+ * and the update lookup share it. */
126
+ upsert(ext: number, code: Uint8Array): void;
127
+ /** The insert core — presence already established by the caller. */
128
+ private insertNew;
129
+ private newCluster;
130
+ private appendToCluster;
131
+ /** Tombstone a live ext. Returns false when absent. */
132
+ remove(ext: number): boolean;
133
+ /** Update the code bound to a live ext. A byte-identical code is a no-op
134
+ * (content-addressed callers re-upsert unchanged vectors wholesale after a
135
+ * restart; each no-op otherwise costs a tombstone + reinsert). */
136
+ update(ext: number, code: Uint8Array): void;
137
+ private updateAt;
138
+ /** Flush buffered chunk/cent/meta rows. Call before COMMIT. */
139
+ commitFlush(): void;
140
+ /** Split a full cluster in two, deterministically. Dead slots are dropped
141
+ * (splits double as incremental compaction). Both halves get fresh
142
+ * majority-bit pivots, so routing sharpens as the corpus grows. */
143
+ private split;
144
+ /** Bit-majority sample size per split side. A pivot is a routing aid, not
145
+ * a stored value: the majority bit of a 512-member deterministic sample
146
+ * agrees with the full-population majority except on near-tied bits,
147
+ * where either choice routes equally well — and counting every member of
148
+ * a 4096-entry cluster was 25% of bulk-load CPU (profiled). */
149
+ private static readonly MAJORITY_SAMPLE;
150
+ /** Majority-bit code of the side's members (the binary centroid), or null
151
+ * when the side is empty. Members are sampled on a deterministic stride
152
+ * when the side exceeds {@link MAJORITY_SAMPLE}. */
153
+ private majorityCode;
154
+ /** Rewrite a cluster's chunks from a member list (one side of a split or a
155
+ * compaction survivor set), updating vmap rows. `dropFrom` deletes any
156
+ * leftover chunk rows beyond the new count. */
157
+ private rebuildCluster;
158
+ /** Multi-row vmap statements, prepared per row count (powers of a fixed
159
+ * batch width) — REPLACE semantics on the ext primary key. */
160
+ private readonly vmapBatchStmts;
161
+ private static readonly VMAP_BATCH;
162
+ private vmapBatchStmt;
163
+ /** Write flat (ext, id, cid, seq, slot) tuples in wide batches. */
164
+ private flushVmapRows;
165
+ /** ef → clusters probed. ef is the familiar "candidate breadth" knob; a
166
+ * probe scans one whole cluster, so nprobe = ef/4 keeps the default
167
+ * (ef 64 → 16 probes) both accurate on trained stores and bounded. */
168
+ private nprobeOf;
169
+ /** k-NN with a full-precision query (accurate RaBitQ estimator). */
170
+ searchKnn(vec: ArrayLike<number>, k: number, ef?: number): IvfHit[];
171
+ /** k-NN with an already-quantized code (sign-bit Hamming distance). */
172
+ searchKnnByCode(code: Uint8Array, k: number, ef?: number): IvfHit[];
173
+ private scanClusters;
174
+ /** Stream live entries whose internal id is > `after`, in id order.
175
+ * Internal ids are monotone at insert and PRESERVED by update/compact, so
176
+ * the largest id a caller has seen is a durable incremental watermark. */
177
+ keysSince(after: number, batch?: number): IterableIterator<{
178
+ ext: number;
179
+ internal: number;
180
+ }>;
181
+ /** Stream every live external id. */
182
+ keys(): IterableIterator<number>;
183
+ /** Drop tombstoned slots by rewriting each cluster that carries any, then
184
+ * VACUUM. Internal ids and cluster assignment are preserved — routing
185
+ * quality is untouched, only dead space is reclaimed. */
186
+ compact(): void;
187
+ /** Heat SQLite's page cache with sequential scans of the chunk and vmap
188
+ * tables — a cold session otherwise warms it through random point reads.
189
+ * Purely a latency optimisation; returns rows touched. */
190
+ warmCache(): number;
191
+ begin(): void;
192
+ commit(): void;
193
+ rollback(): void;
194
+ /** Encode a raw vector to its 1-bit code bytes. */
195
+ encodeToBytes(vec: ArrayLike<number>): Uint8Array;
196
+ codeToBytes(code: ArrayLike<number>): Uint8Array;
197
+ bytesToCode(bytes: Uint8Array): Uint32Array;
198
+ get codeWords(): number;
199
+ close(): void;
200
+ }