@hviana/sema 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +13 -9
- package/dist/src/sema.js +40 -26
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
|
@@ -1,52 +1,52 @@
|
|
|
1
1
|
/** External ids are integers only. */
|
|
2
2
|
export type ExternalId = number;
|
|
3
3
|
export interface DatabaseOptions {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
4
|
+
/** Path to the SQLite database file (use ":memory:" for a transient store). */
|
|
5
|
+
dbPath: string;
|
|
6
|
+
/** Vector dimensionality. Required for a new database; ignored when reopening. */
|
|
7
|
+
dim?: number;
|
|
8
|
+
/** Max neighbours per node on layers >= 1 (layer 0 uses 2*M). Default 16. */
|
|
9
|
+
M?: number;
|
|
10
|
+
/** Candidate list size during construction. Default 200. */
|
|
11
|
+
efConstruction?: number;
|
|
12
|
+
/** Candidate list size during search. Default 100. Tunable at runtime. */
|
|
13
|
+
efSearch?: number;
|
|
14
|
+
/** Bits used to scalar-quantise the query for the RaBitQ estimator.
|
|
15
|
+
* Query-side only (stored codes and the graph are independent of it), so it
|
|
16
|
+
* is tunable at reopen, like efSearch. Default 8: 4 bits cannot resolve a
|
|
17
|
+
* tight cluster's residual and costs ~18% self-recall there (test 2a); 8
|
|
18
|
+
* bits restores it for the price of a Uint16 per-query LUT. */
|
|
19
|
+
queryBits?: number;
|
|
20
|
+
/** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
|
|
21
|
+
rotationRounds?: number;
|
|
22
|
+
/** Seed for the rotation and graph levels. Default fixed. */
|
|
23
|
+
seed?: number;
|
|
24
|
+
/** Optional centroid the vectors are centered by before quantisation. */
|
|
25
|
+
centroid?: ArrayLike<number>;
|
|
26
|
+
/**
|
|
27
|
+
* RAM cache size in MiB -- the single memory knob. It sizes SQLite's page
|
|
28
|
+
* cache AND the immutable-code LRU (whose entry-capacity is derived from this
|
|
29
|
+
* budget and the code size — there is no second knob). Both are purely speed
|
|
30
|
+
* enhancements: correctness and the per-operation storage-read count are
|
|
31
|
+
* identical with it off. Pass 0 to run with essentially no cache. Default 64.
|
|
32
|
+
* Not persisted; set per open.
|
|
33
|
+
*/
|
|
34
|
+
cacheSizeMb?: number;
|
|
35
35
|
}
|
|
36
36
|
export interface QueryResult {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
id: ExternalId;
|
|
38
|
+
/** Estimated cosine distance (1 - cosine). */
|
|
39
|
+
distance: number;
|
|
40
40
|
}
|
|
41
41
|
export interface StorageStats {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
42
|
+
/** Bytes a single Float32 copy of the vector would take. */
|
|
43
|
+
float32BytesPerVector: number;
|
|
44
|
+
/** Bytes of 1-bit code kept per vector (on disk). */
|
|
45
|
+
codeBytesPerVector: number;
|
|
46
|
+
/** Bytes kept per vector (just the code; excludes the graph adjacency). */
|
|
47
|
+
bytesPerVector: number;
|
|
48
|
+
/** float32BytesPerVector / bytesPerVector. */
|
|
49
|
+
compressionRatio: number;
|
|
50
50
|
}
|
|
51
51
|
/**
|
|
52
52
|
* A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
|
|
@@ -62,139 +62,141 @@ export interface StorageStats {
|
|
|
62
62
|
* `insert`/`update`/`query`.
|
|
63
63
|
*/
|
|
64
64
|
export declare class VectorDatabase {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
65
|
+
readonly dim: number;
|
|
66
|
+
private readonly store;
|
|
67
|
+
private readonly quantizer;
|
|
68
|
+
private readonly index;
|
|
69
|
+
private readonly codeWords;
|
|
70
|
+
constructor(options: DatabaseOptions);
|
|
71
|
+
/** Number of live (non-deleted) vectors. */
|
|
72
|
+
get size(): number;
|
|
73
|
+
/**
|
|
74
|
+
* Physical node count including tombstones from deletes/updates. When it grows
|
|
75
|
+
* well beyond `size`, call `compact()` to reclaim the space on disk.
|
|
76
|
+
*/
|
|
77
|
+
get physicalSize(): number;
|
|
78
|
+
get efSearch(): number;
|
|
79
|
+
set efSearch(value: number);
|
|
80
|
+
/** Distance computations performed during the most recent query. */
|
|
81
|
+
get lastQueryDistanceComputations(): number;
|
|
82
|
+
/**
|
|
83
|
+
* Storage row reads issued by the most recent query. This is the honest,
|
|
84
|
+
* cache-independent scalability metric: it counts every node/neighbour fetch
|
|
85
|
+
* that hit the database, so disabling the cache cannot hide a bad access pattern.
|
|
86
|
+
*/
|
|
87
|
+
get lastQueryStorageReads(): number;
|
|
88
|
+
/** Per-vector storage cost of the index versus a Float32 baseline. */
|
|
89
|
+
get storage(): StorageStats;
|
|
90
|
+
has(id: ExternalId): boolean;
|
|
91
|
+
/** Stream every live external id (bounded memory). */
|
|
92
|
+
keys(): IterableIterator<ExternalId>;
|
|
93
|
+
/**
|
|
94
|
+
* Stream live entries whose INTERNAL id is > `after`, as
|
|
95
|
+
* {ext, internal} pairs in internal-id order. Internal ids are assigned
|
|
96
|
+
* monotonically at insert and preserved by {@link compact}, so the largest
|
|
97
|
+
* internal id a caller has seen is a durable incremental watermark: a later
|
|
98
|
+
* call with it yields exactly the entries added since.
|
|
99
|
+
*/
|
|
100
|
+
keysSince(after: number): IterableIterator<{
|
|
101
|
+
ext: ExternalId;
|
|
102
|
+
internal: number;
|
|
103
|
+
}>;
|
|
104
|
+
private checkId;
|
|
105
|
+
/**
|
|
106
|
+
* Convert a value to code bytes, selecting by length:
|
|
107
|
+
* - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
|
|
108
|
+
* - otherwise -> a raw `dim`-vector, encoded first
|
|
109
|
+
* `codeWords < dim` for any dim >= 2, so the two never collide.
|
|
110
|
+
*/
|
|
111
|
+
private toCodeBytes;
|
|
112
|
+
/**
|
|
113
|
+
* Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
|
|
114
|
+
* detected by length. Throws if the id already exists (use `update`/`upsert`).
|
|
115
|
+
*/
|
|
116
|
+
insert(id: ExternalId, value: ArrayLike<number>): void;
|
|
117
|
+
/** Insert with the caller owning the transaction (used by {@link upsertMany}).
|
|
118
|
+
* `ef` optionally narrows the construction beam for this one vector (a
|
|
119
|
+
* caller-declared cheap entry, e.g. a reach-only interior); omitted means
|
|
120
|
+
* the index's configured efConstruction. */
|
|
121
|
+
private insertCore;
|
|
122
|
+
upsert(id: ExternalId, value: ArrayLike<number>): void;
|
|
123
|
+
/**
|
|
124
|
+
* Upsert many vectors under ONE transaction. The HNSW build touches the store
|
|
125
|
+
* on every wired edge, so a transaction per vector is one WAL commit per vector
|
|
126
|
+
* — which dominates a bulk load on disk. Wrapping the whole batch in a single
|
|
127
|
+
* transaction coalesces those commits into one while leaving the graph and its
|
|
128
|
+
* result identical (reads on the connection still see the uncommitted rows).
|
|
129
|
+
*
|
|
130
|
+
* This is purely about commit batching; it changes nothing about the storage
|
|
131
|
+
* model. Codes still live only in SQLite and are read on demand — the
|
|
132
|
+
* cache-independent per-operation read count is unchanged — so a batched load
|
|
133
|
+
* scales exactly as the per-item path does, just with far fewer fsyncs. A throw
|
|
134
|
+
* rolls the whole batch back, so the caller treats it as the per-item path on
|
|
135
|
+
* failure.
|
|
136
|
+
*/
|
|
137
|
+
upsertMany(
|
|
138
|
+
entries: Array<{
|
|
139
|
+
id: ExternalId;
|
|
140
|
+
vector: ArrayLike<number>;
|
|
141
|
+
ef?: number;
|
|
142
|
+
}>,
|
|
143
|
+
): void;
|
|
144
|
+
/**
|
|
145
|
+
* Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
|
|
146
|
+
* original vector is not retained; the code round-trips into insert/update/query.
|
|
147
|
+
*/
|
|
148
|
+
get(id: ExternalId): Uint32Array | null;
|
|
149
|
+
/**
|
|
150
|
+
* Update the vector bound to an id (raw vector or code, detected by length).
|
|
151
|
+
* The previous node is tombstoned and a fresh one inserted. Throws if absent.
|
|
152
|
+
*/
|
|
153
|
+
update(id: ExternalId, value: ArrayLike<number>): void;
|
|
154
|
+
/** Update with the caller owning the transaction (used by {@link upsertMany}). */
|
|
155
|
+
private updateCore;
|
|
156
|
+
/** Update when the live internal node id is already known.
|
|
157
|
+
*
|
|
158
|
+
* Skips the write entirely when the new code equals the stored one — the
|
|
159
|
+
* update would tombstone the node and replay a full graph insert for a
|
|
160
|
+
* byte-identical result. This is not a corner case: content-addressed
|
|
161
|
+
* callers re-upsert unchanged vectors wholesale after a restart (the same
|
|
162
|
+
* content always encodes to the same code), and each such no-op otherwise
|
|
163
|
+
* costs a tombstone (permanent routing/disk overhead until compaction)
|
|
164
|
+
* plus an O(ef·log N) reinsert. */
|
|
165
|
+
private updateAt;
|
|
166
|
+
/**
|
|
167
|
+
* Delete many ids under ONE transaction — same commit-coalescing rationale
|
|
168
|
+
* as {@link upsertMany}: a tombstone per implicit transaction is one WAL
|
|
169
|
+
* commit per id, which dominates a bulk prune. Absent ids are skipped.
|
|
170
|
+
* Returns the number of vectors actually removed.
|
|
171
|
+
*/
|
|
172
|
+
deleteMany(ids: ExternalId[]): number;
|
|
173
|
+
/** Delete the vector bound to an id. Returns false if absent. */
|
|
174
|
+
delete(id: ExternalId): boolean;
|
|
175
|
+
/**
|
|
176
|
+
* Pre-fill the RAM caches (codes + neighbour lists) with sequential table
|
|
177
|
+
* scans, up to their budget-derived caps. Optional and purely a latency
|
|
178
|
+
* optimisation: a cold session otherwise pays the same warming through
|
|
179
|
+
* random point reads over its first minutes. Call once after open on a
|
|
180
|
+
* session that will do sustained inserts/queries. Returns rows warmed;
|
|
181
|
+
* a 0-budget database returns 0 immediately.
|
|
182
|
+
*/
|
|
183
|
+
warmCache(): number;
|
|
184
|
+
/**
|
|
185
|
+
* Rebuild the graph from the live codes only, dropping tombstones and returning
|
|
186
|
+
* the freed pages to the filesystem. Lossless -- equivalent to the original
|
|
187
|
+
* build -- and needs no original vectors.
|
|
188
|
+
*/
|
|
189
|
+
compact(): void;
|
|
190
|
+
/**
|
|
191
|
+
* k-NN search. The argument's length selects the mode:
|
|
192
|
+
* - `dim` elements -> a raw vector (accurate 4-bit-query estimator)
|
|
193
|
+
* - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
|
|
194
|
+
* Detection is by length, so a code is recognised whether it is a Uint32Array
|
|
195
|
+
* or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
|
|
196
|
+
*/
|
|
197
|
+
query(query: ArrayLike<number>, k?: number, opts?: {
|
|
198
|
+
ef?: number;
|
|
199
|
+
}): QueryResult[];
|
|
200
|
+
/** Close the underlying database. The instance must not be used afterwards. */
|
|
201
|
+
close(): void;
|
|
200
202
|
}
|