@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
package/dist/src/store-sqlite.js
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
// vectors) is an independent file. The vector databases persist to their
|
|
28
28
|
// own files on every write — no separate serialisation step is needed.
|
|
29
29
|
import { DatabaseSync } from "node:sqlite";
|
|
30
|
-
import { AbstractStore, flatBytesKids, packKids, unpackKids
|
|
30
|
+
import { AbstractStore, flatBytesKids, packKids, unpackKids } from "./store.js";
|
|
31
31
|
import { DEFAULT_CONFIG } from "./config.js";
|
|
32
32
|
import { VectorDatabase } from "./rabitq-hnsw/src/index.js";
|
|
33
33
|
const SCHEMA = `
|
|
@@ -107,249 +107,276 @@ CREATE TABLE IF NOT EXISTS meta (
|
|
|
107
107
|
);
|
|
108
108
|
`;
|
|
109
109
|
export class SQliteStore extends AbstractStore {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
110
|
+
opts;
|
|
111
|
+
vectorCacheMb;
|
|
112
|
+
vectorSeed;
|
|
113
|
+
// ── the two SEPARATE vector indices ───────────────────────────────────
|
|
114
|
+
content = null; // STORAGE: node gists
|
|
115
|
+
halos = null; // CACHE: halo gists
|
|
116
|
+
sqlite = null;
|
|
117
|
+
// Deferred write transaction guard.
|
|
118
|
+
_inTx = false;
|
|
119
|
+
_insertNode = null;
|
|
120
|
+
_insertKid = null;
|
|
121
|
+
_selContain = null;
|
|
122
|
+
_selParents = null;
|
|
123
|
+
_selHalo = null;
|
|
124
|
+
_upHalo = null;
|
|
125
|
+
_selLeaf = null;
|
|
126
|
+
_selFlat = null;
|
|
127
|
+
_selKids = null;
|
|
128
|
+
_selNode = null;
|
|
129
|
+
_insertEdge = null;
|
|
130
|
+
_selNext = null;
|
|
131
|
+
_selPrev = null;
|
|
132
|
+
_setMeta = null;
|
|
133
|
+
_getMeta = null;
|
|
134
|
+
_delMeta = null;
|
|
135
|
+
_insSnapshot = null;
|
|
136
|
+
_selSnapshot = null;
|
|
137
|
+
constructor(opts = {}) {
|
|
138
|
+
const d = DEFAULT_CONFIG.store;
|
|
139
|
+
// Resolve config from opts, falling back to defaults.
|
|
140
|
+
const config = {
|
|
141
|
+
minHaloMass: opts.minHaloMass ?? d.minHaloMass,
|
|
142
|
+
m: opts.m ?? d.m,
|
|
143
|
+
efConstruction: opts.efConstruction ?? d.efConstruction,
|
|
144
|
+
efConstructionInterior: opts.efConstructionInterior ??
|
|
145
|
+
d.efConstructionInterior,
|
|
146
|
+
efSearch: opts.efSearch ?? d.efSearch,
|
|
147
|
+
compactEveryNWrites: opts.compactEveryNWrites ?? d.compactEveryNWrites,
|
|
148
|
+
overfetch: opts.overfetch ?? d.overfetch,
|
|
149
|
+
batchSize: opts.batchSize ?? d.batchSize,
|
|
150
|
+
dedupCacheMax: opts.dedupCacheMax ?? d.dedupCacheMax,
|
|
151
|
+
bytesCacheMax: opts.bytesCacheMax ?? d.bytesCacheMax,
|
|
152
|
+
recCacheBytes: opts.recCacheBytes ?? d.recCacheBytes,
|
|
153
|
+
ingestCacheBytes: opts.ingestCacheBytes ?? d.ingestCacheBytes,
|
|
154
|
+
pendingGistBytes: opts.pendingGistBytes ?? d.pendingGistBytes,
|
|
155
|
+
haloCacheBytes: opts.haloCacheBytes ?? d.haloCacheBytes,
|
|
156
|
+
vectorCacheMb: opts.vectorCacheMb ?? d.vectorCacheMb,
|
|
157
|
+
coveredIdsMax: opts.coveredIdsMax ?? d.coveredIdsMax,
|
|
158
|
+
chainCacheBytes: opts.chainCacheBytes ?? d.chainCacheBytes,
|
|
159
|
+
};
|
|
160
|
+
const D = opts.D ?? 1024;
|
|
161
|
+
const maxGroup = opts.maxGroup ?? DEFAULT_CONFIG.geometry.maxGroup;
|
|
162
|
+
super(config, D, maxGroup);
|
|
163
|
+
this.opts = opts;
|
|
164
|
+
this.vectorCacheMb = opts.vectorCacheMb ?? d.vectorCacheMb;
|
|
165
|
+
this.vectorSeed = (0x51f15e ^ 0x9e3779b9) >>> 0;
|
|
166
|
+
this._ready = this._dbOpen();
|
|
167
|
+
}
|
|
168
|
+
// ── Vector-DB paths ───────────────────────────────────────────────────
|
|
169
|
+
vectorDbPath(name) {
|
|
170
|
+
const stem = this.opts.path ?? ":memory:";
|
|
171
|
+
if (stem === ":memory:") {
|
|
172
|
+
return ":memory:";
|
|
173
|
+
}
|
|
174
|
+
return `${stem}.${name}.vec`;
|
|
175
|
+
}
|
|
176
|
+
openVectorDB(name) {
|
|
177
|
+
return new VectorDatabase({
|
|
178
|
+
dbPath: this.vectorDbPath(name),
|
|
179
|
+
dim: this.D,
|
|
180
|
+
M: this.m,
|
|
181
|
+
efConstruction: this.efConstruction,
|
|
182
|
+
efSearch: this.efSearch,
|
|
183
|
+
// Query-side estimator precision. 8 bits: 4 bits measurably misranks
|
|
184
|
+
// tight gist clusters (mixture recall@10 37.5% vs 39.0%, self-recall
|
|
185
|
+
// 69% vs 76%; rabitq test 2a), and the codes on disk are independent of
|
|
186
|
+
// it, so existing stores adopt the sharper setting on reopen. The one
|
|
187
|
+
// decision the sharper ranking exposed — a saturated hub winning a
|
|
188
|
+
// sub-noise rank tie and silencing its region — is handled by the
|
|
189
|
+
// tie-band saturation fallback in attention.ts, which is derived from
|
|
190
|
+
// the estimator's own margin noise rather than tuned to a bit width.
|
|
191
|
+
queryBits: this.opts.queryBits ?? 8,
|
|
192
|
+
seed: this.vectorSeed,
|
|
193
|
+
cacheSizeMb: this.vectorCacheMb,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
// ── Abstract method implementations ───────────────────────────────────
|
|
197
|
+
// -- Lifecycle --
|
|
198
|
+
async _dbOpen() {
|
|
199
|
+
const stem = this.opts.path ?? ":memory:";
|
|
200
|
+
const sp = stem === ":memory:" ? ":memory:" : `${stem}.sqlite`;
|
|
201
|
+
this.sqlite = new DatabaseSync(sp);
|
|
202
|
+
// Page size matched to the store's row grain BEFORE any table exists (a
|
|
203
|
+
// no-op on a non-empty database). Every table here holds small rows —
|
|
204
|
+
// packed kid/contain pairs, byte-packed flat branches, quantized halos —
|
|
205
|
+
// so SQLite's default 4 KiB pages carry mostly slack; 1 KiB pages keep
|
|
206
|
+
// leaf-page fill high at a negligible extra tree depth.
|
|
207
|
+
this.sqlite.exec("PRAGMA page_size = 1024;");
|
|
208
|
+
// WAL + NORMAL sync: crash-safe (WAL commits are atomic) without a full
|
|
209
|
+
// fsync per commit — the same discipline the vector databases use.
|
|
210
|
+
this.sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;");
|
|
211
|
+
this.sqlite.exec(SCHEMA);
|
|
212
|
+
// Recover D from a previous run so loadFromStore() can bootstrap with any
|
|
213
|
+
// default — the real value is read from meta before vector indices load.
|
|
214
|
+
// The meta table always exists here (SCHEMA above creates it), so these
|
|
215
|
+
// reads run bare: a throw is a genuine storage fault, and swallowing it
|
|
216
|
+
// would let the store proceed with a default D/maxGroup that silently
|
|
217
|
+
// disagrees with what training persisted.
|
|
218
|
+
{
|
|
219
|
+
const row = this.sqlite.prepare(
|
|
220
|
+
"SELECT val FROM meta WHERE key = 'train.D'",
|
|
221
|
+
).get();
|
|
222
|
+
if (row) {
|
|
223
|
+
const d = Number(row.val);
|
|
224
|
+
if (Number.isInteger(d) && d > 0) {
|
|
225
|
+
this._D = d;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
// Recover maxGroup from meta so indexSubtree uses the training-time value.
|
|
230
|
+
{
|
|
231
|
+
const row = this.sqlite.prepare(
|
|
232
|
+
"SELECT val FROM meta WHERE key = 'geometry.maxGroup'",
|
|
233
|
+
).get();
|
|
234
|
+
if (row) {
|
|
235
|
+
const g = Number(row.val);
|
|
236
|
+
if (Number.isInteger(g) && g > 0) {
|
|
237
|
+
this._maxGroup = g;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
// Persist maxGroup to meta when opening a FRESH store (no rows yet) so
|
|
242
|
+
// indexSubtree always sees the training-time value even when the store is
|
|
243
|
+
// accessed without a Mind / full snapshot.
|
|
244
|
+
if (this._nextId === 0) {
|
|
245
|
+
this.sqlite.prepare(
|
|
246
|
+
"INSERT OR IGNORE INTO meta (key, val) VALUES ('geometry.maxGroup', ?)",
|
|
247
|
+
).run(String(this._maxGroup));
|
|
248
|
+
}
|
|
249
|
+
this.content = this.openVectorDB("content");
|
|
250
|
+
this.halos = this.openVectorDB("halo");
|
|
251
|
+
// Ids are dense (0,1,2,…) and never deleted, so MAX(id)+1 IS the next id
|
|
252
|
+
// — one rightmost B-tree descent on the rowid, O(log n). (COUNT(*) gives
|
|
253
|
+
// the same value but scans every leaf page, so open time grew linearly
|
|
254
|
+
// with the store; a per-row scan into a Set was worse still.)
|
|
255
|
+
this._nextId =
|
|
256
|
+
(this.sqlite.prepare("SELECT MAX(id) AS m FROM node").get().m ?? -1) + 1;
|
|
257
|
+
}
|
|
258
|
+
_dbClose() {
|
|
259
|
+
if (this.content) {
|
|
260
|
+
this.content.close();
|
|
261
|
+
this.content = null;
|
|
262
|
+
}
|
|
263
|
+
if (this.halos) {
|
|
264
|
+
this.halos.close();
|
|
265
|
+
this.halos = null;
|
|
266
|
+
}
|
|
267
|
+
if (this.sqlite) {
|
|
268
|
+
this.sqlite.close();
|
|
269
|
+
this.sqlite = null;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
// -- Transaction --
|
|
273
|
+
_dbBeginTx() {
|
|
274
|
+
if (this._inTx || !this.sqlite) {
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
this.sqlite.exec("BEGIN");
|
|
278
|
+
this._inTx = true;
|
|
279
|
+
}
|
|
280
|
+
_dbCommitTx() {
|
|
281
|
+
if (!this._inTx || !this.sqlite) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
this._inTx = false; // clear first so a throw can't wedge us mid-commit
|
|
285
|
+
this.sqlite.exec("COMMIT");
|
|
286
|
+
}
|
|
287
|
+
// -- Node CRUD --
|
|
288
|
+
_dbInsertNode(id, leaf, kids, h) {
|
|
289
|
+
if (!this._insertNode) {
|
|
290
|
+
this._insertNode = this.sqlite.prepare(
|
|
291
|
+
"INSERT INTO node (id, leaf, kids, h) VALUES (?, ?, ?, ?)",
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
this._insertNode.run(id, leaf, kids, h);
|
|
295
|
+
}
|
|
296
|
+
_dbGetNode(id) {
|
|
297
|
+
if (!this._selNode) {
|
|
298
|
+
this._selNode = this.sqlite.prepare(
|
|
299
|
+
"SELECT id, leaf, kids FROM node WHERE id = ?",
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
const r = this._selNode.get(id);
|
|
303
|
+
if (!r) {
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
// A zero-length kids blob marks a FLAT branch: the leaf column holds its
|
|
307
|
+
// bytes and the kid list is derived, one implicit leaf per byte.
|
|
308
|
+
const flat = r.kids !== null && r.kids.byteLength === 0;
|
|
309
|
+
return {
|
|
310
|
+
id: r.id,
|
|
311
|
+
leaf: r.leaf && !flat ? new Uint8Array(r.leaf) : null,
|
|
312
|
+
kids: flat ? flatBytesKids(r.leaf) : (r.kids ? unpackKids(r.kids) : null),
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
_dbFindLeaf(h, bytes) {
|
|
316
|
+
if (!this._selLeaf) {
|
|
317
|
+
this._selLeaf = this.sqlite.prepare(
|
|
318
|
+
"SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NULL LIMIT 1",
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
const row = this._selLeaf.get(h, bytes);
|
|
322
|
+
return row ? row.id : null;
|
|
323
|
+
}
|
|
324
|
+
_dbFindBranchByLeaf(h, bytes) {
|
|
325
|
+
if (!this._selFlat) {
|
|
326
|
+
this._selFlat = this.sqlite.prepare(
|
|
327
|
+
"SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NOT NULL LIMIT 1",
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
const row = this._selFlat.get(h, bytes);
|
|
331
|
+
return row ? row.id : null;
|
|
332
|
+
}
|
|
333
|
+
_dbFindBranchByKids(h, packed) {
|
|
334
|
+
if (!this._selKids) {
|
|
335
|
+
this._selKids = this.sqlite.prepare(
|
|
336
|
+
"SELECT id FROM node WHERE h = ? AND kids = ? LIMIT 1",
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
const row = this._selKids.get(h, packed);
|
|
340
|
+
return row ? row.id : null;
|
|
341
|
+
}
|
|
342
|
+
// -- Kid (structural parent) edges --
|
|
343
|
+
_dbInsertKid(child, parent) {
|
|
344
|
+
if (!this._insertKid) {
|
|
345
|
+
this._insertKid = this.sqlite.prepare(
|
|
346
|
+
"INSERT OR IGNORE INTO kid (child, parent) VALUES (?, ?)",
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
this._insertKid.run(child, parent);
|
|
350
|
+
}
|
|
351
|
+
_dbGetParents(id) {
|
|
352
|
+
if (!this._selParents) {
|
|
353
|
+
this._selParents = this.sqlite.prepare(
|
|
354
|
+
"SELECT parent FROM kid WHERE child = ?",
|
|
355
|
+
);
|
|
356
|
+
}
|
|
357
|
+
return this._selParents.all(id).map((r) => r.parent);
|
|
358
|
+
}
|
|
359
|
+
// -- Containment --
|
|
360
|
+
_selParentsFirst = null;
|
|
361
|
+
_dbGetParentsFirst(id, limit) {
|
|
362
|
+
if (!this._selParentsFirst) {
|
|
363
|
+
this._selParentsFirst = this.sqlite.prepare(
|
|
364
|
+
"SELECT parent FROM kid WHERE child = ? LIMIT ?",
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
return this._selParentsFirst.all(id, limit)
|
|
368
|
+
.map((r) => r.parent);
|
|
369
|
+
}
|
|
370
|
+
_selChain = null;
|
|
371
|
+
/** {@link Store.chainRun}'s walk as ONE recursive CTE: the whole
|
|
372
|
+
* transparent chain (no edge in or out, exactly one parent) is descended
|
|
373
|
+
* inside SQLite — per-node work is the same three indexed probes as the
|
|
374
|
+
* base class's loop, but without a JS↔SQLite round trip per node, which
|
|
375
|
+
* dominates on the deep single-structure scaffolding this read exists
|
|
376
|
+
* for. */
|
|
377
|
+
_chainWalk(id, cap) {
|
|
378
|
+
if (!this._selChain) {
|
|
379
|
+
this._selChain = this.sqlite.prepare(`WITH RECURSIVE chain(n, d) AS (
|
|
353
380
|
SELECT ?, 0
|
|
354
381
|
UNION ALL
|
|
355
382
|
SELECT (SELECT parent FROM kid WHERE child = chain.n LIMIT 1),
|
|
@@ -363,340 +390,395 @@ export class SQliteStore extends AbstractStore {
|
|
|
363
390
|
) = 1
|
|
364
391
|
)
|
|
365
392
|
SELECT n FROM chain ORDER BY d`);
|
|
366
|
-
}
|
|
367
|
-
return this._selChain.all(id, cap - 1)
|
|
368
|
-
.map((r) => r.n);
|
|
369
|
-
}
|
|
370
|
-
/** Width of the seq band inside the packed contain rowid: rowid =
|
|
371
|
-
* child·SEQ_SPAN + seq. Page sizes are geometric, so a chain of k live
|
|
372
|
-
* pages needs a base of ≥ 2^k · 4 bytes — seq stays ≤ ~50 for any
|
|
373
|
-
* physically possible list; the append path guards the band anyway. */
|
|
374
|
-
static SEQ_SPAN = 1 << 8;
|
|
375
|
-
_selContainExists = null;
|
|
376
|
-
_dbContainExists(child) {
|
|
377
|
-
if (!this._selContainExists) {
|
|
378
|
-
// A probe of the child's rowid range — one descent.
|
|
379
|
-
this._selContainExists = this.sqlite.prepare("SELECT 1 AS one FROM contain WHERE id >= ? AND id < ? LIMIT 1");
|
|
380
|
-
}
|
|
381
|
-
const base = child * SQliteStore.SEQ_SPAN;
|
|
382
|
-
return this._selContainExists.get(base, base + SQliteStore.SEQ_SPAN) !==
|
|
383
|
-
undefined;
|
|
384
|
-
}
|
|
385
|
-
_selContainPages = null;
|
|
386
|
-
/** The child's page directory — (seq, byte length) per page, O(log fan-in)
|
|
387
|
-
* rows, each read from the cell header without loading the blob. */
|
|
388
|
-
containPages(child) {
|
|
389
|
-
if (!this._selContainPages) {
|
|
390
|
-
this._selContainPages = this.sqlite.prepare("SELECT id, length(parents) AS len FROM contain " +
|
|
391
|
-
"WHERE id >= ? AND id < ? ORDER BY id");
|
|
392
|
-
}
|
|
393
|
-
const base = child * SQliteStore.SEQ_SPAN;
|
|
394
|
-
return this._selContainPages.all(base, base + SQliteStore.SEQ_SPAN)
|
|
395
|
-
.map((r) => ({ seq: r.id - base, len: r.len }));
|
|
396
|
-
}
|
|
397
|
-
_selContainPageSub = null;
|
|
398
|
-
_selContainPage = null;
|
|
399
|
-
containPage(child, seq) {
|
|
400
|
-
if (!this._selContainPage) {
|
|
401
|
-
this._selContainPage = this.sqlite.prepare("SELECT parents FROM contain WHERE id = ?");
|
|
402
|
-
}
|
|
403
|
-
return this._selContainPage.get(child * SQliteStore.SEQ_SPAN + seq).parents;
|
|
404
393
|
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
394
|
+
return this._selChain.all(id, cap - 1)
|
|
395
|
+
.map((r) => r.n);
|
|
396
|
+
}
|
|
397
|
+
/** Width of the seq band inside the packed contain rowid: rowid =
|
|
398
|
+
* child·SEQ_SPAN + seq. Page sizes are geometric, so a chain of k live
|
|
399
|
+
* pages needs a base of ≥ 2^k · 4 bytes — seq stays ≤ ~50 for any
|
|
400
|
+
* physically possible list; the append path guards the band anyway. */
|
|
401
|
+
static SEQ_SPAN = 1 << 8;
|
|
402
|
+
_selContainExists = null;
|
|
403
|
+
_dbContainExists(child) {
|
|
404
|
+
if (!this._selContainExists) {
|
|
405
|
+
// A probe of the child's rowid range — one descent.
|
|
406
|
+
this._selContainExists = this.sqlite.prepare(
|
|
407
|
+
"SELECT 1 AS one FROM contain WHERE id >= ? AND id < ? LIMIT 1",
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
const base = child * SQliteStore.SEQ_SPAN;
|
|
411
|
+
return this._selContainExists.get(base, base + SQliteStore.SEQ_SPAN) !==
|
|
412
|
+
undefined;
|
|
413
|
+
}
|
|
414
|
+
_selContainPages = null;
|
|
415
|
+
/** The child's page directory — (seq, byte length) per page, O(log fan-in)
|
|
416
|
+
* rows, each read from the cell header without loading the blob. */
|
|
417
|
+
containPages(child) {
|
|
418
|
+
if (!this._selContainPages) {
|
|
419
|
+
this._selContainPages = this.sqlite.prepare(
|
|
420
|
+
"SELECT id, length(parents) AS len FROM contain " +
|
|
421
|
+
"WHERE id >= ? AND id < ? ORDER BY id",
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
const base = child * SQliteStore.SEQ_SPAN;
|
|
425
|
+
return this._selContainPages.all(base, base + SQliteStore.SEQ_SPAN)
|
|
426
|
+
.map((r) => ({ seq: r.id - base, len: r.len }));
|
|
427
|
+
}
|
|
428
|
+
_selContainPageSub = null;
|
|
429
|
+
_selContainPage = null;
|
|
430
|
+
containPage(child, seq) {
|
|
431
|
+
if (!this._selContainPage) {
|
|
432
|
+
this._selContainPage = this.sqlite.prepare(
|
|
433
|
+
"SELECT parents FROM contain WHERE id = ?",
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
return this._selContainPage.get(child * SQliteStore.SEQ_SPAN + seq).parents;
|
|
437
|
+
}
|
|
438
|
+
_dbGetContainParentsSlice(child, offset, limit) {
|
|
439
|
+
if (!this._selContainPageSub) {
|
|
440
|
+
// substr on a BLOB returns bytes: unpack only the requested span.
|
|
441
|
+
this._selContainPageSub = this.sqlite.prepare(
|
|
442
|
+
"SELECT substr(parents, 1 + ? * 4, ? * 4) AS page " +
|
|
443
|
+
"FROM contain WHERE id = ?",
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
const out = [];
|
|
447
|
+
let skip = offset;
|
|
448
|
+
for (const seg of this.containPages(child)) {
|
|
449
|
+
const n = seg.len >>> 2;
|
|
450
|
+
if (skip >= n) {
|
|
451
|
+
skip -= n;
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
const take = Math.min(limit - out.length, n - skip);
|
|
455
|
+
const row = this._selContainPageSub.get(
|
|
456
|
+
skip,
|
|
457
|
+
take,
|
|
458
|
+
child * SQliteStore.SEQ_SPAN + seg.seq,
|
|
459
|
+
);
|
|
460
|
+
if (row && row.page.length > 0) {
|
|
461
|
+
out.push(...unpackKids(row.page));
|
|
462
|
+
}
|
|
463
|
+
skip = 0;
|
|
464
|
+
if (out.length >= limit) {
|
|
465
|
+
break;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
return out;
|
|
469
|
+
}
|
|
470
|
+
_selContainCount = null;
|
|
471
|
+
_dbGetContainCount(child) {
|
|
472
|
+
if (!this._selContainCount) {
|
|
473
|
+
// Stored ENTRY count (transient duplicates included) — consistent with
|
|
474
|
+
// what the slice streams, which is all the seam math needs.
|
|
475
|
+
this._selContainCount = this.sqlite.prepare(
|
|
476
|
+
"SELECT COALESCE(SUM(length(parents)), 0) / 4 AS n " +
|
|
477
|
+
"FROM contain WHERE id >= ? AND id < ?",
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
const base = child * SQliteStore.SEQ_SPAN;
|
|
481
|
+
return this._selContainCount.get(base, base + SQliteStore.SEQ_SPAN).n;
|
|
482
|
+
}
|
|
483
|
+
_dbGetContainParents(child) {
|
|
484
|
+
if (!this._selContain) {
|
|
485
|
+
this._selContain = this.sqlite.prepare(
|
|
486
|
+
"SELECT parents FROM contain WHERE id >= ? AND id < ? ORDER BY id",
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
const base = child * SQliteStore.SEQ_SPAN;
|
|
490
|
+
const rows = this._selContain.all(base, base + SQliteStore.SEQ_SPAN);
|
|
491
|
+
if (rows.length === 0) {
|
|
492
|
+
return [];
|
|
493
|
+
}
|
|
494
|
+
// Dedup across pages (a pair re-added after its page merged into the base
|
|
495
|
+
// may repeat in the tail until the next base merge), preserving order.
|
|
496
|
+
const seen = new Set();
|
|
497
|
+
const out = [];
|
|
498
|
+
for (const r of rows) {
|
|
499
|
+
for (const p of unpackKids(r.parents)) {
|
|
500
|
+
if (!seen.has(p)) {
|
|
501
|
+
seen.add(p);
|
|
502
|
+
out.push(p);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return out;
|
|
507
|
+
}
|
|
508
|
+
_upsContainPage = null;
|
|
509
|
+
_delContainPage = null;
|
|
510
|
+
_dbAppendContain(child, parents) {
|
|
511
|
+
if (parents.length === 0) {
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
if (!this._upsContainPage) {
|
|
515
|
+
this._upsContainPage = this.sqlite.prepare(
|
|
516
|
+
"INSERT INTO contain (id, parents) VALUES (?, ?) " +
|
|
517
|
+
"ON CONFLICT(id) DO UPDATE SET parents = excluded.parents",
|
|
518
|
+
);
|
|
519
|
+
this._delContainPage = this.sqlite.prepare(
|
|
520
|
+
"DELETE FROM contain WHERE id = ?",
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
// Fold stored tail pages into the new page IN MEMORY, then write ONCE.
|
|
524
|
+
// Merge rule per fold: below a 256-byte floor always merge (most children
|
|
525
|
+
// have a handful of parents — two tiny rows would double their per-row
|
|
526
|
+
// overhead for no amortization benefit); above it, merge only while the
|
|
527
|
+
// stored page is ≤ 2× the accumulated one, which bounds total re-writing
|
|
528
|
+
// to O(fan-in · log fan-in) bytes and keeps the page directory
|
|
529
|
+
// logarithmic. A fold that reaches the seq-0 base page dedups, squeezing
|
|
530
|
+
// out duplicates on the same geometric schedule. Writing the fold's
|
|
531
|
+
// result as one upsert (an in-place row replace in the common
|
|
532
|
+
// small-child case) rather than delete+insert churn keeps B-tree leaf
|
|
533
|
+
// fill at the packed single-row format's level — measured: churn alone
|
|
534
|
+
// cost ~26% extra pages on the same payload.
|
|
535
|
+
const segs = this.containPages(child);
|
|
536
|
+
let cur = packKids(parents);
|
|
537
|
+
let seq = segs.length > 0 ? segs[segs.length - 1].seq + 1 : 0;
|
|
538
|
+
if (seq >= SQliteStore.SEQ_SPAN) {
|
|
539
|
+
throw new Error(`contain page seq overflow for child ${child}`);
|
|
540
|
+
}
|
|
541
|
+
const folded = [];
|
|
542
|
+
while (segs.length > 0) {
|
|
543
|
+
const a = segs[segs.length - 1];
|
|
544
|
+
if (a.len + cur.byteLength > 256 && a.len > 2 * cur.byteLength) {
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
const pa = this.containPage(child, a.seq);
|
|
548
|
+
if (a.seq === 0) {
|
|
450
549
|
const seen = new Set();
|
|
451
|
-
const
|
|
452
|
-
for (const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
return this.halos ? this.halos.size : 0;
|
|
686
|
-
}
|
|
687
|
-
_vecHaloPhysicalSize() {
|
|
688
|
-
return this.halos ? this.halos.physicalSize : 0;
|
|
689
|
-
}
|
|
690
|
-
_vecHaloCompact() {
|
|
691
|
-
this.halos.compact();
|
|
692
|
-
}
|
|
693
|
-
/** Pre-fill both vector indices' RAM caches with sequential scans (up to
|
|
694
|
-
* their budget caps) — seconds of streaming instead of the minutes of
|
|
695
|
-
* random point reads a cold training/query session otherwise pays while
|
|
696
|
-
* warming. Optional; call once after open before sustained work.
|
|
697
|
-
* Returns rows warmed. */
|
|
698
|
-
async warmVectorCaches() {
|
|
699
|
-
await this._ensureReady();
|
|
700
|
-
return (this.content?.warmCache() ?? 0) + (this.halos?.warmCache() ?? 0);
|
|
701
|
-
}
|
|
550
|
+
const ids = [];
|
|
551
|
+
for (const p of unpackKids(pa)) {
|
|
552
|
+
if (!seen.has(p)) {
|
|
553
|
+
seen.add(p);
|
|
554
|
+
ids.push(p);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
for (const p of unpackKids(cur)) {
|
|
558
|
+
if (!seen.has(p)) {
|
|
559
|
+
seen.add(p);
|
|
560
|
+
ids.push(p);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
cur = packKids(ids);
|
|
564
|
+
} else {
|
|
565
|
+
const m = new Uint8Array(pa.byteLength + cur.byteLength);
|
|
566
|
+
m.set(pa, 0);
|
|
567
|
+
m.set(cur, pa.byteLength);
|
|
568
|
+
cur = m;
|
|
569
|
+
}
|
|
570
|
+
folded.push(a.seq);
|
|
571
|
+
seq = a.seq;
|
|
572
|
+
segs.pop();
|
|
573
|
+
}
|
|
574
|
+
const base = child * SQliteStore.SEQ_SPAN;
|
|
575
|
+
for (const s of folded) {
|
|
576
|
+
if (s !== seq) {
|
|
577
|
+
this._delContainPage.run(base + s);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
this._upsContainPage.run(base + seq, cur);
|
|
581
|
+
}
|
|
582
|
+
// -- Continuation edges --
|
|
583
|
+
_dbInsertEdge(src, dst) {
|
|
584
|
+
if (!this._insertEdge) {
|
|
585
|
+
this._insertEdge = this.sqlite.prepare(
|
|
586
|
+
"INSERT OR IGNORE INTO edge (src, dst) VALUES (?, ?)",
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
this._insertEdge.run(src, dst);
|
|
590
|
+
}
|
|
591
|
+
_dbGetNextEdges(id) {
|
|
592
|
+
if (!this._selNext) {
|
|
593
|
+
this._selNext = this.sqlite.prepare(
|
|
594
|
+
"SELECT dst FROM edge WHERE src = ? ORDER BY seq ASC",
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
return this._selNext.all(id).map((r) => r.dst);
|
|
598
|
+
}
|
|
599
|
+
_dbGetPrevEdges(id) {
|
|
600
|
+
if (!this._selPrev) {
|
|
601
|
+
this._selPrev = this.sqlite.prepare(
|
|
602
|
+
"SELECT src FROM edge WHERE dst = ? ORDER BY seq DESC",
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
return this._selPrev.all(id).map((r) => r.src);
|
|
606
|
+
}
|
|
607
|
+
_selNextFirst = null;
|
|
608
|
+
_dbGetNextEdgesFirst(id, limit) {
|
|
609
|
+
if (!this._selNextFirst) {
|
|
610
|
+
this._selNextFirst = this.sqlite.prepare(
|
|
611
|
+
"SELECT dst FROM edge WHERE src = ? ORDER BY seq ASC LIMIT ?",
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
return this._selNextFirst.all(id, limit)
|
|
615
|
+
.map((r) => r.dst);
|
|
616
|
+
}
|
|
617
|
+
_selPrevFirst = null;
|
|
618
|
+
_dbGetPrevEdgesFirst(id, limit) {
|
|
619
|
+
if (!this._selPrevFirst) {
|
|
620
|
+
this._selPrevFirst = this.sqlite.prepare(
|
|
621
|
+
"SELECT src FROM edge WHERE dst = ? ORDER BY seq DESC LIMIT ?",
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
return this._selPrevFirst.all(id, limit)
|
|
625
|
+
.map((r) => r.src);
|
|
626
|
+
}
|
|
627
|
+
_selPrevCount = null;
|
|
628
|
+
/** {@link Store.prevCount} — one indexed COUNT over idx_edge_dst, never a
|
|
629
|
+
* row materialisation (a common continuation's reverse fan-in is
|
|
630
|
+
* corpus-sized). */
|
|
631
|
+
prevCount(id) {
|
|
632
|
+
if (!this._selPrevCount) {
|
|
633
|
+
this._selPrevCount = this.sqlite.prepare(
|
|
634
|
+
"SELECT COUNT(*) AS n FROM edge WHERE dst = ?",
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
return this._selPrevCount.get(id).n;
|
|
638
|
+
}
|
|
639
|
+
_selSrcExists = null;
|
|
640
|
+
_dbEdgeSrcExists(src) {
|
|
641
|
+
if (!this._selSrcExists) {
|
|
642
|
+
// A prefix probe of idx_edge(src, dst) — one B-tree descent.
|
|
643
|
+
this._selSrcExists = this.sqlite.prepare(
|
|
644
|
+
"SELECT 1 AS one FROM edge WHERE src = ? LIMIT 1",
|
|
645
|
+
);
|
|
646
|
+
}
|
|
647
|
+
return this._selSrcExists.get(src) !== undefined;
|
|
648
|
+
}
|
|
649
|
+
_dbEdgeDistinctSrcCount() {
|
|
650
|
+
return this.sqlite.prepare(
|
|
651
|
+
"SELECT COUNT(*) AS n FROM (SELECT DISTINCT src FROM edge)",
|
|
652
|
+
).get().n;
|
|
653
|
+
}
|
|
654
|
+
// -- Halos (durable 2-bit quantized rows) --
|
|
655
|
+
_dbGetHalo(id) {
|
|
656
|
+
if (!this._selHalo) {
|
|
657
|
+
this._selHalo = this.sqlite.prepare(
|
|
658
|
+
"SELECT vec, mass FROM halo WHERE id = ?",
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
const r = this._selHalo.get(id);
|
|
662
|
+
return r ?? null;
|
|
663
|
+
}
|
|
664
|
+
_dbUpsertHalo(id, encodedVec, mass) {
|
|
665
|
+
if (!this._upHalo) {
|
|
666
|
+
this._upHalo = this.sqlite.prepare(
|
|
667
|
+
"INSERT INTO halo (id, vec, mass) VALUES (?, ?, ?) " +
|
|
668
|
+
"ON CONFLICT(id) DO UPDATE SET vec = excluded.vec, mass = excluded.mass",
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
this._upHalo.run(id, encodedVec, mass);
|
|
672
|
+
}
|
|
673
|
+
// -- Meta --
|
|
674
|
+
_dbGetMeta(key) {
|
|
675
|
+
if (!this._getMeta) {
|
|
676
|
+
this._getMeta = this.sqlite.prepare("SELECT val FROM meta WHERE key = ?");
|
|
677
|
+
}
|
|
678
|
+
const row = this._getMeta.get(key);
|
|
679
|
+
return row ? row.val : null;
|
|
680
|
+
}
|
|
681
|
+
_dbSetMeta(key, val) {
|
|
682
|
+
if (!this._setMeta) {
|
|
683
|
+
this._setMeta = this.sqlite.prepare(
|
|
684
|
+
"INSERT INTO meta (key, val) VALUES (?, ?) " +
|
|
685
|
+
"ON CONFLICT(key) DO UPDATE SET val = excluded.val",
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
this._setMeta.run(key, val);
|
|
689
|
+
}
|
|
690
|
+
_dbDeleteMeta(key) {
|
|
691
|
+
if (!this._delMeta) {
|
|
692
|
+
this._delMeta = this.sqlite.prepare("DELETE FROM meta WHERE key = ?");
|
|
693
|
+
}
|
|
694
|
+
this._delMeta.run(key);
|
|
695
|
+
}
|
|
696
|
+
// -- Snapshot --
|
|
697
|
+
_dbSaveSnapshot(bytes) {
|
|
698
|
+
if (!this._insSnapshot) {
|
|
699
|
+
this._insSnapshot = this.sqlite.prepare(
|
|
700
|
+
"INSERT INTO snapshot (id, data) VALUES (1, ?) " +
|
|
701
|
+
"ON CONFLICT(id) DO UPDATE SET data = excluded.data",
|
|
702
|
+
);
|
|
703
|
+
}
|
|
704
|
+
this._insSnapshot.run(bytes);
|
|
705
|
+
}
|
|
706
|
+
_dbLoadSnapshot() {
|
|
707
|
+
if (!this._selSnapshot) {
|
|
708
|
+
this._selSnapshot = this.sqlite.prepare(
|
|
709
|
+
"SELECT data FROM snapshot WHERE id = 1",
|
|
710
|
+
);
|
|
711
|
+
}
|
|
712
|
+
const r = this._selSnapshot.get();
|
|
713
|
+
return r ? new Uint8Array(r.data) : null;
|
|
714
|
+
}
|
|
715
|
+
// -- Vector DB: content (gist) index --
|
|
716
|
+
_vecContentUpsert(entries) {
|
|
717
|
+
this.content.upsertMany(entries);
|
|
718
|
+
}
|
|
719
|
+
_vecContentQuery(v, k, ef) {
|
|
720
|
+
return this.content.query(v, k, { ef });
|
|
721
|
+
}
|
|
722
|
+
_vecContentHas(id) {
|
|
723
|
+
return this.content ? this.content.has(id) : false;
|
|
724
|
+
}
|
|
725
|
+
_vecContentSize() {
|
|
726
|
+
return this.content ? this.content.size : 0;
|
|
727
|
+
}
|
|
728
|
+
_vecContentLastReads() {
|
|
729
|
+
return this.content ? this.content.lastQueryStorageReads : 0;
|
|
730
|
+
}
|
|
731
|
+
_vecContentPhysicalSize() {
|
|
732
|
+
return this.content ? this.content.physicalSize : 0;
|
|
733
|
+
}
|
|
734
|
+
_vecContentCompact() {
|
|
735
|
+
this.content.compact();
|
|
736
|
+
}
|
|
737
|
+
_vecContentDeleteMany(ids) {
|
|
738
|
+
this.content.deleteMany(ids);
|
|
739
|
+
}
|
|
740
|
+
*_vecContentEntriesSince(after) {
|
|
741
|
+
if (this.content) {
|
|
742
|
+
yield* this.content.keysSince(after);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
/** {@link AbstractStore._dbEdgeOrHaloIds} — one C-side scan; UNION dedups
|
|
746
|
+
* and (with the ORDER BY) sorts, so the result is ready for the binary-
|
|
747
|
+
* search membership probes and the ascending-id maintenance walks. */
|
|
748
|
+
_dbEdgeOrHaloIds() {
|
|
749
|
+
const rows = this.sqlite.prepare(
|
|
750
|
+
"SELECT src AS id FROM edge UNION SELECT dst FROM edge " +
|
|
751
|
+
"UNION SELECT id FROM halo ORDER BY 1",
|
|
752
|
+
).all();
|
|
753
|
+
const out = new Array(rows.length);
|
|
754
|
+
for (let i = 0; i < rows.length; i++) {
|
|
755
|
+
out[i] = rows[i].id;
|
|
756
|
+
}
|
|
757
|
+
return out;
|
|
758
|
+
}
|
|
759
|
+
// -- Vector DB: halo index --
|
|
760
|
+
_vecHaloUpsert(entries) {
|
|
761
|
+
this.halos.upsertMany(entries);
|
|
762
|
+
}
|
|
763
|
+
_vecHaloQuery(v, k, ef) {
|
|
764
|
+
return this.halos.query(v, k, { ef });
|
|
765
|
+
}
|
|
766
|
+
_vecHaloSize() {
|
|
767
|
+
return this.halos ? this.halos.size : 0;
|
|
768
|
+
}
|
|
769
|
+
_vecHaloPhysicalSize() {
|
|
770
|
+
return this.halos ? this.halos.physicalSize : 0;
|
|
771
|
+
}
|
|
772
|
+
_vecHaloCompact() {
|
|
773
|
+
this.halos.compact();
|
|
774
|
+
}
|
|
775
|
+
/** Pre-fill both vector indices' RAM caches with sequential scans (up to
|
|
776
|
+
* their budget caps) — seconds of streaming instead of the minutes of
|
|
777
|
+
* random point reads a cold training/query session otherwise pays while
|
|
778
|
+
* warming. Optional; call once after open before sustained work.
|
|
779
|
+
* Returns rows warmed. */
|
|
780
|
+
async warmVectorCaches() {
|
|
781
|
+
await this._ensureReady();
|
|
782
|
+
return (this.content?.warmCache() ?? 0) + (this.halos?.warmCache() ?? 0);
|
|
783
|
+
}
|
|
702
784
|
}
|