@hviana/sema 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/geometry.ts +14 -30
- package/src/mind/mechanisms/cast.ts +12 -7
- package/dist/example/demo.d.ts +0 -1
- package/dist/example/demo.js +0 -39
- package/dist/example/train_base.d.ts +0 -87
- package/dist/example/train_base.js +0 -2252
- package/dist/src/alphabet.d.ts +0 -7
- package/dist/src/alphabet.js +0 -33
- package/dist/src/alu/src/alu.d.ts +0 -185
- package/dist/src/alu/src/alu.js +0 -440
- package/dist/src/alu/src/expr.d.ts +0 -61
- package/dist/src/alu/src/expr.js +0 -318
- package/dist/src/alu/src/index.d.ts +0 -11
- package/dist/src/alu/src/index.js +0 -19
- package/dist/src/alu/src/kernel-arith.d.ts +0 -16
- package/dist/src/alu/src/kernel-arith.js +0 -264
- package/dist/src/alu/src/kernel-bits.d.ts +0 -19
- package/dist/src/alu/src/kernel-bits.js +0 -152
- package/dist/src/alu/src/kernel-logic.d.ts +0 -4
- package/dist/src/alu/src/kernel-logic.js +0 -60
- package/dist/src/alu/src/kernel-nd.d.ts +0 -3
- package/dist/src/alu/src/kernel-nd.js +0 -208
- package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
- package/dist/src/alu/src/kernel-numeric.js +0 -366
- package/dist/src/alu/src/operation.d.ts +0 -168
- package/dist/src/alu/src/operation.js +0 -189
- package/dist/src/alu/src/parser.d.ts +0 -221
- package/dist/src/alu/src/parser.js +0 -577
- package/dist/src/alu/src/resonance.d.ts +0 -55
- package/dist/src/alu/src/resonance.js +0 -126
- package/dist/src/alu/src/text.d.ts +0 -31
- package/dist/src/alu/src/text.js +0 -73
- package/dist/src/alu/src/value.d.ts +0 -109
- package/dist/src/alu/src/value.js +0 -300
- package/dist/src/alu/test/alu.test.d.ts +0 -1
- package/dist/src/alu/test/alu.test.js +0 -764
- package/dist/src/bytes.d.ts +0 -14
- package/dist/src/bytes.js +0 -59
- package/dist/src/canon.d.ts +0 -26
- package/dist/src/canon.js +0 -57
- package/dist/src/config.d.ts +0 -111
- package/dist/src/config.js +0 -91
- package/dist/src/derive/src/deduction.d.ts +0 -125
- package/dist/src/derive/src/deduction.js +0 -155
- package/dist/src/derive/src/index.d.ts +0 -7
- package/dist/src/derive/src/index.js +0 -11
- package/dist/src/derive/src/priority-queue.d.ts +0 -20
- package/dist/src/derive/src/priority-queue.js +0 -73
- package/dist/src/derive/src/rewrite.d.ts +0 -56
- package/dist/src/derive/src/rewrite.js +0 -100
- package/dist/src/derive/src/trie.d.ts +0 -90
- package/dist/src/derive/src/trie.js +0 -217
- package/dist/src/derive/test/derive.test.d.ts +0 -1
- package/dist/src/derive/test/derive.test.js +0 -122
- package/dist/src/extension.d.ts +0 -37
- package/dist/src/extension.js +0 -7
- package/dist/src/geometry.d.ts +0 -148
- package/dist/src/geometry.js +0 -529
- package/dist/src/index.d.ts +0 -16
- package/dist/src/index.js +0 -18
- package/dist/src/ingest-cache.d.ts +0 -41
- package/dist/src/ingest-cache.js +0 -161
- package/dist/src/mind/articulation.d.ts +0 -6
- package/dist/src/mind/articulation.js +0 -99
- package/dist/src/mind/attention.d.ts +0 -72
- package/dist/src/mind/attention.js +0 -898
- package/dist/src/mind/canonical.d.ts +0 -29
- package/dist/src/mind/canonical.js +0 -88
- package/dist/src/mind/graph-search.d.ts +0 -270
- package/dist/src/mind/graph-search.js +0 -847
- package/dist/src/mind/index.d.ts +0 -5
- package/dist/src/mind/index.js +0 -5
- package/dist/src/mind/junction.d.ts +0 -95
- package/dist/src/mind/junction.js +0 -262
- package/dist/src/mind/learning.d.ts +0 -47
- package/dist/src/mind/learning.js +0 -233
- package/dist/src/mind/match.d.ts +0 -113
- package/dist/src/mind/match.js +0 -446
- package/dist/src/mind/mechanisms/alu.d.ts +0 -4
- package/dist/src/mind/mechanisms/alu.js +0 -29
- package/dist/src/mind/mechanisms/cast.d.ts +0 -35
- package/dist/src/mind/mechanisms/cast.js +0 -461
- package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
- package/dist/src/mind/mechanisms/confluence.js +0 -225
- package/dist/src/mind/mechanisms/cover.d.ts +0 -6
- package/dist/src/mind/mechanisms/cover.js +0 -179
- package/dist/src/mind/mechanisms/extraction.d.ts +0 -67
- package/dist/src/mind/mechanisms/extraction.js +0 -342
- package/dist/src/mind/mechanisms/recall.d.ts +0 -13
- package/dist/src/mind/mechanisms/recall.js +0 -235
- package/dist/src/mind/mind.d.ts +0 -273
- package/dist/src/mind/mind.js +0 -562
- package/dist/src/mind/pipeline-mechanism.d.ts +0 -142
- package/dist/src/mind/pipeline-mechanism.js +0 -213
- package/dist/src/mind/pipeline.d.ts +0 -20
- package/dist/src/mind/pipeline.js +0 -185
- package/dist/src/mind/primitives.d.ts +0 -63
- package/dist/src/mind/primitives.js +0 -251
- package/dist/src/mind/rationale.d.ts +0 -134
- package/dist/src/mind/rationale.js +0 -162
- package/dist/src/mind/reasoning.d.ts +0 -15
- package/dist/src/mind/reasoning.js +0 -162
- package/dist/src/mind/recognition.d.ts +0 -20
- package/dist/src/mind/recognition.js +0 -246
- package/dist/src/mind/resonance.d.ts +0 -23
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +0 -15
- package/dist/src/mind/trace.js +0 -73
- package/dist/src/mind/traverse.d.ts +0 -113
- package/dist/src/mind/traverse.js +0 -488
- package/dist/src/mind/types.d.ts +0 -186
- package/dist/src/mind/types.js +0 -84
- package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
- package/dist/src/rabitq-ivf/src/database.js +0 -201
- package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
- package/dist/src/rabitq-ivf/src/index.js +0 -4
- package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
- package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
- package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
- package/dist/src/rabitq-ivf/src/prng.js +0 -36
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-ivf/src/rabitq.js +0 -283
- package/dist/src/sema.d.ts +0 -31
- package/dist/src/sema.js +0 -63
- package/dist/src/store-sqlite.d.ts +0 -184
- package/dist/src/store-sqlite.js +0 -942
- package/dist/src/store.d.ts +0 -654
- package/dist/src/store.js +0 -1612
- package/dist/src/vec.d.ts +0 -31
- package/dist/src/vec.js +0 -109
package/dist/src/store.js
DELETED
|
@@ -1,1612 +0,0 @@
|
|
|
1
|
-
// store.ts — the memory as a content-addressed node graph (a Merkle DAG).
|
|
2
|
-
//
|
|
3
|
-
// There is no "whole", no "leaf vs interior" privilege. There is one kind of
|
|
4
|
-
// thing — a NODE — named by its content. A node is either a leaf (a single
|
|
5
|
-
// byte, with an implicit negative id) or a branch (an ordered list of child
|
|
6
|
-
// node ids). Identical nodes are stored once (hash-consing); a node that was the
|
|
7
|
-
// root of one deposit becomes, by reference, an interior child of a larger
|
|
8
|
-
// deposit that contains it. Storage is therefore O(distinct subtrees), and every
|
|
9
|
-
// span the encoder ever produced is individually addressable.
|
|
10
|
-
//
|
|
11
|
-
// Node ids are signed integers. Single-byte leaves occupy the negative range
|
|
12
|
-
// −256…−1 (derived from the byte value, never stored in a DB row). Branches
|
|
13
|
-
// are dense, monotonically-increasing positive integers (0,1,2,…), never
|
|
14
|
-
// deleted, so the count of minted branch ids is the next id. Integer ids are the
|
|
15
|
-
// most compact key possible: SQLite stores them inline in the rowid B-tree (no
|
|
16
|
-
// secondary index), child lists pack to 4-byte signed integers, and the vector
|
|
17
|
-
// index keys on them as a 4-byte int32.
|
|
18
|
-
//
|
|
19
|
-
// On top of the DAG sit two relations:
|
|
20
|
-
// • the gist index — each node's fold vector, for SOFT (resonant) lookup;
|
|
21
|
-
// • continuation edges — learned "what follows what", the associations recall
|
|
22
|
-
// traverses. Both are keyed by node id, so both deduplicate for free.
|
|
23
|
-
//
|
|
24
|
-
// AbstractStore is the template-method base class that contains ALL domain
|
|
25
|
-
// logic — caching, dedup/merge decisions, geometric
|
|
26
|
-
// halo scheduling, buffer management — and calls down to a handful of
|
|
27
|
-
// protected abstract methods that a concrete persistence adapter implements.
|
|
28
|
-
// SQliteStore (in store-sqlite.ts) extends it and provides only the bare
|
|
29
|
-
// essentials required for SQLite + VectorDatabase communication.
|
|
30
|
-
import { addInto, copy, dot, normalize } from "./vec.js";
|
|
31
|
-
import { identityBar } from "./geometry.js";
|
|
32
|
-
/** Entry cap for the two ANN read caches ({@link AbstractStore._resonateCache}
|
|
33
|
-
* / `_resonateHaloCache`). They are only dropped on index writes, so a long
|
|
34
|
-
* query-only session otherwise grows them without bound — a slow memory/GC
|
|
35
|
-
* degradation that takes hours to manifest. At the cap the cache is cleared
|
|
36
|
-
* wholesale (repeat-query hits re-warm it; a miss only re-runs the ANN). */
|
|
37
|
-
const RESONATE_CACHE_MAX = 4096;
|
|
38
|
-
/** Longest dedup-cache KEY worth retaining, in string length. Branch keys
|
|
39
|
-
* are `kids.join(",")`; structural branches stay tiny (≤ maxGroup kids), but
|
|
40
|
-
* the whole-stream flat branches deposit interns (one leaf id PER INPUT
|
|
41
|
-
* BYTE) produce keys of ~4–5 chars per content byte — one per deposit. The
|
|
42
|
-
* dedup caches are bounded by ENTRY count, so retaining those keys grows
|
|
43
|
-
* memory with total ingested content. Past this bound the cache is
|
|
44
|
-
* bypassed: the durable content-addressed probe still answers (dedup never
|
|
45
|
-
* depends on the cache), at one indexed SQLite lookup per re-encounter —
|
|
46
|
-
* and true input-level repeats are already absorbed by the ingest cache. */
|
|
47
|
-
const DEDUP_KEY_MAX = 4096;
|
|
48
|
-
/** Safety depth bound for one {@link Store.chainRun} read. Semantics never
|
|
49
|
-
* depend on it (a truncated run ends on a still-transparent node and the
|
|
50
|
-
* climber continues from there); it only bounds a single read's size. */
|
|
51
|
-
const CHAIN_DEPTH_CAP = 65_536;
|
|
52
|
-
/** The content key of a Float32Array — one latin1 char per byte, an exact
|
|
53
|
-
* encoding of the 32-bit floats. Content-addressed, not object-identity,
|
|
54
|
-
* so a fresh Float32Array carrying the same values still hits. Vectors are
|
|
55
|
-
* D·4 bytes (max ~4096), well under the argument limit; the key is far
|
|
56
|
-
* cheaper than the ANN query it deduplicates. */
|
|
57
|
-
function vecKey(v) {
|
|
58
|
-
const bytes = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
|
|
59
|
-
// Chunked fromCharCode: one spread of D·4 args (16K at D=4096) flirts with
|
|
60
|
-
// engine argument limits and is slower than a few bounded calls.
|
|
61
|
-
let s = "";
|
|
62
|
-
for (let i = 0; i < bytes.length; i += 8192) {
|
|
63
|
-
s += String.fromCharCode(...bytes.subarray(i, i + 8192));
|
|
64
|
-
}
|
|
65
|
-
return s;
|
|
66
|
-
}
|
|
67
|
-
/** Bounded map with LRU eviction and byte accounting. On get the entry
|
|
68
|
-
* moves to the most-recent end; on set the least-recently-used entries are
|
|
69
|
-
* evicted until total bytes ≤ `maxBytes`. The optional `sizeOf` callback
|
|
70
|
-
* measures each value in bytes (defaults to 1, so `maxBytes` = max entries
|
|
71
|
-
* for uniform caches). A miss only costs a little extra work later, never
|
|
72
|
-
* correctness. */
|
|
73
|
-
export class BoundedMap {
|
|
74
|
-
maxBytes;
|
|
75
|
-
sizeOf;
|
|
76
|
-
evict;
|
|
77
|
-
m = new Map();
|
|
78
|
-
_bytes = 0;
|
|
79
|
-
// Persistent eviction cursor over the Map's insertion order. A fresh
|
|
80
|
-
// `keys()` iterator per eviction re-skips the growing prefix of holes that
|
|
81
|
-
// deletions leave in V8's ordered backing store (compacted only on rehash),
|
|
82
|
-
// making each eviction O(size) once the map first fills — the training-rate
|
|
83
|
-
// cliff at scale. A persistent iterator passes each hole exactly once
|
|
84
|
-
// (V8 map iterators stay valid under mutation): amortized O(1).
|
|
85
|
-
_cursor = null;
|
|
86
|
-
// "smallest" mode: oldest-entry candidates carried between evictions, fed
|
|
87
|
-
// from the cursor, so the LRU window never rescans from the front.
|
|
88
|
-
_candidates = [];
|
|
89
|
-
constructor(maxBytes, sizeOf = () => 1, evict = "lru") {
|
|
90
|
-
this.maxBytes = maxBytes;
|
|
91
|
-
this.sizeOf = sizeOf;
|
|
92
|
-
this.evict = evict;
|
|
93
|
-
}
|
|
94
|
-
/** Next key in insertion (≈ LRU) order, resuming where the last call left
|
|
95
|
-
* off; wraps to the front when exhausted. Undefined only when empty. */
|
|
96
|
-
nextOldest() {
|
|
97
|
-
for (let wrapped = false;;) {
|
|
98
|
-
if (this._cursor === null)
|
|
99
|
-
this._cursor = this.m.keys();
|
|
100
|
-
const n = this._cursor.next();
|
|
101
|
-
if (!n.done)
|
|
102
|
-
return n.value;
|
|
103
|
-
this._cursor = null;
|
|
104
|
-
if (this.m.size === 0 || wrapped)
|
|
105
|
-
return undefined;
|
|
106
|
-
wrapped = true;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
get(k) {
|
|
110
|
-
const v = this.m.get(k);
|
|
111
|
-
if (v !== undefined) {
|
|
112
|
-
this.m.delete(k);
|
|
113
|
-
this.m.set(k, v);
|
|
114
|
-
}
|
|
115
|
-
return v;
|
|
116
|
-
}
|
|
117
|
-
/** Membership without touching LRU order — a pure peek, for callers that only
|
|
118
|
-
* need "is this key present?" and must not promote it to most-recent. */
|
|
119
|
-
has(k) {
|
|
120
|
-
return this.m.has(k);
|
|
121
|
-
}
|
|
122
|
-
set(k, v) {
|
|
123
|
-
const old = this.m.get(k);
|
|
124
|
-
if (old !== undefined) {
|
|
125
|
-
this._bytes -= this.sizeOf(old);
|
|
126
|
-
this.m.delete(k);
|
|
127
|
-
}
|
|
128
|
-
this.m.set(k, v);
|
|
129
|
-
this._bytes += this.sizeOf(v);
|
|
130
|
-
while (this._bytes > this.maxBytes && this.m.size > 0) {
|
|
131
|
-
if (this.evict === "smallest") {
|
|
132
|
-
// Among the oldest LRU candidates, evict the cheapest to rebuild.
|
|
133
|
-
// Window grows logarithmically with cache size — wide enough to
|
|
134
|
-
// find a cheap victim without scanning the whole map. Candidates
|
|
135
|
-
// are pulled from the persistent cursor and carried between
|
|
136
|
-
// evictions, so the window never rescans the map from the front.
|
|
137
|
-
const WINDOW = Math.ceil(Math.log2(this.m.size + 1));
|
|
138
|
-
this._candidates = this._candidates.filter((k) => this.m.has(k));
|
|
139
|
-
while (this._candidates.length < WINDOW) {
|
|
140
|
-
const k = this.nextOldest();
|
|
141
|
-
if (k === undefined)
|
|
142
|
-
break;
|
|
143
|
-
if (!this._candidates.includes(k))
|
|
144
|
-
this._candidates.push(k);
|
|
145
|
-
}
|
|
146
|
-
let bestI = -1;
|
|
147
|
-
let bestSz = Infinity;
|
|
148
|
-
for (let i = 0; i < this._candidates.length; i++) {
|
|
149
|
-
const sz = this.sizeOf(this.m.get(this._candidates[i]));
|
|
150
|
-
if (sz < bestSz) {
|
|
151
|
-
bestSz = sz;
|
|
152
|
-
bestI = i;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
if (bestI < 0)
|
|
156
|
-
break;
|
|
157
|
-
const bestK = this._candidates[bestI];
|
|
158
|
-
this._candidates.splice(bestI, 1);
|
|
159
|
-
this._bytes -= bestSz;
|
|
160
|
-
this.m.delete(bestK);
|
|
161
|
-
}
|
|
162
|
-
else {
|
|
163
|
-
const lru = this.nextOldest();
|
|
164
|
-
if (lru === undefined)
|
|
165
|
-
break;
|
|
166
|
-
const lruv = this.m.get(lru);
|
|
167
|
-
if (lruv === undefined)
|
|
168
|
-
continue;
|
|
169
|
-
this._bytes -= this.sizeOf(lruv);
|
|
170
|
-
this.m.delete(lru);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
get size() {
|
|
175
|
-
return this.m.size;
|
|
176
|
-
}
|
|
177
|
-
get bytes() {
|
|
178
|
-
return this._bytes;
|
|
179
|
-
}
|
|
180
|
-
/** Remove one entry (point invalidation), with byte accounting. */
|
|
181
|
-
delete(k) {
|
|
182
|
-
const v = this.m.get(k);
|
|
183
|
-
if (v === undefined)
|
|
184
|
-
return;
|
|
185
|
-
this._bytes -= this.sizeOf(v);
|
|
186
|
-
this.m.delete(k);
|
|
187
|
-
}
|
|
188
|
-
/** Drop every entry (bulk invalidation) — O(1) amortised via fresh maps. */
|
|
189
|
-
clear() {
|
|
190
|
-
if (this.m.size === 0)
|
|
191
|
-
return;
|
|
192
|
-
this.m = new Map();
|
|
193
|
-
this._bytes = 0;
|
|
194
|
-
this._cursor = null;
|
|
195
|
-
this._candidates = [];
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
// ── Serialisation utilities (pure functions, no DB dependency) ───────────
|
|
199
|
-
const _ZERO = new Uint8Array(0);
|
|
200
|
-
/** Hand control back to the event loop for exactly one turn.
|
|
201
|
-
*
|
|
202
|
-
* `await` on an already-resolved promise only queues a MICROTASK, and the engine
|
|
203
|
-
* drains the entire microtask queue before it runs a single MACROTASK — so
|
|
204
|
-
* micro-awaiting alone never lets a timer, an I/O completion, or any other queued
|
|
205
|
-
* macrotask run. A macrotask primitive (`setImmediate`, else `setTimeout`) does:
|
|
206
|
-
* it parks the continuation behind the loop's next iteration, after pending I/O.
|
|
207
|
-
*
|
|
208
|
-
* Deliberately NOT unref'd: a caller is awaiting this promise, so the one
|
|
209
|
-
* scheduling turn is real pending work. An unref'd wake-up let the process
|
|
210
|
-
* exit mid-`ingest` whenever the event loop had nothing else alive (a bare
|
|
211
|
-
* top-level deposit loop) — the await simply never resolved. */
|
|
212
|
-
function yieldToEventLoop() {
|
|
213
|
-
return new Promise((resolve) => {
|
|
214
|
-
const g = globalThis;
|
|
215
|
-
if (g.setImmediate)
|
|
216
|
-
g.setImmediate(resolve);
|
|
217
|
-
else
|
|
218
|
-
g.setTimeout(resolve, 0);
|
|
219
|
-
});
|
|
220
|
-
}
|
|
221
|
-
/** Concatenate an array of byte arrays. */
|
|
222
|
-
function concat(parts) {
|
|
223
|
-
let n = 0;
|
|
224
|
-
for (const p of parts)
|
|
225
|
-
n += p.length;
|
|
226
|
-
const out = new Uint8Array(n);
|
|
227
|
-
let o = 0;
|
|
228
|
-
for (const p of parts) {
|
|
229
|
-
out.set(p, o);
|
|
230
|
-
o += p.length;
|
|
231
|
-
}
|
|
232
|
-
return out;
|
|
233
|
-
}
|
|
234
|
-
/** The byte string a FLAT branch spans — a branch whose kids are ALL implicit
|
|
235
|
-
* single-byte leaves (ids −256…−1) is fully determined by its bytes, one per
|
|
236
|
-
* kid. Returns null when any kid is a real node. Such branches (the sliding
|
|
237
|
-
* sub-span windows, the leaf-parent chunks, the per-deposit flat root spans —
|
|
238
|
-
* the bulk of what perception explodes text into) are stored as their BYTES
|
|
239
|
-
* (1 byte per kid) in the leaf column, with a zero-length kids blob marking
|
|
240
|
-
* "derive the kid list from the bytes" — 4× smaller than int32-packed ids,
|
|
241
|
-
* and content-addressed through the same leaf index instead of a second
|
|
242
|
-
* blob-duplicating kids index. */
|
|
243
|
-
function flatKidsBytes(kids) {
|
|
244
|
-
const out = new Uint8Array(kids.length);
|
|
245
|
-
for (let i = 0; i < kids.length; i++) {
|
|
246
|
-
const k = kids[i];
|
|
247
|
-
if (k < -256 || k >= 0)
|
|
248
|
-
return null;
|
|
249
|
-
out[i] = -(k + 1);
|
|
250
|
-
}
|
|
251
|
-
return out;
|
|
252
|
-
}
|
|
253
|
-
/** The implicit kid list of a flat branch — the inverse of
|
|
254
|
-
* {@link flatKidsBytes}. */
|
|
255
|
-
export function flatBytesKids(bytes) {
|
|
256
|
-
const out = new Array(bytes.length);
|
|
257
|
-
for (let i = 0; i < bytes.length; i++)
|
|
258
|
-
out[i] = -(bytes[i] + 1);
|
|
259
|
-
return out;
|
|
260
|
-
}
|
|
261
|
-
/** Pack a child-id list as little-endian int32s — 4 bytes per child, far more
|
|
262
|
-
* compact than a space-joined decimal string and trivial to read back. */
|
|
263
|
-
export function packKids(kids) {
|
|
264
|
-
const out = new Int32Array(kids.length);
|
|
265
|
-
for (let i = 0; i < kids.length; i++)
|
|
266
|
-
out[i] = kids[i];
|
|
267
|
-
return new Uint8Array(out.buffer);
|
|
268
|
-
}
|
|
269
|
-
export function unpackKids(blob) {
|
|
270
|
-
// The BLOB may be a view at a non-multiple-of-4 byteOffset, so copy before
|
|
271
|
-
// reinterpreting as int32.
|
|
272
|
-
const i32 = new Int32Array(blob.slice().buffer);
|
|
273
|
-
return Array.from(i32);
|
|
274
|
-
}
|
|
275
|
-
/** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
|
|
276
|
-
* on. Collisions are resolved by verifying the stored blob, never trusted. */
|
|
277
|
-
function hashOf(bytes) {
|
|
278
|
-
let h = 0x811c9dc5 >>> 0;
|
|
279
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
280
|
-
h ^= bytes[i];
|
|
281
|
-
h = Math.imul(h, 0x01000193) >>> 0;
|
|
282
|
-
}
|
|
283
|
-
return h >>> 0;
|
|
284
|
-
}
|
|
285
|
-
/** Fast content key (FNV-1a + length). No array spread onto the call stack. */
|
|
286
|
-
function keyOf(bytes) {
|
|
287
|
-
return hashOf(bytes).toString(16) + ":" + bytes.length;
|
|
288
|
-
}
|
|
289
|
-
/** Re-index a halo only when its mass is small or a power of two — O(log mass)
|
|
290
|
-
* writes per node instead of O(mass). */
|
|
291
|
-
function geometricMass(mass) {
|
|
292
|
-
return mass <= 4 || (mass & (mass - 1)) === 0;
|
|
293
|
-
}
|
|
294
|
-
// Halo accumulators are stored 2-bit quantized: the exact float32 L2 norm
|
|
295
|
-
// followed by D two-bit codes (sign + magnitude class) — 4 + D/4 bytes, 16×
|
|
296
|
-
// smaller than float32 and 4× smaller than int8, which made halo rows the
|
|
297
|
-
// single largest table in an episodically-trained store.
|
|
298
|
-
//
|
|
299
|
-
// Two bits is the COARSEST grain that preserves what halos are read for. A
|
|
300
|
-
// halo is a superposition of high-dimensional random signatures, so its
|
|
301
|
-
// elements are Gaussian; the four levels below are the Lloyd-Max optimal
|
|
302
|
-
// quantizer for a unit Gaussian (decision threshold ±0.9816σ, reconstruction
|
|
303
|
-
// ±0.4528σ / ±1.5104σ), whose output keeps ≥0.88 correlation with the exact
|
|
304
|
-
// accumulator. Every consumer is a thresholded resonance — direct cosines
|
|
305
|
-
// against the concept midpoint, or queries into the halo VectorDatabase whose
|
|
306
|
-
// stored codes are 1-bit RaBitQ anyway — and a 0.88-correlated direction moves
|
|
307
|
-
// none of those comparisons across the midpoint. One bit is NOT enough: a pure
|
|
308
|
-
// sign grid compresses cosine c to (2/π)·asin(c), which drags mid-band concept
|
|
309
|
-
// resonance below the midpoint and severs legitimate cross-name transfer.
|
|
310
|
-
// σ is derived from the stored norm (σ = norm/√D), so the codec carries no
|
|
311
|
-
// per-corpus state; the decoded vector is rescaled to the exact norm, keeping
|
|
312
|
-
// incremental pours magnitude-true.
|
|
313
|
-
const HALO_Q_THRESHOLD = 0.9816; // Lloyd-Max decision point, unit Gaussian
|
|
314
|
-
const HALO_Q_LO = 0.4528; // reconstruction level, inner cell
|
|
315
|
-
const HALO_Q_HI = 1.5104; // reconstruction level, outer cell
|
|
316
|
-
function haloEncode(v) {
|
|
317
|
-
const D = v.length;
|
|
318
|
-
const out = new Uint8Array(4 + ((D + 3) >> 2));
|
|
319
|
-
let n2 = 0;
|
|
320
|
-
for (let i = 0; i < D; i++)
|
|
321
|
-
n2 += v[i] * v[i];
|
|
322
|
-
const norm = Math.sqrt(n2);
|
|
323
|
-
new DataView(out.buffer).setFloat32(0, norm, true);
|
|
324
|
-
const bar = HALO_Q_THRESHOLD * (norm / Math.sqrt(D));
|
|
325
|
-
for (let i = 0; i < D; i++) {
|
|
326
|
-
const x = v[i];
|
|
327
|
-
// bit0: sign, bit1: |x| beyond the Gaussian decision threshold.
|
|
328
|
-
const code = (x > 0 ? 1 : 0) | ((x > bar || -x > bar) ? 2 : 0);
|
|
329
|
-
out[4 + (i >> 2)] |= code << ((i & 3) << 1);
|
|
330
|
-
}
|
|
331
|
-
return out;
|
|
332
|
-
}
|
|
333
|
-
function haloDecode(blob, D) {
|
|
334
|
-
const norm = new DataView(blob.buffer, blob.byteOffset, 4).getFloat32(0, true);
|
|
335
|
-
const out = new Float32Array(D);
|
|
336
|
-
if (norm === 0)
|
|
337
|
-
return out;
|
|
338
|
-
const sigma = norm / Math.sqrt(D);
|
|
339
|
-
let n2 = 0;
|
|
340
|
-
for (let i = 0; i < D; i++) {
|
|
341
|
-
const code = (blob[4 + (i >> 2)] >> ((i & 3) << 1)) & 3;
|
|
342
|
-
const mag = (code & 2 ? HALO_Q_HI : HALO_Q_LO) * sigma;
|
|
343
|
-
const x = code & 1 ? mag : -mag;
|
|
344
|
-
out[i] = x;
|
|
345
|
-
n2 += x * x;
|
|
346
|
-
}
|
|
347
|
-
// Rescale to the exact stored norm so accumulation stays magnitude-true.
|
|
348
|
-
const s = norm / Math.sqrt(n2);
|
|
349
|
-
for (let i = 0; i < D; i++)
|
|
350
|
-
out[i] *= s;
|
|
351
|
-
return out;
|
|
352
|
-
}
|
|
353
|
-
// ── AbstractStore: template-method base with all domain logic ────────────
|
|
354
|
-
/**
|
|
355
|
-
* Template-method base class that contains ALL domain logic for the content-
|
|
356
|
-
* addressed DAG store — caching, dedup/merge decisions, structural-compaction
|
|
357
|
-
* geometric halo scheduling, buffer management. A concrete persistence adapter
|
|
358
|
-
* (e.g. {@link SQliteStore}) extends it and implements only the ~35 one-liner
|
|
359
|
-
* protected abstract methods that talk to the actual storage backend.
|
|
360
|
-
*/
|
|
361
|
-
export class AbstractStore {
|
|
362
|
-
/** Derived query breadth for a partitioned index of C clusters: probe √C
|
|
363
|
-
* of them (the same √-of-the-population convention as the hub bound √N).
|
|
364
|
-
* The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
|
|
365
|
-
* ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
|
|
366
|
-
* collection outgrows it: at 4,270 clusters the default 64 probed 16
|
|
367
|
-
* clusters (0.4%), and an exact stored match of a query routinely sat in
|
|
368
|
-
* an unprobed cluster — recall silently degraded as the store grew. The
|
|
369
|
-
* configured efSearch remains the floor for small collections. */
|
|
370
|
-
efFor(clusterCount) {
|
|
371
|
-
return Math.max(this.efSearch, 4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))));
|
|
372
|
-
}
|
|
373
|
-
// ── Config ─────────────────────────────────────────────────────────────
|
|
374
|
-
_D;
|
|
375
|
-
_maxGroup;
|
|
376
|
-
minHaloMass;
|
|
377
|
-
efSearch;
|
|
378
|
-
overfetch;
|
|
379
|
-
batchSize;
|
|
380
|
-
compactEveryNWrites;
|
|
381
|
-
// ── State ──────────────────────────────────────────────────────────────
|
|
382
|
-
/** Branch node ids are a dense, monotonically-increasing integer sequence
|
|
383
|
-
* (0,1,2,…). Single-byte leaves occupy the implicit negative range −256…−1.
|
|
384
|
-
* They are NEVER deleted, so the count of minted branch ids IS the next id —
|
|
385
|
-
* which doubles as the branch-node count and lets has() be an O(1) check.
|
|
386
|
-
* Set by `_dbOpen()` from the stored node count; incremented by `mintId()`. */
|
|
387
|
-
_nextId = 0;
|
|
388
|
-
_writtenSinceCompact = 0;
|
|
389
|
-
closed = false;
|
|
390
|
-
/** Lifecycle guard — resolved once `_dbOpen()` completes. */
|
|
391
|
-
_ready = null;
|
|
392
|
-
// ── Caches ─────────────────────────────────────────────────────────────
|
|
393
|
-
/** Exact-content dedup: content-key → node id. Intrinsic compression. */
|
|
394
|
-
_leafKey;
|
|
395
|
-
_branchKey;
|
|
396
|
-
/** Reconstructed-bytes read cache (regenerable), keyed by node id. */
|
|
397
|
-
_bytesCache;
|
|
398
|
-
/** contentLen memo — content is immutable, so entries never invalidate. */
|
|
399
|
-
_lenCache;
|
|
400
|
-
/** Node-record cache — avoids repeated persistence queries for shared DAG
|
|
401
|
-
* nodes. Each record is small (a few ints + short leaf buffer). */
|
|
402
|
-
_recCache;
|
|
403
|
-
/** Captured-but-not-yet-indexed gists. Sized in bytes (each is D·4); a deposit
|
|
404
|
-
* links/pours a node right after interning it, so the working set is one
|
|
405
|
-
* deposit's nodes — a small budget captures ~all of it, and an eviction only
|
|
406
|
-
* means that node is reached by the DAG climb instead of by direct
|
|
407
|
-
* resonance. */
|
|
408
|
-
_pendingGist;
|
|
409
|
-
/** EXACT halo accumulators for the session's live pours: full-precision in
|
|
410
|
-
* memory, 2-bit on disk, so within-session accumulate-then-compare never
|
|
411
|
-
* round-trips through the quantizer. Regenerable — a miss reads the durable
|
|
412
|
-
* 2-bit row. */
|
|
413
|
-
_haloExact;
|
|
414
|
-
/** NORMALIZED halo read cache — the decoded, normalized vector {@link halo}
|
|
415
|
-
* returns, cached by id so repeat reads skip the per-call 2-bit decode and
|
|
416
|
-
* normalize of a full D-element row (measured on a trained store: ~15K
|
|
417
|
-
* halo() calls per deep query over ~50 distinct ids — all but the first
|
|
418
|
-
* per id pure re-decode). Point-invalidated by {@link pourHalo}, the one
|
|
419
|
-
* halo mutation site. Callers receive a COPY, so the cached vector is
|
|
420
|
-
* never aliased. Regenerable — a miss re-decodes the durable row. */
|
|
421
|
-
_haloNorm;
|
|
422
|
-
/** Interiors deliberately SKIPPED by indexSubtree (unique nodes with 1 parent
|
|
423
|
-
* that bridge nothing). Remembered so subsequent visits prune the subtree
|
|
424
|
-
* without re-checking parent count. LRU-bounded: an evicted entry is
|
|
425
|
-
* re-checked on next visit — if it gained parents in the meantime, it will
|
|
426
|
-
* be promoted to the index. */
|
|
427
|
-
_coveredIds;
|
|
428
|
-
/** Live content-index id set, LRU-bounded so a massive ingest never leaks
|
|
429
|
-
* memory; an evicted entry is still indexed (the row is durable), so the
|
|
430
|
-
* only cost of an eviction is a duplicate index probe on next visit. */
|
|
431
|
-
_indexedIds;
|
|
432
|
-
// ── ANN read cache ─────────────────────────────────────────────────────
|
|
433
|
-
// The index is read-only between writes, so the same (v,k) always returns
|
|
434
|
-
// the same neighbours. Any index mutation (flush, delete, compact) drops
|
|
435
|
-
// the cache; the next miss recreates it. Content-addressed (vecKey), not
|
|
436
|
-
// identity-addressed — same principle as perceiveMemo.
|
|
437
|
-
/** ANN read cache for {@link resonate} — keyed by vecKey(v) + ":" + k;
|
|
438
|
-
* lazily initialised, dropped on any index mutation. */
|
|
439
|
-
_resonateCache = null;
|
|
440
|
-
/** ANN read cache for {@link resonateHalo} — same scheme. */
|
|
441
|
-
_resonateHaloCache = null;
|
|
442
|
-
// ── Write buffers ──────────────────────────────────────────────────────
|
|
443
|
-
/** Content (gist) index write buffer. */
|
|
444
|
-
_contentBuffer = [];
|
|
445
|
-
/** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
|
|
446
|
-
_haloBuffer = new Map();
|
|
447
|
-
/** Containment write buffer: child → new parents, merged on flush cadence. */
|
|
448
|
-
_containBuf = new Map();
|
|
449
|
-
/** Dedup-target candidates still in the write buffer (keyed by id). Only
|
|
450
|
-
* roots that have gained an edge/halo are targets; a fresh intermediate
|
|
451
|
-
* branch is never folded onto. */
|
|
452
|
-
_nearDedupBuf = new Map();
|
|
453
|
-
/** Ids currently in `_contentBuffer` (not yet flushed) — O(1) membership. */
|
|
454
|
-
_bufferedIds = new Set();
|
|
455
|
-
// ── Transparent-chain cache ────────────────────────────────────────────
|
|
456
|
-
/** {@link Store.chainRun} results, valid for the store's lifetime BETWEEN
|
|
457
|
-
* writes: a chain is a pure function of the kid and edge tables, so any
|
|
458
|
-
* write that could break a node's transparency (a fresh mint inserting kid
|
|
459
|
-
* rows, a link inserting an edge) drops the whole cache — see the two
|
|
460
|
-
* invalidation sites. Regenerable; a miss re-walks. */
|
|
461
|
-
_chainMemo;
|
|
462
|
-
// ── Edge-source-count cache ────────────────────────────────────────────
|
|
463
|
-
/** Distinct edge-source count — the store's DOCUMENT COUNT (how many
|
|
464
|
-
* learnt contexts predict a continuation), the N of every
|
|
465
|
-
* inverse-document-frequency read. −1 until first asked for; from then
|
|
466
|
-
* on maintained INCREMENTALLY by {@link link} (edges are never deleted),
|
|
467
|
-
* so a read is O(1) — never a table scan on the recall path. */
|
|
468
|
-
_edgeSrcCount = -1;
|
|
469
|
-
// ── Constructor ────────────────────────────────────────────────────────
|
|
470
|
-
constructor(config, D, maxGroup) {
|
|
471
|
-
this._D = D;
|
|
472
|
-
this._maxGroup = maxGroup;
|
|
473
|
-
this.minHaloMass = config.minHaloMass;
|
|
474
|
-
this.efSearch = config.efSearch;
|
|
475
|
-
this.overfetch = config.overfetch;
|
|
476
|
-
this.batchSize = config.batchSize;
|
|
477
|
-
this.compactEveryNWrites = config.compactEveryNWrites;
|
|
478
|
-
this._leafKey = new BoundedMap(config.dedupCacheMax);
|
|
479
|
-
this._branchKey = new BoundedMap(config.dedupCacheMax);
|
|
480
|
-
this._bytesCache = new BoundedMap(config.bytesCacheMax, (v) => v.byteLength, "smallest");
|
|
481
|
-
this._lenCache = new BoundedMap(config.bytesCacheMax, () => 16);
|
|
482
|
-
this._recCache = new BoundedMap(config.recCacheBytes, (r) => (r.leaf?.byteLength ?? 0) + (r.kids?.length ?? 0) * 4 + 12);
|
|
483
|
-
this._pendingGist = new BoundedMap(config.pendingGistBytes, (v) => v.byteLength);
|
|
484
|
-
this._haloExact = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
|
|
485
|
-
this._haloNorm = new BoundedMap(config.haloCacheBytes, (v) => v.byteLength);
|
|
486
|
-
this._coveredIds = new BoundedMap(config.coveredIdsMax);
|
|
487
|
-
this._indexedIds = new BoundedMap(config.coveredIdsMax);
|
|
488
|
-
this._chainMemo = new BoundedMap(config.chainCacheBytes, (v) => v.length * 4 + 32);
|
|
489
|
-
}
|
|
490
|
-
// ── Public accessors ───────────────────────────────────────────────────
|
|
491
|
-
get D() {
|
|
492
|
-
return this._D;
|
|
493
|
-
}
|
|
494
|
-
/** Await the async initialisation performed by the concrete constructor. */
|
|
495
|
-
async _ensureReady() {
|
|
496
|
-
if (!this._ready)
|
|
497
|
-
throw new Error("Store: not open");
|
|
498
|
-
await this._ready;
|
|
499
|
-
}
|
|
500
|
-
// ── Id management ──────────────────────────────────────────────────────
|
|
501
|
-
has(id) {
|
|
502
|
-
// Byte leaves (negative ids) always exist.
|
|
503
|
-
if (id < 0)
|
|
504
|
-
return id >= -256;
|
|
505
|
-
return Number.isInteger(id) && id >= 0 && id < this._nextId;
|
|
506
|
-
}
|
|
507
|
-
mintId() {
|
|
508
|
-
return this._nextId++;
|
|
509
|
-
}
|
|
510
|
-
nodeCount() {
|
|
511
|
-
return this._nextId;
|
|
512
|
-
}
|
|
513
|
-
async size() {
|
|
514
|
-
await this._ensureReady();
|
|
515
|
-
return this._nextId;
|
|
516
|
-
}
|
|
517
|
-
// ── DAG traversal ──────────────────────────────────────────────────────
|
|
518
|
-
get(id) {
|
|
519
|
-
// Byte leaves are implicit — fabricate from the id.
|
|
520
|
-
if (id < 0) {
|
|
521
|
-
return { id, leaf: new Uint8Array([-(id + 1)]), kids: null };
|
|
522
|
-
}
|
|
523
|
-
const hit = this._recCache.get(id);
|
|
524
|
-
if (hit !== undefined)
|
|
525
|
-
return hit;
|
|
526
|
-
const rec = this._dbGetNode(id);
|
|
527
|
-
if (rec)
|
|
528
|
-
this._recCache.set(id, rec);
|
|
529
|
-
return rec;
|
|
530
|
-
}
|
|
531
|
-
/** Reconstruct the bytes a node spans by traversing the DAG bottom-up.
|
|
532
|
-
* Iterative post-order on an explicit stack — the call stack never sees the
|
|
533
|
-
* tree depth, so even an adversarial chain of nodes stays safe. */
|
|
534
|
-
/** How many reads hit a MISSING node record this session (a dangling edge
|
|
535
|
-
* or kid id). Zero in a healthy store; a growing count means references
|
|
536
|
-
* outlive their records — the read degrades safely to empty bytes, this
|
|
537
|
-
* counter is what keeps that degradation observable. */
|
|
538
|
-
danglingReads = 0;
|
|
539
|
-
bytes(id) {
|
|
540
|
-
// Fast path.
|
|
541
|
-
const hit = this._bytesCache.get(id);
|
|
542
|
-
if (hit)
|
|
543
|
-
return hit;
|
|
544
|
-
const stack = [id];
|
|
545
|
-
const cache = this._bytesCache;
|
|
546
|
-
while (stack.length > 0) {
|
|
547
|
-
const nid = stack[stack.length - 1]; // peek
|
|
548
|
-
// Already resolved by an earlier traversal.
|
|
549
|
-
if (cache.get(nid)) {
|
|
550
|
-
stack.pop();
|
|
551
|
-
continue;
|
|
552
|
-
}
|
|
553
|
-
const rec = this.get(nid);
|
|
554
|
-
if (!rec) {
|
|
555
|
-
// A DANGLING id (an edge or kid pointing at no record) reads as
|
|
556
|
-
// empty — safe (empty-bytes guards drop it from grounding) but a
|
|
557
|
-
// symptom of store corruption, so count it rather than stay silent.
|
|
558
|
-
// The cache makes the empty read permanent for the session; the
|
|
559
|
-
// counter survives as the visible trace.
|
|
560
|
-
this.danglingReads++;
|
|
561
|
-
cache.set(nid, _ZERO);
|
|
562
|
-
stack.pop();
|
|
563
|
-
continue;
|
|
564
|
-
}
|
|
565
|
-
if (rec.leaf) {
|
|
566
|
-
cache.set(nid, new Uint8Array(rec.leaf));
|
|
567
|
-
stack.pop();
|
|
568
|
-
continue;
|
|
569
|
-
}
|
|
570
|
-
// Branch — push any uncached children (reverse order so they resolve
|
|
571
|
-
// left-to-right). If every child is already cached, concatenate now.
|
|
572
|
-
const kids = rec.kids ?? [];
|
|
573
|
-
let ready = true;
|
|
574
|
-
for (let i = kids.length - 1; i >= 0; i--) {
|
|
575
|
-
if (!cache.get(kids[i])) {
|
|
576
|
-
stack.push(kids[i]);
|
|
577
|
-
ready = false;
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
if (!ready)
|
|
581
|
-
continue;
|
|
582
|
-
stack.pop();
|
|
583
|
-
const out = concat(kids.map((k) => cache.get(k)));
|
|
584
|
-
cache.set(nid, out);
|
|
585
|
-
}
|
|
586
|
-
return cache.get(id) ?? _ZERO;
|
|
587
|
-
}
|
|
588
|
-
/** First `maxLen` bytes of a node. Walks only the leftmost branch,
|
|
589
|
-
* stopping at `maxLen` — so a 1 MB document root costs the same as a
|
|
590
|
-
* 4-byte leaf. Recursive, but tree depth is logarithmic.
|
|
591
|
-
*
|
|
592
|
-
* IMMUTABILITY CONTRACT (applies to {@link bytes} too): returned arrays
|
|
593
|
-
* may be shared with the byte cache and with other callers — treat them
|
|
594
|
-
* as read-only. Mutating one would corrupt every subsequent read. */
|
|
595
|
-
bytesPrefix(id, maxLen) {
|
|
596
|
-
if (maxLen <= 0)
|
|
597
|
-
return _ZERO;
|
|
598
|
-
// A FULL read (the ALL sentinel) routes through bytes(), whose
|
|
599
|
-
// reconstruction enters the byte-budget cache. Without this, the mind's
|
|
600
|
-
// read() — which always passes ALL — re-walked the DAG and re-concatenated
|
|
601
|
-
// on EVERY repeated read of an uncached branch, bypassing the cache that
|
|
602
|
-
// exists precisely for those reconstructions.
|
|
603
|
-
if (maxLen >= 0x7fffffff)
|
|
604
|
-
return this.bytes(id);
|
|
605
|
-
// Full-cache hit: bytes() already reconstructed the whole node.
|
|
606
|
-
const full = this._bytesCache.get(id);
|
|
607
|
-
if (full)
|
|
608
|
-
return full.length <= maxLen ? full : full.subarray(0, maxLen);
|
|
609
|
-
const rec = this.get(id);
|
|
610
|
-
if (!rec)
|
|
611
|
-
return _ZERO;
|
|
612
|
-
if (rec.leaf) {
|
|
613
|
-
// Cache the (small) leaf bytes — cheap and reusable. COPY before
|
|
614
|
-
// caching: rec.leaf is the node record's own buffer, and handing it
|
|
615
|
-
// out would let one mutating caller corrupt the record AND the cache
|
|
616
|
-
// (bytes() makes the same copy for the same reason).
|
|
617
|
-
const leaf = new Uint8Array(rec.leaf);
|
|
618
|
-
this._bytesCache.set(id, leaf);
|
|
619
|
-
return leaf.length <= maxLen ? leaf : leaf.subarray(0, maxLen);
|
|
620
|
-
}
|
|
621
|
-
// Branch — walk children left-to-right, stopping at maxLen.
|
|
622
|
-
const kids = rec.kids ?? [];
|
|
623
|
-
const parts = [];
|
|
624
|
-
let got = 0;
|
|
625
|
-
for (const k of kids) {
|
|
626
|
-
if (got >= maxLen)
|
|
627
|
-
break;
|
|
628
|
-
const child = this.bytesPrefix(k, maxLen - got);
|
|
629
|
-
parts.push(child);
|
|
630
|
-
got += child.length;
|
|
631
|
-
}
|
|
632
|
-
return concat(parts);
|
|
633
|
-
}
|
|
634
|
-
contentLen(id, cap = Infinity) {
|
|
635
|
-
if (id < 0)
|
|
636
|
-
return 1; // implicit single-byte leaf
|
|
637
|
-
const hit = this._lenCache.get(id);
|
|
638
|
-
if (hit !== undefined)
|
|
639
|
-
return hit; // exact — valid under any cap
|
|
640
|
-
if (cap <= 0)
|
|
641
|
-
return 0;
|
|
642
|
-
const rec = this.get(id);
|
|
643
|
-
let n = 0;
|
|
644
|
-
let clamped = false;
|
|
645
|
-
if (rec) {
|
|
646
|
-
if (rec.leaf)
|
|
647
|
-
n = rec.leaf.length;
|
|
648
|
-
else if (rec.kids) {
|
|
649
|
-
for (const k of rec.kids) {
|
|
650
|
-
n += this.contentLen(k, cap - n);
|
|
651
|
-
if (n >= cap) {
|
|
652
|
-
clamped = true; // partial sum — a lower bound, not the length
|
|
653
|
-
break;
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
if (!clamped)
|
|
659
|
-
this._lenCache.set(id, n);
|
|
660
|
-
return n;
|
|
661
|
-
}
|
|
662
|
-
// ── Content-addressed lookup ───────────────────────────────────────────
|
|
663
|
-
findLeaf(bytes) {
|
|
664
|
-
if (bytes.length === 1)
|
|
665
|
-
return -(bytes[0] + 1);
|
|
666
|
-
const key = keyOf(bytes);
|
|
667
|
-
const cached = this._leafKey.get(key);
|
|
668
|
-
if (cached !== undefined)
|
|
669
|
-
return cached;
|
|
670
|
-
const id = this._dbFindLeaf(hashOf(bytes), bytes);
|
|
671
|
-
if (id !== null)
|
|
672
|
-
this._leafKey.set(key, id);
|
|
673
|
-
return id;
|
|
674
|
-
}
|
|
675
|
-
findBranch(kids) {
|
|
676
|
-
const key = kids.join(",");
|
|
677
|
-
const cached = this._branchKey.get(key);
|
|
678
|
-
if (cached !== undefined)
|
|
679
|
-
return cached;
|
|
680
|
-
const flat = flatKidsBytes(kids);
|
|
681
|
-
let id;
|
|
682
|
-
if (flat) {
|
|
683
|
-
id = this._dbFindBranchByLeaf(hashOf(flat), flat);
|
|
684
|
-
}
|
|
685
|
-
else {
|
|
686
|
-
const packed = packKids(kids);
|
|
687
|
-
id = this._dbFindBranchByKids(hashOf(packed), packed);
|
|
688
|
-
}
|
|
689
|
-
if (id !== null && key.length <= DEDUP_KEY_MAX) {
|
|
690
|
-
this._branchKey.set(key, id);
|
|
691
|
-
}
|
|
692
|
-
return id;
|
|
693
|
-
}
|
|
694
|
-
// ── Structural parents ─────────────────────────────────────────────────
|
|
695
|
-
parents(id) {
|
|
696
|
-
return this._dbGetParents(id);
|
|
697
|
-
}
|
|
698
|
-
parentsFirst(id, limit) {
|
|
699
|
-
return this._dbGetParentsFirst(id, limit);
|
|
700
|
-
}
|
|
701
|
-
hasParents(id) {
|
|
702
|
-
return this._dbGetParentsFirst(id, 1).length > 0;
|
|
703
|
-
}
|
|
704
|
-
chainRun(id) {
|
|
705
|
-
const hit = this._chainMemo.get(id);
|
|
706
|
-
if (hit !== undefined)
|
|
707
|
-
return hit;
|
|
708
|
-
const run = this._chainWalk(id, CHAIN_DEPTH_CAP);
|
|
709
|
-
this._chainMemo.set(id, run);
|
|
710
|
-
return run;
|
|
711
|
-
}
|
|
712
|
-
/** {@link Store.chainRun}'s walk, node at a time through the existing
|
|
713
|
-
* probes. Adapters with a set-based query engine should override with a
|
|
714
|
-
* single server-side descent (the SQLite adapter uses a recursive CTE). */
|
|
715
|
-
_chainWalk(id, cap) {
|
|
716
|
-
const run = [id];
|
|
717
|
-
let n = id;
|
|
718
|
-
while (run.length < cap) {
|
|
719
|
-
if (this.hasNext(n) || this.prevCount(n) > 0)
|
|
720
|
-
break;
|
|
721
|
-
const ps = this._dbGetParentsFirst(n, 2);
|
|
722
|
-
if (ps.length !== 1)
|
|
723
|
-
break;
|
|
724
|
-
n = ps[0];
|
|
725
|
-
run.push(n);
|
|
726
|
-
}
|
|
727
|
-
return run;
|
|
728
|
-
}
|
|
729
|
-
// ── Containment ────────────────────────────────────────────────────────
|
|
730
|
-
addContainer(child, parent) {
|
|
731
|
-
let set = this._containBuf.get(child);
|
|
732
|
-
if (set === undefined) {
|
|
733
|
-
set = new Set();
|
|
734
|
-
this._containBuf.set(child, set);
|
|
735
|
-
}
|
|
736
|
-
set.add(parent);
|
|
737
|
-
}
|
|
738
|
-
hasContainers(child) {
|
|
739
|
-
if (this._dbContainExists(child))
|
|
740
|
-
return true;
|
|
741
|
-
const buf = this._containBuf.get(child);
|
|
742
|
-
return buf !== undefined && buf.size > 0;
|
|
743
|
-
}
|
|
744
|
-
containersSlice(child, offset, limit) {
|
|
745
|
-
const out = this._dbGetContainParentsSlice(child, offset, limit);
|
|
746
|
-
if (out.length >= limit)
|
|
747
|
-
return out;
|
|
748
|
-
// Buffered adds page in AFTER the stored ones. A buffered parent that is
|
|
749
|
-
// also stored may repeat across the seam; page consumers dedup by id
|
|
750
|
-
// (they all carry seen-sets), so a repeat costs a skip, never an error.
|
|
751
|
-
const buf = this._containBuf.get(child);
|
|
752
|
-
if (!buf || buf.size === 0)
|
|
753
|
-
return out;
|
|
754
|
-
const storedCount = this._dbGetContainCount(child);
|
|
755
|
-
const bufStart = Math.max(0, offset - storedCount) +
|
|
756
|
-
Math.max(0, out.length - Math.max(0, storedCount - offset));
|
|
757
|
-
let i = 0;
|
|
758
|
-
for (const p of buf) {
|
|
759
|
-
if (out.length >= limit)
|
|
760
|
-
break;
|
|
761
|
-
if (i++ < bufStart)
|
|
762
|
-
continue;
|
|
763
|
-
out.push(p);
|
|
764
|
-
}
|
|
765
|
-
return out;
|
|
766
|
-
}
|
|
767
|
-
containers(child) {
|
|
768
|
-
const stored = this._dbGetContainParents(child);
|
|
769
|
-
const buf = this._containBuf.get(child);
|
|
770
|
-
if (stored.length === 0)
|
|
771
|
-
return buf ? [...buf] : [];
|
|
772
|
-
if (!buf)
|
|
773
|
-
return stored;
|
|
774
|
-
const merged = new Set(stored);
|
|
775
|
-
for (const p of buf)
|
|
776
|
-
merged.add(p);
|
|
777
|
-
return [...merged];
|
|
778
|
-
}
|
|
779
|
-
// ── Walk kids to collect implicit per-byte leaf ids ────────────────────
|
|
780
|
-
flatLeafIds(kids) {
|
|
781
|
-
const out = [];
|
|
782
|
-
const stack = [...kids].reverse();
|
|
783
|
-
while (stack.length > 0) {
|
|
784
|
-
const id = stack.pop();
|
|
785
|
-
if (id < 0) {
|
|
786
|
-
out.push(id);
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
789
|
-
const rec = this.get(id);
|
|
790
|
-
if (!rec)
|
|
791
|
-
return null;
|
|
792
|
-
if (rec.leaf !== null) {
|
|
793
|
-
for (let i = 0; i < rec.leaf.length; i++)
|
|
794
|
-
out.push(-(rec.leaf[i] + 1));
|
|
795
|
-
}
|
|
796
|
-
else if (rec.kids !== null) {
|
|
797
|
-
for (let i = rec.kids.length - 1; i >= 0; i--)
|
|
798
|
-
stack.push(rec.kids[i]);
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
return out;
|
|
802
|
-
}
|
|
803
|
-
/** On a dedup HIT, keep the node's gist available for lazy indexing —
|
|
804
|
-
* EXACTLY when it is not already indexed. Replaces the old id-range
|
|
805
|
-
* "recency" heuristic (id ≥ nextId − cacheWindow), which conflated an LRU
|
|
806
|
-
* entry COUNT with an id RANGE and permanently refused to index any node
|
|
807
|
-
* that first became a resonance target long after it was minted (an early
|
|
808
|
-
* interior later reused as an edge/halo-bearing deposit root was silently
|
|
809
|
-
* unreachable by resonance). The durable index itself is the arbiter:
|
|
810
|
-
* one point query, cached in `_indexedIds` on a hit so repeats are O(1). */
|
|
811
|
-
captureIfUnindexed(id, gist) {
|
|
812
|
-
if (this._indexedIds.has(id) || this._pendingGist.has(id))
|
|
813
|
-
return;
|
|
814
|
-
if (this._vecContentHas(id)) {
|
|
815
|
-
this._indexedIds.set(id, true);
|
|
816
|
-
return;
|
|
817
|
-
}
|
|
818
|
-
this._pendingGist.set(id, normalize(copy(gist)));
|
|
819
|
-
// A node that ALREADY bridges experiences but was never indexed (its 1→2
|
|
820
|
-
// transition fired while its gist was evicted, or in a pre-transition
|
|
821
|
-
// store) is promoted on this re-encounter — the recapture above is
|
|
822
|
-
// exactly what makes its gist available again. Byte leaves (negative
|
|
823
|
-
// ids) never have kid rows, so skip their parent probe.
|
|
824
|
-
if (id >= 0)
|
|
825
|
-
this.promoteBridge(id);
|
|
826
|
-
}
|
|
827
|
-
/** If `id` structurally bridges ≥2 experiences (the post-hoc compaction
|
|
828
|
-
* criterion), promote its gist into the content index NOW — the exact
|
|
829
|
-
* moment it becomes useful for multi-experience recall. The 1→2 parent
|
|
830
|
-
* transition fires on {@link _dbInsertKid} during mint, and nodes that
|
|
831
|
-
* were already bridges but missed indexing (gist evicted, pre-transition
|
|
832
|
-
* store) are recaptured in {@link captureIfUnindexed}.
|
|
833
|
-
*
|
|
834
|
-
* A no-op when the node is already indexed or its gist is evicted from
|
|
835
|
-
* the pending cache — a future re-encounter will retry. */
|
|
836
|
-
promoteBridge(id) {
|
|
837
|
-
// A LIMITed probe: this runs once per kid insert on the MINT hot path,
|
|
838
|
-
// and a shared child's full parent set grows with the corpus.
|
|
839
|
-
if (this._dbGetParentsFirst(id, 2).length < 2)
|
|
840
|
-
return;
|
|
841
|
-
this.indexGist(id, false);
|
|
842
|
-
}
|
|
843
|
-
// ── Core interning: dedup → near-dedup → mint ──────────────────────────
|
|
844
|
-
async intern(leaf, kids, gist) {
|
|
845
|
-
await this._ensureReady();
|
|
846
|
-
// 1. Exact dedup — equal content → one id, no vector work. Primary
|
|
847
|
-
// compression mechanism, intrinsic to the store.
|
|
848
|
-
//
|
|
849
|
-
// findLeaf/findBranch are the content-addressed lookups: in-memory cache
|
|
850
|
-
// first, then a durable probe that repopulates the cache on a hit.
|
|
851
|
-
// Using them here (rather than only the cache) makes dedup survive a cold
|
|
852
|
-
// cache — a resumed/checkpointed training run, or one whose dedup cache
|
|
853
|
-
// has evicted old keys, still recognises content already on disk and
|
|
854
|
-
// reuses its id instead of minting a duplicate.
|
|
855
|
-
const hit = leaf !== null ? this.findLeaf(leaf) : this.findBranch(kids);
|
|
856
|
-
if (hit !== null) {
|
|
857
|
-
this.captureIfUnindexed(hit, gist);
|
|
858
|
-
return hit;
|
|
859
|
-
}
|
|
860
|
-
// 1b. Content-addressed lookup by leaf-id signature. When the same byte
|
|
861
|
-
// sequence was stored as a flat branch (via putBranch during deposit),
|
|
862
|
-
// a branch node spanning those bytes reuses that id even when its tree
|
|
863
|
-
// structure differs — pure content addressing, same bytes → same node.
|
|
864
|
-
if (kids !== null && kids.length >= 2) {
|
|
865
|
-
const leafIds = this.flatLeafIds(kids);
|
|
866
|
-
if (leafIds !== null) {
|
|
867
|
-
const flatHit = this.findBranch(leafIds);
|
|
868
|
-
if (flatHit !== null) {
|
|
869
|
-
this.captureIfUnindexed(flatHit, gist);
|
|
870
|
-
return flatHit;
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
const cache = leaf !== null ? this._leafKey : this._branchKey;
|
|
875
|
-
const key = leaf !== null ? keyOf(leaf) : kids.join(",");
|
|
876
|
-
// 2. Near dedup — BRANCHES ONLY, against RESONANCE TARGETS only.
|
|
877
|
-
// Leaves are single bytes: exact dedup already collapses every identical
|
|
878
|
-
// leaf, and near-merging distinct leaves only corrupts bytes for no real
|
|
879
|
-
// saving. Real near-dedup compression lives in subtree (branch) fusion.
|
|
880
|
-
//
|
|
881
|
-
// There is deliberately NO ANN probe of the FLUSHED index here. It used
|
|
882
|
-
// to fire for EVERY new branch that the buffer scan didn't settle — i.e.
|
|
883
|
-
// ~every interior branch, since interiors are never dedup targets —
|
|
884
|
-
// making one ANN query per branch the dominant training cost (it dwarfed
|
|
885
|
-
// perception and the index write). And it was not merely expensive but
|
|
886
|
-
// WRONG: the 1-bit RaBitQ code can rank a byte-DISTINCT branch as the
|
|
887
|
-
// nearest "target" of a fresh branch, so the fold collapsed two
|
|
888
|
-
// byte-different subtrees onto one id and corrupted exact reconstruction
|
|
889
|
-
// (02-roundtrip's random-byte streams). Real near-duplicate EXPERIENCES
|
|
890
|
-
// are caught two cheaper, exact ways instead: identical content by the
|
|
891
|
-
// exact-dedup hash-cons above, and a near-gist target still in the write
|
|
892
|
-
// buffer by the scan below.
|
|
893
|
-
if (leaf === null) {
|
|
894
|
-
// Near-dedup PREFILTER — the scale-aware identity bar
|
|
895
|
-
// ({@link identityBar}) for THIS branch's own length, not the fixed
|
|
896
|
-
// estimator floor: under the linear fold a long branch crosses
|
|
897
|
-
// 1 − 1/√D while whole windows differ, and every such crossing paid a
|
|
898
|
-
// full differsByOneWindow byte reconstruction. Same final semantics
|
|
899
|
-
// (the byte check below still decides identity); strictly fewer byte
|
|
900
|
-
// reads. The scan runs FIRST: the branch length (Σ kids' contentLen —
|
|
901
|
-
// memoized bottom-up by the interning order itself, O(kids)) is only
|
|
902
|
-
// computed once a nearest candidate actually exists, so the hot
|
|
903
|
-
// no-candidate mint pays nothing.
|
|
904
|
-
let best = -1;
|
|
905
|
-
let bestId = null;
|
|
906
|
-
if (this._nearDedupBuf.size > 0) {
|
|
907
|
-
const g = normalize(copy(gist));
|
|
908
|
-
// Candidates are the buffered DEDUP TARGETS only — genuine whole
|
|
909
|
-
// experiences (edge/halo-bearing roots) not yet flushed.
|
|
910
|
-
for (const [id, vector] of this._nearDedupBuf) {
|
|
911
|
-
const s = dot(g, vector);
|
|
912
|
-
if (s > best) {
|
|
913
|
-
best = s;
|
|
914
|
-
bestId = id;
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
}
|
|
918
|
-
let blen = 0;
|
|
919
|
-
if (bestId !== null) {
|
|
920
|
-
for (const k of kids)
|
|
921
|
-
blen += this.contentLen(k);
|
|
922
|
-
}
|
|
923
|
-
if (bestId !== null &&
|
|
924
|
-
best >= identityBar(this.D, this._maxGroup, blen)) {
|
|
925
|
-
// Scale-aware acceptance. The cosine bar alone is scale-BLIND
|
|
926
|
-
// against a scale-DEPENDENT quantity: the hierarchical fold dilutes
|
|
927
|
-
// a localized difference faster than linearly in form size, so for
|
|
928
|
-
// deep forms ANY fixed bar below 1 is crossed by exactly the one
|
|
929
|
-
// span that distinguishes two experiences. No inversion of the
|
|
930
|
-
// deficit is trustworthy, so the bytes themselves decide: the two
|
|
931
|
-
// forms must be identical except for ONE local span of at most W
|
|
932
|
-
// bytes — the river window, the perception's own resolution quantum.
|
|
933
|
-
const W = this._maxGroup;
|
|
934
|
-
if (this.differsByOneWindow(kids, bestId, W)) {
|
|
935
|
-
if (key.length <= DEDUP_KEY_MAX)
|
|
936
|
-
cache.set(key, bestId);
|
|
937
|
-
return bestId;
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
}
|
|
941
|
-
// 3. Mint a fresh node. A FLAT branch (every kid an implicit single-byte
|
|
942
|
-
// leaf) stores its BYTES in the leaf column with a zero-length kids blob
|
|
943
|
-
// as the marker — the kid list is derived on read.
|
|
944
|
-
const id = this.mintId();
|
|
945
|
-
this._dbBeginTx();
|
|
946
|
-
const flat = kids ? flatKidsBytes(kids) : null;
|
|
947
|
-
const packed = kids && !flat ? packKids(kids) : null;
|
|
948
|
-
this._dbInsertNode(id, leaf ?? flat, packed ?? (flat ? _ZERO : null), hashOf(leaf ?? flat ?? packed));
|
|
949
|
-
// Reverse structural edge: each distinct child → this parent. Lets the graph
|
|
950
|
-
// be climbed upward in index time, with no scan of the kids blobs.
|
|
951
|
-
//
|
|
952
|
-
// Populated NATURALLY here and only here — one write per child, in the SAME
|
|
953
|
-
// mint that creates the node, inside the SAME deferred transaction as the
|
|
954
|
-
// node row, so node and kid rows are always durable together.
|
|
955
|
-
//
|
|
956
|
-
// Implicit single-byte leaves get NO parent edge: a byte belongs to nearly
|
|
957
|
-
// every branch, so its parent set is the corpus-sized hub the climb's
|
|
958
|
-
// saturation guard discards unread.
|
|
959
|
-
if (kids) {
|
|
960
|
-
// Kid rows change parent counts — a child whose parent set grows from
|
|
961
|
-
// one is no longer transparent, so every cached chain that hopped
|
|
962
|
-
// through it is stale. Which chains those are is unknowable without a
|
|
963
|
-
// reverse index, so the WHOLE cache drops (writes come in training
|
|
964
|
-
// bursts where the cache is cold anyway; queries rebuild it lazily).
|
|
965
|
-
this._chainMemo.clear();
|
|
966
|
-
for (const c of kids) {
|
|
967
|
-
if (c < 0 && c >= -256)
|
|
968
|
-
continue;
|
|
969
|
-
this._dbInsertKid(c, id);
|
|
970
|
-
// The 1→2 parent TRANSITION happens here and only here (hash-cons
|
|
971
|
-
// means an existing branch never re-inserts kid rows, so parent sets
|
|
972
|
-
// grow exclusively through fresh mints): the child just became a
|
|
973
|
-
// structural bridge between experiences — the exact set post-hoc
|
|
974
|
-
// compaction keeps — so its gist enters the reach index NOW.
|
|
975
|
-
this.promoteBridge(c);
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
if (key.length <= DEDUP_KEY_MAX)
|
|
979
|
-
cache.set(key, id);
|
|
980
|
-
if (leaf)
|
|
981
|
-
this._bytesCache.set(id, new Uint8Array(leaf));
|
|
982
|
-
else if (flat)
|
|
983
|
-
this._bytesCache.set(id, flat);
|
|
984
|
-
// Capture the gist; it is pushed into the content index lazily, the first
|
|
985
|
-
// time this node becomes a resonance target (link / pourHalo). A node that
|
|
986
|
-
// never does (a pure intermediate DAG node — ~99.5% of them) is never
|
|
987
|
-
// indexed: it costs one persistence row, no vector-index slot and no merge probe.
|
|
988
|
-
this._pendingGist.set(id, normalize(copy(gist)));
|
|
989
|
-
await this.maybeFlush();
|
|
990
|
-
return id;
|
|
991
|
-
}
|
|
992
|
-
/** Whether the byte content under `kids` and the byte content of `targetId`
|
|
993
|
-
* are identical except for ONE local span of at most `W` bytes on each side
|
|
994
|
-
* — the near dedup's byte-grain definition of a near-duplicate. A
|
|
995
|
-
* common-prefix / common-suffix trim: whatever remains after both trims is
|
|
996
|
-
* the single differing span (substitution, insertion or deletion), and both
|
|
997
|
-
* remainders must fit the budget. Scattered differences leave a wide
|
|
998
|
-
* middle and are rejected. */
|
|
999
|
-
differsByOneWindow(kids, targetId, W) {
|
|
1000
|
-
const a = concat(kids.map((k) => this.bytesPrefix(k, Number.MAX_SAFE_INTEGER)));
|
|
1001
|
-
const b = this.bytesPrefix(targetId, a.length + W + 1);
|
|
1002
|
-
if (Math.abs(a.length - b.length) > W)
|
|
1003
|
-
return false;
|
|
1004
|
-
const n = Math.min(a.length, b.length);
|
|
1005
|
-
let i = 0;
|
|
1006
|
-
while (i < n && a[i] === b[i])
|
|
1007
|
-
i++;
|
|
1008
|
-
let j = 0;
|
|
1009
|
-
while (j < n - i && a[a.length - 1 - j] === b[b.length - 1 - j])
|
|
1010
|
-
j++;
|
|
1011
|
-
return a.length - i - j <= W && b.length - i - j <= W;
|
|
1012
|
-
}
|
|
1013
|
-
async putLeaf(bytes, gist) {
|
|
1014
|
-
// Single bytes are implicit — no DB row and no eager index slot. The gist
|
|
1015
|
-
// is captured like any other node's and promoted into the content index
|
|
1016
|
-
// LAZILY, the first time the byte becomes a resonance target.
|
|
1017
|
-
if (bytes.length === 1) {
|
|
1018
|
-
const id = -(bytes[0] + 1);
|
|
1019
|
-
this.captureIfUnindexed(id, gist);
|
|
1020
|
-
return id;
|
|
1021
|
-
}
|
|
1022
|
-
return this.intern(new Uint8Array(bytes), null, gist);
|
|
1023
|
-
}
|
|
1024
|
-
async putBranch(kids, gist) {
|
|
1025
|
-
return this.intern(null, kids, gist);
|
|
1026
|
-
}
|
|
1027
|
-
// ── Lazy content indexing ──────────────────────────────────────────────
|
|
1028
|
-
/** Promote a node's captured gist into the content (resonance) index, once.
|
|
1029
|
-
* Called the first time a node becomes a target — i.e. from `link` (it bears
|
|
1030
|
-
* or receives a continuation edge) or `pourHalo` (it gains distributional
|
|
1031
|
-
* company). Idempotent: a node already indexed, or whose gist has been evicted
|
|
1032
|
-
* from the bounded pending map, is a no-op.
|
|
1033
|
-
*
|
|
1034
|
-
* `dedupTarget` marks the node a candidate the near dedup may fold a fresh
|
|
1035
|
-
* near-gist branch ONTO. Only a genuine target — an edge/halo-bearing ROOT —
|
|
1036
|
-
* is one; a climb-only interior is reach-indexed but never a dedup sink. */
|
|
1037
|
-
indexGist(id, dedupTarget) {
|
|
1038
|
-
if (dedupTarget && this._bufferedIds.has(id)) {
|
|
1039
|
-
// Still buffered (indexed this batch, not yet flushed) — a live dedup
|
|
1040
|
-
// candidate, recorded in O(1) with no scan of the content buffer.
|
|
1041
|
-
const v = this._pendingGist.get(id);
|
|
1042
|
-
if (v !== undefined)
|
|
1043
|
-
this._nearDedupBuf.set(id, v);
|
|
1044
|
-
}
|
|
1045
|
-
if (this._indexedIds.has(id))
|
|
1046
|
-
return;
|
|
1047
|
-
const v = this._pendingGist.get(id);
|
|
1048
|
-
if (v === undefined)
|
|
1049
|
-
return;
|
|
1050
|
-
// Already durably indexed by a previous session? A node id names its
|
|
1051
|
-
// content, and the gist is a pure function of the content, so the stored
|
|
1052
|
-
// vector can only be identical — re-buffering it would spend an encode
|
|
1053
|
-
// and an upsert on a guaranteed no-op. One point query recognises this;
|
|
1054
|
-
// it costs ~nothing for genuinely new nodes (a miss on a covering index).
|
|
1055
|
-
// This is what makes a RESUMED training run replay already-deposited
|
|
1056
|
-
// content at read speed instead of re-upserting the recent-id window.
|
|
1057
|
-
if (this._vecContentHas(id)) {
|
|
1058
|
-
this._indexedIds.set(id, true);
|
|
1059
|
-
return;
|
|
1060
|
-
}
|
|
1061
|
-
this._indexedIds.set(id, true);
|
|
1062
|
-
this._contentBuffer.push({ id, vector: v });
|
|
1063
|
-
this._bufferedIds.add(id);
|
|
1064
|
-
// A node indexed AS a dedup target enters the candidate set immediately.
|
|
1065
|
-
if (dedupTarget)
|
|
1066
|
-
this._nearDedupBuf.set(id, v);
|
|
1067
|
-
}
|
|
1068
|
-
/** {@link Store.indexTarget} — the public hook for marking a deposit root a
|
|
1069
|
-
* resonance target, the one target `link`/`pourHalo` do not cover. A deposit
|
|
1070
|
-
* root is a genuine target (a whole experience), so it is a dedup target
|
|
1071
|
-
* too. */
|
|
1072
|
-
indexTarget(id) {
|
|
1073
|
-
this.indexGist(id, true);
|
|
1074
|
-
}
|
|
1075
|
-
/** Index a node and its interior forms as resonance targets. A node that
|
|
1076
|
-
* gains an edge is a learnt EXPERIENCE, and the consensus climb
|
|
1077
|
-
* ({@link Mind.climbAttention}) answers a query naming only a PORTION of it by
|
|
1078
|
-
* resonating its SUB-REGIONS — branch nodes within the experience — and
|
|
1079
|
-
* climbing their parents back to it.
|
|
1080
|
-
*
|
|
1081
|
-
* EVERY interior branch is indexed unconditionally, and this is
|
|
1082
|
-
* LOAD-BEARING: indexing only structural bridges (nodes with ≥2 parents,
|
|
1083
|
-
* the post-hoc compaction criterion) was tried and REJECTED by the test
|
|
1084
|
-
* suite — partial recall of an experience's interior slices, multi-topic
|
|
1085
|
-
* attention, and counterfactual anchoring all resonate to SINGLE-parent
|
|
1086
|
-
* interiors (13 tests fail without them). Post-hoc structural compaction
|
|
1087
|
-
* ({@link compactContentIndex}) may still remove them, but that is a
|
|
1088
|
-
* storage/recall trade-off for archived stores, not a free optimisation.
|
|
1089
|
-
* The store's hash-cons bounds the index by the number of DISTINCT byte
|
|
1090
|
-
* patterns in the corpus — not by the number of deposits.
|
|
1091
|
-
*
|
|
1092
|
-
* Only the ROOT is a DEDUP TARGET — the whole experience a fresh near-gist
|
|
1093
|
-
* branch may legitimately fold onto. Interior nodes are REACH-ONLY: they
|
|
1094
|
-
* let a partial query resonate and climb, but a fresh branch must never
|
|
1095
|
-
* merge onto an interior node of another experience.
|
|
1096
|
-
*
|
|
1097
|
-
* Iterative explicit-queue walk: the call stack never sees tree depth. */
|
|
1098
|
-
indexSubtree(root) {
|
|
1099
|
-
// The root is the whole experience — always index as a merge target.
|
|
1100
|
-
this.indexGist(root, true);
|
|
1101
|
-
const seen = new Set([root]);
|
|
1102
|
-
const stack = [root];
|
|
1103
|
-
while (stack.length > 0) {
|
|
1104
|
-
const id = stack.pop();
|
|
1105
|
-
const kids = this.get(id)?.kids;
|
|
1106
|
-
if (!kids)
|
|
1107
|
-
continue; // leaf — never a resonance anchor
|
|
1108
|
-
const isRoot = id === root;
|
|
1109
|
-
// Already-indexed or already-skipped nodes PRUNE the walk — their
|
|
1110
|
-
// subtrees were classified in a prior call.
|
|
1111
|
-
if (!isRoot && (this._indexedIds.has(id) || this._coveredIds.has(id))) {
|
|
1112
|
-
continue;
|
|
1113
|
-
}
|
|
1114
|
-
for (const k of kids) {
|
|
1115
|
-
if (!seen.has(k)) {
|
|
1116
|
-
seen.add(k);
|
|
1117
|
-
stack.push(k);
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
if (isRoot)
|
|
1121
|
-
continue;
|
|
1122
|
-
const g = this._pendingGist.get(id);
|
|
1123
|
-
if (g === undefined) {
|
|
1124
|
-
// Gist evicted from the bounded pending cache — the node can't be
|
|
1125
|
-
// indexed now. Mark as covered so subsequent visits prune here
|
|
1126
|
-
// instead of re-walking the subtree (the gist won't return).
|
|
1127
|
-
this._coveredIds.set(id, true);
|
|
1128
|
-
continue;
|
|
1129
|
-
}
|
|
1130
|
-
// Index unconditionally — every interior node is a valid resonance
|
|
1131
|
-
// anchor for partial-query recall. The _indexedIds cache and the
|
|
1132
|
-
// durable-index check in indexGist prevent duplicate indexing across
|
|
1133
|
-
// multiple encounters of the same shared subtree.
|
|
1134
|
-
this.indexGist(id, false);
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
// ── Soft resonance ─────────────────────────────────────────────────────
|
|
1138
|
-
async resonate(v, k) {
|
|
1139
|
-
await this._ensureReady();
|
|
1140
|
-
// Synchronous flush of any buffered index writes: the FIRST resonance
|
|
1141
|
-
// after a large ingest pays that flush here, so it shows up in respond
|
|
1142
|
-
// latency, not ingest latency — correct behaviour, skewed attribution;
|
|
1143
|
-
// profile accordingly.
|
|
1144
|
-
this.flushContent();
|
|
1145
|
-
if (k <= 0)
|
|
1146
|
-
return [];
|
|
1147
|
-
// ANN read cache — content-addressed so a fresh Float32Array with the
|
|
1148
|
-
// same values still hits. Lazy-init: null after any index write; the
|
|
1149
|
-
// first miss after a flush recreates it. When voteRegions resonates
|
|
1150
|
-
// identical perceived sub-regions, only the first call descends the ANN.
|
|
1151
|
-
const rk = vecKey(v) + ":" + k;
|
|
1152
|
-
const cache = this._resonateCache;
|
|
1153
|
-
if (cache) {
|
|
1154
|
-
const hit = cache.get(rk);
|
|
1155
|
-
if (hit !== undefined)
|
|
1156
|
-
return hit;
|
|
1157
|
-
}
|
|
1158
|
-
const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecContentClusterCount()));
|
|
1159
|
-
const out = [];
|
|
1160
|
-
for (const r of results) {
|
|
1161
|
-
const id = r.id;
|
|
1162
|
-
out.push({ id, score: 1 - r.distance });
|
|
1163
|
-
if (out.length >= k)
|
|
1164
|
-
break;
|
|
1165
|
-
}
|
|
1166
|
-
const rc = this._resonateCache ??= new Map();
|
|
1167
|
-
if (rc.size >= RESONATE_CACHE_MAX)
|
|
1168
|
-
rc.clear();
|
|
1169
|
-
rc.set(rk, out);
|
|
1170
|
-
return out;
|
|
1171
|
-
}
|
|
1172
|
-
indexedVectorCount() {
|
|
1173
|
-
this.flushContent();
|
|
1174
|
-
return this._vecContentSize();
|
|
1175
|
-
}
|
|
1176
|
-
lastResonateReads() {
|
|
1177
|
-
return this._vecContentLastReads();
|
|
1178
|
-
}
|
|
1179
|
-
// ── Content index compaction ────────────────────────────────────────────
|
|
1180
|
-
/** How many physical compaction attempts have failed this session. Zero in
|
|
1181
|
-
* a healthy store; a growing count means tombstones are accumulating and
|
|
1182
|
-
* index query cost is drifting up (the first failure also warns once). */
|
|
1183
|
-
compactFailures = 0;
|
|
1184
|
-
/** Meta key holding the incremental scan watermark of
|
|
1185
|
-
* {@link compactContentIndex}: "minParents:maxInternalIdScanned". KEEP
|
|
1186
|
-
* decisions are MONOTONE — parents, edges and halos only ever grow, so an
|
|
1187
|
-
* entry once kept can never become removable — and removed entries are
|
|
1188
|
-
* gone, so a pass only ever needs to examine entries indexed AFTER the
|
|
1189
|
-
* previous pass. Internal ids are monotone and survive the index's
|
|
1190
|
-
* splice compaction; the watermark is reset whenever a PHYSICAL index
|
|
1191
|
-
* compaction runs (id reuse after a dropped top row would otherwise hide
|
|
1192
|
-
* new entries behind it). */
|
|
1193
|
-
static COMPACT_WATERMARK_KEY = "contentCompact.watermark";
|
|
1194
|
-
/** {@link Store.compactContentIndex} */
|
|
1195
|
-
async compactContentIndex(minParents = 2) {
|
|
1196
|
-
await this._ensureReady();
|
|
1197
|
-
this.flush(); // commit any pending writes first
|
|
1198
|
-
// Incremental scan: resume from the last pass's watermark when its
|
|
1199
|
-
// minParents matches; a changed criterion forces a full rescan.
|
|
1200
|
-
let after = 0;
|
|
1201
|
-
{
|
|
1202
|
-
const raw = this._dbGetMeta(AbstractStore.COMPACT_WATERMARK_KEY);
|
|
1203
|
-
if (raw !== null) {
|
|
1204
|
-
const sep = raw.indexOf(":");
|
|
1205
|
-
const mp = Number(raw.slice(0, sep));
|
|
1206
|
-
const wm = Number(raw.slice(sep + 1));
|
|
1207
|
-
if (mp === minParents && Number.isFinite(wm) && wm > 0)
|
|
1208
|
-
after = wm;
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
// The keep criterion "has edges or a halo" as ONE sorted id set (edge
|
|
1212
|
-
// sources ∪ edge targets ∪ halo rows), materialised by a single C-side
|
|
1213
|
-
// scan. A binary-search membership probe replaces the three per-entry
|
|
1214
|
-
// point queries that made this stage minutes long on a trained store.
|
|
1215
|
-
const targets = this._dbEdgeOrHaloIds();
|
|
1216
|
-
const isTarget = (id) => {
|
|
1217
|
-
let lo = 0, hi = targets.length - 1;
|
|
1218
|
-
while (lo <= hi) {
|
|
1219
|
-
const mid = (lo + hi) >> 1;
|
|
1220
|
-
const v = targets[mid];
|
|
1221
|
-
if (v === id)
|
|
1222
|
-
return true;
|
|
1223
|
-
if (v < id)
|
|
1224
|
-
lo = mid + 1;
|
|
1225
|
-
else
|
|
1226
|
-
hi = mid - 1;
|
|
1227
|
-
}
|
|
1228
|
-
return false;
|
|
1229
|
-
};
|
|
1230
|
-
// Collect ids to remove: nodes that are structurally isolated (few
|
|
1231
|
-
// parents), not edge-bearing, and not halo-bearing. These nodes are
|
|
1232
|
-
// unique to one experience tree — they bridge nothing and their index
|
|
1233
|
-
// slots are wasted. Multi-parent nodes are the structural bridges
|
|
1234
|
-
// that let a partial query climb from one experience to another.
|
|
1235
|
-
const toRemove = [];
|
|
1236
|
-
let scanned = 0;
|
|
1237
|
-
let watermark = after;
|
|
1238
|
-
for (const { ext: id, internal } of this._vecContentEntriesSince(after)) {
|
|
1239
|
-
// Yield a real event-loop turn on a fixed cadence: this scan runs over
|
|
1240
|
-
// every unexamined indexed entry, and point probes alone pin the
|
|
1241
|
-
// thread otherwise (the "frozen" symptom).
|
|
1242
|
-
if (++scanned % 8192 === 0)
|
|
1243
|
-
await yieldToEventLoop();
|
|
1244
|
-
watermark = internal;
|
|
1245
|
-
if (isTarget(id))
|
|
1246
|
-
continue;
|
|
1247
|
-
if (this._dbGetParentsFirst(id, minParents).length >= minParents) {
|
|
1248
|
-
continue;
|
|
1249
|
-
}
|
|
1250
|
-
toRemove.push(id);
|
|
1251
|
-
}
|
|
1252
|
-
// Delete in batches — each batch is ONE vector-store transaction (one WAL
|
|
1253
|
-
// commit), with an event-loop turn between batches.
|
|
1254
|
-
const BATCH = 1000;
|
|
1255
|
-
for (let i = 0; i < toRemove.length; i += BATCH) {
|
|
1256
|
-
const batch = toRemove.slice(i, i + BATCH);
|
|
1257
|
-
this._vecContentDeleteMany(batch);
|
|
1258
|
-
// Purge the "already indexed" cache for removed ids: without this, a
|
|
1259
|
-
// node that LATER becomes a resonance target again (gains an edge or
|
|
1260
|
-
// halo) hits the stale cache entry in indexGist and is silently never
|
|
1261
|
-
// re-indexed for the rest of the session.
|
|
1262
|
-
for (const id of batch)
|
|
1263
|
-
this._indexedIds.delete(id);
|
|
1264
|
-
await yieldToEventLoop();
|
|
1265
|
-
}
|
|
1266
|
-
// Persist the scan watermark so the next pass starts where this one
|
|
1267
|
-
// ended (kept beside the criterion that produced it).
|
|
1268
|
-
this._dbBeginTx();
|
|
1269
|
-
this._dbSetMeta(AbstractStore.COMPACT_WATERMARK_KEY, minParents + ":" + watermark);
|
|
1270
|
-
this._dbCommitTx();
|
|
1271
|
-
if (toRemove.length > 0) {
|
|
1272
|
-
// Compact to physically reclaim space from tombstones.
|
|
1273
|
-
try {
|
|
1274
|
-
if (this._vecContentPhysicalSize() > this._vecContentSize() * 1.5) {
|
|
1275
|
-
this._vecContentCompact();
|
|
1276
|
-
// Physical compaction may free the top internal id for reuse —
|
|
1277
|
-
// drop the watermark so the next pass rescans from the start.
|
|
1278
|
-
this._dbBeginTx();
|
|
1279
|
-
this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
|
|
1280
|
-
this._dbCommitTx();
|
|
1281
|
-
}
|
|
1282
|
-
}
|
|
1283
|
-
catch (e) {
|
|
1284
|
-
// Best-effort, but never SILENT: a persistently failing compaction
|
|
1285
|
-
// lets tombstones accumulate and query cost grow with no signal.
|
|
1286
|
-
this.compactFailures++;
|
|
1287
|
-
if (this.compactFailures === 1) {
|
|
1288
|
-
console.warn("sema: content-index compaction failed (will keep " +
|
|
1289
|
-
"counting in store.compactFailures):", e);
|
|
1290
|
-
}
|
|
1291
|
-
}
|
|
1292
|
-
// Index mutations (delete, compact) invalidate the ANN read cache.
|
|
1293
|
-
this._resonateCache = null;
|
|
1294
|
-
}
|
|
1295
|
-
return toRemove.length;
|
|
1296
|
-
}
|
|
1297
|
-
/** {@link Store.repairContentIndex} */
|
|
1298
|
-
async repairContentIndex(regenerateGist, minParents = 2) {
|
|
1299
|
-
await this._ensureReady();
|
|
1300
|
-
this.flush(); // commit any pending writes first
|
|
1301
|
-
if (this._nextId === 0)
|
|
1302
|
-
return 0;
|
|
1303
|
-
let added = 0;
|
|
1304
|
-
// A repairable node MUST carry edges or a halo, so the candidate set IS
|
|
1305
|
-
// the edge/halo id set — corpus-of-experiences-sized (hundreds of
|
|
1306
|
-
// thousands), not node-count-sized (tens of millions). The old walk
|
|
1307
|
-
// visited EVERY branch id ever minted with a parent probe each; driving
|
|
1308
|
-
// from the target set visits only real candidates, in the same ascending
|
|
1309
|
-
// id order (the set is sorted), so the result and its ordering are
|
|
1310
|
-
// identical. Byte leaves (negative ids) were never visited by the old
|
|
1311
|
-
// walk and are skipped here too.
|
|
1312
|
-
const targets = this._dbEdgeOrHaloIds();
|
|
1313
|
-
let scanned = 0;
|
|
1314
|
-
for (const id of targets) {
|
|
1315
|
-
// Yield a real event-loop turn on a fixed cadence so the scan never
|
|
1316
|
-
// pins the thread for its whole duration.
|
|
1317
|
-
if (++scanned % 8192 === 0)
|
|
1318
|
-
await yieldToEventLoop();
|
|
1319
|
-
if (id < 0)
|
|
1320
|
-
continue; // byte leaves: implicit, never repaired
|
|
1321
|
-
// Already indexed in memory — skip.
|
|
1322
|
-
if (this._indexedIds.has(id))
|
|
1323
|
-
continue;
|
|
1324
|
-
// Must be a structural bridge: ≥ minParents parents in the DAG.
|
|
1325
|
-
// One LIMITed probe per candidate.
|
|
1326
|
-
if (this._dbGetParentsFirst(id, minParents).length < minParents) {
|
|
1327
|
-
continue;
|
|
1328
|
-
}
|
|
1329
|
-
// Already durably indexed by a previous session — record and skip.
|
|
1330
|
-
// Probed AFTER the structural filters: candidates are few, so this
|
|
1331
|
-
// point query runs rarely instead of once per node in the store.
|
|
1332
|
-
if (this._vecContentHas(id)) {
|
|
1333
|
-
this._indexedIds.set(id, true);
|
|
1334
|
-
continue;
|
|
1335
|
-
}
|
|
1336
|
-
// Regenerate the gist from bytes. The callback is async (the Mind's
|
|
1337
|
-
// perception is synchronous, but the interface allows a disk-backed
|
|
1338
|
-
// regenerator that yields).
|
|
1339
|
-
const gist = await regenerateGist(id);
|
|
1340
|
-
if (!gist)
|
|
1341
|
-
continue;
|
|
1342
|
-
// Index it — same code path as indexGist, but the vector is injected
|
|
1343
|
-
// directly rather than read from the (empty) pending-gist cache.
|
|
1344
|
-
this._indexedIds.set(id, true);
|
|
1345
|
-
this._contentBuffer.push({ id, vector: gist });
|
|
1346
|
-
this._bufferedIds.add(id);
|
|
1347
|
-
// Repaired nodes are reach-indexed, never dedup targets: their gist is
|
|
1348
|
-
// regenerated (may differ numerically from the original) and they are
|
|
1349
|
-
// interiors, not deposit roots.
|
|
1350
|
-
added++;
|
|
1351
|
-
// Flush periodically to bound the write-transaction size and yield to
|
|
1352
|
-
// the event loop, same cadence compactContentIndex uses.
|
|
1353
|
-
if (added % 1000 === 0)
|
|
1354
|
-
await this.maybeFlush();
|
|
1355
|
-
}
|
|
1356
|
-
// Final flush for the last partial batch.
|
|
1357
|
-
if (added > 0)
|
|
1358
|
-
await this.maybeFlush();
|
|
1359
|
-
return added;
|
|
1360
|
-
}
|
|
1361
|
-
// ── Continuation edges ─────────────────────────────────────────────────
|
|
1362
|
-
async link(from, to) {
|
|
1363
|
-
await this._ensureReady();
|
|
1364
|
-
// Both endpoints become learnt EXPERIENCES whose whole subtree is REACH-
|
|
1365
|
-
// indexed, because the seat is symmetric — a query may name only a PORTION of
|
|
1366
|
-
// either side and must still resonate to its interior and climb to the whole.
|
|
1367
|
-
// Only each ROOT is a MERGE target; the interiors are reach-only.
|
|
1368
|
-
this.indexSubtree(from);
|
|
1369
|
-
this.indexSubtree(to);
|
|
1370
|
-
// Flush the vectors indexSubtree just added to the buffer.
|
|
1371
|
-
// This keeps the buffer bounded and yields to the event loop.
|
|
1372
|
-
await this.maybeFlush();
|
|
1373
|
-
this._dbBeginTx();
|
|
1374
|
-
// Keep the document count exact as it grows: a source gaining its FIRST
|
|
1375
|
-
// edge is one new learnt context. One indexed point probe, and only once
|
|
1376
|
-
// the count has been materialised — before that, the lazy full count in
|
|
1377
|
-
// {@link edgeSourceCount} will see this edge anyway.
|
|
1378
|
-
if (this._edgeSrcCount >= 0 && !this._dbEdgeSrcExists(from)) {
|
|
1379
|
-
this._edgeSrcCount++;
|
|
1380
|
-
}
|
|
1381
|
-
// An edge breaks the transparency of both endpoints — drop cached chains
|
|
1382
|
-
// (same reasoning as the kid-row invalidation in put).
|
|
1383
|
-
this._chainMemo.clear();
|
|
1384
|
-
this._dbInsertEdge(from, to);
|
|
1385
|
-
}
|
|
1386
|
-
next(id) {
|
|
1387
|
-
return this._dbGetNextEdges(id);
|
|
1388
|
-
}
|
|
1389
|
-
/** {@link Store.hasNext} — one indexed point probe, never a range read. */
|
|
1390
|
-
hasNext(id) {
|
|
1391
|
-
return this._dbEdgeSrcExists(id);
|
|
1392
|
-
}
|
|
1393
|
-
prev(id) {
|
|
1394
|
-
return this._dbGetPrevEdges(id);
|
|
1395
|
-
}
|
|
1396
|
-
nextFirst(id, limit) {
|
|
1397
|
-
return this._dbGetNextEdgesFirst(id, limit);
|
|
1398
|
-
}
|
|
1399
|
-
prevFirst(id, limit) {
|
|
1400
|
-
return this._dbGetPrevEdgesFirst(id, limit);
|
|
1401
|
-
}
|
|
1402
|
-
/** {@link Store.prevCount}. Subclasses with an indexed reverse-edge count
|
|
1403
|
-
* should override; this default materialises (correct, not optimal). */
|
|
1404
|
-
prevCount(id) {
|
|
1405
|
-
return this._dbGetPrevEdges(id).length;
|
|
1406
|
-
}
|
|
1407
|
-
edgeSourceCount() {
|
|
1408
|
-
// Materialised once per session (edges written before this moment are
|
|
1409
|
-
// covered by the full count), then kept exact incrementally by link().
|
|
1410
|
-
// The old form re-ran a full COUNT(*) table scan on EVERY call just to
|
|
1411
|
-
// detect staleness — O(edges) on the recall hot path, at every IDF read.
|
|
1412
|
-
if (this._edgeSrcCount < 0) {
|
|
1413
|
-
this._edgeSrcCount = this._dbEdgeDistinctSrcCount();
|
|
1414
|
-
}
|
|
1415
|
-
return this._edgeSrcCount;
|
|
1416
|
-
}
|
|
1417
|
-
// ── Halos ──────────────────────────────────────────────────────────────
|
|
1418
|
-
haloMass(id) {
|
|
1419
|
-
const r = this._dbGetHalo(id);
|
|
1420
|
-
return r ? r.mass : 0;
|
|
1421
|
-
}
|
|
1422
|
-
halo(id) {
|
|
1423
|
-
const cached = this._haloNorm.get(id);
|
|
1424
|
-
if (cached !== undefined)
|
|
1425
|
-
return copy(cached);
|
|
1426
|
-
const r = this._dbGetHalo(id);
|
|
1427
|
-
if (!r || r.mass < this.minHaloMass)
|
|
1428
|
-
return null;
|
|
1429
|
-
const exact = this._haloExact.get(id);
|
|
1430
|
-
const v = normalize(exact ? copy(exact) : haloDecode(r.vec, this.D));
|
|
1431
|
-
this._haloNorm.set(id, v);
|
|
1432
|
-
return copy(v);
|
|
1433
|
-
}
|
|
1434
|
-
/** {@link Store.hasHalo} — MUST mirror {@link halo}'s null condition
|
|
1435
|
-
* exactly (row present AND mass ≥ minHaloMass), minus the decode. */
|
|
1436
|
-
hasHalo(id) {
|
|
1437
|
-
const r = this._dbGetHalo(id);
|
|
1438
|
-
return r !== null && r.mass >= this.minHaloMass;
|
|
1439
|
-
}
|
|
1440
|
-
async pourHalo(id, add) {
|
|
1441
|
-
await this._ensureReady();
|
|
1442
|
-
// A node with a halo is a genuine resonance target — the consensus climb
|
|
1443
|
-
// resonates a query region to it, and articulation reads its halo.
|
|
1444
|
-
this.indexGist(id, true);
|
|
1445
|
-
const r = this._dbGetHalo(id);
|
|
1446
|
-
const acc = this._haloExact.get(id) ??
|
|
1447
|
-
(r ? haloDecode(r.vec, this.D) : new Float32Array(this.D));
|
|
1448
|
-
addInto(acc, add);
|
|
1449
|
-
this._haloExact.set(id, acc);
|
|
1450
|
-
this._haloNorm.delete(id); // the normalized read cache is now stale
|
|
1451
|
-
const mass = (r?.mass ?? 0) + 1;
|
|
1452
|
-
this._dbBeginTx();
|
|
1453
|
-
this._dbUpsertHalo(id, haloEncode(acc), mass);
|
|
1454
|
-
// Re-index on a geometric schedule only (the exact halo is persisted).
|
|
1455
|
-
if (mass >= this.minHaloMass && geometricMass(mass)) {
|
|
1456
|
-
this._haloBuffer.set(id, normalize(copy(acc)));
|
|
1457
|
-
await this.maybeFlush();
|
|
1458
|
-
}
|
|
1459
|
-
}
|
|
1460
|
-
async resonateHalo(v, k) {
|
|
1461
|
-
await this._ensureReady();
|
|
1462
|
-
this.flushHalos();
|
|
1463
|
-
if (k <= 0)
|
|
1464
|
-
return [];
|
|
1465
|
-
// ANN read cache — same scheme as resonate's, but for the halo index.
|
|
1466
|
-
const rk = vecKey(v) + ":" + k;
|
|
1467
|
-
const cache = this._resonateHaloCache;
|
|
1468
|
-
if (cache) {
|
|
1469
|
-
const hit = cache.get(rk);
|
|
1470
|
-
if (hit !== undefined)
|
|
1471
|
-
return hit;
|
|
1472
|
-
}
|
|
1473
|
-
const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecHaloClusterCount()));
|
|
1474
|
-
const out = [];
|
|
1475
|
-
for (const r of results) {
|
|
1476
|
-
const id = r.id;
|
|
1477
|
-
out.push({ id, score: 1 - r.distance });
|
|
1478
|
-
if (out.length >= k)
|
|
1479
|
-
break;
|
|
1480
|
-
}
|
|
1481
|
-
const rhc = this._resonateHaloCache ??= new Map();
|
|
1482
|
-
if (rhc.size >= RESONATE_CACHE_MAX)
|
|
1483
|
-
rhc.clear();
|
|
1484
|
-
rhc.set(rk, out);
|
|
1485
|
-
return out;
|
|
1486
|
-
}
|
|
1487
|
-
// ── Buffering, flushing, compaction ────────────────────────────────────
|
|
1488
|
-
pending() {
|
|
1489
|
-
return this._contentBuffer.length + this._haloBuffer.size;
|
|
1490
|
-
}
|
|
1491
|
-
flushContent() {
|
|
1492
|
-
if (this._contentBuffer.length === 0)
|
|
1493
|
-
return 0;
|
|
1494
|
-
const batch = this._contentBuffer.splice(0);
|
|
1495
|
-
// The merge scan only consults UNFLUSHED candidates, so clear them in
|
|
1496
|
-
// lockstep with the content buffer they mirror.
|
|
1497
|
-
this._nearDedupBuf.clear();
|
|
1498
|
-
this._bufferedIds.clear();
|
|
1499
|
-
this._vecContentUpsert(batch);
|
|
1500
|
-
this._resonateCache = null;
|
|
1501
|
-
return batch.length;
|
|
1502
|
-
}
|
|
1503
|
-
flushHalos() {
|
|
1504
|
-
if (this._haloBuffer.size === 0)
|
|
1505
|
-
return 0;
|
|
1506
|
-
const batch = [...this._haloBuffer.entries()];
|
|
1507
|
-
this._haloBuffer.clear();
|
|
1508
|
-
this._vecHaloUpsert(batch.map(([id, vector]) => ({ id, vector })));
|
|
1509
|
-
this._resonateHaloCache = null;
|
|
1510
|
-
return batch.length;
|
|
1511
|
-
}
|
|
1512
|
-
/** Append the buffered containment pairs, inside the deferred transaction.
|
|
1513
|
-
* Pure appends: durable dedup lives in the adapter (the pair PK), so a
|
|
1514
|
-
* flush never reads a child's stored list back — the old packed-blob
|
|
1515
|
-
* read-merge-rewrite was O(fan-in) per touched child per flush, quadratic
|
|
1516
|
-
* over a long training run on a hot window. */
|
|
1517
|
-
flushContain() {
|
|
1518
|
-
if (this._containBuf.size === 0)
|
|
1519
|
-
return;
|
|
1520
|
-
this._dbBeginTx();
|
|
1521
|
-
for (const [child, set] of this._containBuf) {
|
|
1522
|
-
this._dbAppendContain(child, [...set]);
|
|
1523
|
-
}
|
|
1524
|
-
this._containBuf.clear();
|
|
1525
|
-
}
|
|
1526
|
-
/** Flush all three buffers; compact vector indices on a write-volume cadence. */
|
|
1527
|
-
flush() {
|
|
1528
|
-
this.flushContain();
|
|
1529
|
-
const written = this.flushContent() + this.flushHalos();
|
|
1530
|
-
// Commit the deferred write transaction on the same cadence as the vector
|
|
1531
|
-
// buffers, so node rows / edges / halos become durable in coalesced batches.
|
|
1532
|
-
this._dbCommitTx();
|
|
1533
|
-
if (written === 0)
|
|
1534
|
-
return;
|
|
1535
|
-
this._writtenSinceCompact += written;
|
|
1536
|
-
if (this._writtenSinceCompact >= this.compactEveryNWrites) {
|
|
1537
|
-
this._writtenSinceCompact = 0;
|
|
1538
|
-
try {
|
|
1539
|
-
if (this._vecContentPhysicalSize() > this._vecContentSize() * 2) {
|
|
1540
|
-
this._vecContentCompact();
|
|
1541
|
-
// Physical compaction may free the top internal id for reuse —
|
|
1542
|
-
// invalidate the incremental maintenance watermark.
|
|
1543
|
-
this._dbBeginTx();
|
|
1544
|
-
this._dbDeleteMeta(AbstractStore.COMPACT_WATERMARK_KEY);
|
|
1545
|
-
this._dbCommitTx();
|
|
1546
|
-
}
|
|
1547
|
-
if (this._vecHaloPhysicalSize() > this._vecHaloSize() * 2) {
|
|
1548
|
-
this._vecHaloCompact();
|
|
1549
|
-
}
|
|
1550
|
-
}
|
|
1551
|
-
catch (e) {
|
|
1552
|
-
// Best-effort, but never SILENT (same contract as the prune-path
|
|
1553
|
-
// compaction above): a persistently failing compaction lets
|
|
1554
|
-
// tombstones accumulate and query cost grow with no signal.
|
|
1555
|
-
this.compactFailures++;
|
|
1556
|
-
if (this.compactFailures === 1) {
|
|
1557
|
-
console.warn("sema: vector-index compaction failed (will keep " +
|
|
1558
|
-
"counting in store.compactFailures):", e);
|
|
1559
|
-
}
|
|
1560
|
-
}
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
async maybeFlush() {
|
|
1564
|
-
if (this.pending() >= this.batchSize) {
|
|
1565
|
-
this.flush();
|
|
1566
|
-
// A flush is the one HEAVY, fully-synchronous unit of the ingest path.
|
|
1567
|
-
// Every `await` elsewhere in ingestion resolves as a MICROTASK, which the
|
|
1568
|
-
// engine drains to empty before it ever services a MACROTASK — so a deposit
|
|
1569
|
-
// burst that only micro-awaits pins the single thread for its whole
|
|
1570
|
-
// duration, starving timers, pending I/O and any overlapped work. Here —
|
|
1571
|
-
// buffers empty, transaction committed, nothing mid-write — is the one
|
|
1572
|
-
// safe point to hand the event loop a real turn.
|
|
1573
|
-
await yieldToEventLoop();
|
|
1574
|
-
}
|
|
1575
|
-
}
|
|
1576
|
-
// ── Meta ───────────────────────────────────────────────────────────────
|
|
1577
|
-
async setMeta(key, val) {
|
|
1578
|
-
await this._ensureReady();
|
|
1579
|
-
this._dbBeginTx();
|
|
1580
|
-
this._dbSetMeta(key, val);
|
|
1581
|
-
}
|
|
1582
|
-
async getMeta(key) {
|
|
1583
|
-
await this._ensureReady();
|
|
1584
|
-
return this._dbGetMeta(key);
|
|
1585
|
-
}
|
|
1586
|
-
async deleteMeta(key) {
|
|
1587
|
-
await this._ensureReady();
|
|
1588
|
-
this._dbBeginTx();
|
|
1589
|
-
this._dbDeleteMeta(key);
|
|
1590
|
-
}
|
|
1591
|
-
// ── Snapshot ───────────────────────────────────────────────────────────
|
|
1592
|
-
async saveSnapshot(bytes) {
|
|
1593
|
-
await this._ensureReady();
|
|
1594
|
-
this.flush(); // commits any open write transaction + vector buffers
|
|
1595
|
-
this._dbSaveSnapshot(bytes);
|
|
1596
|
-
}
|
|
1597
|
-
async loadSnapshot() {
|
|
1598
|
-
await this._ensureReady();
|
|
1599
|
-
return this._dbLoadSnapshot();
|
|
1600
|
-
}
|
|
1601
|
-
// ── Lifecycle ──────────────────────────────────────────────────────────
|
|
1602
|
-
commit() {
|
|
1603
|
-
this.flush();
|
|
1604
|
-
}
|
|
1605
|
-
async close() {
|
|
1606
|
-
if (this.closed)
|
|
1607
|
-
return;
|
|
1608
|
-
this.closed = true;
|
|
1609
|
-
this.flush(); // commits any open write transaction + vector buffers
|
|
1610
|
-
this._dbClose();
|
|
1611
|
-
}
|
|
1612
|
-
}
|