@hviana/sema 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.d.ts +1 -0
- package/dist/example/demo.js +39 -0
- package/dist/example/train_base.d.ts +87 -0
- package/dist/example/train_base.js +2252 -0
- package/dist/src/alphabet.d.ts +7 -0
- package/dist/src/alphabet.js +33 -0
- package/dist/src/alu/src/alu.d.ts +185 -0
- package/dist/src/alu/src/alu.js +440 -0
- package/dist/src/alu/src/expr.d.ts +61 -0
- package/dist/src/alu/src/expr.js +318 -0
- package/dist/src/alu/src/index.d.ts +11 -0
- package/dist/src/alu/src/index.js +19 -0
- package/dist/src/alu/src/kernel-arith.d.ts +16 -0
- package/dist/src/alu/src/kernel-arith.js +264 -0
- package/dist/src/alu/src/kernel-bits.d.ts +19 -0
- package/dist/src/alu/src/kernel-bits.js +152 -0
- package/dist/src/alu/src/kernel-logic.d.ts +4 -0
- package/dist/src/alu/src/kernel-logic.js +60 -0
- package/dist/src/alu/src/kernel-nd.d.ts +3 -0
- package/dist/src/alu/src/kernel-nd.js +208 -0
- package/dist/src/alu/src/kernel-numeric.d.ts +54 -0
- package/dist/src/alu/src/kernel-numeric.js +366 -0
- package/dist/src/alu/src/operation.d.ts +168 -0
- package/dist/src/alu/src/operation.js +189 -0
- package/dist/src/alu/src/parser.d.ts +221 -0
- package/dist/src/alu/src/parser.js +577 -0
- package/dist/src/alu/src/resonance.d.ts +55 -0
- package/dist/src/alu/src/resonance.js +126 -0
- package/dist/src/alu/src/text.d.ts +31 -0
- package/dist/src/alu/src/text.js +73 -0
- package/dist/src/alu/src/value.d.ts +109 -0
- package/dist/src/alu/src/value.js +300 -0
- package/dist/src/alu/test/alu.test.d.ts +1 -0
- package/dist/src/alu/test/alu.test.js +764 -0
- package/dist/src/bytes.d.ts +14 -0
- package/dist/src/bytes.js +59 -0
- package/dist/src/canon.d.ts +26 -0
- package/dist/src/canon.js +57 -0
- package/dist/src/config.d.ts +111 -0
- package/dist/src/config.js +91 -0
- package/dist/src/derive/src/deduction.d.ts +125 -0
- package/dist/src/derive/src/deduction.js +155 -0
- package/dist/src/derive/src/index.d.ts +7 -0
- package/dist/src/derive/src/index.js +11 -0
- package/dist/src/derive/src/priority-queue.d.ts +20 -0
- package/dist/src/derive/src/priority-queue.js +73 -0
- package/dist/src/derive/src/rewrite.d.ts +56 -0
- package/dist/src/derive/src/rewrite.js +100 -0
- package/dist/src/derive/src/trie.d.ts +90 -0
- package/dist/src/derive/src/trie.js +217 -0
- package/dist/src/derive/test/derive.test.d.ts +1 -0
- package/dist/src/derive/test/derive.test.js +122 -0
- package/dist/src/extension.d.ts +37 -0
- package/dist/src/extension.js +7 -0
- package/dist/src/geometry.d.ts +202 -0
- package/dist/src/geometry.js +575 -0
- package/dist/src/index.d.ts +16 -0
- package/dist/src/index.js +18 -0
- package/dist/src/ingest-cache.d.ts +41 -0
- package/dist/src/ingest-cache.js +161 -0
- package/dist/src/mind/articulation.d.ts +6 -0
- package/dist/src/mind/articulation.js +99 -0
- package/dist/src/mind/attention.d.ts +405 -0
- package/dist/src/mind/attention.js +1829 -0
- package/dist/src/mind/bridge.d.ts +30 -0
- package/dist/src/mind/bridge.js +569 -0
- package/dist/src/mind/canonical.d.ts +29 -0
- package/dist/src/mind/canonical.js +88 -0
- package/dist/src/mind/graph-search.d.ts +285 -0
- package/dist/src/mind/graph-search.js +956 -0
- package/dist/src/mind/index.d.ts +7 -0
- package/dist/src/mind/index.js +5 -0
- package/dist/src/mind/junction.d.ts +124 -0
- package/dist/src/mind/junction.js +317 -0
- package/dist/src/mind/learning.d.ts +47 -0
- package/dist/src/mind/learning.js +248 -0
- package/dist/src/mind/match.d.ts +126 -0
- package/dist/src/mind/match.js +441 -0
- package/dist/src/mind/mechanisms/alu.d.ts +4 -0
- package/dist/src/mind/mechanisms/alu.js +36 -0
- package/dist/src/mind/mechanisms/cast.d.ts +89 -0
- package/dist/src/mind/mechanisms/cast.js +716 -0
- package/dist/src/mind/mechanisms/confluence.d.ts +24 -0
- package/dist/src/mind/mechanisms/confluence.js +225 -0
- package/dist/src/mind/mechanisms/cover.d.ts +6 -0
- package/dist/src/mind/mechanisms/cover.js +203 -0
- package/dist/src/mind/mechanisms/extraction.d.ts +67 -0
- package/dist/src/mind/mechanisms/extraction.js +387 -0
- package/dist/src/mind/mechanisms/recall.d.ts +13 -0
- package/dist/src/mind/mechanisms/recall.js +325 -0
- package/dist/src/mind/mind.d.ts +276 -0
- package/dist/src/mind/mind.js +572 -0
- package/dist/src/mind/pipeline-mechanism.d.ts +142 -0
- package/dist/src/mind/pipeline-mechanism.js +213 -0
- package/dist/src/mind/pipeline.d.ts +20 -0
- package/dist/src/mind/pipeline.js +217 -0
- package/dist/src/mind/primitives.d.ts +70 -0
- package/dist/src/mind/primitives.js +287 -0
- package/dist/src/mind/rationale.d.ts +139 -0
- package/dist/src/mind/rationale.js +163 -0
- package/dist/src/mind/reasoning.d.ts +34 -0
- package/dist/src/mind/reasoning.js +240 -0
- package/dist/src/mind/recognition.d.ts +20 -0
- package/dist/src/mind/recognition.js +390 -0
- package/dist/src/mind/resonance.d.ts +23 -0
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +15 -0
- package/dist/src/mind/trace.js +73 -0
- package/dist/src/mind/traverse.d.ts +113 -0
- package/dist/src/mind/traverse.js +568 -0
- package/dist/src/mind/types.d.ts +289 -0
- package/dist/src/mind/types.js +130 -0
- package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
- package/dist/src/rabitq-ivf/src/database.js +201 -0
- package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
- package/dist/src/rabitq-ivf/src/index.js +4 -0
- package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
- package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
- package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
- package/dist/src/rabitq-ivf/src/prng.js +36 -0
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
- package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
- package/dist/src/sema.d.ts +31 -0
- package/dist/src/sema.js +63 -0
- package/dist/src/store-sqlite.d.ts +184 -0
- package/dist/src/store-sqlite.js +942 -0
- package/dist/src/store.d.ts +663 -0
- package/dist/src/store.js +1618 -0
- package/dist/src/vec.d.ts +31 -0
- package/dist/src/vec.js +109 -0
- package/package.json +1 -1
- package/src/mind/attention.ts +178 -63
- package/test/51-structural-resonance-ladder.test.mjs +6 -3
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import type { Vec } from "../vec.js";
|
|
2
|
+
import type { Sema } from "../sema.js";
|
|
3
|
+
import type { BoundedMap, Store } from "../store.js";
|
|
4
|
+
import type { Space } from "../sema.js";
|
|
5
|
+
import type { Alphabet } from "../alphabet.js";
|
|
6
|
+
import type { MindConfig } from "../config.js";
|
|
7
|
+
import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
|
|
8
|
+
import type { Rationale } from "./rationale.js";
|
|
9
|
+
import type { FoldPyramid, Grid, StableFold } from "../geometry.js";
|
|
10
|
+
/** One {@link MindContext._depositTrees} entry — see that field's doc. */
|
|
11
|
+
export interface DepositCacheEntry {
|
|
12
|
+
/** Turn boundaries accumulated over this content's deposit chain —
|
|
13
|
+
* strictly increasing proper offsets, each a previously-deposited
|
|
14
|
+
* whole-context length. Empty for a first-seen (single-turn) input. */
|
|
15
|
+
boundaries: number[];
|
|
16
|
+
/** Plain-fold pyramid (first-seen inputs only). */
|
|
17
|
+
pyramid?: FoldPyramid;
|
|
18
|
+
/** Stable-prefix segment folds (grown-context inputs only). */
|
|
19
|
+
stable?: StableFold;
|
|
20
|
+
/** The continuation bytes this ctxInput was paired with in ingestPair, if
|
|
21
|
+
* any — the ONLY thing that makes a later, longer ctxInput a genuine next
|
|
22
|
+
* TURN of the same conversation rather than an unrelated fact that
|
|
23
|
+
* happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5"). A
|
|
24
|
+
* later deposit only takes this entry as its stable-prefix `prev` when
|
|
25
|
+
* its own suffix bytes-equal this exactly. */
|
|
26
|
+
nextBytes?: Uint8Array;
|
|
27
|
+
}
|
|
28
|
+
export type Input = string | Uint8Array | Grid | Grid[];
|
|
29
|
+
/** The host capabilities GraphSearch consults during a cover. MindContext
|
|
30
|
+
* extends this so the Mind can pass itself as the host. */
|
|
31
|
+
export interface GraphSearchHost {
|
|
32
|
+
resolve(bytes: Uint8Array): number | null;
|
|
33
|
+
recogniseSpan?(bytes: Uint8Array): {
|
|
34
|
+
sites: ReadonlyArray<Site>;
|
|
35
|
+
leaves: ReadonlyArray<Leaf>;
|
|
36
|
+
splits: ReadonlySet<number>;
|
|
37
|
+
starts: ReadonlySet<number>;
|
|
38
|
+
};
|
|
39
|
+
chooseNext?(node: number): number | undefined;
|
|
40
|
+
}
|
|
41
|
+
export interface Recognition {
|
|
42
|
+
/** Forms that can lead somewhere — they have an edge or a halo. */
|
|
43
|
+
sites: Site[];
|
|
44
|
+
/** The query's perceived leaves (the search's covering axioms). */
|
|
45
|
+
leaves: Leaf[];
|
|
46
|
+
/** Sub-leaf positions where a form boundary falls between leaf edges. */
|
|
47
|
+
splits: Set<number>;
|
|
48
|
+
/** Leaf-parent (chunk) start positions from the query's OWN perceived
|
|
49
|
+
* fold — the positions the fold itself chose as a grouping boundary, as
|
|
50
|
+
* opposed to an offset a byte-level scan merely happens to land on. The
|
|
51
|
+
* one boundary signal opportunistic cross-leaf recovery (recognition's
|
|
52
|
+
* own canonical chains, the search's `fuse`) can lean on instead of
|
|
53
|
+
* ASCII/word heuristics: see the `boundary` gate in recognition.ts. */
|
|
54
|
+
starts: Set<number>;
|
|
55
|
+
}
|
|
56
|
+
/** How the consensus climb weights a region's Document-Frequency reach. */
|
|
57
|
+
export type DFMode = "inverse" | "direct" | "combined";
|
|
58
|
+
/** One POINT OF ATTENTION the consensus climb resolved. */
|
|
59
|
+
export interface Attention {
|
|
60
|
+
/** The learnt context this point resolves to. */
|
|
61
|
+
anchor: number;
|
|
62
|
+
/** IDF-weighted consensus vote — the strength that orders points. */
|
|
63
|
+
vote: number;
|
|
64
|
+
/** The union of the query byte-spans whose evidence supports this point. */
|
|
65
|
+
start: number;
|
|
66
|
+
end: number;
|
|
67
|
+
/** SCALE-INVARIANT confidence: the fraction of the query's OWN regions
|
|
68
|
+
* whose evidence this point accounts for (Σ RegionVote.absorbed among
|
|
69
|
+
* its contributors, over the query's total region count) — read PER-
|
|
70
|
+
* ANCHOR, unlike the raw IDF vote (an absolute, ln(N)-scaled quantity
|
|
71
|
+
* that means "strong" on a small store and "weak" on a large one for
|
|
72
|
+
* the SAME degree of genuine consensus). A point whose breadth clears
|
|
73
|
+
* `dominates` (> half the query's regions corroborate it) is real
|
|
74
|
+
* consensus; one that does not is a coincidental single-region echo —
|
|
75
|
+
* see test/35-attention-confidence.test.mjs. */
|
|
76
|
+
breadth: number;
|
|
77
|
+
/** DISPERSION: the number of distinct clusters this point's contributing
|
|
78
|
+
* regions form, merging any two whose gap is under one river-fold
|
|
79
|
+
* quantum W. Neither breadth NOR raw region count discriminates a
|
|
80
|
+
* genuine further topic from a coincidental echo (both were tried and
|
|
81
|
+
* falsified — breadth starves a genuine, evenly-split multi-topic query,
|
|
82
|
+
* since no root in a real N-way split can exceed half the vote; raw
|
|
83
|
+
* count doesn't separate them either, since a short, structurally simple
|
|
84
|
+
* echo racks up as many corroborating regions as a real topic does).
|
|
85
|
+
* Dispersion asks a different question: not how MUCH evidence, but how
|
|
86
|
+
* many separate PLACES in the query corroborate it. A coincidental
|
|
87
|
+
* match — one local phrase resonating with an unrelated stored form —
|
|
88
|
+
* is structurally confined to ONE cluster no matter how strong its vote;
|
|
89
|
+
* a genuine further topic is named in its own distinctive wording
|
|
90
|
+
* somewhere the query's scaffolding does not reach, always a SEPARATE
|
|
91
|
+
* cluster from whatever else corroborates it. See
|
|
92
|
+
* test/37-cluster-dispersion-fusion.test.mjs. */
|
|
93
|
+
clusters: number;
|
|
94
|
+
}
|
|
95
|
+
/** Both read-outs of one consensus climb. */
|
|
96
|
+
export interface AttentionRead {
|
|
97
|
+
roots: Attention[];
|
|
98
|
+
ranked: Attention[];
|
|
99
|
+
}
|
|
100
|
+
/** A positioned region of a byte stream paired with its gist. */
|
|
101
|
+
export interface Segment {
|
|
102
|
+
start: number;
|
|
103
|
+
end: number;
|
|
104
|
+
v: Vec;
|
|
105
|
+
}
|
|
106
|
+
/** A region of the query's perceived tree for the consensus climb. */
|
|
107
|
+
export interface Region {
|
|
108
|
+
v: Vec;
|
|
109
|
+
start: number;
|
|
110
|
+
end: number;
|
|
111
|
+
chunk: boolean;
|
|
112
|
+
/** Whether the region's bytes resolve to a KNOWN node (content-addressed,
|
|
113
|
+
* exact). Exact regions vote with full weight; approximate ones pay the
|
|
114
|
+
* contrastive margin (see voteRegions) — under the linear fold a raw
|
|
115
|
+
* resonance score is byte-overlap, evidence only in excess of its best
|
|
116
|
+
* rival conclusion. */
|
|
117
|
+
known: boolean;
|
|
118
|
+
}
|
|
119
|
+
/** Per-region vote data from the consensus climb's resonance pass. */
|
|
120
|
+
export interface RegionVote {
|
|
121
|
+
start: number;
|
|
122
|
+
end: number;
|
|
123
|
+
canonicalFailed: boolean;
|
|
124
|
+
roots: readonly number[];
|
|
125
|
+
w: number;
|
|
126
|
+
wFocus: number;
|
|
127
|
+
/** How many of the query's ORIGINAL regions this one vote's evidence
|
|
128
|
+
* accounts for. 1 for an ordinary per-region vote (itself); for a
|
|
129
|
+
* cross-region junction vote, 1 (itself) plus however many individual
|
|
130
|
+
* votes it explained away (see crossRegionVotes) — the junction speaks
|
|
131
|
+
* for all of them at once, and breadth accounting must not undercount it
|
|
132
|
+
* to "one region" just because it collapsed to one pooled axiom.
|
|
133
|
+
* Defaults to 1 when absent. */
|
|
134
|
+
absorbed?: number;
|
|
135
|
+
}
|
|
136
|
+
/** The structural gate that first decided an {@link edgeAncestors} climb was
|
|
137
|
+
* saturated (an abstention, not a discriminative conclusion) — pure
|
|
138
|
+
* instrumentation for {@link ClimbConsensusData}'s reach trace; it never
|
|
139
|
+
* feeds back into the climb itself. */
|
|
140
|
+
export type SaturationReason = "byte-atom-commonality" | "predecessor-fan-in" | "distinct-context-limit" | "parent-fan-out" | "lateral-cone-limit";
|
|
141
|
+
/** One saturation stop's provenance: which reason fired, at which node, the
|
|
142
|
+
* observed count against the bound that decided it. */
|
|
143
|
+
export interface SaturationStop {
|
|
144
|
+
reason: SaturationReason;
|
|
145
|
+
node: number;
|
|
146
|
+
observed: number;
|
|
147
|
+
limit: number;
|
|
148
|
+
}
|
|
149
|
+
/** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
|
|
150
|
+
export interface AncestorReach {
|
|
151
|
+
roots: number[];
|
|
152
|
+
contextsReached: number;
|
|
153
|
+
saturated: boolean;
|
|
154
|
+
/** The saturation gate that stopped this climb, when {@link saturated} is
|
|
155
|
+
* true and a trace was requested — see {@link edgeAncestors}. Absent for
|
|
156
|
+
* a non-saturated reach, and absent (even when saturated) when no trace
|
|
157
|
+
* was requested — instrumentation must not allocate when tracing is off. */
|
|
158
|
+
saturation?: SaturationStop;
|
|
159
|
+
}
|
|
160
|
+
/** Saturated-interval information for the noise-drop gate. */
|
|
161
|
+
export interface SaturationInfo {
|
|
162
|
+
leadingEnd: number;
|
|
163
|
+
hasLeading: boolean;
|
|
164
|
+
intervals: Array<{
|
|
165
|
+
start: number;
|
|
166
|
+
end: number;
|
|
167
|
+
}>;
|
|
168
|
+
}
|
|
169
|
+
/** The items of poolVotes' deduction system. */
|
|
170
|
+
export type AItem = {
|
|
171
|
+
kind: "region";
|
|
172
|
+
ri: number;
|
|
173
|
+
} | {
|
|
174
|
+
kind: "anchor";
|
|
175
|
+
id: number;
|
|
176
|
+
} | {
|
|
177
|
+
kind: "anchorFocus";
|
|
178
|
+
id: number;
|
|
179
|
+
};
|
|
180
|
+
export interface MindContext extends GraphSearchHost {
|
|
181
|
+
store: Store;
|
|
182
|
+
space: Space;
|
|
183
|
+
alphabet: Alphabet;
|
|
184
|
+
cfg: MindConfig;
|
|
185
|
+
search: GraphSearch;
|
|
186
|
+
trace: Rationale | null;
|
|
187
|
+
/** The content canonicalizer for THIS response, or null — injected by the
|
|
188
|
+
* modality entry point (respondText passes the text canonicalizer; a
|
|
189
|
+
* binary respond passes none). Resolution uses it as a fallback: when
|
|
190
|
+
* the exact content-addressed lookup misses, the span's canonical key is
|
|
191
|
+
* probed against the store's canon index (see src/canon.ts). The core
|
|
192
|
+
* never inspects what the equivalence IS. */
|
|
193
|
+
canon: ((bytes: Uint8Array) => Uint8Array) | null;
|
|
194
|
+
/** Per-response memo of canonical-fallback resolutions, keyed by the
|
|
195
|
+
* span's latin1 content key. Null outside respond(). */
|
|
196
|
+
canonMemo: Map<string, number | null> | null;
|
|
197
|
+
/** Memo of the consensus climb — content-keyed (latin1) so results
|
|
198
|
+
* persist across conversation turns where the same byte spans recur.
|
|
199
|
+
* Null outside respond(); during respondTurn() the conversation's
|
|
200
|
+
* persistent map is swapped in. */
|
|
201
|
+
climbMemo: Map<string, Map<string, AttentionRead>> | null;
|
|
202
|
+
/** Memo of {@link recognise} — content-keyed (latin1) so recognised
|
|
203
|
+
* forms carry forward across conversation turns. Bypassed while a
|
|
204
|
+
* trace is attached. Null outside respond(). */
|
|
205
|
+
recogniseMemo: Map<string, Recognition> | null;
|
|
206
|
+
/** Memo of {@link perceive} — content-keyed (latin1). The general
|
|
207
|
+
* cache the result-level memos each partially compensate for. NOT
|
|
208
|
+
* bypassed under trace — perception emits no rationale steps.
|
|
209
|
+
* Null outside respond(). */
|
|
210
|
+
perceiveMemo: Map<string, Sema> | null;
|
|
211
|
+
/** Subtree-resolution cache: Sema node → its store id and byte length.
|
|
212
|
+
* Populated by {@link foldTree} during inference; checked before
|
|
213
|
+
* walking children. When a conversation's pyramid reuses prefix
|
|
214
|
+
* subtrees, this cache lets {@link recognise} skip them entirely —
|
|
215
|
+
* O(suffix) instead of O(context). Mind-lifetime (WeakMap keys are
|
|
216
|
+
* the Sema objects the pyramid keeps alive). */
|
|
217
|
+
_resolvedSubtrees: WeakMap<Sema, {
|
|
218
|
+
id: number;
|
|
219
|
+
len: number;
|
|
220
|
+
}> | null;
|
|
221
|
+
_edgeGuide: Vec | null;
|
|
222
|
+
_edgeChoice: Map<number, number>;
|
|
223
|
+
_prevSeen: Set<number> | null;
|
|
224
|
+
/** Session cache of node-id → perceived gist, for candidate scoring
|
|
225
|
+
* ({@link chooseAmong} in the reverse projection's recall path re-gists up to
|
|
226
|
+
* √N contexts per pick — the measured bottleneck there). `chooseNext` does
|
|
227
|
+
* NOT use this cache; forward-edge disambiguation uses prevOf counts
|
|
228
|
+
* (distributional evidence) instead of gist comparison, because for short
|
|
229
|
+
* answer candidates the gist is dominated by accidental byte-pattern
|
|
230
|
+
* correlations. A node's bytes are immutable and perception is a pure
|
|
231
|
+
* function of bytes, so an entry stays valid for the store's lifetime —
|
|
232
|
+
* never invalidated. Bounded LRU (byte-sized); a miss only re-perceives,
|
|
233
|
+
* never a correctness risk. */
|
|
234
|
+
_gistCache: BoundedMap<number, Vec>;
|
|
235
|
+
/** DEPOSIT-path perception cache: content key (latin1) of a deposited
|
|
236
|
+
* input → its accumulated turn BOUNDARIES plus reusable fold state. A
|
|
237
|
+
* deposit whose content extends a cached entry IS a conversation context
|
|
238
|
+
* grown by one turn — the cached length is the new boundary — so it
|
|
239
|
+
* folds with the SAME stable-prefix fold query-time perception uses
|
|
240
|
+
* (structural train/inference agreement, load-bearing for recall),
|
|
241
|
+
* reusing every already-folded segment via `stable` (see StableFold) —
|
|
242
|
+
* O(turn) per deposit instead of O(context). A first-seen input keeps
|
|
243
|
+
* the plain fold and caches its `pyramid` (see FoldPyramid). Purely a
|
|
244
|
+
* performance cache for the FOLD STATE; the boundaries are semantic but
|
|
245
|
+
* derived only from the deposit sequence itself (an evicted chain falls
|
|
246
|
+
* back to plain-fold behavior, exactly the pre-boundary shape). */
|
|
247
|
+
_depositTrees: BoundedMap<string, DepositCacheEntry>;
|
|
248
|
+
/** The byte lengths present in {@link _depositTrees} — the candidate
|
|
249
|
+
* prefix lengths probed (longest first). Drifts on eviction (a stale
|
|
250
|
+
* length only costs a miss); cleared with the map when it outgrows the
|
|
251
|
+
* probe budget. */
|
|
252
|
+
_depositLens: Set<number>;
|
|
253
|
+
/** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
|
|
254
|
+
* content-addressed id. Valid forever (ids are permanent, Sema nodes
|
|
255
|
+
* immutable); WeakMap, so entries live exactly as long as the pyramid
|
|
256
|
+
* cache keeps the shared subtrees alive. Lets internTreeIds skip whole
|
|
257
|
+
* shared subtrees and indexSubSpans keep its seenBefore window skip. */
|
|
258
|
+
_internIds: WeakMap<Sema, number>;
|
|
259
|
+
}
|
|
260
|
+
/** Read a whole node's bytes. */
|
|
261
|
+
export declare const ALL = 2147483647;
|
|
262
|
+
/** Splice every chosen span in order — the whole cover as one byte string. */
|
|
263
|
+
export declare function spliceAll(segs: Seg[]): Uint8Array | null;
|
|
264
|
+
/** Whether a chosen span RESTATES the query rather than answering it: its
|
|
265
|
+
* SUBSTITUTED bytes (an edge followed from a recognised site, not the
|
|
266
|
+
* site's own literal text read back) already occur elsewhere in the query
|
|
267
|
+
* — the same principle recall.ts's tiers apply to a whole-query projection
|
|
268
|
+
* ("a projection that is a proper byte-subspan of the query restates part
|
|
269
|
+
* of the question"). A LITERAL span (the site's own bytes, unchanged) is
|
|
270
|
+
* exempt: naming what's already there at its OWN position is not a
|
|
271
|
+
* substitution. A recognised site that is itself an entire PRIOR TURN of
|
|
272
|
+
* a multi-turn query is exactly this shape: it carries a genuine learnt
|
|
273
|
+
* continuation, but that continuation is something the asker already said
|
|
274
|
+
* moments later in the SAME query, not a new answer. Below one river
|
|
275
|
+
* window, byte overlap is chance, not evidence — the same floor
|
|
276
|
+
* identityBar and reachThreshold hold every other structural-overlap claim
|
|
277
|
+
* to. */
|
|
278
|
+
export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: number, W: number): boolean;
|
|
279
|
+
/** Lift the answer out of the cover for think: the recognised region, free of
|
|
280
|
+
* the asker's surrounding (unrecognised) framing — and free of any chosen
|
|
281
|
+
* span that only RESTATES content the query already contains (see {@link
|
|
282
|
+
* segRestatesQuery}). A restating span is excluded from both the framing
|
|
283
|
+
* (lo/hi) decision and the final concatenation: it is stale, not a second
|
|
284
|
+
* answer, but the OTHER spans a derivation chose are independent evidence
|
|
285
|
+
* and must not be discarded along with it. */
|
|
286
|
+
export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
|
|
287
|
+
/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
|
|
288
|
+
* tracked deposit interned (`prevSeen`). */
|
|
289
|
+
export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// types.ts — all interfaces, types, and free functions for the mind.
|
|
2
|
+
//
|
|
3
|
+
// GraphSearchHost is defined first (minimal imports) so GraphSearch can import
|
|
4
|
+
// it without pulling in the full MindContext.
|
|
5
|
+
import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
|
|
6
|
+
import { dominates } from "../geometry.js";
|
|
7
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
8
|
+
// FREE FUNCTIONS (pure, no state)
|
|
9
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
10
|
+
/** Read a whole node's bytes. */
|
|
11
|
+
export const ALL = 0x7fffffff;
|
|
12
|
+
/** Splice every chosen span in order — the whole cover as one byte string. */
|
|
13
|
+
export function spliceAll(segs) {
|
|
14
|
+
if (!segs.some((s) => s.rec))
|
|
15
|
+
return null;
|
|
16
|
+
return concatBytes(segs.map((s) => s.bytes));
|
|
17
|
+
}
|
|
18
|
+
/** Whether a chosen span RESTATES the query rather than answering it: its
|
|
19
|
+
* SUBSTITUTED bytes (an edge followed from a recognised site, not the
|
|
20
|
+
* site's own literal text read back) already occur elsewhere in the query
|
|
21
|
+
* — the same principle recall.ts's tiers apply to a whole-query projection
|
|
22
|
+
* ("a projection that is a proper byte-subspan of the query restates part
|
|
23
|
+
* of the question"). A LITERAL span (the site's own bytes, unchanged) is
|
|
24
|
+
* exempt: naming what's already there at its OWN position is not a
|
|
25
|
+
* substitution. A recognised site that is itself an entire PRIOR TURN of
|
|
26
|
+
* a multi-turn query is exactly this shape: it carries a genuine learnt
|
|
27
|
+
* continuation, but that continuation is something the asker already said
|
|
28
|
+
* moments later in the SAME query, not a new answer. Below one river
|
|
29
|
+
* window, byte overlap is chance, not evidence — the same floor
|
|
30
|
+
* identityBar and reachThreshold hold every other structural-overlap claim
|
|
31
|
+
* to. */
|
|
32
|
+
export function segRestatesQuery(s, query, queryLen, W) {
|
|
33
|
+
if (!s.rec)
|
|
34
|
+
return false;
|
|
35
|
+
const literal = s.j - s.i === s.bytes.length &&
|
|
36
|
+
bytesEqual(s.bytes, query.subarray(s.i, s.j));
|
|
37
|
+
if (literal)
|
|
38
|
+
return false;
|
|
39
|
+
return s.bytes.length >= W && s.bytes.length < queryLen &&
|
|
40
|
+
indexOf(query, s.bytes, 0) >= 0;
|
|
41
|
+
}
|
|
42
|
+
/** Lift the answer out of the cover for think: the recognised region, free of
|
|
43
|
+
* the asker's surrounding (unrecognised) framing — and free of any chosen
|
|
44
|
+
* span that only RESTATES content the query already contains (see {@link
|
|
45
|
+
* segRestatesQuery}). A restating span is excluded from both the framing
|
|
46
|
+
* (lo/hi) decision and the final concatenation: it is stale, not a second
|
|
47
|
+
* answer, but the OTHER spans a derivation chose are independent evidence
|
|
48
|
+
* and must not be discarded along with it. */
|
|
49
|
+
export function liftAnswer(segs, queryLen, query, W) {
|
|
50
|
+
const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
|
|
51
|
+
const recognised = [];
|
|
52
|
+
for (let k = 0; k < segs.length; k++) {
|
|
53
|
+
if (segs[k].rec && !restated[k])
|
|
54
|
+
recognised.push(k);
|
|
55
|
+
}
|
|
56
|
+
if (recognised.length === 0)
|
|
57
|
+
return null;
|
|
58
|
+
if (recognised.length === 1) {
|
|
59
|
+
const s = segs[recognised[0]];
|
|
60
|
+
// A COMPUTED span's query-side width is operand digit-count, not
|
|
61
|
+
// evidence of how much of the query's meaning it accounts for — the
|
|
62
|
+
// half-dominance check below (built for a genuinely RECOGNISED learned
|
|
63
|
+
// form) is not a valid framing signal for it (see the `computed` field
|
|
64
|
+
// doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
|
|
65
|
+
// because the operands are big, not because the framing matters less.
|
|
66
|
+
// A LITERAL PREFIX before a computed span is unambiguous framing
|
|
67
|
+
// regardless of width — an arithmetic expression is never itself
|
|
68
|
+
// preceded by more literal computed content, so anything literal before
|
|
69
|
+
// it is question wording ("what is ", "compute ") to lift clear of.
|
|
70
|
+
// With no prefix (s.i === 0) the span is judged by the ordinary
|
|
71
|
+
// half-dominance rule below, which already correctly keeps a short
|
|
72
|
+
// trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
|
|
73
|
+
if (s.computed && s.i > 0)
|
|
74
|
+
return s.bytes;
|
|
75
|
+
if (dominates(s.j - s.i, queryLen)) {
|
|
76
|
+
return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
|
|
77
|
+
}
|
|
78
|
+
return s.bytes;
|
|
79
|
+
}
|
|
80
|
+
const lo = recognised[0];
|
|
81
|
+
const hi = recognised[recognised.length - 1];
|
|
82
|
+
return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
|
|
83
|
+
}
|
|
84
|
+
/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
|
|
85
|
+
* tracked deposit interned (`prevSeen`). */
|
|
86
|
+
export function changedNodes(tree, ids, prevSeen) {
|
|
87
|
+
const newCount = new Map();
|
|
88
|
+
const count = (n) => {
|
|
89
|
+
const memo = newCount.get(n);
|
|
90
|
+
if (memo !== undefined)
|
|
91
|
+
return memo;
|
|
92
|
+
const id = ids.get(n);
|
|
93
|
+
// PRUNE: a node whose id the previous deposit already interned is old,
|
|
94
|
+
// and content addressing makes that transitive — the same id names the
|
|
95
|
+
// same content, so every descendant was interned then too. The whole
|
|
96
|
+
// subtree counts 0 without walking it; with the pyramid fold sharing a
|
|
97
|
+
// conversation's prefix subtree, this is what keeps the changed-nodes
|
|
98
|
+
// read O(new nodes) instead of O(context). (A node internTreeIds
|
|
99
|
+
// memo-skipped has an id here exactly when it is such a shared root.)
|
|
100
|
+
if (id !== undefined && prevSeen.has(id)) {
|
|
101
|
+
newCount.set(n, 0);
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
let c = 1; // reachable only when NOT pruned above ⇒ this node is new
|
|
105
|
+
if (n.kids) {
|
|
106
|
+
for (const k of n.kids)
|
|
107
|
+
c += count(k);
|
|
108
|
+
}
|
|
109
|
+
newCount.set(n, c);
|
|
110
|
+
return c;
|
|
111
|
+
};
|
|
112
|
+
const total = count(tree);
|
|
113
|
+
if (total === 0)
|
|
114
|
+
return [tree];
|
|
115
|
+
let n = tree;
|
|
116
|
+
for (;;) {
|
|
117
|
+
if (n.kids === null)
|
|
118
|
+
return [n];
|
|
119
|
+
let holder = null;
|
|
120
|
+
for (const k of n.kids) {
|
|
121
|
+
if (newCount.get(k) === total) {
|
|
122
|
+
holder = k;
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (holder === null)
|
|
127
|
+
return [n];
|
|
128
|
+
n = holder;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/** External ids are integers only. */
|
|
2
|
+
export type ExternalId = number;
|
|
3
|
+
export interface DatabaseOptions {
|
|
4
|
+
/** Path to the SQLite database file (use ":memory:" for a transient store). */
|
|
5
|
+
dbPath: string;
|
|
6
|
+
/** Vector dimensionality. Required for a new database; ignored when reopening. */
|
|
7
|
+
dim?: number;
|
|
8
|
+
/** Query breadth: clusters probed per query = ceil(efSearch / 4). Default
|
|
9
|
+
* 64 (16 probes). Tunable at runtime and at reopen. */
|
|
10
|
+
efSearch?: number;
|
|
11
|
+
/** Bits used to scalar-quantise the query for the RaBitQ estimator.
|
|
12
|
+
* Query-side only (stored codes are independent of it), so it is tunable
|
|
13
|
+
* at reopen, like efSearch. Default 8. */
|
|
14
|
+
queryBits?: number;
|
|
15
|
+
/** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
|
|
16
|
+
rotationRounds?: number;
|
|
17
|
+
/** Seed for the rotation. Default fixed. */
|
|
18
|
+
seed?: number;
|
|
19
|
+
/** Optional centroid the vectors are centered by before quantisation. */
|
|
20
|
+
centroid?: ArrayLike<number>;
|
|
21
|
+
/** SQLite page-cache budget in MiB. Purely a latency knob: correctness and
|
|
22
|
+
* the per-operation storage-read count are identical with it at 0. */
|
|
23
|
+
cacheSizeMb?: number;
|
|
24
|
+
}
|
|
25
|
+
export interface QueryResult {
|
|
26
|
+
id: ExternalId;
|
|
27
|
+
/** Estimated cosine distance (1 - cosine). */
|
|
28
|
+
distance: number;
|
|
29
|
+
}
|
|
30
|
+
export interface StorageStats {
|
|
31
|
+
float32BytesPerVector: number;
|
|
32
|
+
codeBytesPerVector: number;
|
|
33
|
+
bytesPerVector: number;
|
|
34
|
+
compressionRatio: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* A persistent vector database: an adaptive PARTITIONED (IVF) index over
|
|
38
|
+
* 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`. See
|
|
39
|
+
* ivf.ts for the index design. The original float vectors are never
|
|
40
|
+
* retained — only the sign codes — so a 1024-d vector costs 128 bytes.
|
|
41
|
+
*
|
|
42
|
+
* Inserting is route-and-append: cost is essentially FLAT in collection size
|
|
43
|
+
* (one RAM scan of the pivot table + one chunk append — no per-insert graph
|
|
44
|
+
* walk). Query cost is bounded by nprobe × cluster size — constant in N
|
|
45
|
+
* once the collection is past its first splits.
|
|
46
|
+
*/
|
|
47
|
+
export declare class VectorDatabase {
|
|
48
|
+
private readonly index;
|
|
49
|
+
constructor(options: DatabaseOptions);
|
|
50
|
+
get dim(): number;
|
|
51
|
+
/** Number of live (non-deleted) vectors. */
|
|
52
|
+
get size(): number;
|
|
53
|
+
/** Physical slot count including tombstones from deletes/updates. */
|
|
54
|
+
get physicalSize(): number;
|
|
55
|
+
/** Number of clusters (partitions) currently in the index. */
|
|
56
|
+
get clusterCount(): number;
|
|
57
|
+
get efSearch(): number;
|
|
58
|
+
set efSearch(value: number);
|
|
59
|
+
/** Distance computations performed during the most recent query. */
|
|
60
|
+
get lastQueryDistanceComputations(): number;
|
|
61
|
+
/** Storage row reads issued by the most recent query — the honest,
|
|
62
|
+
* cache-independent scalability metric. */
|
|
63
|
+
get lastQueryStorageReads(): number;
|
|
64
|
+
get storage(): StorageStats;
|
|
65
|
+
private checkId;
|
|
66
|
+
/** Convert a value to code bytes, selecting by length: `codeWords`
|
|
67
|
+
* elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
|
|
68
|
+
private toCodeBytes;
|
|
69
|
+
has(id: ExternalId): boolean;
|
|
70
|
+
/** Stream every live external id (bounded memory). */
|
|
71
|
+
keys(): IterableIterator<ExternalId>;
|
|
72
|
+
/** Stream live entries whose INTERNAL id is > `after` — a durable
|
|
73
|
+
* incremental watermark (internal ids are monotone at insert and preserved
|
|
74
|
+
* by update and compact). */
|
|
75
|
+
keysSince(after: number): IterableIterator<{
|
|
76
|
+
ext: ExternalId;
|
|
77
|
+
internal: number;
|
|
78
|
+
}>;
|
|
79
|
+
/** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
|
|
80
|
+
get(id: ExternalId): Uint32Array | null;
|
|
81
|
+
insert(id: ExternalId, value: ArrayLike<number>): void;
|
|
82
|
+
update(id: ExternalId, value: ArrayLike<number>): void;
|
|
83
|
+
upsert(id: ExternalId, value: ArrayLike<number>): void;
|
|
84
|
+
/**
|
|
85
|
+
* Upsert many vectors under ONE transaction — one WAL commit for the whole
|
|
86
|
+
* batch instead of one per vector, with chunk appends to the same cluster
|
|
87
|
+
* coalesced in the index's write-back buffer.
|
|
88
|
+
*/
|
|
89
|
+
upsertMany(entries: Array<{
|
|
90
|
+
id: ExternalId;
|
|
91
|
+
vector: ArrayLike<number>;
|
|
92
|
+
}>): void;
|
|
93
|
+
/** Delete many ids under ONE transaction. Absent ids are skipped.
|
|
94
|
+
* Returns the number of vectors actually removed. */
|
|
95
|
+
deleteMany(ids: ExternalId[]): number;
|
|
96
|
+
/** Delete the vector bound to an id. Returns false if absent. */
|
|
97
|
+
delete(id: ExternalId): boolean;
|
|
98
|
+
/** Heat the SQLite page cache with sequential scans (latency only). */
|
|
99
|
+
warmCache(): number;
|
|
100
|
+
/** Reclaim tombstoned slots by rewriting the clusters that carry any, then
|
|
101
|
+
* VACUUM. Internal ids and cluster assignment are preserved. */
|
|
102
|
+
compact(): void;
|
|
103
|
+
/**
|
|
104
|
+
* k-NN search. The argument's length selects the mode:
|
|
105
|
+
* - `dim` elements → a raw vector (accurate estimator)
|
|
106
|
+
* - `codeWords` elements → a 1-bit code, by sign-bit Hamming
|
|
107
|
+
*/
|
|
108
|
+
query(query: ArrayLike<number>, k?: number, opts?: {
|
|
109
|
+
ef?: number;
|
|
110
|
+
}): QueryResult[];
|
|
111
|
+
/** Close the underlying database. The instance must not be used afterwards. */
|
|
112
|
+
close(): void;
|
|
113
|
+
}
|