@hviana/sema 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/dist/example/demo.d.ts +1 -0
  2. package/dist/example/demo.js +39 -0
  3. package/dist/example/train_base.d.ts +87 -0
  4. package/dist/example/train_base.js +2252 -0
  5. package/dist/src/alphabet.d.ts +7 -0
  6. package/dist/src/alphabet.js +33 -0
  7. package/dist/src/alu/src/alu.d.ts +185 -0
  8. package/dist/src/alu/src/alu.js +440 -0
  9. package/dist/src/alu/src/expr.d.ts +61 -0
  10. package/dist/src/alu/src/expr.js +318 -0
  11. package/dist/src/alu/src/index.d.ts +11 -0
  12. package/dist/src/alu/src/index.js +19 -0
  13. package/dist/src/alu/src/kernel-arith.d.ts +16 -0
  14. package/dist/src/alu/src/kernel-arith.js +264 -0
  15. package/dist/src/alu/src/kernel-bits.d.ts +19 -0
  16. package/dist/src/alu/src/kernel-bits.js +152 -0
  17. package/dist/src/alu/src/kernel-logic.d.ts +4 -0
  18. package/dist/src/alu/src/kernel-logic.js +60 -0
  19. package/dist/src/alu/src/kernel-nd.d.ts +3 -0
  20. package/dist/src/alu/src/kernel-nd.js +208 -0
  21. package/dist/src/alu/src/kernel-numeric.d.ts +54 -0
  22. package/dist/src/alu/src/kernel-numeric.js +366 -0
  23. package/dist/src/alu/src/operation.d.ts +168 -0
  24. package/dist/src/alu/src/operation.js +189 -0
  25. package/dist/src/alu/src/parser.d.ts +221 -0
  26. package/dist/src/alu/src/parser.js +577 -0
  27. package/dist/src/alu/src/resonance.d.ts +55 -0
  28. package/dist/src/alu/src/resonance.js +126 -0
  29. package/dist/src/alu/src/text.d.ts +31 -0
  30. package/dist/src/alu/src/text.js +73 -0
  31. package/dist/src/alu/src/value.d.ts +109 -0
  32. package/dist/src/alu/src/value.js +300 -0
  33. package/dist/src/alu/test/alu.test.d.ts +1 -0
  34. package/dist/src/alu/test/alu.test.js +764 -0
  35. package/dist/src/bytes.d.ts +14 -0
  36. package/dist/src/bytes.js +59 -0
  37. package/dist/src/canon.d.ts +26 -0
  38. package/dist/src/canon.js +57 -0
  39. package/dist/src/config.d.ts +111 -0
  40. package/dist/src/config.js +91 -0
  41. package/dist/src/derive/src/deduction.d.ts +125 -0
  42. package/dist/src/derive/src/deduction.js +155 -0
  43. package/dist/src/derive/src/index.d.ts +7 -0
  44. package/dist/src/derive/src/index.js +11 -0
  45. package/dist/src/derive/src/priority-queue.d.ts +20 -0
  46. package/dist/src/derive/src/priority-queue.js +73 -0
  47. package/dist/src/derive/src/rewrite.d.ts +56 -0
  48. package/dist/src/derive/src/rewrite.js +100 -0
  49. package/dist/src/derive/src/trie.d.ts +90 -0
  50. package/dist/src/derive/src/trie.js +217 -0
  51. package/dist/src/derive/test/derive.test.d.ts +1 -0
  52. package/dist/src/derive/test/derive.test.js +122 -0
  53. package/dist/src/extension.d.ts +37 -0
  54. package/dist/src/extension.js +7 -0
  55. package/dist/src/geometry.d.ts +202 -0
  56. package/dist/src/geometry.js +575 -0
  57. package/dist/src/index.d.ts +16 -0
  58. package/dist/src/index.js +18 -0
  59. package/dist/src/ingest-cache.d.ts +41 -0
  60. package/dist/src/ingest-cache.js +161 -0
  61. package/dist/src/mind/articulation.d.ts +6 -0
  62. package/dist/src/mind/articulation.js +99 -0
  63. package/dist/src/mind/attention.d.ts +405 -0
  64. package/dist/src/mind/attention.js +1829 -0
  65. package/dist/src/mind/bridge.d.ts +30 -0
  66. package/dist/src/mind/bridge.js +569 -0
  67. package/dist/src/mind/canonical.d.ts +29 -0
  68. package/dist/src/mind/canonical.js +88 -0
  69. package/dist/src/mind/graph-search.d.ts +285 -0
  70. package/dist/src/mind/graph-search.js +956 -0
  71. package/dist/src/mind/index.d.ts +7 -0
  72. package/dist/src/mind/index.js +5 -0
  73. package/dist/src/mind/junction.d.ts +124 -0
  74. package/dist/src/mind/junction.js +317 -0
  75. package/dist/src/mind/learning.d.ts +47 -0
  76. package/dist/src/mind/learning.js +248 -0
  77. package/dist/src/mind/match.d.ts +126 -0
  78. package/dist/src/mind/match.js +441 -0
  79. package/dist/src/mind/mechanisms/alu.d.ts +4 -0
  80. package/dist/src/mind/mechanisms/alu.js +36 -0
  81. package/dist/src/mind/mechanisms/cast.d.ts +89 -0
  82. package/dist/src/mind/mechanisms/cast.js +716 -0
  83. package/dist/src/mind/mechanisms/confluence.d.ts +24 -0
  84. package/dist/src/mind/mechanisms/confluence.js +225 -0
  85. package/dist/src/mind/mechanisms/cover.d.ts +6 -0
  86. package/dist/src/mind/mechanisms/cover.js +203 -0
  87. package/dist/src/mind/mechanisms/extraction.d.ts +67 -0
  88. package/dist/src/mind/mechanisms/extraction.js +387 -0
  89. package/dist/src/mind/mechanisms/recall.d.ts +13 -0
  90. package/dist/src/mind/mechanisms/recall.js +325 -0
  91. package/dist/src/mind/mind.d.ts +276 -0
  92. package/dist/src/mind/mind.js +572 -0
  93. package/dist/src/mind/pipeline-mechanism.d.ts +142 -0
  94. package/dist/src/mind/pipeline-mechanism.js +213 -0
  95. package/dist/src/mind/pipeline.d.ts +20 -0
  96. package/dist/src/mind/pipeline.js +217 -0
  97. package/dist/src/mind/primitives.d.ts +70 -0
  98. package/dist/src/mind/primitives.js +287 -0
  99. package/dist/src/mind/rationale.d.ts +139 -0
  100. package/dist/src/mind/rationale.js +163 -0
  101. package/dist/src/mind/reasoning.d.ts +34 -0
  102. package/dist/src/mind/reasoning.js +240 -0
  103. package/dist/src/mind/recognition.d.ts +20 -0
  104. package/dist/src/mind/recognition.js +390 -0
  105. package/dist/src/mind/resonance.d.ts +23 -0
  106. package/dist/src/mind/resonance.js +0 -0
  107. package/dist/src/mind/trace.d.ts +15 -0
  108. package/dist/src/mind/trace.js +73 -0
  109. package/dist/src/mind/traverse.d.ts +113 -0
  110. package/dist/src/mind/traverse.js +568 -0
  111. package/dist/src/mind/types.d.ts +289 -0
  112. package/dist/src/mind/types.js +130 -0
  113. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  114. package/dist/src/rabitq-ivf/src/database.js +201 -0
  115. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  116. package/dist/src/rabitq-ivf/src/index.js +4 -0
  117. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  118. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  119. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  120. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  121. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  122. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  123. package/dist/src/sema.d.ts +31 -0
  124. package/dist/src/sema.js +63 -0
  125. package/dist/src/store-sqlite.d.ts +184 -0
  126. package/dist/src/store-sqlite.js +942 -0
  127. package/dist/src/store.d.ts +663 -0
  128. package/dist/src/store.js +1618 -0
  129. package/dist/src/vec.d.ts +31 -0
  130. package/dist/src/vec.js +109 -0
  131. package/package.json +1 -1
  132. package/src/mind/attention.ts +178 -63
  133. package/test/51-structural-resonance-ladder.test.mjs +6 -3
@@ -0,0 +1,289 @@
1
+ import type { Vec } from "../vec.js";
2
+ import type { Sema } from "../sema.js";
3
+ import type { BoundedMap, Store } from "../store.js";
4
+ import type { Space } from "../sema.js";
5
+ import type { Alphabet } from "../alphabet.js";
6
+ import type { MindConfig } from "../config.js";
7
+ import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
8
+ import type { Rationale } from "./rationale.js";
9
+ import type { FoldPyramid, Grid, StableFold } from "../geometry.js";
10
+ /** One {@link MindContext._depositTrees} entry — see that field's doc. */
11
+ export interface DepositCacheEntry {
12
+ /** Turn boundaries accumulated over this content's deposit chain —
13
+ * strictly increasing proper offsets, each a previously-deposited
14
+ * whole-context length. Empty for a first-seen (single-turn) input. */
15
+ boundaries: number[];
16
+ /** Plain-fold pyramid (first-seen inputs only). */
17
+ pyramid?: FoldPyramid;
18
+ /** Stable-prefix segment folds (grown-context inputs only). */
19
+ stable?: StableFold;
20
+ /** The continuation bytes this ctxInput was paired with in ingestPair, if
21
+ * any — the ONLY thing that makes a later, longer ctxInput a genuine next
22
+ * TURN of the same conversation rather than an unrelated fact that
23
+ * happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5"). A
24
+ * later deposit only takes this entry as its stable-prefix `prev` when
25
+ * its own suffix bytes-equal this exactly. */
26
+ nextBytes?: Uint8Array;
27
+ }
28
+ export type Input = string | Uint8Array | Grid | Grid[];
29
+ /** The host capabilities GraphSearch consults during a cover. MindContext
30
+ * extends this so the Mind can pass itself as the host. */
31
+ export interface GraphSearchHost {
32
+ resolve(bytes: Uint8Array): number | null;
33
+ recogniseSpan?(bytes: Uint8Array): {
34
+ sites: ReadonlyArray<Site>;
35
+ leaves: ReadonlyArray<Leaf>;
36
+ splits: ReadonlySet<number>;
37
+ starts: ReadonlySet<number>;
38
+ };
39
+ chooseNext?(node: number): number | undefined;
40
+ }
41
+ export interface Recognition {
42
+ /** Forms that can lead somewhere — they have an edge or a halo. */
43
+ sites: Site[];
44
+ /** The query's perceived leaves (the search's covering axioms). */
45
+ leaves: Leaf[];
46
+ /** Sub-leaf positions where a form boundary falls between leaf edges. */
47
+ splits: Set<number>;
48
+ /** Leaf-parent (chunk) start positions from the query's OWN perceived
49
+ * fold — the positions the fold itself chose as a grouping boundary, as
50
+ * opposed to an offset a byte-level scan merely happens to land on. The
51
+ * one boundary signal opportunistic cross-leaf recovery (recognition's
52
+ * own canonical chains, the search's `fuse`) can lean on instead of
53
+ * ASCII/word heuristics: see the `boundary` gate in recognition.ts. */
54
+ starts: Set<number>;
55
+ }
56
+ /** How the consensus climb weights a region's Document-Frequency reach. */
57
+ export type DFMode = "inverse" | "direct" | "combined";
58
+ /** One POINT OF ATTENTION the consensus climb resolved. */
59
+ export interface Attention {
60
+ /** The learnt context this point resolves to. */
61
+ anchor: number;
62
+ /** IDF-weighted consensus vote — the strength that orders points. */
63
+ vote: number;
64
+ /** The union of the query byte-spans whose evidence supports this point. */
65
+ start: number;
66
+ end: number;
67
+ /** SCALE-INVARIANT confidence: the fraction of the query's OWN regions
68
+ * whose evidence this point accounts for (Σ RegionVote.absorbed among
69
+ * its contributors, over the query's total region count) — read PER-
70
+ * ANCHOR, unlike the raw IDF vote (an absolute, ln(N)-scaled quantity
71
+ * that means "strong" on a small store and "weak" on a large one for
72
+ * the SAME degree of genuine consensus). A point whose breadth clears
73
+ * `dominates` (> half the query's regions corroborate it) is real
74
+ * consensus; one that does not is a coincidental single-region echo —
75
+ * see test/35-attention-confidence.test.mjs. */
76
+ breadth: number;
77
+ /** DISPERSION: the number of distinct clusters this point's contributing
78
+ * regions form, merging any two whose gap is under one river-fold
79
+ * quantum W. Neither breadth NOR raw region count discriminates a
80
+ * genuine further topic from a coincidental echo (both were tried and
81
+ * falsified — breadth starves a genuine, evenly-split multi-topic query,
82
+ * since no root in a real N-way split can exceed half the vote; raw
83
+ * count doesn't separate them either, since a short, structurally simple
84
+ * echo racks up as many corroborating regions as a real topic does).
85
+ * Dispersion asks a different question: not how MUCH evidence, but how
86
+ * many separate PLACES in the query corroborate it. A coincidental
87
+ * match — one local phrase resonating with an unrelated stored form —
88
+ * is structurally confined to ONE cluster no matter how strong its vote;
89
+ * a genuine further topic is named in its own distinctive wording
90
+ * somewhere the query's scaffolding does not reach, always a SEPARATE
91
+ * cluster from whatever else corroborates it. See
92
+ * test/37-cluster-dispersion-fusion.test.mjs. */
93
+ clusters: number;
94
+ }
95
+ /** Both read-outs of one consensus climb. */
96
+ export interface AttentionRead {
97
+ roots: Attention[];
98
+ ranked: Attention[];
99
+ }
100
+ /** A positioned region of a byte stream paired with its gist. */
101
+ export interface Segment {
102
+ start: number;
103
+ end: number;
104
+ v: Vec;
105
+ }
106
+ /** A region of the query's perceived tree for the consensus climb. */
107
+ export interface Region {
108
+ v: Vec;
109
+ start: number;
110
+ end: number;
111
+ chunk: boolean;
112
+ /** Whether the region's bytes resolve to a KNOWN node (content-addressed,
113
+ * exact). Exact regions vote with full weight; approximate ones pay the
114
+ * contrastive margin (see voteRegions) — under the linear fold a raw
115
+ * resonance score is byte-overlap, evidence only in excess of its best
116
+ * rival conclusion. */
117
+ known: boolean;
118
+ }
119
+ /** Per-region vote data from the consensus climb's resonance pass. */
120
+ export interface RegionVote {
121
+ start: number;
122
+ end: number;
123
+ canonicalFailed: boolean;
124
+ roots: readonly number[];
125
+ w: number;
126
+ wFocus: number;
127
+ /** How many of the query's ORIGINAL regions this one vote's evidence
128
+ * accounts for. 1 for an ordinary per-region vote (itself); for a
129
+ * cross-region junction vote, 1 (itself) plus however many individual
130
+ * votes it explained away (see crossRegionVotes) — the junction speaks
131
+ * for all of them at once, and breadth accounting must not undercount it
132
+ * to "one region" just because it collapsed to one pooled axiom.
133
+ * Defaults to 1 when absent. */
134
+ absorbed?: number;
135
+ }
136
+ /** The structural gate that first decided an {@link edgeAncestors} climb was
137
+ * saturated (an abstention, not a discriminative conclusion) — pure
138
+ * instrumentation for {@link ClimbConsensusData}'s reach trace; it never
139
+ * feeds back into the climb itself. */
140
+ export type SaturationReason = "byte-atom-commonality" | "predecessor-fan-in" | "distinct-context-limit" | "parent-fan-out" | "lateral-cone-limit";
141
+ /** One saturation stop's provenance: which reason fired, at which node, the
142
+ * observed count against the bound that decided it. */
143
+ export interface SaturationStop {
144
+ reason: SaturationReason;
145
+ node: number;
146
+ observed: number;
147
+ limit: number;
148
+ }
149
+ /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
150
+ export interface AncestorReach {
151
+ roots: number[];
152
+ contextsReached: number;
153
+ saturated: boolean;
154
+ /** The saturation gate that stopped this climb, when {@link saturated} is
155
+ * true and a trace was requested — see {@link edgeAncestors}. Absent for
156
+ * a non-saturated reach, and absent (even when saturated) when no trace
157
+ * was requested — instrumentation must not allocate when tracing is off. */
158
+ saturation?: SaturationStop;
159
+ }
160
+ /** Saturated-interval information for the noise-drop gate. */
161
+ export interface SaturationInfo {
162
+ leadingEnd: number;
163
+ hasLeading: boolean;
164
+ intervals: Array<{
165
+ start: number;
166
+ end: number;
167
+ }>;
168
+ }
169
+ /** The items of poolVotes' deduction system. */
170
+ export type AItem = {
171
+ kind: "region";
172
+ ri: number;
173
+ } | {
174
+ kind: "anchor";
175
+ id: number;
176
+ } | {
177
+ kind: "anchorFocus";
178
+ id: number;
179
+ };
180
+ export interface MindContext extends GraphSearchHost {
181
+ store: Store;
182
+ space: Space;
183
+ alphabet: Alphabet;
184
+ cfg: MindConfig;
185
+ search: GraphSearch;
186
+ trace: Rationale | null;
187
+ /** The content canonicalizer for THIS response, or null — injected by the
188
+ * modality entry point (respondText passes the text canonicalizer; a
189
+ * binary respond passes none). Resolution uses it as a fallback: when
190
+ * the exact content-addressed lookup misses, the span's canonical key is
191
+ * probed against the store's canon index (see src/canon.ts). The core
192
+ * never inspects what the equivalence IS. */
193
+ canon: ((bytes: Uint8Array) => Uint8Array) | null;
194
+ /** Per-response memo of canonical-fallback resolutions, keyed by the
195
+ * span's latin1 content key. Null outside respond(). */
196
+ canonMemo: Map<string, number | null> | null;
197
+ /** Memo of the consensus climb — content-keyed (latin1) so results
198
+ * persist across conversation turns where the same byte spans recur.
199
+ * Null outside respond(); during respondTurn() the conversation's
200
+ * persistent map is swapped in. */
201
+ climbMemo: Map<string, Map<string, AttentionRead>> | null;
202
+ /** Memo of {@link recognise} — content-keyed (latin1) so recognised
203
+ * forms carry forward across conversation turns. Bypassed while a
204
+ * trace is attached. Null outside respond(). */
205
+ recogniseMemo: Map<string, Recognition> | null;
206
+ /** Memo of {@link perceive} — content-keyed (latin1). The general
207
+ * cache the result-level memos each partially compensate for. NOT
208
+ * bypassed under trace — perception emits no rationale steps.
209
+ * Null outside respond(). */
210
+ perceiveMemo: Map<string, Sema> | null;
211
+ /** Subtree-resolution cache: Sema node → its store id and byte length.
212
+ * Populated by {@link foldTree} during inference; checked before
213
+ * walking children. When a conversation's pyramid reuses prefix
214
+ * subtrees, this cache lets {@link recognise} skip them entirely —
215
+ * O(suffix) instead of O(context). Mind-lifetime (WeakMap keys are
216
+ * the Sema objects the pyramid keeps alive). */
217
+ _resolvedSubtrees: WeakMap<Sema, {
218
+ id: number;
219
+ len: number;
220
+ }> | null;
221
+ _edgeGuide: Vec | null;
222
+ _edgeChoice: Map<number, number>;
223
+ _prevSeen: Set<number> | null;
224
+ /** Session cache of node-id → perceived gist, for candidate scoring
225
+ * ({@link chooseAmong} in the reverse projection's recall path re-gists up to
226
+ * √N contexts per pick — the measured bottleneck there). `chooseNext` does
227
+ * NOT use this cache; forward-edge disambiguation uses prevOf counts
228
+ * (distributional evidence) instead of gist comparison, because for short
229
+ * answer candidates the gist is dominated by accidental byte-pattern
230
+ * correlations. A node's bytes are immutable and perception is a pure
231
+ * function of bytes, so an entry stays valid for the store's lifetime —
232
+ * never invalidated. Bounded LRU (byte-sized); a miss only re-perceives,
233
+ * never a correctness risk. */
234
+ _gistCache: BoundedMap<number, Vec>;
235
+ /** DEPOSIT-path perception cache: content key (latin1) of a deposited
236
+ * input → its accumulated turn BOUNDARIES plus reusable fold state. A
237
+ * deposit whose content extends a cached entry IS a conversation context
238
+ * grown by one turn — the cached length is the new boundary — so it
239
+ * folds with the SAME stable-prefix fold query-time perception uses
240
+ * (structural train/inference agreement, load-bearing for recall),
241
+ * reusing every already-folded segment via `stable` (see StableFold) —
242
+ * O(turn) per deposit instead of O(context). A first-seen input keeps
243
+ * the plain fold and caches its `pyramid` (see FoldPyramid). Purely a
244
+ * performance cache for the FOLD STATE; the boundaries are semantic but
245
+ * derived only from the deposit sequence itself (an evicted chain falls
246
+ * back to plain-fold behavior, exactly the pre-boundary shape). */
247
+ _depositTrees: BoundedMap<string, DepositCacheEntry>;
248
+ /** The byte lengths present in {@link _depositTrees} — the candidate
249
+ * prefix lengths probed (longest first). Drifts on eviction (a stale
250
+ * length only costs a miss); cleared with the map when it outgrows the
251
+ * probe budget. */
252
+ _depositLens: Set<number>;
253
+ /** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
254
+ * content-addressed id. Valid forever (ids are permanent, Sema nodes
255
+ * immutable); WeakMap, so entries live exactly as long as the pyramid
256
+ * cache keeps the shared subtrees alive. Lets internTreeIds skip whole
257
+ * shared subtrees and indexSubSpans keep its seenBefore window skip. */
258
+ _internIds: WeakMap<Sema, number>;
259
+ }
260
+ /** Read a whole node's bytes. */
261
+ export declare const ALL = 2147483647;
262
+ /** Splice every chosen span in order — the whole cover as one byte string. */
263
+ export declare function spliceAll(segs: Seg[]): Uint8Array | null;
264
+ /** Whether a chosen span RESTATES the query rather than answering it: its
265
+ * SUBSTITUTED bytes (an edge followed from a recognised site, not the
266
+ * site's own literal text read back) already occur elsewhere in the query
267
+ * — the same principle recall.ts's tiers apply to a whole-query projection
268
+ * ("a projection that is a proper byte-subspan of the query restates part
269
+ * of the question"). A LITERAL span (the site's own bytes, unchanged) is
270
+ * exempt: naming what's already there at its OWN position is not a
271
+ * substitution. A recognised site that is itself an entire PRIOR TURN of
272
+ * a multi-turn query is exactly this shape: it carries a genuine learnt
273
+ * continuation, but that continuation is something the asker already said
274
+ * moments later in the SAME query, not a new answer. Below one river
275
+ * window, byte overlap is chance, not evidence — the same floor
276
+ * identityBar and reachThreshold hold every other structural-overlap claim
277
+ * to. */
278
+ export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: number, W: number): boolean;
279
+ /** Lift the answer out of the cover for think: the recognised region, free of
280
+ * the asker's surrounding (unrecognised) framing — and free of any chosen
281
+ * span that only RESTATES content the query already contains (see {@link
282
+ * segRestatesQuery}). A restating span is excluded from both the framing
283
+ * (lo/hi) decision and the final concatenation: it is stale, not a second
284
+ * answer, but the OTHER spans a derivation chose are independent evidence
285
+ * and must not be discarded along with it. */
286
+ export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
287
+ /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
288
+ * tracked deposit interned (`prevSeen`). */
289
+ export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];
@@ -0,0 +1,130 @@
1
+ // types.ts — all interfaces, types, and free functions for the mind.
2
+ //
3
+ // GraphSearchHost is defined first (minimal imports) so GraphSearch can import
4
+ // it without pulling in the full MindContext.
5
+ import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
6
+ import { dominates } from "../geometry.js";
7
+ // ═══════════════════════════════════════════════════════════════════════════
8
+ // FREE FUNCTIONS (pure, no state)
9
+ // ═══════════════════════════════════════════════════════════════════════════
10
+ /** Read a whole node's bytes. */
11
+ export const ALL = 0x7fffffff;
12
+ /** Splice every chosen span in order — the whole cover as one byte string. */
13
+ export function spliceAll(segs) {
14
+ if (!segs.some((s) => s.rec))
15
+ return null;
16
+ return concatBytes(segs.map((s) => s.bytes));
17
+ }
18
+ /** Whether a chosen span RESTATES the query rather than answering it: its
19
+ * SUBSTITUTED bytes (an edge followed from a recognised site, not the
20
+ * site's own literal text read back) already occur elsewhere in the query
21
+ * — the same principle recall.ts's tiers apply to a whole-query projection
22
+ * ("a projection that is a proper byte-subspan of the query restates part
23
+ * of the question"). A LITERAL span (the site's own bytes, unchanged) is
24
+ * exempt: naming what's already there at its OWN position is not a
25
+ * substitution. A recognised site that is itself an entire PRIOR TURN of
26
+ * a multi-turn query is exactly this shape: it carries a genuine learnt
27
+ * continuation, but that continuation is something the asker already said
28
+ * moments later in the SAME query, not a new answer. Below one river
29
+ * window, byte overlap is chance, not evidence — the same floor
30
+ * identityBar and reachThreshold hold every other structural-overlap claim
31
+ * to. */
32
+ export function segRestatesQuery(s, query, queryLen, W) {
33
+ if (!s.rec)
34
+ return false;
35
+ const literal = s.j - s.i === s.bytes.length &&
36
+ bytesEqual(s.bytes, query.subarray(s.i, s.j));
37
+ if (literal)
38
+ return false;
39
+ return s.bytes.length >= W && s.bytes.length < queryLen &&
40
+ indexOf(query, s.bytes, 0) >= 0;
41
+ }
42
+ /** Lift the answer out of the cover for think: the recognised region, free of
43
+ * the asker's surrounding (unrecognised) framing — and free of any chosen
44
+ * span that only RESTATES content the query already contains (see {@link
45
+ * segRestatesQuery}). A restating span is excluded from both the framing
46
+ * (lo/hi) decision and the final concatenation: it is stale, not a second
47
+ * answer, but the OTHER spans a derivation chose are independent evidence
48
+ * and must not be discarded along with it. */
49
+ export function liftAnswer(segs, queryLen, query, W) {
50
+ const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
51
+ const recognised = [];
52
+ for (let k = 0; k < segs.length; k++) {
53
+ if (segs[k].rec && !restated[k])
54
+ recognised.push(k);
55
+ }
56
+ if (recognised.length === 0)
57
+ return null;
58
+ if (recognised.length === 1) {
59
+ const s = segs[recognised[0]];
60
+ // A COMPUTED span's query-side width is operand digit-count, not
61
+ // evidence of how much of the query's meaning it accounts for — the
62
+ // half-dominance check below (built for a genuinely RECOGNISED learned
63
+ // form) is not a valid framing signal for it (see the `computed` field
64
+ // doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
65
+ // because the operands are big, not because the framing matters less.
66
+ // A LITERAL PREFIX before a computed span is unambiguous framing
67
+ // regardless of width — an arithmetic expression is never itself
68
+ // preceded by more literal computed content, so anything literal before
69
+ // it is question wording ("what is ", "compute ") to lift clear of.
70
+ // With no prefix (s.i === 0) the span is judged by the ordinary
71
+ // half-dominance rule below, which already correctly keeps a short
72
+ // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
73
+ if (s.computed && s.i > 0)
74
+ return s.bytes;
75
+ if (dominates(s.j - s.i, queryLen)) {
76
+ return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
77
+ }
78
+ return s.bytes;
79
+ }
80
+ const lo = recognised[0];
81
+ const hi = recognised[recognised.length - 1];
82
+ return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
83
+ }
84
+ /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
85
+ * tracked deposit interned (`prevSeen`). */
86
+ export function changedNodes(tree, ids, prevSeen) {
87
+ const newCount = new Map();
88
+ const count = (n) => {
89
+ const memo = newCount.get(n);
90
+ if (memo !== undefined)
91
+ return memo;
92
+ const id = ids.get(n);
93
+ // PRUNE: a node whose id the previous deposit already interned is old,
94
+ // and content addressing makes that transitive — the same id names the
95
+ // same content, so every descendant was interned then too. The whole
96
+ // subtree counts 0 without walking it; with the pyramid fold sharing a
97
+ // conversation's prefix subtree, this is what keeps the changed-nodes
98
+ // read O(new nodes) instead of O(context). (A node internTreeIds
99
+ // memo-skipped has an id here exactly when it is such a shared root.)
100
+ if (id !== undefined && prevSeen.has(id)) {
101
+ newCount.set(n, 0);
102
+ return 0;
103
+ }
104
+ let c = 1; // reachable only when NOT pruned above ⇒ this node is new
105
+ if (n.kids) {
106
+ for (const k of n.kids)
107
+ c += count(k);
108
+ }
109
+ newCount.set(n, c);
110
+ return c;
111
+ };
112
+ const total = count(tree);
113
+ if (total === 0)
114
+ return [tree];
115
+ let n = tree;
116
+ for (;;) {
117
+ if (n.kids === null)
118
+ return [n];
119
+ let holder = null;
120
+ for (const k of n.kids) {
121
+ if (newCount.get(k) === total) {
122
+ holder = k;
123
+ break;
124
+ }
125
+ }
126
+ if (holder === null)
127
+ return [n];
128
+ n = holder;
129
+ }
130
+ }
@@ -0,0 +1,113 @@
1
+ /** External ids are integers only. */
2
+ export type ExternalId = number;
3
+ export interface DatabaseOptions {
4
+ /** Path to the SQLite database file (use ":memory:" for a transient store). */
5
+ dbPath: string;
6
+ /** Vector dimensionality. Required for a new database; ignored when reopening. */
7
+ dim?: number;
8
+ /** Query breadth: clusters probed per query = ceil(efSearch / 4). Default
9
+ * 64 (16 probes). Tunable at runtime and at reopen. */
10
+ efSearch?: number;
11
+ /** Bits used to scalar-quantise the query for the RaBitQ estimator.
12
+ * Query-side only (stored codes are independent of it), so it is tunable
13
+ * at reopen, like efSearch. Default 8. */
14
+ queryBits?: number;
15
+ /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
16
+ rotationRounds?: number;
17
+ /** Seed for the rotation. Default fixed. */
18
+ seed?: number;
19
+ /** Optional centroid the vectors are centered by before quantisation. */
20
+ centroid?: ArrayLike<number>;
21
+ /** SQLite page-cache budget in MiB. Purely a latency knob: correctness and
22
+ * the per-operation storage-read count are identical with it at 0. */
23
+ cacheSizeMb?: number;
24
+ }
25
+ export interface QueryResult {
26
+ id: ExternalId;
27
+ /** Estimated cosine distance (1 - cosine). */
28
+ distance: number;
29
+ }
30
+ export interface StorageStats {
31
+ float32BytesPerVector: number;
32
+ codeBytesPerVector: number;
33
+ bytesPerVector: number;
34
+ compressionRatio: number;
35
+ }
36
+ /**
37
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
38
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`. See
39
+ * ivf.ts for the index design. The original float vectors are never
40
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
41
+ *
42
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
43
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
44
+ * walk). Query cost is bounded by nprobe × cluster size — constant in N
45
+ * once the collection is past its first splits.
46
+ */
47
+ export declare class VectorDatabase {
48
+ private readonly index;
49
+ constructor(options: DatabaseOptions);
50
+ get dim(): number;
51
+ /** Number of live (non-deleted) vectors. */
52
+ get size(): number;
53
+ /** Physical slot count including tombstones from deletes/updates. */
54
+ get physicalSize(): number;
55
+ /** Number of clusters (partitions) currently in the index. */
56
+ get clusterCount(): number;
57
+ get efSearch(): number;
58
+ set efSearch(value: number);
59
+ /** Distance computations performed during the most recent query. */
60
+ get lastQueryDistanceComputations(): number;
61
+ /** Storage row reads issued by the most recent query — the honest,
62
+ * cache-independent scalability metric. */
63
+ get lastQueryStorageReads(): number;
64
+ get storage(): StorageStats;
65
+ private checkId;
66
+ /** Convert a value to code bytes, selecting by length: `codeWords`
67
+ * elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
68
+ private toCodeBytes;
69
+ has(id: ExternalId): boolean;
70
+ /** Stream every live external id (bounded memory). */
71
+ keys(): IterableIterator<ExternalId>;
72
+ /** Stream live entries whose INTERNAL id is > `after` — a durable
73
+ * incremental watermark (internal ids are monotone at insert and preserved
74
+ * by update and compact). */
75
+ keysSince(after: number): IterableIterator<{
76
+ ext: ExternalId;
77
+ internal: number;
78
+ }>;
79
+ /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
80
+ get(id: ExternalId): Uint32Array | null;
81
+ insert(id: ExternalId, value: ArrayLike<number>): void;
82
+ update(id: ExternalId, value: ArrayLike<number>): void;
83
+ upsert(id: ExternalId, value: ArrayLike<number>): void;
84
+ /**
85
+ * Upsert many vectors under ONE transaction — one WAL commit for the whole
86
+ * batch instead of one per vector, with chunk appends to the same cluster
87
+ * coalesced in the index's write-back buffer.
88
+ */
89
+ upsertMany(entries: Array<{
90
+ id: ExternalId;
91
+ vector: ArrayLike<number>;
92
+ }>): void;
93
+ /** Delete many ids under ONE transaction. Absent ids are skipped.
94
+ * Returns the number of vectors actually removed. */
95
+ deleteMany(ids: ExternalId[]): number;
96
+ /** Delete the vector bound to an id. Returns false if absent. */
97
+ delete(id: ExternalId): boolean;
98
+ /** Heat the SQLite page cache with sequential scans (latency only). */
99
+ warmCache(): number;
100
+ /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
101
+ * VACUUM. Internal ids and cluster assignment are preserved. */
102
+ compact(): void;
103
+ /**
104
+ * k-NN search. The argument's length selects the mode:
105
+ * - `dim` elements → a raw vector (accurate estimator)
106
+ * - `codeWords` elements → a 1-bit code, by sign-bit Hamming
107
+ */
108
+ query(query: ArrayLike<number>, k?: number, opts?: {
109
+ ef?: number;
110
+ }): QueryResult[];
111
+ /** Close the underlying database. The instance must not be used afterwards. */
112
+ close(): void;
113
+ }