@hviana/sema 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/package.json +1 -1
  2. package/src/geometry.ts +52 -3
  3. package/src/mind/attention.ts +1469 -120
  4. package/src/mind/index.ts +16 -0
  5. package/src/mind/junction.ts +125 -16
  6. package/src/mind/mind.ts +15 -0
  7. package/src/mind/rationale.ts +12 -4
  8. package/src/mind/traverse.ts +75 -7
  9. package/src/mind/types.ts +25 -0
  10. package/test/51-structural-resonance-ladder.test.mjs +552 -0
  11. package/test/52-climb-consensus-instrumentation.test.mjs +324 -0
  12. package/test/53-cross-region-probe-instrumentation.test.mjs +356 -0
  13. package/dist/example/demo.d.ts +0 -1
  14. package/dist/example/demo.js +0 -39
  15. package/dist/example/train_base.d.ts +0 -87
  16. package/dist/example/train_base.js +0 -2252
  17. package/dist/src/alphabet.d.ts +0 -7
  18. package/dist/src/alphabet.js +0 -33
  19. package/dist/src/alu/src/alu.d.ts +0 -185
  20. package/dist/src/alu/src/alu.js +0 -440
  21. package/dist/src/alu/src/expr.d.ts +0 -61
  22. package/dist/src/alu/src/expr.js +0 -318
  23. package/dist/src/alu/src/index.d.ts +0 -11
  24. package/dist/src/alu/src/index.js +0 -19
  25. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  26. package/dist/src/alu/src/kernel-arith.js +0 -264
  27. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  28. package/dist/src/alu/src/kernel-bits.js +0 -152
  29. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  30. package/dist/src/alu/src/kernel-logic.js +0 -60
  31. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  32. package/dist/src/alu/src/kernel-nd.js +0 -208
  33. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  34. package/dist/src/alu/src/kernel-numeric.js +0 -366
  35. package/dist/src/alu/src/operation.d.ts +0 -168
  36. package/dist/src/alu/src/operation.js +0 -189
  37. package/dist/src/alu/src/parser.d.ts +0 -221
  38. package/dist/src/alu/src/parser.js +0 -577
  39. package/dist/src/alu/src/resonance.d.ts +0 -55
  40. package/dist/src/alu/src/resonance.js +0 -126
  41. package/dist/src/alu/src/text.d.ts +0 -31
  42. package/dist/src/alu/src/text.js +0 -73
  43. package/dist/src/alu/src/value.d.ts +0 -109
  44. package/dist/src/alu/src/value.js +0 -300
  45. package/dist/src/alu/test/alu.test.d.ts +0 -1
  46. package/dist/src/alu/test/alu.test.js +0 -764
  47. package/dist/src/bytes.d.ts +0 -14
  48. package/dist/src/bytes.js +0 -59
  49. package/dist/src/canon.d.ts +0 -26
  50. package/dist/src/canon.js +0 -57
  51. package/dist/src/config.d.ts +0 -111
  52. package/dist/src/config.js +0 -91
  53. package/dist/src/derive/src/deduction.d.ts +0 -125
  54. package/dist/src/derive/src/deduction.js +0 -155
  55. package/dist/src/derive/src/index.d.ts +0 -7
  56. package/dist/src/derive/src/index.js +0 -11
  57. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  58. package/dist/src/derive/src/priority-queue.js +0 -73
  59. package/dist/src/derive/src/rewrite.d.ts +0 -56
  60. package/dist/src/derive/src/rewrite.js +0 -100
  61. package/dist/src/derive/src/trie.d.ts +0 -90
  62. package/dist/src/derive/src/trie.js +0 -217
  63. package/dist/src/derive/test/derive.test.d.ts +0 -1
  64. package/dist/src/derive/test/derive.test.js +0 -122
  65. package/dist/src/extension.d.ts +0 -37
  66. package/dist/src/extension.js +0 -7
  67. package/dist/src/geometry.d.ts +0 -176
  68. package/dist/src/geometry.js +0 -546
  69. package/dist/src/index.d.ts +0 -16
  70. package/dist/src/index.js +0 -18
  71. package/dist/src/ingest-cache.d.ts +0 -41
  72. package/dist/src/ingest-cache.js +0 -161
  73. package/dist/src/mind/articulation.d.ts +0 -6
  74. package/dist/src/mind/articulation.js +0 -99
  75. package/dist/src/mind/attention.d.ts +0 -88
  76. package/dist/src/mind/attention.js +0 -997
  77. package/dist/src/mind/bridge.d.ts +0 -30
  78. package/dist/src/mind/bridge.js +0 -569
  79. package/dist/src/mind/canonical.d.ts +0 -29
  80. package/dist/src/mind/canonical.js +0 -88
  81. package/dist/src/mind/graph-search.d.ts +0 -285
  82. package/dist/src/mind/graph-search.js +0 -956
  83. package/dist/src/mind/index.d.ts +0 -5
  84. package/dist/src/mind/index.js +0 -5
  85. package/dist/src/mind/junction.d.ts +0 -95
  86. package/dist/src/mind/junction.js +0 -262
  87. package/dist/src/mind/learning.d.ts +0 -47
  88. package/dist/src/mind/learning.js +0 -248
  89. package/dist/src/mind/match.d.ts +0 -126
  90. package/dist/src/mind/match.js +0 -441
  91. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  92. package/dist/src/mind/mechanisms/alu.js +0 -36
  93. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  94. package/dist/src/mind/mechanisms/cast.js +0 -716
  95. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  96. package/dist/src/mind/mechanisms/confluence.js +0 -225
  97. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  98. package/dist/src/mind/mechanisms/cover.js +0 -203
  99. package/dist/src/mind/mechanisms/extraction.d.ts +0 -67
  100. package/dist/src/mind/mechanisms/extraction.js +0 -387
  101. package/dist/src/mind/mechanisms/recall.d.ts +0 -13
  102. package/dist/src/mind/mechanisms/recall.js +0 -325
  103. package/dist/src/mind/mind.d.ts +0 -274
  104. package/dist/src/mind/mind.js +0 -572
  105. package/dist/src/mind/pipeline-mechanism.d.ts +0 -142
  106. package/dist/src/mind/pipeline-mechanism.js +0 -213
  107. package/dist/src/mind/pipeline.d.ts +0 -20
  108. package/dist/src/mind/pipeline.js +0 -217
  109. package/dist/src/mind/primitives.d.ts +0 -70
  110. package/dist/src/mind/primitives.js +0 -287
  111. package/dist/src/mind/rationale.d.ts +0 -134
  112. package/dist/src/mind/rationale.js +0 -162
  113. package/dist/src/mind/reasoning.d.ts +0 -34
  114. package/dist/src/mind/reasoning.js +0 -240
  115. package/dist/src/mind/recognition.d.ts +0 -20
  116. package/dist/src/mind/recognition.js +0 -390
  117. package/dist/src/mind/resonance.d.ts +0 -23
  118. package/dist/src/mind/resonance.js +0 -0
  119. package/dist/src/mind/trace.d.ts +0 -15
  120. package/dist/src/mind/trace.js +0 -73
  121. package/dist/src/mind/traverse.d.ts +0 -113
  122. package/dist/src/mind/traverse.js +0 -504
  123. package/dist/src/mind/types.d.ts +0 -271
  124. package/dist/src/mind/types.js +0 -130
  125. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  126. package/dist/src/rabitq-ivf/src/database.js +0 -201
  127. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  128. package/dist/src/rabitq-ivf/src/index.js +0 -4
  129. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  130. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  131. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  132. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  133. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  134. package/dist/src/rabitq-ivf/src/rabitq.js +0 -283
  135. package/dist/src/sema.d.ts +0 -31
  136. package/dist/src/sema.js +0 -63
  137. package/dist/src/store-sqlite.d.ts +0 -184
  138. package/dist/src/store-sqlite.js +0 -942
  139. package/dist/src/store.d.ts +0 -663
  140. package/dist/src/store.js +0 -1618
  141. package/dist/src/vec.d.ts +0 -31
  142. package/dist/src/vec.js +0 -109
@@ -1,271 +0,0 @@
1
- import type { Vec } from "../vec.js";
2
- import type { Sema } from "../sema.js";
3
- import type { BoundedMap, Store } from "../store.js";
4
- import type { Space } from "../sema.js";
5
- import type { Alphabet } from "../alphabet.js";
6
- import type { MindConfig } from "../config.js";
7
- import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
8
- import type { Rationale } from "./rationale.js";
9
- import type { FoldPyramid, Grid, StableFold } from "../geometry.js";
10
- /** One {@link MindContext._depositTrees} entry — see that field's doc. */
11
- export interface DepositCacheEntry {
12
- /** Turn boundaries accumulated over this content's deposit chain —
13
- * strictly increasing proper offsets, each a previously-deposited
14
- * whole-context length. Empty for a first-seen (single-turn) input. */
15
- boundaries: number[];
16
- /** Plain-fold pyramid (first-seen inputs only). */
17
- pyramid?: FoldPyramid;
18
- /** Stable-prefix segment folds (grown-context inputs only). */
19
- stable?: StableFold;
20
- /** The continuation bytes this ctxInput was paired with in ingestPair, if
21
- * any — the ONLY thing that makes a later, longer ctxInput a genuine next
22
- * TURN of the same conversation rather than an unrelated fact that
23
- * happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5"). A
24
- * later deposit only takes this entry as its stable-prefix `prev` when
25
- * its own suffix bytes-equal this exactly. */
26
- nextBytes?: Uint8Array;
27
- }
28
- export type Input = string | Uint8Array | Grid | Grid[];
29
- /** The host capabilities GraphSearch consults during a cover. MindContext
30
- * extends this so the Mind can pass itself as the host. */
31
- export interface GraphSearchHost {
32
- resolve(bytes: Uint8Array): number | null;
33
- recogniseSpan?(bytes: Uint8Array): {
34
- sites: ReadonlyArray<Site>;
35
- leaves: ReadonlyArray<Leaf>;
36
- splits: ReadonlySet<number>;
37
- starts: ReadonlySet<number>;
38
- };
39
- chooseNext?(node: number): number | undefined;
40
- }
41
- export interface Recognition {
42
- /** Forms that can lead somewhere — they have an edge or a halo. */
43
- sites: Site[];
44
- /** The query's perceived leaves (the search's covering axioms). */
45
- leaves: Leaf[];
46
- /** Sub-leaf positions where a form boundary falls between leaf edges. */
47
- splits: Set<number>;
48
- /** Leaf-parent (chunk) start positions from the query's OWN perceived
49
- * fold — the positions the fold itself chose as a grouping boundary, as
50
- * opposed to an offset a byte-level scan merely happens to land on. The
51
- * one boundary signal opportunistic cross-leaf recovery (recognition's
52
- * own canonical chains, the search's `fuse`) can lean on instead of
53
- * ASCII/word heuristics: see the `boundary` gate in recognition.ts. */
54
- starts: Set<number>;
55
- }
56
- /** How the consensus climb weights a region's Document-Frequency reach. */
57
- export type DFMode = "inverse" | "direct" | "combined";
58
- /** One POINT OF ATTENTION the consensus climb resolved. */
59
- export interface Attention {
60
- /** The learnt context this point resolves to. */
61
- anchor: number;
62
- /** IDF-weighted consensus vote — the strength that orders points. */
63
- vote: number;
64
- /** The union of the query byte-spans whose evidence supports this point. */
65
- start: number;
66
- end: number;
67
- /** SCALE-INVARIANT confidence: the fraction of the query's OWN regions
68
- * whose evidence this point accounts for (Σ RegionVote.absorbed among
69
- * its contributors, over the query's total region count) — read PER-
70
- * ANCHOR, unlike the raw IDF vote (an absolute, ln(N)-scaled quantity
71
- * that means "strong" on a small store and "weak" on a large one for
72
- * the SAME degree of genuine consensus). A point whose breadth clears
73
- * `dominates` (> half the query's regions corroborate it) is real
74
- * consensus; one that does not is a coincidental single-region echo —
75
- * see test/35-attention-confidence.test.mjs. */
76
- breadth: number;
77
- /** DISPERSION: the number of distinct clusters this point's contributing
78
- * regions form, merging any two whose gap is under one river-fold
79
- * quantum W. Neither breadth NOR raw region count discriminates a
80
- * genuine further topic from a coincidental echo (both were tried and
81
- * falsified — breadth starves a genuine, evenly-split multi-topic query,
82
- * since no root in a real N-way split can exceed half the vote; raw
83
- * count doesn't separate them either, since a short, structurally simple
84
- * echo racks up as many corroborating regions as a real topic does).
85
- * Dispersion asks a different question: not how MUCH evidence, but how
86
- * many separate PLACES in the query corroborate it. A coincidental
87
- * match — one local phrase resonating with an unrelated stored form —
88
- * is structurally confined to ONE cluster no matter how strong its vote;
89
- * a genuine further topic is named in its own distinctive wording
90
- * somewhere the query's scaffolding does not reach, always a SEPARATE
91
- * cluster from whatever else corroborates it. See
92
- * test/37-cluster-dispersion-fusion.test.mjs. */
93
- clusters: number;
94
- }
95
- /** Both read-outs of one consensus climb. */
96
- export interface AttentionRead {
97
- roots: Attention[];
98
- ranked: Attention[];
99
- }
100
- /** A positioned region of a byte stream paired with its gist. */
101
- export interface Segment {
102
- start: number;
103
- end: number;
104
- v: Vec;
105
- }
106
- /** A region of the query's perceived tree for the consensus climb. */
107
- export interface Region {
108
- v: Vec;
109
- start: number;
110
- end: number;
111
- chunk: boolean;
112
- /** Whether the region's bytes resolve to a KNOWN node (content-addressed,
113
- * exact). Exact regions vote with full weight; approximate ones pay the
114
- * contrastive margin (see voteRegions) — under the linear fold a raw
115
- * resonance score is byte-overlap, evidence only in excess of its best
116
- * rival conclusion. */
117
- known: boolean;
118
- }
119
- /** Per-region vote data from the consensus climb's resonance pass. */
120
- export interface RegionVote {
121
- start: number;
122
- end: number;
123
- canonicalFailed: boolean;
124
- roots: readonly number[];
125
- w: number;
126
- wFocus: number;
127
- /** How many of the query's ORIGINAL regions this one vote's evidence
128
- * accounts for. 1 for an ordinary per-region vote (itself); for a
129
- * cross-region junction vote, 1 (itself) plus however many individual
130
- * votes it explained away (see crossRegionVotes) — the junction speaks
131
- * for all of them at once, and breadth accounting must not undercount it
132
- * to "one region" just because it collapsed to one pooled axiom.
133
- * Defaults to 1 when absent. */
134
- absorbed?: number;
135
- }
136
- /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
137
- export interface AncestorReach {
138
- roots: number[];
139
- contextsReached: number;
140
- saturated: boolean;
141
- }
142
- /** Saturated-interval information for the noise-drop gate. */
143
- export interface SaturationInfo {
144
- leadingEnd: number;
145
- hasLeading: boolean;
146
- intervals: Array<{
147
- start: number;
148
- end: number;
149
- }>;
150
- }
151
- /** The items of poolVotes' deduction system. */
152
- export type AItem = {
153
- kind: "region";
154
- ri: number;
155
- } | {
156
- kind: "anchor";
157
- id: number;
158
- } | {
159
- kind: "anchorFocus";
160
- id: number;
161
- };
162
- export interface MindContext extends GraphSearchHost {
163
- store: Store;
164
- space: Space;
165
- alphabet: Alphabet;
166
- cfg: MindConfig;
167
- search: GraphSearch;
168
- trace: Rationale | null;
169
- /** The content canonicalizer for THIS response, or null — injected by the
170
- * modality entry point (respondText passes the text canonicalizer; a
171
- * binary respond passes none). Resolution uses it as a fallback: when
172
- * the exact content-addressed lookup misses, the span's canonical key is
173
- * probed against the store's canon index (see src/canon.ts). The core
174
- * never inspects what the equivalence IS. */
175
- canon: ((bytes: Uint8Array) => Uint8Array) | null;
176
- /** Per-response memo of canonical-fallback resolutions, keyed by the
177
- * span's latin1 content key. Null outside respond(). */
178
- canonMemo: Map<string, number | null> | null;
179
- /** Memo of the consensus climb — content-keyed (latin1) so results
180
- * persist across conversation turns where the same byte spans recur.
181
- * Null outside respond(); during respondTurn() the conversation's
182
- * persistent map is swapped in. */
183
- climbMemo: Map<string, Map<string, AttentionRead>> | null;
184
- /** Memo of {@link recognise} — content-keyed (latin1) so recognised
185
- * forms carry forward across conversation turns. Bypassed while a
186
- * trace is attached. Null outside respond(). */
187
- recogniseMemo: Map<string, Recognition> | null;
188
- /** Memo of {@link perceive} — content-keyed (latin1). The general
189
- * cache the result-level memos each partially compensate for. NOT
190
- * bypassed under trace — perception emits no rationale steps.
191
- * Null outside respond(). */
192
- perceiveMemo: Map<string, Sema> | null;
193
- /** Subtree-resolution cache: Sema node → its store id and byte length.
194
- * Populated by {@link foldTree} during inference; checked before
195
- * walking children. When a conversation's pyramid reuses prefix
196
- * subtrees, this cache lets {@link recognise} skip them entirely —
197
- * O(suffix) instead of O(context). Mind-lifetime (WeakMap keys are
198
- * the Sema objects the pyramid keeps alive). */
199
- _resolvedSubtrees: WeakMap<Sema, {
200
- id: number;
201
- len: number;
202
- }> | null;
203
- _edgeGuide: Vec | null;
204
- _edgeChoice: Map<number, number>;
205
- _prevSeen: Set<number> | null;
206
- /** Session cache of node-id → perceived gist, for candidate scoring
207
- * ({@link chooseAmong} in the reverse projection's recall path re-gists up to
208
- * √N contexts per pick — the measured bottleneck there). `chooseNext` does
209
- * NOT use this cache; forward-edge disambiguation uses prevOf counts
210
- * (distributional evidence) instead of gist comparison, because for short
211
- * answer candidates the gist is dominated by accidental byte-pattern
212
- * correlations. A node's bytes are immutable and perception is a pure
213
- * function of bytes, so an entry stays valid for the store's lifetime —
214
- * never invalidated. Bounded LRU (byte-sized); a miss only re-perceives,
215
- * never a correctness risk. */
216
- _gistCache: BoundedMap<number, Vec>;
217
- /** DEPOSIT-path perception cache: content key (latin1) of a deposited
218
- * input → its accumulated turn BOUNDARIES plus reusable fold state. A
219
- * deposit whose content extends a cached entry IS a conversation context
220
- * grown by one turn — the cached length is the new boundary — so it
221
- * folds with the SAME stable-prefix fold query-time perception uses
222
- * (structural train/inference agreement, load-bearing for recall),
223
- * reusing every already-folded segment via `stable` (see StableFold) —
224
- * O(turn) per deposit instead of O(context). A first-seen input keeps
225
- * the plain fold and caches its `pyramid` (see FoldPyramid). Purely a
226
- * performance cache for the FOLD STATE; the boundaries are semantic but
227
- * derived only from the deposit sequence itself (an evicted chain falls
228
- * back to plain-fold behavior, exactly the pre-boundary shape). */
229
- _depositTrees: BoundedMap<string, DepositCacheEntry>;
230
- /** The byte lengths present in {@link _depositTrees} — the candidate
231
- * prefix lengths probed (longest first). Drifts on eviction (a stale
232
- * length only costs a miss); cleared with the map when it outgrows the
233
- * probe budget. */
234
- _depositLens: Set<number>;
235
- /** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
236
- * content-addressed id. Valid forever (ids are permanent, Sema nodes
237
- * immutable); WeakMap, so entries live exactly as long as the pyramid
238
- * cache keeps the shared subtrees alive. Lets internTreeIds skip whole
239
- * shared subtrees and indexSubSpans keep its seenBefore window skip. */
240
- _internIds: WeakMap<Sema, number>;
241
- }
242
- /** Read a whole node's bytes. */
243
- export declare const ALL = 2147483647;
244
- /** Splice every chosen span in order — the whole cover as one byte string. */
245
- export declare function spliceAll(segs: Seg[]): Uint8Array | null;
246
- /** Whether a chosen span RESTATES the query rather than answering it: its
247
- * SUBSTITUTED bytes (an edge followed from a recognised site, not the
248
- * site's own literal text read back) already occur elsewhere in the query
249
- * — the same principle recall.ts's tiers apply to a whole-query projection
250
- * ("a projection that is a proper byte-subspan of the query restates part
251
- * of the question"). A LITERAL span (the site's own bytes, unchanged) is
252
- * exempt: naming what's already there at its OWN position is not a
253
- * substitution. A recognised site that is itself an entire PRIOR TURN of
254
- * a multi-turn query is exactly this shape: it carries a genuine learnt
255
- * continuation, but that continuation is something the asker already said
256
- * moments later in the SAME query, not a new answer. Below one river
257
- * window, byte overlap is chance, not evidence — the same floor
258
- * identityBar and reachThreshold hold every other structural-overlap claim
259
- * to. */
260
- export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: number, W: number): boolean;
261
- /** Lift the answer out of the cover for think: the recognised region, free of
262
- * the asker's surrounding (unrecognised) framing — and free of any chosen
263
- * span that only RESTATES content the query already contains (see {@link
264
- * segRestatesQuery}). A restating span is excluded from both the framing
265
- * (lo/hi) decision and the final concatenation: it is stale, not a second
266
- * answer, but the OTHER spans a derivation chose are independent evidence
267
- * and must not be discarded along with it. */
268
- export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
269
- /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
270
- * tracked deposit interned (`prevSeen`). */
271
- export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];
@@ -1,130 +0,0 @@
1
- // types.ts — all interfaces, types, and free functions for the mind.
2
- //
3
- // GraphSearchHost is defined first (minimal imports) so GraphSearch can import
4
- // it without pulling in the full MindContext.
5
- import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
6
- import { dominates } from "../geometry.js";
7
- // ═══════════════════════════════════════════════════════════════════════════
8
- // FREE FUNCTIONS (pure, no state)
9
- // ═══════════════════════════════════════════════════════════════════════════
10
- /** Read a whole node's bytes. */
11
- export const ALL = 0x7fffffff;
12
- /** Splice every chosen span in order — the whole cover as one byte string. */
13
- export function spliceAll(segs) {
14
- if (!segs.some((s) => s.rec))
15
- return null;
16
- return concatBytes(segs.map((s) => s.bytes));
17
- }
18
- /** Whether a chosen span RESTATES the query rather than answering it: its
19
- * SUBSTITUTED bytes (an edge followed from a recognised site, not the
20
- * site's own literal text read back) already occur elsewhere in the query
21
- * — the same principle recall.ts's tiers apply to a whole-query projection
22
- * ("a projection that is a proper byte-subspan of the query restates part
23
- * of the question"). A LITERAL span (the site's own bytes, unchanged) is
24
- * exempt: naming what's already there at its OWN position is not a
25
- * substitution. A recognised site that is itself an entire PRIOR TURN of
26
- * a multi-turn query is exactly this shape: it carries a genuine learnt
27
- * continuation, but that continuation is something the asker already said
28
- * moments later in the SAME query, not a new answer. Below one river
29
- * window, byte overlap is chance, not evidence — the same floor
30
- * identityBar and reachThreshold hold every other structural-overlap claim
31
- * to. */
32
- export function segRestatesQuery(s, query, queryLen, W) {
33
- if (!s.rec)
34
- return false;
35
- const literal = s.j - s.i === s.bytes.length &&
36
- bytesEqual(s.bytes, query.subarray(s.i, s.j));
37
- if (literal)
38
- return false;
39
- return s.bytes.length >= W && s.bytes.length < queryLen &&
40
- indexOf(query, s.bytes, 0) >= 0;
41
- }
42
- /** Lift the answer out of the cover for think: the recognised region, free of
43
- * the asker's surrounding (unrecognised) framing — and free of any chosen
44
- * span that only RESTATES content the query already contains (see {@link
45
- * segRestatesQuery}). A restating span is excluded from both the framing
46
- * (lo/hi) decision and the final concatenation: it is stale, not a second
47
- * answer, but the OTHER spans a derivation chose are independent evidence
48
- * and must not be discarded along with it. */
49
- export function liftAnswer(segs, queryLen, query, W) {
50
- const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
51
- const recognised = [];
52
- for (let k = 0; k < segs.length; k++) {
53
- if (segs[k].rec && !restated[k])
54
- recognised.push(k);
55
- }
56
- if (recognised.length === 0)
57
- return null;
58
- if (recognised.length === 1) {
59
- const s = segs[recognised[0]];
60
- // A COMPUTED span's query-side width is operand digit-count, not
61
- // evidence of how much of the query's meaning it accounts for — the
62
- // half-dominance check below (built for a genuinely RECOGNISED learned
63
- // form) is not a valid framing signal for it (see the `computed` field
64
- // doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
65
- // because the operands are big, not because the framing matters less.
66
- // A LITERAL PREFIX before a computed span is unambiguous framing
67
- // regardless of width — an arithmetic expression is never itself
68
- // preceded by more literal computed content, so anything literal before
69
- // it is question wording ("what is ", "compute ") to lift clear of.
70
- // With no prefix (s.i === 0) the span is judged by the ordinary
71
- // half-dominance rule below, which already correctly keeps a short
72
- // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
73
- if (s.computed && s.i > 0)
74
- return s.bytes;
75
- if (dominates(s.j - s.i, queryLen)) {
76
- return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
77
- }
78
- return s.bytes;
79
- }
80
- const lo = recognised[0];
81
- const hi = recognised[recognised.length - 1];
82
- return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
83
- }
84
- /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
85
- * tracked deposit interned (`prevSeen`). */
86
- export function changedNodes(tree, ids, prevSeen) {
87
- const newCount = new Map();
88
- const count = (n) => {
89
- const memo = newCount.get(n);
90
- if (memo !== undefined)
91
- return memo;
92
- const id = ids.get(n);
93
- // PRUNE: a node whose id the previous deposit already interned is old,
94
- // and content addressing makes that transitive — the same id names the
95
- // same content, so every descendant was interned then too. The whole
96
- // subtree counts 0 without walking it; with the pyramid fold sharing a
97
- // conversation's prefix subtree, this is what keeps the changed-nodes
98
- // read O(new nodes) instead of O(context). (A node internTreeIds
99
- // memo-skipped has an id here exactly when it is such a shared root.)
100
- if (id !== undefined && prevSeen.has(id)) {
101
- newCount.set(n, 0);
102
- return 0;
103
- }
104
- let c = 1; // reachable only when NOT pruned above ⇒ this node is new
105
- if (n.kids) {
106
- for (const k of n.kids)
107
- c += count(k);
108
- }
109
- newCount.set(n, c);
110
- return c;
111
- };
112
- const total = count(tree);
113
- if (total === 0)
114
- return [tree];
115
- let n = tree;
116
- for (;;) {
117
- if (n.kids === null)
118
- return [n];
119
- let holder = null;
120
- for (const k of n.kids) {
121
- if (newCount.get(k) === total) {
122
- holder = k;
123
- break;
124
- }
125
- }
126
- if (holder === null)
127
- return [n];
128
- n = holder;
129
- }
130
- }
@@ -1,113 +0,0 @@
1
- /** External ids are integers only. */
2
- export type ExternalId = number;
3
- export interface DatabaseOptions {
4
- /** Path to the SQLite database file (use ":memory:" for a transient store). */
5
- dbPath: string;
6
- /** Vector dimensionality. Required for a new database; ignored when reopening. */
7
- dim?: number;
8
- /** Query breadth: clusters probed per query = ceil(efSearch / 4). Default
9
- * 64 (16 probes). Tunable at runtime and at reopen. */
10
- efSearch?: number;
11
- /** Bits used to scalar-quantise the query for the RaBitQ estimator.
12
- * Query-side only (stored codes are independent of it), so it is tunable
13
- * at reopen, like efSearch. Default 8. */
14
- queryBits?: number;
15
- /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
16
- rotationRounds?: number;
17
- /** Seed for the rotation. Default fixed. */
18
- seed?: number;
19
- /** Optional centroid the vectors are centered by before quantisation. */
20
- centroid?: ArrayLike<number>;
21
- /** SQLite page-cache budget in MiB. Purely a latency knob: correctness and
22
- * the per-operation storage-read count are identical with it at 0. */
23
- cacheSizeMb?: number;
24
- }
25
- export interface QueryResult {
26
- id: ExternalId;
27
- /** Estimated cosine distance (1 - cosine). */
28
- distance: number;
29
- }
30
- export interface StorageStats {
31
- float32BytesPerVector: number;
32
- codeBytesPerVector: number;
33
- bytesPerVector: number;
34
- compressionRatio: number;
35
- }
36
- /**
37
- * A persistent vector database: an adaptive PARTITIONED (IVF) index over
38
- * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`. See
39
- * ivf.ts for the index design. The original float vectors are never
40
- * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
41
- *
42
- * Inserting is route-and-append: cost is essentially FLAT in collection size
43
- * (one RAM scan of the pivot table + one chunk append — no per-insert graph
44
- * walk). Query cost is bounded by nprobe × cluster size — constant in N
45
- * once the collection is past its first splits.
46
- */
47
- export declare class VectorDatabase {
48
- private readonly index;
49
- constructor(options: DatabaseOptions);
50
- get dim(): number;
51
- /** Number of live (non-deleted) vectors. */
52
- get size(): number;
53
- /** Physical slot count including tombstones from deletes/updates. */
54
- get physicalSize(): number;
55
- /** Number of clusters (partitions) currently in the index. */
56
- get clusterCount(): number;
57
- get efSearch(): number;
58
- set efSearch(value: number);
59
- /** Distance computations performed during the most recent query. */
60
- get lastQueryDistanceComputations(): number;
61
- /** Storage row reads issued by the most recent query — the honest,
62
- * cache-independent scalability metric. */
63
- get lastQueryStorageReads(): number;
64
- get storage(): StorageStats;
65
- private checkId;
66
- /** Convert a value to code bytes, selecting by length: `codeWords`
67
- * elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
68
- private toCodeBytes;
69
- has(id: ExternalId): boolean;
70
- /** Stream every live external id (bounded memory). */
71
- keys(): IterableIterator<ExternalId>;
72
- /** Stream live entries whose INTERNAL id is > `after` — a durable
73
- * incremental watermark (internal ids are monotone at insert and preserved
74
- * by update and compact). */
75
- keysSince(after: number): IterableIterator<{
76
- ext: ExternalId;
77
- internal: number;
78
- }>;
79
- /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
80
- get(id: ExternalId): Uint32Array | null;
81
- insert(id: ExternalId, value: ArrayLike<number>): void;
82
- update(id: ExternalId, value: ArrayLike<number>): void;
83
- upsert(id: ExternalId, value: ArrayLike<number>): void;
84
- /**
85
- * Upsert many vectors under ONE transaction — one WAL commit for the whole
86
- * batch instead of one per vector, with chunk appends to the same cluster
87
- * coalesced in the index's write-back buffer.
88
- */
89
- upsertMany(entries: Array<{
90
- id: ExternalId;
91
- vector: ArrayLike<number>;
92
- }>): void;
93
- /** Delete many ids under ONE transaction. Absent ids are skipped.
94
- * Returns the number of vectors actually removed. */
95
- deleteMany(ids: ExternalId[]): number;
96
- /** Delete the vector bound to an id. Returns false if absent. */
97
- delete(id: ExternalId): boolean;
98
- /** Heat the SQLite page cache with sequential scans (latency only). */
99
- warmCache(): number;
100
- /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
101
- * VACUUM. Internal ids and cluster assignment are preserved. */
102
- compact(): void;
103
- /**
104
- * k-NN search. The argument's length selects the mode:
105
- * - `dim` elements → a raw vector (accurate estimator)
106
- * - `codeWords` elements → a 1-bit code, by sign-bit Hamming
107
- */
108
- query(query: ArrayLike<number>, k?: number, opts?: {
109
- ef?: number;
110
- }): QueryResult[];
111
- /** Close the underlying database. The instance must not be used afterwards. */
112
- close(): void;
113
- }