@hviana/sema 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/AGENTS.md +115 -53
  2. package/HOW_IT_WORKS.md +291 -184
  3. package/jsr.json +6 -0
  4. package/package.json +1 -1
  5. package/src/mind/bridge.ts +10 -104
  6. package/src/mind/match.ts +497 -7
  7. package/src/mind/{prefix-completion.ts → mechanisms/prefix-completion.ts} +66 -92
  8. package/src/mind/mechanisms/recall.ts +38 -129
  9. package/src/mind/mechanisms/reference.ts +343 -0
  10. package/src/mind/mind.ts +12 -8
  11. package/src/mind/pipeline-mechanism.ts +120 -3
  12. package/src/mind/pipeline.ts +16 -2
  13. package/src/mind/traverse.ts +92 -1
  14. package/src/store.ts +13 -4
  15. package/test/33-multi-candidate.test.mjs +21 -11
  16. package/test/70-prefix-completion.test.mjs +1 -1
  17. package/test/72-prefix-candidate-supply.test.mjs +7 -9
  18. package/test/74-prefix-trap-not-sprung-early.test.mjs +1 -1
  19. package/test/76-reference-binding.test.mjs +543 -0
  20. package/dist/example/demo.d.ts +0 -1
  21. package/dist/example/demo.js +0 -39
  22. package/dist/example/train_base.d.ts +0 -93
  23. package/dist/example/train_base.js +0 -2269
  24. package/dist/src/alphabet.d.ts +0 -7
  25. package/dist/src/alphabet.js +0 -33
  26. package/dist/src/alu/src/alu.d.ts +0 -185
  27. package/dist/src/alu/src/alu.js +0 -440
  28. package/dist/src/alu/src/expr.d.ts +0 -61
  29. package/dist/src/alu/src/expr.js +0 -318
  30. package/dist/src/alu/src/index.d.ts +0 -11
  31. package/dist/src/alu/src/index.js +0 -19
  32. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  33. package/dist/src/alu/src/kernel-arith.js +0 -264
  34. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  35. package/dist/src/alu/src/kernel-bits.js +0 -152
  36. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  37. package/dist/src/alu/src/kernel-logic.js +0 -60
  38. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  39. package/dist/src/alu/src/kernel-nd.js +0 -208
  40. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  41. package/dist/src/alu/src/kernel-numeric.js +0 -366
  42. package/dist/src/alu/src/operation.d.ts +0 -168
  43. package/dist/src/alu/src/operation.js +0 -189
  44. package/dist/src/alu/src/parser.d.ts +0 -221
  45. package/dist/src/alu/src/parser.js +0 -577
  46. package/dist/src/alu/src/resonance.d.ts +0 -55
  47. package/dist/src/alu/src/resonance.js +0 -126
  48. package/dist/src/alu/src/text.d.ts +0 -31
  49. package/dist/src/alu/src/text.js +0 -73
  50. package/dist/src/alu/src/value.d.ts +0 -109
  51. package/dist/src/alu/src/value.js +0 -300
  52. package/dist/src/alu/test/alu.test.d.ts +0 -1
  53. package/dist/src/alu/test/alu.test.js +0 -764
  54. package/dist/src/bytes.d.ts +0 -14
  55. package/dist/src/bytes.js +0 -59
  56. package/dist/src/canon.d.ts +0 -45
  57. package/dist/src/canon.js +0 -85
  58. package/dist/src/config.d.ts +0 -111
  59. package/dist/src/config.js +0 -91
  60. package/dist/src/derive/src/deduction.d.ts +0 -136
  61. package/dist/src/derive/src/deduction.js +0 -159
  62. package/dist/src/derive/src/index.d.ts +0 -8
  63. package/dist/src/derive/src/index.js +0 -11
  64. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  65. package/dist/src/derive/src/priority-queue.js +0 -73
  66. package/dist/src/derive/src/rewrite.d.ts +0 -56
  67. package/dist/src/derive/src/rewrite.js +0 -100
  68. package/dist/src/derive/src/trie.d.ts +0 -90
  69. package/dist/src/derive/src/trie.js +0 -217
  70. package/dist/src/derive/test/derive.test.d.ts +0 -1
  71. package/dist/src/derive/test/derive.test.js +0 -122
  72. package/dist/src/extension.d.ts +0 -37
  73. package/dist/src/extension.js +0 -7
  74. package/dist/src/geometry.d.ts +0 -233
  75. package/dist/src/geometry.js +0 -1089
  76. package/dist/src/index.d.ts +0 -17
  77. package/dist/src/index.js +0 -19
  78. package/dist/src/ingest-cache.d.ts +0 -41
  79. package/dist/src/ingest-cache.js +0 -165
  80. package/dist/src/meter.d.ts +0 -176
  81. package/dist/src/meter.js +0 -274
  82. package/dist/src/mind/articulation.d.ts +0 -6
  83. package/dist/src/mind/articulation.js +0 -99
  84. package/dist/src/mind/attention.d.ts +0 -430
  85. package/dist/src/mind/attention.js +0 -2418
  86. package/dist/src/mind/bridge.d.ts +0 -39
  87. package/dist/src/mind/bridge.js +0 -998
  88. package/dist/src/mind/canonical.d.ts +0 -34
  89. package/dist/src/mind/canonical.js +0 -93
  90. package/dist/src/mind/frame-filler.d.ts +0 -15
  91. package/dist/src/mind/frame-filler.js +0 -535
  92. package/dist/src/mind/graph-search.d.ts +0 -294
  93. package/dist/src/mind/graph-search.js +0 -996
  94. package/dist/src/mind/index.d.ts +0 -9
  95. package/dist/src/mind/index.js +0 -5
  96. package/dist/src/mind/junction.d.ts +0 -137
  97. package/dist/src/mind/junction.js +0 -358
  98. package/dist/src/mind/learning.d.ts +0 -75
  99. package/dist/src/mind/learning.js +0 -265
  100. package/dist/src/mind/match.d.ts +0 -192
  101. package/dist/src/mind/match.js +0 -666
  102. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  103. package/dist/src/mind/mechanisms/alu.js +0 -36
  104. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  105. package/dist/src/mind/mechanisms/cast.js +0 -1186
  106. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  107. package/dist/src/mind/mechanisms/confluence.js +0 -271
  108. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  109. package/dist/src/mind/mechanisms/cover.js +0 -232
  110. package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
  111. package/dist/src/mind/mechanisms/extraction.js +0 -327
  112. package/dist/src/mind/mechanisms/recall.d.ts +0 -16
  113. package/dist/src/mind/mechanisms/recall.js +0 -558
  114. package/dist/src/mind/mind.d.ts +0 -386
  115. package/dist/src/mind/mind.js +0 -754
  116. package/dist/src/mind/pipeline-mechanism.d.ts +0 -183
  117. package/dist/src/mind/pipeline-mechanism.js +0 -521
  118. package/dist/src/mind/pipeline.d.ts +0 -49
  119. package/dist/src/mind/pipeline.js +0 -331
  120. package/dist/src/mind/prefix-completion.d.ts +0 -59
  121. package/dist/src/mind/prefix-completion.js +0 -270
  122. package/dist/src/mind/primitives.d.ts +0 -85
  123. package/dist/src/mind/primitives.js +0 -333
  124. package/dist/src/mind/rationale.d.ts +0 -139
  125. package/dist/src/mind/rationale.js +0 -163
  126. package/dist/src/mind/reasoning.d.ts +0 -42
  127. package/dist/src/mind/reasoning.js +0 -305
  128. package/dist/src/mind/recognition.d.ts +0 -20
  129. package/dist/src/mind/recognition.js +0 -696
  130. package/dist/src/mind/resonance.d.ts +0 -35
  131. package/dist/src/mind/resonance.js +0 -365
  132. package/dist/src/mind/trace.d.ts +0 -15
  133. package/dist/src/mind/trace.js +0 -73
  134. package/dist/src/mind/traverse.d.ts +0 -158
  135. package/dist/src/mind/traverse.js +0 -716
  136. package/dist/src/mind/types.d.ts +0 -418
  137. package/dist/src/mind/types.js +0 -179
  138. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  139. package/dist/src/rabitq-ivf/src/database.js +0 -201
  140. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  141. package/dist/src/rabitq-ivf/src/index.js +0 -4
  142. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  143. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  144. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  145. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  146. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  147. package/dist/src/rabitq-ivf/src/rabitq.js +0 -313
  148. package/dist/src/sema.d.ts +0 -41
  149. package/dist/src/sema.js +0 -77
  150. package/dist/src/store-sqlite.d.ts +0 -184
  151. package/dist/src/store-sqlite.js +0 -942
  152. package/dist/src/store.d.ts +0 -762
  153. package/dist/src/store.js +0 -1823
  154. package/dist/src/vec.d.ts +0 -31
  155. package/dist/src/vec.js +0 -109
  156. package/src/mind/frame-filler.ts +0 -604
  157. package/test/69-frame-filler.test.mjs +0 -115
@@ -1,1089 +0,0 @@
1
- // geometry.ts — every modality is a stream; geometry is only a reading order.
2
- //
3
- // 1. Each byte is a leaf — an atom carrying its own vector straight from
4
- // the alphabet.
5
- // 2. The river folds leaves upward in fixed-size groups (maxGroup). Items
6
- // that cross the stable-prefix boundary are split so the prefix folds
7
- // identically regardless of what follows — pure structural stability.
8
- // 3. The same rule recurses level after level until one root remains.
9
- import { addInto, copy, normalize, zeros } from "./vec.js";
10
- import { sema, twoEndedSeat } from "./sema.js";
11
- // ---- geometric constants ----
12
- //
13
- // Every threshold below is a derived function of the fold's own geometry —
14
- // dimension D, maxGroup, etc. — never a tuned magic number. They live here
15
- // (not in a config file) because they follow from the structure itself.
16
- //
17
- // MEASUREMENT CAVEAT: these thresholds are compared against RaBitQ-ESTIMATED
18
- // cosines (1-bit stored codes scored against a 4-bit-quantized query; the
19
- // index never reranks with exact vectors). The derivations assume an exact
20
- // cosine; the estimator adds a small, rotation-uniformised error the bars do
21
- // not model. This is benign for the inequality thresholds (they gate broad
22
- // regions), but it means NO decision may treat an estimated score as exact —
23
- // identity in particular is decided by content-addressed resolve(), never by
24
- // `score >= 1` (see recallByResonance tier 0).
25
- /** The store's geometric identity bar: cosine ≥ 1 − 1/√D is the similarity at
26
- * which `intern` already treats two gists as the SAME node. Recall reuses it
27
- * to accept a near-identical query, and the climb to accept a containing form —
28
- * one derived constant, never a tuned threshold. NOTE: this fixed bar is
29
- * the ESTIMATOR floor of an identity claim; a whole-span claim over a span
30
- * longer than the perception quantum must use the scale-aware
31
- * {@link identityBar}, which converts the tolerated fraction into bytes. */
32
- export function mergeThreshold(D) {
33
- return 1 - 1 / Math.sqrt(D);
34
- }
35
- /** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
36
- * of `len` bytes. Under the linear fold a cosine reads "fraction of aligned
37
- * shared bytes", so a FIXED cosine bar admits a byte budget that grows with
38
- * the span: 1 − 1/√D over a 4·√D-byte span tolerates four whole river
39
- * windows of foreign content while still claiming "near-identical". An
40
- * identity claim may tolerate at most ONE river window W — the perception
41
- * quantum, the same single-window budget near-dedup's differsByOneWindow
42
- * grants — so the bar is 1 − W/len, floored at mergeThreshold(D), below
43
- * which the RaBitQ estimator cannot certify identity anyway. This is the
44
- * angle+magnitude form of the identity test: the ANGLE carries the shared
45
- * fraction, the span's MAGNITUDE (√len, the linear fold's own norm) converts
46
- * the tolerated fraction into tolerated bytes. Derived from W, D and the
47
- * span; never tuned. */
48
- export function identityBar(D, maxGroup, len) {
49
- return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
50
- }
51
- /** The reach bar: half a river quantum, derived from the fold's own geometry.
52
- * A branch folds up to `maxGroup` children, so two forms that differ in ONE
53
- * whole child — the smallest distinction perception can mean — sit at cosine
54
- * ≈ 1 − 1/maxGroup. Half that quantum, 1 − 1/(2·maxGroup), is closer than any
55
- * single-child difference can be: a positional echo of the same content.
56
- *
57
- * This is an EQUAL-ARITY replacement law. The two-ended coordinate frame is
58
- * a bijective relabelling of the seats inside that node, so it does not change
59
- * the one-child overlap or this bar. Stability under a leading/trailing
60
- * insertion comes from preserving content-defined subtrees and their anchored
61
- * coordinates — never from lowering the confidence floor.
62
- *
63
- * Recall uses this as its confidence floor: a query whose nearest resonant
64
- * form sits below this bar is structurally unrelated to everything in the store
65
- * — further than any single-child variant — and the system returns null rather
66
- * than fabricate an answer from an unrelated form. Derived, never tuned. */
67
- export function reachThreshold(maxGroup) {
68
- return 1 - 1 / (2 * maxGroup);
69
- }
70
- /** The estimator's own noise floor: 1/√D — ONE standard deviation of the
71
- * cosine between two independent random vectors in D dimensions (the same σ
72
- * {@link significanceBar} takes three of). It is the smallest difference in
73
- * cosine that is distinguishable from the rotation-uniformised RaBitQ
74
- * estimation error (see the MEASUREMENT CAVEAT above): a contrastive margin
75
- * below it is quantisation noise, not evidence. The consensus climb gates a
76
- * region's vote on its discriminative margin clearing this floor — the
77
- * minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
78
- * Derived, never tuned. */
79
- export function estimatorNoise(D) {
80
- return 1 / Math.sqrt(D);
81
- }
82
- /** The statistical-significance bar for whole-query resonance: 3/√D.
83
- * In D dimensions the expected cosine of two independent random vectors is 0
84
- * with standard deviation 1/√D. A cosine ≥ 3/√D is three standard deviations
85
- * above chance — the query is statistically related to the store, not merely
86
- * sharing random byte noise. Below this bar the consensus climb (which trusts
87
- * sub-region resonance) is skipped: there is no evidence the query belongs to
88
- * the same distribution as the stored content. Derived, never tuned. */
89
- export function significanceBar(D) {
90
- return 3 / Math.sqrt(D);
91
- }
92
- /** The concept (halo) threshold: the cosine above which two nodes share a
93
- * distributional concept. A halo is a superposition of episode signatures in
94
- * D-dimensional space, so the expected cosine between two unrelated halos is 0
95
- * with standard deviation 1/√D. The structural midpoint 0.5 separates "more
96
- * similar than not" from noise; the +0.5/√D term adds one half-sigma margin
97
- * that vanishes as D → ∞, accounting for the wider noise band at lower D
98
- * without inventing a tuned constant. At D=1024 this gives 0.516, within
99
- * 3% of 0.5 — existing behavior is preserved while threshold and D move
100
- * together. Derived, never tuned. */
101
- export function conceptThreshold(D) {
102
- return 0.5 + 0.5 / Math.sqrt(D);
103
- }
104
- /** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
105
- * whole of `wholeLen` covers STRICTLY more than half of it. A span that
106
- * dominates its whole can no longer discriminate the whole's own content —
107
- * the one test behind liftAnswer's keep-the-frame rule, collectRegions'
108
- * wrapper exclusion, and CAST's frame-depth majority (each cites this).
109
- * CAST's frame-FRACTION gate is the deliberately CLOSED variant (≥ ½ is
110
- * already unusable there) and stays inline where it is documented.
111
- * Derived from the structural midpoint, never tuned. */
112
- export function dominates(partLen, wholeLen) {
113
- return partLen * 2 > wholeLen;
114
- }
115
- /** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
116
- * of learnt contexts (edge sources). A single region's IDF-weighted vote for
117
- * an anchor reached through c contexts is at most ln(N/c) ≤ ln(N); the +1/2
118
- * demands the pooled vote exceed what ONE maximally-specific region could
119
- * contribute by half a unit — i.e. genuine corroboration beyond a lone
120
- * region's echo at this corpus scale. The ONE floor both consumers gate on:
121
- * recallByResonance trusting a climb anchor, and commitVotes admitting a
122
- * further point of attention. Defined once here so the two can never
123
- * drift apart. Derived from N, never tuned. */
124
- export function consensusFloor(N) {
125
- return Math.log(N) + 1 / 2;
126
- }
127
- /** The coverage bar for the reach (interior) index, when vector-similarity
128
- * gating is used. Returns the concept threshold — the structural midpoint
129
- * (~0.5 at D=1024) where two forms are "more similar than not."
130
- *
131
- * Currently UNUSED in the hot training path: interior nodes are indexed
132
- * unconditionally (hash-cons dedup bounds the index naturally).
133
- * Post-hoc structural compaction ({@link Store.compactContentIndex})
134
- * replaces runtime coverage gating with a batch pass that removes
135
- * structurally-isolated entries. Derived, never tuned. */
136
- export function coverageBar(_maxGroup, D) {
137
- return conceptThreshold(D);
138
- }
139
- // ---- folding ----
140
- //
141
- // The river fold is a hierarchical prefix network: each level contracts
142
- // groups of `maxGroup` adjacent items into one via permute-then-add
143
- // (positional seat binding), recursing until one root remains.
144
- //
145
- // FLAT per-level fold — one inline loop per level (foldSlice): no per-group
146
- // function calls, no Array.slice per group, the permute and add FUSED
147
- // (`gist[d] += v[seat[d]]`, no scratch buffer), and subtree byte lengths
148
- // carried incrementally on Folded (the old boundary scan re-walked subtrees
149
- // every level — O(n log n)). The per-level SUPERPOSITION is byte-identical
150
- // to the original recursive foldGroup: the same FP additions in the same
151
- // order.
152
- //
153
- // LINEAR fold — intermediate gists are NOT normalized; only the final root is
154
- // (riverFold's single normalize). This is a deliberate change of similarity
155
- // semantics from the original per-group normalize, not a cached optimization:
156
- // the fold is now a pure linear operator — a superposition of positionally-
157
- // bound leaf vectors — so an interior node carries its span's natural
158
- // magnitude and a resonance score reads as byte-proportional overlap rather
159
- // than a scale-free cosine. The mechanisms that depend on that reading
160
- // compensate for it EXPLICITLY, never silently: the contrastive margin on
161
- // approximate votes (voteRegions), company signatures decoupling halo company
162
- // from gist content (sema.ts), and the shared-frame analogy tier (match.ts).
163
- /** Fold `items[start .. start+count)` in groups of `mg` into `out`.
164
- *
165
- * With `force`, the trailing incomplete group (2..mg-1 items) is folded as
166
- * well — only a lone singleton passes through. The river always folds with
167
- * force: every level contracts by ~mg, so the tree's DEPTH is a function of
168
- * ceil(log_mg(n)) alone. Letting leftovers pass through unfolded made depth
169
- * depend on the exact byte count (39 bytes folded in 3 levels, 41 in 4), and
170
- * each extra level applies another seat permutation to the whole gist —
171
- * near-identical inputs straddling such a cliff read as orthogonal
172
- * (measured: 33-byte-identical prefixes at cos ≈ 0). */
173
- function foldSlice(space, items, start, count, out, force) {
174
- const mg = space.maxGroup;
175
- const D = space.D;
176
- const complete = count - (count % mg);
177
- const foldAt = (at, size) => {
178
- const gist = new Float32Array(D);
179
- const kids = new Array(size);
180
- let len = 0;
181
- for (let k = 0; k < size; k++) {
182
- const f = items[at + k];
183
- const slot = twoEndedSeat(space.seats.length, size, k);
184
- const seat = space.seats[slot].fwd;
185
- const v = f.tree.v;
186
- // Fused permute-and-accumulate — same FP ops, same order as the old
187
- // permuteInto + addInto pair, with no scratch buffer.
188
- for (let d = 0; d < D; d++)
189
- gist[d] += v[seat[d]];
190
- kids[k] = f.tree;
191
- len += f.len;
192
- }
193
- out.push({ tree: sema(gist, null, kids), len });
194
- };
195
- for (let i = 0; i < complete; i += mg)
196
- foldAt(start + i, mg);
197
- const leftover = count - complete;
198
- if (leftover === 0)
199
- return;
200
- if (force && leftover >= 2)
201
- foldAt(start + complete, leftover);
202
- else
203
- for (let i = complete; i < count; i++)
204
- out.push(items[start + i]);
205
- }
206
- function riverFold(space, row, stableBytes) {
207
- if (row.length === 0) {
208
- const z = new Float32Array(space.D);
209
- return { tree: sema(z, new Uint8Array(0), null), len: 0 };
210
- }
211
- let level = row;
212
- while (level.length > 1) {
213
- // Find the item index where accumulated bytes reaches stableBytes.
214
- let boundary = level.length;
215
- if (stableBytes > 0) {
216
- let acc = 0;
217
- for (let i = 0; i < level.length; i++) {
218
- acc += level[i].len;
219
- if (acc >= stableBytes) {
220
- boundary = i + 1;
221
- break;
222
- }
223
- }
224
- }
225
- const next = [];
226
- if (boundary < level.length) {
227
- // Prefix folds independently of the suffix — structural stability.
228
- foldSlice(space, level, 0, boundary, next, true);
229
- foldSlice(space, level, boundary, level.length - boundary, next, true);
230
- }
231
- else {
232
- foldSlice(space, level, 0, level.length, next, true);
233
- }
234
- level = next;
235
- }
236
- // LINEAR fold — this root normalize is the ONLY normalize of the entire
237
- // fold; every intermediate gist stays unnormalized (see the folding
238
- // header). Skipped for a single-leaf input: that root IS the shared
239
- // alphabet vector (already unit), and normalizing in place would mutate the
240
- // alphabet itself.
241
- if (row.length > 1)
242
- normalize(level[0].tree.v);
243
- return level[0];
244
- }
245
- // ---- public API ----
246
- function bytesToLeaves(alphabet, bytes) {
247
- return Array.from(bytes, (b, i) => {
248
- const v = alphabet.vecs[b];
249
- return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
250
- });
251
- }
252
- /** CONTENT-DEFINED FOLD BOUNDARIES — where a byte stream segments, chosen by
253
- * the bytes rather than by arithmetic.
254
- *
255
- * `riverFold` groups FIXED arity from byte 0 and permutes item k by
256
- * `seats[k]`, k = index mod W, so a byte's contribution is a function of its
257
- * ABSOLUTE OFFSET: the same byte run is a different vector at a different
258
- * position, and the same content is a different SUBTREE. W's size has nothing
259
- * to do with it — any fixed modulus does this, and identity must not depend on
260
- * W at all.
261
- *
262
- * A rolling hash removes the dependence without touching the fold's arity: the
263
- * cut lands where the hash of the recent bytes vanishes mod W, so a change
264
- * upstream moves only the cut it falls inside — every downstream boundary, and
265
- * therefore every downstream segment, is unchanged. Because
266
- * {@link stablePrefixFold} folds each segment independently from its own slot
267
- * 0, byte-identical content then produces byte-identical subtrees, and
268
- * hash-consing makes it the SAME NODE ID wherever it appears. That — not the
269
- * root cosine, which should and does still move when content is added — is
270
- * what recognition, cover and resolve need.
271
- *
272
- * Measured over 400 real deposits, shifts of 1..7 bytes, downstream cuts
273
- * preserved / segments byte-identical:
274
- *
275
- * content-defined, deposit text 99.7% / 98.4% mean seg 6.61 B
276
- * content-defined, non-Latin scripts 99.6% / 98.3% mean seg 6.43 B
277
- * content-defined, random binary 99.9% / 99.2% mean seg 6.92 B
278
- * the arithmetic grid, same corpus 14.3% (only the k≡0 mod W
279
- * shifts survive)
280
- *
281
- * The three content-defined rows agreeing is the load-bearing part: this reads
282
- * BYTES, never text. Mind is not a text engine — the same fold carries grids
283
- * and any other modality — so a boundary rule justified by where words or
284
- * sentences fall would be importing an assumption the architecture rejects.
285
- * Random binary must, and does, behave exactly like prose.
286
- *
287
- * Every constant is derived (§2.2): the cut mask is W, so a cut is offered once
288
- * per quantum of bytes — which, composed with the minimum below, puts the
289
- * expected segment at minLen + W − 1 ≈ 6 B rather than at W, deliberately (see
290
- * the refutation recorded at `cutRate` in {@link contentLevels}: a segment is
291
- * the flat PHRASE-scale unit the W-ary groups are built from, not a group of W
292
- * children, and forcing E[len] = W costs 15 tests). The minimum is W−1, `canonicalWindows`'s
293
- * straddle neighbour and the write side's own floor for a unit; and the maximum
294
- * is the KEYRING's seat count, because a segment folds as ONE flat node and
295
- * `fold` has exactly that many seats to bind children into. Capping there is
296
- * what keeps the fold light: a segment of 3..seats leaves is a single node,
297
- * where splitting it into W-groups plus a remainder would cost two or three
298
- * and the remainders barely share (measured: partial-arity nodes 504 → 3,590,
299
- * and total distinct nodes 8,142 → 9,712, when segments folded as [W][rest]). */
300
- /** {@link contentBoundaries} plus, for each cut, its LEVEL — how deep in the
301
- * tree that cut reaches.
302
- *
303
- * One rolling hash serves every scale. A cut is level 0 when its hash vanishes
304
- * mod W, level 1 when mod W², and so on: level-L cuts are by construction a
305
- * subset of level-(L−1) cuts, which is exactly the nesting a tree needs. The
306
- * expected span of a level-L node is therefore W^(L+1) bytes — the same growth
307
- * the grid fold had, but with boundaries the content chose, so a shift moves
308
- * one node at each level instead of all of them.
309
- *
310
- * Levels are read from the hash the cut was ACCEPTED at, not recomputed, so
311
- * they cost nothing beyond the divisions already being done. */
312
- // Cyclic-polynomial table for the bounded-window cut hash. Derived once from
313
- // the fold's own mixing constant — no seed, no tuning. A byte contributes
314
- // BUZ[b] on entering the window; the hash rotates by one per byte, so by the
315
- // time that byte leaves, its contribution has travelled k places and is
316
- // removed rotated by k. The rotation is taken at the use site rather than
317
- // precomputed into a second table so the window width follows maxGroup
318
- // instead of being frozen at one value.
319
- const BUZ = new Uint32Array(256);
320
- {
321
- let x = 0x9e3779b9 >>> 0;
322
- for (let i = 0; i < 256; i++) {
323
- x = Math.imul(x ^ (x >>> 15), 2654435761) >>> 0;
324
- x = (x ^ (x >>> 13)) >>> 0;
325
- BUZ[i] = x;
326
- }
327
- }
328
- /** BUZ rotated by the window width — what a byte's contribution has become by
329
- * the time it leaves. Cached because the width follows `maxGroup`, which is
330
- * fixed for a given space: built once, then a plain table lookup per byte. */
331
- let buzOutTable = null;
332
- let buzOutWidth = -1;
333
- function buzOut(k) {
334
- if (buzOutWidth !== k || buzOutTable === null) {
335
- const t = new Uint32Array(256);
336
- for (let i = 0; i < 256; i++) {
337
- const v = BUZ[i];
338
- t[i] = ((v << k) | (v >>> (32 - k))) >>> 0;
339
- }
340
- buzOutTable = t;
341
- buzOutWidth = k;
342
- }
343
- return buzOutTable;
344
- }
345
- function contentLevels(space, bytes) {
346
- const W = space.maxGroup;
347
- const minLen = W - 1;
348
- const maxLen = space.seats.length;
349
- // MEASURED AND REFUTED — making E[segment] equal W. A segment is at least
350
- // `minLen` bytes and then cuts with probability p, so E[len] = minLen +
351
- // (1−p)/p; with `h % W === 0` that is minLen + W − 1 = 6 for W = 4 (measured
352
- // mean 6.61 B on real deposits), half again coarser than the fold's own
353
- // arity, and the doc above used to claim the mask made the two equal. It
354
- // does not, and the mismatch looked like the cause of a real deficit: the
355
- // climb perceives one region per segment, so on `Michelangelo is to sculpture
356
- // as who is to literature?` it sees 9 regions where the grid saw 19, reaches
357
- // one ranked anchor instead of two, and CAST's weave never includes the
358
- // writing-domain exemplar it needs (test/29 A2, C1).
359
- //
360
- // Fixing the rate to hit E[len] = W (a threshold `h % W < W − minLen + 1` on
361
- // the same rolling value, keeping the two bytes of history) does exactly what
362
- // it says — mean segment 3.60 B on text, 3.93 B on random binary, 14 segments
363
- // on that query instead of 9 — and takes the suite from 3 failures to 18,
364
- // across think, universality, scaling, decomposition, bridge, generalization,
365
- // saturation, audit and recognition. The coarser-than-arity scale is
366
- // load-bearing: W is the arity `riverFold` groups CHILDREN at, and a segment
367
- // is not a group of W children but the flat unit those groups are built from
368
- // — a phrase-scale unit, and the mechanisms above all read it that way.
369
- // Do not re-derive the rate from W.
370
- //
371
- // MEASURED AND REFUTED — reading the hash's HIGH bits instead of its low
372
- // ones. The accumulator shifts LEFT, so `h % W` is the part the mixing has
373
- // had no chance to reach: with `h = (h << 1) + byte·K` and K odd, `h mod 4`
374
- // reduces to `(2·(prev mod 2) + byte) mod 4` — two bytes of history, and only
375
- // their parity. It reads like a hash test that is really a byte-parity test,
376
- // and it shows: on `Michelangelo is to sculpture as who is to literature?`
377
- // the segments come out 8,8,8,8,8,7,4,2 — almost every boundary the FORCED
378
- // one at maxLen, which is the one offset-dependent cut in the rule.
379
- //
380
- // Testing the top bits instead (`h < 2^32 / W`, the same 1/W rate, with the
381
- // level test falling out as `h < 2^32 / W^(L+1)` — cleaner nesting than
382
- // divisibility) takes the suite from 2 failures to 21 AND breaks the
383
- // invariance floor in test/59. The reason is the whole point: in a
384
- // shift-accumulator the LOW bits have a short effective window and the HIGH
385
- // bits a long one, so a disturbance upstream perturbs the high bits for the
386
- // next ~32 bytes while the low bits re-sync within two. The narrow window is
387
- // not a defect in this hash — it IS the invariance. A boundary rule here is
388
- // choosing how far a change may propagate, not how well the bits mix.
389
- //
390
- // MEASURED AND REFUTED — normalized chunking, to shrink the forced cut. The
391
- // hard cut at maxLen is the ONE offset-dependent boundary left in the rule,
392
- // and it is not rare: 32% of segments on a mixed sample end there, five of
393
- // the eight in the query above, so for those streams the fold IS a grid of 2W
394
- // with all of the grid's phase. The standard remedy applies cleanly here —
395
- // past a target length, relax the mask by one power of the same radix
396
- // (W → W/2), still content-defined and so still invariant, just coarser
397
- // evidence for a boundary; both quantities are already in the rule, nothing
398
- // introduced. It works as advertised: forced cuts fall from 32% to 8.3% and
399
- // test/59's invariance floors still hold.
400
- //
401
- // And the suite goes from 2 failures to 5 relaxing at the expected length
402
- // (minLen + W − 1), or to 6 relaxing at the last opportunity (maxLen − 1) —
403
- // 22-multihop, 24-generalization, 29, 36. Taken with the two results above
404
- // (changing the rate costs 15 tests; changing which bits are read costs 19),
405
- // the reading is that the segment DISTRIBUTION is what the mechanisms
406
- // downstream are fitted to, not the purity of the rule that produces it. The
407
- // forced cut is part of that distribution. Do not tidy it away without
408
- // re-measuring everything that reads a region.
409
- // A BOUNDED-WINDOW rolling hash — the cut decision reads only the last
410
- // `k` bytes, so nothing before the window can reach it.
411
- //
412
- // The old hash, `h = (h<<1) + byte*K`, needed 32 shifts to drop a byte, so
413
- // it carried ~32 bytes of history; and on PERIODIC content its value was
414
- // periodic too, so the threshold either never fired or fired at a fixed
415
- // phase. Then the `maxLen` fallback placed every boundary at a fixed
416
- // offset from the previous one and the segmentation could never recover
417
- // from a shift. Measured fraction of cuts that re-align after a prepend:
418
- //
419
- // text uniform sparse lowent records ramp
420
- // old 0.870 0.902 0.492 0.879 0.888 0.441
421
- // this 0.935 0.952 0.732 0.920 0.916 0.935
422
- //
423
- // The cyclic polynomial (each byte enters as a table value, leaves rotated
424
- // by the window width) has EXACTLY k bytes of memory and scrambles periodic
425
- // input, so the threshold fires at content-chosen positions on a gradient
426
- // just as it does on text — which is what leaves the `maxLen` fallback
427
- // rarely engaged instead of carrying the phase. Segment lengths are
428
- // unchanged in distribution (mean 5.2-7.2 against the old 5.4-6.0), so the
429
- // mechanisms fitted to that distribution see the same scale.
430
- //
431
- // Cost is the same shape as before: shifts, XORs and two table lookups per
432
- // byte, no multiply and no auxiliary structure. (An exact sliding-window
433
- // minimum — winnowing — aligns slightly better still, 0.91-0.999, but its
434
- // deque costs 51 MB/s against this rule's 112 and buys nothing the
435
- // scrambling hash does not already give.)
436
- const k = W;
437
- const OUT = buzOut(k);
438
- const cuts = [];
439
- const levels = [];
440
- const n = bytes.length;
441
- let h = 0;
442
- let prev = 0;
443
- let recent = 0; // the last GAP raw hits, one bit each
444
- // A boundary is a property of a 4-GRAM, not of a position. The register IS
445
- // the window — it holds exactly the last `k` raw bytes — so the decision is
446
- // a pure function of those bytes and nothing else can reach it.
447
- //
448
- // What kept the OLD rule position-dependent was `minLen`, counted from the
449
- // previous cut: on periodic content that count carried the initial phase
450
- // forever and the segmentation never recovered from a shift. But its only
451
- // job was to stop segments being too short, and that can be said locally —
452
- // take a hit only when the previous GAP positions did NOT hit. Every term
453
- // is then a function of a bounded byte window (k + GAP), so the rule stays
454
- // a pure content property while still setting the segment scale. Measured
455
- // fraction of cuts that survive a prepend, worst case over six byte types:
456
- // 0.441 for the original rule, 0.769 counting from `last`, 0.847 for this.
457
- const GAP = 2;
458
- const GAPMASK = (1 << GAP) - 1;
459
- // The keyring bound is restored WITHOUT reintroducing a count: because
460
- // boundaries are content-determined, an over-long segment carries identical
461
- // bytes wherever it occurs, so splitting it at strides from ITS OWN start is
462
- // content-relative. (This holds only while such splits stay RARE — a split
463
- // leaves a right edge the content did not choose, so the next segment's
464
- // start is not content-determined either. At this rate they are: mean
465
- // segment 5.4 against a bound of 8. Lowering the cut rate to lengthen
466
- // segments makes forced splits dominant and alignment collapses — measured,
467
- // 0.000 on two-symbol data at rate 1/16.)
468
- const emit = (at, lvl) => {
469
- while (at - prev > maxLen) {
470
- prev += maxLen;
471
- cuts.push(prev);
472
- levels.push(0);
473
- }
474
- if (at <= prev || at >= n)
475
- return;
476
- cuts.push(at);
477
- levels.push(lvl);
478
- prev = at;
479
- };
480
- for (let i = 0; i < n; i++) {
481
- h = ((h << 8) | bytes[i]) >>> 0;
482
- if (i + 1 >= n)
483
- break;
484
- if (i < k - 1)
485
- continue;
486
- // Two-round avalanche. The window holds four RAW bytes, whose entropy may
487
- // sit in only a few bits (a gradient's low bits, a sparse stream's zeros);
488
- // one multiply leaves that structure partly intact and the boundary test
489
- // inherits it. A second round spreads every input bit across the word,
490
- // which is what makes the rule behave the same on a ramp as on prose.
491
- let mixv = Math.imul(h ^ (h >>> 16), 0x85ebca6b) >>> 0;
492
- mixv = Math.imul(mixv ^ (mixv >>> 13), 0xc2b2ae35) >>> 0;
493
- mixv = (mixv ^ (mixv >>> 16)) >>> 0;
494
- const hit = mixv % W === 0;
495
- if (hit && (recent & GAPMASK) === 0) {
496
- // Level: how many further powers of W divide the mixed value — level-L
497
- // cuts stay a subset of level-(L-1) cuts, the nesting the tree needs.
498
- let lvl = 0;
499
- let m = W;
500
- while (lvl < 24 && m <= 0x40000000 && mixv % (m * W) === 0) {
501
- lvl++;
502
- m *= W;
503
- }
504
- emit(i + 1, lvl);
505
- }
506
- recent = ((recent << 1) | (hit ? 1 : 0)) & GAPMASK;
507
- }
508
- while (n - prev > maxLen) {
509
- prev += maxLen;
510
- cuts.push(prev);
511
- levels.push(0);
512
- }
513
- return { cuts, levels };
514
- }
515
- export function contentBoundaries(space, bytes) {
516
- // ONE implementation of the rule. This used to carry its own copy of the
517
- // rolling-hash loop, which is exactly how a write side and a read side drift
518
- // apart without a type error; the levels are computed from the hash the cut
519
- // was accepted at, so asking for them costs nothing but an array.
520
- return contentLevels(space, bytes).cuts;
521
- }
522
- /** Find the longest prefix of `bytes` whose leaf-id signature matches a
523
- * known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
524
- export function knownPrefixLength(bytes, leafAt, lookup) {
525
- const leafIds = [];
526
- for (let i = 0; i < bytes.length; i++) {
527
- const lid = leafAt(i);
528
- if (lid === null)
529
- break;
530
- leafIds.push(lid);
531
- }
532
- // Match the longest PROPER prefix — a full-length match means the entire
533
- // input already exists as a stored form (e.g. the flat leaf-id branch
534
- // stored alongside the structural root). That would hide the true split
535
- // point and prevent the river from producing the same tree it folded
536
- // during training, so the structural recognition cannot find the right
537
- // forms. A proper prefix guarantees at least two regions.
538
- for (let len = bytes.length - 1; len >= 2; len--) {
539
- if (lookup(leafIds.slice(0, len)) !== null)
540
- return len;
541
- }
542
- return 0;
543
- }
544
- /** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
545
- * detecting previously-stored prefixes so the river can split at the
546
- * correct boundary. Pass them through from `perceive`; the geometry
547
- * computes the stable prefix internally.
548
- *
549
- * `boundaries` is the CALLER-computed stable-prefix boundary set (§10.3):
550
- * strictly-increasing proper byte offsets, each the length of a prefix that
551
- * is already a stored whole-stream form. When given, the fold splits into
552
- * the segments between consecutive boundaries — each folded independently,
553
- * exactly as it folded when it was learned — and the segment roots join
554
- * LEFT-NESTED (((s₀·s₁)·s₂)…), so every learnt cumulative-context root
555
- * reappears as an identical subtree (and, by hash-consing, the very same
556
- * node) inside the grown stream. This is what lets a conversation's next
557
- * turn extend perception instead of refolding it: identical prefixes
558
- * produce identical subtrees regardless of what follows them. */
559
- export function bytesToTree(space, alphabet, bytes, leafAt, lookup, boundaries) {
560
- if (bytes.length === 0) {
561
- return sema(alphabet.vecs[0], new Uint8Array(0), null);
562
- }
563
- // WHERE A STREAM SEGMENTS IS DECIDED BY ITS BYTES ({@link contentBoundaries}),
564
- // and the segments then fold BALANCED, W-ary, like any other row of items.
565
- //
566
- // The two must not be confused. Content cuts are what make identity
567
- // offset-free: a segment folds from its own slot 0, so the same bytes give the
568
- // same subtree wherever they sit. The shape ABOVE the segments is a separate
569
- // question, and it must stay the river's own — grouping W at a time, depth
570
- // log_W(n). Joining segments left-nested instead (as the stable-prefix fold
571
- // does, for its own good reason) costs a node per segment on a single spine:
572
- // at a cut every ~6.6 B a 3 KB deposit becomes a 450-deep spine of 450 fresh
573
- // D-vectors, ~1.8 MB for one deposit, and every walker above inherits the
574
- // depth. That is an implementation blunder, not a property of content-defined
575
- // folding, and it is what riverFold below avoids.
576
- //
577
- // Caller-supplied boundaries stay left-nested (see stablePrefixFold): there are
578
- // a handful of them, one per conversation turn, and the cumulative-context-root
579
- // contract depends on that shape. Each SPAN between them content-folds.
580
- const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
581
- const outer = new Set();
582
- if (boundaries !== undefined) {
583
- for (const b of boundaries)
584
- outer.add(b);
585
- }
586
- if (sb > 0)
587
- outer.add(sb);
588
- if (outer.size === 0) {
589
- return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
590
- }
591
- return stablePrefixFold(space, alphabet, bytes, [...outer].sort((a, b) => a - b));
592
- }
593
- /** One span, folded over its own content cuts — AT EVERY LEVEL.
594
- *
595
- * A segment becomes ONE FLAT NODE: every leaf bound into its own seat and
596
- * summed, arity = the segment's length. `contentBoundaries` caps a segment at
597
- * the keyring's seat count so this is always possible, and the flat form is
598
- * both lighter (one node per segment instead of a [W][remainder] pair) and the
599
- * natural unit — a segment IS the smallest thing the cuts claim is a unit.
600
- *
601
- * Above the segments the cutting RECURSES rather than reverting to the grid.
602
- * Grouping segment roots W-at-a-time from index 0 would reintroduce the very
603
- * bug content cuts exist to remove, one level up: a form spanning segments
604
- * 12..17 straddles the [12-15] and [16-17] groups and is no node at all, so
605
- * recognition can only reach it by an alignment accident (test/44 pins exactly
606
- * this — at HEAD the grid happened to put a node one byte before the target).
607
- * {@link contentLevels} assigns each cut a level from how divisible its hash
608
- * is, so level-L cuts are a subset of level-(L−1) cuts and every node at every
609
- * scale is delimited by content. Identity is then offset-free at all scales,
610
- * which is the whole requirement — it must not depend on W. */
611
- function contentFoldSpan(space, alphabet, bytes, from, to) {
612
- const span = bytes.subarray(from, to);
613
- const { cuts, levels } = contentLevels(space, span);
614
- const edges = [0, ...cuts, span.length];
615
- const segs = [];
616
- for (let i = 0; i + 1 < edges.length; i++) {
617
- segs.push(flatFold(space, alphabet, span, edges[i], edges[i + 1]));
618
- }
619
- if (segs.length > 1)
620
- return groupByLevel(space, segs, levels, 1);
621
- return segs[0];
622
- }
623
- /** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
624
- * fold of a byte-identical prefix already produced.
625
- *
626
- * WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES. A level-0 segment is a
627
- * pure function of its own bytes ({@link flatFold} reads nothing else), so
628
- * reusing one whose [start,end) is unchanged is bit-identical to refolding it
629
- * — the cache can never change the tree, only skip work. And the cuts
630
- * themselves are stable under APPEND: {@link contentLevels} decides each cut
631
- * from a rolling hash over a local window, so bytes added at the right edge
632
- * cannot move a cut to their left (measured over a growing 12-turn context:
633
- * 100% of prior cuts survive every append, zero tail churn). Together those
634
- * two facts are the whole optimisation — a grown stream refolds only the
635
- * segments at its right edge.
636
- *
637
- * This is the reuse the conversation path wants, and it costs NOTHING in
638
- * structure: the tree is exactly the tree {@link bytesToTree} builds for the
639
- * same bytes with no boundary set at all. Turn boundaries buy prefix-ROOT
640
- * identity, which is a different property from incremental reuse; conflating
641
- * the two is what put an imposed boundary set on the inference path and left
642
- * it folding differently from the deposits it was querying.
643
- *
644
- * `groupByLevel` above the segments is re-run whole. It operates on segment
645
- * ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
646
- * and only its right edge actually changes shape — measured at ~40 rebuilt
647
- * nodes per turn, flat as the context grows sevenfold.
648
- *
649
- * PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
650
- * `bytes`. Reuse is keyed on a segment's [start,end) OFFSETS, which is what
651
- * makes it O(1) per segment; offsets alone cannot witness that the underlying
652
- * bytes agree. Hand it a fold of DIFFERENT bytes whose cuts happen to land
653
- * in the same places and it will splice those foreign segments in — measured,
654
- * a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
655
- * streams. Verifying the bytes here would cost O(prefix) and defeat the
656
- * whole point, so the obligation sits with the caller, and every caller
657
- * discharges it structurally rather than by care: `perceiveDeposit` looks the
658
- * entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
659
- * ARE the cache key — and a conversation's fold state advances only by
660
- * append. A new caller that cannot make the same structural argument must
661
- * pass no `prev` at all; the cold path is always correct.
662
- * ({@link stablePrefixFoldIncremental} carries the identical precondition for
663
- * the identical reason.) */
664
- export function contentFoldIncremental(space, alphabet, bytes, prev) {
665
- if (bytes.length === 0) {
666
- return {
667
- tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
668
- fold: { edges: [0], segs: [] },
669
- };
670
- }
671
- const { cuts, levels } = contentLevels(space, bytes);
672
- const edges = [0, ...cuts, bytes.length];
673
- const segs = [];
674
- for (let i = 0; i + 1 < edges.length; i++) {
675
- const hit = prev !== undefined && prev.edges[i] === edges[i] &&
676
- prev.edges[i + 1] === edges[i + 1]
677
- ? prev.segs[i]
678
- : undefined;
679
- segs.push(hit ?? flatFold(space, alphabet, bytes, edges[i], edges[i + 1]));
680
- }
681
- const folded = segs.length > 1
682
- ? groupByLevel(space, segs, levels, 1)
683
- : segs[0];
684
- // THE ROOT IS NORMALIZED IN PLACE, A CACHED SEGMENT NEVER IS. With one
685
- // segment — or with a grouping that passes a lone item through — `folded`
686
- // IS a cached seg, and a later turn will reuse it as an interior node whose
687
- // magnitude must stay byte-proportional. Copy before normalizing, exactly
688
- // as the stable-prefix twin does. A single LEAF is copied too: its vector
689
- // is the shared alphabet entry and must never be written.
690
- const aliased = segs.some((s) => s.tree === folded.tree);
691
- let tree = folded.tree;
692
- if (aliased) {
693
- tree = tree.kids === null
694
- ? sema(tree.v, tree.leaf, null)
695
- : sema(Float32Array.from(tree.v), null, tree.kids);
696
- }
697
- if (tree.kids !== null)
698
- normalize(tree.v);
699
- return { tree, fold: { edges, segs } };
700
- }
701
- /** Group a row of items by the level of the cut BETWEEN them: items separated
702
- * by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
703
- * it. Recurses upward until one root remains, so the shape at every level is
704
- * the content's, not an index's. `levels[i]` is the level of the cut that
705
- * precedes item i+1 (there are items.length − 1 of them).
706
- *
707
- * A level that fails to split (every cut below L) or that would exceed the
708
- * keyring falls through to the plain river fold for that row — the fold stays
709
- * total on any input, and the fallback is rare enough not to reintroduce a
710
- * systematic alignment. */
711
- /** A content key for a folded item: a cheap hash of its gist's leading
712
- * coordinates. Used to choose a split point inside an over-long row, where
713
- * the cut levels are uniformly 0 and carry no signal. Identical subtrees
714
- * fold to identical vectors, so the same items in the same order always
715
- * choose the same split — the property the whole fold rests on. */
716
- function itemKey(v) {
717
- let h = 0x811c9dc5;
718
- for (let d = 0; d < 8; d++) {
719
- h = Math.imul(h ^ ((v[d] * 8192) | 0), 0x01000193) >>> 0;
720
- }
721
- return h >>> 0;
722
- }
723
- function groupByLevel(space, items, levels, level) {
724
- if (items.length === 1)
725
- return items[0];
726
- const maxSeats = space.seats.length;
727
- const groups = [];
728
- const groupLevels = [];
729
- // Emit [from, to) as one group, splitting it at its STRONGEST interior cut
730
- // whenever it would exceed the keyring.
731
- //
732
- // The old rule force-cut at the arity limit counted from the group's start
733
- // — the last index-derived boundary in the grouping. Measured, the
734
- // grouping loses one node in six even when EVERY child survives a prepend
735
- // (P(node | all kids survive) = 0.82-0.84), and those losses spike at
736
- // arity 8: exactly this boundary. Choosing the highest-level cut inside
737
- // the feasible window instead makes the split a function of content, and
738
- // the window bound keeps arity <= maxSeats so the seat algebra is untouched.
739
- const emit = (from, to) => {
740
- let at = from;
741
- while (to - at > maxSeats) {
742
- // Strongest cut in the window that still leaves a legal group. Ties
743
- // take the LATEST, so equal levels give the widest legal group rather
744
- // than a degenerate spine of singletons.
745
- let best = at + maxSeats - 1;
746
- let bestKey = -1;
747
- let bestLevel = -1;
748
- for (let j = at; j < at + maxSeats && j < to - 1; j++) {
749
- // Prefer a real level boundary; among equals — and inside an
750
- // over-long stretch the levels are almost all 0, so they usually ARE
751
- // equal — fall back to the ITEMS' own content. A group's gist is
752
- // diverse where its cut level is not, so hashing it gives a
753
- // content-determined split point where the level array has none.
754
- const key = itemKey(items[j].tree.v);
755
- if (levels[j] > bestLevel ||
756
- (levels[j] === bestLevel && key > bestKey)) {
757
- bestLevel = levels[j];
758
- bestKey = key;
759
- best = j;
760
- }
761
- }
762
- const part = items.slice(at, best + 1);
763
- groups.push(part.length === 1 ? part[0] : joinFlat(space, part));
764
- groupLevels.push(levels[best]);
765
- at = best + 1;
766
- }
767
- const slice = items.slice(at, to);
768
- groups.push(slice.length === 1 ? slice[0] : joinFlat(space, slice));
769
- };
770
- let start = 0;
771
- for (let i = 0; i <= levels.length; i++) {
772
- const atEnd = i === levels.length;
773
- if (!atEnd && levels[i] < level)
774
- continue;
775
- emit(start, i + 1);
776
- if (!atEnd)
777
- groupLevels.push(levels[i]);
778
- start = i + 1;
779
- }
780
- if (groups.length === items.length) {
781
- // This level split nothing — climb rather than spin.
782
- return level < 24
783
- ? groupByLevel(space, items, levels, level + 1)
784
- : riverFoldRaw(space, items);
785
- }
786
- return groupByLevel(space, groups, groupLevels, level + 1);
787
- }
788
- /** Join a row of already-folded items as one unnormalized node — the same
789
- * two-ended seat binding as {@link flatFold}, one level up. A group formed
790
- * by content-level cuts inherits the same robustness: interior items keep
791
- * their seats when a leading or trailing segment is perturbed. */
792
- function joinFlat(space, items) {
793
- const n = items.length;
794
- const gist = new Float32Array(space.D);
795
- const kids = new Array(n);
796
- let len = 0;
797
- for (let k = 0; k < n; k++) {
798
- const v = items[k].tree.v;
799
- const slot = twoEndedSeat(space.seats.length, n, k);
800
- const seat = space.seats[slot].fwd;
801
- for (let d = 0; d < space.D; d++)
802
- gist[d] += v[seat[d]];
803
- kids[k] = items[k].tree;
804
- len += items[k].len;
805
- }
806
- return { tree: sema(gist, null, kids), len };
807
- }
808
- /** One segment as a single unnormalized node: leaf per byte, each bound into
809
- * a seat derived from its position relative to BOTH segment ends.
810
- *
811
- * Binding from both ends — first bytes use the lowest seat slots, last
812
- * bytes use the highest — makes the gist of the segment interior robust
813
- * under a leading or trailing insertion: a byte prepended or appended
814
- * shifts only the boundary seat, not every interior position. The same
815
- * rule gives the shift-invariant knife its re-synchronising window in the
816
- * ancestral fold (sema-old, KNIFE_WINDOW trailing items bound with
817
- * relative seat keys). Ported to the current river: a content-defined
818
- * segment always starts at seat 0, so the "relative" binding is the
819
- * segment's own two-ended assignment.
820
- *
821
- * Never normalizes: the linear-fold contract keeps every interior gist
822
- * raw and normalizes once at the root. Magnitude still ∝ √n — seat
823
- * permutation preserves vector length, and the sum of n near-orthogonal
824
- * vectors grows as √n. */
825
- function flatFold(space, alphabet, bytes, from, to) {
826
- const n = to - from;
827
- if (n === 1) {
828
- const b = bytes[from];
829
- return {
830
- tree: sema(alphabet.vecs[b], bytes.slice(from, to), null),
831
- len: 1,
832
- };
833
- }
834
- const gist = new Float32Array(space.D);
835
- const kids = new Array(n);
836
- for (let k = 0; k < n; k++) {
837
- const b = bytes[from + k];
838
- const v = alphabet.vecs[b];
839
- // Two-ended: the first half uses low seats and the second half uses
840
- // high seats, inward from the tail of the FULL keyring.
841
- const slot = twoEndedSeat(space.seats.length, n, k);
842
- const seat = space.seats[slot].fwd;
843
- for (let d = 0; d < space.D; d++)
844
- gist[d] += v[seat[d]];
845
- kids[k] = sema(v, bytes.slice(from + k, from + k + 1), null);
846
- }
847
- return { tree: sema(gist, null, kids), len: n };
848
- }
849
- /** The stable-prefix segmented fold (§10.3). Each segment between
850
- * consecutive boundaries folds PLAINLY and independently; segment roots
851
- * join left-nested, and only the final root is normalized (the linear-fold
852
- * contract: one normalize per perception). A segment's own inner splits
853
- * need no recursion here: a nested learnt prefix is itself an earlier
854
- * boundary, so the left-nested join reproduces every intermediate learnt
855
- * root ((s₀·s₁) IS the root the store learnt for the first two segments'
856
- * bytes, and so on). */
857
- /** A fold's ROOT: ONE normalize per perception, at the root, exactly as
858
- * riverFold did — the interior stays raw (the linear-fold contract).
859
- *
860
- * Normalizes EXCEPT when the whole stream folded to a single
861
- * leaf: a leaf's vector IS the alphabet's own, shared by every occurrence of
862
- * that byte, and `normalize` writes in place. Every fold entry point returns
863
- * through here, because the guard is exactly the kind that gets written at one
864
- * site and missed at the next two — which is what had happened: only
865
- * {@link bytesToTree} carried it, while `stablePrefixFold` and its incremental
866
- * twin normalized unconditionally, reachable by a one-byte stream whose only
867
- * boundary is its own length. */
868
- function rootOf(f) {
869
- if (f.tree.kids !== null)
870
- normalize(f.tree.v);
871
- return f.tree;
872
- }
873
- function stablePrefixFold(space, alphabet, bytes, boundaries) {
874
- const cuts = [];
875
- let prev = 0;
876
- for (const b of boundaries) {
877
- if (b > prev && b < bytes.length) {
878
- cuts.push(b);
879
- prev = b;
880
- }
881
- }
882
- if (cuts.length === 0) {
883
- return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
884
- }
885
- const edges = [0, ...cuts, bytes.length];
886
- const segs = [];
887
- for (let i = 0; i + 1 < edges.length; i++) {
888
- segs.push(contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
889
- }
890
- let cur = segs[0];
891
- for (let i = 1; i < segs.length; i++)
892
- cur = fold2(space, cur, segs[i]);
893
- return rootOf(cur);
894
- }
895
- /** {@link stablePrefixFold} with incremental segment reuse — same cuts, same
896
- * segment folds, same left-nested join, same single root normalize; `prev`
897
- * only elides recomputing segments whose [start,end) offsets it already
898
- * folded over a byte-identical prefix (the caller keys the cache by
899
- * content). Requires a non-empty effective boundary set. */
900
- export function stablePrefixFoldIncremental(space, alphabet, bytes, boundaries, prev) {
901
- // SORTED, like {@link bytesToTree} does before calling the non-incremental
902
- // twin. The filter below is sequential (`b > prevB`), so an out-of-order
903
- // entry is silently DROPPED rather than rejected — and these two functions
904
- // are documented as producing the same cuts, so a caller that hands the
905
- // same set to each and gets different trees has hit a trap, not a contract.
906
- // Sorting here makes the twins genuinely interchangeable; the set is one
907
- // entry per conversation turn, so the cost is nil.
908
- const sorted = [...boundaries].sort((a, b) => a - b);
909
- const cuts = [];
910
- let prevB = 0;
911
- for (const b of sorted) {
912
- if (b > prevB && b < bytes.length) {
913
- cuts.push(b);
914
- prevB = b;
915
- }
916
- }
917
- const edges = [0, ...cuts, bytes.length];
918
- const segs = [];
919
- for (let i = 0; i + 1 < edges.length; i++) {
920
- const hit = prev !== undefined && prev.edges[i] === edges[i] &&
921
- prev.edges[i + 1] === edges[i + 1]
922
- ? prev.segs[i]
923
- : undefined;
924
- segs.push(hit ?? contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
925
- }
926
- if (segs.length === 1) {
927
- // Degenerate boundary set — one span, which IS the whole stream, and it was
928
- // just folded (or reused from `prev`) right above. It cannot go through
929
- // `rootOf`: the ROOT is normalized, a cached SEGMENT never is (a later turn
930
- // reuses it as one), and `normalize` writes in place. Re-folding `bytes`
931
- // to get a separate object is what this used to do — and a first-seen
932
- // deposit has no boundaries, so it always lands here, paying the fold
933
- // twice. Copying the gist is the same result for one vector copy. A
934
- // single LEAF needs neither: its vector is the shared alphabet entry and
935
- // must not be written at all.
936
- const only = segs[0].tree;
937
- const tree = only.kids === null
938
- ? only
939
- : sema(Float32Array.from(only.v), null, only.kids);
940
- if (tree.kids !== null)
941
- normalize(tree.v);
942
- return { tree, fold: { edges, segs } };
943
- }
944
- let cur = segs[0];
945
- for (let i = 1; i < segs.length; i++)
946
- cur = fold2(space, cur, segs[i]);
947
- return { tree: rootOf(cur), fold: { edges, segs } };
948
- }
949
- /** Join two folded items as one 2-kid branch — the top-level join of the
950
- * stable-prefix fold, delegated to {@link joinFlat} (same two-ended seat
951
- * binding as every other group fold). Unnormalized (interior). */
952
- function fold2(space, a, b) {
953
- return joinFlat(space, [a, b]);
954
- }
955
- /** Plain river fold WITHOUT the final root normalize — the segment-level
956
- * building block of {@link stablePrefixFold} (interiors must keep their
957
- * byte-proportional magnitude; only the whole perception's root is ever
958
- * normalized). Exported so callers that COMPOSE already-existing structural
959
- * parts into a hypothetical synthetic root (see {@link composeStructuralGist})
960
- * can feed the same raw primitive instead of duplicating its mathematics. */
961
- export function riverFoldRaw(space, row) {
962
- if (row.length === 0) {
963
- const z = new Float32Array(space.D);
964
- return { tree: sema(z, new Uint8Array(0), null), len: 0 };
965
- }
966
- if (row.length === 1)
967
- return row[0];
968
- let level = row;
969
- while (level.length > 1) {
970
- const next = [];
971
- foldSlice(space, level, 0, level.length, next, true);
972
- level = next;
973
- }
974
- return level[0];
975
- }
976
- /** Synthesize a hypothetical internal structure from already-existing
977
- * structural vectors — NOT from bytes. This is the raw positional
978
- * composition the linear river fold already uses (see the folding header
979
- * above): each part is positionally bound into its own seat, its natural
980
- * span magnitude is preserved, the parts are linearly superposed, and only
981
- * the final synthetic root is normalized. It never calls {@link gistOf}
982
- * (there is no `gistOf` here — geometry.ts has no store), never perceives a
983
- * concatenated byte string, and never interns or stores a new node: the
984
- * result is an opaque, ungrounded Vec for an ANN probe only. */
985
- export function composeStructuralGist(space, parts) {
986
- const foldedParts = [];
987
- for (const part of parts) {
988
- if (part.len <= 0)
989
- continue;
990
- const direction = copy(part.v);
991
- normalize(direction);
992
- const scaled = zeros(space.D);
993
- addInto(scaled, direction, Math.sqrt(part.len));
994
- foldedParts.push({ tree: sema(scaled), len: part.len });
995
- }
996
- if (foldedParts.length === 0)
997
- return zeros(space.D);
998
- const rawRoot = riverFoldRaw(space, foldedParts);
999
- const result = copy(rawRoot.tree.v);
1000
- normalize(result);
1001
- return result;
1002
- }
1003
- // ---- n-D Hilbert curve ----
1004
- function gridDims(grid) {
1005
- if (grid.dims && grid.dims.length > 0)
1006
- return grid.dims.slice();
1007
- const dims = [grid.height, grid.width];
1008
- if (grid.channels > 1)
1009
- dims.push(grid.channels);
1010
- return dims;
1011
- }
1012
- function hilbertPoint(index, n, bits) {
1013
- const x = new Array(n).fill(0);
1014
- for (let b = 0; b < bits; b++) {
1015
- for (let d = 0; d < n; d++) {
1016
- const bit = (index >>> (b * n + (n - 1 - d))) & 1;
1017
- x[d] |= bit << b;
1018
- }
1019
- }
1020
- const N = 1 << bits;
1021
- let t = x[n - 1] >> 1;
1022
- for (let i = n - 1; i > 0; i--)
1023
- x[i] ^= x[i - 1];
1024
- x[0] ^= t;
1025
- for (let q = 2; q !== N; q <<= 1) {
1026
- const p = q - 1;
1027
- for (let i = n - 1; i >= 0; i--) {
1028
- if (x[i] & q)
1029
- x[0] ^= p;
1030
- else {
1031
- t = (x[0] ^ x[i]) & p;
1032
- x[0] ^= t;
1033
- x[i] ^= t;
1034
- }
1035
- }
1036
- }
1037
- return x;
1038
- }
1039
- export function hilbertBytes(grid) {
1040
- const dims = gridDims(grid);
1041
- const n = dims.length;
1042
- if (n === 0 || grid.data.length === 0)
1043
- return new Uint8Array(0);
1044
- if (n === 1)
1045
- return grid.data.slice(0, dims[0]);
1046
- const maxAxis = Math.max(...dims);
1047
- const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
1048
- const side = 1 << bits;
1049
- const total = Math.pow(side, n);
1050
- const stride = new Array(n);
1051
- stride[n - 1] = 1;
1052
- for (let d = n - 2; d >= 0; d--)
1053
- stride[d] = stride[d + 1] * dims[d + 1];
1054
- const out = [];
1055
- for (let h = 0; h < total; h++) {
1056
- const pt = hilbertPoint(h, n, bits);
1057
- let inside = true, flat = 0;
1058
- for (let d = 0; d < n; d++) {
1059
- if (pt[d] >= dims[d]) {
1060
- inside = false;
1061
- break;
1062
- }
1063
- flat += pt[d] * stride[d];
1064
- }
1065
- if (inside)
1066
- out.push(grid.data[flat]);
1067
- }
1068
- return Uint8Array.from(out);
1069
- }
1070
- export function gridToTree(space, alphabet, grid) {
1071
- return bytesToTree(space, alphabet, hilbertBytes(grid));
1072
- }
1073
- export function stackGrids(frames) {
1074
- if (frames.length === 0) {
1075
- return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
1076
- }
1077
- const frameDims = gridDims(frames[0]);
1078
- const per = frames[0].data.length;
1079
- const data = new Uint8Array(per * frames.length);
1080
- for (let i = 0; i < frames.length; i++)
1081
- data.set(frames[i].data, i * per);
1082
- return {
1083
- width: 0,
1084
- height: 0,
1085
- channels: 0,
1086
- dims: [frames.length, ...frameDims],
1087
- data,
1088
- };
1089
- }