@hviana/sema 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +115 -53
- package/HOW_IT_WORKS.md +291 -184
- package/jsr.json +6 -0
- package/package.json +1 -1
- package/src/mind/bridge.ts +10 -104
- package/src/mind/match.ts +497 -7
- package/src/mind/{prefix-completion.ts → mechanisms/prefix-completion.ts} +66 -92
- package/src/mind/mechanisms/recall.ts +38 -129
- package/src/mind/mechanisms/reference.ts +343 -0
- package/src/mind/mind.ts +12 -8
- package/src/mind/pipeline-mechanism.ts +120 -3
- package/src/mind/pipeline.ts +16 -2
- package/src/mind/traverse.ts +92 -1
- package/src/store.ts +13 -4
- package/test/33-multi-candidate.test.mjs +21 -11
- package/test/70-prefix-completion.test.mjs +1 -1
- package/test/72-prefix-candidate-supply.test.mjs +7 -9
- package/test/74-prefix-trap-not-sprung-early.test.mjs +1 -1
- package/test/76-reference-binding.test.mjs +543 -0
- package/dist/example/demo.d.ts +0 -1
- package/dist/example/demo.js +0 -39
- package/dist/example/train_base.d.ts +0 -93
- package/dist/example/train_base.js +0 -2269
- package/dist/src/alphabet.d.ts +0 -7
- package/dist/src/alphabet.js +0 -33
- package/dist/src/alu/src/alu.d.ts +0 -185
- package/dist/src/alu/src/alu.js +0 -440
- package/dist/src/alu/src/expr.d.ts +0 -61
- package/dist/src/alu/src/expr.js +0 -318
- package/dist/src/alu/src/index.d.ts +0 -11
- package/dist/src/alu/src/index.js +0 -19
- package/dist/src/alu/src/kernel-arith.d.ts +0 -16
- package/dist/src/alu/src/kernel-arith.js +0 -264
- package/dist/src/alu/src/kernel-bits.d.ts +0 -19
- package/dist/src/alu/src/kernel-bits.js +0 -152
- package/dist/src/alu/src/kernel-logic.d.ts +0 -4
- package/dist/src/alu/src/kernel-logic.js +0 -60
- package/dist/src/alu/src/kernel-nd.d.ts +0 -3
- package/dist/src/alu/src/kernel-nd.js +0 -208
- package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
- package/dist/src/alu/src/kernel-numeric.js +0 -366
- package/dist/src/alu/src/operation.d.ts +0 -168
- package/dist/src/alu/src/operation.js +0 -189
- package/dist/src/alu/src/parser.d.ts +0 -221
- package/dist/src/alu/src/parser.js +0 -577
- package/dist/src/alu/src/resonance.d.ts +0 -55
- package/dist/src/alu/src/resonance.js +0 -126
- package/dist/src/alu/src/text.d.ts +0 -31
- package/dist/src/alu/src/text.js +0 -73
- package/dist/src/alu/src/value.d.ts +0 -109
- package/dist/src/alu/src/value.js +0 -300
- package/dist/src/alu/test/alu.test.d.ts +0 -1
- package/dist/src/alu/test/alu.test.js +0 -764
- package/dist/src/bytes.d.ts +0 -14
- package/dist/src/bytes.js +0 -59
- package/dist/src/canon.d.ts +0 -45
- package/dist/src/canon.js +0 -85
- package/dist/src/config.d.ts +0 -111
- package/dist/src/config.js +0 -91
- package/dist/src/derive/src/deduction.d.ts +0 -136
- package/dist/src/derive/src/deduction.js +0 -159
- package/dist/src/derive/src/index.d.ts +0 -8
- package/dist/src/derive/src/index.js +0 -11
- package/dist/src/derive/src/priority-queue.d.ts +0 -20
- package/dist/src/derive/src/priority-queue.js +0 -73
- package/dist/src/derive/src/rewrite.d.ts +0 -56
- package/dist/src/derive/src/rewrite.js +0 -100
- package/dist/src/derive/src/trie.d.ts +0 -90
- package/dist/src/derive/src/trie.js +0 -217
- package/dist/src/derive/test/derive.test.d.ts +0 -1
- package/dist/src/derive/test/derive.test.js +0 -122
- package/dist/src/extension.d.ts +0 -37
- package/dist/src/extension.js +0 -7
- package/dist/src/geometry.d.ts +0 -233
- package/dist/src/geometry.js +0 -1089
- package/dist/src/index.d.ts +0 -17
- package/dist/src/index.js +0 -19
- package/dist/src/ingest-cache.d.ts +0 -41
- package/dist/src/ingest-cache.js +0 -165
- package/dist/src/meter.d.ts +0 -176
- package/dist/src/meter.js +0 -274
- package/dist/src/mind/articulation.d.ts +0 -6
- package/dist/src/mind/articulation.js +0 -99
- package/dist/src/mind/attention.d.ts +0 -430
- package/dist/src/mind/attention.js +0 -2418
- package/dist/src/mind/bridge.d.ts +0 -39
- package/dist/src/mind/bridge.js +0 -998
- package/dist/src/mind/canonical.d.ts +0 -34
- package/dist/src/mind/canonical.js +0 -93
- package/dist/src/mind/frame-filler.d.ts +0 -15
- package/dist/src/mind/frame-filler.js +0 -535
- package/dist/src/mind/graph-search.d.ts +0 -294
- package/dist/src/mind/graph-search.js +0 -996
- package/dist/src/mind/index.d.ts +0 -9
- package/dist/src/mind/index.js +0 -5
- package/dist/src/mind/junction.d.ts +0 -137
- package/dist/src/mind/junction.js +0 -358
- package/dist/src/mind/learning.d.ts +0 -75
- package/dist/src/mind/learning.js +0 -265
- package/dist/src/mind/match.d.ts +0 -192
- package/dist/src/mind/match.js +0 -666
- package/dist/src/mind/mechanisms/alu.d.ts +0 -4
- package/dist/src/mind/mechanisms/alu.js +0 -36
- package/dist/src/mind/mechanisms/cast.d.ts +0 -89
- package/dist/src/mind/mechanisms/cast.js +0 -1186
- package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
- package/dist/src/mind/mechanisms/confluence.js +0 -271
- package/dist/src/mind/mechanisms/cover.d.ts +0 -6
- package/dist/src/mind/mechanisms/cover.js +0 -232
- package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
- package/dist/src/mind/mechanisms/extraction.js +0 -327
- package/dist/src/mind/mechanisms/recall.d.ts +0 -16
- package/dist/src/mind/mechanisms/recall.js +0 -558
- package/dist/src/mind/mind.d.ts +0 -386
- package/dist/src/mind/mind.js +0 -754
- package/dist/src/mind/pipeline-mechanism.d.ts +0 -183
- package/dist/src/mind/pipeline-mechanism.js +0 -521
- package/dist/src/mind/pipeline.d.ts +0 -49
- package/dist/src/mind/pipeline.js +0 -331
- package/dist/src/mind/prefix-completion.d.ts +0 -59
- package/dist/src/mind/prefix-completion.js +0 -270
- package/dist/src/mind/primitives.d.ts +0 -85
- package/dist/src/mind/primitives.js +0 -333
- package/dist/src/mind/rationale.d.ts +0 -139
- package/dist/src/mind/rationale.js +0 -163
- package/dist/src/mind/reasoning.d.ts +0 -42
- package/dist/src/mind/reasoning.js +0 -305
- package/dist/src/mind/recognition.d.ts +0 -20
- package/dist/src/mind/recognition.js +0 -696
- package/dist/src/mind/resonance.d.ts +0 -35
- package/dist/src/mind/resonance.js +0 -365
- package/dist/src/mind/trace.d.ts +0 -15
- package/dist/src/mind/trace.js +0 -73
- package/dist/src/mind/traverse.d.ts +0 -158
- package/dist/src/mind/traverse.js +0 -716
- package/dist/src/mind/types.d.ts +0 -418
- package/dist/src/mind/types.js +0 -179
- package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
- package/dist/src/rabitq-ivf/src/database.js +0 -201
- package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
- package/dist/src/rabitq-ivf/src/index.js +0 -4
- package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
- package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
- package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
- package/dist/src/rabitq-ivf/src/prng.js +0 -36
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-ivf/src/rabitq.js +0 -313
- package/dist/src/sema.d.ts +0 -41
- package/dist/src/sema.js +0 -77
- package/dist/src/store-sqlite.d.ts +0 -184
- package/dist/src/store-sqlite.js +0 -942
- package/dist/src/store.d.ts +0 -762
- package/dist/src/store.js +0 -1823
- package/dist/src/vec.d.ts +0 -31
- package/dist/src/vec.js +0 -109
- package/src/mind/frame-filler.ts +0 -604
- package/test/69-frame-filler.test.mjs +0 -115
package/dist/src/geometry.js
DELETED
|
@@ -1,1089 +0,0 @@
|
|
|
1
|
-
// geometry.ts — every modality is a stream; geometry is only a reading order.
|
|
2
|
-
//
|
|
3
|
-
// 1. Each byte is a leaf — an atom carrying its own vector straight from
|
|
4
|
-
// the alphabet.
|
|
5
|
-
// 2. The river folds leaves upward in fixed-size groups (maxGroup). Items
|
|
6
|
-
// that cross the stable-prefix boundary are split so the prefix folds
|
|
7
|
-
// identically regardless of what follows — pure structural stability.
|
|
8
|
-
// 3. The same rule recurses level after level until one root remains.
|
|
9
|
-
import { addInto, copy, normalize, zeros } from "./vec.js";
|
|
10
|
-
import { sema, twoEndedSeat } from "./sema.js";
|
|
11
|
-
// ---- geometric constants ----
|
|
12
|
-
//
|
|
13
|
-
// Every threshold below is a derived function of the fold's own geometry —
|
|
14
|
-
// dimension D, maxGroup, etc. — never a tuned magic number. They live here
|
|
15
|
-
// (not in a config file) because they follow from the structure itself.
|
|
16
|
-
//
|
|
17
|
-
// MEASUREMENT CAVEAT: these thresholds are compared against RaBitQ-ESTIMATED
|
|
18
|
-
// cosines (1-bit stored codes scored against a 4-bit-quantized query; the
|
|
19
|
-
// index never reranks with exact vectors). The derivations assume an exact
|
|
20
|
-
// cosine; the estimator adds a small, rotation-uniformised error the bars do
|
|
21
|
-
// not model. This is benign for the inequality thresholds (they gate broad
|
|
22
|
-
// regions), but it means NO decision may treat an estimated score as exact —
|
|
23
|
-
// identity in particular is decided by content-addressed resolve(), never by
|
|
24
|
-
// `score >= 1` (see recallByResonance tier 0).
|
|
25
|
-
/** The store's geometric identity bar: cosine ≥ 1 − 1/√D is the similarity at
|
|
26
|
-
* which `intern` already treats two gists as the SAME node. Recall reuses it
|
|
27
|
-
* to accept a near-identical query, and the climb to accept a containing form —
|
|
28
|
-
* one derived constant, never a tuned threshold. NOTE: this fixed bar is
|
|
29
|
-
* the ESTIMATOR floor of an identity claim; a whole-span claim over a span
|
|
30
|
-
* longer than the perception quantum must use the scale-aware
|
|
31
|
-
* {@link identityBar}, which converts the tolerated fraction into bytes. */
|
|
32
|
-
export function mergeThreshold(D) {
|
|
33
|
-
return 1 - 1 / Math.sqrt(D);
|
|
34
|
-
}
|
|
35
|
-
/** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
|
|
36
|
-
* of `len` bytes. Under the linear fold a cosine reads "fraction of aligned
|
|
37
|
-
* shared bytes", so a FIXED cosine bar admits a byte budget that grows with
|
|
38
|
-
* the span: 1 − 1/√D over a 4·√D-byte span tolerates four whole river
|
|
39
|
-
* windows of foreign content while still claiming "near-identical". An
|
|
40
|
-
* identity claim may tolerate at most ONE river window W — the perception
|
|
41
|
-
* quantum, the same single-window budget near-dedup's differsByOneWindow
|
|
42
|
-
* grants — so the bar is 1 − W/len, floored at mergeThreshold(D), below
|
|
43
|
-
* which the RaBitQ estimator cannot certify identity anyway. This is the
|
|
44
|
-
* angle+magnitude form of the identity test: the ANGLE carries the shared
|
|
45
|
-
* fraction, the span's MAGNITUDE (√len, the linear fold's own norm) converts
|
|
46
|
-
* the tolerated fraction into tolerated bytes. Derived from W, D and the
|
|
47
|
-
* span; never tuned. */
|
|
48
|
-
export function identityBar(D, maxGroup, len) {
|
|
49
|
-
return Math.max(mergeThreshold(D), 1 - maxGroup / Math.max(1, len));
|
|
50
|
-
}
|
|
51
|
-
/** The reach bar: half a river quantum, derived from the fold's own geometry.
|
|
52
|
-
* A branch folds up to `maxGroup` children, so two forms that differ in ONE
|
|
53
|
-
* whole child — the smallest distinction perception can mean — sit at cosine
|
|
54
|
-
* ≈ 1 − 1/maxGroup. Half that quantum, 1 − 1/(2·maxGroup), is closer than any
|
|
55
|
-
* single-child difference can be: a positional echo of the same content.
|
|
56
|
-
*
|
|
57
|
-
* This is an EQUAL-ARITY replacement law. The two-ended coordinate frame is
|
|
58
|
-
* a bijective relabelling of the seats inside that node, so it does not change
|
|
59
|
-
* the one-child overlap or this bar. Stability under a leading/trailing
|
|
60
|
-
* insertion comes from preserving content-defined subtrees and their anchored
|
|
61
|
-
* coordinates — never from lowering the confidence floor.
|
|
62
|
-
*
|
|
63
|
-
* Recall uses this as its confidence floor: a query whose nearest resonant
|
|
64
|
-
* form sits below this bar is structurally unrelated to everything in the store
|
|
65
|
-
* — further than any single-child variant — and the system returns null rather
|
|
66
|
-
* than fabricate an answer from an unrelated form. Derived, never tuned. */
|
|
67
|
-
export function reachThreshold(maxGroup) {
|
|
68
|
-
return 1 - 1 / (2 * maxGroup);
|
|
69
|
-
}
|
|
70
|
-
/** The estimator's own noise floor: 1/√D — ONE standard deviation of the
|
|
71
|
-
* cosine between two independent random vectors in D dimensions (the same σ
|
|
72
|
-
* {@link significanceBar} takes three of). It is the smallest difference in
|
|
73
|
-
* cosine that is distinguishable from the rotation-uniformised RaBitQ
|
|
74
|
-
* estimation error (see the MEASUREMENT CAVEAT above): a contrastive margin
|
|
75
|
-
* below it is quantisation noise, not evidence. The consensus climb gates a
|
|
76
|
-
* region's vote on its discriminative margin clearing this floor — the
|
|
77
|
-
* minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
|
|
78
|
-
* Derived, never tuned. */
|
|
79
|
-
export function estimatorNoise(D) {
|
|
80
|
-
return 1 / Math.sqrt(D);
|
|
81
|
-
}
|
|
82
|
-
/** The statistical-significance bar for whole-query resonance: 3/√D.
|
|
83
|
-
* In D dimensions the expected cosine of two independent random vectors is 0
|
|
84
|
-
* with standard deviation 1/√D. A cosine ≥ 3/√D is three standard deviations
|
|
85
|
-
* above chance — the query is statistically related to the store, not merely
|
|
86
|
-
* sharing random byte noise. Below this bar the consensus climb (which trusts
|
|
87
|
-
* sub-region resonance) is skipped: there is no evidence the query belongs to
|
|
88
|
-
* the same distribution as the stored content. Derived, never tuned. */
|
|
89
|
-
export function significanceBar(D) {
|
|
90
|
-
return 3 / Math.sqrt(D);
|
|
91
|
-
}
|
|
92
|
-
/** The concept (halo) threshold: the cosine above which two nodes share a
|
|
93
|
-
* distributional concept. A halo is a superposition of episode signatures in
|
|
94
|
-
* D-dimensional space, so the expected cosine between two unrelated halos is 0
|
|
95
|
-
* with standard deviation 1/√D. The structural midpoint 0.5 separates "more
|
|
96
|
-
* similar than not" from noise; the +0.5/√D term adds one half-sigma margin
|
|
97
|
-
* that vanishes as D → ∞, accounting for the wider noise band at lower D
|
|
98
|
-
* without inventing a tuned constant. At D=1024 this gives 0.516, within
|
|
99
|
-
* 3% of 0.5 — existing behavior is preserved while threshold and D move
|
|
100
|
-
* together. Derived, never tuned. */
|
|
101
|
-
export function conceptThreshold(D) {
|
|
102
|
-
return 0.5 + 0.5 / Math.sqrt(D);
|
|
103
|
-
}
|
|
104
|
-
/** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
|
|
105
|
-
* whole of `wholeLen` covers STRICTLY more than half of it. A span that
|
|
106
|
-
* dominates its whole can no longer discriminate the whole's own content —
|
|
107
|
-
* the one test behind liftAnswer's keep-the-frame rule, collectRegions'
|
|
108
|
-
* wrapper exclusion, and CAST's frame-depth majority (each cites this).
|
|
109
|
-
* CAST's frame-FRACTION gate is the deliberately CLOSED variant (≥ ½ is
|
|
110
|
-
* already unusable there) and stays inline where it is documented.
|
|
111
|
-
* Derived from the structural midpoint, never tuned. */
|
|
112
|
-
export function dominates(partLen, wholeLen) {
|
|
113
|
-
return partLen * 2 > wholeLen;
|
|
114
|
-
}
|
|
115
|
-
/** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
|
|
116
|
-
* of learnt contexts (edge sources). A single region's IDF-weighted vote for
|
|
117
|
-
* an anchor reached through c contexts is at most ln(N/c) ≤ ln(N); the +1/2
|
|
118
|
-
* demands the pooled vote exceed what ONE maximally-specific region could
|
|
119
|
-
* contribute by half a unit — i.e. genuine corroboration beyond a lone
|
|
120
|
-
* region's echo at this corpus scale. The ONE floor both consumers gate on:
|
|
121
|
-
* recallByResonance trusting a climb anchor, and commitVotes admitting a
|
|
122
|
-
* further point of attention. Defined once here so the two can never
|
|
123
|
-
* drift apart. Derived from N, never tuned. */
|
|
124
|
-
export function consensusFloor(N) {
|
|
125
|
-
return Math.log(N) + 1 / 2;
|
|
126
|
-
}
|
|
127
|
-
/** The coverage bar for the reach (interior) index, when vector-similarity
|
|
128
|
-
* gating is used. Returns the concept threshold — the structural midpoint
|
|
129
|
-
* (~0.5 at D=1024) where two forms are "more similar than not."
|
|
130
|
-
*
|
|
131
|
-
* Currently UNUSED in the hot training path: interior nodes are indexed
|
|
132
|
-
* unconditionally (hash-cons dedup bounds the index naturally).
|
|
133
|
-
* Post-hoc structural compaction ({@link Store.compactContentIndex})
|
|
134
|
-
* replaces runtime coverage gating with a batch pass that removes
|
|
135
|
-
* structurally-isolated entries. Derived, never tuned. */
|
|
136
|
-
export function coverageBar(_maxGroup, D) {
|
|
137
|
-
return conceptThreshold(D);
|
|
138
|
-
}
|
|
139
|
-
// ---- folding ----
|
|
140
|
-
//
|
|
141
|
-
// The river fold is a hierarchical prefix network: each level contracts
|
|
142
|
-
// groups of `maxGroup` adjacent items into one via permute-then-add
|
|
143
|
-
// (positional seat binding), recursing until one root remains.
|
|
144
|
-
//
|
|
145
|
-
// FLAT per-level fold — one inline loop per level (foldSlice): no per-group
|
|
146
|
-
// function calls, no Array.slice per group, the permute and add FUSED
|
|
147
|
-
// (`gist[d] += v[seat[d]]`, no scratch buffer), and subtree byte lengths
|
|
148
|
-
// carried incrementally on Folded (the old boundary scan re-walked subtrees
|
|
149
|
-
// every level — O(n log n)). The per-level SUPERPOSITION is byte-identical
|
|
150
|
-
// to the original recursive foldGroup: the same FP additions in the same
|
|
151
|
-
// order.
|
|
152
|
-
//
|
|
153
|
-
// LINEAR fold — intermediate gists are NOT normalized; only the final root is
|
|
154
|
-
// (riverFold's single normalize). This is a deliberate change of similarity
|
|
155
|
-
// semantics from the original per-group normalize, not a cached optimization:
|
|
156
|
-
// the fold is now a pure linear operator — a superposition of positionally-
|
|
157
|
-
// bound leaf vectors — so an interior node carries its span's natural
|
|
158
|
-
// magnitude and a resonance score reads as byte-proportional overlap rather
|
|
159
|
-
// than a scale-free cosine. The mechanisms that depend on that reading
|
|
160
|
-
// compensate for it EXPLICITLY, never silently: the contrastive margin on
|
|
161
|
-
// approximate votes (voteRegions), company signatures decoupling halo company
|
|
162
|
-
// from gist content (sema.ts), and the shared-frame analogy tier (match.ts).
|
|
163
|
-
/** Fold `items[start .. start+count)` in groups of `mg` into `out`.
|
|
164
|
-
*
|
|
165
|
-
* With `force`, the trailing incomplete group (2..mg-1 items) is folded as
|
|
166
|
-
* well — only a lone singleton passes through. The river always folds with
|
|
167
|
-
* force: every level contracts by ~mg, so the tree's DEPTH is a function of
|
|
168
|
-
* ceil(log_mg(n)) alone. Letting leftovers pass through unfolded made depth
|
|
169
|
-
* depend on the exact byte count (39 bytes folded in 3 levels, 41 in 4), and
|
|
170
|
-
* each extra level applies another seat permutation to the whole gist —
|
|
171
|
-
* near-identical inputs straddling such a cliff read as orthogonal
|
|
172
|
-
* (measured: 33-byte-identical prefixes at cos ≈ 0). */
|
|
173
|
-
function foldSlice(space, items, start, count, out, force) {
|
|
174
|
-
const mg = space.maxGroup;
|
|
175
|
-
const D = space.D;
|
|
176
|
-
const complete = count - (count % mg);
|
|
177
|
-
const foldAt = (at, size) => {
|
|
178
|
-
const gist = new Float32Array(D);
|
|
179
|
-
const kids = new Array(size);
|
|
180
|
-
let len = 0;
|
|
181
|
-
for (let k = 0; k < size; k++) {
|
|
182
|
-
const f = items[at + k];
|
|
183
|
-
const slot = twoEndedSeat(space.seats.length, size, k);
|
|
184
|
-
const seat = space.seats[slot].fwd;
|
|
185
|
-
const v = f.tree.v;
|
|
186
|
-
// Fused permute-and-accumulate — same FP ops, same order as the old
|
|
187
|
-
// permuteInto + addInto pair, with no scratch buffer.
|
|
188
|
-
for (let d = 0; d < D; d++)
|
|
189
|
-
gist[d] += v[seat[d]];
|
|
190
|
-
kids[k] = f.tree;
|
|
191
|
-
len += f.len;
|
|
192
|
-
}
|
|
193
|
-
out.push({ tree: sema(gist, null, kids), len });
|
|
194
|
-
};
|
|
195
|
-
for (let i = 0; i < complete; i += mg)
|
|
196
|
-
foldAt(start + i, mg);
|
|
197
|
-
const leftover = count - complete;
|
|
198
|
-
if (leftover === 0)
|
|
199
|
-
return;
|
|
200
|
-
if (force && leftover >= 2)
|
|
201
|
-
foldAt(start + complete, leftover);
|
|
202
|
-
else
|
|
203
|
-
for (let i = complete; i < count; i++)
|
|
204
|
-
out.push(items[start + i]);
|
|
205
|
-
}
|
|
206
|
-
function riverFold(space, row, stableBytes) {
|
|
207
|
-
if (row.length === 0) {
|
|
208
|
-
const z = new Float32Array(space.D);
|
|
209
|
-
return { tree: sema(z, new Uint8Array(0), null), len: 0 };
|
|
210
|
-
}
|
|
211
|
-
let level = row;
|
|
212
|
-
while (level.length > 1) {
|
|
213
|
-
// Find the item index where accumulated bytes reaches stableBytes.
|
|
214
|
-
let boundary = level.length;
|
|
215
|
-
if (stableBytes > 0) {
|
|
216
|
-
let acc = 0;
|
|
217
|
-
for (let i = 0; i < level.length; i++) {
|
|
218
|
-
acc += level[i].len;
|
|
219
|
-
if (acc >= stableBytes) {
|
|
220
|
-
boundary = i + 1;
|
|
221
|
-
break;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
const next = [];
|
|
226
|
-
if (boundary < level.length) {
|
|
227
|
-
// Prefix folds independently of the suffix — structural stability.
|
|
228
|
-
foldSlice(space, level, 0, boundary, next, true);
|
|
229
|
-
foldSlice(space, level, boundary, level.length - boundary, next, true);
|
|
230
|
-
}
|
|
231
|
-
else {
|
|
232
|
-
foldSlice(space, level, 0, level.length, next, true);
|
|
233
|
-
}
|
|
234
|
-
level = next;
|
|
235
|
-
}
|
|
236
|
-
// LINEAR fold — this root normalize is the ONLY normalize of the entire
|
|
237
|
-
// fold; every intermediate gist stays unnormalized (see the folding
|
|
238
|
-
// header). Skipped for a single-leaf input: that root IS the shared
|
|
239
|
-
// alphabet vector (already unit), and normalizing in place would mutate the
|
|
240
|
-
// alphabet itself.
|
|
241
|
-
if (row.length > 1)
|
|
242
|
-
normalize(level[0].tree.v);
|
|
243
|
-
return level[0];
|
|
244
|
-
}
|
|
245
|
-
// ---- public API ----
|
|
246
|
-
function bytesToLeaves(alphabet, bytes) {
|
|
247
|
-
return Array.from(bytes, (b, i) => {
|
|
248
|
-
const v = alphabet.vecs[b];
|
|
249
|
-
return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
|
|
250
|
-
});
|
|
251
|
-
}
|
|
252
|
-
/** CONTENT-DEFINED FOLD BOUNDARIES — where a byte stream segments, chosen by
|
|
253
|
-
* the bytes rather than by arithmetic.
|
|
254
|
-
*
|
|
255
|
-
* `riverFold` groups FIXED arity from byte 0 and permutes item k by
|
|
256
|
-
* `seats[k]`, k = index mod W, so a byte's contribution is a function of its
|
|
257
|
-
* ABSOLUTE OFFSET: the same byte run is a different vector at a different
|
|
258
|
-
* position, and the same content is a different SUBTREE. W's size has nothing
|
|
259
|
-
* to do with it — any fixed modulus does this, and identity must not depend on
|
|
260
|
-
* W at all.
|
|
261
|
-
*
|
|
262
|
-
* A rolling hash removes the dependence without touching the fold's arity: the
|
|
263
|
-
* cut lands where the hash of the recent bytes vanishes mod W, so a change
|
|
264
|
-
* upstream moves only the cut it falls inside — every downstream boundary, and
|
|
265
|
-
* therefore every downstream segment, is unchanged. Because
|
|
266
|
-
* {@link stablePrefixFold} folds each segment independently from its own slot
|
|
267
|
-
* 0, byte-identical content then produces byte-identical subtrees, and
|
|
268
|
-
* hash-consing makes it the SAME NODE ID wherever it appears. That — not the
|
|
269
|
-
* root cosine, which should and does still move when content is added — is
|
|
270
|
-
* what recognition, cover and resolve need.
|
|
271
|
-
*
|
|
272
|
-
* Measured over 400 real deposits, shifts of 1..7 bytes, downstream cuts
|
|
273
|
-
* preserved / segments byte-identical:
|
|
274
|
-
*
|
|
275
|
-
* content-defined, deposit text 99.7% / 98.4% mean seg 6.61 B
|
|
276
|
-
* content-defined, non-Latin scripts 99.6% / 98.3% mean seg 6.43 B
|
|
277
|
-
* content-defined, random binary 99.9% / 99.2% mean seg 6.92 B
|
|
278
|
-
* the arithmetic grid, same corpus 14.3% (only the k≡0 mod W
|
|
279
|
-
* shifts survive)
|
|
280
|
-
*
|
|
281
|
-
* The three content-defined rows agreeing is the load-bearing part: this reads
|
|
282
|
-
* BYTES, never text. Mind is not a text engine — the same fold carries grids
|
|
283
|
-
* and any other modality — so a boundary rule justified by where words or
|
|
284
|
-
* sentences fall would be importing an assumption the architecture rejects.
|
|
285
|
-
* Random binary must, and does, behave exactly like prose.
|
|
286
|
-
*
|
|
287
|
-
* Every constant is derived (§2.2): the cut mask is W, so a cut is offered once
|
|
288
|
-
* per quantum of bytes — which, composed with the minimum below, puts the
|
|
289
|
-
* expected segment at minLen + W − 1 ≈ 6 B rather than at W, deliberately (see
|
|
290
|
-
* the refutation recorded at `cutRate` in {@link contentLevels}: a segment is
|
|
291
|
-
* the flat PHRASE-scale unit the W-ary groups are built from, not a group of W
|
|
292
|
-
* children, and forcing E[len] = W costs 15 tests). The minimum is W−1, `canonicalWindows`'s
|
|
293
|
-
* straddle neighbour and the write side's own floor for a unit; and the maximum
|
|
294
|
-
* is the KEYRING's seat count, because a segment folds as ONE flat node and
|
|
295
|
-
* `fold` has exactly that many seats to bind children into. Capping there is
|
|
296
|
-
* what keeps the fold light: a segment of 3..seats leaves is a single node,
|
|
297
|
-
* where splitting it into W-groups plus a remainder would cost two or three
|
|
298
|
-
* and the remainders barely share (measured: partial-arity nodes 504 → 3,590,
|
|
299
|
-
* and total distinct nodes 8,142 → 9,712, when segments folded as [W][rest]). */
|
|
300
|
-
/** {@link contentBoundaries} plus, for each cut, its LEVEL — how deep in the
|
|
301
|
-
* tree that cut reaches.
|
|
302
|
-
*
|
|
303
|
-
* One rolling hash serves every scale. A cut is level 0 when its hash vanishes
|
|
304
|
-
* mod W, level 1 when mod W², and so on: level-L cuts are by construction a
|
|
305
|
-
* subset of level-(L−1) cuts, which is exactly the nesting a tree needs. The
|
|
306
|
-
* expected span of a level-L node is therefore W^(L+1) bytes — the same growth
|
|
307
|
-
* the grid fold had, but with boundaries the content chose, so a shift moves
|
|
308
|
-
* one node at each level instead of all of them.
|
|
309
|
-
*
|
|
310
|
-
* Levels are read from the hash the cut was ACCEPTED at, not recomputed, so
|
|
311
|
-
* they cost nothing beyond the divisions already being done. */
|
|
312
|
-
// Cyclic-polynomial table for the bounded-window cut hash. Derived once from
|
|
313
|
-
// the fold's own mixing constant — no seed, no tuning. A byte contributes
|
|
314
|
-
// BUZ[b] on entering the window; the hash rotates by one per byte, so by the
|
|
315
|
-
// time that byte leaves, its contribution has travelled k places and is
|
|
316
|
-
// removed rotated by k. The rotation is taken at the use site rather than
|
|
317
|
-
// precomputed into a second table so the window width follows maxGroup
|
|
318
|
-
// instead of being frozen at one value.
|
|
319
|
-
const BUZ = new Uint32Array(256);
|
|
320
|
-
{
|
|
321
|
-
let x = 0x9e3779b9 >>> 0;
|
|
322
|
-
for (let i = 0; i < 256; i++) {
|
|
323
|
-
x = Math.imul(x ^ (x >>> 15), 2654435761) >>> 0;
|
|
324
|
-
x = (x ^ (x >>> 13)) >>> 0;
|
|
325
|
-
BUZ[i] = x;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
/** BUZ rotated by the window width — what a byte's contribution has become by
|
|
329
|
-
* the time it leaves. Cached because the width follows `maxGroup`, which is
|
|
330
|
-
* fixed for a given space: built once, then a plain table lookup per byte. */
|
|
331
|
-
let buzOutTable = null;
|
|
332
|
-
let buzOutWidth = -1;
|
|
333
|
-
function buzOut(k) {
|
|
334
|
-
if (buzOutWidth !== k || buzOutTable === null) {
|
|
335
|
-
const t = new Uint32Array(256);
|
|
336
|
-
for (let i = 0; i < 256; i++) {
|
|
337
|
-
const v = BUZ[i];
|
|
338
|
-
t[i] = ((v << k) | (v >>> (32 - k))) >>> 0;
|
|
339
|
-
}
|
|
340
|
-
buzOutTable = t;
|
|
341
|
-
buzOutWidth = k;
|
|
342
|
-
}
|
|
343
|
-
return buzOutTable;
|
|
344
|
-
}
|
|
345
|
-
function contentLevels(space, bytes) {
|
|
346
|
-
const W = space.maxGroup;
|
|
347
|
-
const minLen = W - 1;
|
|
348
|
-
const maxLen = space.seats.length;
|
|
349
|
-
// MEASURED AND REFUTED — making E[segment] equal W. A segment is at least
|
|
350
|
-
// `minLen` bytes and then cuts with probability p, so E[len] = minLen +
|
|
351
|
-
// (1−p)/p; with `h % W === 0` that is minLen + W − 1 = 6 for W = 4 (measured
|
|
352
|
-
// mean 6.61 B on real deposits), half again coarser than the fold's own
|
|
353
|
-
// arity, and the doc above used to claim the mask made the two equal. It
|
|
354
|
-
// does not, and the mismatch looked like the cause of a real deficit: the
|
|
355
|
-
// climb perceives one region per segment, so on `Michelangelo is to sculpture
|
|
356
|
-
// as who is to literature?` it sees 9 regions where the grid saw 19, reaches
|
|
357
|
-
// one ranked anchor instead of two, and CAST's weave never includes the
|
|
358
|
-
// writing-domain exemplar it needs (test/29 A2, C1).
|
|
359
|
-
//
|
|
360
|
-
// Fixing the rate to hit E[len] = W (a threshold `h % W < W − minLen + 1` on
|
|
361
|
-
// the same rolling value, keeping the two bytes of history) does exactly what
|
|
362
|
-
// it says — mean segment 3.60 B on text, 3.93 B on random binary, 14 segments
|
|
363
|
-
// on that query instead of 9 — and takes the suite from 3 failures to 18,
|
|
364
|
-
// across think, universality, scaling, decomposition, bridge, generalization,
|
|
365
|
-
// saturation, audit and recognition. The coarser-than-arity scale is
|
|
366
|
-
// load-bearing: W is the arity `riverFold` groups CHILDREN at, and a segment
|
|
367
|
-
// is not a group of W children but the flat unit those groups are built from
|
|
368
|
-
// — a phrase-scale unit, and the mechanisms above all read it that way.
|
|
369
|
-
// Do not re-derive the rate from W.
|
|
370
|
-
//
|
|
371
|
-
// MEASURED AND REFUTED — reading the hash's HIGH bits instead of its low
|
|
372
|
-
// ones. The accumulator shifts LEFT, so `h % W` is the part the mixing has
|
|
373
|
-
// had no chance to reach: with `h = (h << 1) + byte·K` and K odd, `h mod 4`
|
|
374
|
-
// reduces to `(2·(prev mod 2) + byte) mod 4` — two bytes of history, and only
|
|
375
|
-
// their parity. It reads like a hash test that is really a byte-parity test,
|
|
376
|
-
// and it shows: on `Michelangelo is to sculpture as who is to literature?`
|
|
377
|
-
// the segments come out 8,8,8,8,8,7,4,2 — almost every boundary the FORCED
|
|
378
|
-
// one at maxLen, which is the one offset-dependent cut in the rule.
|
|
379
|
-
//
|
|
380
|
-
// Testing the top bits instead (`h < 2^32 / W`, the same 1/W rate, with the
|
|
381
|
-
// level test falling out as `h < 2^32 / W^(L+1)` — cleaner nesting than
|
|
382
|
-
// divisibility) takes the suite from 2 failures to 21 AND breaks the
|
|
383
|
-
// invariance floor in test/59. The reason is the whole point: in a
|
|
384
|
-
// shift-accumulator the LOW bits have a short effective window and the HIGH
|
|
385
|
-
// bits a long one, so a disturbance upstream perturbs the high bits for the
|
|
386
|
-
// next ~32 bytes while the low bits re-sync within two. The narrow window is
|
|
387
|
-
// not a defect in this hash — it IS the invariance. A boundary rule here is
|
|
388
|
-
// choosing how far a change may propagate, not how well the bits mix.
|
|
389
|
-
//
|
|
390
|
-
// MEASURED AND REFUTED — normalized chunking, to shrink the forced cut. The
|
|
391
|
-
// hard cut at maxLen is the ONE offset-dependent boundary left in the rule,
|
|
392
|
-
// and it is not rare: 32% of segments on a mixed sample end there, five of
|
|
393
|
-
// the eight in the query above, so for those streams the fold IS a grid of 2W
|
|
394
|
-
// with all of the grid's phase. The standard remedy applies cleanly here —
|
|
395
|
-
// past a target length, relax the mask by one power of the same radix
|
|
396
|
-
// (W → W/2), still content-defined and so still invariant, just coarser
|
|
397
|
-
// evidence for a boundary; both quantities are already in the rule, nothing
|
|
398
|
-
// introduced. It works as advertised: forced cuts fall from 32% to 8.3% and
|
|
399
|
-
// test/59's invariance floors still hold.
|
|
400
|
-
//
|
|
401
|
-
// And the suite goes from 2 failures to 5 relaxing at the expected length
|
|
402
|
-
// (minLen + W − 1), or to 6 relaxing at the last opportunity (maxLen − 1) —
|
|
403
|
-
// 22-multihop, 24-generalization, 29, 36. Taken with the two results above
|
|
404
|
-
// (changing the rate costs 15 tests; changing which bits are read costs 19),
|
|
405
|
-
// the reading is that the segment DISTRIBUTION is what the mechanisms
|
|
406
|
-
// downstream are fitted to, not the purity of the rule that produces it. The
|
|
407
|
-
// forced cut is part of that distribution. Do not tidy it away without
|
|
408
|
-
// re-measuring everything that reads a region.
|
|
409
|
-
// A BOUNDED-WINDOW rolling hash — the cut decision reads only the last
|
|
410
|
-
// `k` bytes, so nothing before the window can reach it.
|
|
411
|
-
//
|
|
412
|
-
// The old hash, `h = (h<<1) + byte*K`, needed 32 shifts to drop a byte, so
|
|
413
|
-
// it carried ~32 bytes of history; and on PERIODIC content its value was
|
|
414
|
-
// periodic too, so the threshold either never fired or fired at a fixed
|
|
415
|
-
// phase. Then the `maxLen` fallback placed every boundary at a fixed
|
|
416
|
-
// offset from the previous one and the segmentation could never recover
|
|
417
|
-
// from a shift. Measured fraction of cuts that re-align after a prepend:
|
|
418
|
-
//
|
|
419
|
-
// text uniform sparse lowent records ramp
|
|
420
|
-
// old 0.870 0.902 0.492 0.879 0.888 0.441
|
|
421
|
-
// this 0.935 0.952 0.732 0.920 0.916 0.935
|
|
422
|
-
//
|
|
423
|
-
// The cyclic polynomial (each byte enters as a table value, leaves rotated
|
|
424
|
-
// by the window width) has EXACTLY k bytes of memory and scrambles periodic
|
|
425
|
-
// input, so the threshold fires at content-chosen positions on a gradient
|
|
426
|
-
// just as it does on text — which is what leaves the `maxLen` fallback
|
|
427
|
-
// rarely engaged instead of carrying the phase. Segment lengths are
|
|
428
|
-
// unchanged in distribution (mean 5.2-7.2 against the old 5.4-6.0), so the
|
|
429
|
-
// mechanisms fitted to that distribution see the same scale.
|
|
430
|
-
//
|
|
431
|
-
// Cost is the same shape as before: shifts, XORs and two table lookups per
|
|
432
|
-
// byte, no multiply and no auxiliary structure. (An exact sliding-window
|
|
433
|
-
// minimum — winnowing — aligns slightly better still, 0.91-0.999, but its
|
|
434
|
-
// deque costs 51 MB/s against this rule's 112 and buys nothing the
|
|
435
|
-
// scrambling hash does not already give.)
|
|
436
|
-
const k = W;
|
|
437
|
-
const OUT = buzOut(k);
|
|
438
|
-
const cuts = [];
|
|
439
|
-
const levels = [];
|
|
440
|
-
const n = bytes.length;
|
|
441
|
-
let h = 0;
|
|
442
|
-
let prev = 0;
|
|
443
|
-
let recent = 0; // the last GAP raw hits, one bit each
|
|
444
|
-
// A boundary is a property of a 4-GRAM, not of a position. The register IS
|
|
445
|
-
// the window — it holds exactly the last `k` raw bytes — so the decision is
|
|
446
|
-
// a pure function of those bytes and nothing else can reach it.
|
|
447
|
-
//
|
|
448
|
-
// What kept the OLD rule position-dependent was `minLen`, counted from the
|
|
449
|
-
// previous cut: on periodic content that count carried the initial phase
|
|
450
|
-
// forever and the segmentation never recovered from a shift. But its only
|
|
451
|
-
// job was to stop segments being too short, and that can be said locally —
|
|
452
|
-
// take a hit only when the previous GAP positions did NOT hit. Every term
|
|
453
|
-
// is then a function of a bounded byte window (k + GAP), so the rule stays
|
|
454
|
-
// a pure content property while still setting the segment scale. Measured
|
|
455
|
-
// fraction of cuts that survive a prepend, worst case over six byte types:
|
|
456
|
-
// 0.441 for the original rule, 0.769 counting from `last`, 0.847 for this.
|
|
457
|
-
const GAP = 2;
|
|
458
|
-
const GAPMASK = (1 << GAP) - 1;
|
|
459
|
-
// The keyring bound is restored WITHOUT reintroducing a count: because
|
|
460
|
-
// boundaries are content-determined, an over-long segment carries identical
|
|
461
|
-
// bytes wherever it occurs, so splitting it at strides from ITS OWN start is
|
|
462
|
-
// content-relative. (This holds only while such splits stay RARE — a split
|
|
463
|
-
// leaves a right edge the content did not choose, so the next segment's
|
|
464
|
-
// start is not content-determined either. At this rate they are: mean
|
|
465
|
-
// segment 5.4 against a bound of 8. Lowering the cut rate to lengthen
|
|
466
|
-
// segments makes forced splits dominant and alignment collapses — measured,
|
|
467
|
-
// 0.000 on two-symbol data at rate 1/16.)
|
|
468
|
-
const emit = (at, lvl) => {
|
|
469
|
-
while (at - prev > maxLen) {
|
|
470
|
-
prev += maxLen;
|
|
471
|
-
cuts.push(prev);
|
|
472
|
-
levels.push(0);
|
|
473
|
-
}
|
|
474
|
-
if (at <= prev || at >= n)
|
|
475
|
-
return;
|
|
476
|
-
cuts.push(at);
|
|
477
|
-
levels.push(lvl);
|
|
478
|
-
prev = at;
|
|
479
|
-
};
|
|
480
|
-
for (let i = 0; i < n; i++) {
|
|
481
|
-
h = ((h << 8) | bytes[i]) >>> 0;
|
|
482
|
-
if (i + 1 >= n)
|
|
483
|
-
break;
|
|
484
|
-
if (i < k - 1)
|
|
485
|
-
continue;
|
|
486
|
-
// Two-round avalanche. The window holds four RAW bytes, whose entropy may
|
|
487
|
-
// sit in only a few bits (a gradient's low bits, a sparse stream's zeros);
|
|
488
|
-
// one multiply leaves that structure partly intact and the boundary test
|
|
489
|
-
// inherits it. A second round spreads every input bit across the word,
|
|
490
|
-
// which is what makes the rule behave the same on a ramp as on prose.
|
|
491
|
-
let mixv = Math.imul(h ^ (h >>> 16), 0x85ebca6b) >>> 0;
|
|
492
|
-
mixv = Math.imul(mixv ^ (mixv >>> 13), 0xc2b2ae35) >>> 0;
|
|
493
|
-
mixv = (mixv ^ (mixv >>> 16)) >>> 0;
|
|
494
|
-
const hit = mixv % W === 0;
|
|
495
|
-
if (hit && (recent & GAPMASK) === 0) {
|
|
496
|
-
// Level: how many further powers of W divide the mixed value — level-L
|
|
497
|
-
// cuts stay a subset of level-(L-1) cuts, the nesting the tree needs.
|
|
498
|
-
let lvl = 0;
|
|
499
|
-
let m = W;
|
|
500
|
-
while (lvl < 24 && m <= 0x40000000 && mixv % (m * W) === 0) {
|
|
501
|
-
lvl++;
|
|
502
|
-
m *= W;
|
|
503
|
-
}
|
|
504
|
-
emit(i + 1, lvl);
|
|
505
|
-
}
|
|
506
|
-
recent = ((recent << 1) | (hit ? 1 : 0)) & GAPMASK;
|
|
507
|
-
}
|
|
508
|
-
while (n - prev > maxLen) {
|
|
509
|
-
prev += maxLen;
|
|
510
|
-
cuts.push(prev);
|
|
511
|
-
levels.push(0);
|
|
512
|
-
}
|
|
513
|
-
return { cuts, levels };
|
|
514
|
-
}
|
|
515
|
-
export function contentBoundaries(space, bytes) {
|
|
516
|
-
// ONE implementation of the rule. This used to carry its own copy of the
|
|
517
|
-
// rolling-hash loop, which is exactly how a write side and a read side drift
|
|
518
|
-
// apart without a type error; the levels are computed from the hash the cut
|
|
519
|
-
// was accepted at, so asking for them costs nothing but an array.
|
|
520
|
-
return contentLevels(space, bytes).cuts;
|
|
521
|
-
}
|
|
522
|
-
/** Find the longest prefix of `bytes` whose leaf-id signature matches a
|
|
523
|
-
* known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
|
|
524
|
-
export function knownPrefixLength(bytes, leafAt, lookup) {
|
|
525
|
-
const leafIds = [];
|
|
526
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
527
|
-
const lid = leafAt(i);
|
|
528
|
-
if (lid === null)
|
|
529
|
-
break;
|
|
530
|
-
leafIds.push(lid);
|
|
531
|
-
}
|
|
532
|
-
// Match the longest PROPER prefix — a full-length match means the entire
|
|
533
|
-
// input already exists as a stored form (e.g. the flat leaf-id branch
|
|
534
|
-
// stored alongside the structural root). That would hide the true split
|
|
535
|
-
// point and prevent the river from producing the same tree it folded
|
|
536
|
-
// during training, so the structural recognition cannot find the right
|
|
537
|
-
// forms. A proper prefix guarantees at least two regions.
|
|
538
|
-
for (let len = bytes.length - 1; len >= 2; len--) {
|
|
539
|
-
if (lookup(leafIds.slice(0, len)) !== null)
|
|
540
|
-
return len;
|
|
541
|
-
}
|
|
542
|
-
return 0;
|
|
543
|
-
}
|
|
544
|
-
/** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
|
|
545
|
-
* detecting previously-stored prefixes so the river can split at the
|
|
546
|
-
* correct boundary. Pass them through from `perceive`; the geometry
|
|
547
|
-
* computes the stable prefix internally.
|
|
548
|
-
*
|
|
549
|
-
* `boundaries` is the CALLER-computed stable-prefix boundary set (§10.3):
|
|
550
|
-
* strictly-increasing proper byte offsets, each the length of a prefix that
|
|
551
|
-
* is already a stored whole-stream form. When given, the fold splits into
|
|
552
|
-
* the segments between consecutive boundaries — each folded independently,
|
|
553
|
-
* exactly as it folded when it was learned — and the segment roots join
|
|
554
|
-
* LEFT-NESTED (((s₀·s₁)·s₂)…), so every learnt cumulative-context root
|
|
555
|
-
* reappears as an identical subtree (and, by hash-consing, the very same
|
|
556
|
-
* node) inside the grown stream. This is what lets a conversation's next
|
|
557
|
-
* turn extend perception instead of refolding it: identical prefixes
|
|
558
|
-
* produce identical subtrees regardless of what follows them. */
|
|
559
|
-
export function bytesToTree(space, alphabet, bytes, leafAt, lookup, boundaries) {
|
|
560
|
-
if (bytes.length === 0) {
|
|
561
|
-
return sema(alphabet.vecs[0], new Uint8Array(0), null);
|
|
562
|
-
}
|
|
563
|
-
// WHERE A STREAM SEGMENTS IS DECIDED BY ITS BYTES ({@link contentBoundaries}),
|
|
564
|
-
// and the segments then fold BALANCED, W-ary, like any other row of items.
|
|
565
|
-
//
|
|
566
|
-
// The two must not be confused. Content cuts are what make identity
|
|
567
|
-
// offset-free: a segment folds from its own slot 0, so the same bytes give the
|
|
568
|
-
// same subtree wherever they sit. The shape ABOVE the segments is a separate
|
|
569
|
-
// question, and it must stay the river's own — grouping W at a time, depth
|
|
570
|
-
// log_W(n). Joining segments left-nested instead (as the stable-prefix fold
|
|
571
|
-
// does, for its own good reason) costs a node per segment on a single spine:
|
|
572
|
-
// at a cut every ~6.6 B a 3 KB deposit becomes a 450-deep spine of 450 fresh
|
|
573
|
-
// D-vectors, ~1.8 MB for one deposit, and every walker above inherits the
|
|
574
|
-
// depth. That is an implementation blunder, not a property of content-defined
|
|
575
|
-
// folding, and it is what riverFold below avoids.
|
|
576
|
-
//
|
|
577
|
-
// Caller-supplied boundaries stay left-nested (see stablePrefixFold): there are
|
|
578
|
-
// a handful of them, one per conversation turn, and the cumulative-context-root
|
|
579
|
-
// contract depends on that shape. Each SPAN between them content-folds.
|
|
580
|
-
const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
|
|
581
|
-
const outer = new Set();
|
|
582
|
-
if (boundaries !== undefined) {
|
|
583
|
-
for (const b of boundaries)
|
|
584
|
-
outer.add(b);
|
|
585
|
-
}
|
|
586
|
-
if (sb > 0)
|
|
587
|
-
outer.add(sb);
|
|
588
|
-
if (outer.size === 0) {
|
|
589
|
-
return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
|
|
590
|
-
}
|
|
591
|
-
return stablePrefixFold(space, alphabet, bytes, [...outer].sort((a, b) => a - b));
|
|
592
|
-
}
|
|
593
|
-
/** One span, folded over its own content cuts — AT EVERY LEVEL.
|
|
594
|
-
*
|
|
595
|
-
* A segment becomes ONE FLAT NODE: every leaf bound into its own seat and
|
|
596
|
-
* summed, arity = the segment's length. `contentBoundaries` caps a segment at
|
|
597
|
-
* the keyring's seat count so this is always possible, and the flat form is
|
|
598
|
-
* both lighter (one node per segment instead of a [W][remainder] pair) and the
|
|
599
|
-
* natural unit — a segment IS the smallest thing the cuts claim is a unit.
|
|
600
|
-
*
|
|
601
|
-
* Above the segments the cutting RECURSES rather than reverting to the grid.
|
|
602
|
-
* Grouping segment roots W-at-a-time from index 0 would reintroduce the very
|
|
603
|
-
* bug content cuts exist to remove, one level up: a form spanning segments
|
|
604
|
-
* 12..17 straddles the [12-15] and [16-17] groups and is no node at all, so
|
|
605
|
-
* recognition can only reach it by an alignment accident (test/44 pins exactly
|
|
606
|
-
* this — at HEAD the grid happened to put a node one byte before the target).
|
|
607
|
-
* {@link contentLevels} assigns each cut a level from how divisible its hash
|
|
608
|
-
* is, so level-L cuts are a subset of level-(L−1) cuts and every node at every
|
|
609
|
-
* scale is delimited by content. Identity is then offset-free at all scales,
|
|
610
|
-
* which is the whole requirement — it must not depend on W. */
|
|
611
|
-
function contentFoldSpan(space, alphabet, bytes, from, to) {
|
|
612
|
-
const span = bytes.subarray(from, to);
|
|
613
|
-
const { cuts, levels } = contentLevels(space, span);
|
|
614
|
-
const edges = [0, ...cuts, span.length];
|
|
615
|
-
const segs = [];
|
|
616
|
-
for (let i = 0; i + 1 < edges.length; i++) {
|
|
617
|
-
segs.push(flatFold(space, alphabet, span, edges[i], edges[i + 1]));
|
|
618
|
-
}
|
|
619
|
-
if (segs.length > 1)
|
|
620
|
-
return groupByLevel(space, segs, levels, 1);
|
|
621
|
-
return segs[0];
|
|
622
|
-
}
|
|
623
|
-
/** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
|
|
624
|
-
* fold of a byte-identical prefix already produced.
|
|
625
|
-
*
|
|
626
|
-
* WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES. A level-0 segment is a
|
|
627
|
-
* pure function of its own bytes ({@link flatFold} reads nothing else), so
|
|
628
|
-
* reusing one whose [start,end) is unchanged is bit-identical to refolding it
|
|
629
|
-
* — the cache can never change the tree, only skip work. And the cuts
|
|
630
|
-
* themselves are stable under APPEND: {@link contentLevels} decides each cut
|
|
631
|
-
* from a rolling hash over a local window, so bytes added at the right edge
|
|
632
|
-
* cannot move a cut to their left (measured over a growing 12-turn context:
|
|
633
|
-
* 100% of prior cuts survive every append, zero tail churn). Together those
|
|
634
|
-
* two facts are the whole optimisation — a grown stream refolds only the
|
|
635
|
-
* segments at its right edge.
|
|
636
|
-
*
|
|
637
|
-
* This is the reuse the conversation path wants, and it costs NOTHING in
|
|
638
|
-
* structure: the tree is exactly the tree {@link bytesToTree} builds for the
|
|
639
|
-
* same bytes with no boundary set at all. Turn boundaries buy prefix-ROOT
|
|
640
|
-
* identity, which is a different property from incremental reuse; conflating
|
|
641
|
-
* the two is what put an imposed boundary set on the inference path and left
|
|
642
|
-
* it folding differently from the deposits it was querying.
|
|
643
|
-
*
|
|
644
|
-
* `groupByLevel` above the segments is re-run whole. It operates on segment
|
|
645
|
-
* ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
|
|
646
|
-
* and only its right edge actually changes shape — measured at ~40 rebuilt
|
|
647
|
-
* nodes per turn, flat as the context grows sevenfold.
|
|
648
|
-
*
|
|
649
|
-
* PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
|
|
650
|
-
* `bytes`. Reuse is keyed on a segment's [start,end) OFFSETS, which is what
|
|
651
|
-
* makes it O(1) per segment; offsets alone cannot witness that the underlying
|
|
652
|
-
* bytes agree. Hand it a fold of DIFFERENT bytes whose cuts happen to land
|
|
653
|
-
* in the same places and it will splice those foreign segments in — measured,
|
|
654
|
-
* a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
|
|
655
|
-
* streams. Verifying the bytes here would cost O(prefix) and defeat the
|
|
656
|
-
* whole point, so the obligation sits with the caller, and every caller
|
|
657
|
-
* discharges it structurally rather than by care: `perceiveDeposit` looks the
|
|
658
|
-
* entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
|
|
659
|
-
* ARE the cache key — and a conversation's fold state advances only by
|
|
660
|
-
* append. A new caller that cannot make the same structural argument must
|
|
661
|
-
* pass no `prev` at all; the cold path is always correct.
|
|
662
|
-
* ({@link stablePrefixFoldIncremental} carries the identical precondition for
|
|
663
|
-
* the identical reason.) */
|
|
664
|
-
export function contentFoldIncremental(space, alphabet, bytes, prev) {
|
|
665
|
-
if (bytes.length === 0) {
|
|
666
|
-
return {
|
|
667
|
-
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
668
|
-
fold: { edges: [0], segs: [] },
|
|
669
|
-
};
|
|
670
|
-
}
|
|
671
|
-
const { cuts, levels } = contentLevels(space, bytes);
|
|
672
|
-
const edges = [0, ...cuts, bytes.length];
|
|
673
|
-
const segs = [];
|
|
674
|
-
for (let i = 0; i + 1 < edges.length; i++) {
|
|
675
|
-
const hit = prev !== undefined && prev.edges[i] === edges[i] &&
|
|
676
|
-
prev.edges[i + 1] === edges[i + 1]
|
|
677
|
-
? prev.segs[i]
|
|
678
|
-
: undefined;
|
|
679
|
-
segs.push(hit ?? flatFold(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
680
|
-
}
|
|
681
|
-
const folded = segs.length > 1
|
|
682
|
-
? groupByLevel(space, segs, levels, 1)
|
|
683
|
-
: segs[0];
|
|
684
|
-
// THE ROOT IS NORMALIZED IN PLACE, A CACHED SEGMENT NEVER IS. With one
|
|
685
|
-
// segment — or with a grouping that passes a lone item through — `folded`
|
|
686
|
-
// IS a cached seg, and a later turn will reuse it as an interior node whose
|
|
687
|
-
// magnitude must stay byte-proportional. Copy before normalizing, exactly
|
|
688
|
-
// as the stable-prefix twin does. A single LEAF is copied too: its vector
|
|
689
|
-
// is the shared alphabet entry and must never be written.
|
|
690
|
-
const aliased = segs.some((s) => s.tree === folded.tree);
|
|
691
|
-
let tree = folded.tree;
|
|
692
|
-
if (aliased) {
|
|
693
|
-
tree = tree.kids === null
|
|
694
|
-
? sema(tree.v, tree.leaf, null)
|
|
695
|
-
: sema(Float32Array.from(tree.v), null, tree.kids);
|
|
696
|
-
}
|
|
697
|
-
if (tree.kids !== null)
|
|
698
|
-
normalize(tree.v);
|
|
699
|
-
return { tree, fold: { edges, segs } };
|
|
700
|
-
}
|
|
701
|
-
/** Group a row of items by the level of the cut BETWEEN them: items separated
|
|
702
|
-
* by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
|
|
703
|
-
* it. Recurses upward until one root remains, so the shape at every level is
|
|
704
|
-
* the content's, not an index's. `levels[i]` is the level of the cut that
|
|
705
|
-
* precedes item i+1 (there are items.length − 1 of them).
|
|
706
|
-
*
|
|
707
|
-
* A level that fails to split (every cut below L) or that would exceed the
|
|
708
|
-
* keyring falls through to the plain river fold for that row — the fold stays
|
|
709
|
-
* total on any input, and the fallback is rare enough not to reintroduce a
|
|
710
|
-
* systematic alignment. */
|
|
711
|
-
/** A content key for a folded item: a cheap hash of its gist's leading
|
|
712
|
-
* coordinates. Used to choose a split point inside an over-long row, where
|
|
713
|
-
* the cut levels are uniformly 0 and carry no signal. Identical subtrees
|
|
714
|
-
* fold to identical vectors, so the same items in the same order always
|
|
715
|
-
* choose the same split — the property the whole fold rests on. */
|
|
716
|
-
function itemKey(v) {
|
|
717
|
-
let h = 0x811c9dc5;
|
|
718
|
-
for (let d = 0; d < 8; d++) {
|
|
719
|
-
h = Math.imul(h ^ ((v[d] * 8192) | 0), 0x01000193) >>> 0;
|
|
720
|
-
}
|
|
721
|
-
return h >>> 0;
|
|
722
|
-
}
|
|
723
|
-
function groupByLevel(space, items, levels, level) {
|
|
724
|
-
if (items.length === 1)
|
|
725
|
-
return items[0];
|
|
726
|
-
const maxSeats = space.seats.length;
|
|
727
|
-
const groups = [];
|
|
728
|
-
const groupLevels = [];
|
|
729
|
-
// Emit [from, to) as one group, splitting it at its STRONGEST interior cut
|
|
730
|
-
// whenever it would exceed the keyring.
|
|
731
|
-
//
|
|
732
|
-
// The old rule force-cut at the arity limit counted from the group's start
|
|
733
|
-
// — the last index-derived boundary in the grouping. Measured, the
|
|
734
|
-
// grouping loses one node in six even when EVERY child survives a prepend
|
|
735
|
-
// (P(node | all kids survive) = 0.82-0.84), and those losses spike at
|
|
736
|
-
// arity 8: exactly this boundary. Choosing the highest-level cut inside
|
|
737
|
-
// the feasible window instead makes the split a function of content, and
|
|
738
|
-
// the window bound keeps arity <= maxSeats so the seat algebra is untouched.
|
|
739
|
-
const emit = (from, to) => {
|
|
740
|
-
let at = from;
|
|
741
|
-
while (to - at > maxSeats) {
|
|
742
|
-
// Strongest cut in the window that still leaves a legal group. Ties
|
|
743
|
-
// take the LATEST, so equal levels give the widest legal group rather
|
|
744
|
-
// than a degenerate spine of singletons.
|
|
745
|
-
let best = at + maxSeats - 1;
|
|
746
|
-
let bestKey = -1;
|
|
747
|
-
let bestLevel = -1;
|
|
748
|
-
for (let j = at; j < at + maxSeats && j < to - 1; j++) {
|
|
749
|
-
// Prefer a real level boundary; among equals — and inside an
|
|
750
|
-
// over-long stretch the levels are almost all 0, so they usually ARE
|
|
751
|
-
// equal — fall back to the ITEMS' own content. A group's gist is
|
|
752
|
-
// diverse where its cut level is not, so hashing it gives a
|
|
753
|
-
// content-determined split point where the level array has none.
|
|
754
|
-
const key = itemKey(items[j].tree.v);
|
|
755
|
-
if (levels[j] > bestLevel ||
|
|
756
|
-
(levels[j] === bestLevel && key > bestKey)) {
|
|
757
|
-
bestLevel = levels[j];
|
|
758
|
-
bestKey = key;
|
|
759
|
-
best = j;
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
const part = items.slice(at, best + 1);
|
|
763
|
-
groups.push(part.length === 1 ? part[0] : joinFlat(space, part));
|
|
764
|
-
groupLevels.push(levels[best]);
|
|
765
|
-
at = best + 1;
|
|
766
|
-
}
|
|
767
|
-
const slice = items.slice(at, to);
|
|
768
|
-
groups.push(slice.length === 1 ? slice[0] : joinFlat(space, slice));
|
|
769
|
-
};
|
|
770
|
-
let start = 0;
|
|
771
|
-
for (let i = 0; i <= levels.length; i++) {
|
|
772
|
-
const atEnd = i === levels.length;
|
|
773
|
-
if (!atEnd && levels[i] < level)
|
|
774
|
-
continue;
|
|
775
|
-
emit(start, i + 1);
|
|
776
|
-
if (!atEnd)
|
|
777
|
-
groupLevels.push(levels[i]);
|
|
778
|
-
start = i + 1;
|
|
779
|
-
}
|
|
780
|
-
if (groups.length === items.length) {
|
|
781
|
-
// This level split nothing — climb rather than spin.
|
|
782
|
-
return level < 24
|
|
783
|
-
? groupByLevel(space, items, levels, level + 1)
|
|
784
|
-
: riverFoldRaw(space, items);
|
|
785
|
-
}
|
|
786
|
-
return groupByLevel(space, groups, groupLevels, level + 1);
|
|
787
|
-
}
|
|
788
|
-
/** Join a row of already-folded items as one unnormalized node — the same
|
|
789
|
-
* two-ended seat binding as {@link flatFold}, one level up. A group formed
|
|
790
|
-
* by content-level cuts inherits the same robustness: interior items keep
|
|
791
|
-
* their seats when a leading or trailing segment is perturbed. */
|
|
792
|
-
function joinFlat(space, items) {
|
|
793
|
-
const n = items.length;
|
|
794
|
-
const gist = new Float32Array(space.D);
|
|
795
|
-
const kids = new Array(n);
|
|
796
|
-
let len = 0;
|
|
797
|
-
for (let k = 0; k < n; k++) {
|
|
798
|
-
const v = items[k].tree.v;
|
|
799
|
-
const slot = twoEndedSeat(space.seats.length, n, k);
|
|
800
|
-
const seat = space.seats[slot].fwd;
|
|
801
|
-
for (let d = 0; d < space.D; d++)
|
|
802
|
-
gist[d] += v[seat[d]];
|
|
803
|
-
kids[k] = items[k].tree;
|
|
804
|
-
len += items[k].len;
|
|
805
|
-
}
|
|
806
|
-
return { tree: sema(gist, null, kids), len };
|
|
807
|
-
}
|
|
808
|
-
/** One segment as a single unnormalized node: leaf per byte, each bound into
|
|
809
|
-
* a seat derived from its position relative to BOTH segment ends.
|
|
810
|
-
*
|
|
811
|
-
* Binding from both ends — first bytes use the lowest seat slots, last
|
|
812
|
-
* bytes use the highest — makes the gist of the segment interior robust
|
|
813
|
-
* under a leading or trailing insertion: a byte prepended or appended
|
|
814
|
-
* shifts only the boundary seat, not every interior position. The same
|
|
815
|
-
* rule gives the shift-invariant knife its re-synchronising window in the
|
|
816
|
-
* ancestral fold (sema-old, KNIFE_WINDOW trailing items bound with
|
|
817
|
-
* relative seat keys). Ported to the current river: a content-defined
|
|
818
|
-
* segment always starts at seat 0, so the "relative" binding is the
|
|
819
|
-
* segment's own two-ended assignment.
|
|
820
|
-
*
|
|
821
|
-
* Never normalizes: the linear-fold contract keeps every interior gist
|
|
822
|
-
* raw and normalizes once at the root. Magnitude still ∝ √n — seat
|
|
823
|
-
* permutation preserves vector length, and the sum of n near-orthogonal
|
|
824
|
-
* vectors grows as √n. */
|
|
825
|
-
function flatFold(space, alphabet, bytes, from, to) {
|
|
826
|
-
const n = to - from;
|
|
827
|
-
if (n === 1) {
|
|
828
|
-
const b = bytes[from];
|
|
829
|
-
return {
|
|
830
|
-
tree: sema(alphabet.vecs[b], bytes.slice(from, to), null),
|
|
831
|
-
len: 1,
|
|
832
|
-
};
|
|
833
|
-
}
|
|
834
|
-
const gist = new Float32Array(space.D);
|
|
835
|
-
const kids = new Array(n);
|
|
836
|
-
for (let k = 0; k < n; k++) {
|
|
837
|
-
const b = bytes[from + k];
|
|
838
|
-
const v = alphabet.vecs[b];
|
|
839
|
-
// Two-ended: the first half uses low seats and the second half uses
|
|
840
|
-
// high seats, inward from the tail of the FULL keyring.
|
|
841
|
-
const slot = twoEndedSeat(space.seats.length, n, k);
|
|
842
|
-
const seat = space.seats[slot].fwd;
|
|
843
|
-
for (let d = 0; d < space.D; d++)
|
|
844
|
-
gist[d] += v[seat[d]];
|
|
845
|
-
kids[k] = sema(v, bytes.slice(from + k, from + k + 1), null);
|
|
846
|
-
}
|
|
847
|
-
return { tree: sema(gist, null, kids), len: n };
|
|
848
|
-
}
|
|
849
|
-
/** The stable-prefix segmented fold (§10.3). Each segment between
|
|
850
|
-
* consecutive boundaries folds PLAINLY and independently; segment roots
|
|
851
|
-
* join left-nested, and only the final root is normalized (the linear-fold
|
|
852
|
-
* contract: one normalize per perception). A segment's own inner splits
|
|
853
|
-
* need no recursion here: a nested learnt prefix is itself an earlier
|
|
854
|
-
* boundary, so the left-nested join reproduces every intermediate learnt
|
|
855
|
-
* root ((s₀·s₁) IS the root the store learnt for the first two segments'
|
|
856
|
-
* bytes, and so on). */
|
|
857
|
-
/** A fold's ROOT: ONE normalize per perception, at the root, exactly as
|
|
858
|
-
* riverFold did — the interior stays raw (the linear-fold contract).
|
|
859
|
-
*
|
|
860
|
-
* Normalizes EXCEPT when the whole stream folded to a single
|
|
861
|
-
* leaf: a leaf's vector IS the alphabet's own, shared by every occurrence of
|
|
862
|
-
* that byte, and `normalize` writes in place. Every fold entry point returns
|
|
863
|
-
* through here, because the guard is exactly the kind that gets written at one
|
|
864
|
-
* site and missed at the next two — which is what had happened: only
|
|
865
|
-
* {@link bytesToTree} carried it, while `stablePrefixFold` and its incremental
|
|
866
|
-
* twin normalized unconditionally, reachable by a one-byte stream whose only
|
|
867
|
-
* boundary is its own length. */
|
|
868
|
-
function rootOf(f) {
|
|
869
|
-
if (f.tree.kids !== null)
|
|
870
|
-
normalize(f.tree.v);
|
|
871
|
-
return f.tree;
|
|
872
|
-
}
|
|
873
|
-
function stablePrefixFold(space, alphabet, bytes, boundaries) {
|
|
874
|
-
const cuts = [];
|
|
875
|
-
let prev = 0;
|
|
876
|
-
for (const b of boundaries) {
|
|
877
|
-
if (b > prev && b < bytes.length) {
|
|
878
|
-
cuts.push(b);
|
|
879
|
-
prev = b;
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
if (cuts.length === 0) {
|
|
883
|
-
return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
|
|
884
|
-
}
|
|
885
|
-
const edges = [0, ...cuts, bytes.length];
|
|
886
|
-
const segs = [];
|
|
887
|
-
for (let i = 0; i + 1 < edges.length; i++) {
|
|
888
|
-
segs.push(contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
889
|
-
}
|
|
890
|
-
let cur = segs[0];
|
|
891
|
-
for (let i = 1; i < segs.length; i++)
|
|
892
|
-
cur = fold2(space, cur, segs[i]);
|
|
893
|
-
return rootOf(cur);
|
|
894
|
-
}
|
|
895
|
-
/** {@link stablePrefixFold} with incremental segment reuse — same cuts, same
|
|
896
|
-
* segment folds, same left-nested join, same single root normalize; `prev`
|
|
897
|
-
* only elides recomputing segments whose [start,end) offsets it already
|
|
898
|
-
* folded over a byte-identical prefix (the caller keys the cache by
|
|
899
|
-
* content). Requires a non-empty effective boundary set. */
|
|
900
|
-
export function stablePrefixFoldIncremental(space, alphabet, bytes, boundaries, prev) {
|
|
901
|
-
// SORTED, like {@link bytesToTree} does before calling the non-incremental
|
|
902
|
-
// twin. The filter below is sequential (`b > prevB`), so an out-of-order
|
|
903
|
-
// entry is silently DROPPED rather than rejected — and these two functions
|
|
904
|
-
// are documented as producing the same cuts, so a caller that hands the
|
|
905
|
-
// same set to each and gets different trees has hit a trap, not a contract.
|
|
906
|
-
// Sorting here makes the twins genuinely interchangeable; the set is one
|
|
907
|
-
// entry per conversation turn, so the cost is nil.
|
|
908
|
-
const sorted = [...boundaries].sort((a, b) => a - b);
|
|
909
|
-
const cuts = [];
|
|
910
|
-
let prevB = 0;
|
|
911
|
-
for (const b of sorted) {
|
|
912
|
-
if (b > prevB && b < bytes.length) {
|
|
913
|
-
cuts.push(b);
|
|
914
|
-
prevB = b;
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
const edges = [0, ...cuts, bytes.length];
|
|
918
|
-
const segs = [];
|
|
919
|
-
for (let i = 0; i + 1 < edges.length; i++) {
|
|
920
|
-
const hit = prev !== undefined && prev.edges[i] === edges[i] &&
|
|
921
|
-
prev.edges[i + 1] === edges[i + 1]
|
|
922
|
-
? prev.segs[i]
|
|
923
|
-
: undefined;
|
|
924
|
-
segs.push(hit ?? contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
925
|
-
}
|
|
926
|
-
if (segs.length === 1) {
|
|
927
|
-
// Degenerate boundary set — one span, which IS the whole stream, and it was
|
|
928
|
-
// just folded (or reused from `prev`) right above. It cannot go through
|
|
929
|
-
// `rootOf`: the ROOT is normalized, a cached SEGMENT never is (a later turn
|
|
930
|
-
// reuses it as one), and `normalize` writes in place. Re-folding `bytes`
|
|
931
|
-
// to get a separate object is what this used to do — and a first-seen
|
|
932
|
-
// deposit has no boundaries, so it always lands here, paying the fold
|
|
933
|
-
// twice. Copying the gist is the same result for one vector copy. A
|
|
934
|
-
// single LEAF needs neither: its vector is the shared alphabet entry and
|
|
935
|
-
// must not be written at all.
|
|
936
|
-
const only = segs[0].tree;
|
|
937
|
-
const tree = only.kids === null
|
|
938
|
-
? only
|
|
939
|
-
: sema(Float32Array.from(only.v), null, only.kids);
|
|
940
|
-
if (tree.kids !== null)
|
|
941
|
-
normalize(tree.v);
|
|
942
|
-
return { tree, fold: { edges, segs } };
|
|
943
|
-
}
|
|
944
|
-
let cur = segs[0];
|
|
945
|
-
for (let i = 1; i < segs.length; i++)
|
|
946
|
-
cur = fold2(space, cur, segs[i]);
|
|
947
|
-
return { tree: rootOf(cur), fold: { edges, segs } };
|
|
948
|
-
}
|
|
949
|
-
/** Join two folded items as one 2-kid branch — the top-level join of the
|
|
950
|
-
* stable-prefix fold, delegated to {@link joinFlat} (same two-ended seat
|
|
951
|
-
* binding as every other group fold). Unnormalized (interior). */
|
|
952
|
-
function fold2(space, a, b) {
|
|
953
|
-
return joinFlat(space, [a, b]);
|
|
954
|
-
}
|
|
955
|
-
/** Plain river fold WITHOUT the final root normalize — the segment-level
|
|
956
|
-
* building block of {@link stablePrefixFold} (interiors must keep their
|
|
957
|
-
* byte-proportional magnitude; only the whole perception's root is ever
|
|
958
|
-
* normalized). Exported so callers that COMPOSE already-existing structural
|
|
959
|
-
* parts into a hypothetical synthetic root (see {@link composeStructuralGist})
|
|
960
|
-
* can feed the same raw primitive instead of duplicating its mathematics. */
|
|
961
|
-
export function riverFoldRaw(space, row) {
|
|
962
|
-
if (row.length === 0) {
|
|
963
|
-
const z = new Float32Array(space.D);
|
|
964
|
-
return { tree: sema(z, new Uint8Array(0), null), len: 0 };
|
|
965
|
-
}
|
|
966
|
-
if (row.length === 1)
|
|
967
|
-
return row[0];
|
|
968
|
-
let level = row;
|
|
969
|
-
while (level.length > 1) {
|
|
970
|
-
const next = [];
|
|
971
|
-
foldSlice(space, level, 0, level.length, next, true);
|
|
972
|
-
level = next;
|
|
973
|
-
}
|
|
974
|
-
return level[0];
|
|
975
|
-
}
|
|
976
|
-
/** Synthesize a hypothetical internal structure from already-existing
|
|
977
|
-
* structural vectors — NOT from bytes. This is the raw positional
|
|
978
|
-
* composition the linear river fold already uses (see the folding header
|
|
979
|
-
* above): each part is positionally bound into its own seat, its natural
|
|
980
|
-
* span magnitude is preserved, the parts are linearly superposed, and only
|
|
981
|
-
* the final synthetic root is normalized. It never calls {@link gistOf}
|
|
982
|
-
* (there is no `gistOf` here — geometry.ts has no store), never perceives a
|
|
983
|
-
* concatenated byte string, and never interns or stores a new node: the
|
|
984
|
-
* result is an opaque, ungrounded Vec for an ANN probe only. */
|
|
985
|
-
export function composeStructuralGist(space, parts) {
|
|
986
|
-
const foldedParts = [];
|
|
987
|
-
for (const part of parts) {
|
|
988
|
-
if (part.len <= 0)
|
|
989
|
-
continue;
|
|
990
|
-
const direction = copy(part.v);
|
|
991
|
-
normalize(direction);
|
|
992
|
-
const scaled = zeros(space.D);
|
|
993
|
-
addInto(scaled, direction, Math.sqrt(part.len));
|
|
994
|
-
foldedParts.push({ tree: sema(scaled), len: part.len });
|
|
995
|
-
}
|
|
996
|
-
if (foldedParts.length === 0)
|
|
997
|
-
return zeros(space.D);
|
|
998
|
-
const rawRoot = riverFoldRaw(space, foldedParts);
|
|
999
|
-
const result = copy(rawRoot.tree.v);
|
|
1000
|
-
normalize(result);
|
|
1001
|
-
return result;
|
|
1002
|
-
}
|
|
1003
|
-
// ---- n-D Hilbert curve ----
|
|
1004
|
-
function gridDims(grid) {
|
|
1005
|
-
if (grid.dims && grid.dims.length > 0)
|
|
1006
|
-
return grid.dims.slice();
|
|
1007
|
-
const dims = [grid.height, grid.width];
|
|
1008
|
-
if (grid.channels > 1)
|
|
1009
|
-
dims.push(grid.channels);
|
|
1010
|
-
return dims;
|
|
1011
|
-
}
|
|
1012
|
-
function hilbertPoint(index, n, bits) {
|
|
1013
|
-
const x = new Array(n).fill(0);
|
|
1014
|
-
for (let b = 0; b < bits; b++) {
|
|
1015
|
-
for (let d = 0; d < n; d++) {
|
|
1016
|
-
const bit = (index >>> (b * n + (n - 1 - d))) & 1;
|
|
1017
|
-
x[d] |= bit << b;
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
const N = 1 << bits;
|
|
1021
|
-
let t = x[n - 1] >> 1;
|
|
1022
|
-
for (let i = n - 1; i > 0; i--)
|
|
1023
|
-
x[i] ^= x[i - 1];
|
|
1024
|
-
x[0] ^= t;
|
|
1025
|
-
for (let q = 2; q !== N; q <<= 1) {
|
|
1026
|
-
const p = q - 1;
|
|
1027
|
-
for (let i = n - 1; i >= 0; i--) {
|
|
1028
|
-
if (x[i] & q)
|
|
1029
|
-
x[0] ^= p;
|
|
1030
|
-
else {
|
|
1031
|
-
t = (x[0] ^ x[i]) & p;
|
|
1032
|
-
x[0] ^= t;
|
|
1033
|
-
x[i] ^= t;
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
|
-
}
|
|
1037
|
-
return x;
|
|
1038
|
-
}
|
|
1039
|
-
export function hilbertBytes(grid) {
|
|
1040
|
-
const dims = gridDims(grid);
|
|
1041
|
-
const n = dims.length;
|
|
1042
|
-
if (n === 0 || grid.data.length === 0)
|
|
1043
|
-
return new Uint8Array(0);
|
|
1044
|
-
if (n === 1)
|
|
1045
|
-
return grid.data.slice(0, dims[0]);
|
|
1046
|
-
const maxAxis = Math.max(...dims);
|
|
1047
|
-
const bits = Math.max(1, Math.ceil(Math.log2(maxAxis)));
|
|
1048
|
-
const side = 1 << bits;
|
|
1049
|
-
const total = Math.pow(side, n);
|
|
1050
|
-
const stride = new Array(n);
|
|
1051
|
-
stride[n - 1] = 1;
|
|
1052
|
-
for (let d = n - 2; d >= 0; d--)
|
|
1053
|
-
stride[d] = stride[d + 1] * dims[d + 1];
|
|
1054
|
-
const out = [];
|
|
1055
|
-
for (let h = 0; h < total; h++) {
|
|
1056
|
-
const pt = hilbertPoint(h, n, bits);
|
|
1057
|
-
let inside = true, flat = 0;
|
|
1058
|
-
for (let d = 0; d < n; d++) {
|
|
1059
|
-
if (pt[d] >= dims[d]) {
|
|
1060
|
-
inside = false;
|
|
1061
|
-
break;
|
|
1062
|
-
}
|
|
1063
|
-
flat += pt[d] * stride[d];
|
|
1064
|
-
}
|
|
1065
|
-
if (inside)
|
|
1066
|
-
out.push(grid.data[flat]);
|
|
1067
|
-
}
|
|
1068
|
-
return Uint8Array.from(out);
|
|
1069
|
-
}
|
|
1070
|
-
export function gridToTree(space, alphabet, grid) {
|
|
1071
|
-
return bytesToTree(space, alphabet, hilbertBytes(grid));
|
|
1072
|
-
}
|
|
1073
|
-
export function stackGrids(frames) {
|
|
1074
|
-
if (frames.length === 0) {
|
|
1075
|
-
return { width: 0, height: 0, channels: 0, data: new Uint8Array(0) };
|
|
1076
|
-
}
|
|
1077
|
-
const frameDims = gridDims(frames[0]);
|
|
1078
|
-
const per = frames[0].data.length;
|
|
1079
|
-
const data = new Uint8Array(per * frames.length);
|
|
1080
|
-
for (let i = 0; i < frames.length; i++)
|
|
1081
|
-
data.set(frames[i].data, i * per);
|
|
1082
|
-
return {
|
|
1083
|
-
width: 0,
|
|
1084
|
-
height: 0,
|
|
1085
|
-
channels: 0,
|
|
1086
|
-
dims: [frames.length, ...frameDims],
|
|
1087
|
-
data,
|
|
1088
|
-
};
|
|
1089
|
-
}
|