@hviana/sema 0.4.4 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORS.md +0 -1
- package/LICENSE.md +1 -1
- package/README.md +2 -2
- package/dist/src/geometry.d.ts +6 -0
- package/dist/src/geometry.js +224 -44
- package/dist/src/mind/attention.d.ts +11 -0
- package/dist/src/mind/attention.js +344 -13
- package/dist/src/mind/junction.js +18 -2
- package/dist/src/mind/match.d.ts +11 -0
- package/dist/src/mind/match.js +13 -2
- package/dist/src/mind/mechanisms/cast.js +366 -34
- package/dist/src/mind/mechanisms/confluence.js +17 -1
- package/dist/src/mind/mechanisms/recall.js +17 -3
- package/dist/src/mind/pipeline-mechanism.d.ts +4 -0
- package/dist/src/mind/pipeline-mechanism.js +96 -40
- package/dist/src/mind/pipeline.js +31 -3
- package/dist/src/mind/reasoning.d.ts +4 -2
- package/dist/src/mind/reasoning.js +29 -4
- package/dist/src/mind/recognition.js +67 -2
- package/dist/src/mind/resonance.d.ts +14 -2
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/types.d.ts +43 -1
- package/dist/src/rabitq-ivf/src/rabitq.js +31 -1
- package/dist/src/sema.d.ts +11 -1
- package/dist/src/sema.js +16 -2
- package/dist/src/store.d.ts +64 -1
- package/dist/src/store.js +107 -8
- package/index.html +2 -3
- package/package.json +1 -1
- package/src/geometry.ts +231 -43
- package/src/mind/attention.ts +366 -15
- package/src/mind/junction.ts +18 -2
- package/src/mind/match.ts +18 -2
- package/src/mind/mechanisms/cast.ts +376 -43
- package/src/mind/mechanisms/confluence.ts +16 -1
- package/src/mind/mechanisms/recall.ts +17 -2
- package/src/mind/pipeline-mechanism.ts +96 -36
- package/src/mind/pipeline.ts +33 -3
- package/src/mind/reasoning.ts +31 -4
- package/src/mind/recognition.ts +65 -2
- package/src/mind/resonance.ts +0 -0
- package/src/mind/types.ts +43 -1
- package/src/rabitq-ivf/src/rabitq.ts +31 -1
- package/src/sema.ts +21 -2
- package/src/store.ts +106 -5
- package/test/00-extract.test.mjs +28 -0
- package/test/15-decomposition-gap.test.mjs +0 -0
- package/test/24-generalization.test.mjs +67 -19
- package/test/29-counterfactual.test.mjs +106 -42
- package/test/33-multi-candidate.test.mjs +56 -12
- package/test/53-cross-region-probe-instrumentation.test.mjs +16 -1
- package/test/63-fold-invariants.test.mjs +489 -0
- package/test/64-two-ended-thresholds.test.mjs +76 -0
- package/test/65-ann-recall.test.mjs +331 -0
|
@@ -282,15 +282,30 @@ test("6. structural-resonance: eligible probes report variants, merged proposals
|
|
|
282
282
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
283
283
|
|
|
284
284
|
test("7. resonance outcomes: ineligible reasons are named and a margin-rejected probe reports a sub-noise-floor margin", async () => {
|
|
285
|
+
// BOTH readings come from the SPREAD query. An ineligible probe is only
|
|
286
|
+
// observable when resonance is REACHED and declines — which needs more
|
|
287
|
+
// than one pair in play. On the tight "red then square" the query's own
|
|
288
|
+
// regions leave a single maximal pair, so the one probe attempted is
|
|
289
|
+
// accepted and there is nothing ineligible to inspect: the assertion below
|
|
290
|
+
// would then be reporting the fixture's pair count, not the classifier.
|
|
291
|
+
// The premise is asserted rather than assumed, so a fixture that stops
|
|
292
|
+
// exercising the mechanism fails loudly instead of vacuously passing.
|
|
285
293
|
const mIneligible = mk(1);
|
|
286
294
|
await mIneligible.ingest(NO_BRIDGE_CORPUS);
|
|
287
295
|
const { steps: ineligibleSteps } = await trace(
|
|
288
296
|
mIneligible,
|
|
289
|
-
"red
|
|
297
|
+
"red and a very long interior gap before square",
|
|
290
298
|
);
|
|
291
299
|
await mIneligible.store.close();
|
|
292
300
|
|
|
293
301
|
const ineligibleStep = climbStep(ineligibleSteps);
|
|
302
|
+
const attempted = ineligibleStep.data.crossRegion?.probesAttempted ?? 0;
|
|
303
|
+
assert.ok(
|
|
304
|
+
attempted > 1,
|
|
305
|
+
`fixture premise broken: only ${attempted} cross-region probe(s) were ` +
|
|
306
|
+
`attempted, so no probe can be observed DECLINING — widen the query ` +
|
|
307
|
+
`(more regions in play) rather than relaxing the assertion.`,
|
|
308
|
+
);
|
|
294
309
|
const ineligible = (ineligibleStep.data.crossRegion?.probes ?? []).filter(
|
|
295
310
|
(p) => p.resonance?.outcome === "ineligible",
|
|
296
311
|
);
|
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
// 63-fold-invariants.test.mjs — the fold's complete contract, in one place.
|
|
2
|
+
//
|
|
3
|
+
// Three jobs:
|
|
4
|
+
// 1. VALIDATE every structural invariant the fold must hold, exactly.
|
|
5
|
+
// 2. MEASURE the shift-invariance the fold currently achieves.
|
|
6
|
+
// 3. PIN those measurements as a baseline, so a future change that trades a
|
|
7
|
+
// contract for a number — or quietly loses ground — fails here.
|
|
8
|
+
//
|
|
9
|
+
// BYTE-AGNOSTIC BY CONSTRUCTION. Text is one case, not the case. Every
|
|
10
|
+
// measurement runs over six corpora, five of which are not text: uniform
|
|
11
|
+
// random, sparse (mostly-zero, like headers and bitmaps), two-symbol
|
|
12
|
+
// low-entropy, periodic fixed-size records, and smooth gradients (audio and
|
|
13
|
+
// image ramps). This is not decoration — conclusions drawn from an English
|
|
14
|
+
// corpus have inverted twice when finally tested on bytes, and a degenerate
|
|
15
|
+
// case on a gradient was invisible until a ramp corpus existed.
|
|
16
|
+
//
|
|
17
|
+
// Every generator must VARY PER STREAM. A deterministic ramp once made all
|
|
18
|
+
// twelve "unrelated" samples byte-identical and reported an unrelated max of
|
|
19
|
+
// 1.000, which reads as a catastrophic result and was a bug in the harness.
|
|
20
|
+
//
|
|
21
|
+
// The pinned numbers are FLOORS AND CEILINGS with slack, never equalities:
|
|
22
|
+
// they exist to catch regressions and silent trades, not to freeze the
|
|
23
|
+
// geometry. Raising a floor after a genuine improvement is expected.
|
|
24
|
+
|
|
25
|
+
import { test } from "node:test";
|
|
26
|
+
import assert from "node:assert/strict";
|
|
27
|
+
import {
|
|
28
|
+
bytesToTree,
|
|
29
|
+
contentBoundaries,
|
|
30
|
+
estimatorNoise,
|
|
31
|
+
identityBar,
|
|
32
|
+
mergeThreshold,
|
|
33
|
+
reachThreshold,
|
|
34
|
+
significanceBar,
|
|
35
|
+
} from "../dist/src/geometry.js";
|
|
36
|
+
import { Alphabet } from "../dist/src/alphabet.js";
|
|
37
|
+
import { fold, sema } from "../dist/src/sema.js";
|
|
38
|
+
import { cosine, makeKeyring, normalize, rng } from "../dist/src/vec.js";
|
|
39
|
+
|
|
40
|
+
const D = 1024;
|
|
41
|
+
const W = 4;
|
|
42
|
+
const SEATS = 8;
|
|
43
|
+
const mkSpace = () => ({
|
|
44
|
+
D,
|
|
45
|
+
seats: makeKeyring(D, SEATS, rng(1)),
|
|
46
|
+
rand: rng(2),
|
|
47
|
+
maxGroup: W,
|
|
48
|
+
});
|
|
49
|
+
const space = mkSpace();
|
|
50
|
+
const alphabet = new Alphabet(7, D, { roughness: 0.65, seedMask: 0xa1fa17 });
|
|
51
|
+
|
|
52
|
+
const E = (s) => new TextEncoder().encode(s);
|
|
53
|
+
const tree = (b) =>
|
|
54
|
+
bytesToTree(space, alphabet, b instanceof Uint8Array ? b : E(b));
|
|
55
|
+
const gist = (b) => tree(b).v;
|
|
56
|
+
const mag = (v) => Math.sqrt(v.reduce((a, x) => a + x * x, 0));
|
|
57
|
+
const span = (n) =>
|
|
58
|
+
n.kids
|
|
59
|
+
? n.kids.reduce((a, k) => a + span(k), 0)
|
|
60
|
+
: (n.leaf ? n.leaf.length : 0);
|
|
61
|
+
const med = (a) => {
|
|
62
|
+
const s = a.slice().sort((x, y) => x - y);
|
|
63
|
+
return s[s.length >> 1];
|
|
64
|
+
};
|
|
65
|
+
const cat = (...xs) => {
|
|
66
|
+
const n = xs.reduce((a, x) => a + x.length, 0), o = new Uint8Array(n);
|
|
67
|
+
let k = 0;
|
|
68
|
+
for (const x of xs) {
|
|
69
|
+
o.set(x, k);
|
|
70
|
+
k += x.length;
|
|
71
|
+
}
|
|
72
|
+
return o;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// ── corpora ──────────────────────────────────────────────────────────────
|
|
76
|
+
function gen(kind, R, n) {
|
|
77
|
+
const b = new Uint8Array(n);
|
|
78
|
+
if (kind === "uniform") {
|
|
79
|
+
for (let i = 0; i < n; i++) b[i] = Math.floor(R() * 256);
|
|
80
|
+
} else if (kind === "sparse") {
|
|
81
|
+
for (let i = 0; i < n; i++) {
|
|
82
|
+
b[i] = R() < 0.15 ? Math.floor(R() * 256) : 0;
|
|
83
|
+
}
|
|
84
|
+
} else if (kind === "lowent") {
|
|
85
|
+
for (let i = 0; i < n; i++) {
|
|
86
|
+
b[i] = R() < 0.5 ? 0x41 : 0x42;
|
|
87
|
+
}
|
|
88
|
+
} else if (kind === "records") {
|
|
89
|
+
const tag = Math.floor(R() * 256);
|
|
90
|
+
for (let i = 0; i < n; i++) {
|
|
91
|
+
b[i] = (i % 8 === 0) ? tag : (i % 4) * 17 + Math.floor(R() * 4);
|
|
92
|
+
}
|
|
93
|
+
} else if (kind === "ramp") {
|
|
94
|
+
const step = 1 + Math.floor(R() * 13), phase = Math.floor(R() * 256);
|
|
95
|
+
for (let i = 0; i < n; i++) b[i] = (phase + i * step) & 0xff;
|
|
96
|
+
}
|
|
97
|
+
return b;
|
|
98
|
+
}
|
|
99
|
+
const SENTENCES = [
|
|
100
|
+
"The owl is named Sage and lives in the old barn",
|
|
101
|
+
"Michelangelo sculpted the David from a single block of marble",
|
|
102
|
+
"Photosynthesis converts light energy into chemical energy in plants",
|
|
103
|
+
"The treaty was signed after three years of difficult negotiation",
|
|
104
|
+
"Quicksort partitions an array around a chosen pivot element",
|
|
105
|
+
"Volcanic ash drifted across the island for several weeks",
|
|
106
|
+
"She repaired the bicycle chain with a small steel link",
|
|
107
|
+
"Neutron stars rotate hundreds of times every single second",
|
|
108
|
+
"The bakery on Fifth Street closes early on Sundays",
|
|
109
|
+
"Chess engines evaluate millions of positions before moving",
|
|
110
|
+
"Antarctic ice cores preserve a record of ancient climate",
|
|
111
|
+
"He translated the poem without losing its original meter",
|
|
112
|
+
];
|
|
113
|
+
const PREFIX = ["Well, ", "Actually, ", "I think ", "Listen, "];
|
|
114
|
+
const SUFFIX = [" indeed.", " you know.", " of course.", " really."];
|
|
115
|
+
const CORPORA = ["text", "uniform", "sparse", "lowent", "records", "ramp"];
|
|
116
|
+
|
|
117
|
+
/** Twelve base streams plus a prefix/suffix generator, per corpus. */
|
|
118
|
+
function corpus(kind, seed = 9, len = 60) {
|
|
119
|
+
const R = rng(seed);
|
|
120
|
+
const base = kind === "text"
|
|
121
|
+
? SENTENCES.map(E)
|
|
122
|
+
: Array.from({ length: 12 }, () => gen(kind, R, len));
|
|
123
|
+
return {
|
|
124
|
+
base,
|
|
125
|
+
pre: () =>
|
|
126
|
+
kind === "text" ? E(PREFIX[Math.floor(R() * 4)]) : gen(kind, R, 6),
|
|
127
|
+
post: () =>
|
|
128
|
+
kind === "text" ? E(SUFFIX[Math.floor(R() * 4)]) : gen(kind, R, 6),
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
133
|
+
// PART 1 — STRUCTURAL INVARIANTS. Exact contracts; no tolerance to tune.
|
|
134
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
135
|
+
|
|
136
|
+
test("determinism: identical bytes fold to an identical tree, on every byte type", () => {
|
|
137
|
+
for (const kind of CORPORA) {
|
|
138
|
+
for (const b of corpus(kind).base) {
|
|
139
|
+
assert.deepEqual(Array.from(gist(b)), Array.from(gist(b)), kind);
|
|
140
|
+
assert.deepEqual(
|
|
141
|
+
contentBoundaries(space, b),
|
|
142
|
+
contentBoundaries(space, b),
|
|
143
|
+
kind,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("a node's gist does not depend on its ROLE — the stable-prefix contract", () => {
|
|
150
|
+
// fold(prefix) standing alone must equal the node covering that prefix
|
|
151
|
+
// inside a longer stream: "(s₀·s₁) IS the root the store learnt for the
|
|
152
|
+
// first two segments' bytes". A rule applied at only some join sites
|
|
153
|
+
// breaks this, and with it express() and recognition of a stored form
|
|
154
|
+
// appearing inside a longer query.
|
|
155
|
+
const cases = [
|
|
156
|
+
[E("The owl is "), E("named Sage.")],
|
|
157
|
+
[E("Michelangelo "), E("sculpted David.")],
|
|
158
|
+
[E("abcdefgh"), E("ijklmnop")],
|
|
159
|
+
];
|
|
160
|
+
const R = rng(5);
|
|
161
|
+
for (let i = 0; i < 8; i++) {
|
|
162
|
+
cases.push([gen("uniform", R, 20), gen("uniform", R, 30)]);
|
|
163
|
+
}
|
|
164
|
+
for (let i = 0; i < 4; i++) {
|
|
165
|
+
cases.push([gen("records", R, 24), gen("records", R, 24)]);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
for (const [pre, rest] of cases) {
|
|
169
|
+
const whole = bytesToTree(
|
|
170
|
+
space,
|
|
171
|
+
alphabet,
|
|
172
|
+
cat(pre, rest),
|
|
173
|
+
undefined,
|
|
174
|
+
undefined,
|
|
175
|
+
[pre.length],
|
|
176
|
+
);
|
|
177
|
+
let found = null;
|
|
178
|
+
const walk = (n) => {
|
|
179
|
+
if (span(n) === pre.length && !found) found = n;
|
|
180
|
+
(n.kids ?? []).forEach(walk);
|
|
181
|
+
};
|
|
182
|
+
walk(whole);
|
|
183
|
+
assert.ok(found, "no node covers the declared prefix");
|
|
184
|
+
assert.ok(
|
|
185
|
+
cosine(found.v, gist(pre)) > 0.999,
|
|
186
|
+
`prefix node must BE fold(prefix): cos ${
|
|
187
|
+
cosine(found.v, gist(pre)).toFixed(5)
|
|
188
|
+
}`,
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
test("linearity: interior gists stay raw, |v| ∝ √len, and only the root is normalized", () => {
|
|
194
|
+
// identityBar converts a cosine into a tolerated BYTE budget using this
|
|
195
|
+
// norm, so drift here silently changes what counts as identity. A rule
|
|
196
|
+
// emitting more than one term per child (any n-gram or pair superposition)
|
|
197
|
+
// breaks it.
|
|
198
|
+
const R = rng(6);
|
|
199
|
+
let worst = 0;
|
|
200
|
+
for (const kind of ["uniform", "records", "lowent", "ramp"]) {
|
|
201
|
+
const root = tree(gen(kind, R, 400));
|
|
202
|
+
assert.ok(
|
|
203
|
+
Math.abs(mag(root.v) - 1) < 1e-5,
|
|
204
|
+
"the root, and only the root, is normalized",
|
|
205
|
+
);
|
|
206
|
+
const seen = new Map();
|
|
207
|
+
const walk = (n) => {
|
|
208
|
+
if (!n.kids) return;
|
|
209
|
+
const s = span(n);
|
|
210
|
+
if (!seen.has(s)) seen.set(s, mag(n.v) / Math.sqrt(s));
|
|
211
|
+
n.kids.forEach(walk);
|
|
212
|
+
};
|
|
213
|
+
root.kids.forEach(walk);
|
|
214
|
+
for (const r of seen.values()) worst = Math.max(worst, Math.abs(r - 1));
|
|
215
|
+
}
|
|
216
|
+
assert.ok(
|
|
217
|
+
worst < 0.25,
|
|
218
|
+
`interior |v|/√len drifts by ${
|
|
219
|
+
worst.toFixed(4)
|
|
220
|
+
} — the linear-fold norm is gone`,
|
|
221
|
+
);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
test("order-bearing: permuting children changes the gist, for bytes as well as prose", () => {
|
|
225
|
+
// A plain superposition is commutative and made a stream equal its own
|
|
226
|
+
// reversal. Checked at three scales: segments, clauses, and whole halves.
|
|
227
|
+
const segSwap = cosine(
|
|
228
|
+
gist(Uint8Array.of(0, 64, 0, 0, 0, 0, 0, 0)),
|
|
229
|
+
gist(Uint8Array.of(0, 0, 0, 0, 0, 64, 0, 0)),
|
|
230
|
+
);
|
|
231
|
+
const clause = cosine(
|
|
232
|
+
gist("Steel is hard. Ice is cold."),
|
|
233
|
+
gist("Ice is cold. Steel is hard."),
|
|
234
|
+
);
|
|
235
|
+
const R = rng(7);
|
|
236
|
+
const x = gen("uniform", R, 24), y = gen("uniform", R, 24);
|
|
237
|
+
const halves = cosine(gist(cat(x, y)), gist(cat(y, x)));
|
|
238
|
+
for (
|
|
239
|
+
const [name, c] of [["segment", segSwap], ["clause", clause], [
|
|
240
|
+
"halves",
|
|
241
|
+
halves,
|
|
242
|
+
]]
|
|
243
|
+
) {
|
|
244
|
+
assert.ok(
|
|
245
|
+
c < 0.99,
|
|
246
|
+
`${name} permutation left the gist unchanged (cos ${c.toFixed(4)})`,
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
test("repetition is not erased: a repeated unit differs from the unit", () => {
|
|
252
|
+
// normalize(n·v) = v̂ for every n, so a superposition cannot see repeats and
|
|
253
|
+
// "ab" interned as the same node as "abab".
|
|
254
|
+
const bar = mergeThreshold(D);
|
|
255
|
+
const pairs = [["ab", "abab"], ["aaaa", "aaaaaaaa"], ["aaaa", "aaaaaa"], [
|
|
256
|
+
"xyz",
|
|
257
|
+
"xyzxyz",
|
|
258
|
+
]];
|
|
259
|
+
for (const [a, b] of pairs) {
|
|
260
|
+
const c = cosine(gist(a), gist(b));
|
|
261
|
+
assert.ok(
|
|
262
|
+
c < bar,
|
|
263
|
+
`"${a}" vs "${b}": cos ${c.toFixed(5)} ≥ mergeThreshold ${
|
|
264
|
+
bar.toFixed(4)
|
|
265
|
+
} — interned as ONE node`,
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
const u = Uint8Array.of(7, 200, 19, 4);
|
|
269
|
+
const c = cosine(gist(u), gist(cat(u, u)));
|
|
270
|
+
assert.ok(c < bar, `binary unit vs its repetition: cos ${c.toFixed(5)}`);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
test("seats are injective: no two positions in a node alias onto one key", () => {
|
|
274
|
+
// The original point of this file. Two streams differing only by a
|
|
275
|
+
// positional swap must not be geometrically identical.
|
|
276
|
+
const a = Uint8Array.of(1, 1, 1, 3, 1);
|
|
277
|
+
const b = Uint8Array.of(1, 1, 3, 1, 1);
|
|
278
|
+
assert.notDeepEqual(Array.from(gist(a)), Array.from(gist(b)));
|
|
279
|
+
assert.ok(
|
|
280
|
+
cosine(gist(a), gist(b)) < mergeThreshold(D),
|
|
281
|
+
"a positional swap is not geometric identity",
|
|
282
|
+
);
|
|
283
|
+
|
|
284
|
+
// Length must survive too: n copies of one byte must not collapse onto one.
|
|
285
|
+
for (const [m, n] of [[4, 8], [3, 6], [5, 7]]) {
|
|
286
|
+
const c = cosine(
|
|
287
|
+
gist(new Uint8Array(m).fill(97)),
|
|
288
|
+
gist(new Uint8Array(n).fill(97)),
|
|
289
|
+
);
|
|
290
|
+
assert.ok(
|
|
291
|
+
c < mergeThreshold(D),
|
|
292
|
+
`${m} vs ${n} equal bytes collapsed (cos ${c.toFixed(5)})`,
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
test("the public fold and the perceived fold share ONE positional algebra", () => {
|
|
298
|
+
// A synthetic/canonical fold must agree with perception, or composed gists
|
|
299
|
+
// match nothing perception produces.
|
|
300
|
+
for (let n = 2; n <= SEATS; n++) {
|
|
301
|
+
const bytes = new Uint8Array(n).fill(97);
|
|
302
|
+
if (contentBoundaries(space, bytes).length !== 0) continue; // only flat cases
|
|
303
|
+
assert.deepEqual(
|
|
304
|
+
Array.from(gist(bytes)),
|
|
305
|
+
Array.from(fold(space, Array.from(bytes, (b) => alphabet.vecs[b]))),
|
|
306
|
+
`flat ${n}-item nodes must have one geometry`,
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
test("separation: unrelated streams stay inside the noise the derived bars assume", () => {
|
|
312
|
+
// Every bar — mergeThreshold, identityBar, reachThreshold, estimatorNoise,
|
|
313
|
+
// significanceBar, conceptThreshold — is a function of D assuming unrelated
|
|
314
|
+
// ≈ 0 with σ = 1/√D. Raising that floor invalidates all of them at once.
|
|
315
|
+
for (const kind of ["uniform", "records"]) {
|
|
316
|
+
const vs = corpus(kind).base.map(gist);
|
|
317
|
+
const u = [];
|
|
318
|
+
for (let i = 0; i < vs.length; i++) {
|
|
319
|
+
for (let j = i + 1; j < vs.length; j++) {
|
|
320
|
+
u.push(Math.abs(cosine(vs[i], vs[j])));
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
const mean = u.reduce((a, x) => a + x, 0) / u.length;
|
|
324
|
+
assert.ok(
|
|
325
|
+
mean < 3 * estimatorNoise(D),
|
|
326
|
+
`${kind}: unrelated mean |cos| ${mean.toFixed(4)} ≥ 3σ ${
|
|
327
|
+
(3 * estimatorNoise(D)).toFixed(4)
|
|
328
|
+
}`,
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
test("the derived bars stay derived — no tuned constants creep in", () => {
|
|
334
|
+
assert.equal(mergeThreshold(D), 1 - 1 / Math.sqrt(D));
|
|
335
|
+
assert.equal(reachThreshold(W), 1 - 1 / (2 * W));
|
|
336
|
+
assert.equal(estimatorNoise(D), 1 / Math.sqrt(D));
|
|
337
|
+
assert.equal(significanceBar(D), 3 / Math.sqrt(D));
|
|
338
|
+
assert.equal(identityBar(D, W, 64), Math.max(mergeThreshold(D), 1 - W / 64));
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
342
|
+
// PART 2 — MEASURED BASELINE. Floors and ceilings with slack.
|
|
343
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
344
|
+
|
|
345
|
+
/** Measured 2026-07-28. `lead` is the cosine between a stream and a PREFIXED
|
|
346
|
+
* copy of itself — the shift-invariance this fold exists to provide; `trail`
|
|
347
|
+
* the same for an APPENDED copy; `unrelMax` the most similar unrelated
|
|
348
|
+
* stream in the corpus. Floors sit ~0.08 under the measurement, ceilings
|
|
349
|
+
* ~0.08 over, which is well outside run-to-run spread (these are medians of
|
|
350
|
+
* 48 pairs) and well inside the margin that separates the current geometry
|
|
351
|
+
* from its predecessors — HEAD's worst-case lead was 0.020 against 0.537. */
|
|
352
|
+
const BASELINE = {
|
|
353
|
+
// lead≥ trail≥ unrelMax≤
|
|
354
|
+
text: [0.68, 0.77, 0.28],
|
|
355
|
+
uniform: [0.70, 0.79, 0.17],
|
|
356
|
+
sparse: [0.68, 0.84, 0.64],
|
|
357
|
+
lowent: [0.64, 0.81, 0.71],
|
|
358
|
+
records: [0.68, 0.79, 0.26],
|
|
359
|
+
ramp: [0.44, 0.80, 0.17],
|
|
360
|
+
};
|
|
361
|
+
|
|
362
|
+
function shiftProfile(kind) {
|
|
363
|
+
const { base, pre, post } = corpus(kind);
|
|
364
|
+
const vs = base.map(gist);
|
|
365
|
+
const L = [], T = [], U = [];
|
|
366
|
+
base.forEach((b, i) => {
|
|
367
|
+
for (let k = 0; k < 4; k++) L.push(cosine(vs[i], gist(cat(pre(), b))));
|
|
368
|
+
for (let k = 0; k < 4; k++) T.push(cosine(vs[i], gist(cat(b, post()))));
|
|
369
|
+
for (let j = i + 1; j < base.length; j++) U.push(cosine(vs[i], vs[j]));
|
|
370
|
+
});
|
|
371
|
+
return { lead: med(L), trail: med(T), unrelMax: Math.max(...U) };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
test("shift invariance holds its measured baseline on every byte type", () => {
|
|
375
|
+
const worst = { lead: 1, trail: 1 };
|
|
376
|
+
for (const kind of CORPORA) {
|
|
377
|
+
const { lead, trail, unrelMax } = shiftProfile(kind);
|
|
378
|
+
const [leadFloor, trailFloor, unrelCeil] = BASELINE[kind];
|
|
379
|
+
assert.ok(
|
|
380
|
+
lead >= leadFloor,
|
|
381
|
+
`${kind}: lead ${lead.toFixed(3)} fell below the pinned ${leadFloor}`,
|
|
382
|
+
);
|
|
383
|
+
assert.ok(
|
|
384
|
+
trail >= trailFloor,
|
|
385
|
+
`${kind}: trail ${trail.toFixed(3)} fell below the pinned ${trailFloor}`,
|
|
386
|
+
);
|
|
387
|
+
assert.ok(
|
|
388
|
+
unrelMax <= unrelCeil,
|
|
389
|
+
`${kind}: unrelated max ${
|
|
390
|
+
unrelMax.toFixed(3)
|
|
391
|
+
} rose above the pinned ${unrelCeil}`,
|
|
392
|
+
);
|
|
393
|
+
worst.lead = Math.min(worst.lead, lead);
|
|
394
|
+
worst.trail = Math.min(worst.trail, trail);
|
|
395
|
+
}
|
|
396
|
+
// The headline pair. A change may trade between corpora; it may not lower
|
|
397
|
+
// the worst case, which is what "works on all byte types" means.
|
|
398
|
+
assert.ok(
|
|
399
|
+
worst.lead >= 0.44,
|
|
400
|
+
`worst-case lead across byte types: ${worst.lead.toFixed(3)}`,
|
|
401
|
+
);
|
|
402
|
+
assert.ok(
|
|
403
|
+
worst.trail >= 0.77,
|
|
404
|
+
`worst-case trail across byte types: ${worst.trail.toFixed(3)}`,
|
|
405
|
+
);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test("a shifted copy is more similar than any unrelated stream — the usable bar", () => {
|
|
409
|
+
// The defect this whole line of work removed: under the previous geometry a
|
|
410
|
+
// prefixed copy of a sentence (0.199) was LESS similar than the nearest
|
|
411
|
+
// unrelated sentence (0.265) — not separable at all. Every corpus must now
|
|
412
|
+
// clear its own unrelated maximum.
|
|
413
|
+
for (const kind of CORPORA) {
|
|
414
|
+
const { lead, unrelMax } = shiftProfile(kind);
|
|
415
|
+
assert.ok(
|
|
416
|
+
lead > unrelMax,
|
|
417
|
+
`${kind}: lead ${lead.toFixed(3)} ≤ unrelated max ${
|
|
418
|
+
unrelMax.toFixed(3)
|
|
419
|
+
} — a shifted copy is unrecognisable`,
|
|
420
|
+
);
|
|
421
|
+
}
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
test("content cuts re-synchronise after an insertion, on every byte type", () => {
|
|
425
|
+
// The cut rule reads a bounded window, so a prepend can only disturb
|
|
426
|
+
// boundaries until the window clears. Long-memory or last-gated rules
|
|
427
|
+
// phase-lock instead and never recover: the predecessor scored 0.441 on a
|
|
428
|
+
// gradient and 0.492 on sparse data.
|
|
429
|
+
const FLOOR = {
|
|
430
|
+
text: 0.90,
|
|
431
|
+
uniform: 0.88,
|
|
432
|
+
sparse: 0.78,
|
|
433
|
+
lowent: 0.84,
|
|
434
|
+
records: 0.87,
|
|
435
|
+
ramp: 0.88,
|
|
436
|
+
};
|
|
437
|
+
for (const kind of CORPORA) {
|
|
438
|
+
const { base, pre } = corpus(kind);
|
|
439
|
+
let sum = 0;
|
|
440
|
+
for (const b of base) {
|
|
441
|
+
const p = pre();
|
|
442
|
+
const A = contentBoundaries(space, b);
|
|
443
|
+
const B = contentBoundaries(space, cat(p, b)).map((c) => c - p.length)
|
|
444
|
+
.filter((c) => c > 0);
|
|
445
|
+
let k = 0;
|
|
446
|
+
while (
|
|
447
|
+
k < Math.min(A.length, B.length) &&
|
|
448
|
+
A[A.length - 1 - k] === B[B.length - 1 - k]
|
|
449
|
+
) k++;
|
|
450
|
+
sum += A.length ? k / A.length : 1;
|
|
451
|
+
}
|
|
452
|
+
const frac = sum / base.length;
|
|
453
|
+
assert.ok(
|
|
454
|
+
frac >= FLOOR[kind],
|
|
455
|
+
`${kind}: only ${frac.toFixed(3)} of cuts re-aligned (pinned ${
|
|
456
|
+
FLOOR[kind]
|
|
457
|
+
})`,
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
test("segments keep their scale, and never exceed the keyring", () => {
|
|
463
|
+
// Segment SCALE is load-bearing: the mechanisms downstream are fitted to
|
|
464
|
+
// this distribution, and a rule that halves it costs more than the
|
|
465
|
+
// structural purity it buys. The keyring bound is what makes a segment
|
|
466
|
+
// foldable at all — flatFold seats one child per seat.
|
|
467
|
+
for (const kind of CORPORA) {
|
|
468
|
+
const lens = [];
|
|
469
|
+
for (const b of corpus(kind, 9, 120).base) {
|
|
470
|
+
let prev = 0;
|
|
471
|
+
for (const c of contentBoundaries(space, b)) {
|
|
472
|
+
lens.push(c - prev);
|
|
473
|
+
prev = c;
|
|
474
|
+
}
|
|
475
|
+
lens.push(b.length - prev);
|
|
476
|
+
}
|
|
477
|
+
const mean = lens.reduce((a, x) => a + x, 0) / lens.length;
|
|
478
|
+
assert.ok(
|
|
479
|
+
Math.max(...lens) <= SEATS,
|
|
480
|
+
`${kind}: a ${
|
|
481
|
+
Math.max(...lens)
|
|
482
|
+
}-byte segment exceeds the ${SEATS}-seat keyring`,
|
|
483
|
+
);
|
|
484
|
+
assert.ok(
|
|
485
|
+
mean >= 3.5 && mean <= 7.5,
|
|
486
|
+
`${kind}: mean segment ${mean.toFixed(2)} left the 3.5–7.5 band`,
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// 64-two-ended-thresholds.test.mjs — changing the coordinate frame must not
|
|
2
|
+
// silently retune the geometry's statistical and structural decision bars.
|
|
3
|
+
|
|
4
|
+
import { test } from "node:test";
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import {
|
|
7
|
+
estimatorNoise,
|
|
8
|
+
identityBar,
|
|
9
|
+
mergeThreshold,
|
|
10
|
+
reachThreshold,
|
|
11
|
+
significanceBar,
|
|
12
|
+
} from "../dist/src/geometry.js";
|
|
13
|
+
import { fold } from "../dist/src/sema.js";
|
|
14
|
+
import { cosine, makeKeyring, randomUnit, rng } from "../dist/src/vec.js";
|
|
15
|
+
|
|
16
|
+
const D = 1024;
|
|
17
|
+
const W = 4;
|
|
18
|
+
const space = {
|
|
19
|
+
D,
|
|
20
|
+
seats: makeKeyring(D, 8, rng(1)),
|
|
21
|
+
rand: rng(2),
|
|
22
|
+
maxGroup: W,
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const mean = (xs) => xs.reduce((sum, x) => sum + x, 0) / xs.length;
|
|
26
|
+
|
|
27
|
+
test("the two-ended coordinate frame preserves the one-child overlap law", () => {
|
|
28
|
+
const rand = rng(991);
|
|
29
|
+
const replacements = [];
|
|
30
|
+
const unrelated = [];
|
|
31
|
+
|
|
32
|
+
for (let trial = 0; trial < 128; trial++) {
|
|
33
|
+
const original = Array.from({ length: W }, () => randomUnit(D, rand));
|
|
34
|
+
const changed = original.slice();
|
|
35
|
+
changed[trial % W] = randomUnit(D, rand);
|
|
36
|
+
const other = Array.from({ length: W }, () => randomUnit(D, rand));
|
|
37
|
+
|
|
38
|
+
replacements.push(cosine(fold(space, original), fold(space, changed)));
|
|
39
|
+
unrelated.push(cosine(fold(space, original), fold(space, other)));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const oneChildOverlap = 1 - 1 / W;
|
|
43
|
+
assert.ok(
|
|
44
|
+
Math.abs(mean(replacements) - oneChildOverlap) < estimatorNoise(D),
|
|
45
|
+
"renaming seats must preserve the expected 1 - 1/W overlap",
|
|
46
|
+
);
|
|
47
|
+
assert.ok(
|
|
48
|
+
replacements.every((score) => score < reachThreshold(W)),
|
|
49
|
+
"reach must remain stricter than replacing one complete child",
|
|
50
|
+
);
|
|
51
|
+
assert.ok(
|
|
52
|
+
Math.abs(mean(unrelated)) < estimatorNoise(D),
|
|
53
|
+
"unrelated folds must remain centred on zero",
|
|
54
|
+
);
|
|
55
|
+
assert.ok(
|
|
56
|
+
unrelated.every((score) => Math.abs(score) < significanceBar(D)),
|
|
57
|
+
"the seeded unrelated sample must remain inside the 3-sigma noise band",
|
|
58
|
+
);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("identity, reach, and significance remain derived laws", () => {
|
|
62
|
+
assert.equal(mergeThreshold(D), 1 - 1 / Math.sqrt(D));
|
|
63
|
+
assert.equal(reachThreshold(W), 1 - 1 / (2 * W));
|
|
64
|
+
assert.equal(significanceBar(D), 3 / Math.sqrt(D));
|
|
65
|
+
assert.equal(
|
|
66
|
+
identityBar(D, W, 64),
|
|
67
|
+
Math.max(mergeThreshold(D), 1 - W / 64),
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
const rand = rng(1234);
|
|
71
|
+
const children = Array.from({ length: W }, () => randomUnit(D, rand));
|
|
72
|
+
assert.ok(
|
|
73
|
+
cosine(fold(space, children), fold(space, children)) >= mergeThreshold(D),
|
|
74
|
+
"an identical structural form must still clear the identity floor",
|
|
75
|
+
);
|
|
76
|
+
});
|