@hviana/sema 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +71 -5
- package/CONTRIBUTING.md +92 -10
- package/LICENSE.md +2 -2
- package/dist/src/derive/src/deduction.d.ts +12 -1
- package/dist/src/derive/src/deduction.js +5 -1
- package/dist/src/derive/src/index.d.ts +1 -0
- package/dist/src/geometry.d.ts +3 -30
- package/dist/src/geometry.js +330 -82
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/meter.d.ts +171 -0
- package/dist/src/meter.js +269 -0
- package/dist/src/mind/attention.d.ts +0 -4
- package/dist/src/mind/attention.js +251 -23
- package/dist/src/mind/bridge.d.ts +10 -1
- package/dist/src/mind/bridge.js +179 -10
- package/dist/src/mind/canonical.d.ts +6 -1
- package/dist/src/mind/canonical.js +6 -1
- package/dist/src/mind/graph-search.d.ts +9 -0
- package/dist/src/mind/graph-search.js +59 -19
- package/dist/src/mind/junction.d.ts +10 -0
- package/dist/src/mind/junction.js +14 -0
- package/dist/src/mind/match.d.ts +40 -0
- package/dist/src/mind/match.js +125 -1
- package/dist/src/mind/mechanisms/cast.js +63 -6
- package/dist/src/mind/mechanisms/extraction.d.ts +0 -34
- package/dist/src/mind/mechanisms/extraction.js +1 -88
- package/dist/src/mind/mechanisms/recall.d.ts +3 -0
- package/dist/src/mind/mechanisms/recall.js +77 -14
- package/dist/src/mind/mind.d.ts +55 -4
- package/dist/src/mind/mind.js +110 -91
- package/dist/src/mind/pipeline-mechanism.d.ts +33 -3
- package/dist/src/mind/pipeline-mechanism.js +179 -10
- package/dist/src/mind/pipeline.js +52 -10
- package/dist/src/mind/primitives.d.ts +11 -15
- package/dist/src/mind/primitives.js +47 -28
- package/dist/src/mind/reasoning.d.ts +7 -1
- package/dist/src/mind/reasoning.js +40 -8
- package/dist/src/mind/recognition.js +93 -20
- package/dist/src/mind/traverse.d.ts +11 -0
- package/dist/src/mind/traverse.js +58 -5
- package/dist/src/mind/types.d.ts +28 -5
- package/dist/src/store.d.ts +15 -0
- package/dist/src/store.js +91 -6
- package/package.json +1 -1
- package/src/derive/src/deduction.ts +15 -0
- package/src/derive/src/index.ts +1 -0
- package/src/geometry.ts +350 -122
- package/src/index.ts +1 -0
- package/src/meter.ts +333 -0
- package/src/mind/attention.ts +268 -31
- package/src/mind/bridge.ts +187 -10
- package/src/mind/canonical.ts +6 -1
- package/src/mind/graph-search.ts +60 -21
- package/src/mind/junction.ts +12 -0
- package/src/mind/match.ts +146 -1
- package/src/mind/mechanisms/cast.ts +62 -6
- package/src/mind/mechanisms/extraction.ts +2 -103
- package/src/mind/mechanisms/recall.ts +84 -17
- package/src/mind/mind.ts +139 -98
- package/src/mind/pipeline-mechanism.ts +203 -13
- package/src/mind/pipeline.ts +65 -8
- package/src/mind/primitives.ts +49 -33
- package/src/mind/reasoning.ts +39 -7
- package/src/mind/recognition.ts +89 -19
- package/src/mind/traverse.ts +59 -5
- package/src/mind/types.ts +28 -5
- package/src/store.ts +75 -6
- package/test/14-scaling.test.mjs +17 -7
- package/test/31-audit.test.mjs +4 -1
- package/test/33-multi-candidate.test.mjs +13 -1
- package/test/36-already-answered-fusion.test.mjs +10 -3
- package/test/46-recognise-multibyte-edge.test.mjs +3 -3
- package/test/53-cross-region-probe-instrumentation.test.mjs +36 -6
- package/test/55-cost-meter.test.mjs +284 -0
- package/test/56-bridge-identity-admission.test.mjs +209 -0
- package/test/57-fusion-order.test.mjs +104 -0
- package/test/58-subquantum-sites.test.mjs +112 -0
- package/test/59-fold-invariance.test.mjs +226 -0
package/dist/src/geometry.js
CHANGED
|
@@ -242,6 +242,163 @@ function bytesToLeaves(alphabet, bytes) {
|
|
|
242
242
|
return { tree: sema(v, bytes.slice(i, i + 1), null), len: 1 };
|
|
243
243
|
});
|
|
244
244
|
}
|
|
245
|
+
/** CONTENT-DEFINED FOLD BOUNDARIES — where a byte stream segments, chosen by
|
|
246
|
+
* the bytes rather than by arithmetic.
|
|
247
|
+
*
|
|
248
|
+
* `riverFold` groups FIXED arity from byte 0 and permutes item k by
|
|
249
|
+
* `seats[k]`, k = index mod W, so a byte's contribution is a function of its
|
|
250
|
+
* ABSOLUTE OFFSET: the same byte run is a different vector at a different
|
|
251
|
+
* position, and the same content is a different SUBTREE. W's size has nothing
|
|
252
|
+
* to do with it — any fixed modulus does this, and identity must not depend on
|
|
253
|
+
* W at all.
|
|
254
|
+
*
|
|
255
|
+
* A rolling hash removes the dependence without touching the fold's arity: the
|
|
256
|
+
* cut lands where the hash of the recent bytes vanishes mod W, so a change
|
|
257
|
+
* upstream moves only the cut it falls inside — every downstream boundary, and
|
|
258
|
+
* therefore every downstream segment, is unchanged. Because
|
|
259
|
+
* {@link stablePrefixFold} folds each segment independently from its own slot
|
|
260
|
+
* 0, byte-identical content then produces byte-identical subtrees, and
|
|
261
|
+
* hash-consing makes it the SAME NODE ID wherever it appears. That — not the
|
|
262
|
+
* root cosine, which should and does still move when content is added — is
|
|
263
|
+
* what recognition, cover and resolve need.
|
|
264
|
+
*
|
|
265
|
+
* Measured over 400 real deposits, shifts of 1..7 bytes, downstream cuts
|
|
266
|
+
* preserved / segments byte-identical:
|
|
267
|
+
*
|
|
268
|
+
* content-defined, deposit text 99.7% / 98.4% mean seg 6.61 B
|
|
269
|
+
* content-defined, non-Latin scripts 99.6% / 98.3% mean seg 6.43 B
|
|
270
|
+
* content-defined, random binary 99.9% / 99.2% mean seg 6.92 B
|
|
271
|
+
* the arithmetic grid, same corpus 14.3% (only the k≡0 mod W
|
|
272
|
+
* shifts survive)
|
|
273
|
+
*
|
|
274
|
+
* The three content-defined rows agreeing is the load-bearing part: this reads
|
|
275
|
+
* BYTES, never text. Mind is not a text engine — the same fold carries grids
|
|
276
|
+
* and any other modality — so a boundary rule justified by where words or
|
|
277
|
+
* sentences fall would be importing an assumption the architecture rejects.
|
|
278
|
+
* Random binary must, and does, behave exactly like prose.
|
|
279
|
+
*
|
|
280
|
+
* Every constant is derived (§2.2): the cut mask is W, so a cut is offered once
|
|
281
|
+
* per quantum of bytes — which, composed with the minimum below, puts the
|
|
282
|
+
* expected segment at minLen + W − 1 ≈ 6 B rather than at W, deliberately (see
|
|
283
|
+
* the refutation recorded at `cutRate` in {@link contentLevels}: a segment is
|
|
284
|
+
* the flat PHRASE-scale unit the W-ary groups are built from, not a group of W
|
|
285
|
+
* children, and forcing E[len] = W costs 15 tests). The minimum is W−1, `canonicalWindows`'s
|
|
286
|
+
* straddle neighbour and the write side's own floor for a unit; and the maximum
|
|
287
|
+
* is the KEYRING's seat count, because a segment folds as ONE flat node and
|
|
288
|
+
* `fold` has exactly that many seats to bind children into. Capping there is
|
|
289
|
+
* what keeps the fold light: a segment of 3..seats leaves is a single node,
|
|
290
|
+
* where splitting it into W-groups plus a remainder would cost two or three
|
|
291
|
+
* and the remainders barely share (measured: partial-arity nodes 504 → 3,590,
|
|
292
|
+
* and total distinct nodes 8,142 → 9,712, when segments folded as [W][rest]). */
|
|
293
|
+
/** {@link contentBoundaries} plus, for each cut, its LEVEL — how deep in the
|
|
294
|
+
* tree that cut reaches.
|
|
295
|
+
*
|
|
296
|
+
* One rolling hash serves every scale. A cut is level 0 when its hash vanishes
|
|
297
|
+
* mod W, level 1 when mod W², and so on: level-L cuts are by construction a
|
|
298
|
+
* subset of level-(L−1) cuts, which is exactly the nesting a tree needs. The
|
|
299
|
+
* expected span of a level-L node is therefore W^(L+1) bytes — the same growth
|
|
300
|
+
* the grid fold had, but with boundaries the content chose, so a shift moves
|
|
301
|
+
* one node at each level instead of all of them.
|
|
302
|
+
*
|
|
303
|
+
* Levels are read from the hash the cut was ACCEPTED at, not recomputed, so
|
|
304
|
+
* they cost nothing beyond the divisions already being done. */
|
|
305
|
+
function contentLevels(space, bytes) {
|
|
306
|
+
const W = space.maxGroup;
|
|
307
|
+
const minLen = W - 1;
|
|
308
|
+
const maxLen = space.seats.length;
|
|
309
|
+
// MEASURED AND REFUTED — making E[segment] equal W. A segment is at least
|
|
310
|
+
// `minLen` bytes and then cuts with probability p, so E[len] = minLen +
|
|
311
|
+
// (1−p)/p; with `h % W === 0` that is minLen + W − 1 = 6 for W = 4 (measured
|
|
312
|
+
// mean 6.61 B on real deposits), half again coarser than the fold's own
|
|
313
|
+
// arity, and the doc above used to claim the mask made the two equal. It
|
|
314
|
+
// does not, and the mismatch looked like the cause of a real deficit: the
|
|
315
|
+
// climb perceives one region per segment, so on `Michelangelo is to sculpture
|
|
316
|
+
// as who is to literature?` it sees 9 regions where the grid saw 19, reaches
|
|
317
|
+
// one ranked anchor instead of two, and CAST's weave never includes the
|
|
318
|
+
// writing-domain exemplar it needs (test/29 A2, C1).
|
|
319
|
+
//
|
|
320
|
+
// Fixing the rate to hit E[len] = W (a threshold `h % W < W − minLen + 1` on
|
|
321
|
+
// the same rolling value, keeping the two bytes of history) does exactly what
|
|
322
|
+
// it says — mean segment 3.60 B on text, 3.93 B on random binary, 14 segments
|
|
323
|
+
// on that query instead of 9 — and takes the suite from 3 failures to 18,
|
|
324
|
+
// across think, universality, scaling, decomposition, bridge, generalization,
|
|
325
|
+
// saturation, audit and recognition. The coarser-than-arity scale is
|
|
326
|
+
// load-bearing: W is the arity `riverFold` groups CHILDREN at, and a segment
|
|
327
|
+
// is not a group of W children but the flat unit those groups are built from
|
|
328
|
+
// — a phrase-scale unit, and the mechanisms above all read it that way.
|
|
329
|
+
// Do not re-derive the rate from W.
|
|
330
|
+
//
|
|
331
|
+
// MEASURED AND REFUTED — reading the hash's HIGH bits instead of its low
|
|
332
|
+
// ones. The accumulator shifts LEFT, so `h % W` is the part the mixing has
|
|
333
|
+
// had no chance to reach: with `h = (h << 1) + byte·K` and K odd, `h mod 4`
|
|
334
|
+
// reduces to `(2·(prev mod 2) + byte) mod 4` — two bytes of history, and only
|
|
335
|
+
// their parity. It reads like a hash test that is really a byte-parity test,
|
|
336
|
+
// and it shows: on `Michelangelo is to sculpture as who is to literature?`
|
|
337
|
+
// the segments come out 8,8,8,8,8,7,4,2 — almost every boundary the FORCED
|
|
338
|
+
// one at maxLen, which is the one offset-dependent cut in the rule.
|
|
339
|
+
//
|
|
340
|
+
// Testing the top bits instead (`h < 2^32 / W`, the same 1/W rate, with the
|
|
341
|
+
// level test falling out as `h < 2^32 / W^(L+1)` — cleaner nesting than
|
|
342
|
+
// divisibility) takes the suite from 2 failures to 21 AND breaks the
|
|
343
|
+
// invariance floor in test/59. The reason is the whole point: in a
|
|
344
|
+
// shift-accumulator the LOW bits have a short effective window and the HIGH
|
|
345
|
+
// bits a long one, so a disturbance upstream perturbs the high bits for the
|
|
346
|
+
// next ~32 bytes while the low bits re-sync within two. The narrow window is
|
|
347
|
+
// not a defect in this hash — it IS the invariance. A boundary rule here is
|
|
348
|
+
// choosing how far a change may propagate, not how well the bits mix.
|
|
349
|
+
//
|
|
350
|
+
// MEASURED AND REFUTED — normalized chunking, to shrink the forced cut. The
|
|
351
|
+
// hard cut at maxLen is the ONE offset-dependent boundary left in the rule,
|
|
352
|
+
// and it is not rare: 32% of segments on a mixed sample end there, five of
|
|
353
|
+
// the eight in the query above, so for those streams the fold IS a grid of 2W
|
|
354
|
+
// with all of the grid's phase. The standard remedy applies cleanly here —
|
|
355
|
+
// past a target length, relax the mask by one power of the same radix
|
|
356
|
+
// (W → W/2), still content-defined and so still invariant, just coarser
|
|
357
|
+
// evidence for a boundary; both quantities are already in the rule, nothing
|
|
358
|
+
// introduced. It works as advertised: forced cuts fall from 32% to 8.3% and
|
|
359
|
+
// test/59's invariance floors still hold.
|
|
360
|
+
//
|
|
361
|
+
// And the suite goes from 2 failures to 5 relaxing at the expected length
|
|
362
|
+
// (minLen + W − 1), or to 6 relaxing at the last opportunity (maxLen − 1) —
|
|
363
|
+
// 22-multihop, 24-generalization, 29, 36. Taken with the two results above
|
|
364
|
+
// (changing the rate costs 15 tests; changing which bits are read costs 19),
|
|
365
|
+
// the reading is that the segment DISTRIBUTION is what the mechanisms
|
|
366
|
+
// downstream are fitted to, not the purity of the rule that produces it. The
|
|
367
|
+
// forced cut is part of that distribution. Do not tidy it away without
|
|
368
|
+
// re-measuring everything that reads a region.
|
|
369
|
+
const cuts = [];
|
|
370
|
+
const levels = [];
|
|
371
|
+
let h = 0;
|
|
372
|
+
let last = 0;
|
|
373
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
374
|
+
h = (((h << 1) >>> 0) + Math.imul(bytes[i], 2654435761)) >>> 0;
|
|
375
|
+
if (i + 1 >= bytes.length)
|
|
376
|
+
break;
|
|
377
|
+
const hit = h % W === 0;
|
|
378
|
+
if (i - last >= minLen && (hit || i - last + 1 >= maxLen)) {
|
|
379
|
+
cuts.push(i + 1);
|
|
380
|
+
// How many further powers of W divide the hash — 0 for a forced cut.
|
|
381
|
+
let lvl = 0;
|
|
382
|
+
if (hit) {
|
|
383
|
+
let m = W;
|
|
384
|
+
while (lvl < 24 && m <= 0x40000000 && h % (m * W) === 0) {
|
|
385
|
+
lvl++;
|
|
386
|
+
m *= W;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
levels.push(lvl);
|
|
390
|
+
last = i + 1;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return { cuts, levels };
|
|
394
|
+
}
|
|
395
|
+
export function contentBoundaries(space, bytes) {
|
|
396
|
+
// ONE implementation of the rule. This used to carry its own copy of the
|
|
397
|
+
// rolling-hash loop, which is exactly how a write side and a read side drift
|
|
398
|
+
// apart without a type error; the levels are computed from the hash the cut
|
|
399
|
+
// was accepted at, so asking for them costs nothing but an array.
|
|
400
|
+
return contentLevels(space, bytes).cuts;
|
|
401
|
+
}
|
|
245
402
|
/** Find the longest prefix of `bytes` whose leaf-id signature matches a
|
|
246
403
|
* known branch via `lookup`. Returns the byte-length of that prefix, or 0. */
|
|
247
404
|
export function knownPrefixLength(bytes, leafAt, lookup) {
|
|
@@ -283,11 +440,144 @@ export function bytesToTree(space, alphabet, bytes, leafAt, lookup, boundaries)
|
|
|
283
440
|
if (bytes.length === 0) {
|
|
284
441
|
return sema(alphabet.vecs[0], new Uint8Array(0), null);
|
|
285
442
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
443
|
+
// WHERE A STREAM SEGMENTS IS DECIDED BY ITS BYTES ({@link contentBoundaries}),
|
|
444
|
+
// and the segments then fold BALANCED, W-ary, like any other row of items.
|
|
445
|
+
//
|
|
446
|
+
// The two must not be confused. Content cuts are what make identity
|
|
447
|
+
// offset-free: a segment folds from its own slot 0, so the same bytes give the
|
|
448
|
+
// same subtree wherever they sit. The shape ABOVE the segments is a separate
|
|
449
|
+
// question, and it must stay the river's own — grouping W at a time, depth
|
|
450
|
+
// log_W(n). Joining segments left-nested instead (as the stable-prefix fold
|
|
451
|
+
// does, for its own good reason) costs a node per segment on a single spine:
|
|
452
|
+
// at a cut every ~6.6 B a 3 KB deposit becomes a 450-deep spine of 450 fresh
|
|
453
|
+
// D-vectors, ~1.8 MB for one deposit, and every walker above inherits the
|
|
454
|
+
// depth. That is an implementation blunder, not a property of content-defined
|
|
455
|
+
// folding, and it is what riverFold below avoids.
|
|
456
|
+
//
|
|
457
|
+
// Caller-supplied boundaries stay left-nested (see stablePrefixFold): there are
|
|
458
|
+
// a handful of them, one per conversation turn, and the cumulative-context-root
|
|
459
|
+
// contract depends on that shape. Each SPAN between them content-folds.
|
|
289
460
|
const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
|
|
290
|
-
|
|
461
|
+
const outer = new Set();
|
|
462
|
+
if (boundaries !== undefined) {
|
|
463
|
+
for (const b of boundaries)
|
|
464
|
+
outer.add(b);
|
|
465
|
+
}
|
|
466
|
+
if (sb > 0)
|
|
467
|
+
outer.add(sb);
|
|
468
|
+
if (outer.size === 0) {
|
|
469
|
+
return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
|
|
470
|
+
}
|
|
471
|
+
return stablePrefixFold(space, alphabet, bytes, [...outer].sort((a, b) => a - b));
|
|
472
|
+
}
|
|
473
|
+
/** One span, folded over its own content cuts — AT EVERY LEVEL.
|
|
474
|
+
*
|
|
475
|
+
* A segment becomes ONE FLAT NODE: every leaf bound into its own seat and
|
|
476
|
+
* summed, arity = the segment's length. `contentBoundaries` caps a segment at
|
|
477
|
+
* the keyring's seat count so this is always possible, and the flat form is
|
|
478
|
+
* both lighter (one node per segment instead of a [W][remainder] pair) and the
|
|
479
|
+
* natural unit — a segment IS the smallest thing the cuts claim is a unit.
|
|
480
|
+
*
|
|
481
|
+
* Above the segments the cutting RECURSES rather than reverting to the grid.
|
|
482
|
+
* Grouping segment roots W-at-a-time from index 0 would reintroduce the very
|
|
483
|
+
* bug content cuts exist to remove, one level up: a form spanning segments
|
|
484
|
+
* 12..17 straddles the [12-15] and [16-17] groups and is no node at all, so
|
|
485
|
+
* recognition can only reach it by an alignment accident (test/44 pins exactly
|
|
486
|
+
* this — at HEAD the grid happened to put a node one byte before the target).
|
|
487
|
+
* {@link contentLevels} assigns each cut a level from how divisible its hash
|
|
488
|
+
* is, so level-L cuts are a subset of level-(L−1) cuts and every node at every
|
|
489
|
+
* scale is delimited by content. Identity is then offset-free at all scales,
|
|
490
|
+
* which is the whole requirement — it must not depend on W. */
|
|
491
|
+
function contentFoldSpan(space, alphabet, bytes, from, to) {
|
|
492
|
+
const span = bytes.subarray(from, to);
|
|
493
|
+
const { cuts, levels } = contentLevels(space, span);
|
|
494
|
+
const edges = [0, ...cuts, span.length];
|
|
495
|
+
const segs = [];
|
|
496
|
+
for (let i = 0; i + 1 < edges.length; i++) {
|
|
497
|
+
segs.push(flatFold(space, alphabet, span, edges[i], edges[i + 1]));
|
|
498
|
+
}
|
|
499
|
+
if (segs.length > 1)
|
|
500
|
+
return groupByLevel(space, segs, levels, 1);
|
|
501
|
+
return segs[0];
|
|
502
|
+
}
|
|
503
|
+
/** Group a row of items by the level of the cut BETWEEN them: items separated
|
|
504
|
+
* by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
|
|
505
|
+
* it. Recurses upward until one root remains, so the shape at every level is
|
|
506
|
+
* the content's, not an index's. `levels[i]` is the level of the cut that
|
|
507
|
+
* precedes item i+1 (there are items.length − 1 of them).
|
|
508
|
+
*
|
|
509
|
+
* A level that fails to split (every cut below L) or that would exceed the
|
|
510
|
+
* keyring falls through to the plain river fold for that row — the fold stays
|
|
511
|
+
* total on any input, and the fallback is rare enough not to reintroduce a
|
|
512
|
+
* systematic alignment. */
|
|
513
|
+
function groupByLevel(space, items, levels, level) {
|
|
514
|
+
if (items.length === 1)
|
|
515
|
+
return items[0];
|
|
516
|
+
const maxSeats = space.seats.length;
|
|
517
|
+
const groups = [];
|
|
518
|
+
const groupLevels = [];
|
|
519
|
+
let start = 0;
|
|
520
|
+
for (let i = 0; i <= levels.length; i++) {
|
|
521
|
+
const atEnd = i === levels.length;
|
|
522
|
+
const cutHere = atEnd || levels[i] >= level;
|
|
523
|
+
const wouldOverflow = i - start + 1 >= maxSeats;
|
|
524
|
+
if (!cutHere && !wouldOverflow)
|
|
525
|
+
continue;
|
|
526
|
+
const slice = items.slice(start, i + 1);
|
|
527
|
+
groups.push(slice.length === 1 ? slice[0] : joinFlat(space, slice));
|
|
528
|
+
if (!atEnd)
|
|
529
|
+
groupLevels.push(levels[i]);
|
|
530
|
+
start = i + 1;
|
|
531
|
+
}
|
|
532
|
+
if (groups.length === items.length) {
|
|
533
|
+
// This level split nothing — climb rather than spin.
|
|
534
|
+
return level < 24
|
|
535
|
+
? groupByLevel(space, items, levels, level + 1)
|
|
536
|
+
: riverFoldRaw(space, items);
|
|
537
|
+
}
|
|
538
|
+
return groupByLevel(space, groups, groupLevels, level + 1);
|
|
539
|
+
}
|
|
540
|
+
/** Join a row of already-folded items as one unnormalized node — the same
|
|
541
|
+
* seat-bound accumulate `flatFold` does for bytes, one level up. */
|
|
542
|
+
function joinFlat(space, items) {
|
|
543
|
+
const gist = new Float32Array(space.D);
|
|
544
|
+
const kids = new Array(items.length);
|
|
545
|
+
let len = 0;
|
|
546
|
+
for (let k = 0; k < items.length; k++) {
|
|
547
|
+
const v = items[k].tree.v;
|
|
548
|
+
const seat = space.seats[k].fwd;
|
|
549
|
+
for (let d = 0; d < space.D; d++)
|
|
550
|
+
gist[d] += v[seat[d]];
|
|
551
|
+
kids[k] = items[k].tree;
|
|
552
|
+
len += items[k].len;
|
|
553
|
+
}
|
|
554
|
+
return { tree: sema(gist, null, kids), len };
|
|
555
|
+
}
|
|
556
|
+
/** One segment as a single unnormalized node: leaf per byte, each bound into
|
|
557
|
+
* seat k, summed. Same FP ops and same seat order as foldSlice's group fold —
|
|
558
|
+
* only the arity is the segment's own length rather than a fixed W. Never
|
|
559
|
+
* normalizes: the linear-fold contract keeps every interior gist raw and
|
|
560
|
+
* normalizes once at the root. */
|
|
561
|
+
function flatFold(space, alphabet, bytes, from, to) {
|
|
562
|
+
const n = to - from;
|
|
563
|
+
if (n === 1) {
|
|
564
|
+
const b = bytes[from];
|
|
565
|
+
return {
|
|
566
|
+
tree: sema(alphabet.vecs[b], bytes.slice(from, to), null),
|
|
567
|
+
len: 1,
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
const gist = new Float32Array(space.D);
|
|
571
|
+
const kids = new Array(n);
|
|
572
|
+
for (let k = 0; k < n; k++) {
|
|
573
|
+
const b = bytes[from + k];
|
|
574
|
+
const v = alphabet.vecs[b];
|
|
575
|
+
const seat = space.seats[k].fwd;
|
|
576
|
+
for (let d = 0; d < space.D; d++)
|
|
577
|
+
gist[d] += v[seat[d]];
|
|
578
|
+
kids[k] = sema(v, bytes.slice(from + k, from + k + 1), null);
|
|
579
|
+
}
|
|
580
|
+
return { tree: sema(gist, null, kids), len: n };
|
|
291
581
|
}
|
|
292
582
|
/** The stable-prefix segmented fold (§10.3). Each segment between
|
|
293
583
|
* consecutive boundaries folds PLAINLY and independently; segment roots
|
|
@@ -297,6 +587,22 @@ export function bytesToTree(space, alphabet, bytes, leafAt, lookup, boundaries)
|
|
|
297
587
|
* boundary, so the left-nested join reproduces every intermediate learnt
|
|
298
588
|
* root ((s₀·s₁) IS the root the store learnt for the first two segments'
|
|
299
589
|
* bytes, and so on). */
|
|
590
|
+
/** A fold's ROOT: ONE normalize per perception, at the root, exactly as
|
|
591
|
+
* riverFold did — the interior stays raw (the linear-fold contract).
|
|
592
|
+
*
|
|
593
|
+
* Normalizes EXCEPT when the whole stream folded to a single
|
|
594
|
+
* leaf: a leaf's vector IS the alphabet's own, shared by every occurrence of
|
|
595
|
+
* that byte, and `normalize` writes in place. Every fold entry point returns
|
|
596
|
+
* through here, because the guard is exactly the kind that gets written at one
|
|
597
|
+
* site and missed at the next two — which is what had happened: only
|
|
598
|
+
* {@link bytesToTree} carried it, while `stablePrefixFold` and its incremental
|
|
599
|
+
* twin normalized unconditionally, reachable by a one-byte stream whose only
|
|
600
|
+
* boundary is its own length. */
|
|
601
|
+
function rootOf(f) {
|
|
602
|
+
if (f.tree.kids !== null)
|
|
603
|
+
normalize(f.tree.v);
|
|
604
|
+
return f.tree;
|
|
605
|
+
}
|
|
300
606
|
function stablePrefixFold(space, alphabet, bytes, boundaries) {
|
|
301
607
|
const cuts = [];
|
|
302
608
|
let prev = 0;
|
|
@@ -307,19 +613,17 @@ function stablePrefixFold(space, alphabet, bytes, boundaries) {
|
|
|
307
613
|
}
|
|
308
614
|
}
|
|
309
615
|
if (cuts.length === 0) {
|
|
310
|
-
return
|
|
616
|
+
return rootOf(contentFoldSpan(space, alphabet, bytes, 0, bytes.length));
|
|
311
617
|
}
|
|
312
618
|
const edges = [0, ...cuts, bytes.length];
|
|
313
619
|
const segs = [];
|
|
314
620
|
for (let i = 0; i + 1 < edges.length; i++) {
|
|
315
|
-
|
|
316
|
-
segs.push(riverFoldRaw(space, bytesToLeaves(alphabet, seg)));
|
|
621
|
+
segs.push(contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
317
622
|
}
|
|
318
623
|
let cur = segs[0];
|
|
319
624
|
for (let i = 1; i < segs.length; i++)
|
|
320
625
|
cur = fold2(space, cur, segs[i]);
|
|
321
|
-
|
|
322
|
-
return cur.tree;
|
|
626
|
+
return rootOf(cur);
|
|
323
627
|
}
|
|
324
628
|
/** {@link stablePrefixFold} with incremental segment reuse — same cuts, same
|
|
325
629
|
* segment folds, same left-nested join, same single root normalize; `prev`
|
|
@@ -342,19 +646,30 @@ export function stablePrefixFoldIncremental(space, alphabet, bytes, boundaries,
|
|
|
342
646
|
prev.edges[i + 1] === edges[i + 1]
|
|
343
647
|
? prev.segs[i]
|
|
344
648
|
: undefined;
|
|
345
|
-
segs.push(hit ??
|
|
346
|
-
riverFoldRaw(space, bytesToLeaves(alphabet, bytes.subarray(edges[i], edges[i + 1]))));
|
|
649
|
+
segs.push(hit ?? contentFoldSpan(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
347
650
|
}
|
|
348
651
|
if (segs.length === 1) {
|
|
349
|
-
// Degenerate boundary set — the
|
|
350
|
-
|
|
652
|
+
// Degenerate boundary set — one span, which IS the whole stream, and it was
|
|
653
|
+
// just folded (or reused from `prev`) right above. It cannot go through
|
|
654
|
+
// `rootOf`: the ROOT is normalized, a cached SEGMENT never is (a later turn
|
|
655
|
+
// reuses it as one), and `normalize` writes in place. Re-folding `bytes`
|
|
656
|
+
// to get a separate object is what this used to do — and a first-seen
|
|
657
|
+
// deposit has no boundaries, so it always lands here, paying the fold
|
|
658
|
+
// twice. Copying the gist is the same result for one vector copy. A
|
|
659
|
+
// single LEAF needs neither: its vector is the shared alphabet entry and
|
|
660
|
+
// must not be written at all.
|
|
661
|
+
const only = segs[0].tree;
|
|
662
|
+
const tree = only.kids === null
|
|
663
|
+
? only
|
|
664
|
+
: sema(Float32Array.from(only.v), null, only.kids);
|
|
665
|
+
if (tree.kids !== null)
|
|
666
|
+
normalize(tree.v);
|
|
351
667
|
return { tree, fold: { edges, segs } };
|
|
352
668
|
}
|
|
353
669
|
let cur = segs[0];
|
|
354
670
|
for (let i = 1; i < segs.length; i++)
|
|
355
671
|
cur = fold2(space, cur, segs[i]);
|
|
356
|
-
|
|
357
|
-
return { tree: cur.tree, fold: { edges, segs } };
|
|
672
|
+
return { tree: rootOf(cur), fold: { edges, segs } };
|
|
358
673
|
}
|
|
359
674
|
/** Join two folded items as one 2-kid branch — the top-level join of the
|
|
360
675
|
* stable-prefix fold, identical FP ops to foldSlice's seat-bound
|
|
@@ -419,73 +734,6 @@ export function composeStructuralGist(space, parts) {
|
|
|
419
734
|
normalize(result);
|
|
420
735
|
return result;
|
|
421
736
|
}
|
|
422
|
-
/** Plain bytes→tree (identical to capability-less {@link bytesToTree}) that
|
|
423
|
-
* also RETURNS its pyramid, reusing `prev` — the pyramid of a PROPER
|
|
424
|
-
* prefix of `bytes` (caller guarantees content match and
|
|
425
|
-
* prev.bytes < bytes.length). */
|
|
426
|
-
export function bytesToTreePyramid(space, alphabet, bytes, prev) {
|
|
427
|
-
if (bytes.length === 0) {
|
|
428
|
-
return {
|
|
429
|
-
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
430
|
-
pyramid: { levels: [], bytes: 0 },
|
|
431
|
-
};
|
|
432
|
-
}
|
|
433
|
-
// ZERO GROWTH: the previous pyramid already covers every byte — its top
|
|
434
|
-
// level holds the finished, normalized root. Refolding here would not
|
|
435
|
-
// only waste the work: when the whole span is one full block, the refold's
|
|
436
|
-
// top item is a REUSED raw interior of `prev`, and the final normalize
|
|
437
|
-
// below would mutate that shared raw block in place, corrupting every
|
|
438
|
-
// later incremental fold built on `prev`.
|
|
439
|
-
if (prev && prev.bytes === bytes.length && prev.levels.length > 0) {
|
|
440
|
-
return {
|
|
441
|
-
tree: prev.levels[prev.levels.length - 1][0].tree,
|
|
442
|
-
pyramid: prev,
|
|
443
|
-
};
|
|
444
|
-
}
|
|
445
|
-
const mg = space.maxGroup;
|
|
446
|
-
const reusable = (L) => {
|
|
447
|
-
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
448
|
-
// be raw interiors, so the top level is always excluded.
|
|
449
|
-
if (!prev || L > prev.levels.length - 2)
|
|
450
|
-
return null;
|
|
451
|
-
return prev.levels[L];
|
|
452
|
-
};
|
|
453
|
-
// Level 0: reuse the prefix's leaf items wholesale (all are full blocks).
|
|
454
|
-
const lv0 = reusable(0);
|
|
455
|
-
const row = lv0
|
|
456
|
-
? [...lv0, ...bytesToLeaves(alphabet, bytes.subarray(prev.bytes))]
|
|
457
|
-
: bytesToLeaves(alphabet, bytes);
|
|
458
|
-
const levels = [row];
|
|
459
|
-
let level = row;
|
|
460
|
-
let L = 0;
|
|
461
|
-
while (level.length > 1) {
|
|
462
|
-
let next = [];
|
|
463
|
-
// Reuse the leading FULL blocks of prev's level L+1 (len == mg^(L+1),
|
|
464
|
-
// wholly inside the prefix); the rule below folds the rest from this
|
|
465
|
-
// level's items exactly as the from-scratch fold would.
|
|
466
|
-
const blockLen = mg ** (L + 1);
|
|
467
|
-
let reused = 0;
|
|
468
|
-
const cand = reusable(L + 1);
|
|
469
|
-
if (cand) {
|
|
470
|
-
const maxFull = Math.floor(prev.bytes / blockLen);
|
|
471
|
-
while (reused < maxFull && reused < cand.length &&
|
|
472
|
-
cand[reused].len === blockLen)
|
|
473
|
-
reused++;
|
|
474
|
-
for (let i = 0; i < reused; i++)
|
|
475
|
-
next.push(cand[i]);
|
|
476
|
-
}
|
|
477
|
-
foldSlice(space, level, reused * mg, level.length - reused * mg, next, true);
|
|
478
|
-
levels.push(next);
|
|
479
|
-
level = next;
|
|
480
|
-
L++;
|
|
481
|
-
}
|
|
482
|
-
if (row.length > 1)
|
|
483
|
-
normalize(level[0].tree.v);
|
|
484
|
-
return {
|
|
485
|
-
tree: level[0].tree,
|
|
486
|
-
pyramid: { levels, bytes: bytes.length },
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
737
|
// ---- n-D Hilbert curve ----
|
|
490
738
|
function gridDims(grid) {
|
|
491
739
|
if (grid.dims && grid.dims.length > 0)
|
package/dist/src/index.d.ts
CHANGED
package/dist/src/index.js
CHANGED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/** Per-phase accumulation — one entry per mechanism `floor`/`run` and per
|
|
2
|
+
* named stage.
|
|
3
|
+
*
|
|
4
|
+
* PHASES NEST, AND ARE NOT DISJOINT. `think` contains every mechanism
|
|
5
|
+
* phase; a mechanism's `floor` contains whatever shared analysis it
|
|
6
|
+
* first-touched (`attention`, `weave`); `recall.run` contains
|
|
7
|
+
* `substitutionBridge`, which contains `recall.exhaustiveResonate`. Read a
|
|
8
|
+
* phase as "wall-clock spent inside this, inclusive" — never sum them and
|
|
9
|
+
* expect the total. `CostReport.elapsedMs` is the only whole.
|
|
10
|
+
*
|
|
11
|
+
* Shared analyses are charged to THEMSELVES, not to the mechanism that
|
|
12
|
+
* first-touched them: the first toucher pays the wall clock, but every
|
|
13
|
+
* later consumer gets the result free, so blaming it would misread which
|
|
14
|
+
* work is expensive. (Before this, the profile read "cast.floor costs
|
|
15
|
+
* 2.9 s"; what actually cost 2.7 s of that was the consensus climb, which
|
|
16
|
+
* cast merely paid for on everyone's behalf.) */
|
|
17
|
+
export interface PhaseCost {
|
|
18
|
+
/** How many times the phase ran. */
|
|
19
|
+
calls: number;
|
|
20
|
+
/** Wall-clock milliseconds spent inside it (non-deterministic). */
|
|
21
|
+
ms: number;
|
|
22
|
+
/** Work counters accrued INSIDE this phase — the same names as
|
|
23
|
+
* {@link CostReport.counters}, differenced across the phase's entry and
|
|
24
|
+
* exit, summed over its calls. Deterministic, unlike `ms`, and the field
|
|
25
|
+
* that answers "which phase did those 75,000 byte reads?" — a question a
|
|
26
|
+
* whole-response counter total cannot. Inclusive, like `ms`: a nested
|
|
27
|
+
* phase's work is counted in its parent too. */
|
|
28
|
+
counters: Record<string, number>;
|
|
29
|
+
}
|
|
30
|
+
/** A snapshot of one inference call's computational usage. Plain data —
|
|
31
|
+
* JSON-serialisable, diffable between runs. */
|
|
32
|
+
export interface CostReport {
|
|
33
|
+
version: 1;
|
|
34
|
+
/** Wall-clock milliseconds of the whole call (non-deterministic). */
|
|
35
|
+
elapsedMs: number;
|
|
36
|
+
/** Query length in bytes — the scale every other number is read against. */
|
|
37
|
+
queryBytes: number;
|
|
38
|
+
/** Deterministic work counters, by layer. */
|
|
39
|
+
counters: Record<string, number>;
|
|
40
|
+
/** Per-phase call counts and millisecond totals, insertion-ordered.
|
|
41
|
+
* NESTED — see {@link PhaseCost}; do not sum. */
|
|
42
|
+
phases: Record<string, PhaseCost>;
|
|
43
|
+
}
|
|
44
|
+
/** The mutable accumulator. One is created per profiled inference call and
|
|
45
|
+
* handed to every layer through `ctx.meter` / `store.meter`. */
|
|
46
|
+
export declare class Meter {
|
|
47
|
+
/** `store.get` — one node record materialised (cache hit or DB row). */
|
|
48
|
+
nodeRecords: number;
|
|
49
|
+
/** `store.bytes` / `store.bytesPrefix` — one reconstruction request. */
|
|
50
|
+
byteReads: number;
|
|
51
|
+
/** Bytes actually handed back by those reads — the real I/O volume, and
|
|
52
|
+
* the number that exposes an unbounded read (AGENTS §2.8) that a call
|
|
53
|
+
* count alone hides. */
|
|
54
|
+
bytesRead: number;
|
|
55
|
+
/** `store.contentLen`. */
|
|
56
|
+
lenReads: number;
|
|
57
|
+
/** `store.findLeaf`. */
|
|
58
|
+
leafLookups: number;
|
|
59
|
+
/** `store.findBranch`. */
|
|
60
|
+
branchLookups: number;
|
|
61
|
+
/** `store.canonFind` — equivalence-class candidate proposal. */
|
|
62
|
+
canonLookups: number;
|
|
63
|
+
/** `parents` + `parentsFirst` — materialising parent reads. */
|
|
64
|
+
parentReads: number;
|
|
65
|
+
/** `hasParents` — indexed existence probe. */
|
|
66
|
+
parentProbes: number;
|
|
67
|
+
/** `chainRun` — one bounded transparent-chain climb. */
|
|
68
|
+
chainRuns: number;
|
|
69
|
+
/** `containers` + `containersSlice`. */
|
|
70
|
+
containerReads: number;
|
|
71
|
+
/** `hasContainers` — indexed existence probe. */
|
|
72
|
+
containerProbes: number;
|
|
73
|
+
/** `next` + `nextFirst`. */
|
|
74
|
+
edgeReads: number;
|
|
75
|
+
/** `hasNext` — indexed existence probe. */
|
|
76
|
+
edgeProbes: number;
|
|
77
|
+
/** `prev` + `prevFirst`. */
|
|
78
|
+
prevReads: number;
|
|
79
|
+
/** `prevCount` — indexed count probe. */
|
|
80
|
+
prevProbes: number;
|
|
81
|
+
/** `halo` — one halo vector decoded. */
|
|
82
|
+
haloReads: number;
|
|
83
|
+
/** `hasHalo` + `haloMass` — probes that never decode a vector. */
|
|
84
|
+
haloProbes: number;
|
|
85
|
+
/** `resonate` calls that descended the content index. */
|
|
86
|
+
annQueries: number;
|
|
87
|
+
/** `resonate` calls served from the per-flush read cache. */
|
|
88
|
+
annCacheHits: number;
|
|
89
|
+
/** Stored vectors the content index actually scored — the dominant cost of
|
|
90
|
+
* a query on a large store, and the one counter that grows with N when a
|
|
91
|
+
* budget is missing. */
|
|
92
|
+
annVectorReads: number;
|
|
93
|
+
/** `resonateHalo` calls. */
|
|
94
|
+
haloQueries: number;
|
|
95
|
+
/** `perceive` calls that actually folded (memo misses only). */
|
|
96
|
+
perceptions: number;
|
|
97
|
+
/** Bytes folded by those perceptions — perception is O(bytes), so this is
|
|
98
|
+
* its true cost, and it is what a multi-turn regression shows up in first
|
|
99
|
+
* (re-folding the whole context instead of the new turn). */
|
|
100
|
+
perceivedBytes: number;
|
|
101
|
+
/** `perceive` calls served from the per-response / conversation memo. */
|
|
102
|
+
perceiveHits: number;
|
|
103
|
+
/** `recognise` calls that actually ran. */
|
|
104
|
+
recognitions: number;
|
|
105
|
+
/** Bytes recognised by those calls. */
|
|
106
|
+
recognisedBytes: number;
|
|
107
|
+
/** `recognise` calls served from the memo. */
|
|
108
|
+
recogniseHits: number;
|
|
109
|
+
/** `resolve` — whole-span content-addressed identity requests. */
|
|
110
|
+
resolves: number;
|
|
111
|
+
/** `climbAttentionAll` calls that actually climbed. */
|
|
112
|
+
climbs: number;
|
|
113
|
+
/** `climbAttentionAll` calls served from `climbMemo`. */
|
|
114
|
+
climbHits: number;
|
|
115
|
+
/** Query regions the climb voted on, summed over climbs. */
|
|
116
|
+
climbRegions: number;
|
|
117
|
+
/** Nodes popped and examined by `edgeAncestors` ascents — the climb's
|
|
118
|
+
* inner loop, and the thing `hubBound` is supposed to be bounding. */
|
|
119
|
+
ancestorVisits: number;
|
|
120
|
+
/** `alignGraded` / `alignRuns` invocations. */
|
|
121
|
+
alignments: number;
|
|
122
|
+
/** Σ (query bytes × context bytes) over those alignments — the alignment
|
|
123
|
+
* family is quadratic, so this is the honest unit. */
|
|
124
|
+
alignCells: number;
|
|
125
|
+
/** `junctionContainersFrom` ascents started — the cross-region ladder's
|
|
126
|
+
* and the bridge's shared "which learnt whole contains these two forms?"
|
|
127
|
+
* walk. */
|
|
128
|
+
junctionWalks: number;
|
|
129
|
+
/** Nodes popped by those ascents, against their √N·W budget — the counter
|
|
130
|
+
* that shows whether the walks are deciding early or burning the budget. */
|
|
131
|
+
junctionPops: number;
|
|
132
|
+
/** `lightestDerivation` searches started. */
|
|
133
|
+
searches: number;
|
|
134
|
+
/** Chart items popped by those searches. */
|
|
135
|
+
searchPops: number;
|
|
136
|
+
/** Chart items pushed by those searches. */
|
|
137
|
+
searchPushes: number;
|
|
138
|
+
/** `floor()` calls that returned a bound (the mechanism could fire). */
|
|
139
|
+
mechanismFloors: number;
|
|
140
|
+
/** `floor()` calls that returned null (structurally impossible). */
|
|
141
|
+
mechanismSkips: number;
|
|
142
|
+
/** `run()` calls — the ones the floor pruning let through. */
|
|
143
|
+
mechanismRuns: number;
|
|
144
|
+
/** Candidates the decider weighed. */
|
|
145
|
+
candidates: number;
|
|
146
|
+
/** Candidates refused before the competition for explaining less than 1/W
|
|
147
|
+
* of the query — the honesty-density floor (see pipeline.ts `consider`). */
|
|
148
|
+
thinRejects: number;
|
|
149
|
+
private readonly _phases;
|
|
150
|
+
private readonly _t0;
|
|
151
|
+
/** Every work counter's current value, by name — the snapshot `time`
|
|
152
|
+
* differences to attribute work to a phase. */
|
|
153
|
+
private snapshot;
|
|
154
|
+
/** Charge `ms`, one call, and a counter delta to a named phase.
|
|
155
|
+
* Insertion-ordered, so a report reads in execution order. */
|
|
156
|
+
charge(phase: string, ms: number, delta?: Record<string, number>): void;
|
|
157
|
+
/** Time one async phase and attribute the work done inside it. Returns
|
|
158
|
+
* the awaited value untouched — a meter never changes what a layer
|
|
159
|
+
* computes, only what is known about it. */
|
|
160
|
+
time<T>(phase: string, fn: () => Promise<T>): Promise<T>;
|
|
161
|
+
/** The finished report. Zero-valued counters are dropped: a report should
|
|
162
|
+
* show what a query DID, not the whole vocabulary of what it might have. */
|
|
163
|
+
report(queryBytes: number): CostReport;
|
|
164
|
+
}
|
|
165
|
+
/** Sum a set of reports into one — for aggregating a battery of probes or a
|
|
166
|
+
* multi-turn session. `elapsedMs` and `queryBytes` add; counters and phases
|
|
167
|
+
* merge by key. */
|
|
168
|
+
export declare function sumReports(reports: readonly CostReport[]): CostReport;
|
|
169
|
+
/** A human-readable rendering of a report — the shape a profiling run prints.
|
|
170
|
+
* Pure formatting; no ANSI, so it is safe to log anywhere. */
|
|
171
|
+
export declare function formatReport(r: CostReport): string;
|