@hviana/sema 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/train_base.js +42 -9
- package/dist/src/alu/src/parser.d.ts +9 -0
- package/dist/src/alu/src/parser.js +110 -2
- package/dist/src/canon.d.ts +26 -0
- package/dist/src/canon.js +57 -0
- package/dist/src/geometry.d.ts +13 -2
- package/dist/src/geometry.js +101 -2
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/mind/attention.js +11 -7
- package/dist/src/mind/learning.js +33 -1
- package/dist/src/mind/match.d.ts +4 -2
- package/dist/src/mind/match.js +30 -6
- package/dist/src/mind/mechanisms/cast.js +18 -4
- package/dist/src/mind/mechanisms/confluence.js +13 -1
- package/dist/src/mind/mechanisms/recall.js +115 -31
- package/dist/src/mind/mind.d.ts +138 -12
- package/dist/src/mind/mind.js +284 -22
- package/dist/src/mind/primitives.d.ts +22 -2
- package/dist/src/mind/primitives.js +95 -6
- package/dist/src/mind/recognition.js +31 -8
- package/dist/src/mind/traverse.d.ts +13 -0
- package/dist/src/mind/traverse.js +41 -0
- package/dist/src/mind/types.d.ts +33 -21
- package/dist/src/store-sqlite.d.ts +10 -0
- package/dist/src/store-sqlite.js +58 -0
- package/dist/src/store.d.ts +23 -0
- package/dist/src/store.js +13 -2
- package/example/train_base.ts +49 -9
- package/index.html +65 -0
- package/package.json +1 -1
- package/src/alu/src/parser.ts +105 -4
- package/src/canon.ts +65 -0
- package/src/geometry.ts +105 -1
- package/src/index.ts +1 -0
- package/src/mind/attention.ts +11 -6
- package/src/mind/learning.ts +39 -1
- package/src/mind/match.ts +29 -6
- package/src/mind/mechanisms/cast.ts +21 -6
- package/src/mind/mechanisms/confluence.ts +15 -1
- package/src/mind/mechanisms/recall.ts +132 -41
- package/src/mind/mind.ts +396 -22
- package/src/mind/primitives.ts +109 -7
- package/src/mind/recognition.ts +36 -8
- package/src/mind/traverse.ts +47 -0
- package/src/mind/types.ts +30 -21
- package/src/store-sqlite.ts +76 -0
- package/src/store.ts +46 -2
- package/test/13-conversation.test.mjs +240 -20
- package/test/35-prefix-edge.test.mjs +86 -0
|
@@ -152,8 +152,11 @@ const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir of *.zip
|
|
|
152
152
|
const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
|
|
153
153
|
const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
|
|
154
154
|
const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel refresh cadence
|
|
155
|
-
// Index maintenance at checkpoints: compact (remove garbage)
|
|
156
|
-
// gaps)
|
|
155
|
+
// Index maintenance at checkpoints: compact (remove garbage), repair (fill
|
|
156
|
+
// gaps), then refresh the canonical-form index (equivalence-class resolution —
|
|
157
|
+
// src/canon.ts). All three are idempotent batch operations (the canon build is
|
|
158
|
+
// additionally incremental via the store's `canon.upto` cursor);
|
|
159
|
+
// INDEX_MAINTENANCE=0 disables.
|
|
157
160
|
const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
|
|
158
161
|
const DOWNLOAD_TRIES = 5;
|
|
159
162
|
// In-progress downloads are written to a sibling "<dest>.part" and atomically
|
|
@@ -1485,11 +1488,25 @@ async function main() {
|
|
|
1485
1488
|
// final exit for tidiness. Same lesson as waitMs above (deliberately un-unref'd).
|
|
1486
1489
|
const keepAlive = setInterval(() => { }, 1 << 30);
|
|
1487
1490
|
const checkpoint = () => mind.save();
|
|
1488
|
-
/** Run index maintenance: compact (remove garbage)
|
|
1489
|
-
*
|
|
1490
|
-
*
|
|
1491
|
-
*
|
|
1492
|
-
*
|
|
1491
|
+
/** Run index maintenance: compact (remove garbage), repair (fill gaps),
|
|
1492
|
+
* then refresh the canonical-form index (see below). All three are
|
|
1493
|
+
* idempotent — running twice produces the same result as once.
|
|
1494
|
+
* Compaction frees index space first; repair then adds back every
|
|
1495
|
+
* edge/halo-bearing node whose gist was evicted from the pending cache
|
|
1496
|
+
* before it reached the content index, completing the coverage that
|
|
1497
|
+
* incremental promotion alone cannot guarantee.
|
|
1498
|
+
*
|
|
1499
|
+
* repair runs with minParents = 0, NOT the library default of 2. The
|
|
1500
|
+
* default repairs only structural BRIDGES (≥2 parents), but this
|
|
1501
|
+
* trainer's fact deposits also leave answer-side DEPOSIT ROOTS with 0
|
|
1502
|
+
* structural parents ("The capital of France is Paris." as the dst of a
|
|
1503
|
+
* Q→A edge is a root of its own tree, contained in nothing). Those are
|
|
1504
|
+
* resonance targets recall depends on — a trained store shipped without
|
|
1505
|
+
* them cannot ground statement-shaped queries against its own answers
|
|
1506
|
+
* (observed: 33 such roots missing after a full curriculum, including
|
|
1507
|
+
* high-traffic conversation replies). minParents = 0 admits every
|
|
1508
|
+
* edge/halo bearer; the candidate set is still corpus-of-experiences-
|
|
1509
|
+
* sized, so the pass stays cheap.
|
|
1493
1510
|
*
|
|
1494
1511
|
* Logs the number of entries removed/added so a run that silently degrades
|
|
1495
1512
|
* (growing compaction count, or repair never recovering anything) is
|
|
@@ -1507,14 +1524,30 @@ async function main() {
|
|
|
1507
1524
|
progress.log(` ${YEL}⚠ index compact failed${R}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1508
1525
|
}
|
|
1509
1526
|
try {
|
|
1510
|
-
const added = await mind.repairContentIndex();
|
|
1527
|
+
const added = await mind.repairContentIndex(0);
|
|
1511
1528
|
if (added > 0) {
|
|
1512
|
-
progress.log(` ${GRN}index repair: added ${int(added)} missing
|
|
1529
|
+
progress.log(` ${GRN}index repair: added ${int(added)} missing resonance targets${R}`);
|
|
1513
1530
|
}
|
|
1514
1531
|
}
|
|
1515
1532
|
catch (err) {
|
|
1516
1533
|
progress.log(` ${YEL}⚠ index repair failed${R}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1517
1534
|
}
|
|
1535
|
+
// Canonical-form index (src/canon.ts): lets resolution find stored forms
|
|
1536
|
+
// across surface variation (case, width, whitespace). Incremental and
|
|
1537
|
+
// idempotent by construction — the `canon.upto` meta cursor scans only
|
|
1538
|
+
// nodes newer than the last pass, and the (h, id) primary key ignores
|
|
1539
|
+
// re-inserted rows — so it composes with the resume model exactly like
|
|
1540
|
+
// compact/repair: every checkpoint (and finish) leaves the index
|
|
1541
|
+
// covering all content trained so far.
|
|
1542
|
+
try {
|
|
1543
|
+
const added = await mind.buildCanonIndex();
|
|
1544
|
+
if (added > 0) {
|
|
1545
|
+
progress.log(` ${GRN}canon index: added ${int(added)} canonical-form entries${R}`);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
catch (err) {
|
|
1549
|
+
progress.log(` ${YEL}⚠ canon index build failed${R}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1550
|
+
}
|
|
1518
1551
|
};
|
|
1519
1552
|
// The checkpoint recall is a best-effort diagnostic. It is time-bounded so a
|
|
1520
1553
|
// slow/large store can never freeze the deposit loop, and guarded so a still
|
|
@@ -144,6 +144,15 @@ export declare class QueryParser {
|
|
|
144
144
|
* same judgement the perception tree makes about spacing — so no character
|
|
145
145
|
* is privileged as "the" separator. */
|
|
146
146
|
private arithmeticRuns;
|
|
147
|
+
/** The maximal BRACKETED arithmetic runs: like {@link arithmeticRuns} but
|
|
148
|
+
* grouping brackets participate — an all-'(' term opens depth where an
|
|
149
|
+
* operand is expected, a term beginning with ')' closes it where an
|
|
150
|
+
* operator could follow (only its bracket prefix joins the run; anything
|
|
151
|
+
* after — a trailing "?" — ends it). A run is recorded only when it is
|
|
152
|
+
* BALANCED, actually crossed a bracket, and contains an operator — the
|
|
153
|
+
* bracket-free case is {@link arithmeticRuns}' own. Evaluation is the
|
|
154
|
+
* same registry-derived grammar, whose lexer already reads the brackets. */
|
|
155
|
+
private bracketedRuns;
|
|
147
156
|
/** Evaluate an infix-arithmetic run to its canonical result bytes, through
|
|
148
157
|
* the kernel's recursive expression evaluator, or null if it does not
|
|
149
158
|
* evaluate. A whole result stays an exact int; otherwise the canonical
|
|
@@ -113,8 +113,26 @@ export class QueryParser {
|
|
|
113
113
|
j,
|
|
114
114
|
bytes: this.evalRun(query.subarray(i, j)),
|
|
115
115
|
}));
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
// BRACKETED runs — "(3 + 4) * (10 - 2)" — extend across grouping
|
|
117
|
+
// notation the flat alternation cannot cross. The expression grammar
|
|
118
|
+
// (expr.ts) already nests and parenthesizes; this only hands it the
|
|
119
|
+
// WHOLE bracketed span instead of the fragments between brackets. A
|
|
120
|
+
// bracketed run that evaluates ABSORBS the flat runs inside it (the
|
|
121
|
+
// authority law: contained spans are its material); one that does not
|
|
122
|
+
// evaluate leaves the flat readings untouched.
|
|
123
|
+
const bracketed = this.bracketedRuns(query, tokens)
|
|
124
|
+
.map(([i, j]) => ({ i, j, bytes: this.evalRun(query.subarray(i, j)) }))
|
|
125
|
+
.filter((r) => r.bytes !== null);
|
|
126
|
+
const absorbed = (r) => bracketed.some((b) => b.i <= r.i && r.j <= b.j);
|
|
127
|
+
const composedRuns = [
|
|
128
|
+
...runs.filter((r) => !absorbed(r)),
|
|
129
|
+
...bracketed,
|
|
130
|
+
].sort((a, b) => a.i - b.i);
|
|
131
|
+
const spans = [
|
|
132
|
+
...runs.filter((r) => r.bytes !== null),
|
|
133
|
+
...bracketed,
|
|
134
|
+
];
|
|
135
|
+
let stream = compose(tokens, composedRuns);
|
|
118
136
|
const readings = new Map();
|
|
119
137
|
for (;;) {
|
|
120
138
|
const fired = await this.operations(query, stream, readings);
|
|
@@ -231,6 +249,96 @@ export class QueryParser {
|
|
|
231
249
|
}
|
|
232
250
|
return runs;
|
|
233
251
|
}
|
|
252
|
+
/** The maximal BRACKETED arithmetic runs: like {@link arithmeticRuns} but
|
|
253
|
+
* grouping brackets participate — an all-'(' term opens depth where an
|
|
254
|
+
* operand is expected, a term beginning with ')' closes it where an
|
|
255
|
+
* operator could follow (only its bracket prefix joins the run; anything
|
|
256
|
+
* after — a trailing "?" — ends it). A run is recorded only when it is
|
|
257
|
+
* BALANCED, actually crossed a bracket, and contains an operator — the
|
|
258
|
+
* bracket-free case is {@link arithmeticRuns}' own. Evaluation is the
|
|
259
|
+
* same registry-derived grammar, whose lexer already reads the brackets. */
|
|
260
|
+
bracketedRuns(query, tokens) {
|
|
261
|
+
const OPEN = 0x28, CLOSE = 0x29;
|
|
262
|
+
const bracketPrefix = (t, b) => {
|
|
263
|
+
if (t.kind !== "term")
|
|
264
|
+
return 0;
|
|
265
|
+
let n = 0;
|
|
266
|
+
while (t.i + n < t.j && query[t.i + n] === b)
|
|
267
|
+
n++;
|
|
268
|
+
return n;
|
|
269
|
+
};
|
|
270
|
+
const allOf = (t, b) => bracketPrefix(t, b) === t.j - t.i;
|
|
271
|
+
const bridged = (from, to) => from >= to || this.host.segment(query.subarray(from, to)).length <= 1;
|
|
272
|
+
const runs = [];
|
|
273
|
+
let k = 0;
|
|
274
|
+
while (k < tokens.length) {
|
|
275
|
+
const first = tokens[k];
|
|
276
|
+
if (first.kind !== "operand" && !allOf(first, OPEN)) {
|
|
277
|
+
k++;
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
let depth = 0;
|
|
281
|
+
let ops = 0;
|
|
282
|
+
let brackets = 0;
|
|
283
|
+
let expectOperand = true;
|
|
284
|
+
let end = -1; // byte end of the last VALID run state (balanced, after operand/close)
|
|
285
|
+
let endTok = k; // token index just past the accepted prefix
|
|
286
|
+
let m = k;
|
|
287
|
+
let prevJ = -1;
|
|
288
|
+
while (m < tokens.length) {
|
|
289
|
+
const t = tokens[m];
|
|
290
|
+
if (prevJ >= 0 && !bridged(prevJ, t.i))
|
|
291
|
+
break;
|
|
292
|
+
if (expectOperand) {
|
|
293
|
+
if (allOf(t, OPEN)) {
|
|
294
|
+
depth += t.j - t.i;
|
|
295
|
+
brackets++;
|
|
296
|
+
}
|
|
297
|
+
else if (t.kind === "operand") {
|
|
298
|
+
expectOperand = false;
|
|
299
|
+
if (depth === 0) {
|
|
300
|
+
end = t.j;
|
|
301
|
+
endTok = m + 1;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
else
|
|
305
|
+
break;
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
const c = bracketPrefix(t, CLOSE);
|
|
309
|
+
if (c > 0 && depth > 0) {
|
|
310
|
+
const take = Math.min(c, depth);
|
|
311
|
+
depth -= take;
|
|
312
|
+
brackets++;
|
|
313
|
+
if (depth === 0) {
|
|
314
|
+
end = t.i + take;
|
|
315
|
+
endTok = m + 1;
|
|
316
|
+
}
|
|
317
|
+
// A term with content beyond its usable bracket prefix ("?")
|
|
318
|
+
// ends the run inside the term.
|
|
319
|
+
if (take < t.j - t.i)
|
|
320
|
+
break;
|
|
321
|
+
}
|
|
322
|
+
else if (t.kind === "operator") {
|
|
323
|
+
expectOperand = true;
|
|
324
|
+
ops++;
|
|
325
|
+
}
|
|
326
|
+
else
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
prevJ = t.j;
|
|
330
|
+
m++;
|
|
331
|
+
}
|
|
332
|
+
if (end >= 0 && ops > 0 && brackets > 0) {
|
|
333
|
+
runs.push([first.i, end]);
|
|
334
|
+
k = endTok;
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
k++;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return runs;
|
|
341
|
+
}
|
|
234
342
|
/** Evaluate an infix-arithmetic run to its canonical result bytes, through
|
|
235
343
|
* the kernel's recursive expression evaluator, or null if it does not
|
|
236
344
|
* evaluate. A whole result stays an exact int; otherwise the canonical
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** A content canonicalizer: maps a byte span to the canonical representative
|
|
2
|
+
* of its equivalence class. Must be pure and deterministic. Returning the
|
|
3
|
+
* input unchanged is always sound (the class is then {input}). */
|
|
4
|
+
export type Canon = (bytes: Uint8Array) => Uint8Array;
|
|
5
|
+
/** The TEXT canonicalizer: Unicode-aware equivalence over every character
|
|
6
|
+
* variation that does not change what the text SAYS —
|
|
7
|
+
*
|
|
8
|
+
* • compatibility normalization (NFKC): full-width forms, ligatures,
|
|
9
|
+
* composed vs decomposed accents collapse to one representation;
|
|
10
|
+
* • case folding (locale-independent lowercase after NFKC — the standard
|
|
11
|
+
* simple fold);
|
|
12
|
+
* • whitespace: every INTERIOR run of Unicode whitespace becomes one plain
|
|
13
|
+
* space. EDGE whitespace is preserved verbatim: a span's leading or
|
|
14
|
+
* trailing separator belongs BETWEEN forms, not to the form — trimming
|
|
15
|
+
* it would let a recognised span swallow the boundary byte that
|
|
16
|
+
* separates it from its neighbour (observed: "ice fire" composing to
|
|
17
|
+
* "coldhot" because the span "ice " matched the stored "ice").
|
|
18
|
+
*
|
|
19
|
+
* "WHAT IS", "What is" and "what is" share one canonical form. This is
|
|
20
|
+
* deliberately conservative: punctuation, digits and word order are content
|
|
21
|
+
* and pass through untouched. */
|
|
22
|
+
export declare function textCanon(bytes: Uint8Array): Uint8Array;
|
|
23
|
+
/** 32-bit FNV-1a over a canonical key — the integer the store's canon index
|
|
24
|
+
* is keyed on. Same construction as the node table's content hash; a
|
|
25
|
+
* collision is resolved by verifying canon(stored) === key, never trusted. */
|
|
26
|
+
export declare function canonHash(key: Uint8Array): number;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// canon.ts — content canonicalization for equivalence-class resolution.
|
|
2
|
+
//
|
|
3
|
+
// The store is content-addressed on RAW bytes: "What", "WHAT" and "what" are
|
|
4
|
+
// three different hashes, so a query whose surface form varies from the
|
|
5
|
+
// trained form resolves to nothing even though the CONTENT is the same. A
|
|
6
|
+
// CANONICALIZER maps every surface variant of the same content onto one
|
|
7
|
+
// canonical byte string; the store keeps a small hash index from canonical
|
|
8
|
+
// keys to node ids (see Store.canonAdd/canonFind), and resolution falls back
|
|
9
|
+
// to that index when the exact content-addressed lookup misses.
|
|
10
|
+
//
|
|
11
|
+
// The canonicalizer is MODALITY-SPECIFIC and always INJECTED — nothing in the
|
|
12
|
+
// store or the mind's core knows what "case" or "whitespace" is. The text
|
|
13
|
+
// canonicalizer below is the one `respondText`/`respondTurnText` pass down;
|
|
14
|
+
// a grid or audio modality would supply its own (or none).
|
|
15
|
+
//
|
|
16
|
+
// Canonical keys are equivalence-class LABELS, never content: they are hashed
|
|
17
|
+
// and verified (canon(stored bytes) must equal canon(query bytes) before an
|
|
18
|
+
// id is accepted), so a hash collision costs a read, never a wrong id — the
|
|
19
|
+
// same discipline as the node table's own `h` index.
|
|
20
|
+
const dec = new TextDecoder("utf-8", { fatal: false });
|
|
21
|
+
const enc = new TextEncoder();
|
|
22
|
+
/** The TEXT canonicalizer: Unicode-aware equivalence over every character
|
|
23
|
+
* variation that does not change what the text SAYS —
|
|
24
|
+
*
|
|
25
|
+
* • compatibility normalization (NFKC): full-width forms, ligatures,
|
|
26
|
+
* composed vs decomposed accents collapse to one representation;
|
|
27
|
+
* • case folding (locale-independent lowercase after NFKC — the standard
|
|
28
|
+
* simple fold);
|
|
29
|
+
* • whitespace: every INTERIOR run of Unicode whitespace becomes one plain
|
|
30
|
+
* space. EDGE whitespace is preserved verbatim: a span's leading or
|
|
31
|
+
* trailing separator belongs BETWEEN forms, not to the form — trimming
|
|
32
|
+
* it would let a recognised span swallow the boundary byte that
|
|
33
|
+
* separates it from its neighbour (observed: "ice fire" composing to
|
|
34
|
+
* "coldhot" because the span "ice " matched the stored "ice").
|
|
35
|
+
*
|
|
36
|
+
* "WHAT IS", "What is" and "what is" share one canonical form. This is
|
|
37
|
+
* deliberately conservative: punctuation, digits and word order are content
|
|
38
|
+
* and pass through untouched. */
|
|
39
|
+
export function textCanon(bytes) {
|
|
40
|
+
const s = dec
|
|
41
|
+
.decode(bytes)
|
|
42
|
+
.normalize("NFKC")
|
|
43
|
+
.toLowerCase()
|
|
44
|
+
.replace(/(\S)\s+(?=\S)/g, "$1 ");
|
|
45
|
+
return enc.encode(s);
|
|
46
|
+
}
|
|
47
|
+
/** 32-bit FNV-1a over a canonical key — the integer the store's canon index
|
|
48
|
+
* is keyed on. Same construction as the node table's content hash; a
|
|
49
|
+
* collision is resolved by verifying canon(stored) === key, never trusted. */
|
|
50
|
+
export function canonHash(key) {
|
|
51
|
+
let h = 0x811c9dc5 >>> 0;
|
|
52
|
+
for (let i = 0; i < key.length; i++) {
|
|
53
|
+
h ^= key[i];
|
|
54
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
55
|
+
}
|
|
56
|
+
return h >>> 0;
|
|
57
|
+
}
|
package/dist/src/geometry.d.ts
CHANGED
|
@@ -103,8 +103,19 @@ export declare function knownPrefixLength(bytes: Uint8Array, leafAt: (i: number)
|
|
|
103
103
|
/** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
|
|
104
104
|
* detecting previously-stored prefixes so the river can split at the
|
|
105
105
|
* correct boundary. Pass them through from `perceive`; the geometry
|
|
106
|
-
* computes the stable prefix internally.
|
|
107
|
-
|
|
106
|
+
* computes the stable prefix internally.
|
|
107
|
+
*
|
|
108
|
+
* `boundaries` is the CALLER-computed stable-prefix boundary set (§10.3):
|
|
109
|
+
* strictly-increasing proper byte offsets, each the length of a prefix that
|
|
110
|
+
* is already a stored whole-stream form. When given, the fold splits into
|
|
111
|
+
* the segments between consecutive boundaries — each folded independently,
|
|
112
|
+
* exactly as it folded when it was learned — and the segment roots join
|
|
113
|
+
* LEFT-NESTED (((s₀·s₁)·s₂)…), so every learnt cumulative-context root
|
|
114
|
+
* reappears as an identical subtree (and, by hash-consing, the very same
|
|
115
|
+
* node) inside the grown stream. This is what lets a conversation's next
|
|
116
|
+
* turn extend perception instead of refolding it: identical prefixes
|
|
117
|
+
* produce identical subtrees regardless of what follows them. */
|
|
118
|
+
export declare function bytesToTree(space: Space, alphabet: Alphabet, bytes: Uint8Array, leafAt?: (i: number) => number | null, lookup?: (leafIds: number[]) => number | null, boundaries?: readonly number[]): Sema;
|
|
108
119
|
/** The PLAIN fold's full level pyramid — every level's item list, bottom
|
|
109
120
|
* (leaves) to top (root). Left-grouped folding is RADIX-ALIGNED: the item
|
|
110
121
|
* at level L, index i, covers exactly bytes [i·mg^L, (i+1)·mg^L) whenever
|
package/dist/src/geometry.js
CHANGED
|
@@ -270,14 +270,101 @@ export function knownPrefixLength(bytes, leafAt, lookup) {
|
|
|
270
270
|
/** Bytes → Sema tree. `leafAt` and `lookup` are store capabilities for
|
|
271
271
|
* detecting previously-stored prefixes so the river can split at the
|
|
272
272
|
* correct boundary. Pass them through from `perceive`; the geometry
|
|
273
|
-
* computes the stable prefix internally.
|
|
274
|
-
|
|
273
|
+
* computes the stable prefix internally.
|
|
274
|
+
*
|
|
275
|
+
* `boundaries` is the CALLER-computed stable-prefix boundary set (§10.3):
|
|
276
|
+
* strictly-increasing proper byte offsets, each the length of a prefix that
|
|
277
|
+
* is already a stored whole-stream form. When given, the fold splits into
|
|
278
|
+
* the segments between consecutive boundaries — each folded independently,
|
|
279
|
+
* exactly as it folded when it was learned — and the segment roots join
|
|
280
|
+
* LEFT-NESTED (((s₀·s₁)·s₂)…), so every learnt cumulative-context root
|
|
281
|
+
* reappears as an identical subtree (and, by hash-consing, the very same
|
|
282
|
+
* node) inside the grown stream. This is what lets a conversation's next
|
|
283
|
+
* turn extend perception instead of refolding it: identical prefixes
|
|
284
|
+
* produce identical subtrees regardless of what follows them. */
|
|
285
|
+
export function bytesToTree(space, alphabet, bytes, leafAt, lookup, boundaries) {
|
|
275
286
|
if (bytes.length === 0) {
|
|
276
287
|
return sema(alphabet.vecs[0], new Uint8Array(0), null);
|
|
277
288
|
}
|
|
289
|
+
if (boundaries !== undefined && boundaries.length > 0) {
|
|
290
|
+
return stablePrefixFold(space, alphabet, bytes, boundaries);
|
|
291
|
+
}
|
|
278
292
|
const sb = (leafAt && lookup) ? knownPrefixLength(bytes, leafAt, lookup) : 0;
|
|
279
293
|
return riverFold(space, bytesToLeaves(alphabet, bytes), sb > 0 ? sb : bytes.length).tree;
|
|
280
294
|
}
|
|
295
|
+
/** The stable-prefix segmented fold (§10.3). Each segment between
|
|
296
|
+
* consecutive boundaries folds PLAINLY and independently; segment roots
|
|
297
|
+
* join left-nested, and only the final root is normalized (the linear-fold
|
|
298
|
+
* contract: one normalize per perception). A segment's own inner splits
|
|
299
|
+
* need no recursion here: a nested learnt prefix is itself an earlier
|
|
300
|
+
* boundary, so the left-nested join reproduces every intermediate learnt
|
|
301
|
+
* root ((s₀·s₁) IS the root the store learnt for the first two segments'
|
|
302
|
+
* bytes, and so on). */
|
|
303
|
+
function stablePrefixFold(space, alphabet, bytes, boundaries) {
|
|
304
|
+
const cuts = [];
|
|
305
|
+
let prev = 0;
|
|
306
|
+
for (const b of boundaries) {
|
|
307
|
+
if (b > prev && b < bytes.length) {
|
|
308
|
+
cuts.push(b);
|
|
309
|
+
prev = b;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
if (cuts.length === 0) {
|
|
313
|
+
return riverFold(space, bytesToLeaves(alphabet, bytes), bytes.length).tree;
|
|
314
|
+
}
|
|
315
|
+
const edges = [0, ...cuts, bytes.length];
|
|
316
|
+
const segs = [];
|
|
317
|
+
for (let i = 0; i + 1 < edges.length; i++) {
|
|
318
|
+
const seg = bytes.subarray(edges[i], edges[i + 1]);
|
|
319
|
+
segs.push(riverFoldRaw(space, bytesToLeaves(alphabet, seg)));
|
|
320
|
+
}
|
|
321
|
+
let cur = segs[0];
|
|
322
|
+
for (let i = 1; i < segs.length; i++)
|
|
323
|
+
cur = fold2(space, cur, segs[i]);
|
|
324
|
+
normalize(cur.tree.v);
|
|
325
|
+
return cur.tree;
|
|
326
|
+
}
|
|
327
|
+
/** Join two folded items as one 2-kid branch — the top-level join of the
|
|
328
|
+
* stable-prefix fold, identical FP ops to foldSlice's seat-bound
|
|
329
|
+
* accumulation over a group of two. Unnormalized (interior). */
|
|
330
|
+
function fold2(space, a, b) {
|
|
331
|
+
const D = space.D;
|
|
332
|
+
const gist = new Float32Array(D);
|
|
333
|
+
const kids = [a.tree, b.tree];
|
|
334
|
+
for (let k = 0; k < 2; k++) {
|
|
335
|
+
const seat = space.seats[k].fwd;
|
|
336
|
+
const v = kids[k].v;
|
|
337
|
+
for (let d = 0; d < D; d++)
|
|
338
|
+
gist[d] += v[seat[d]];
|
|
339
|
+
}
|
|
340
|
+
return { tree: sema(gist, null, kids), len: a.len + b.len };
|
|
341
|
+
}
|
|
342
|
+
/** Plain river fold WITHOUT the final root normalize — the segment-level
|
|
343
|
+
* building block of {@link stablePrefixFold} (interiors must keep their
|
|
344
|
+
* byte-proportional magnitude; only the whole perception's root is ever
|
|
345
|
+
* normalized). */
|
|
346
|
+
function riverFoldRaw(space, row) {
|
|
347
|
+
if (row.length === 0) {
|
|
348
|
+
const z = new Float32Array(space.D);
|
|
349
|
+
return { tree: sema(z, new Uint8Array(0), null), len: 0 };
|
|
350
|
+
}
|
|
351
|
+
if (row.length === 1)
|
|
352
|
+
return row[0];
|
|
353
|
+
let level = row;
|
|
354
|
+
while (level.length > 1) {
|
|
355
|
+
const next = [];
|
|
356
|
+
foldSlice(space, level, 0, level.length, next, false);
|
|
357
|
+
if (next.length === level.length) {
|
|
358
|
+
const forced = [];
|
|
359
|
+
foldSlice(space, next, 0, next.length, forced, true);
|
|
360
|
+
level = forced;
|
|
361
|
+
}
|
|
362
|
+
else {
|
|
363
|
+
level = next;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
return level[0];
|
|
367
|
+
}
|
|
281
368
|
/** Plain bytes→tree (identical to capability-less {@link bytesToTree}) that
|
|
282
369
|
* also RETURNS its pyramid, reusing `prev` — the pyramid of a PROPER
|
|
283
370
|
* prefix of `bytes` (caller guarantees content match and
|
|
@@ -289,6 +376,18 @@ export function bytesToTreePyramid(space, alphabet, bytes, prev) {
|
|
|
289
376
|
pyramid: { levels: [], bytes: 0 },
|
|
290
377
|
};
|
|
291
378
|
}
|
|
379
|
+
// ZERO GROWTH: the previous pyramid already covers every byte — its top
|
|
380
|
+
// level holds the finished, normalized root. Refolding here would not
|
|
381
|
+
// only waste the work: when the whole span is one full block, the refold's
|
|
382
|
+
// top item is a REUSED raw interior of `prev`, and the final normalize
|
|
383
|
+
// below would mutate that shared raw block in place, corrupting every
|
|
384
|
+
// later incremental fold built on `prev`.
|
|
385
|
+
if (prev && prev.bytes === bytes.length && prev.levels.length > 0) {
|
|
386
|
+
return {
|
|
387
|
+
tree: prev.levels[prev.levels.length - 1][0].tree,
|
|
388
|
+
pyramid: prev,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
292
391
|
const mg = space.maxGroup;
|
|
293
392
|
const reusable = (L) => {
|
|
294
393
|
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
package/dist/src/index.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ export * from "./mind/index.js";
|
|
|
9
9
|
export * from "./store-sqlite.js";
|
|
10
10
|
export * from "./config.js";
|
|
11
11
|
export * from "./extension.js";
|
|
12
|
+
export * from "./canon.js";
|
|
12
13
|
export * from "./ingest-cache.js";
|
|
13
14
|
export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
|
|
14
15
|
export type { DatabaseOptions, ExternalId, IvfConfig, IvfHit, QueryContext, QueryResult, RaBitQOptions, StorageStats, } from "./rabitq-ivf/src/index.js";
|
package/dist/src/index.js
CHANGED
|
@@ -11,6 +11,7 @@ export * from "./mind/index.js";
|
|
|
11
11
|
export * from "./store-sqlite.js";
|
|
12
12
|
export * from "./config.js";
|
|
13
13
|
export * from "./extension.js";
|
|
14
|
+
export * from "./canon.js";
|
|
14
15
|
export * from "./ingest-cache.js";
|
|
15
16
|
// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
|
|
16
17
|
export { IvfIndex, Prng, RaBitQuantizer, VectorDatabase, } from "./rabitq-ivf/src/index.js";
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { isChunk } from "../sema.js";
|
|
10
10
|
import { lightestDerivation, } from "../derive/src/index.js";
|
|
11
11
|
import { consensusFloor, dominates, estimatorNoise } from "../geometry.js";
|
|
12
|
-
import { foldTree, gistOf, perceive, read } from "./primitives.js";
|
|
12
|
+
import { foldTree, gistOf, latin1Key, perceive, read } from "./primitives.js";
|
|
13
13
|
import { recognise } from "./recognition.js";
|
|
14
14
|
import { leafIdRun } from "./canonical.js";
|
|
15
15
|
import { corpusN, edgeAncestors } from "./traverse.js";
|
|
@@ -27,16 +27,20 @@ export async function climbAttention(ctx, query, k, mode = "inverse") {
|
|
|
27
27
|
* attention) and the entire ranked list. Cached via ctx.climbMemo when
|
|
28
28
|
* ctx.trace is null. */
|
|
29
29
|
export async function climbAttentionAll(ctx, query, k, mode = "inverse") {
|
|
30
|
+
// Content-keyed memo — works for both single-turn respond() and multi-turn
|
|
31
|
+
// respondTurn(). Skipped while tracing.
|
|
30
32
|
if (ctx.climbMemo && !ctx.trace) {
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
const contentKey = latin1Key(query);
|
|
34
|
+
const modeKey = `${k}:${mode}`;
|
|
35
|
+
let byRead = ctx.climbMemo.get(contentKey);
|
|
36
|
+
if (byRead === undefined) {
|
|
37
|
+
ctx.climbMemo.set(contentKey, byRead = new Map());
|
|
38
|
+
}
|
|
39
|
+
const hit = byRead.get(modeKey);
|
|
36
40
|
if (hit !== undefined)
|
|
37
41
|
return hit;
|
|
38
42
|
const read = await computeAttention(ctx, query, k, mode);
|
|
39
|
-
byRead.set(
|
|
43
|
+
byRead.set(modeKey, read);
|
|
40
44
|
return read;
|
|
41
45
|
}
|
|
42
46
|
return computeAttention(ctx, query, k, mode);
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// node. A fact is an EDGE between node ids; recall traverses edges.
|
|
5
5
|
import { bindSeat, companySignature, isChunk } from "../sema.js";
|
|
6
6
|
import { changedNodes } from "./types.js";
|
|
7
|
-
import { inputBytes, perceiveDeposit } from "./primitives.js";
|
|
7
|
+
import { inputBytes, perceiveDeposit, resolve, } from "./primitives.js";
|
|
8
8
|
import { canonicalWindows, leafIdPrefix } from "./canonical.js";
|
|
9
9
|
import { fold as foldVecs } from "../sema.js";
|
|
10
10
|
/** Intern a perceived tree into node ids, bottom-up, sharing equal subtrees.
|
|
@@ -152,12 +152,44 @@ export async function ingestOne(ctx, input) {
|
|
|
152
152
|
}
|
|
153
153
|
return Object.assign(tree, { id: rootId });
|
|
154
154
|
}
|
|
155
|
+
/** For each right-edge suffix of the context bytes, resolve it against the
|
|
156
|
+
* store. A suffix whose resolved node is already a known form inherits the
|
|
157
|
+
* continuation edge. Gate: ≥ 2 structural parents (reused across deposits),
|
|
158
|
+
* or (halo > 0 ∧ already an edge source). Pure answers do not qualify. */
|
|
159
|
+
async function propagateSuffixes(ctx, src, dst) {
|
|
160
|
+
const W = ctx.space.maxGroup;
|
|
161
|
+
const bytes = ctx.store.bytes(src);
|
|
162
|
+
const n = bytes.length;
|
|
163
|
+
if (n < 2 * W)
|
|
164
|
+
return;
|
|
165
|
+
// Existence prefilter — the write side of the canonical contract: every
|
|
166
|
+
// deposit interns its WHOLE byte stream as a flat branch of per-byte leaf
|
|
167
|
+
// ids (deposit(), canonical.ts). A suffix is a stored form exactly when
|
|
168
|
+
// that flat twin exists, so one content-hash probe per offset decides;
|
|
169
|
+
// only a hit pays for resolve()'s deposit-shaped perception. This keeps
|
|
170
|
+
// the scan free of river folds — O(1) probes over cheap byte hashes
|
|
171
|
+
// instead of O(suffix) vector folds per offset.
|
|
172
|
+
const leafIds = leafIdPrefix(ctx, bytes);
|
|
173
|
+
for (let i = 1; i <= n - W; i++) {
|
|
174
|
+
if (ctx.store.findBranch(leafIds.slice(i)) === null)
|
|
175
|
+
continue;
|
|
176
|
+
const id = resolve(ctx, bytes.subarray(i));
|
|
177
|
+
if (id === null || id === src)
|
|
178
|
+
continue;
|
|
179
|
+
const known = ctx.store.parentsFirst(id, 2).length >= 2 ||
|
|
180
|
+
(ctx.store.haloMass(id) > 0 && ctx.store.hasNext(id));
|
|
181
|
+
if (!known)
|
|
182
|
+
continue;
|
|
183
|
+
await ctx.store.link(id, dst);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
155
186
|
/** Ingest a pair (context, continuation) — learn an edge and pour halos. */
|
|
156
187
|
export async function ingestPair(ctx, ctxInput, cont) {
|
|
157
188
|
const c = await deposit(ctx, ctxInput, true);
|
|
158
189
|
const cont_ = await deposit(ctx, cont, false);
|
|
159
190
|
const ctxId = c.rootId, contId = cont_.rootId;
|
|
160
191
|
await ctx.store.link(ctxId, contId);
|
|
192
|
+
await propagateSuffixes(ctx, ctxId, contId);
|
|
161
193
|
// Halos pour company SIGNATURES (identity), not gists (content) — see
|
|
162
194
|
// companySignature in sema.ts.
|
|
163
195
|
const contSeat = bindSeat(ctx.space, companySignature(ctx.space, contId), 1);
|
package/dist/src/mind/match.d.ts
CHANGED
|
@@ -94,8 +94,10 @@ export declare function follow(ctx: MindContext, id: number, guide?: Vec | null)
|
|
|
94
94
|
* `guide` the context whose gist resonates with the query wins (seat
|
|
95
95
|
* symmetry) — without one, the most-corroborated context wins (poured halo
|
|
96
96
|
* MASS, the direct measure of how many episodes established it), falling
|
|
97
|
-
* back to first-learnt on equal mass.
|
|
98
|
-
*
|
|
97
|
+
* back to first-learnt on equal mass. Among many predecessors RECIPROCAL
|
|
98
|
+
* ones (mutual edges) are preferred when any exist (RC5). Callers that
|
|
99
|
+
* HAVE a query gist must pass it, or they silently change disambiguation
|
|
100
|
+
* regime.
|
|
99
101
|
*
|
|
100
102
|
* `rev`, when the caller has already materialised prevOf (one read per
|
|
101
103
|
* relation — a hub's reverse fan-in is corpus-sized), is reused instead of
|
package/dist/src/mind/match.js
CHANGED
|
@@ -370,8 +370,10 @@ export async function follow(ctx, id, guide) {
|
|
|
370
370
|
* `guide` the context whose gist resonates with the query wins (seat
|
|
371
371
|
* symmetry) — without one, the most-corroborated context wins (poured halo
|
|
372
372
|
* MASS, the direct measure of how many episodes established it), falling
|
|
373
|
-
* back to first-learnt on equal mass.
|
|
374
|
-
*
|
|
373
|
+
* back to first-learnt on equal mass. Among many predecessors RECIPROCAL
|
|
374
|
+
* ones (mutual edges) are preferred when any exist (RC5). Callers that
|
|
375
|
+
* HAVE a query gist must pass it, or they silently change disambiguation
|
|
376
|
+
* regime.
|
|
375
377
|
*
|
|
376
378
|
* `rev`, when the caller has already materialised prevOf (one read per
|
|
377
379
|
* relation — a hub's reverse fan-in is corpus-sized), is reused instead of
|
|
@@ -386,11 +388,33 @@ export function reverseContext(ctx, id, guide, rev) {
|
|
|
386
388
|
const candidates = rev ?? ctx.store.prevFirst(id, hubBound(ctx));
|
|
387
389
|
if (candidates.length === 0)
|
|
388
390
|
return null;
|
|
389
|
-
|
|
390
|
-
|
|
391
|
+
// RECIPROCAL PREFERENCE: among many predecessors, one that `id` also
|
|
392
|
+
// continues TO (cand → id AND id → cand both learnt) is a mutually
|
|
393
|
+
// established pairing — the strongest structural evidence a predecessor
|
|
394
|
+
// can carry (bidirectional training deposits both directions of a genuine
|
|
395
|
+
// pair). A bare predecessor is one episode's adjacency; guide-resonance
|
|
396
|
+
// over bare predecessors favours whichever stored document merely
|
|
397
|
+
// CONTAINS the query's bytes (the linear fold's cosine is byte overlap —
|
|
398
|
+
// the observed "merci → unrelated French document" failure). One capped
|
|
399
|
+
// forward read decides; when no reciprocal exists, behaviour is unchanged
|
|
400
|
+
// — bare predecessors ARE the honest answer for a shared deposited
|
|
401
|
+
// continuation (two questions → one answer; audited by 31-audit C1), and
|
|
402
|
+
// this arm serves every mechanism's reverse projection, so abstaining
|
|
403
|
+
// here starves far more than the one containment failure it would fix.
|
|
404
|
+
let pool = candidates;
|
|
405
|
+
if (candidates.length > 1) {
|
|
406
|
+
const fwd = new Set(ctx.store.nextFirst(id, hubBound(ctx)));
|
|
407
|
+
if (fwd.size > 0) {
|
|
408
|
+
const mutual = candidates.filter((c) => fwd.has(c));
|
|
409
|
+
if (mutual.length > 0)
|
|
410
|
+
pool = mutual;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
const pick = pool.length === 1
|
|
414
|
+
? pool[0]
|
|
391
415
|
: guide
|
|
392
|
-
? chooseAmong(ctx,
|
|
393
|
-
: pickByMass(ctx,
|
|
416
|
+
? chooseAmong(ctx, pool, guide).id
|
|
417
|
+
: pickByMass(ctx, pool);
|
|
394
418
|
const g = read(ctx, pick);
|
|
395
419
|
return g.length > 0 ? g : null;
|
|
396
420
|
}
|