@hviana/sema 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example/train_base.ts +29 -4
- package/package.json +1 -1
- package/src/alu/src/parser.ts +105 -4
- package/src/canon.ts +65 -0
- package/src/geometry.ts +26 -30
- package/src/index.ts +1 -0
- package/src/mind/learning.ts +39 -1
- package/src/mind/match.ts +29 -6
- package/src/mind/mechanisms/cast.ts +33 -13
- package/src/mind/mechanisms/confluence.ts +15 -1
- package/src/mind/mechanisms/recall.ts +116 -41
- package/src/mind/mind.ts +172 -29
- package/src/mind/primitives.ts +66 -1
- package/src/mind/recognition.ts +10 -0
- package/src/mind/types.ts +10 -0
- package/src/store-sqlite.ts +68 -0
- package/src/store.ts +27 -0
- package/test/13-conversation.test.mjs +77 -27
- package/test/35-prefix-edge.test.mjs +86 -0
- package/dist/example/demo.d.ts +0 -1
- package/dist/example/demo.js +0 -39
- package/dist/example/train_base.d.ts +0 -87
- package/dist/example/train_base.js +0 -2232
- package/dist/src/alphabet.d.ts +0 -7
- package/dist/src/alphabet.js +0 -33
- package/dist/src/alu/src/alu.d.ts +0 -185
- package/dist/src/alu/src/alu.js +0 -440
- package/dist/src/alu/src/expr.d.ts +0 -61
- package/dist/src/alu/src/expr.js +0 -318
- package/dist/src/alu/src/index.d.ts +0 -11
- package/dist/src/alu/src/index.js +0 -19
- package/dist/src/alu/src/kernel-arith.d.ts +0 -16
- package/dist/src/alu/src/kernel-arith.js +0 -264
- package/dist/src/alu/src/kernel-bits.d.ts +0 -19
- package/dist/src/alu/src/kernel-bits.js +0 -152
- package/dist/src/alu/src/kernel-logic.d.ts +0 -4
- package/dist/src/alu/src/kernel-logic.js +0 -60
- package/dist/src/alu/src/kernel-nd.d.ts +0 -3
- package/dist/src/alu/src/kernel-nd.js +0 -208
- package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
- package/dist/src/alu/src/kernel-numeric.js +0 -366
- package/dist/src/alu/src/operation.d.ts +0 -168
- package/dist/src/alu/src/operation.js +0 -189
- package/dist/src/alu/src/parser.d.ts +0 -212
- package/dist/src/alu/src/parser.js +0 -469
- package/dist/src/alu/src/resonance.d.ts +0 -55
- package/dist/src/alu/src/resonance.js +0 -126
- package/dist/src/alu/src/text.d.ts +0 -31
- package/dist/src/alu/src/text.js +0 -73
- package/dist/src/alu/src/value.d.ts +0 -109
- package/dist/src/alu/src/value.js +0 -300
- package/dist/src/alu/test/alu.test.d.ts +0 -1
- package/dist/src/alu/test/alu.test.js +0 -764
- package/dist/src/bytes.d.ts +0 -14
- package/dist/src/bytes.js +0 -59
- package/dist/src/config.d.ts +0 -111
- package/dist/src/config.js +0 -91
- package/dist/src/derive/src/deduction.d.ts +0 -125
- package/dist/src/derive/src/deduction.js +0 -155
- package/dist/src/derive/src/index.d.ts +0 -7
- package/dist/src/derive/src/index.js +0 -11
- package/dist/src/derive/src/priority-queue.d.ts +0 -20
- package/dist/src/derive/src/priority-queue.js +0 -73
- package/dist/src/derive/src/rewrite.d.ts +0 -56
- package/dist/src/derive/src/rewrite.js +0 -100
- package/dist/src/derive/src/trie.d.ts +0 -90
- package/dist/src/derive/src/trie.js +0 -217
- package/dist/src/derive/test/derive.test.d.ts +0 -1
- package/dist/src/derive/test/derive.test.js +0 -122
- package/dist/src/extension.d.ts +0 -37
- package/dist/src/extension.js +0 -7
- package/dist/src/geometry.d.ts +0 -148
- package/dist/src/geometry.js +0 -517
- package/dist/src/index.d.ts +0 -15
- package/dist/src/index.js +0 -17
- package/dist/src/ingest-cache.d.ts +0 -41
- package/dist/src/ingest-cache.js +0 -161
- package/dist/src/mind/articulation.d.ts +0 -6
- package/dist/src/mind/articulation.js +0 -99
- package/dist/src/mind/attention.d.ts +0 -72
- package/dist/src/mind/attention.js +0 -898
- package/dist/src/mind/canonical.d.ts +0 -29
- package/dist/src/mind/canonical.js +0 -88
- package/dist/src/mind/graph-search.d.ts +0 -270
- package/dist/src/mind/graph-search.js +0 -847
- package/dist/src/mind/index.d.ts +0 -5
- package/dist/src/mind/index.js +0 -5
- package/dist/src/mind/junction.d.ts +0 -95
- package/dist/src/mind/junction.js +0 -262
- package/dist/src/mind/learning.d.ts +0 -47
- package/dist/src/mind/learning.js +0 -201
- package/dist/src/mind/match.d.ts +0 -111
- package/dist/src/mind/match.js +0 -422
- package/dist/src/mind/mechanisms/alu.d.ts +0 -4
- package/dist/src/mind/mechanisms/alu.js +0 -29
- package/dist/src/mind/mechanisms/cast.d.ts +0 -35
- package/dist/src/mind/mechanisms/cast.js +0 -447
- package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
- package/dist/src/mind/mechanisms/confluence.js +0 -213
- package/dist/src/mind/mechanisms/cover.d.ts +0 -6
- package/dist/src/mind/mechanisms/cover.js +0 -179
- package/dist/src/mind/mechanisms/extraction.d.ts +0 -67
- package/dist/src/mind/mechanisms/extraction.js +0 -342
- package/dist/src/mind/mechanisms/recall.d.ts +0 -13
- package/dist/src/mind/mechanisms/recall.js +0 -166
- package/dist/src/mind/mind.d.ts +0 -211
- package/dist/src/mind/mind.js +0 -433
- package/dist/src/mind/pipeline-mechanism.d.ts +0 -142
- package/dist/src/mind/pipeline-mechanism.js +0 -213
- package/dist/src/mind/pipeline.d.ts +0 -20
- package/dist/src/mind/pipeline.js +0 -185
- package/dist/src/mind/primitives.d.ts +0 -54
- package/dist/src/mind/primitives.js +0 -184
- package/dist/src/mind/rationale.d.ts +0 -134
- package/dist/src/mind/rationale.js +0 -162
- package/dist/src/mind/reasoning.d.ts +0 -15
- package/dist/src/mind/reasoning.js +0 -162
- package/dist/src/mind/recognition.d.ts +0 -20
- package/dist/src/mind/recognition.js +0 -236
- package/dist/src/mind/resonance.d.ts +0 -23
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +0 -15
- package/dist/src/mind/trace.js +0 -73
- package/dist/src/mind/traverse.d.ts +0 -113
- package/dist/src/mind/traverse.js +0 -488
- package/dist/src/mind/types.d.ts +0 -176
- package/dist/src/mind/types.js +0 -84
- package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
- package/dist/src/rabitq-ivf/src/database.js +0 -201
- package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
- package/dist/src/rabitq-ivf/src/index.js +0 -4
- package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
- package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
- package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
- package/dist/src/rabitq-ivf/src/prng.js +0 -36
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-ivf/src/rabitq.js +0 -283
- package/dist/src/sema.d.ts +0 -31
- package/dist/src/sema.js +0 -63
- package/dist/src/store-sqlite.d.ts +0 -176
- package/dist/src/store-sqlite.js +0 -890
- package/dist/src/store.d.ts +0 -642
- package/dist/src/store.js +0 -1612
- package/dist/src/vec.d.ts +0 -31
- package/dist/src/vec.js +0 -109
package/example/train_base.ts
CHANGED
|
@@ -177,8 +177,11 @@ const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir of *.zip
|
|
|
177
177
|
const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
|
|
178
178
|
const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
|
|
179
179
|
const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel refresh cadence
|
|
180
|
-
// Index maintenance at checkpoints: compact (remove garbage)
|
|
181
|
-
// gaps)
|
|
180
|
+
// Index maintenance at checkpoints: compact (remove garbage), repair (fill
|
|
181
|
+
// gaps), then refresh the canonical-form index (equivalence-class resolution —
|
|
182
|
+
// src/canon.ts). All three are idempotent batch operations (the canon build is
|
|
183
|
+
// additionally incremental via the store's `canon.upto` cursor);
|
|
184
|
+
// INDEX_MAINTENANCE=0 disables.
|
|
182
185
|
const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
|
|
183
186
|
const DOWNLOAD_TRIES = 5;
|
|
184
187
|
// In-progress downloads are written to a sibling "<dest>.part" and atomically
|
|
@@ -1823,8 +1826,9 @@ async function main(): Promise<void> {
|
|
|
1823
1826
|
|
|
1824
1827
|
const checkpoint = () => mind.save();
|
|
1825
1828
|
|
|
1826
|
-
/** Run index maintenance: compact (remove garbage)
|
|
1827
|
-
*
|
|
1829
|
+
/** Run index maintenance: compact (remove garbage), repair (fill gaps),
|
|
1830
|
+
* then refresh the canonical-form index (see below). All three are
|
|
1831
|
+
* idempotent — running twice produces the same result as once.
|
|
1828
1832
|
* Compaction frees index space first; repair then adds back every
|
|
1829
1833
|
* edge/halo-bearing node whose gist was evicted from the pending cache
|
|
1830
1834
|
* before it reached the content index, completing the coverage that
|
|
@@ -1877,6 +1881,27 @@ async function main(): Promise<void> {
|
|
|
1877
1881
|
}`,
|
|
1878
1882
|
);
|
|
1879
1883
|
}
|
|
1884
|
+
// Canonical-form index (src/canon.ts): lets resolution find stored forms
|
|
1885
|
+
// across surface variation (case, width, whitespace). Incremental and
|
|
1886
|
+
// idempotent by construction — the `canon.upto` meta cursor scans only
|
|
1887
|
+
// nodes newer than the last pass, and the (h, id) primary key ignores
|
|
1888
|
+
// re-inserted rows — so it composes with the resume model exactly like
|
|
1889
|
+
// compact/repair: every checkpoint (and finish) leaves the index
|
|
1890
|
+
// covering all content trained so far.
|
|
1891
|
+
try {
|
|
1892
|
+
const added = await mind.buildCanonIndex();
|
|
1893
|
+
if (added > 0) {
|
|
1894
|
+
progress.log(
|
|
1895
|
+
` ${GRN}canon index: added ${int(added)} canonical-form entries${R}`,
|
|
1896
|
+
);
|
|
1897
|
+
}
|
|
1898
|
+
} catch (err) {
|
|
1899
|
+
progress.log(
|
|
1900
|
+
` ${YEL}⚠ canon index build failed${R}: ${
|
|
1901
|
+
err instanceof Error ? err.message : String(err)
|
|
1902
|
+
}`,
|
|
1903
|
+
);
|
|
1904
|
+
}
|
|
1880
1905
|
};
|
|
1881
1906
|
|
|
1882
1907
|
// The checkpoint recall is a best-effort diagnostic. It is time-bounded so a
|
package/package.json
CHANGED
package/src/alu/src/parser.ts
CHANGED
|
@@ -172,10 +172,27 @@ export class QueryParser {
|
|
|
172
172
|
j,
|
|
173
173
|
bytes: this.evalRun(query.subarray(i, j)),
|
|
174
174
|
}));
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
);
|
|
178
|
-
|
|
175
|
+
// BRACKETED runs — "(3 + 4) * (10 - 2)" — extend across grouping
|
|
176
|
+
// notation the flat alternation cannot cross. The expression grammar
|
|
177
|
+
// (expr.ts) already nests and parenthesizes; this only hands it the
|
|
178
|
+
// WHOLE bracketed span instead of the fragments between brackets. A
|
|
179
|
+
// bracketed run that evaluates ABSORBS the flat runs inside it (the
|
|
180
|
+
// authority law: contained spans are its material); one that does not
|
|
181
|
+
// evaluate leaves the flat readings untouched.
|
|
182
|
+
const bracketed = this.bracketedRuns(query, tokens)
|
|
183
|
+
.map(([i, j]) => ({ i, j, bytes: this.evalRun(query.subarray(i, j)) }))
|
|
184
|
+
.filter((r): r is ComputedSpan => r.bytes !== null);
|
|
185
|
+
const absorbed = (r: { i: number; j: number }) =>
|
|
186
|
+
bracketed.some((b) => b.i <= r.i && r.j <= b.j);
|
|
187
|
+
const composedRuns = [
|
|
188
|
+
...runs.filter((r) => !absorbed(r)),
|
|
189
|
+
...bracketed,
|
|
190
|
+
].sort((a, b) => a.i - b.i);
|
|
191
|
+
const spans: ComputedSpan[] = [
|
|
192
|
+
...runs.filter((r): r is ComputedSpan => r.bytes !== null),
|
|
193
|
+
...bracketed,
|
|
194
|
+
];
|
|
195
|
+
let stream = compose(tokens, composedRuns);
|
|
179
196
|
const readings = new Map<Token, readonly string[]>();
|
|
180
197
|
for (;;) {
|
|
181
198
|
const fired = await this.operations(query, stream, readings);
|
|
@@ -302,6 +319,90 @@ export class QueryParser {
|
|
|
302
319
|
return runs;
|
|
303
320
|
}
|
|
304
321
|
|
|
322
|
+
/** The maximal BRACKETED arithmetic runs: like {@link arithmeticRuns} but
|
|
323
|
+
* grouping brackets participate — an all-'(' term opens depth where an
|
|
324
|
+
* operand is expected, a term beginning with ')' closes it where an
|
|
325
|
+
* operator could follow (only its bracket prefix joins the run; anything
|
|
326
|
+
* after — a trailing "?" — ends it). A run is recorded only when it is
|
|
327
|
+
* BALANCED, actually crossed a bracket, and contains an operator — the
|
|
328
|
+
* bracket-free case is {@link arithmeticRuns}' own. Evaluation is the
|
|
329
|
+
* same registry-derived grammar, whose lexer already reads the brackets. */
|
|
330
|
+
private bracketedRuns(
|
|
331
|
+
query: Uint8Array,
|
|
332
|
+
tokens: Token[],
|
|
333
|
+
): Array<[number, number]> {
|
|
334
|
+
const OPEN = 0x28, CLOSE = 0x29;
|
|
335
|
+
const bracketPrefix = (t: Token, b: number): number => {
|
|
336
|
+
if (t.kind !== "term") return 0;
|
|
337
|
+
let n = 0;
|
|
338
|
+
while (t.i + n < t.j && query[t.i + n] === b) n++;
|
|
339
|
+
return n;
|
|
340
|
+
};
|
|
341
|
+
const allOf = (t: Token, b: number): boolean =>
|
|
342
|
+
bracketPrefix(t, b) === t.j - t.i;
|
|
343
|
+
const bridged = (from: number, to: number): boolean =>
|
|
344
|
+
from >= to || this.host.segment(query.subarray(from, to)).length <= 1;
|
|
345
|
+
const runs: Array<[number, number]> = [];
|
|
346
|
+
let k = 0;
|
|
347
|
+
while (k < tokens.length) {
|
|
348
|
+
const first = tokens[k];
|
|
349
|
+
if (first.kind !== "operand" && !allOf(first, OPEN)) {
|
|
350
|
+
k++;
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
let depth = 0;
|
|
354
|
+
let ops = 0;
|
|
355
|
+
let brackets = 0;
|
|
356
|
+
let expectOperand = true;
|
|
357
|
+
let end = -1; // byte end of the last VALID run state (balanced, after operand/close)
|
|
358
|
+
let endTok = k; // token index just past the accepted prefix
|
|
359
|
+
let m = k;
|
|
360
|
+
let prevJ = -1;
|
|
361
|
+
while (m < tokens.length) {
|
|
362
|
+
const t = tokens[m];
|
|
363
|
+
if (prevJ >= 0 && !bridged(prevJ, t.i)) break;
|
|
364
|
+
if (expectOperand) {
|
|
365
|
+
if (allOf(t, OPEN)) {
|
|
366
|
+
depth += t.j - t.i;
|
|
367
|
+
brackets++;
|
|
368
|
+
} else if (t.kind === "operand") {
|
|
369
|
+
expectOperand = false;
|
|
370
|
+
if (depth === 0) {
|
|
371
|
+
end = t.j;
|
|
372
|
+
endTok = m + 1;
|
|
373
|
+
}
|
|
374
|
+
} else break;
|
|
375
|
+
} else {
|
|
376
|
+
const c = bracketPrefix(t, CLOSE);
|
|
377
|
+
if (c > 0 && depth > 0) {
|
|
378
|
+
const take = Math.min(c, depth);
|
|
379
|
+
depth -= take;
|
|
380
|
+
brackets++;
|
|
381
|
+
if (depth === 0) {
|
|
382
|
+
end = t.i + take;
|
|
383
|
+
endTok = m + 1;
|
|
384
|
+
}
|
|
385
|
+
// A term with content beyond its usable bracket prefix ("?")
|
|
386
|
+
// ends the run inside the term.
|
|
387
|
+
if (take < t.j - t.i) break;
|
|
388
|
+
} else if (t.kind === "operator") {
|
|
389
|
+
expectOperand = true;
|
|
390
|
+
ops++;
|
|
391
|
+
} else break;
|
|
392
|
+
}
|
|
393
|
+
prevJ = t.j;
|
|
394
|
+
m++;
|
|
395
|
+
}
|
|
396
|
+
if (end >= 0 && ops > 0 && brackets > 0) {
|
|
397
|
+
runs.push([first.i, end]);
|
|
398
|
+
k = endTok;
|
|
399
|
+
} else {
|
|
400
|
+
k++;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return runs;
|
|
404
|
+
}
|
|
405
|
+
|
|
305
406
|
/** Evaluate an infix-arithmetic run to its canonical result bytes, through
|
|
306
407
|
* the kernel's recursive expression evaluator, or null if it does not
|
|
307
408
|
* evaluate. A whole result stays an exact int; otherwise the canonical
|
package/src/canon.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// canon.ts — content canonicalization for equivalence-class resolution.
|
|
2
|
+
//
|
|
3
|
+
// The store is content-addressed on RAW bytes: "What", "WHAT" and "what" are
|
|
4
|
+
// three different hashes, so a query whose surface form varies from the
|
|
5
|
+
// trained form resolves to nothing even though the CONTENT is the same. A
|
|
6
|
+
// CANONICALIZER maps every surface variant of the same content onto one
|
|
7
|
+
// canonical byte string; the store keeps a small hash index from canonical
|
|
8
|
+
// keys to node ids (see Store.canonAdd/canonFind), and resolution falls back
|
|
9
|
+
// to that index when the exact content-addressed lookup misses.
|
|
10
|
+
//
|
|
11
|
+
// The canonicalizer is MODALITY-SPECIFIC and always INJECTED — nothing in the
|
|
12
|
+
// store or the mind's core knows what "case" or "whitespace" is. The text
|
|
13
|
+
// canonicalizer below is the one `respondText`/`respondTurnText` pass down;
|
|
14
|
+
// a grid or audio modality would supply its own (or none).
|
|
15
|
+
//
|
|
16
|
+
// Canonical keys are equivalence-class LABELS, never content: they are hashed
|
|
17
|
+
// and verified (canon(stored bytes) must equal canon(query bytes) before an
|
|
18
|
+
// id is accepted), so a hash collision costs a read, never a wrong id — the
|
|
19
|
+
// same discipline as the node table's own `h` index.
|
|
20
|
+
|
|
21
|
+
/** A content canonicalizer: maps a byte span to the canonical representative
|
|
22
|
+
* of its equivalence class. Must be pure and deterministic. Returning the
|
|
23
|
+
* input unchanged is always sound (the class is then {input}). */
|
|
24
|
+
export type Canon = (bytes: Uint8Array) => Uint8Array;
|
|
25
|
+
|
|
26
|
+
const dec = new TextDecoder("utf-8", { fatal: false });
|
|
27
|
+
const enc = new TextEncoder();
|
|
28
|
+
|
|
29
|
+
/** The TEXT canonicalizer: Unicode-aware equivalence over every character
|
|
30
|
+
* variation that does not change what the text SAYS —
|
|
31
|
+
*
|
|
32
|
+
* • compatibility normalization (NFKC): full-width forms, ligatures,
|
|
33
|
+
* composed vs decomposed accents collapse to one representation;
|
|
34
|
+
* • case folding (locale-independent lowercase after NFKC — the standard
|
|
35
|
+
* simple fold);
|
|
36
|
+
* • whitespace: every INTERIOR run of Unicode whitespace becomes one plain
|
|
37
|
+
* space. EDGE whitespace is preserved verbatim: a span's leading or
|
|
38
|
+
* trailing separator belongs BETWEEN forms, not to the form — trimming
|
|
39
|
+
* it would let a recognised span swallow the boundary byte that
|
|
40
|
+
* separates it from its neighbour (observed: "ice fire" composing to
|
|
41
|
+
* "coldhot" because the span "ice " matched the stored "ice").
|
|
42
|
+
*
|
|
43
|
+
* "WHAT IS", "What is" and "what is" share one canonical form. This is
|
|
44
|
+
* deliberately conservative: punctuation, digits and word order are content
|
|
45
|
+
* and pass through untouched. */
|
|
46
|
+
export function textCanon(bytes: Uint8Array): Uint8Array {
|
|
47
|
+
const s = dec
|
|
48
|
+
.decode(bytes)
|
|
49
|
+
.normalize("NFKC")
|
|
50
|
+
.toLowerCase()
|
|
51
|
+
.replace(/(\S)\s+(?=\S)/g, "$1 ");
|
|
52
|
+
return enc.encode(s);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 32-bit FNV-1a over a canonical key — the integer the store's canon index
|
|
56
|
+
* is keyed on. Same construction as the node table's content hash; a
|
|
57
|
+
* collision is resolved by verifying canon(stored) === key, never trusted. */
|
|
58
|
+
export function canonHash(key: Uint8Array): number {
|
|
59
|
+
let h = 0x811c9dc5 >>> 0;
|
|
60
|
+
for (let i = 0; i < key.length; i++) {
|
|
61
|
+
h ^= key[i];
|
|
62
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
63
|
+
}
|
|
64
|
+
return h >>> 0;
|
|
65
|
+
}
|
package/src/geometry.ts
CHANGED
|
@@ -189,8 +189,13 @@ export interface Grid {
|
|
|
189
189
|
/** Fold `items[start .. start+count)` in groups of `mg` into `out`.
|
|
190
190
|
*
|
|
191
191
|
* With `force`, the trailing incomplete group (2..mg-1 items) is folded as
|
|
192
|
-
* well — only a lone singleton passes through
|
|
193
|
-
* the
|
|
192
|
+
* well — only a lone singleton passes through. The river always folds with
|
|
193
|
+
* force: every level contracts by ~mg, so the tree's DEPTH is a function of
|
|
194
|
+
* ceil(log_mg(n)) alone. Letting leftovers pass through unfolded made depth
|
|
195
|
+
* depend on the exact byte count (39 bytes folded in 3 levels, 41 in 4), and
|
|
196
|
+
* each extra level applies another seat permutation to the whole gist —
|
|
197
|
+
* near-identical inputs straddling such a cliff read as orthogonal
|
|
198
|
+
* (measured: 33-byte-identical prefixes at cos ≈ 0). */
|
|
194
199
|
function foldSlice(
|
|
195
200
|
space: Space,
|
|
196
201
|
items: Folded[],
|
|
@@ -251,19 +256,12 @@ function riverFold(space: Space, row: Folded[], stableBytes: number): Folded {
|
|
|
251
256
|
const next: Folded[] = [];
|
|
252
257
|
if (boundary < level.length) {
|
|
253
258
|
// Prefix folds independently of the suffix — structural stability.
|
|
254
|
-
foldSlice(space, level, 0, boundary, next,
|
|
255
|
-
foldSlice(space, level, boundary, level.length - boundary, next,
|
|
259
|
+
foldSlice(space, level, 0, boundary, next, true);
|
|
260
|
+
foldSlice(space, level, boundary, level.length - boundary, next, true);
|
|
256
261
|
} else {
|
|
257
|
-
foldSlice(space, level, 0, level.length, next,
|
|
258
|
-
}
|
|
259
|
-
if (next.length === level.length) {
|
|
260
|
-
// Stuck — every group was incomplete. Force-fold to break the stall.
|
|
261
|
-
const forced: Folded[] = [];
|
|
262
|
-
foldSlice(space, next, 0, next.length, forced, true);
|
|
263
|
-
level = forced;
|
|
264
|
-
} else {
|
|
265
|
-
level = next;
|
|
262
|
+
foldSlice(space, level, 0, level.length, next, true);
|
|
266
263
|
}
|
|
264
|
+
level = next;
|
|
267
265
|
}
|
|
268
266
|
// LINEAR fold — this root normalize is the ONLY normalize of the entire
|
|
269
267
|
// fold; every intermediate gist stays unnormalized (see the folding
|
|
@@ -413,14 +411,8 @@ function riverFoldRaw(space: Space, row: Folded[]): Folded {
|
|
|
413
411
|
let level = row;
|
|
414
412
|
while (level.length > 1) {
|
|
415
413
|
const next: Folded[] = [];
|
|
416
|
-
foldSlice(space, level, 0, level.length, next,
|
|
417
|
-
|
|
418
|
-
const forced: Folded[] = [];
|
|
419
|
-
foldSlice(space, next, 0, next.length, forced, true);
|
|
420
|
-
level = forced;
|
|
421
|
-
} else {
|
|
422
|
-
level = next;
|
|
423
|
-
}
|
|
414
|
+
foldSlice(space, level, 0, level.length, next, true);
|
|
415
|
+
level = next;
|
|
424
416
|
}
|
|
425
417
|
return level[0];
|
|
426
418
|
}
|
|
@@ -460,6 +452,18 @@ export function bytesToTreePyramid(
|
|
|
460
452
|
pyramid: { levels: [], bytes: 0 },
|
|
461
453
|
};
|
|
462
454
|
}
|
|
455
|
+
// ZERO GROWTH: the previous pyramid already covers every byte — its top
|
|
456
|
+
// level holds the finished, normalized root. Refolding here would not
|
|
457
|
+
// only waste the work: when the whole span is one full block, the refold's
|
|
458
|
+
// top item is a REUSED raw interior of `prev`, and the final normalize
|
|
459
|
+
// below would mutate that shared raw block in place, corrupting every
|
|
460
|
+
// later incremental fold built on `prev`.
|
|
461
|
+
if (prev && prev.bytes === bytes.length && prev.levels.length > 0) {
|
|
462
|
+
return {
|
|
463
|
+
tree: prev.levels[prev.levels.length - 1][0].tree,
|
|
464
|
+
pyramid: prev,
|
|
465
|
+
};
|
|
466
|
+
}
|
|
463
467
|
const mg = space.maxGroup;
|
|
464
468
|
const reusable = (L: number): ReadonlyArray<Folded> | null => {
|
|
465
469
|
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
|
@@ -497,16 +501,8 @@ export function bytesToTreePyramid(
|
|
|
497
501
|
reused * mg,
|
|
498
502
|
level.length - reused * mg,
|
|
499
503
|
next,
|
|
500
|
-
|
|
504
|
+
true,
|
|
501
505
|
);
|
|
502
|
-
if (next.length === level.length) {
|
|
503
|
-
// Stuck — same force-fold as riverFold (reuse cannot cause it: any
|
|
504
|
-
// reused block replaces a whole folded group, so reused > 0 implies
|
|
505
|
-
// progress).
|
|
506
|
-
const forced: Folded[] = [];
|
|
507
|
-
foldSlice(space, next, 0, next.length, forced, true);
|
|
508
|
-
next = forced;
|
|
509
|
-
}
|
|
510
506
|
levels.push(next);
|
|
511
507
|
level = next;
|
|
512
508
|
L++;
|
package/src/index.ts
CHANGED
|
@@ -12,6 +12,7 @@ export * from "./mind/index.js";
|
|
|
12
12
|
export * from "./store-sqlite.js";
|
|
13
13
|
export * from "./config.js";
|
|
14
14
|
export * from "./extension.js";
|
|
15
|
+
export * from "./canon.js";
|
|
15
16
|
export * from "./ingest-cache.js";
|
|
16
17
|
// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
|
|
17
18
|
export {
|
package/src/mind/learning.ts
CHANGED
|
@@ -7,7 +7,12 @@ import { Vec } from "../vec.js";
|
|
|
7
7
|
import { bindSeat, companySignature, isChunk, Sema } from "../sema.js";
|
|
8
8
|
import type { Input, MindContext } from "./types.js";
|
|
9
9
|
import { changedNodes } from "./types.js";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
inputBytes,
|
|
12
|
+
perceive,
|
|
13
|
+
perceiveDeposit,
|
|
14
|
+
resolve,
|
|
15
|
+
} from "./primitives.js";
|
|
11
16
|
import { canonicalWindows, leafIdPrefix } from "./canonical.js";
|
|
12
17
|
import { fold as foldVecs } from "../sema.js";
|
|
13
18
|
|
|
@@ -171,6 +176,38 @@ export async function ingestOne(
|
|
|
171
176
|
return Object.assign(tree, { id: rootId });
|
|
172
177
|
}
|
|
173
178
|
|
|
179
|
+
/** For each right-edge suffix of the context bytes, resolve it against the
|
|
180
|
+
* store. A suffix whose resolved node is already a known form inherits the
|
|
181
|
+
* continuation edge. Gate: ≥ 2 structural parents (reused across deposits),
|
|
182
|
+
* or (halo > 0 ∧ already an edge source). Pure answers do not qualify. */
|
|
183
|
+
async function propagateSuffixes(
|
|
184
|
+
ctx: MindContext,
|
|
185
|
+
src: number,
|
|
186
|
+
dst: number,
|
|
187
|
+
): Promise<void> {
|
|
188
|
+
const W = ctx.space.maxGroup;
|
|
189
|
+
const bytes = ctx.store.bytes(src);
|
|
190
|
+
const n = bytes.length;
|
|
191
|
+
if (n < 2 * W) return;
|
|
192
|
+
// Existence prefilter — the write side of the canonical contract: every
|
|
193
|
+
// deposit interns its WHOLE byte stream as a flat branch of per-byte leaf
|
|
194
|
+
// ids (deposit(), canonical.ts). A suffix is a stored form exactly when
|
|
195
|
+
// that flat twin exists, so one content-hash probe per offset decides;
|
|
196
|
+
// only a hit pays for resolve()'s deposit-shaped perception. This keeps
|
|
197
|
+
// the scan free of river folds — O(1) probes over cheap byte hashes
|
|
198
|
+
// instead of O(suffix) vector folds per offset.
|
|
199
|
+
const leafIds = leafIdPrefix(ctx, bytes);
|
|
200
|
+
for (let i = 1; i <= n - W; i++) {
|
|
201
|
+
if (ctx.store.findBranch(leafIds.slice(i)) === null) continue;
|
|
202
|
+
const id = resolve(ctx, bytes.subarray(i));
|
|
203
|
+
if (id === null || id === src) continue;
|
|
204
|
+
const known = ctx.store.parentsFirst(id, 2).length >= 2 ||
|
|
205
|
+
(ctx.store.haloMass(id) > 0 && ctx.store.hasNext(id));
|
|
206
|
+
if (!known) continue;
|
|
207
|
+
await ctx.store.link(id, dst);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
174
211
|
/** Ingest a pair (context, continuation) — learn an edge and pour halos. */
|
|
175
212
|
export async function ingestPair(
|
|
176
213
|
ctx: MindContext,
|
|
@@ -182,6 +219,7 @@ export async function ingestPair(
|
|
|
182
219
|
const ctxId = c.rootId, contId = cont_.rootId;
|
|
183
220
|
|
|
184
221
|
await ctx.store.link(ctxId, contId);
|
|
222
|
+
await propagateSuffixes(ctx, ctxId, contId);
|
|
185
223
|
|
|
186
224
|
// Halos pour company SIGNATURES (identity), not gists (content) — see
|
|
187
225
|
// companySignature in sema.ts.
|
package/src/mind/match.ts
CHANGED
|
@@ -446,8 +446,10 @@ export async function follow(
|
|
|
446
446
|
* `guide` the context whose gist resonates with the query wins (seat
|
|
447
447
|
* symmetry) — without one, the most-corroborated context wins (poured halo
|
|
448
448
|
* MASS, the direct measure of how many episodes established it), falling
|
|
449
|
-
* back to first-learnt on equal mass.
|
|
450
|
-
*
|
|
449
|
+
* back to first-learnt on equal mass. Among many predecessors RECIPROCAL
|
|
450
|
+
* ones (mutual edges) are preferred when any exist (RC5). Callers that
|
|
451
|
+
* HAVE a query gist must pass it, or they silently change disambiguation
|
|
452
|
+
* regime.
|
|
451
453
|
*
|
|
452
454
|
* `rev`, when the caller has already materialised prevOf (one read per
|
|
453
455
|
* relation — a hub's reverse fan-in is corpus-sized), is reused instead of
|
|
@@ -466,11 +468,32 @@ export function reverseContext(
|
|
|
466
468
|
// keeps the single-predecessor shortcut exact.
|
|
467
469
|
const candidates = rev ?? ctx.store.prevFirst(id, hubBound(ctx));
|
|
468
470
|
if (candidates.length === 0) return null;
|
|
469
|
-
|
|
470
|
-
|
|
471
|
+
// RECIPROCAL PREFERENCE: among many predecessors, one that `id` also
|
|
472
|
+
// continues TO (cand → id AND id → cand both learnt) is a mutually
|
|
473
|
+
// established pairing — the strongest structural evidence a predecessor
|
|
474
|
+
// can carry (bidirectional training deposits both directions of a genuine
|
|
475
|
+
// pair). A bare predecessor is one episode's adjacency; guide-resonance
|
|
476
|
+
// over bare predecessors favours whichever stored document merely
|
|
477
|
+
// CONTAINS the query's bytes (the linear fold's cosine is byte overlap —
|
|
478
|
+
// the observed "merci → unrelated French document" failure). One capped
|
|
479
|
+
// forward read decides; when no reciprocal exists, behaviour is unchanged
|
|
480
|
+
// — bare predecessors ARE the honest answer for a shared deposited
|
|
481
|
+
// continuation (two questions → one answer; audited by 31-audit C1), and
|
|
482
|
+
// this arm serves every mechanism's reverse projection, so abstaining
|
|
483
|
+
// here starves far more than the one containment failure it would fix.
|
|
484
|
+
let pool: readonly number[] = candidates;
|
|
485
|
+
if (candidates.length > 1) {
|
|
486
|
+
const fwd = new Set(ctx.store.nextFirst(id, hubBound(ctx)));
|
|
487
|
+
if (fwd.size > 0) {
|
|
488
|
+
const mutual = candidates.filter((c) => fwd.has(c));
|
|
489
|
+
if (mutual.length > 0) pool = mutual;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
const pick = pool.length === 1
|
|
493
|
+
? pool[0]
|
|
471
494
|
: guide
|
|
472
|
-
? chooseAmong(ctx,
|
|
473
|
-
: pickByMass(ctx,
|
|
495
|
+
? chooseAmong(ctx, pool, guide).id
|
|
496
|
+
: pickByMass(ctx, pool);
|
|
474
497
|
const g = read(ctx, pick);
|
|
475
498
|
return g.length > 0 ? g : null;
|
|
476
499
|
}
|
|
@@ -345,14 +345,34 @@ export async function counterfactualTransfer(
|
|
|
345
345
|
// corrupted-store read now falls back here instead of voicing a hollow
|
|
346
346
|
// seat into the comparison — the same "empty bytes are no grounding"
|
|
347
347
|
// invariant project() has always enforced.
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
348
|
+
// A node that only ever CONTINUES — an edge SOURCE with no predecessor —
|
|
349
|
+
// is a learnt CONTEXT (a stored question-shaped frame), not an entity.
|
|
350
|
+
// Voicing it verbatim answers a question with a question (the observed
|
|
351
|
+
// cast echo: two stored questions concatenated as an "answer"). The
|
|
352
|
+
// context's role is established by what it LEADS TO, so its seat is its
|
|
353
|
+
// own continuation — the fact — with the reverse projection and the raw
|
|
354
|
+
// bytes as the ordinary fallbacks for genuine entities.
|
|
355
|
+
const seatOfNode = async (
|
|
356
|
+
id: number,
|
|
357
|
+
fallback: Uint8Array,
|
|
358
|
+
): Promise<Uint8Array> => {
|
|
359
|
+
if (ctx.store.prevCount(id) === 0 && ctx.store.hasNext(id)) {
|
|
360
|
+
const fwd = await follow(ctx, id, qv);
|
|
361
|
+
if (fwd !== null) return fwd;
|
|
362
|
+
}
|
|
363
|
+
return reverseContext(ctx, id, qv) ?? fallback;
|
|
364
|
+
};
|
|
365
|
+
const seatOf = (p: Point): Promise<Uint8Array> => seatOfNode(p.anchor, p.ctx);
|
|
351
366
|
interface AnalogCandidate {
|
|
352
367
|
anchor: number;
|
|
353
368
|
/** The point this candidate came from, or null when it is a nextOf
|
|
354
369
|
* descendant — then seatOfNode supplies the seat. */
|
|
355
370
|
point: Point | null;
|
|
371
|
+
/** For a nextOf descendant: the aligned point whose continuation edge
|
|
372
|
+
* named it. Its runs ARE the query evidence this analog rests on
|
|
373
|
+
* (the hop was reached THROUGH that alignment), so the comparison
|
|
374
|
+
* schema accounts them. */
|
|
375
|
+
src: Point;
|
|
356
376
|
}
|
|
357
377
|
const analogs: AnalogCandidate[] = [];
|
|
358
378
|
for (const p of points) {
|
|
@@ -365,7 +385,7 @@ export async function counterfactualTransfer(
|
|
|
365
385
|
indexOf(p.ctx, dominant.ctx, 0) < 0 &&
|
|
366
386
|
indexOf(query, p.ctx, 0) < 0
|
|
367
387
|
) {
|
|
368
|
-
analogs.push({ anchor: p.anchor, point: p });
|
|
388
|
+
analogs.push({ anchor: p.anchor, point: p, src: p });
|
|
369
389
|
}
|
|
370
390
|
// Reach through to the point's continuation targets regardless
|
|
371
391
|
// of the point's own context length: when the point is a leaf
|
|
@@ -384,7 +404,7 @@ export async function counterfactualTransfer(
|
|
|
384
404
|
indexOf(nctx, dominant.ctx, 0) >= 0 ||
|
|
385
405
|
indexOf(query, nctx, 0) >= 0
|
|
386
406
|
) continue;
|
|
387
|
-
analogs.push({ anchor: nid, point: null });
|
|
407
|
+
analogs.push({ anchor: nid, point: null, src: p });
|
|
388
408
|
}
|
|
389
409
|
}
|
|
390
410
|
let bestAnalog: AnalogCandidate | null = null;
|
|
@@ -494,10 +514,10 @@ export async function counterfactualTransfer(
|
|
|
494
514
|
[],
|
|
495
515
|
"the two structures keep distributional company beyond chance — genuine analogs",
|
|
496
516
|
);
|
|
497
|
-
const a = seatOf(dominant);
|
|
517
|
+
const a = await seatOf(dominant);
|
|
498
518
|
const b = bestAnalog.point !== null
|
|
499
|
-
? seatOf(bestAnalog.point)
|
|
500
|
-
: seatOfNode(bestAnalog.anchor, read(ctx, bestAnalog.anchor));
|
|
519
|
+
? await seatOf(bestAnalog.point)
|
|
520
|
+
: await seatOfNode(bestAnalog.anchor, read(ctx, bestAnalog.anchor));
|
|
501
521
|
const answer = await joinWithBridge(ctx, a, b);
|
|
502
522
|
record(
|
|
503
523
|
answer,
|
|
@@ -506,13 +526,13 @@ export async function counterfactualTransfer(
|
|
|
506
526
|
// A halo-mediated act (the analogy gate) plus two seat projections.
|
|
507
527
|
CONCEPT + STEP + STEP,
|
|
508
528
|
// What comparison READ: the dominant's own aligned runs, plus the
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
//
|
|
512
|
-
//
|
|
529
|
+
// aligned runs of the point that named the analog — the analog itself
|
|
530
|
+
// when it was an aligned point, else the source point whose
|
|
531
|
+
// continuation edge reached it (that alignment IS the query evidence
|
|
532
|
+
// the hop rests on).
|
|
513
533
|
[
|
|
514
534
|
...runSpans(dominant),
|
|
515
|
-
...(bestAnalog.point
|
|
535
|
+
...runSpans(bestAnalog.point ?? bestAnalog.src),
|
|
516
536
|
],
|
|
517
537
|
);
|
|
518
538
|
}
|
|
@@ -116,6 +116,20 @@ export async function confluenceJoin(
|
|
|
116
116
|
* recognised common form). */
|
|
117
117
|
held: Array<[number, number]>;
|
|
118
118
|
}
|
|
119
|
+
// A constraint must bind a CONSTITUENT, not a shard. A genuinely shared
|
|
120
|
+
// form weaves a contiguous RUN of shared discriminative windows — its
|
|
121
|
+
// merged cover span is the form's own length, beyond one perception
|
|
122
|
+
// quantum (2W: the same "two quanta of structure" bar the conjunctive
|
|
123
|
+
// precondition `query.length < 2W` and CAST's weave live under). A long
|
|
124
|
+
// stored document holds thousands of W-windows and will hold a FEW of any
|
|
125
|
+
// query's by accident, but accidental sharing is one window (a merged
|
|
126
|
+
// span of W, at most ~W+overlap bytes: "ow ma", "ys a", "ías " —
|
|
127
|
+
// observed), never a run (" translucent", " featherlight" — the genuine
|
|
128
|
+
// constraints). Shard-bound streams are no constraints, and their meets
|
|
129
|
+
// are connective debris (". Sure,", "ngul" — observed).
|
|
130
|
+
const bindsAConstituent = (cover: Array<[number, number]>): boolean =>
|
|
131
|
+
cover.some(([cs, ce]) => ce - cs >= 2 * W);
|
|
132
|
+
|
|
119
133
|
const streams: Stream[] = [];
|
|
120
134
|
const rankedCapped = ranked.length > pre.k ? ranked.slice(0, pre.k) : ranked;
|
|
121
135
|
for (const cand of rankedCapped) {
|
|
@@ -134,7 +148,7 @@ export async function confluenceJoin(
|
|
|
134
148
|
if (curC !== null && off <= curC[1]) curC[1] = off + W;
|
|
135
149
|
else cover.push(curC = [off, off + W]);
|
|
136
150
|
}
|
|
137
|
-
if (cover.length > 0) {
|
|
151
|
+
if (cover.length > 0 && bindsAConstituent(cover)) {
|
|
138
152
|
streams.push({ anchor: cand.anchor, vote: cand.vote, ids, cover, held });
|
|
139
153
|
}
|
|
140
154
|
}
|