@hviana/sema 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/train_base.js +24 -4
- package/dist/src/alu/src/parser.d.ts +9 -0
- package/dist/src/alu/src/parser.js +110 -2
- package/dist/src/canon.d.ts +26 -0
- package/dist/src/canon.js +57 -0
- package/dist/src/geometry.js +12 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/mind/learning.js +33 -1
- package/dist/src/mind/match.d.ts +4 -2
- package/dist/src/mind/match.js +30 -6
- package/dist/src/mind/mechanisms/cast.js +18 -4
- package/dist/src/mind/mechanisms/confluence.js +13 -1
- package/dist/src/mind/mechanisms/recall.js +97 -28
- package/dist/src/mind/mind.d.ts +64 -2
- package/dist/src/mind/mind.js +152 -23
- package/dist/src/mind/primitives.d.ts +9 -0
- package/dist/src/mind/primitives.js +68 -1
- package/dist/src/mind/recognition.js +11 -1
- package/dist/src/mind/types.d.ts +10 -0
- package/dist/src/store-sqlite.d.ts +8 -0
- package/dist/src/store-sqlite.js +52 -0
- package/dist/src/store.d.ts +12 -0
- package/example/train_base.ts +29 -4
- package/package.json +1 -1
- package/src/alu/src/parser.ts +105 -4
- package/src/canon.ts +65 -0
- package/src/geometry.ts +12 -0
- package/src/index.ts +1 -0
- package/src/mind/learning.ts +39 -1
- package/src/mind/match.ts +29 -6
- package/src/mind/mechanisms/cast.ts +21 -6
- package/src/mind/mechanisms/confluence.ts +15 -1
- package/src/mind/mechanisms/recall.ts +116 -41
- package/src/mind/mind.ts +172 -29
- package/src/mind/primitives.ts +66 -1
- package/src/mind/recognition.ts +10 -0
- package/src/mind/types.ts +10 -0
- package/src/store-sqlite.ts +68 -0
- package/src/store.ts +27 -0
- package/test/13-conversation.test.mjs +77 -27
- package/test/35-prefix-edge.test.mjs +86 -0
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// Address — bytes → node (perceive, foldTree, resolve)
|
|
4
4
|
// Read — node → bytes (read)
|
|
5
5
|
import { bytesToTree, bytesToTreePyramid, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
|
|
6
|
+
import { canonHash } from "../canon.js";
|
|
7
|
+
import { bytesEqual } from "../bytes.js";
|
|
6
8
|
import { ALL } from "./types.js";
|
|
7
9
|
// ── Address: bytes → node ──────────────────────────────────────────────
|
|
8
10
|
/** The content key of a byte span — one latin1 char per byte, an exact,
|
|
@@ -155,7 +157,72 @@ export function foldTree(ctx, n, start, visit) {
|
|
|
155
157
|
export function resolve(ctx, bytes) {
|
|
156
158
|
if (bytes.length === 0)
|
|
157
159
|
return null;
|
|
158
|
-
|
|
160
|
+
const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
|
|
161
|
+
if (exact !== null)
|
|
162
|
+
return exact;
|
|
163
|
+
return canonResolve(ctx, bytes);
|
|
164
|
+
}
|
|
165
|
+
/** Equivalence-class resolution: when the exact content-addressed lookup
|
|
166
|
+
* misses, find a stored node whose CANONICAL key equals the span's — the
|
|
167
|
+
* store's canon index proposes candidates by key hash, and each is verified
|
|
168
|
+
* by re-canonicalizing its bytes (hash-then-verify, like every content
|
|
169
|
+
* lookup). Among verified candidates, one that leads somewhere (has a
|
|
170
|
+
* continuation edge) is preferred; ties break to the lowest id — a corpus
|
|
171
|
+
* property, not a seed property. Null when the response carries no
|
|
172
|
+
* canonicalizer, the store has no canon index, or nothing verifies. */
|
|
173
|
+
export function canonResolve(ctx, bytes) {
|
|
174
|
+
const canon = ctx.canon;
|
|
175
|
+
const store = ctx.store;
|
|
176
|
+
if (canon === null || !store.canonFind)
|
|
177
|
+
return null;
|
|
178
|
+
if (bytes.length < 2)
|
|
179
|
+
return null;
|
|
180
|
+
const memo = ctx.canonMemo;
|
|
181
|
+
const memoKey = memo ? latin1Key(bytes) : "";
|
|
182
|
+
if (memo) {
|
|
183
|
+
const hit = memo.get(memoKey);
|
|
184
|
+
if (hit !== undefined)
|
|
185
|
+
return hit;
|
|
186
|
+
}
|
|
187
|
+
const set = (v) => {
|
|
188
|
+
memo?.set(memoKey, v);
|
|
189
|
+
return v;
|
|
190
|
+
};
|
|
191
|
+
const key = canon(bytes);
|
|
192
|
+
if (key.length === 0)
|
|
193
|
+
return set(null);
|
|
194
|
+
// A stored form that IS canonical is not in the index (buildCanonIndex
|
|
195
|
+
// skips identity rows) — the exact content-addressed lookup of the
|
|
196
|
+
// canonical bytes finds it directly.
|
|
197
|
+
if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
|
|
198
|
+
const direct = foldTree(ctx, perceive(ctx, key), 0).node;
|
|
199
|
+
if (direct !== null)
|
|
200
|
+
return set(direct);
|
|
201
|
+
}
|
|
202
|
+
const candidates = store.canonFind(canonHash(key));
|
|
203
|
+
if (candidates.length === 0)
|
|
204
|
+
return set(null);
|
|
205
|
+
let best = null;
|
|
206
|
+
let bestLeads = false;
|
|
207
|
+
for (const id of candidates) {
|
|
208
|
+
const bytesOf = read(ctx, id);
|
|
209
|
+
const stored = canon(bytesOf);
|
|
210
|
+
if (stored.length !== key.length || !bytesEqual(stored, key))
|
|
211
|
+
continue;
|
|
212
|
+
// The index stores FLAT content twins; the id the exact path would have
|
|
213
|
+
// resolved for these bytes is their FOLD — the deposit-shaped node that
|
|
214
|
+
// carries the edges and halos. Re-folding the candidate's bytes lands
|
|
215
|
+
// on exactly the node the canonical-case query would have found.
|
|
216
|
+
const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
|
|
217
|
+
const use = folded ?? id;
|
|
218
|
+
const leads = store.hasNext(use) || store.haloMass(use) > 0;
|
|
219
|
+
if (best === null || (leads && !bestLeads) ||
|
|
220
|
+
(leads === bestLeads && use < best)) {
|
|
221
|
+
best = use;
|
|
222
|
+
bestLeads = leads;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return set(best);
|
|
159
226
|
}
|
|
160
227
|
/** Walk a perceived tree in POST-ORDER with byte offsets — children before
|
|
161
228
|
* their parent, `visit(node, start, end)` for every node including leaves.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// that leads somewhere (has a continuation edge or a halo).
|
|
6
6
|
// segment — leaf-parent segmentation using the geometry's own groupings.
|
|
7
7
|
import { rItem } from "./trace.js";
|
|
8
|
-
import { foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
|
|
8
|
+
import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
|
|
9
9
|
import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
|
|
10
10
|
import { chainReach, leafIdAt } from "./canonical.js";
|
|
11
11
|
import { isChunk } from "../sema.js";
|
|
@@ -86,6 +86,16 @@ function recogniseImpl(ctx, bytes) {
|
|
|
86
86
|
}
|
|
87
87
|
if (node !== null)
|
|
88
88
|
emit(start, end, node);
|
|
89
|
+
// Canonical fallback: a subtree whose exact content-addressed lookup
|
|
90
|
+
// missed may still be a stored form under the response's equivalence
|
|
91
|
+
// (case, width, whitespace — whatever the injected canonicalizer says).
|
|
92
|
+
// O(subtree bytes) per miss, memoised per response; a no-op when no
|
|
93
|
+
// canonicalizer was injected or the store has no canon index.
|
|
94
|
+
else if (end - start >= 2) {
|
|
95
|
+
const cid = canonResolve(ctx, bytes.subarray(start, end));
|
|
96
|
+
if (cid !== null)
|
|
97
|
+
emit(start, end, cid);
|
|
98
|
+
}
|
|
89
99
|
if (isChunk(n)) {
|
|
90
100
|
starts.add(start);
|
|
91
101
|
// Try every sub-span within this leaf-parent.
|
package/dist/src/mind/types.d.ts
CHANGED
|
@@ -105,6 +105,16 @@ export interface MindContext extends GraphSearchHost {
|
|
|
105
105
|
cfg: MindConfig;
|
|
106
106
|
search: GraphSearch;
|
|
107
107
|
trace: Rationale | null;
|
|
108
|
+
/** The content canonicalizer for THIS response, or null — injected by the
|
|
109
|
+
* modality entry point (respondText passes the text canonicalizer; a
|
|
110
|
+
* binary respond passes none). Resolution uses it as a fallback: when
|
|
111
|
+
* the exact content-addressed lookup misses, the span's canonical key is
|
|
112
|
+
* probed against the store's canon index (see src/canon.ts). The core
|
|
113
|
+
* never inspects what the equivalence IS. */
|
|
114
|
+
canon: ((bytes: Uint8Array) => Uint8Array) | null;
|
|
115
|
+
/** Per-response memo of canonical-fallback resolutions, keyed by the
|
|
116
|
+
* span's latin1 content key. Null outside respond(). */
|
|
117
|
+
canonMemo: Map<string, number | null> | null;
|
|
108
118
|
/** Memo of the consensus climb — content-keyed (latin1) so results
|
|
109
119
|
* persist across conversation turns where the same byte spans recur.
|
|
110
120
|
* Null outside respond(); during respondTurn() the conversation's
|
|
@@ -43,6 +43,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
43
43
|
private _selPrev;
|
|
44
44
|
private _setMeta;
|
|
45
45
|
private _getMeta;
|
|
46
|
+
private _insCanon;
|
|
47
|
+
private _selCanon;
|
|
48
|
+
private _cntCanon;
|
|
49
|
+
private _selContentFrom;
|
|
46
50
|
private _delMeta;
|
|
47
51
|
private _insSnapshot;
|
|
48
52
|
private _selSnapshot;
|
|
@@ -130,6 +134,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
130
134
|
protected _dbGetMeta(key: string): string | null;
|
|
131
135
|
protected _dbSetMeta(key: string, val: string): void;
|
|
132
136
|
protected _dbDeleteMeta(key: string): void;
|
|
137
|
+
canonAdd(h: number, id: number): void;
|
|
138
|
+
canonFind(h: number): number[];
|
|
139
|
+
canonCount(): number;
|
|
140
|
+
eachContent(cb: (id: number, bytes: Uint8Array) => void, fromId?: number): void;
|
|
133
141
|
protected _dbSaveSnapshot(bytes: Uint8Array): void;
|
|
134
142
|
protected _dbLoadSnapshot(): Uint8Array | null;
|
|
135
143
|
protected _vecContentUpsert(entries: Array<{
|
package/dist/src/store-sqlite.js
CHANGED
|
@@ -112,6 +112,18 @@ CREATE TABLE IF NOT EXISTS contain (
|
|
|
112
112
|
id INTEGER PRIMARY KEY,
|
|
113
113
|
parents BLOB NOT NULL
|
|
114
114
|
);
|
|
115
|
+
-- CANONICAL-FORM index (Store.canonAdd/canonFind): h is the 32-bit hash of a
|
|
116
|
+
-- node's CANONICAL content key (the modality's canonicalizer output — case-
|
|
117
|
+
-- folded, whitespace-collapsed text, etc.; the store never sees the key
|
|
118
|
+
-- itself, only its hash). Same hash-then-verify discipline as idx_node_h:
|
|
119
|
+
-- the caller re-canonicalizes each candidate's bytes before trusting it, so
|
|
120
|
+
-- a collision costs a read, never a wrong id. WITHOUT ROWID on (h, id):
|
|
121
|
+
-- the composite key IS the lookup index and gives free dedup.
|
|
122
|
+
CREATE TABLE IF NOT EXISTS canon (
|
|
123
|
+
h INTEGER NOT NULL,
|
|
124
|
+
id INTEGER NOT NULL,
|
|
125
|
+
PRIMARY KEY (h, id)
|
|
126
|
+
) WITHOUT ROWID;
|
|
115
127
|
CREATE TABLE IF NOT EXISTS snapshot (
|
|
116
128
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
117
129
|
data BLOB NOT NULL
|
|
@@ -212,6 +224,10 @@ export class SQliteStore extends AbstractStore {
|
|
|
212
224
|
_selPrev = null;
|
|
213
225
|
_setMeta = null;
|
|
214
226
|
_getMeta = null;
|
|
227
|
+
_insCanon = null;
|
|
228
|
+
_selCanon = null;
|
|
229
|
+
_cntCanon = null;
|
|
230
|
+
_selContentFrom = null;
|
|
215
231
|
_delMeta = null;
|
|
216
232
|
_insSnapshot = null;
|
|
217
233
|
_selSnapshot = null;
|
|
@@ -797,6 +813,42 @@ export class SQliteStore extends AbstractStore {
|
|
|
797
813
|
}
|
|
798
814
|
this._delMeta.run(key);
|
|
799
815
|
}
|
|
816
|
+
// -- Canonical-form index (Store optional capability) --
|
|
817
|
+
canonAdd(h, id) {
|
|
818
|
+
if (!this._insCanon) {
|
|
819
|
+
this._insCanon = this.sqlite.prepare("INSERT OR IGNORE INTO canon (h, id) VALUES (?, ?)");
|
|
820
|
+
}
|
|
821
|
+
// Join the deferred write transaction (committed by flush/commit), so a
|
|
822
|
+
// bulk index build coalesces instead of paying autocommit per row.
|
|
823
|
+
this._dbBeginTx();
|
|
824
|
+
this._insCanon.run(h, id);
|
|
825
|
+
}
|
|
826
|
+
canonFind(h) {
|
|
827
|
+
if (!this._selCanon) {
|
|
828
|
+
this._selCanon = this.sqlite.prepare("SELECT id FROM canon WHERE h = ?");
|
|
829
|
+
}
|
|
830
|
+
return this._selCanon.all(h).map((r) => r.id);
|
|
831
|
+
}
|
|
832
|
+
canonCount() {
|
|
833
|
+
if (!this._cntCanon) {
|
|
834
|
+
this._cntCanon = this.sqlite.prepare("SELECT count(*) AS c FROM canon");
|
|
835
|
+
}
|
|
836
|
+
return this._cntCanon.get().c;
|
|
837
|
+
}
|
|
838
|
+
eachContent(cb, fromId = 0) {
|
|
839
|
+
if (!this._selContentFrom) {
|
|
840
|
+
// Content-bearing nodes are FLAT branches: leaf present, kids an
|
|
841
|
+
// empty (zero-length) blob — the same population PART 2 of the
|
|
842
|
+
// diagnostics calls "distinct content spans".
|
|
843
|
+
this._selContentFrom = this.sqlite.prepare("SELECT id, leaf FROM node " +
|
|
844
|
+
"WHERE id >= ? AND leaf IS NOT NULL " +
|
|
845
|
+
"AND (kids IS NULL OR length(kids) = 0)");
|
|
846
|
+
}
|
|
847
|
+
for (const row of this._selContentFrom.iterate(fromId)) {
|
|
848
|
+
const r = row;
|
|
849
|
+
cb(r.id, new Uint8Array(r.leaf));
|
|
850
|
+
}
|
|
851
|
+
}
|
|
800
852
|
// -- Snapshot --
|
|
801
853
|
_dbSaveSnapshot(bytes) {
|
|
802
854
|
if (!this._insSnapshot) {
|
package/dist/src/store.d.ts
CHANGED
|
@@ -240,6 +240,18 @@ export interface Store {
|
|
|
240
240
|
* repetition, but repetition outranks insertion-order accident. 0 when
|
|
241
241
|
* the node has no halo row. */
|
|
242
242
|
haloMass(id: NodeId): number;
|
|
243
|
+
/** Record that node `id`'s canonical key hashes to `h`. Idempotent. */
|
|
244
|
+
canonAdd?(h: number, id: NodeId): void;
|
|
245
|
+
/** All candidate node ids whose canonical key hashes to `h` (collisions
|
|
246
|
+
* included — the caller verifies). */
|
|
247
|
+
canonFind?(h: number): NodeId[];
|
|
248
|
+
/** Number of (h, id) rows in the canon index — 0 means never built. */
|
|
249
|
+
canonCount?(): number;
|
|
250
|
+
/** Visit every content-bearing node (flat branch: `leaf` present, no
|
|
251
|
+
* kids) — the population a canonical index is built over. `fromId`
|
|
252
|
+
* restricts the scan to ids ≥ fromId, so an index refresh after further
|
|
253
|
+
* training only visits the new rows. */
|
|
254
|
+
eachContent?(cb: (id: NodeId, bytes: Uint8Array) => void, fromId?: NodeId): void;
|
|
243
255
|
size(): Promise<number>;
|
|
244
256
|
saveSnapshot(bytes: Uint8Array): Promise<void>;
|
|
245
257
|
loadSnapshot(): Promise<Uint8Array | null>;
|
package/example/train_base.ts
CHANGED
|
@@ -177,8 +177,11 @@ const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir of *.zip
|
|
|
177
177
|
const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
|
|
178
178
|
const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
|
|
179
179
|
const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel refresh cadence
|
|
180
|
-
// Index maintenance at checkpoints: compact (remove garbage)
|
|
181
|
-
// gaps)
|
|
180
|
+
// Index maintenance at checkpoints: compact (remove garbage), repair (fill
|
|
181
|
+
// gaps), then refresh the canonical-form index (equivalence-class resolution —
|
|
182
|
+
// src/canon.ts). All three are idempotent batch operations (the canon build is
|
|
183
|
+
// additionally incremental via the store's `canon.upto` cursor);
|
|
184
|
+
// INDEX_MAINTENANCE=0 disables.
|
|
182
185
|
const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
|
|
183
186
|
const DOWNLOAD_TRIES = 5;
|
|
184
187
|
// In-progress downloads are written to a sibling "<dest>.part" and atomically
|
|
@@ -1823,8 +1826,9 @@ async function main(): Promise<void> {
|
|
|
1823
1826
|
|
|
1824
1827
|
const checkpoint = () => mind.save();
|
|
1825
1828
|
|
|
1826
|
-
/** Run index maintenance: compact (remove garbage)
|
|
1827
|
-
*
|
|
1829
|
+
/** Run index maintenance: compact (remove garbage), repair (fill gaps),
|
|
1830
|
+
* then refresh the canonical-form index (see below). All three are
|
|
1831
|
+
* idempotent — running twice produces the same result as once.
|
|
1828
1832
|
* Compaction frees index space first; repair then adds back every
|
|
1829
1833
|
* edge/halo-bearing node whose gist was evicted from the pending cache
|
|
1830
1834
|
* before it reached the content index, completing the coverage that
|
|
@@ -1877,6 +1881,27 @@ async function main(): Promise<void> {
|
|
|
1877
1881
|
}`,
|
|
1878
1882
|
);
|
|
1879
1883
|
}
|
|
1884
|
+
// Canonical-form index (src/canon.ts): lets resolution find stored forms
|
|
1885
|
+
// across surface variation (case, width, whitespace). Incremental and
|
|
1886
|
+
// idempotent by construction — the `canon.upto` meta cursor scans only
|
|
1887
|
+
// nodes newer than the last pass, and the (h, id) primary key ignores
|
|
1888
|
+
// re-inserted rows — so it composes with the resume model exactly like
|
|
1889
|
+
// compact/repair: every checkpoint (and finish) leaves the index
|
|
1890
|
+
// covering all content trained so far.
|
|
1891
|
+
try {
|
|
1892
|
+
const added = await mind.buildCanonIndex();
|
|
1893
|
+
if (added > 0) {
|
|
1894
|
+
progress.log(
|
|
1895
|
+
` ${GRN}canon index: added ${int(added)} canonical-form entries${R}`,
|
|
1896
|
+
);
|
|
1897
|
+
}
|
|
1898
|
+
} catch (err) {
|
|
1899
|
+
progress.log(
|
|
1900
|
+
` ${YEL}⚠ canon index build failed${R}: ${
|
|
1901
|
+
err instanceof Error ? err.message : String(err)
|
|
1902
|
+
}`,
|
|
1903
|
+
);
|
|
1904
|
+
}
|
|
1880
1905
|
};
|
|
1881
1906
|
|
|
1882
1907
|
// The checkpoint recall is a best-effort diagnostic. It is time-bounded so a
|
package/package.json
CHANGED
package/src/alu/src/parser.ts
CHANGED
|
@@ -172,10 +172,27 @@ export class QueryParser {
|
|
|
172
172
|
j,
|
|
173
173
|
bytes: this.evalRun(query.subarray(i, j)),
|
|
174
174
|
}));
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
);
|
|
178
|
-
|
|
175
|
+
// BRACKETED runs — "(3 + 4) * (10 - 2)" — extend across grouping
|
|
176
|
+
// notation the flat alternation cannot cross. The expression grammar
|
|
177
|
+
// (expr.ts) already nests and parenthesizes; this only hands it the
|
|
178
|
+
// WHOLE bracketed span instead of the fragments between brackets. A
|
|
179
|
+
// bracketed run that evaluates ABSORBS the flat runs inside it (the
|
|
180
|
+
// authority law: contained spans are its material); one that does not
|
|
181
|
+
// evaluate leaves the flat readings untouched.
|
|
182
|
+
const bracketed = this.bracketedRuns(query, tokens)
|
|
183
|
+
.map(([i, j]) => ({ i, j, bytes: this.evalRun(query.subarray(i, j)) }))
|
|
184
|
+
.filter((r): r is ComputedSpan => r.bytes !== null);
|
|
185
|
+
const absorbed = (r: { i: number; j: number }) =>
|
|
186
|
+
bracketed.some((b) => b.i <= r.i && r.j <= b.j);
|
|
187
|
+
const composedRuns = [
|
|
188
|
+
...runs.filter((r) => !absorbed(r)),
|
|
189
|
+
...bracketed,
|
|
190
|
+
].sort((a, b) => a.i - b.i);
|
|
191
|
+
const spans: ComputedSpan[] = [
|
|
192
|
+
...runs.filter((r): r is ComputedSpan => r.bytes !== null),
|
|
193
|
+
...bracketed,
|
|
194
|
+
];
|
|
195
|
+
let stream = compose(tokens, composedRuns);
|
|
179
196
|
const readings = new Map<Token, readonly string[]>();
|
|
180
197
|
for (;;) {
|
|
181
198
|
const fired = await this.operations(query, stream, readings);
|
|
@@ -302,6 +319,90 @@ export class QueryParser {
|
|
|
302
319
|
return runs;
|
|
303
320
|
}
|
|
304
321
|
|
|
322
|
+
/** The maximal BRACKETED arithmetic runs: like {@link arithmeticRuns} but
|
|
323
|
+
* grouping brackets participate — an all-'(' term opens depth where an
|
|
324
|
+
* operand is expected, a term beginning with ')' closes it where an
|
|
325
|
+
* operator could follow (only its bracket prefix joins the run; anything
|
|
326
|
+
* after — a trailing "?" — ends it). A run is recorded only when it is
|
|
327
|
+
* BALANCED, actually crossed a bracket, and contains an operator — the
|
|
328
|
+
* bracket-free case is {@link arithmeticRuns}' own. Evaluation is the
|
|
329
|
+
* same registry-derived grammar, whose lexer already reads the brackets. */
|
|
330
|
+
private bracketedRuns(
|
|
331
|
+
query: Uint8Array,
|
|
332
|
+
tokens: Token[],
|
|
333
|
+
): Array<[number, number]> {
|
|
334
|
+
const OPEN = 0x28, CLOSE = 0x29;
|
|
335
|
+
const bracketPrefix = (t: Token, b: number): number => {
|
|
336
|
+
if (t.kind !== "term") return 0;
|
|
337
|
+
let n = 0;
|
|
338
|
+
while (t.i + n < t.j && query[t.i + n] === b) n++;
|
|
339
|
+
return n;
|
|
340
|
+
};
|
|
341
|
+
const allOf = (t: Token, b: number): boolean =>
|
|
342
|
+
bracketPrefix(t, b) === t.j - t.i;
|
|
343
|
+
const bridged = (from: number, to: number): boolean =>
|
|
344
|
+
from >= to || this.host.segment(query.subarray(from, to)).length <= 1;
|
|
345
|
+
const runs: Array<[number, number]> = [];
|
|
346
|
+
let k = 0;
|
|
347
|
+
while (k < tokens.length) {
|
|
348
|
+
const first = tokens[k];
|
|
349
|
+
if (first.kind !== "operand" && !allOf(first, OPEN)) {
|
|
350
|
+
k++;
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
let depth = 0;
|
|
354
|
+
let ops = 0;
|
|
355
|
+
let brackets = 0;
|
|
356
|
+
let expectOperand = true;
|
|
357
|
+
let end = -1; // byte end of the last VALID run state (balanced, after operand/close)
|
|
358
|
+
let endTok = k; // token index just past the accepted prefix
|
|
359
|
+
let m = k;
|
|
360
|
+
let prevJ = -1;
|
|
361
|
+
while (m < tokens.length) {
|
|
362
|
+
const t = tokens[m];
|
|
363
|
+
if (prevJ >= 0 && !bridged(prevJ, t.i)) break;
|
|
364
|
+
if (expectOperand) {
|
|
365
|
+
if (allOf(t, OPEN)) {
|
|
366
|
+
depth += t.j - t.i;
|
|
367
|
+
brackets++;
|
|
368
|
+
} else if (t.kind === "operand") {
|
|
369
|
+
expectOperand = false;
|
|
370
|
+
if (depth === 0) {
|
|
371
|
+
end = t.j;
|
|
372
|
+
endTok = m + 1;
|
|
373
|
+
}
|
|
374
|
+
} else break;
|
|
375
|
+
} else {
|
|
376
|
+
const c = bracketPrefix(t, CLOSE);
|
|
377
|
+
if (c > 0 && depth > 0) {
|
|
378
|
+
const take = Math.min(c, depth);
|
|
379
|
+
depth -= take;
|
|
380
|
+
brackets++;
|
|
381
|
+
if (depth === 0) {
|
|
382
|
+
end = t.i + take;
|
|
383
|
+
endTok = m + 1;
|
|
384
|
+
}
|
|
385
|
+
// A term with content beyond its usable bracket prefix ("?")
|
|
386
|
+
// ends the run inside the term.
|
|
387
|
+
if (take < t.j - t.i) break;
|
|
388
|
+
} else if (t.kind === "operator") {
|
|
389
|
+
expectOperand = true;
|
|
390
|
+
ops++;
|
|
391
|
+
} else break;
|
|
392
|
+
}
|
|
393
|
+
prevJ = t.j;
|
|
394
|
+
m++;
|
|
395
|
+
}
|
|
396
|
+
if (end >= 0 && ops > 0 && brackets > 0) {
|
|
397
|
+
runs.push([first.i, end]);
|
|
398
|
+
k = endTok;
|
|
399
|
+
} else {
|
|
400
|
+
k++;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return runs;
|
|
404
|
+
}
|
|
405
|
+
|
|
305
406
|
/** Evaluate an infix-arithmetic run to its canonical result bytes, through
|
|
306
407
|
* the kernel's recursive expression evaluator, or null if it does not
|
|
307
408
|
* evaluate. A whole result stays an exact int; otherwise the canonical
|
package/src/canon.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// canon.ts — content canonicalization for equivalence-class resolution.
|
|
2
|
+
//
|
|
3
|
+
// The store is content-addressed on RAW bytes: "What", "WHAT" and "what" are
|
|
4
|
+
// three different hashes, so a query whose surface form varies from the
|
|
5
|
+
// trained form resolves to nothing even though the CONTENT is the same. A
|
|
6
|
+
// CANONICALIZER maps every surface variant of the same content onto one
|
|
7
|
+
// canonical byte string; the store keeps a small hash index from canonical
|
|
8
|
+
// keys to node ids (see Store.canonAdd/canonFind), and resolution falls back
|
|
9
|
+
// to that index when the exact content-addressed lookup misses.
|
|
10
|
+
//
|
|
11
|
+
// The canonicalizer is MODALITY-SPECIFIC and always INJECTED — nothing in the
|
|
12
|
+
// store or the mind's core knows what "case" or "whitespace" is. The text
|
|
13
|
+
// canonicalizer below is the one `respondText`/`respondTurnText` pass down;
|
|
14
|
+
// a grid or audio modality would supply its own (or none).
|
|
15
|
+
//
|
|
16
|
+
// Canonical keys are equivalence-class LABELS, never content: they are hashed
|
|
17
|
+
// and verified (canon(stored bytes) must equal canon(query bytes) before an
|
|
18
|
+
// id is accepted), so a hash collision costs a read, never a wrong id — the
|
|
19
|
+
// same discipline as the node table's own `h` index.
|
|
20
|
+
|
|
21
|
+
/** A content canonicalizer: maps a byte span to the canonical representative
|
|
22
|
+
* of its equivalence class. Must be pure and deterministic. Returning the
|
|
23
|
+
* input unchanged is always sound (the class is then {input}). */
|
|
24
|
+
export type Canon = (bytes: Uint8Array) => Uint8Array;
|
|
25
|
+
|
|
26
|
+
const dec = new TextDecoder("utf-8", { fatal: false });
|
|
27
|
+
const enc = new TextEncoder();
|
|
28
|
+
|
|
29
|
+
/** The TEXT canonicalizer: Unicode-aware equivalence over every character
|
|
30
|
+
* variation that does not change what the text SAYS —
|
|
31
|
+
*
|
|
32
|
+
* • compatibility normalization (NFKC): full-width forms, ligatures,
|
|
33
|
+
* composed vs decomposed accents collapse to one representation;
|
|
34
|
+
* • case folding (locale-independent lowercase after NFKC — the standard
|
|
35
|
+
* simple fold);
|
|
36
|
+
* • whitespace: every INTERIOR run of Unicode whitespace becomes one plain
|
|
37
|
+
* space. EDGE whitespace is preserved verbatim: a span's leading or
|
|
38
|
+
* trailing separator belongs BETWEEN forms, not to the form — trimming
|
|
39
|
+
* it would let a recognised span swallow the boundary byte that
|
|
40
|
+
* separates it from its neighbour (observed: "ice fire" composing to
|
|
41
|
+
* "coldhot" because the span "ice " matched the stored "ice").
|
|
42
|
+
*
|
|
43
|
+
* "WHAT IS", "What is" and "what is" share one canonical form. This is
|
|
44
|
+
* deliberately conservative: punctuation, digits and word order are content
|
|
45
|
+
* and pass through untouched. */
|
|
46
|
+
export function textCanon(bytes: Uint8Array): Uint8Array {
|
|
47
|
+
const s = dec
|
|
48
|
+
.decode(bytes)
|
|
49
|
+
.normalize("NFKC")
|
|
50
|
+
.toLowerCase()
|
|
51
|
+
.replace(/(\S)\s+(?=\S)/g, "$1 ");
|
|
52
|
+
return enc.encode(s);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 32-bit FNV-1a over a canonical key — the integer the store's canon index
|
|
56
|
+
* is keyed on. Same construction as the node table's content hash; a
|
|
57
|
+
* collision is resolved by verifying canon(stored) === key, never trusted. */
|
|
58
|
+
export function canonHash(key: Uint8Array): number {
|
|
59
|
+
let h = 0x811c9dc5 >>> 0;
|
|
60
|
+
for (let i = 0; i < key.length; i++) {
|
|
61
|
+
h ^= key[i];
|
|
62
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
63
|
+
}
|
|
64
|
+
return h >>> 0;
|
|
65
|
+
}
|
package/src/geometry.ts
CHANGED
|
@@ -460,6 +460,18 @@ export function bytesToTreePyramid(
|
|
|
460
460
|
pyramid: { levels: [], bytes: 0 },
|
|
461
461
|
};
|
|
462
462
|
}
|
|
463
|
+
// ZERO GROWTH: the previous pyramid already covers every byte — its top
|
|
464
|
+
// level holds the finished, normalized root. Refolding here would not
|
|
465
|
+
// only waste the work: when the whole span is one full block, the refold's
|
|
466
|
+
// top item is a REUSED raw interior of `prev`, and the final normalize
|
|
467
|
+
// below would mutate that shared raw block in place, corrupting every
|
|
468
|
+
// later incremental fold built on `prev`.
|
|
469
|
+
if (prev && prev.bytes === bytes.length && prev.levels.length > 0) {
|
|
470
|
+
return {
|
|
471
|
+
tree: prev.levels[prev.levels.length - 1][0].tree,
|
|
472
|
+
pyramid: prev,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
463
475
|
const mg = space.maxGroup;
|
|
464
476
|
const reusable = (L: number): ReadonlyArray<Folded> | null => {
|
|
465
477
|
// prev's TOPMOST level holds its normalized ROOT — reusable blocks must
|
package/src/index.ts
CHANGED
|
@@ -12,6 +12,7 @@ export * from "./mind/index.js";
|
|
|
12
12
|
export * from "./store-sqlite.js";
|
|
13
13
|
export * from "./config.js";
|
|
14
14
|
export * from "./extension.js";
|
|
15
|
+
export * from "./canon.js";
|
|
15
16
|
export * from "./ingest-cache.js";
|
|
16
17
|
// rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
|
|
17
18
|
export {
|
package/src/mind/learning.ts
CHANGED
|
@@ -7,7 +7,12 @@ import { Vec } from "../vec.js";
|
|
|
7
7
|
import { bindSeat, companySignature, isChunk, Sema } from "../sema.js";
|
|
8
8
|
import type { Input, MindContext } from "./types.js";
|
|
9
9
|
import { changedNodes } from "./types.js";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
inputBytes,
|
|
12
|
+
perceive,
|
|
13
|
+
perceiveDeposit,
|
|
14
|
+
resolve,
|
|
15
|
+
} from "./primitives.js";
|
|
11
16
|
import { canonicalWindows, leafIdPrefix } from "./canonical.js";
|
|
12
17
|
import { fold as foldVecs } from "../sema.js";
|
|
13
18
|
|
|
@@ -171,6 +176,38 @@ export async function ingestOne(
|
|
|
171
176
|
return Object.assign(tree, { id: rootId });
|
|
172
177
|
}
|
|
173
178
|
|
|
179
|
+
/** For each right-edge suffix of the context bytes, resolve it against the
|
|
180
|
+
* store. A suffix whose resolved node is already a known form inherits the
|
|
181
|
+
* continuation edge. Gate: ≥ 2 structural parents (reused across deposits),
|
|
182
|
+
* or (halo > 0 ∧ already an edge source). Pure answers do not qualify. */
|
|
183
|
+
async function propagateSuffixes(
|
|
184
|
+
ctx: MindContext,
|
|
185
|
+
src: number,
|
|
186
|
+
dst: number,
|
|
187
|
+
): Promise<void> {
|
|
188
|
+
const W = ctx.space.maxGroup;
|
|
189
|
+
const bytes = ctx.store.bytes(src);
|
|
190
|
+
const n = bytes.length;
|
|
191
|
+
if (n < 2 * W) return;
|
|
192
|
+
// Existence prefilter — the write side of the canonical contract: every
|
|
193
|
+
// deposit interns its WHOLE byte stream as a flat branch of per-byte leaf
|
|
194
|
+
// ids (deposit(), canonical.ts). A suffix is a stored form exactly when
|
|
195
|
+
// that flat twin exists, so one content-hash probe per offset decides;
|
|
196
|
+
// only a hit pays for resolve()'s deposit-shaped perception. This keeps
|
|
197
|
+
// the scan free of river folds — O(1) probes over cheap byte hashes
|
|
198
|
+
// instead of O(suffix) vector folds per offset.
|
|
199
|
+
const leafIds = leafIdPrefix(ctx, bytes);
|
|
200
|
+
for (let i = 1; i <= n - W; i++) {
|
|
201
|
+
if (ctx.store.findBranch(leafIds.slice(i)) === null) continue;
|
|
202
|
+
const id = resolve(ctx, bytes.subarray(i));
|
|
203
|
+
if (id === null || id === src) continue;
|
|
204
|
+
const known = ctx.store.parentsFirst(id, 2).length >= 2 ||
|
|
205
|
+
(ctx.store.haloMass(id) > 0 && ctx.store.hasNext(id));
|
|
206
|
+
if (!known) continue;
|
|
207
|
+
await ctx.store.link(id, dst);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
174
211
|
/** Ingest a pair (context, continuation) — learn an edge and pour halos. */
|
|
175
212
|
export async function ingestPair(
|
|
176
213
|
ctx: MindContext,
|
|
@@ -182,6 +219,7 @@ export async function ingestPair(
|
|
|
182
219
|
const ctxId = c.rootId, contId = cont_.rootId;
|
|
183
220
|
|
|
184
221
|
await ctx.store.link(ctxId, contId);
|
|
222
|
+
await propagateSuffixes(ctx, ctxId, contId);
|
|
185
223
|
|
|
186
224
|
// Halos pour company SIGNATURES (identity), not gists (content) — see
|
|
187
225
|
// companySignature in sema.ts.
|
package/src/mind/match.ts
CHANGED
|
@@ -446,8 +446,10 @@ export async function follow(
|
|
|
446
446
|
* `guide` the context whose gist resonates with the query wins (seat
|
|
447
447
|
* symmetry) — without one, the most-corroborated context wins (poured halo
|
|
448
448
|
* MASS, the direct measure of how many episodes established it), falling
|
|
449
|
-
* back to first-learnt on equal mass.
|
|
450
|
-
*
|
|
449
|
+
* back to first-learnt on equal mass. Among many predecessors RECIPROCAL
|
|
450
|
+
* ones (mutual edges) are preferred when any exist (RC5). Callers that
|
|
451
|
+
* HAVE a query gist must pass it, or they silently change disambiguation
|
|
452
|
+
* regime.
|
|
451
453
|
*
|
|
452
454
|
* `rev`, when the caller has already materialised prevOf (one read per
|
|
453
455
|
* relation — a hub's reverse fan-in is corpus-sized), is reused instead of
|
|
@@ -466,11 +468,32 @@ export function reverseContext(
|
|
|
466
468
|
// keeps the single-predecessor shortcut exact.
|
|
467
469
|
const candidates = rev ?? ctx.store.prevFirst(id, hubBound(ctx));
|
|
468
470
|
if (candidates.length === 0) return null;
|
|
469
|
-
|
|
470
|
-
|
|
471
|
+
// RECIPROCAL PREFERENCE: among many predecessors, one that `id` also
|
|
472
|
+
// continues TO (cand → id AND id → cand both learnt) is a mutually
|
|
473
|
+
// established pairing — the strongest structural evidence a predecessor
|
|
474
|
+
// can carry (bidirectional training deposits both directions of a genuine
|
|
475
|
+
// pair). A bare predecessor is one episode's adjacency; guide-resonance
|
|
476
|
+
// over bare predecessors favours whichever stored document merely
|
|
477
|
+
// CONTAINS the query's bytes (the linear fold's cosine is byte overlap —
|
|
478
|
+
// the observed "merci → unrelated French document" failure). One capped
|
|
479
|
+
// forward read decides; when no reciprocal exists, behaviour is unchanged
|
|
480
|
+
// — bare predecessors ARE the honest answer for a shared deposited
|
|
481
|
+
// continuation (two questions → one answer; audited by 31-audit C1), and
|
|
482
|
+
// this arm serves every mechanism's reverse projection, so abstaining
|
|
483
|
+
// here starves far more than the one containment failure it would fix.
|
|
484
|
+
let pool: readonly number[] = candidates;
|
|
485
|
+
if (candidates.length > 1) {
|
|
486
|
+
const fwd = new Set(ctx.store.nextFirst(id, hubBound(ctx)));
|
|
487
|
+
if (fwd.size > 0) {
|
|
488
|
+
const mutual = candidates.filter((c) => fwd.has(c));
|
|
489
|
+
if (mutual.length > 0) pool = mutual;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
const pick = pool.length === 1
|
|
493
|
+
? pool[0]
|
|
471
494
|
: guide
|
|
472
|
-
? chooseAmong(ctx,
|
|
473
|
-
: pickByMass(ctx,
|
|
495
|
+
? chooseAmong(ctx, pool, guide).id
|
|
496
|
+
: pickByMass(ctx, pool);
|
|
474
497
|
const g = read(ctx, pick);
|
|
475
498
|
return g.length > 0 ? g : null;
|
|
476
499
|
}
|