@hviana/sema 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/train_base.js +42 -9
- package/dist/src/alu/src/parser.d.ts +9 -0
- package/dist/src/alu/src/parser.js +110 -2
- package/dist/src/canon.d.ts +26 -0
- package/dist/src/canon.js +57 -0
- package/dist/src/geometry.d.ts +13 -2
- package/dist/src/geometry.js +101 -2
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/mind/attention.js +11 -7
- package/dist/src/mind/learning.js +33 -1
- package/dist/src/mind/match.d.ts +4 -2
- package/dist/src/mind/match.js +30 -6
- package/dist/src/mind/mechanisms/cast.js +18 -4
- package/dist/src/mind/mechanisms/confluence.js +13 -1
- package/dist/src/mind/mechanisms/recall.js +115 -31
- package/dist/src/mind/mind.d.ts +138 -12
- package/dist/src/mind/mind.js +284 -22
- package/dist/src/mind/primitives.d.ts +22 -2
- package/dist/src/mind/primitives.js +95 -6
- package/dist/src/mind/recognition.js +31 -8
- package/dist/src/mind/traverse.d.ts +13 -0
- package/dist/src/mind/traverse.js +41 -0
- package/dist/src/mind/types.d.ts +33 -21
- package/dist/src/store-sqlite.d.ts +10 -0
- package/dist/src/store-sqlite.js +58 -0
- package/dist/src/store.d.ts +23 -0
- package/dist/src/store.js +13 -2
- package/example/train_base.ts +49 -9
- package/index.html +65 -0
- package/package.json +1 -1
- package/src/alu/src/parser.ts +105 -4
- package/src/canon.ts +65 -0
- package/src/geometry.ts +105 -1
- package/src/index.ts +1 -0
- package/src/mind/attention.ts +11 -6
- package/src/mind/learning.ts +39 -1
- package/src/mind/match.ts +29 -6
- package/src/mind/mechanisms/cast.ts +21 -6
- package/src/mind/mechanisms/confluence.ts +15 -1
- package/src/mind/mechanisms/recall.ts +132 -41
- package/src/mind/mind.ts +396 -22
- package/src/mind/primitives.ts +109 -7
- package/src/mind/recognition.ts +36 -8
- package/src/mind/traverse.ts +47 -0
- package/src/mind/types.ts +30 -21
- package/src/store-sqlite.ts +76 -0
- package/src/store.ts +46 -2
- package/test/13-conversation.test.mjs +240 -20
- package/test/35-prefix-edge.test.mjs +86 -0
package/dist/src/mind/mind.js
CHANGED
|
@@ -10,15 +10,16 @@
|
|
|
10
10
|
// Implementation split across src/mind/*.ts — this file assembles the Mind class.
|
|
11
11
|
import { makeKeyring, rng, setVecConfig } from "../vec.js";
|
|
12
12
|
import { Alphabet } from "../alphabet.js";
|
|
13
|
-
import { reachThreshold, } from "../geometry.js";
|
|
13
|
+
import { bytesToTreePyramid, reachThreshold, } from "../geometry.js";
|
|
14
14
|
import { BoundedMap } from "../store.js";
|
|
15
15
|
import { SQliteStore } from "../store-sqlite.js";
|
|
16
16
|
import { resolveConfig } from "../config.js";
|
|
17
|
-
import {
|
|
17
|
+
import { canonHash, textCanon } from "../canon.js";
|
|
18
|
+
import { bytesEqual, concat2 } from "../bytes.js";
|
|
18
19
|
import { GraphSearch, } from "./graph-search.js";
|
|
19
20
|
import { Alu } from "../alu/src/index.js";
|
|
20
21
|
import { decodeText, Rationale, } from "./rationale.js";
|
|
21
|
-
import { gistOf, inputBytes, perceive as perceiveImpl, resolve as resolveImpl, } from "./primitives.js";
|
|
22
|
+
import { gistOf, inputBytes, latin1Key, perceive as perceiveImpl, resolve as resolveImpl, } from "./primitives.js";
|
|
22
23
|
import { chooseNext, edgeAncestors as edgeAncestorsFn } from "./traverse.js";
|
|
23
24
|
import { follow } from "./match.js";
|
|
24
25
|
import { recognise, segment } from "./recognition.js";
|
|
@@ -42,16 +43,24 @@ export class Mind {
|
|
|
42
43
|
mechanisms = [];
|
|
43
44
|
/** The live rationale tracer for the inference currently in flight, or null. */
|
|
44
45
|
trace = null;
|
|
45
|
-
/**
|
|
46
|
-
* {@link
|
|
47
|
-
*
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
/** The content canonicalizer for the response in flight — see
|
|
47
|
+
* {@link MindContext.canon}. Injected per response by the modality entry
|
|
48
|
+
* point; null when the response carries no equivalence. */
|
|
49
|
+
canon = null;
|
|
50
|
+
/** Per-response canonical-resolution memo — see {@link MindContext.canonMemo}. */
|
|
51
|
+
canonMemo = null;
|
|
52
|
+
/** The Mind-level canon option: a canonicalizer to use for EVERY response,
|
|
53
|
+
* `false` to disable canonical resolution, or null to let each entry
|
|
54
|
+
* point decide (text entry points inject {@link textCanon}). */
|
|
55
|
+
_canonOpt = null;
|
|
56
|
+
/** Memo of the consensus climb — content-keyed. See {@link MindContext.climbMemo}. */
|
|
50
57
|
climbMemo = null;
|
|
51
|
-
/**
|
|
58
|
+
/** Memo of recognise() — content-keyed. See {@link MindContext.recogniseMemo}. */
|
|
52
59
|
recogniseMemo = null;
|
|
53
|
-
/**
|
|
60
|
+
/** Memo of perceive() — content-keyed. See {@link MindContext.perceiveMemo}. */
|
|
54
61
|
perceiveMemo = null;
|
|
62
|
+
/** Subtree-resolution cache. See {@link MindContext._resolvedSubtrees}. */
|
|
63
|
+
_resolvedSubtrees = null;
|
|
55
64
|
/** The perceived gist of the query currently being answered. Set by `think`
|
|
56
65
|
* before the graph search runs; `chooseNext` consults it as a gate (a null
|
|
57
66
|
* guide means no query is in flight, so structural walkers keep plain
|
|
@@ -75,6 +84,9 @@ export class Mind {
|
|
|
75
84
|
_depositTrees = new BoundedMap(8);
|
|
76
85
|
_depositLens = new Set();
|
|
77
86
|
_internIds = new WeakMap();
|
|
87
|
+
// ── Conversation state ──────────────────────────────────────────────────
|
|
88
|
+
_nextConvId = 1;
|
|
89
|
+
_conversations = new Map();
|
|
78
90
|
// ── GraphSearchHost implementation ─────────────────────────────────────
|
|
79
91
|
/** Canonical node id of a byte span. Required by GraphSearchHost & MindContext. */
|
|
80
92
|
resolve(bytes) {
|
|
@@ -103,7 +115,8 @@ export class Mind {
|
|
|
103
115
|
this.store = storeArg;
|
|
104
116
|
}
|
|
105
117
|
else {
|
|
106
|
-
const { store: optsStore, mechanisms: userMechs, mechanismFactories: userFacts, ...rest } = (optsOrCfg ?? {});
|
|
118
|
+
const { store: optsStore, mechanisms: userMechs, mechanismFactories: userFacts, canon: optsCanon, ...rest } = (optsOrCfg ?? {});
|
|
119
|
+
this._canonOpt = optsCanon ?? null;
|
|
107
120
|
this.cfg = resolveConfig(rest);
|
|
108
121
|
this.store = optsStore ?? new SQliteStore({
|
|
109
122
|
maxGroup: this.cfg.geometry.maxGroup,
|
|
@@ -165,11 +178,21 @@ export class Mind {
|
|
|
165
178
|
/** Open one response's transient state — the tracer and the per-response
|
|
166
179
|
* memos. Paired with {@link endResponse}; the ONE place this state is
|
|
167
180
|
* created, so adding a memo cannot forget its reset. */
|
|
168
|
-
beginResponse(inspectRationale) {
|
|
181
|
+
beginResponse(inspectRationale, canon) {
|
|
169
182
|
this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
|
|
170
|
-
this.climbMemo = new
|
|
171
|
-
this.recogniseMemo = new
|
|
183
|
+
this.climbMemo = new Map();
|
|
184
|
+
this.recogniseMemo = new Map();
|
|
172
185
|
this.perceiveMemo = new Map();
|
|
186
|
+
this.canon = canon ?? null;
|
|
187
|
+
this.canonMemo = canon ? new Map() : null;
|
|
188
|
+
}
|
|
189
|
+
/** The canonicalizer a response should carry: the Mind-level option when
|
|
190
|
+
* set (or none when explicitly disabled), else the entry point's own
|
|
191
|
+
* default — text entry points pass {@link textCanon}, binary ones null. */
|
|
192
|
+
_canonFor(entryDefault) {
|
|
193
|
+
if (this._canonOpt === false)
|
|
194
|
+
return null;
|
|
195
|
+
return this._canonOpt ?? entryDefault;
|
|
173
196
|
}
|
|
174
197
|
/** Close one response's transient state — every per-response field, incl.
|
|
175
198
|
* the edge guide/choices `think` sets mid-flight. */
|
|
@@ -178,22 +201,26 @@ export class Mind {
|
|
|
178
201
|
this.climbMemo = null;
|
|
179
202
|
this.recogniseMemo = null;
|
|
180
203
|
this.perceiveMemo = null;
|
|
204
|
+
this.canon = null;
|
|
205
|
+
this.canonMemo = null;
|
|
181
206
|
this._edgeGuide = null;
|
|
182
207
|
this._edgeChoice.clear();
|
|
183
208
|
}
|
|
184
|
-
|
|
185
|
-
|
|
209
|
+
/** Shared response core — the one path from bytes to voiced answer.
|
|
210
|
+
* `respond` calls this directly; `respondTurn` has its own path
|
|
211
|
+
* with conversation-persistent memos and incremental perception. */
|
|
212
|
+
async _respondImpl(queryBytes, inspectRationale, traceLabel = "respond", canon = null) {
|
|
213
|
+
this.beginResponse(inspectRationale, canon);
|
|
186
214
|
try {
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
rItem(inBytes, "query"),
|
|
215
|
+
const top = this.trace?.enter(traceLabel, [
|
|
216
|
+
rItem(queryBytes, "query"),
|
|
190
217
|
]);
|
|
191
|
-
const thought = await think(this,
|
|
218
|
+
const thought = await think(this, queryBytes, this.mechanisms);
|
|
192
219
|
if (thought === null) {
|
|
193
220
|
top?.done([], "nothing to perceive or an empty store — no answer");
|
|
194
221
|
return { v: null, bytes: new Uint8Array(0) };
|
|
195
222
|
}
|
|
196
|
-
const voiced = await articulate(this, thought.bytes,
|
|
223
|
+
const voiced = await articulate(this, thought.bytes, queryBytes);
|
|
197
224
|
top?.done([rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)], "the answer, re-voiced in the asker's words");
|
|
198
225
|
return {
|
|
199
226
|
v: gistOf(this, voiced),
|
|
@@ -205,14 +232,207 @@ export class Mind {
|
|
|
205
232
|
this.endResponse();
|
|
206
233
|
}
|
|
207
234
|
}
|
|
235
|
+
async respond(input, inspectRationale) {
|
|
236
|
+
// A STRING input is text by nature: it carries the text equivalence even
|
|
237
|
+
// through the generic entry point. Raw bytes / grids carry only the
|
|
238
|
+
// Mind-level canon option, if any.
|
|
239
|
+
const canon = this._canonFor(typeof input === "string" ? textCanon : null);
|
|
240
|
+
return this._respondImpl(inputBytes(this, input), inspectRationale, "respond", canon);
|
|
241
|
+
}
|
|
208
242
|
/** Text view of {@link respond}. NUL bytes (0x00) are stripped before
|
|
209
243
|
* decoding — they are structural padding in text answers. LOSSY for a
|
|
210
244
|
* binary answer that legitimately contains NULs: use {@link respond} and
|
|
211
|
-
* read `bytes` directly for binary/grid modalities.
|
|
245
|
+
* read `bytes` directly for binary/grid modalities.
|
|
246
|
+
*
|
|
247
|
+
* Injects the TEXT canonicalizer (src/canon.ts) so resolution treats
|
|
248
|
+
* every character variation of the same text — case, width, whitespace —
|
|
249
|
+
* as one form, provided the store's canon index is built
|
|
250
|
+
* ({@link buildCanonIndex}). */
|
|
212
251
|
async respondText(input, inspectRationale) {
|
|
213
252
|
const r = await this.respond(input, inspectRationale);
|
|
214
253
|
return decodeText(r.bytes);
|
|
215
254
|
}
|
|
255
|
+
// ── Conversation API ────────────────────────────────────────────────────
|
|
256
|
+
/** Begin a new conversation, optionally restoring from a previously-saved
|
|
257
|
+
* {@link ConversationState}. The returned handle is required for
|
|
258
|
+
* {@link respondTurn} and {@link endConversation}.
|
|
259
|
+
*
|
|
260
|
+
* Conversations are independent — a Mind can manage several concurrently.
|
|
261
|
+
* Each tracks the fold pyramid (accumulated internal processing) and
|
|
262
|
+
* turn-boundary offsets; the geometry never inspects content to guess
|
|
263
|
+
* where one turn ends and the next begins. */
|
|
264
|
+
beginConversation(state) {
|
|
265
|
+
const id = this._nextConvId++;
|
|
266
|
+
// Build the initial pyramid: from scratch for a new conversation,
|
|
267
|
+
// or from the saved context bytes when restoring.
|
|
268
|
+
const initBytes = state?.context ?? new Uint8Array(0);
|
|
269
|
+
const { pyramid } = bytesToTreePyramid(this.space, this.alphabet, initBytes);
|
|
270
|
+
this._conversations.set(id, {
|
|
271
|
+
pyramid,
|
|
272
|
+
bytes: initBytes,
|
|
273
|
+
boundaries: state?.boundaries ? [...state.boundaries] : [],
|
|
274
|
+
perceiveMemo: new Map(),
|
|
275
|
+
recogniseMemo: new Map(),
|
|
276
|
+
climbMemo: new Map(),
|
|
277
|
+
resolvedSubtrees: new WeakMap(),
|
|
278
|
+
});
|
|
279
|
+
return { id };
|
|
280
|
+
}
|
|
281
|
+
/** End a conversation, releasing its internal resources (accumulated
|
|
282
|
+
* context, boundary offsets, and the fold-pyramid cache). Idempotent. */
|
|
283
|
+
endConversation(conv) {
|
|
284
|
+
this._conversations.delete(conv.id);
|
|
285
|
+
}
|
|
286
|
+
/** The current serialisable state of an active conversation. Save this
|
|
287
|
+
* to resume the conversation later via {@link beginConversation}. */
|
|
288
|
+
conversationState(conv) {
|
|
289
|
+
const data = this._conversations.get(conv.id);
|
|
290
|
+
if (!data)
|
|
291
|
+
return null;
|
|
292
|
+
return {
|
|
293
|
+
context: data.bytes,
|
|
294
|
+
boundaries: [...data.boundaries],
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
/** Append a turn to a conversation's accumulated context WITHOUT
|
|
298
|
+
* responding — raw byte append plus a boundary offset, never a
|
|
299
|
+
* separator; the fold pyramid advances by O(turn).
|
|
300
|
+
*
|
|
301
|
+
* This is the primitive for turns the Mind should hear but not answer:
|
|
302
|
+
* replaying a transcript, feeding the OTHER speaker's line in a
|
|
303
|
+
* prediction harness, or restoring context piecewise. {@link
|
|
304
|
+
* respondTurn} = addTurn + think + its own reply appended the same way. */
|
|
305
|
+
addTurn(conv, turn) {
|
|
306
|
+
const data = this._conversations.get(conv.id);
|
|
307
|
+
if (!data)
|
|
308
|
+
throw new Error(`Conversation ${conv.id} not found`);
|
|
309
|
+
const turnBytes = inputBytes(this, turn);
|
|
310
|
+
this._growContext(data, turnBytes);
|
|
311
|
+
return this.conversationState(conv);
|
|
312
|
+
}
|
|
313
|
+
/** Grow a conversation's accumulated context by one turn's bytes — raw
|
|
314
|
+
* append plus a boundary offset, pyramid advanced by O(turn), the grown
|
|
315
|
+
* context's tree seeded into the conversation's perceive memo. The ONE
|
|
316
|
+
* place a context grows ({@link addTurn} and {@link respondTurn} both
|
|
317
|
+
* come through here), so the append semantics cannot drift. */
|
|
318
|
+
_growContext(data, turnBytes) {
|
|
319
|
+
const prevLen = data.pyramid.bytes;
|
|
320
|
+
// An empty turn neither grows the context nor marks a boundary —
|
|
321
|
+
// boundaries are documented strictly increasing, and a zero-length
|
|
322
|
+
// "turn" is no turn. The pyramid's zero-growth shortcut makes the
|
|
323
|
+
// refold below O(1), so the existing tree is returned unchanged.
|
|
324
|
+
const grow = turnBytes.length > 0;
|
|
325
|
+
const grown = !grow
|
|
326
|
+
? data.bytes
|
|
327
|
+
: prevLen > 0
|
|
328
|
+
? concat2(data.bytes, turnBytes)
|
|
329
|
+
: turnBytes;
|
|
330
|
+
if (grow && prevLen > 0)
|
|
331
|
+
data.boundaries.push(prevLen);
|
|
332
|
+
const { tree, pyramid } = bytesToTreePyramid(this.space, this.alphabet, grown, data.pyramid);
|
|
333
|
+
data.pyramid = pyramid;
|
|
334
|
+
data.bytes = grown;
|
|
335
|
+
data.perceiveMemo.set(latin1Key(grown), tree);
|
|
336
|
+
return tree;
|
|
337
|
+
}
|
|
338
|
+
/** Process one turn of a conversation.
|
|
339
|
+
*
|
|
340
|
+
* `turn` is the raw input for the latest turn — its bytes are appended
|
|
341
|
+
* to the accumulated context directly (raw concatenation). The Mind
|
|
342
|
+
* tracks the byte offset where each turn ends; no separator is ever
|
|
343
|
+
* inserted or inspected.
|
|
344
|
+
*
|
|
345
|
+
* Returns the response AND the updated {@link ConversationState} so the
|
|
346
|
+
* caller can persist it. The conversation handle's internal state is
|
|
347
|
+
* updated in place — the returned state is a snapshot for storage.
|
|
348
|
+
*
|
|
349
|
+
* SINGLE FLIGHT: at most one respondTurn may be in flight per Mind. The
|
|
350
|
+
* conversation's memo caches are swapped into the Mind-level per-response
|
|
351
|
+
* pointers for the duration of the turn, so a concurrently-running
|
|
352
|
+
* respond()/respondTurn() on the SAME Mind would interleave state.
|
|
353
|
+
* Different Minds (or sequential awaits, as in every test) are safe. */
|
|
354
|
+
async respondTurn(conv, turn, inspectRationale) {
|
|
355
|
+
const data = this._conversations.get(conv.id);
|
|
356
|
+
if (!data)
|
|
357
|
+
throw new Error(`Conversation ${conv.id} not found`);
|
|
358
|
+
const turnBytes = inputBytes(this, turn);
|
|
359
|
+
// Incremental perception — O(turn) instead of O(context).
|
|
360
|
+
const tree = this._growContext(data, turnBytes);
|
|
361
|
+
const newContext = data.bytes;
|
|
362
|
+
// Swap in the conversation's persistent state so the inference
|
|
363
|
+
// pipeline does not re-process the prefix from scratch.
|
|
364
|
+
// perceiveMemo / recogniseMemo / climbMemo — content-keyed;
|
|
365
|
+
// the prefix's results from the previous turn are found by
|
|
366
|
+
// the current turn's sub-span calls.
|
|
367
|
+
// _resolvedSubtrees — Sema-node-keyed; when the pyramid reuses
|
|
368
|
+
// prefix subtrees (identical objects), foldTree returns their
|
|
369
|
+
// ids immediately — O(suffix) instead of O(context).
|
|
370
|
+
this.perceiveMemo = data.perceiveMemo;
|
|
371
|
+
this.recogniseMemo = data.recogniseMemo;
|
|
372
|
+
this.climbMemo = data.climbMemo;
|
|
373
|
+
this._resolvedSubtrees = data.resolvedSubtrees;
|
|
374
|
+
this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
|
|
375
|
+
// A string turn is text by nature — carry the text equivalence, same as
|
|
376
|
+
// respond() (see _canonFor).
|
|
377
|
+
this.canon = this._canonFor(typeof turn === "string" ? textCanon : null);
|
|
378
|
+
this.canonMemo = this.canon ? new Map() : null;
|
|
379
|
+
try {
|
|
380
|
+
const top = this.trace?.enter("respondTurn", [
|
|
381
|
+
rItem(newContext, "query"),
|
|
382
|
+
]);
|
|
383
|
+
const thought = await think(this, newContext, this.mechanisms);
|
|
384
|
+
if (thought === null) {
|
|
385
|
+
top?.done([], "nothing to perceive or an empty store — no answer");
|
|
386
|
+
return {
|
|
387
|
+
response: { v: null, bytes: new Uint8Array(0) },
|
|
388
|
+
state: this.conversationState(conv),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
const voiced = await articulate(this, thought.bytes, newContext);
|
|
392
|
+
top?.done([rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)], "the answer, re-voiced in the asker's words");
|
|
393
|
+
// The REPLY joins the accumulated context the same way a turn does
|
|
394
|
+
// ({@link addTurn}): raw byte append plus a boundary offset — never a
|
|
395
|
+
// separator. A conversation's context is the full exchange, exactly
|
|
396
|
+
// the cumulative continuous shape multi-turn training deposits, so a
|
|
397
|
+
// later turn can refer to what was ANSWERED ("which of those two…"),
|
|
398
|
+
// not only to what was asked.
|
|
399
|
+
if (voiced.length > 0)
|
|
400
|
+
this.addTurn(conv, voiced);
|
|
401
|
+
return {
|
|
402
|
+
response: {
|
|
403
|
+
v: gistOf(this, voiced),
|
|
404
|
+
bytes: voiced,
|
|
405
|
+
provenance: thought.provenance,
|
|
406
|
+
},
|
|
407
|
+
state: this.conversationState(conv),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
finally {
|
|
411
|
+
// No save-back: the conversation's memo MAPS were mutated in place —
|
|
412
|
+
// `data.*` still points at them. Re-assigning from the Mind-level
|
|
413
|
+
// pointers here was a no-op in the single-flight case and, under a
|
|
414
|
+
// concurrently-started respond() (which swaps its own fresh maps into
|
|
415
|
+
// those pointers), would inject a FOREIGN response's memos into this
|
|
416
|
+
// conversation. Clear Mind references — non-conversation respond()
|
|
417
|
+
// calls get fresh per-response memos.
|
|
418
|
+
this.trace = null;
|
|
419
|
+
this.perceiveMemo = null;
|
|
420
|
+
this.recogniseMemo = null;
|
|
421
|
+
this.climbMemo = null;
|
|
422
|
+
this.canon = null;
|
|
423
|
+
this.canonMemo = null;
|
|
424
|
+
this._resolvedSubtrees = null;
|
|
425
|
+
this._edgeGuide = null;
|
|
426
|
+
this._edgeChoice.clear();
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
/** Text view of {@link respondTurn}. See {@link respondText} for the
|
|
430
|
+
* NUL-stripping caveat. For binary or grid turns use {@link respondTurn}
|
|
431
|
+
* directly — this is a text-only convenience, like {@link respondText}. */
|
|
432
|
+
async respondTurnText(conv, turn, inspectRationale) {
|
|
433
|
+
const { response, state } = await this.respondTurn(conv, turn, inspectRationale);
|
|
434
|
+
return { response: decodeText(response.bytes), state };
|
|
435
|
+
}
|
|
216
436
|
async embedding(input) {
|
|
217
437
|
return (await this.respond(input)).v;
|
|
218
438
|
}
|
|
@@ -281,6 +501,48 @@ export class Mind {
|
|
|
281
501
|
return gistOf(this, bytes);
|
|
282
502
|
}, minParents);
|
|
283
503
|
}
|
|
504
|
+
// ── Canonical-form index ───────────────────────────────────────────────
|
|
505
|
+
/** Build (or incrementally refresh) the store's canonical-form index: for
|
|
506
|
+
* every content-bearing node, record the hash of its CANONICAL key so
|
|
507
|
+
* resolution can find stored forms across surface variation (case, width,
|
|
508
|
+
* whitespace — whatever `canon` equates; see src/canon.ts).
|
|
509
|
+
*
|
|
510
|
+
* Incremental and idempotent: the last indexed node id is remembered in
|
|
511
|
+
* store meta (`canon.upto`), so a refresh after further training scans
|
|
512
|
+
* only the new rows. Run once after training, and again after ingests —
|
|
513
|
+
* the same operational shape as {@link repairContentIndex}.
|
|
514
|
+
*
|
|
515
|
+
* @param canon the canonicalizer to index under — MUST be the same one
|
|
516
|
+
* queries will carry (text queries carry {@link textCanon}
|
|
517
|
+
* unless the Mind was constructed with its own)
|
|
518
|
+
* @returns number of index rows added */
|
|
519
|
+
async buildCanonIndex(canon) {
|
|
520
|
+
const c = canon ?? this._canonFor(textCanon);
|
|
521
|
+
const store = this.store;
|
|
522
|
+
if (c === null || !store.canonAdd || !store.eachContent)
|
|
523
|
+
return 0;
|
|
524
|
+
const from = Number(await store.getMeta("canon.upto") ?? 0);
|
|
525
|
+
let added = 0;
|
|
526
|
+
let maxId = from - 1;
|
|
527
|
+
store.eachContent((id, bytes) => {
|
|
528
|
+
if (id > maxId)
|
|
529
|
+
maxId = id;
|
|
530
|
+
const key = c(bytes);
|
|
531
|
+
if (key.length === 0)
|
|
532
|
+
return;
|
|
533
|
+
// Only index content whose canonical key DIFFERS from its raw bytes —
|
|
534
|
+
// an already-canonical span is found by the exact lookup (and by the
|
|
535
|
+
// fallback's own exact probe of the canonical bytes), so indexing it
|
|
536
|
+
// would only add rows.
|
|
537
|
+
if (bytesEqual(key, bytes))
|
|
538
|
+
return;
|
|
539
|
+
store.canonAdd(canonHash(key), id);
|
|
540
|
+
added++;
|
|
541
|
+
}, from);
|
|
542
|
+
await store.setMeta("canon.upto", String(maxId + 1));
|
|
543
|
+
store.commit();
|
|
544
|
+
return added;
|
|
545
|
+
}
|
|
284
546
|
// ── Persistence ──────────────────────────────────────────────────────────
|
|
285
547
|
async save() {
|
|
286
548
|
const meta = new TextEncoder().encode(JSON.stringify(this.cfg));
|
|
@@ -1,9 +1,20 @@
|
|
|
1
1
|
import { Vec } from "../vec.js";
|
|
2
2
|
import { Sema } from "../sema.js";
|
|
3
3
|
import type { Input, MindContext } from "./types.js";
|
|
4
|
+
/** The content key of a byte span — one latin1 char per byte, an exact,
|
|
5
|
+
* collision-free encoding. Spans on the perception path are query-scale
|
|
6
|
+
* (windows, regions, candidate spans), so key construction is far cheaper
|
|
7
|
+
* than the river fold it deduplicates. */
|
|
8
|
+
export declare function latin1Key(bytes: Uint8Array): string;
|
|
4
9
|
/** Perceive input into a content-defined tree (the river fold).
|
|
5
|
-
* Deterministic — identical bytes always produce an identical tree.
|
|
6
|
-
|
|
10
|
+
* Deterministic — identical bytes always produce an identical tree.
|
|
11
|
+
*
|
|
12
|
+
* `boundaries` is an optional sorted list of proper byte offsets where the
|
|
13
|
+
* fold must split so that each prefix segment folds identically to how it
|
|
14
|
+
* folded when it was learned (§10.3 stable-prefix contract). Only the
|
|
15
|
+
* CALLER — who assembled the multi-turn context — knows where those
|
|
16
|
+
* boundaries are; the geometry never guesses them from the bytes. */
|
|
17
|
+
export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
|
|
7
18
|
/** The DEPOSIT-shaped perceive: the PLAIN fold (bit-identical to inference
|
|
8
19
|
* perception — that structural train/inference agreement is load-bearing
|
|
9
20
|
* for exact recall), computed INCREMENTALLY via the fold's level pyramid
|
|
@@ -30,6 +41,15 @@ export declare function foldTree(ctx: MindContext, n: Sema, start: number, visit
|
|
|
30
41
|
* training did — and recover its root bottom-up. Returns null if any part is
|
|
31
42
|
* unknown. */
|
|
32
43
|
export declare function resolve(ctx: MindContext, bytes: Uint8Array): number | null;
|
|
44
|
+
/** Equivalence-class resolution: when the exact content-addressed lookup
|
|
45
|
+
* misses, find a stored node whose CANONICAL key equals the span's — the
|
|
46
|
+
* store's canon index proposes candidates by key hash, and each is verified
|
|
47
|
+
* by re-canonicalizing its bytes (hash-then-verify, like every content
|
|
48
|
+
* lookup). Among verified candidates, one that leads somewhere (has a
|
|
49
|
+
* continuation edge) is preferred; ties break to the lowest id — a corpus
|
|
50
|
+
* property, not a seed property. Null when the response carries no
|
|
51
|
+
* canonicalizer, the store has no canon index, or nothing verifies. */
|
|
52
|
+
export declare function canonResolve(ctx: MindContext, bytes: Uint8Array): number | null;
|
|
33
53
|
/** Walk a perceived tree in POST-ORDER with byte offsets — children before
|
|
34
54
|
* their parent, `visit(node, start, end)` for every node including leaves.
|
|
35
55
|
* Returns the byte end. The one shared traversal the offset-carrying tree
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
// Address — bytes → node (perceive, foldTree, resolve)
|
|
4
4
|
// Read — node → bytes (read)
|
|
5
5
|
import { bytesToTree, bytesToTreePyramid, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
|
|
6
|
+
import { canonHash } from "../canon.js";
|
|
7
|
+
import { bytesEqual } from "../bytes.js";
|
|
6
8
|
import { ALL } from "./types.js";
|
|
7
9
|
// ── Address: bytes → node ──────────────────────────────────────────────
|
|
8
10
|
/** The content key of a byte span — one latin1 char per byte, an exact,
|
|
9
11
|
* collision-free encoding. Spans on the perception path are query-scale
|
|
10
12
|
* (windows, regions, candidate spans), so key construction is far cheaper
|
|
11
13
|
* than the river fold it deduplicates. */
|
|
12
|
-
function latin1Key(bytes) {
|
|
14
|
+
export function latin1Key(bytes) {
|
|
13
15
|
// Batched String.fromCharCode — avoids the O(n²) cost of repeated += on
|
|
14
16
|
// potentially-large query spans, and stays well under the ~65536 arg limit.
|
|
15
17
|
const n = bytes.length;
|
|
@@ -20,8 +22,14 @@ function latin1Key(bytes) {
|
|
|
20
22
|
return s;
|
|
21
23
|
}
|
|
22
24
|
/** Perceive input into a content-defined tree (the river fold).
|
|
23
|
-
* Deterministic — identical bytes always produce an identical tree.
|
|
24
|
-
|
|
25
|
+
* Deterministic — identical bytes always produce an identical tree.
|
|
26
|
+
*
|
|
27
|
+
* `boundaries` is an optional sorted list of proper byte offsets where the
|
|
28
|
+
* fold must split so that each prefix segment folds identically to how it
|
|
29
|
+
* folded when it was learned (§10.3 stable-prefix contract). Only the
|
|
30
|
+
* CALLER — who assembled the multi-turn context — knows where those
|
|
31
|
+
* boundaries are; the geometry never guesses them from the bytes. */
|
|
32
|
+
export function perceive(ctx, input, leafAt, lookup, boundaries) {
|
|
25
33
|
if (typeof input === "string" || input instanceof Uint8Array) {
|
|
26
34
|
const bytes = typeof input === "string"
|
|
27
35
|
? new TextEncoder().encode(input)
|
|
@@ -37,11 +45,11 @@ export function perceive(ctx, input, leafAt, lookup) {
|
|
|
37
45
|
const hit = memo.get(key);
|
|
38
46
|
if (hit !== undefined)
|
|
39
47
|
return hit;
|
|
40
|
-
const tree = bytesToTree(ctx.space, ctx.alphabet, bytes);
|
|
48
|
+
const tree = bytesToTree(ctx.space, ctx.alphabet, bytes, undefined, undefined, boundaries);
|
|
41
49
|
memo.set(key, tree);
|
|
42
50
|
return tree;
|
|
43
51
|
}
|
|
44
|
-
return bytesToTree(ctx.space, ctx.alphabet, bytes);
|
|
52
|
+
return bytesToTree(ctx.space, ctx.alphabet, bytes, undefined, undefined, boundaries);
|
|
45
53
|
}
|
|
46
54
|
return bytesToTree(ctx.space, ctx.alphabet, bytes, leafAt, lookup);
|
|
47
55
|
}
|
|
@@ -105,11 +113,24 @@ export function gistOf(ctx, bytes) {
|
|
|
105
113
|
* node with its byte span and resolved id. Returns the node's byte end and
|
|
106
114
|
* resolved id. */
|
|
107
115
|
export function foldTree(ctx, n, start, visit) {
|
|
116
|
+
// Fast path: subtree already resolved (from a previous conversation turn
|
|
117
|
+
// or an earlier recognition pass). The pyramid reuses prefix subtrees as
|
|
118
|
+
// identical Sema objects, so this cache turns foldTree into O(suffix)
|
|
119
|
+
// instead of O(context) for multi-turn recognition.
|
|
120
|
+
const cached = ctx._resolvedSubtrees?.get(n);
|
|
121
|
+
if (cached !== undefined) {
|
|
122
|
+
const end = start + cached.len;
|
|
123
|
+
visit?.(n, start, end, cached.id);
|
|
124
|
+
return { end, node: cached.id };
|
|
125
|
+
}
|
|
108
126
|
if (n.kids === null) {
|
|
109
127
|
const b = n.leaf ?? new Uint8Array(0);
|
|
110
128
|
const end = start + b.length;
|
|
111
129
|
const node = ctx.store.findLeaf(b);
|
|
112
130
|
visit?.(n, start, end, node);
|
|
131
|
+
if (node !== null && ctx._resolvedSubtrees) {
|
|
132
|
+
ctx._resolvedSubtrees.set(n, { id: node, len: b.length });
|
|
133
|
+
}
|
|
113
134
|
return { end, node };
|
|
114
135
|
}
|
|
115
136
|
let pos = start;
|
|
@@ -125,6 +146,9 @@ export function foldTree(ctx, n, start, visit) {
|
|
|
125
146
|
}
|
|
126
147
|
const node = known ? ctx.store.findBranch(kids) : null;
|
|
127
148
|
visit?.(n, start, pos, node);
|
|
149
|
+
if (node !== null && ctx._resolvedSubtrees) {
|
|
150
|
+
ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
|
|
151
|
+
}
|
|
128
152
|
return { end: pos, node };
|
|
129
153
|
}
|
|
130
154
|
/** The canonical node id of a byte span: perceive it in isolation — the way
|
|
@@ -133,7 +157,72 @@ export function foldTree(ctx, n, start, visit) {
|
|
|
133
157
|
export function resolve(ctx, bytes) {
|
|
134
158
|
if (bytes.length === 0)
|
|
135
159
|
return null;
|
|
136
|
-
|
|
160
|
+
const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
|
|
161
|
+
if (exact !== null)
|
|
162
|
+
return exact;
|
|
163
|
+
return canonResolve(ctx, bytes);
|
|
164
|
+
}
|
|
165
|
+
/** Equivalence-class resolution: when the exact content-addressed lookup
|
|
166
|
+
* misses, find a stored node whose CANONICAL key equals the span's — the
|
|
167
|
+
* store's canon index proposes candidates by key hash, and each is verified
|
|
168
|
+
* by re-canonicalizing its bytes (hash-then-verify, like every content
|
|
169
|
+
* lookup). Among verified candidates, one that leads somewhere (has a
|
|
170
|
+
* continuation edge) is preferred; ties break to the lowest id — a corpus
|
|
171
|
+
* property, not a seed property. Null when the response carries no
|
|
172
|
+
* canonicalizer, the store has no canon index, or nothing verifies. */
|
|
173
|
+
export function canonResolve(ctx, bytes) {
|
|
174
|
+
const canon = ctx.canon;
|
|
175
|
+
const store = ctx.store;
|
|
176
|
+
if (canon === null || !store.canonFind)
|
|
177
|
+
return null;
|
|
178
|
+
if (bytes.length < 2)
|
|
179
|
+
return null;
|
|
180
|
+
const memo = ctx.canonMemo;
|
|
181
|
+
const memoKey = memo ? latin1Key(bytes) : "";
|
|
182
|
+
if (memo) {
|
|
183
|
+
const hit = memo.get(memoKey);
|
|
184
|
+
if (hit !== undefined)
|
|
185
|
+
return hit;
|
|
186
|
+
}
|
|
187
|
+
const set = (v) => {
|
|
188
|
+
memo?.set(memoKey, v);
|
|
189
|
+
return v;
|
|
190
|
+
};
|
|
191
|
+
const key = canon(bytes);
|
|
192
|
+
if (key.length === 0)
|
|
193
|
+
return set(null);
|
|
194
|
+
// A stored form that IS canonical is not in the index (buildCanonIndex
|
|
195
|
+
// skips identity rows) — the exact content-addressed lookup of the
|
|
196
|
+
// canonical bytes finds it directly.
|
|
197
|
+
if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
|
|
198
|
+
const direct = foldTree(ctx, perceive(ctx, key), 0).node;
|
|
199
|
+
if (direct !== null)
|
|
200
|
+
return set(direct);
|
|
201
|
+
}
|
|
202
|
+
const candidates = store.canonFind(canonHash(key));
|
|
203
|
+
if (candidates.length === 0)
|
|
204
|
+
return set(null);
|
|
205
|
+
let best = null;
|
|
206
|
+
let bestLeads = false;
|
|
207
|
+
for (const id of candidates) {
|
|
208
|
+
const bytesOf = read(ctx, id);
|
|
209
|
+
const stored = canon(bytesOf);
|
|
210
|
+
if (stored.length !== key.length || !bytesEqual(stored, key))
|
|
211
|
+
continue;
|
|
212
|
+
// The index stores FLAT content twins; the id the exact path would have
|
|
213
|
+
// resolved for these bytes is their FOLD — the deposit-shaped node that
|
|
214
|
+
// carries the edges and halos. Re-folding the candidate's bytes lands
|
|
215
|
+
// on exactly the node the canonical-case query would have found.
|
|
216
|
+
const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
|
|
217
|
+
const use = folded ?? id;
|
|
218
|
+
const leads = store.hasNext(use) || store.haloMass(use) > 0;
|
|
219
|
+
if (best === null || (leads && !bestLeads) ||
|
|
220
|
+
(leads === bestLeads && use < best)) {
|
|
221
|
+
best = use;
|
|
222
|
+
bestLeads = leads;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return set(best);
|
|
137
226
|
}
|
|
138
227
|
/** Walk a perceived tree in POST-ORDER with byte offsets — children before
|
|
139
228
|
* their parent, `visit(node, start, end)` for every node including leaves.
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
// that leads somewhere (has a continuation edge or a halo).
|
|
6
6
|
// segment — leaf-parent segmentation using the geometry's own groupings.
|
|
7
7
|
import { rItem } from "./trace.js";
|
|
8
|
-
import { foldTree, gistOf, perceive, resolve } from "./primitives.js";
|
|
9
|
-
import { leadsSomewhere } from "./traverse.js";
|
|
8
|
+
import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
|
|
9
|
+
import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
|
|
10
10
|
import { chainReach, leafIdAt } from "./canonical.js";
|
|
11
11
|
import { isChunk } from "../sema.js";
|
|
12
12
|
/** Decompose a byte stream into every stored form that leads somewhere
|
|
@@ -22,16 +22,16 @@ import { isChunk } from "../sema.js";
|
|
|
22
22
|
*
|
|
23
23
|
* Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
|
|
24
24
|
export function recognise(ctx, bytes) {
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
// emits its rationale step.
|
|
25
|
+
// Content-keyed memo — works for both single-turn respond() and multi-turn
|
|
26
|
+
// respondTurn() (where the map persists across calls). Skipped while
|
|
27
|
+
// tracing so every call still emits its rationale step.
|
|
29
28
|
if (ctx.recogniseMemo && !ctx.trace) {
|
|
30
|
-
const
|
|
29
|
+
const key = latin1Key(bytes);
|
|
30
|
+
const hit = ctx.recogniseMemo.get(key);
|
|
31
31
|
if (hit !== undefined)
|
|
32
32
|
return hit;
|
|
33
33
|
const fresh = recogniseImpl(ctx, bytes);
|
|
34
|
-
ctx.recogniseMemo.set(
|
|
34
|
+
ctx.recogniseMemo.set(key, fresh);
|
|
35
35
|
return fresh;
|
|
36
36
|
}
|
|
37
37
|
return recogniseImpl(ctx, bytes);
|
|
@@ -59,7 +59,20 @@ function recogniseImpl(ctx, bytes) {
|
|
|
59
59
|
}
|
|
60
60
|
return id;
|
|
61
61
|
};
|
|
62
|
+
// Byte atoms (implicit negative-id single-byte leaves) are admitted as
|
|
63
|
+
// recognised sites only while atoms can still DISCRIMINATE at this corpus
|
|
64
|
+
// scale (see {@link atomIsHub}). On a small store a single-letter fact
|
|
65
|
+
// ("a" → "A") is genuine learnt content and its site is essential; on a
|
|
66
|
+
// large one every letter of every query would otherwise become a
|
|
67
|
+
// "recognised form" — the bridge then finds junction connectors between
|
|
68
|
+
// bare letters, cover follows edges hanging off them, and pure noise
|
|
69
|
+
// ("qq8f3kz9…") grounds to an arbitrary learnt sentence instead of
|
|
70
|
+
// silence. Atoms stay available as leaves (PASS-carried literals) and
|
|
71
|
+
// through exact tier-0 resolution regardless.
|
|
72
|
+
const atomsAreHubs = atomIsHub(ctx, corpusN(ctx));
|
|
62
73
|
const emit = (start, end, id) => {
|
|
74
|
+
if (id < 0 && atomsAreHubs)
|
|
75
|
+
return;
|
|
63
76
|
if (leadsSomewhere(ctx, id)) {
|
|
64
77
|
sites.push({ start, end, payload: id });
|
|
65
78
|
}
|
|
@@ -73,6 +86,16 @@ function recogniseImpl(ctx, bytes) {
|
|
|
73
86
|
}
|
|
74
87
|
if (node !== null)
|
|
75
88
|
emit(start, end, node);
|
|
89
|
+
// Canonical fallback: a subtree whose exact content-addressed lookup
|
|
90
|
+
// missed may still be a stored form under the response's equivalence
|
|
91
|
+
// (case, width, whitespace — whatever the injected canonicalizer says).
|
|
92
|
+
// O(subtree bytes) per miss, memoised per response; a no-op when no
|
|
93
|
+
// canonicalizer was injected or the store has no canon index.
|
|
94
|
+
else if (end - start >= 2) {
|
|
95
|
+
const cid = canonResolve(ctx, bytes.subarray(start, end));
|
|
96
|
+
if (cid !== null)
|
|
97
|
+
emit(start, end, cid);
|
|
98
|
+
}
|
|
76
99
|
if (isChunk(n)) {
|
|
77
100
|
starts.add(start);
|
|
78
101
|
// Try every sub-span within this leaf-parent.
|