@hviana/sema 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/AGENTS.md +470 -0
  2. package/AUTHORS.md +12 -0
  3. package/COMMERCIAL-LICENSE.md +19 -0
  4. package/CONTRIBUTING.md +15 -0
  5. package/HOW_IT_WORKS.md +4210 -0
  6. package/LICENSE.md +117 -0
  7. package/README.md +290 -0
  8. package/TRADEMARKS.md +19 -0
  9. package/example/demo.ts +46 -0
  10. package/example/train_base.ts +2675 -0
  11. package/index.html +893 -0
  12. package/package.json +25 -0
  13. package/src/alphabet.ts +34 -0
  14. package/src/alu/README.md +332 -0
  15. package/src/alu/src/alu.ts +541 -0
  16. package/src/alu/src/expr.ts +339 -0
  17. package/src/alu/src/index.ts +115 -0
  18. package/src/alu/src/kernel-arith.ts +377 -0
  19. package/src/alu/src/kernel-bits.ts +199 -0
  20. package/src/alu/src/kernel-logic.ts +102 -0
  21. package/src/alu/src/kernel-nd.ts +235 -0
  22. package/src/alu/src/kernel-numeric.ts +466 -0
  23. package/src/alu/src/operation.ts +344 -0
  24. package/src/alu/src/parser.ts +574 -0
  25. package/src/alu/src/resonance.ts +161 -0
  26. package/src/alu/src/text.ts +83 -0
  27. package/src/alu/src/value.ts +327 -0
  28. package/src/alu/test/alu.test.ts +1004 -0
  29. package/src/bytes.ts +62 -0
  30. package/src/config.ts +227 -0
  31. package/src/derive/README.md +295 -0
  32. package/src/derive/src/deduction.ts +263 -0
  33. package/src/derive/src/index.ts +24 -0
  34. package/src/derive/src/priority-queue.ts +70 -0
  35. package/src/derive/src/rewrite.ts +127 -0
  36. package/src/derive/src/trie.ts +242 -0
  37. package/src/derive/test/derive.test.ts +137 -0
  38. package/src/extension.ts +42 -0
  39. package/src/geometry.ts +511 -0
  40. package/src/index.ts +38 -0
  41. package/src/ingest-cache.ts +224 -0
  42. package/src/mind/articulation.ts +137 -0
  43. package/src/mind/attention.ts +1061 -0
  44. package/src/mind/canonical.ts +107 -0
  45. package/src/mind/graph-search.ts +1100 -0
  46. package/src/mind/index.ts +18 -0
  47. package/src/mind/junction.ts +347 -0
  48. package/src/mind/learning.ts +246 -0
  49. package/src/mind/match.ts +507 -0
  50. package/src/mind/mechanisms/alu.ts +33 -0
  51. package/src/mind/mechanisms/cast.ts +568 -0
  52. package/src/mind/mechanisms/confluence.ts +268 -0
  53. package/src/mind/mechanisms/cover.ts +248 -0
  54. package/src/mind/mechanisms/extraction.ts +422 -0
  55. package/src/mind/mechanisms/recall.ts +241 -0
  56. package/src/mind/mind.ts +483 -0
  57. package/src/mind/pipeline-mechanism.ts +326 -0
  58. package/src/mind/pipeline.ts +300 -0
  59. package/src/mind/primitives.ts +201 -0
  60. package/src/mind/rationale.ts +275 -0
  61. package/src/mind/reasoning.ts +198 -0
  62. package/src/mind/recognition.ts +234 -0
  63. package/src/mind/resonance.ts +0 -0
  64. package/src/mind/trace.ts +108 -0
  65. package/src/mind/traverse.ts +556 -0
  66. package/src/mind/types.ts +281 -0
  67. package/src/rabitq-hnsw/README.md +303 -0
  68. package/src/rabitq-hnsw/src/database.ts +492 -0
  69. package/src/rabitq-hnsw/src/heap.ts +90 -0
  70. package/src/rabitq-hnsw/src/hnsw.ts +514 -0
  71. package/src/rabitq-hnsw/src/index.ts +15 -0
  72. package/src/rabitq-hnsw/src/prng.ts +39 -0
  73. package/src/rabitq-hnsw/src/rabitq.ts +308 -0
  74. package/src/rabitq-hnsw/src/store.ts +994 -0
  75. package/src/rabitq-hnsw/test/hnsw.test.ts +1213 -0
  76. package/src/store-sqlite.ts +928 -0
  77. package/src/store.ts +2148 -0
  78. package/src/vec.ts +124 -0
  79. package/test/00-extract.test.mjs +151 -0
  80. package/test/01-floor.test.mjs +20 -0
  81. package/test/02-roundtrip.test.mjs +83 -0
  82. package/test/03-recall.test.mjs +98 -0
  83. package/test/04-think.test.mjs +627 -0
  84. package/test/05-concepts.test.mjs +84 -0
  85. package/test/08-storage.test.mjs +948 -0
  86. package/test/09-edges.test.mjs +65 -0
  87. package/test/11-universality.test.mjs +180 -0
  88. package/test/13-conversation.test.mjs +228 -0
  89. package/test/14-scaling.test.mjs +503 -0
  90. package/test/15-decomposition-gap.test.mjs +0 -0
  91. package/test/16-bridge.test.mjs +250 -0
  92. package/test/17-intelligence.test.mjs +240 -0
  93. package/test/18-alu.test.mjs +209 -0
  94. package/test/19-nd.test.mjs +254 -0
  95. package/test/20-stability.test.mjs +489 -0
  96. package/test/21-partial.test.mjs +158 -0
  97. package/test/22-multihop.test.mjs +130 -0
  98. package/test/23-rationale.test.mjs +316 -0
  99. package/test/24-generalization.test.mjs +416 -0
  100. package/test/25-embedding.test.mjs +75 -0
  101. package/test/26-cooperative.test.mjs +89 -0
  102. package/test/27-saturation-drop.test.mjs +637 -0
  103. package/test/28-unknowable.test.mjs +88 -0
  104. package/test/29-counterfactual.test.mjs +476 -0
  105. package/test/30-conflict-resolution.test.mjs +166 -0
  106. package/test/31-audit.test.mjs +288 -0
  107. package/test/32-confluence.test.mjs +173 -0
  108. package/test/33-multi-candidate.test.mjs +222 -0
  109. package/test/34-cross-region.test.mjs +252 -0
  110. package/tsconfig.json +16 -0
@@ -0,0 +1,18 @@
1
+ // mind/index.ts — public surface of the mind module.
2
+ //
3
+ // Re-exports the Mind class and all public types that were previously
4
+ // exported from mind/mind.ts directly.
5
+
6
+ export { Mind } from "./mind.js";
7
+ export type { Input, Response } from "./mind.js";
8
+ export type { ComputedSpan, ExtensionHost } from "./mind.js";
9
+ export type {
10
+ MechanismResult,
11
+ PipelineMechanism,
12
+ Precomputed,
13
+ } from "./pipeline-mechanism.js";
14
+ export type {
15
+ InspectRationale,
16
+ RationaleItem,
17
+ RationaleStep,
18
+ } from "./rationale.js";
@@ -0,0 +1,347 @@
1
+ // junction.ts — content-addressed junction search (bridge Tier 1).
2
+ //
3
+ // "Which learnt wholes ran L and R together?" answered by DAG ascent, not a
4
+ // similarity guess: hash-consing means any deposit containing L's bytes shares
5
+ // L's node (or L's canonical-window ids — position-independent identities), so
6
+ // climbing parents + containment links from L's and R's seeds reaches every
7
+ // container that literally holds L-then-R. A resonance seed (the gist of the
8
+ // bare concatenation — an object never learnt) could rank the true container
9
+ // out of its top-k; the ascent cannot.
10
+ //
11
+ // Extracted from resonance.ts so BOTH the bridge (a connector between two
12
+ // adjacent ANSWER pieces) and cross-region attention (the joint CONTEXT of two
13
+ // non-adjacent QUERY regions) ascend by the same disciplined, bounded walk.
14
+
15
+ import type { MindContext } from "./types.js";
16
+ import { read, resolve } from "./primitives.js";
17
+ import { windowIds } from "./canonical.js";
18
+ import { hubBound } from "./traverse.js";
19
+ import { haloSiblings } from "./match.js";
20
+ import { indexOf } from "../bytes.js";
21
+
22
+ export interface Junction {
23
+ /** The node whose learnt bytes evidence this junction (a container form,
24
+ * a continuation, or a context). */
25
+ id: number;
26
+ /** The bytes that belong between left and right. */
27
+ interior: Uint8Array;
28
+ }
29
+
30
+ /** Seed node ids to ascend from for one side of a junction: the side's own
31
+ * node when it is a stored form, plus — when the node has no structural
32
+ * parents — its canonical window ids. A non-W-aligned node may have no
33
+ * parents, but its constituent W-grams typically do; the window ids
34
+ * provide alternative ascent paths. The `parentsFirst(…, 1)` probe is a
35
+ * single indexed lookup, far cheaper than computing every window id, so
36
+ * the window-id path is only taken when the node alone cannot ascend.
37
+ * Exported for callers (synonym junctions) that hold one side FIXED across
38
+ * several calls and so compute its seeds once instead of per call. */
39
+ export function junctionSeeds(ctx: MindContext, b: Uint8Array): number[] {
40
+ const r = resolve(ctx, b);
41
+ if (r !== null) {
42
+ if (ctx.store.parentsFirst(r, 1).length > 0) return [r];
43
+ const wids = [...windowIds(ctx, b).values()];
44
+ return [r, ...wids];
45
+ }
46
+ const wids = [...windowIds(ctx, b).values()];
47
+ if (wids.length <= 2) return wids;
48
+ return [wids[0], wids[wids.length - 1]];
49
+ }
50
+
51
+ /** Per-response cache of the identity walks' pure reads (capped bytes,
52
+ * parent pages, container pages), keyed by the response lifecycle object
53
+ * (ctx.climbMemo). One response issues many walks whose ancestries overlap
54
+ * heavily (pair sides repeat across combos, and synonym walks revisit the
55
+ * same neighbourhoods); the store is read-only while a response is in flight,
56
+ * so every one of these reads is a pure function of the id — repeats cost a
57
+ * Map hit instead of a SQL statement or a byte reconstruction. */
58
+ export interface WalkCache {
59
+ /** id → prefix bytes read so far + whether they are the COMPLETE bytes
60
+ * (shorter than the cap that read them). */
61
+ reads: Map<number, { b: Uint8Array; complete: boolean }>;
62
+ parents: Map<number, number[]>;
63
+ containers: Map<number, number[]>;
64
+ }
65
+ const walkCaches = new WeakMap<object, WalkCache>();
66
+ export function walkCache(ctx: MindContext): WalkCache | null {
67
+ if (ctx.climbMemo === null) return null;
68
+ let c = walkCaches.get(ctx.climbMemo);
69
+ if (c === undefined) {
70
+ walkCaches.set(
71
+ ctx.climbMemo,
72
+ c = { reads: new Map(), parents: new Map(), containers: new Map() },
73
+ );
74
+ }
75
+ return c;
76
+ }
77
+ export function cachedRead(
78
+ ctx: MindContext,
79
+ cache: WalkCache | null,
80
+ id: number,
81
+ cap: number,
82
+ ): Uint8Array {
83
+ if (cache === null) return read(ctx, id, cap + 1);
84
+ const hit = cache.reads.get(id);
85
+ // A cached COMPLETE read serves any cap; a cached truncated read serves
86
+ // any cap it already covers (the caller only checks `length > cap`).
87
+ if (hit !== undefined && (hit.complete || hit.b.length > cap)) return hit.b;
88
+ const b = read(ctx, id, cap + 1);
89
+ cache.reads.set(id, { b, complete: b.length <= cap });
90
+ return b;
91
+ }
92
+ function cachedParents(
93
+ ctx: MindContext,
94
+ cache: WalkCache | null,
95
+ id: number,
96
+ limit: number,
97
+ ): number[] {
98
+ if (cache === null) return ctx.store.parentsFirst(id, limit);
99
+ let v = cache.parents.get(id);
100
+ if (v === undefined) {
101
+ v = ctx.store.parentsFirst(id, limit);
102
+ cache.parents.set(id, v);
103
+ }
104
+ return v;
105
+ }
106
+ function cachedContainers(
107
+ ctx: MindContext,
108
+ cache: WalkCache | null,
109
+ id: number,
110
+ limit: number,
111
+ ): number[] {
112
+ if (cache === null) return ctx.store.containersSlice(id, 0, limit);
113
+ let v = cache.containers.get(id);
114
+ if (v === undefined) {
115
+ v = ctx.store.containersSlice(id, 0, limit);
116
+ cache.containers.set(id, v);
117
+ }
118
+ return v;
119
+ }
120
+
121
+ /** Tier 1 body, parameterised on already-resolved seed lists so a caller
122
+ * holding one side FIXED across several calls (synonym junctions) pays for
123
+ * that side's seeds once, not once per call. The byte-containment check
124
+ * below ensures only genuine containers are returned regardless of seeds.
125
+ *
126
+ * BOUNDED at corpus scale by three disciplines (profiled on a 17.7M-node
127
+ * store, where the unbounded form spent >90% of a query's CPU here):
128
+ * • PHRASE-SCALE READS — a junction container is by contract a whole the
129
+ * pair nearly exhausts (glue from a period to a phrase), so every visit
130
+ * reads at most `maxContainer + 1` bytes (`bytesPrefix` stops early).
131
+ * A node whose bytes exceed the cap cannot be a junction container, and
132
+ * its ancestors are strictly larger — the branch is PRUNED, never
133
+ * reconstructing a corpus-sized deposit (an oasst2 conversation) just
134
+ * to reject it.
135
+ * • EXPANSION BUDGET — at most √N·W nodes are popped in total: a √N-wide
136
+ * frontier (the one fan-out convention) through the ~W structural
137
+ * levels that separate phrase-scale content from its containers
138
+ * (perception trees are W-ary, so a junction container lies within a
139
+ * few levels of its parts). A side too common to decide within the
140
+ * budget abstains here and falls through to the resonance tier (the
141
+ * climb's own saturation semantics).
142
+ * • per-node hub guards — parent fan-outs beyond √N are hubs (not
143
+ * expanded); each node contributes at most one √N page of containers;
144
+ * √N collected candidates decide. */
145
+ export function junctionContainersFrom(
146
+ ctx: MindContext,
147
+ left: Uint8Array,
148
+ right: Uint8Array,
149
+ maxContainer: number,
150
+ leftSeeds: number[],
151
+ rightSeeds: number[],
152
+ /** Shared expansion budget — a TIER's √N pops, not each walk's, when one
153
+ * tier issues several walks (synonym junctions try up to 2·haloQueryK
154
+ * siblings; without a shared budget each sibling would spend its own √N). */
155
+ budget?: { n: number },
156
+ /** ORDER-FREE containment: also accept containers holding right-then-left.
157
+ * A junction is evidence that the two forms were LEARNT TOGETHER; which
158
+ * one the query happened to mention first is a fact about the query, not
159
+ * about the learnt whole. The walk is identical (the seed ascent does not
160
+ * depend on order) — only the byte-containment test gains a second probe,
161
+ * so order-freedom costs two indexOf calls per visited node, never a
162
+ * second walk. */
163
+ unordered = false,
164
+ ): Junction[] {
165
+ const bound = hubBound(ctx);
166
+ const joinedLength = left.length + right.length;
167
+ const seeds = [...new Set([...leftSeeds, ...rightSeeds])];
168
+ if (seeds.length === 0) return [];
169
+
170
+ const b = budget ?? { n: bound * ctx.space.maxGroup };
171
+ // DEPTH CAP: perception trees are W-ary and a junction container is
172
+ // phrase-scale, so it sits within ~log_W(maxContainer) structural levels
173
+ // of its parts — at most W levels for any practical W (plus the
174
+ // containment hop the seeds already are). Ancestry beyond that depth is
175
+ // strictly larger than any admissible container; walking it can only burn
176
+ // budget, never find a junction.
177
+ const maxDepth = ctx.space.maxGroup;
178
+ const out: Junction[] = [];
179
+ const cache = walkCache(ctx);
180
+ const seen = new Set<number>(seeds);
181
+ const stack: Array<{ id: number; d: number }> = seeds.map((id) => ({
182
+ id,
183
+ d: 0,
184
+ }));
185
+ while (stack.length > 0 && out.length < bound && b.n-- > 0) {
186
+ const { id: x, d } = stack.pop()!;
187
+ const f = cachedRead(ctx, cache, x, maxContainer);
188
+ if (f.length > maxContainer) continue; // beyond phrase scale: prune branch
189
+ if (unordered) {
190
+ // Order-free containment does NOT require disjoint occurrences: two
191
+ // grid-aligned fragments of the same whole legitimately OVERLAP inside
192
+ // it ("red " at 0 and " cir" at 3 in `red circle`), and both being
193
+ // literal substrings is the evidence. The interior is the gap between
194
+ // them when they are disjoint, empty otherwise. Only the containment
195
+ // test differs from the ordered form — and because occurrences may
196
+ // overlap or abut, `f.length > joinedLength` is too strict (grid
197
+ // fragments of one whole sum past it; "red " + "circle" exactly equals
198
+ // `red circle`). The container must be a STRICT super-form of each
199
+ // side, so that holding both is more than restating either.
200
+ const li = indexOf(f, left, 0);
201
+ const ri = li >= 0 ? indexOf(f, right, 0) : -1;
202
+ if (
203
+ li >= 0 && ri >= 0 && f.length > Math.max(left.length, right.length)
204
+ ) {
205
+ const lo = Math.min(li + left.length, ri + right.length);
206
+ const hi = Math.max(li, ri);
207
+ out.push({
208
+ id: x,
209
+ interior: lo < hi ? f.subarray(lo, hi) : f.subarray(0, 0),
210
+ });
211
+ }
212
+ } else if (f.length > joinedLength) {
213
+ const li = indexOf(f, left, 0);
214
+ if (li >= 0) {
215
+ const ri = indexOf(f, right, li + left.length);
216
+ if (ri >= 0) {
217
+ out.push({ id: x, interior: f.subarray(li + left.length, ri) });
218
+ }
219
+ }
220
+ }
221
+ if (d >= maxDepth) continue; // deeper ancestry is beyond phrase scale
222
+ const parents = cachedParents(ctx, cache, x, bound + 1);
223
+ if (parents.length <= bound) { // beyond √N parents: a hub, not expanded
224
+ for (const p of parents) {
225
+ if (!seen.has(p)) {
226
+ seen.add(p);
227
+ stack.push({ id: p, d: d + 1 });
228
+ }
229
+ }
230
+ }
231
+ // Containment fan-out under the SAME hub reading as parents: a node
232
+ // whose containers fill a whole √N page is COMMON content — its
233
+ // containment ancestry reaches a non-discriminative slice of the corpus
234
+ // (the climb's saturation semantics), and walking it would spend the
235
+ // entire budget discovering nothing a junction could use. Such a node
236
+ // is not expanded through containment; a pair whose sides are common
237
+ // abstains here in a handful of pops and falls through to the resonance
238
+ // tier. Below the page bound the read IS the full container list, so
239
+ // the walk stays exact exactly where identity evidence discriminates.
240
+ const containers = cachedContainers(ctx, cache, x, bound);
241
+ if (containers.length < bound) {
242
+ for (const c of containers) {
243
+ if (!seen.has(c)) {
244
+ seen.add(c);
245
+ stack.push({ id: c, d: d + 1 });
246
+ }
247
+ }
248
+ }
249
+ }
250
+ return out;
251
+ }
252
+
253
+ /** Tier 1 entry point: every learnt whole that literally contains
254
+ * left-then-right, found by ascending the structural DAG (parents +
255
+ * containment links) from the two sides' content-addressed identities.
256
+ * Both sides' seeds resolved fresh, one call. */
257
+ export function junctionContainers(
258
+ ctx: MindContext,
259
+ left: Uint8Array,
260
+ right: Uint8Array,
261
+ maxContainer: number,
262
+ unordered = false,
263
+ ): Junction[] {
264
+ return junctionContainersFrom(
265
+ ctx,
266
+ left,
267
+ right,
268
+ maxContainer,
269
+ junctionSeeds(ctx, left),
270
+ junctionSeeds(ctx, right),
271
+ undefined,
272
+ unordered,
273
+ );
274
+ }
275
+
276
+ /** Tier 2.5: synonym junctions — the container ascent (tier 1) applied to
277
+ * halo siblings of left and right. When a distributional synonym of one
278
+ * form participates in a learnt whole with the other form, the container
279
+ * between the synonym and the other side is valid evidence for the
280
+ * original pair. The container evidence is exact (content-addressed DAG
281
+ * ascent, with window-id-enhanced seeds so non-W-aligned siblings still
282
+ * ascend); the relaxation is only in which form occupies one side — a
283
+ * distributional sibling rather than the exact form.
284
+ *
285
+ * ONE expansion budget is shared by every sibling walk in this call, so
286
+ * cost is bounded at √N·W pops total regardless of how many siblings are
287
+ * tried. A sibling whose bytes exceed `maxInterior` is skipped (it
288
+ * cannot be junction-sized). */
289
+ export async function junctionSynonyms(
290
+ ctx: MindContext,
291
+ left: Uint8Array,
292
+ right: Uint8Array,
293
+ maxInterior: number,
294
+ unordered = false,
295
+ ): Promise<Junction[]> {
296
+ const out: Junction[] = [];
297
+
298
+ const lId = resolve(ctx, left);
299
+ const rId = resolve(ctx, right);
300
+ if (lId === null && rId === null) return out;
301
+
302
+ const budget = { n: hubBound(ctx) * ctx.space.maxGroup };
303
+
304
+ // Left-side synonyms: containers of sibling+right. `right`'s seeds are
305
+ // FIXED across every sibling this loop tries.
306
+ if (lId !== null) {
307
+ const rightSeeds = junctionSeeds(ctx, right);
308
+ for (const sib of await haloSiblings(ctx, lId)) {
309
+ const sibBytes = read(ctx, sib.id, maxInterior + 1);
310
+ if (sibBytes.length === 0 || sibBytes.length > maxInterior) continue;
311
+ const containers = junctionContainersFrom(
312
+ ctx,
313
+ sibBytes,
314
+ right,
315
+ sibBytes.length + right.length + maxInterior,
316
+ junctionSeeds(ctx, sibBytes),
317
+ rightSeeds,
318
+ budget,
319
+ unordered,
320
+ );
321
+ for (const c of containers) out.push(c);
322
+ }
323
+ }
324
+
325
+ // Right-side synonyms: containers of left+sibling. `left`'s seeds are
326
+ // likewise fixed across this loop.
327
+ if (rId !== null) {
328
+ const leftSeeds = junctionSeeds(ctx, left);
329
+ for (const sib of await haloSiblings(ctx, rId)) {
330
+ const sibBytes = read(ctx, sib.id, maxInterior + 1);
331
+ if (sibBytes.length === 0 || sibBytes.length > maxInterior) continue;
332
+ const containers = junctionContainersFrom(
333
+ ctx,
334
+ left,
335
+ sibBytes,
336
+ left.length + sibBytes.length + maxInterior,
337
+ leftSeeds,
338
+ junctionSeeds(ctx, sibBytes),
339
+ budget,
340
+ unordered,
341
+ );
342
+ for (const c of containers) out.push(c);
343
+ }
344
+ }
345
+
346
+ return out;
347
+ }
@@ -0,0 +1,246 @@
1
+ // learning.ts — ingest and deposition (Section 7 of the mind).
2
+ //
3
+ // Learning is DEPOSITION: perceive a stream into a tree and intern every
4
+ // node. A fact is an EDGE between node ids; recall traverses edges.
5
+
6
+ import { Vec } from "../vec.js";
7
+ import { bindSeat, companySignature, isChunk, Sema } from "../sema.js";
8
+ import type { Input, MindContext } from "./types.js";
9
+ import { changedNodes } from "./types.js";
10
+ import { inputBytes, perceive, perceiveDeposit } from "./primitives.js";
11
+ import { canonicalWindows, leafIdPrefix } from "./canonical.js";
12
+ import { fold as foldVecs } from "../sema.js";
13
+
14
+ /** Intern a perceived tree into node ids, bottom-up, sharing equal subtrees.
15
+ * Returns the root node id and a map from tree nodes to their ids.
16
+ *
17
+ * Memoized by NODE IDENTITY (ctx._internIds): the pyramid fold shares a
18
+ * prefix's subtree OBJECTS across an accumulated context's deposits, and a
19
+ * node already interned needs nothing again — its id is permanent
20
+ * (content-addressed) and its intern-time side effects (gist capture, kid
21
+ * rows) fired at first mint; re-interning was pure lookups. A memo hit
22
+ * therefore skips the WHOLE shared subtree, making the intern walk
23
+ * O(new nodes) per deposit instead of O(context). Only the hit node
24
+ * itself enters `ids`; descendants stay reachable via the memo (see
25
+ * idOf in indexSubSpans and the changedNodes prune). */
26
+ export async function internTreeIds(
27
+ ctx: MindContext,
28
+ node: Sema,
29
+ ids: Map<Sema, number>,
30
+ ): Promise<number> {
31
+ const known = ctx._internIds.get(node);
32
+ if (known !== undefined) {
33
+ ids.set(node, known);
34
+ return known;
35
+ }
36
+ let id: number;
37
+ if (node.kids === null) {
38
+ id = await ctx.store.putLeaf(node.leaf ?? new Uint8Array(0), node.v);
39
+ } else {
40
+ const kds: number[] = [];
41
+ for (const k of node.kids) kds.push(await internTreeIds(ctx, k, ids));
42
+ id = await ctx.store.putBranch(kds, node.v);
43
+ }
44
+ ids.set(node, id);
45
+ ctx._internIds.set(node, id);
46
+ return id;
47
+ }
48
+
49
+ /** Index flat branches for sub-spans of a deposit's byte stream, linked to
50
+ * their structural chunks via durable CONTAINMENT edges. */
51
+ export async function indexSubSpans(
52
+ ctx: MindContext,
53
+ tree: Sema,
54
+ ids: Map<Sema, number>,
55
+ ): Promise<boolean> {
56
+ const chunkOf: Array<number | undefined> = [];
57
+ const streamIds: number[] = [];
58
+ const streamVecs: Vec[] = [];
59
+ const collect = (n: Sema): boolean => {
60
+ if (isChunk(n)) {
61
+ // A chunk inside a memo-skipped shared subtree is absent from `ids`;
62
+ // the intern memo still knows it (same object). A miss on both (the
63
+ // WeakMap entry was collected) only forfeits the seenBefore skip.
64
+ const chunkId = ids.get(n) ?? ctx._internIds.get(n);
65
+ for (const k of n.kids) {
66
+ const lid = k.leaf ? ctx.store.findLeaf(k.leaf) : null;
67
+ if (lid === null) return false;
68
+ streamIds.push(lid);
69
+ streamVecs.push(k.v);
70
+ chunkOf.push(chunkId);
71
+ }
72
+ return true;
73
+ }
74
+ if (n.kids) {
75
+ for (const k of n.kids) if (!collect(k)) return false;
76
+ }
77
+ return true;
78
+ };
79
+ if (!collect(tree)) return false;
80
+
81
+ const W = ctx.space.maxGroup; // write side of the canonical contract
82
+ const prev = ctx._prevSeen;
83
+ const seenBefore = (off: number, len: number): boolean => {
84
+ if (!prev) return false;
85
+ for (let i = off; i < off + len; i++) {
86
+ const c = chunkOf[i];
87
+ if (c === undefined || !prev.has(c)) return false;
88
+ }
89
+ return true;
90
+ };
91
+ const lens = streamIds.length >= W ? canonicalWindows(W) : [streamIds.length];
92
+ for (const len of lens) {
93
+ if (len < 1) continue;
94
+ for (let off = 0; off + len <= streamIds.length; off++) {
95
+ if (seenBefore(off, len)) continue;
96
+ const winIds = streamIds.slice(off, off + len);
97
+ const flatId = ctx.store.findBranch(winIds) ??
98
+ await ctx.store.putBranch(
99
+ winIds,
100
+ foldVecs(ctx.space, streamVecs.slice(off, off + len)),
101
+ );
102
+ for (let i = off; i < off + len; i++) {
103
+ const c = chunkOf[i];
104
+ if (c !== undefined) ctx.store.addContainer(flatId, c);
105
+ }
106
+ }
107
+ }
108
+ return true;
109
+ }
110
+
111
+ /** Perceive, intern, and index a single input. Returns the perceived tree,
112
+ * root id, id map, and the changed (new) subtrees for halo reinforcement. */
113
+ export async function deposit(
114
+ ctx: MindContext,
115
+ input: Input,
116
+ track: boolean,
117
+ ): Promise<
118
+ { tree: Sema; rootId: number; ids: Map<Sema, number>; changed: Sema[] }
119
+ > {
120
+ const bytes = inputBytes(ctx, input);
121
+ // Deposit-shaped perception: stable-prefix tree SEEDING (see
122
+ // perceiveDeposit) — an accumulated context re-folds only its new suffix,
123
+ // O(turn) instead of O(context) per conversation turn. Cache-only here
124
+ // (no store-probe fallback): a knownPrefixLength scan on every novel fact
125
+ // would cost O(n²) hashing, while conversation replays are always warm —
126
+ // re-deposition replays from the first turn, rebuilding the cache as it
127
+ // goes.
128
+ const tree = perceiveDeposit(ctx, bytes);
129
+
130
+ const ids = new Map<Sema, number>();
131
+ const rootId = await internTreeIds(ctx, tree, ids);
132
+
133
+ const indexed = await indexSubSpans(ctx, tree, ids);
134
+
135
+ const leafIds = leafIdPrefix(ctx, bytes);
136
+ if (leafIds.length === bytes.length && leafIds.length >= 2) {
137
+ await ctx.store.putBranch(leafIds, tree.v);
138
+ }
139
+
140
+ const changed = (track && ctx._prevSeen)
141
+ ? changedNodes(tree, ids, ctx._prevSeen)
142
+ : [tree];
143
+ if (track) ctx._prevSeen = indexed ? new Set(ids.values()) : null;
144
+ return { tree, rootId, ids, changed };
145
+ }
146
+
147
+ /** Ingest a single input (a bare experience, no continuation). */
148
+ export async function ingestOne(
149
+ ctx: MindContext,
150
+ input: Input,
151
+ ): Promise<Sema & { id: number }> {
152
+ const { tree, rootId, ids } = await deposit(ctx, input, true);
153
+ ctx.store.indexTarget(rootId);
154
+ const parts: number[] = tree.kids
155
+ ? tree.kids.map((k) => ids.get(k)!)
156
+ : [rootId];
157
+ const stride = ctx.space.maxGroup;
158
+ if (parts.length > stride) {
159
+ for (let i = 0; i + stride < parts.length; i += stride) {
160
+ await ctx.store.link(parts[i], parts[i + stride]);
161
+ }
162
+ if ((parts.length - 1) % stride !== 0) {
163
+ const lastStart = Math.floor((parts.length - 1) / stride) * stride;
164
+ if (lastStart < parts.length - 1) {
165
+ await ctx.store.link(parts[lastStart], parts[parts.length - 1]);
166
+ }
167
+ }
168
+ } else {
169
+ for (const id of parts) ctx.store.indexTarget(id);
170
+ }
171
+ return Object.assign(tree, { id: rootId });
172
+ }
173
+
174
+ /** Ingest a pair (context, continuation) — learn an edge and pour halos. */
175
+ export async function ingestPair(
176
+ ctx: MindContext,
177
+ ctxInput: Input,
178
+ cont: Input,
179
+ ): Promise<void> {
180
+ const c = await deposit(ctx, ctxInput, true);
181
+ const cont_ = await deposit(ctx, cont, false);
182
+ const ctxId = c.rootId, contId = cont_.rootId;
183
+
184
+ await ctx.store.link(ctxId, contId);
185
+
186
+ // Halos pour company SIGNATURES (identity), not gists (content) — see
187
+ // companySignature in sema.ts.
188
+ const contSeat = bindSeat(ctx.space, companySignature(ctx.space, contId), 1);
189
+ for (const part of c.changed) {
190
+ const partId = c.ids.get(part)!;
191
+ await ctx.store.pourHalo(partId, contSeat);
192
+ await ctx.store.pourHalo(
193
+ contId,
194
+ bindSeat(ctx.space, companySignature(ctx.space, partId), 0),
195
+ );
196
+ }
197
+ }
198
+
199
+ /** Dispatch the public ingest input shapes onto one-input / pair handlers —
200
+ * THE one reading of ingest's polymorphic surface (scalar, (context,
201
+ * continuation) pair, or a list mixing bare inputs and pairs). Both ingest
202
+ * paths — the direct one below and {@link CachedIngest} — route through
203
+ * this, so the shape-detection can never drift between them again (the
204
+ * ingest cache once re-implemented it and drifted). */
205
+ export async function dispatchIngest(
206
+ input: Input | (Input | [Input, Input])[],
207
+ second: Input | undefined,
208
+ onOne: (input: Input) => Promise<Sema & { id: number }>,
209
+ onPair: (ctxInput: Input, cont: Input) => Promise<void>,
210
+ ): Promise<(Sema & { id: number }) | undefined> {
211
+ if (
212
+ Array.isArray(input) && !(input instanceof Uint8Array) &&
213
+ (input as { width?: unknown }).width === undefined
214
+ ) {
215
+ const arr = input as (Input | [Input, Input])[];
216
+ if (
217
+ arr.length === 2 && !Array.isArray(arr[0]) && !Array.isArray(arr[1])
218
+ ) {
219
+ await onPair(arr[0] as Input, arr[1] as Input);
220
+ return undefined;
221
+ }
222
+ for (const item of arr) {
223
+ if (Array.isArray(item) && item.length === 2) {
224
+ await onPair(item[0], item[1]);
225
+ } else await onOne(item as Input);
226
+ }
227
+ return undefined;
228
+ }
229
+ if (second === undefined) return onOne(input as Input);
230
+ await onPair(input as Input, second);
231
+ return undefined;
232
+ }
233
+
234
+ /** Ingest an input or array of inputs/pairs. The public ingest entry point. */
235
+ export async function ingest(
236
+ ctx: MindContext,
237
+ input: Input | (Input | [Input, Input])[],
238
+ second?: Input,
239
+ ): Promise<(Sema & { id: number }) | undefined> {
240
+ return dispatchIngest(
241
+ input,
242
+ second,
243
+ (i) => ingestOne(ctx, i),
244
+ (a, b) => ingestPair(ctx, a, b),
245
+ );
246
+ }