@hviana/sema 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/mind/attention.d.ts +14 -2
- package/dist/src/mind/attention.js +54 -8
- package/dist/src/mind/bridge.d.ts +30 -0
- package/dist/src/mind/bridge.js +569 -0
- package/dist/src/mind/match.d.ts +15 -2
- package/dist/src/mind/match.js +3 -8
- package/dist/src/mind/mechanisms/cast.d.ts +54 -0
- package/dist/src/mind/mechanisms/cast.js +268 -37
- package/dist/src/mind/mechanisms/cover.js +16 -31
- package/dist/src/mind/mechanisms/recall.js +66 -0
- package/dist/src/mind/reasoning.d.ts +11 -0
- package/dist/src/mind/reasoning.js +58 -2
- package/dist/src/mind/recognition.js +127 -7
- package/dist/src/mind/traverse.js +16 -15
- package/dist/src/mind/types.d.ts +39 -2
- package/dist/src/mind/types.js +38 -7
- package/dist/src/store.d.ts +12 -3
- package/dist/src/store.js +9 -3
- package/package.json +1 -1
- package/src/mind/attention.ts +65 -7
- package/src/mind/bridge.ts +596 -0
- package/src/mind/match.ts +19 -5
- package/src/mind/mechanisms/cast.ts +290 -38
- package/src/mind/mechanisms/cover.ts +23 -36
- package/src/mind/mechanisms/recall.ts +79 -0
- package/src/mind/reasoning.ts +71 -2
- package/src/mind/recognition.ts +132 -7
- package/src/mind/traverse.ts +16 -17
- package/src/mind/types.ts +70 -6
- package/src/store.ts +19 -5
- package/test/36-already-answered-fusion.test.mjs +128 -0
- package/test/37-cluster-dispersion-fusion.test.mjs +190 -0
- package/test/38-reason-restate-guard.test.mjs +94 -0
- package/test/39-cast-restate-guard.test.mjs +102 -0
- package/test/40-choosenext-scale-guard.test.mjs +75 -0
- package/test/41-seatofnode-direction.test.mjs +85 -0
- package/test/42-recognise-trace-idempotence.test.mjs +106 -0
- package/test/43-cast-analog-seat.test.mjs +244 -0
- package/test/44-recognise-edge-whitespace.test.mjs +63 -0
- package/test/45-liftanswer-restated-trim.test.mjs +60 -0
- package/test/46-recognise-multibyte-edge.test.mjs +85 -0
- package/test/47-cast-comparison-coverage.test.mjs +134 -0
- package/test/48-recognise-turn-connective.test.mjs +125 -0
- package/test/49-natural-units-synonym-bridge.test.mjs +175 -0
- package/test/50-cast-analog-consensus-floor.test.mjs +179 -0
package/src/mind/reasoning.ts
CHANGED
|
@@ -13,6 +13,20 @@ import { joinWithBridge, pivotInto } from "./resonance.js";
|
|
|
13
13
|
import type { Precomputed } from "./pipeline-mechanism.js";
|
|
14
14
|
import type { Rationale } from "./rationale.js";
|
|
15
15
|
|
|
16
|
+
/** Whether `bytes` is a proper byte-subspan of `query` — already present in
|
|
17
|
+
* the question, so voicing it back only restates part of what was asked,
|
|
18
|
+
* never answers it. The exact guard recallByResonance already applies to
|
|
19
|
+
* its OWN grounding candidates (tier 1's `restates`, tier 2's subspan
|
|
20
|
+
* check, tier 0b's argument-binding subspan check) — every mechanism that
|
|
21
|
+
* walks a LEARNT CONTINUATION EDGE past an already-vetted grounding
|
|
22
|
+
* (reason()'s own hops below, and CAST's `projectCounterfactual` seat
|
|
23
|
+
* substitution — see cast.ts) needs the same guard applied to what the
|
|
24
|
+
* walk turns up, since `follow()`/`chooseNext`/`pivotInto` know nothing of
|
|
25
|
+
* the query at all — only of what structurally continues what. */
|
|
26
|
+
export function restatesQuery(query: Uint8Array, bytes: Uint8Array): boolean {
|
|
27
|
+
return bytes.length < query.length && indexOf(query, bytes, 0) >= 0;
|
|
28
|
+
}
|
|
29
|
+
|
|
16
30
|
/** Extend a grounded answer forward across facts (multi-hop reasoning).
|
|
17
31
|
* Pivots on the longest unconsumed learnt context each answer contains,
|
|
18
32
|
* then follows the pivot's continuation to the next fact. Repeats up
|
|
@@ -91,7 +105,8 @@ export async function reason(
|
|
|
91
105
|
const fwdId = fwd !== null ? resolve(ctx, fwd) : null;
|
|
92
106
|
if (
|
|
93
107
|
fwd !== null && !bytesEqual(fwd, cur) &&
|
|
94
|
-
(fwdId === null || !consumed.has(fwdId))
|
|
108
|
+
(fwdId === null || !consumed.has(fwdId)) &&
|
|
109
|
+
!restatesQuery(query, fwd)
|
|
95
110
|
) {
|
|
96
111
|
consumeAll(curId);
|
|
97
112
|
t ??= ctx.trace?.enter("reason", [
|
|
@@ -115,7 +130,7 @@ export async function reason(
|
|
|
115
130
|
|
|
116
131
|
const fc = await follow(ctx, pivot, qv);
|
|
117
132
|
consumeAll(pivot);
|
|
118
|
-
if (fc === null || bytesEqual(fc, cur)) break;
|
|
133
|
+
if (fc === null || bytesEqual(fc, cur) || restatesQuery(query, fc)) break;
|
|
119
134
|
t ??= ctx.trace?.enter("reason", [rItem(startedFrom, "grounded")]);
|
|
120
135
|
ctx.trace?.step(
|
|
121
136
|
"pivotStep",
|
|
@@ -189,6 +204,60 @@ export async function fuseAttention(
|
|
|
189
204
|
...rest.map((r) => rNode(ctx, r.anchor, "point", r.vote)),
|
|
190
205
|
]);
|
|
191
206
|
for (const root of rest) {
|
|
207
|
+
// DISPERSION: this root's contributing regions are confined to a
|
|
208
|
+
// single cluster (see Attention.clusters) — one local neighbourhood of
|
|
209
|
+
// the query, not several separate places. Raw region count already
|
|
210
|
+
// failed to discriminate a coincidental match from a genuine further
|
|
211
|
+
// topic (test/24 gap 3.1 vs test/35's echo); dispersion is a different
|
|
212
|
+
// question — not how MUCH evidence, but how many separate PLACES in the
|
|
213
|
+
// query corroborate it — and a coincidental match is structurally
|
|
214
|
+
// confined to one cluster no matter how strongly it resonates.
|
|
215
|
+
//
|
|
216
|
+
// EXCEPTION: crossRegionVotes' own joint conclusions (a query naming
|
|
217
|
+
// two attributes that were only ever learnt TOGETHER — test/34's own
|
|
218
|
+
// binding corpus) are inherently ONE fused context and are pooled from
|
|
219
|
+
// a single synthetic region, so they always read as one cluster even
|
|
220
|
+
// though they already, by construction, account for both original
|
|
221
|
+
// mentions. `breadth` (dominates — the same > half-the-query bar used
|
|
222
|
+
// everywhere else) still correctly recognises these: a genuine joint
|
|
223
|
+
// binding explains the MAJORITY of the query's regions on its own,
|
|
224
|
+
// which a coincidental echo never does (verified: test/35's echo tops
|
|
225
|
+
// out at 0.40). So a root is trusted when EITHER measure alone
|
|
226
|
+
// indicates real signal — excluded only when BOTH are weak. Cheap and
|
|
227
|
+
// synchronous — checked before the async already-answered walk below.
|
|
228
|
+
if (root.clusters < 2 && root.breadth <= 0.5) {
|
|
229
|
+
ctx.trace?.step(
|
|
230
|
+
"singleCluster",
|
|
231
|
+
[rNode(ctx, root.anchor, "point", root.vote)],
|
|
232
|
+
[],
|
|
233
|
+
"this point's evidence is confined to one local neighbourhood of the query — not trusted as an independent topic",
|
|
234
|
+
);
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
// ALREADY ANSWERED: this root's own learnt continuation — the same
|
|
238
|
+
// content-addressed walk reason()'s echo guard already trusts
|
|
239
|
+
// (`ctx.store.prevCount(qId) > 0`), here applied per-candidate instead
|
|
240
|
+
// of to the whole query — is VERBATIM present later in the query. A
|
|
241
|
+
// query that embeds both an exchange's ask and its own already-given
|
|
242
|
+
// reply (a conversation's turn plus its own prior answer, concatenated
|
|
243
|
+
// raw by addTurn — or any caller pasting a transcript into one
|
|
244
|
+
// respond() call; the check is Mind-bookkeeping-free, so it treats both
|
|
245
|
+
// identically) has already spoken this root's answer — fusing it in
|
|
246
|
+
// again would only restate it. Deliberately NOT a magnitude measure:
|
|
247
|
+
// it fires on exact content-addressed recurrence, not on how strongly
|
|
248
|
+
// the root resonates.
|
|
249
|
+
const cont = await follow(ctx, root.anchor, qv);
|
|
250
|
+
if (
|
|
251
|
+
cont !== null && cont.length > 0 && indexOf(query, cont, root.end) >= 0
|
|
252
|
+
) {
|
|
253
|
+
ctx.trace?.step(
|
|
254
|
+
"alreadyAnswered",
|
|
255
|
+
[rNode(ctx, root.anchor, "point", root.vote)],
|
|
256
|
+
[rItem(cont, "continuation")],
|
|
257
|
+
"this point's own learnt continuation already appears later in the query — already answered, not fused",
|
|
258
|
+
);
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
192
261
|
const g = await project(ctx, root.anchor, qv);
|
|
193
262
|
if (g === null || g.length === 0) continue;
|
|
194
263
|
if (pieces.some((p) => indexOf(p.bytes, g, 0) >= 0)) continue;
|
package/src/mind/recognition.ts
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
resolve,
|
|
17
17
|
} from "./primitives.js";
|
|
18
18
|
import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
|
|
19
|
-
import { chainReach, leafIdAt } from "./canonical.js";
|
|
19
|
+
import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
|
|
20
20
|
import { isChunk, type Sema } from "../sema.js";
|
|
21
21
|
import type { Leaf, Site } from "./graph-search.js";
|
|
22
22
|
|
|
@@ -34,12 +34,44 @@ import type { Leaf, Site } from "./graph-search.js";
|
|
|
34
34
|
* Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
|
|
35
35
|
export function recognise(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
36
36
|
// Content-keyed memo — works for both single-turn respond() and multi-turn
|
|
37
|
-
// respondTurn() (where the map persists across calls).
|
|
38
|
-
// tracing
|
|
39
|
-
|
|
37
|
+
// respondTurn() (where the map persists across calls). ALWAYS consulted,
|
|
38
|
+
// regardless of tracing — matching perceive()'s own memo, which carries no
|
|
39
|
+
// trace gate at all. This memo is NOT an optional accelerator: recogniseImpl
|
|
40
|
+
// walks the query's perceived tree via foldTree, whose subtree-resolution
|
|
41
|
+
// fast path (see primitives.ts) skips invoking `visit` — and therefore
|
|
42
|
+
// skips EMITTING SITES — for any subtree already cached in
|
|
43
|
+
// ctx._resolvedSubtrees. A multi-turn conversation's stable-prefix fold
|
|
44
|
+
// deliberately shares node OBJECTS across turns, so by the second call on
|
|
45
|
+
// the exact same bytes, large swaths of the tree are already cached and
|
|
46
|
+
// foldTree stops short of recursing into them — a second recogniseImpl
|
|
47
|
+
// call on the SAME bytes is not idempotent; it silently finds FEWER sites
|
|
48
|
+
// than the first (observed live: 31 sites → 5 on an immediate repeat
|
|
49
|
+
// call). Skipping this memo "only while tracing" used to mean every
|
|
50
|
+
// traced turn re-ran recogniseImpl from scratch at every one of the many
|
|
51
|
+
// call sites that recognise the same query (cover, reason, articulate...),
|
|
52
|
+
// each subsequent call silently more incomplete than the last — measurably
|
|
53
|
+
// changing which mechanism grounds the answer, not just costing time. The
|
|
54
|
+
// trace step must still fire on every call regardless (a cache hit is not
|
|
55
|
+
// silent), so it is emitted here directly instead of only inside
|
|
56
|
+
// recogniseImpl.
|
|
57
|
+
if (ctx.recogniseMemo) {
|
|
40
58
|
const key = latin1Key(bytes);
|
|
41
59
|
const hit = ctx.recogniseMemo.get(key);
|
|
42
|
-
if (hit !== undefined)
|
|
60
|
+
if (hit !== undefined) {
|
|
61
|
+
ctx.trace?.step(
|
|
62
|
+
"recognise",
|
|
63
|
+
[rItem(bytes, "query")],
|
|
64
|
+
hit.sites.map((s) =>
|
|
65
|
+
rItem(bytes.subarray(s.start, s.end), "form", s.payload, [
|
|
66
|
+
s.start,
|
|
67
|
+
s.end,
|
|
68
|
+
])
|
|
69
|
+
),
|
|
70
|
+
`decompose the query into ${hit.sites.length} learnt form(s) that ` +
|
|
71
|
+
`lead somewhere (over ${hit.leaves.length} perceived leaves) [cached]`,
|
|
72
|
+
);
|
|
73
|
+
return hit;
|
|
74
|
+
}
|
|
43
75
|
const fresh = recogniseImpl(ctx, bytes);
|
|
44
76
|
ctx.recogniseMemo.set(key, fresh);
|
|
45
77
|
return fresh;
|
|
@@ -83,8 +115,18 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
|
83
115
|
// silence. Atoms stay available as leaves (PASS-carried literals) and
|
|
84
116
|
// through exact tier-0 resolution regardless.
|
|
85
117
|
const atomsAreHubs = atomIsHub(ctx, corpusN(ctx));
|
|
118
|
+
// Distinct probes (structural exact match, canon fallback, edge trims at
|
|
119
|
+
// several offsets) can legitimately re-derive the SAME (start, end, id)
|
|
120
|
+
// site from different tree nodes — a wide edge-trim search is exactly
|
|
121
|
+
// this on purpose (see below). Duplicate site entries are not wrong
|
|
122
|
+
// evidence, but they double the weight cover's derivation search gives
|
|
123
|
+
// that span, distorting its cost model — the same span must count once.
|
|
124
|
+
const seen = new Set<string>();
|
|
86
125
|
const emit = (start: number, end: number, id: number) => {
|
|
87
126
|
if (id < 0 && atomsAreHubs) return;
|
|
127
|
+
const key = start + "," + end + "," + id;
|
|
128
|
+
if (seen.has(key)) return;
|
|
129
|
+
seen.add(key);
|
|
88
130
|
if (leadsSomewhere(ctx, id)) {
|
|
89
131
|
sites.push({ start, end, payload: id });
|
|
90
132
|
}
|
|
@@ -101,10 +143,93 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
|
101
143
|
// missed may still be a stored form under the response's equivalence
|
|
102
144
|
// (case, width, whitespace — whatever the injected canonicalizer says).
|
|
103
145
|
// O(subtree bytes) per miss, memoised per response; a no-op when no
|
|
104
|
-
// canonicalizer was injected or the store has no canon index.
|
|
105
|
-
|
|
146
|
+
// canonicalizer was injected or the store has no canon index. A raw
|
|
147
|
+
// leaf (n.kids === null) is single-byte and handled by the byte-atom
|
|
148
|
+
// path above instead — canon equivalence only applies to composites.
|
|
149
|
+
else if (n.kids !== null) {
|
|
106
150
|
const cid = canonResolve(ctx, bytes.subarray(start, end));
|
|
107
151
|
if (cid !== null) emit(start, end, cid);
|
|
152
|
+
// The edge-trim fallbacks below remove 1 byte from a side; the
|
|
153
|
+
// remainder must still be a composite (>= 2 bytes, the same floor
|
|
154
|
+
// n.kids !== null enforces above) rather than degenerate into
|
|
155
|
+
// single-byte-atom territory, which atomIsHub already governs
|
|
156
|
+
// separately.
|
|
157
|
+
else if (end - start - 1 >= 2) {
|
|
158
|
+
// The chunk's own boundary is drawn by content geometry, not by
|
|
159
|
+
// any notion of "form" — it can include one edge byte the query's
|
|
160
|
+
// fold happened to attach here that the trained span never had
|
|
161
|
+
// (e.g. a separator from the preceding chunk). The core has no
|
|
162
|
+
// idea what that byte means; it only knows resolve()/canonResolve
|
|
163
|
+
// are self-verifying (hash-then-verify, same discipline as every
|
|
164
|
+
// content lookup here), so a blind one-byte-shorter guess on
|
|
165
|
+
// either edge costs nothing when wrong and is trustworthy when it
|
|
166
|
+
// hits. Two extra probes, only on the already-failed miss path.
|
|
167
|
+
const left = resolve(ctx, bytes.subarray(start + 1, end));
|
|
168
|
+
if (left !== null) emit(start + 1, end, left);
|
|
169
|
+
const right = resolve(ctx, bytes.subarray(start, end - 1));
|
|
170
|
+
if (right !== null) emit(start, end - 1, right);
|
|
171
|
+
// A misalignment wider than one byte (e.g. more than one edge
|
|
172
|
+
// separator swallowed) is not itself geometry-quantized — the
|
|
173
|
+
// WRITE side's canonical index (canonicalWindows) interns sliding
|
|
174
|
+
// W−1/W-length windows over leaf ids at EVERY offset, not just
|
|
175
|
+
// radix-aligned ones (see canonical.ts) — so the offset that
|
|
176
|
+
// recovers a trained span can be anything, not a multiple of W.
|
|
177
|
+
// What IS bounded is how far it's worth looking: chainReach(W)=W²,
|
|
178
|
+
// the same reach the canonical pass (tryChain) trusts for a chain
|
|
179
|
+
// rebuilt off the query's own fold. Every candidate offset is
|
|
180
|
+
// gated by store.findBranch(leafIds) first — the SAME cheap,
|
|
181
|
+
// fold-free existence check tryChain already uses — so the extra
|
|
182
|
+
// resolve() fold (the real cost) is only paid when a branch could
|
|
183
|
+
// plausibly exist there, not for every offset. The node itself is
|
|
184
|
+
// also bounded to chunk-scale (end - start <= W²): widening this at
|
|
185
|
+
// whole-query/root scale can rediscover a smaller subtree's own
|
|
186
|
+
// content as a second, overlapping site the structural walk's own
|
|
187
|
+
// finer recursion already emits correctly on its own — a duplicate
|
|
188
|
+
// that downstream derivation can stitch into a wrong answer.
|
|
189
|
+
const W = ctx.space.maxGroup;
|
|
190
|
+
for (
|
|
191
|
+
let k = 1;
|
|
192
|
+
end - start <= W * W && k <= W * W && start + k < end - 1;
|
|
193
|
+
k++
|
|
194
|
+
) {
|
|
195
|
+
const lIds = leafIdRun(ctx, bytes, start + k, end);
|
|
196
|
+
if (lIds !== null && store.findBranch(lIds) !== null) {
|
|
197
|
+
const eLeft = resolve(ctx, bytes.subarray(start + k, end));
|
|
198
|
+
if (eLeft !== null) emit(start + k, end, eLeft);
|
|
199
|
+
}
|
|
200
|
+
const rIds = leafIdRun(ctx, bytes, start, end - k);
|
|
201
|
+
if (rIds !== null && store.findBranch(rIds) !== null) {
|
|
202
|
+
const eRight = resolve(ctx, bytes.subarray(start, end - k));
|
|
203
|
+
if (eRight !== null) emit(start, end - k, eRight);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// A REAL extra word at the left edge (a discourse connective like
|
|
207
|
+
// "And " prepended to a follow-up turn — not boundary noise, actual
|
|
208
|
+
// content the injected canonicalizer has no equivalence for) shows
|
|
209
|
+
// up as a canon-miss too big for the chunk-scale search above: the
|
|
210
|
+
// turn is its OWN segment (stable-prefix folded independently — see
|
|
211
|
+
// mind.ts's _growContext), so it can be turn/segment-scale, not
|
|
212
|
+
// chunk-scale. Widening the size bound itself reopens the root-
|
|
213
|
+
// scale false-positive this module already fixed once (test/46);
|
|
214
|
+
// widening the SEARCH instead — trying every position up to W
|
|
215
|
+
// chunk-widths from the left edge that the query's OWN fold treats
|
|
216
|
+
// as a chunk boundary (`starts`, the same set the canonical pass
|
|
217
|
+
// privileges with full chain reach) — does not: `starts.has(p)` is
|
|
218
|
+
// fold EVIDENCE the query produced on its own (a leaf-parent chunk
|
|
219
|
+
// like "And " is visited, and its start added to `starts`, before
|
|
220
|
+
// this composite ever runs — foldTree is post-order), never a blind
|
|
221
|
+
// guess. Bounded to W candidates, each an O(1) set lookup before
|
|
222
|
+
// paying for the real canonResolve fold — canonResolve, not
|
|
223
|
+
// resolve()/findBranch, because the gap here is often exactly the
|
|
224
|
+
// kind of equivalence (case, in the live trace) canon exists for,
|
|
225
|
+
// not just an exact-content coincidence.
|
|
226
|
+
for (let k = 1; k <= W; k++) {
|
|
227
|
+
const p = start + k * W;
|
|
228
|
+
if (p >= end - 1 || !starts.has(p)) continue;
|
|
229
|
+
const cid = canonResolve(ctx, bytes.subarray(p, end));
|
|
230
|
+
if (cid !== null) emit(p, end, cid);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
108
233
|
}
|
|
109
234
|
if (isChunk(n)) {
|
|
110
235
|
starts.add(start);
|
package/src/mind/traverse.ts
CHANGED
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
// project) live in match.ts — the elementary match-and-project operation.
|
|
8
8
|
|
|
9
9
|
import { cosine, Vec } from "../vec.js";
|
|
10
|
-
import { consensusFloor } from "../geometry.js";
|
|
11
10
|
import type { AncestorReach, MindContext } from "./types.js";
|
|
12
11
|
import { gistOf, read } from "./primitives.js";
|
|
13
12
|
|
|
@@ -528,22 +527,22 @@ export function chooseNext(
|
|
|
528
527
|
}
|
|
529
528
|
}
|
|
530
529
|
|
|
531
|
-
//
|
|
532
|
-
//
|
|
533
|
-
//
|
|
534
|
-
//
|
|
535
|
-
//
|
|
536
|
-
//
|
|
537
|
-
//
|
|
538
|
-
//
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
)
|
|
545
|
-
|
|
546
|
-
|
|
530
|
+
// NO consensusFloor gate here (tried and reverted — see
|
|
531
|
+
// test/40-choosenext-scale-guard.test.mjs): that floor is calibrated for
|
|
532
|
+
// POOLED, IDF-weighted CLIMB VOTES (recallByResonance, commitVotes), where
|
|
533
|
+
// each corroborating region contributes at most ln N and the floor grows
|
|
534
|
+
// with N exactly as that per-region ceiling does (HOW_IT_WORKS.md §8.6).
|
|
535
|
+
// `bestSupport` here is a different kind of quantity — a raw prevCount of
|
|
536
|
+
// how many training contexts predicted ONE destination, bounded by how
|
|
537
|
+
// often that specific fact was retold, never by corpus size N. Gating an
|
|
538
|
+
// N-invariant count against an N-growing threshold guarantees failure
|
|
539
|
+
// once N is large enough, discarding genuinely, structurally dominant
|
|
540
|
+
// edges (observed: a fact corroborated 2-to-1-1-1 refused at N≈325K,
|
|
541
|
+
// falling back to a noisy concept-hop). The loop above already IS the
|
|
542
|
+
// "genuinely competing" test: a tie leaves first-inserted as the pick
|
|
543
|
+
// (test/30's own pinned behaviour); a strict winner is real evidence
|
|
544
|
+
// regardless of corpus scale. Matches HOW_IT_WORKS.md §25's own
|
|
545
|
+
// chooseNext pseudocode, which has no such floor.
|
|
547
546
|
|
|
548
547
|
// Trace is built lazily — the filter + map below only execute when a
|
|
549
548
|
// trace listener is attached, so the common (no-trace) path pays only
|
package/src/mind/types.ts
CHANGED
|
@@ -39,7 +39,7 @@ export interface DepositCacheEntry {
|
|
|
39
39
|
* its own suffix bytes-equal this exactly. */
|
|
40
40
|
nextBytes?: Uint8Array;
|
|
41
41
|
}
|
|
42
|
-
import { concatBytes } from "../bytes.js";
|
|
42
|
+
import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
|
|
43
43
|
import { dominates } from "../geometry.js";
|
|
44
44
|
|
|
45
45
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
@@ -111,6 +111,23 @@ export interface Attention {
|
|
|
111
111
|
* consensus; one that does not is a coincidental single-region echo —
|
|
112
112
|
* see test/35-attention-confidence.test.mjs. */
|
|
113
113
|
breadth: number;
|
|
114
|
+
/** DISPERSION: the number of distinct clusters this point's contributing
|
|
115
|
+
* regions form, merging any two whose gap is under one river-fold
|
|
116
|
+
* quantum W. Neither breadth NOR raw region count discriminates a
|
|
117
|
+
* genuine further topic from a coincidental echo (both were tried and
|
|
118
|
+
* falsified — breadth starves a genuine, evenly-split multi-topic query,
|
|
119
|
+
* since no root in a real N-way split can exceed half the vote; raw
|
|
120
|
+
* count doesn't separate them either, since a short, structurally simple
|
|
121
|
+
* echo racks up as many corroborating regions as a real topic does).
|
|
122
|
+
* Dispersion asks a different question: not how MUCH evidence, but how
|
|
123
|
+
* many separate PLACES in the query corroborate it. A coincidental
|
|
124
|
+
* match — one local phrase resonating with an unrelated stored form —
|
|
125
|
+
* is structurally confined to ONE cluster no matter how strong its vote;
|
|
126
|
+
* a genuine further topic is named in its own distinctive wording
|
|
127
|
+
* somewhere the query's scaffolding does not reach, always a SEPARATE
|
|
128
|
+
* cluster from whatever else corroborates it. See
|
|
129
|
+
* test/37-cluster-dispersion-fusion.test.mjs. */
|
|
130
|
+
clusters: number;
|
|
114
131
|
}
|
|
115
132
|
|
|
116
133
|
/** Both read-outs of one consensus climb. */
|
|
@@ -273,11 +290,52 @@ export function spliceAll(segs: Seg[]): Uint8Array | null {
|
|
|
273
290
|
return concatBytes(segs.map((s) => s.bytes));
|
|
274
291
|
}
|
|
275
292
|
|
|
293
|
+
/** Whether a chosen span RESTATES the query rather than answering it: its
|
|
294
|
+
* SUBSTITUTED bytes (an edge followed from a recognised site, not the
|
|
295
|
+
* site's own literal text read back) already occur elsewhere in the query
|
|
296
|
+
* — the same principle recall.ts's tiers apply to a whole-query projection
|
|
297
|
+
* ("a projection that is a proper byte-subspan of the query restates part
|
|
298
|
+
* of the question"). A LITERAL span (the site's own bytes, unchanged) is
|
|
299
|
+
* exempt: naming what's already there at its OWN position is not a
|
|
300
|
+
* substitution. A recognised site that is itself an entire PRIOR TURN of
|
|
301
|
+
* a multi-turn query is exactly this shape: it carries a genuine learnt
|
|
302
|
+
* continuation, but that continuation is something the asker already said
|
|
303
|
+
* moments later in the SAME query, not a new answer. Below one river
|
|
304
|
+
* window, byte overlap is chance, not evidence — the same floor
|
|
305
|
+
* identityBar and reachThreshold hold every other structural-overlap claim
|
|
306
|
+
* to. */
|
|
307
|
+
export function segRestatesQuery(
|
|
308
|
+
s: Seg,
|
|
309
|
+
query: Uint8Array,
|
|
310
|
+
queryLen: number,
|
|
311
|
+
W: number,
|
|
312
|
+
): boolean {
|
|
313
|
+
if (!s.rec) return false;
|
|
314
|
+
const literal = s.j - s.i === s.bytes.length &&
|
|
315
|
+
bytesEqual(s.bytes, query.subarray(s.i, s.j));
|
|
316
|
+
if (literal) return false;
|
|
317
|
+
return s.bytes.length >= W && s.bytes.length < queryLen &&
|
|
318
|
+
indexOf(query, s.bytes, 0) >= 0;
|
|
319
|
+
}
|
|
320
|
+
|
|
276
321
|
/** Lift the answer out of the cover for think: the recognised region, free of
|
|
277
|
-
* the asker's surrounding (unrecognised) framing
|
|
278
|
-
|
|
322
|
+
* the asker's surrounding (unrecognised) framing — and free of any chosen
|
|
323
|
+
* span that only RESTATES content the query already contains (see {@link
|
|
324
|
+
* segRestatesQuery}). A restating span is excluded from both the framing
|
|
325
|
+
* (lo/hi) decision and the final concatenation: it is stale, not a second
|
|
326
|
+
* answer, but the OTHER spans a derivation chose are independent evidence
|
|
327
|
+
* and must not be discarded along with it. */
|
|
328
|
+
export function liftAnswer(
|
|
329
|
+
segs: Seg[],
|
|
330
|
+
queryLen: number,
|
|
331
|
+
query: Uint8Array,
|
|
332
|
+
W: number,
|
|
333
|
+
): Uint8Array | null {
|
|
334
|
+
const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
|
|
279
335
|
const recognised: number[] = [];
|
|
280
|
-
for (let k = 0; k < segs.length; k++)
|
|
336
|
+
for (let k = 0; k < segs.length; k++) {
|
|
337
|
+
if (segs[k].rec && !restated[k]) recognised.push(k);
|
|
338
|
+
}
|
|
281
339
|
if (recognised.length === 0) return null;
|
|
282
340
|
|
|
283
341
|
if (recognised.length === 1) {
|
|
@@ -297,13 +355,19 @@ export function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null {
|
|
|
297
355
|
// trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
|
|
298
356
|
if (s.computed && s.i > 0) return s.bytes;
|
|
299
357
|
if (dominates(s.j - s.i, queryLen)) {
|
|
300
|
-
return concatBytes(
|
|
358
|
+
return concatBytes(
|
|
359
|
+
segs.filter((_, k) => !restated[k]).map((x) => x.bytes),
|
|
360
|
+
);
|
|
301
361
|
}
|
|
302
362
|
return s.bytes;
|
|
303
363
|
}
|
|
304
364
|
const lo = recognised[0];
|
|
305
365
|
const hi = recognised[recognised.length - 1];
|
|
306
|
-
return concatBytes(
|
|
366
|
+
return concatBytes(
|
|
367
|
+
segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) =>
|
|
368
|
+
x.bytes
|
|
369
|
+
),
|
|
370
|
+
);
|
|
307
371
|
}
|
|
308
372
|
|
|
309
373
|
/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
|
package/src/store.ts
CHANGED
|
@@ -306,8 +306,17 @@ export interface Store {
|
|
|
306
306
|
nodeCount(): number;
|
|
307
307
|
|
|
308
308
|
// ── soft resonance over node gists ─────────────────────────────────────
|
|
309
|
-
/** The k nodes whose gist resonates most with v.
|
|
310
|
-
|
|
309
|
+
/** The k nodes whose gist resonates most with v. `exhaustive` widens the
|
|
310
|
+
* IVF probe to every cluster (see {@link AbstractStore.efFor}'s doc) —
|
|
311
|
+
* for refusal-path-only callers where an approximate top-√C-clusters
|
|
312
|
+
* search is not the same discriminator as the caller's own byte-exact
|
|
313
|
+
* verification (the substitution bridge's proposal channel): a rarer
|
|
314
|
+
* paraphrase can score lower than hundreds of unrelated hits by pure
|
|
315
|
+
* fold-geometry structural distance (a middle-of-string mismatch
|
|
316
|
+
* perturbs the tree hash far more than a tail mismatch of the same
|
|
317
|
+
* byte length) and so never even reach a probed cluster, no matter how
|
|
318
|
+
* large k is — k only reorders WITHIN the clusters already probed. */
|
|
319
|
+
resonate(v: Vec, k: number, exhaustive?: boolean): Promise<Hit[]>;
|
|
311
320
|
/** Mark a node as a RESONANCE TARGET — promote its gist into the content
|
|
312
321
|
* index so {@link resonate} can find it. A node's gist is captured at intern
|
|
313
322
|
* but indexed LAZILY (only targets are indexed; the ~99.5% intermediate DAG
|
|
@@ -1636,7 +1645,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1636
1645
|
|
|
1637
1646
|
// ── Soft resonance ─────────────────────────────────────────────────────
|
|
1638
1647
|
|
|
1639
|
-
async resonate(v: Vec, k: number): Promise<Hit[]> {
|
|
1648
|
+
async resonate(v: Vec, k: number, exhaustive = false): Promise<Hit[]> {
|
|
1640
1649
|
await this._ensureReady();
|
|
1641
1650
|
// Synchronous flush of any buffered index writes: the FIRST resonance
|
|
1642
1651
|
// after a large ingest pays that flush here, so it shows up in respond
|
|
@@ -1649,17 +1658,22 @@ export abstract class AbstractStore implements Store {
|
|
|
1649
1658
|
// same values still hits. Lazy-init: null after any index write; the
|
|
1650
1659
|
// first miss after a flush recreates it. When voteRegions resonates
|
|
1651
1660
|
// identical perceived sub-regions, only the first call descends the ANN.
|
|
1652
|
-
const rk = vecKey(v) + ":" + k;
|
|
1661
|
+
const rk = vecKey(v) + ":" + k + (exhaustive ? ":x" : "");
|
|
1653
1662
|
const cache = this._resonateCache;
|
|
1654
1663
|
if (cache) {
|
|
1655
1664
|
const hit = cache.get(rk);
|
|
1656
1665
|
if (hit !== undefined) return hit;
|
|
1657
1666
|
}
|
|
1658
1667
|
|
|
1668
|
+
const clusters = this._vecContentClusterCount();
|
|
1659
1669
|
const results = this._vecContentQuery(
|
|
1660
1670
|
normalize(copy(v)),
|
|
1661
1671
|
k * this.overfetch,
|
|
1662
|
-
|
|
1672
|
+
// Exhaustive: probe every cluster (ef ≥ 4·clusters guarantees the
|
|
1673
|
+
// IVF's own ef→nprobe=ceil(ef/4) mapping reaches all of them) — the
|
|
1674
|
+
// natural ceiling for a search that is ALREADY refusal-path-only and
|
|
1675
|
+
// must not miss a candidate hiding in an unprobed cluster.
|
|
1676
|
+
exhaustive ? 4 * clusters : this.efFor(clusters),
|
|
1663
1677
|
);
|
|
1664
1678
|
const out: Hit[] = [];
|
|
1665
1679
|
for (const r of results) {
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// 36-already-answered-fusion.test.mjs — a point of attention whose own
|
|
2
|
+
// learnt continuation is ALREADY present later in the query must not be
|
|
3
|
+
// fused in again.
|
|
4
|
+
//
|
|
5
|
+
// This is Category A of a two-part defect found while investigating a
|
|
6
|
+
// multi-turn dialogue's garbled fusion ("Thank you very much!" fused with
|
|
7
|
+
// unrelated points of attention). One of those points (root "Hello",
|
|
8
|
+
// anchor of the greeting) traced back to substantial, genuine regions —
|
|
9
|
+
// not noise — and its learnt continuation ("Hello! How can I assist you
|
|
10
|
+
// today?") was found to be VERBATIM present earlier in the query, because
|
|
11
|
+
// it was Sema's own prior reply, appended by addTurn the same way any turn
|
|
12
|
+
// is appended. Re-surfacing it is redundant: the query has already spoken
|
|
13
|
+
// its own answer.
|
|
14
|
+
//
|
|
15
|
+
// Deliberately turn-agnostic and Mind-bookkeeping-free: Mind's multi-turn
|
|
16
|
+
// API is strictly a computational optimization (incremental fold reuse) —
|
|
17
|
+
// it must never be the thing inference depends on for correctness. This
|
|
18
|
+
// check uses only what already exists for ANY accumulated byte stream,
|
|
19
|
+
// single-shot or multi-turn: `follow()` (the same content-addressed
|
|
20
|
+
// continuation walk `reason()`'s own echo guard already uses,
|
|
21
|
+
// `ctx.store.prevCount(qId) > 0`, just applied per-candidate-root instead
|
|
22
|
+
// of to the whole query) and plain byte containment. A query that embeds
|
|
23
|
+
// its own prior exchange — via real respondTurn(), or a caller manually
|
|
24
|
+
// pasting a transcript into one respond() call — is caught identically.
|
|
25
|
+
//
|
|
26
|
+
// (Category B — coincidental echo of a short, generic CURRENT-turn phrase
|
|
27
|
+
// against unrelated corpus content, unrelated to staleness — is a SEPARATE
|
|
28
|
+
// defect, not addressed here; see the investigation notes for why breadth
|
|
29
|
+
// and regionSupport both fail to discriminate it from genuine fusion.)
|
|
30
|
+
|
|
31
|
+
import { test } from "node:test";
|
|
32
|
+
import assert from "node:assert/strict";
|
|
33
|
+
import { Mind } from "../dist/src/index.js";
|
|
34
|
+
import { SQliteStore } from "../dist/src/store-sqlite.js";
|
|
35
|
+
import { fuseAttention } from "../dist/src/mind/reasoning.js";
|
|
36
|
+
import { gistOf } from "../dist/src/mind/primitives.js";
|
|
37
|
+
|
|
38
|
+
const enc = (s) => new TextEncoder().encode(s);
|
|
39
|
+
const dec = (b) =>
|
|
40
|
+
new TextDecoder().decode(b.filter((x) => x !== 0)).replace(/\s+/g, " ")
|
|
41
|
+
.trim();
|
|
42
|
+
|
|
43
|
+
// "greet" leads to "reply-greet" — a learnt exchange the query embeds BOTH
|
|
44
|
+
// halves of, exactly the shape addTurn produces (ask, then the system's own
|
|
45
|
+
// reply, concatenated raw). "red circle" is a genuine, unrelated second
|
|
46
|
+
// topic (test/34's own binding corpus) whose own continuation is nowhere
|
|
47
|
+
// in the query — the ordinary multi-topic case, which must still fuse.
|
|
48
|
+
const CORPUS = [
|
|
49
|
+
["greet", "reply-greet"],
|
|
50
|
+
["red", "is a color"],
|
|
51
|
+
["blue", "is a color"],
|
|
52
|
+
["circle", "is a shape"],
|
|
53
|
+
["square", "is a shape"],
|
|
54
|
+
["red circle", "answer alpha"],
|
|
55
|
+
["red square", "answer beta"],
|
|
56
|
+
["blue circle", "answer gamma"],
|
|
57
|
+
["blue square", "answer delta"],
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
const mk = (seed) =>
|
|
61
|
+
new Mind({ seed, store: new SQliteStore({ path: ":memory:" }) });
|
|
62
|
+
|
|
63
|
+
test("fuseAttention: a root whose continuation is already answered in the query is excluded", async () => {
|
|
64
|
+
const m = mk(1);
|
|
65
|
+
await m.ingest(CORPUS);
|
|
66
|
+
// "greet"'s own continuation, "reply-greet", is embedded right in the
|
|
67
|
+
// query — the ask and its answer both present, exactly like a
|
|
68
|
+
// conversation's own turn + reply. "red circle" is a genuine further
|
|
69
|
+
// topic with no such embedded answer.
|
|
70
|
+
const q = enc("greet reply-greet then red then circle");
|
|
71
|
+
const roots = await m.climbAttention(q, 24);
|
|
72
|
+
const greet = roots.find((r) => r.start === 0);
|
|
73
|
+
const redCircle = roots.find((r) => r.start !== 0);
|
|
74
|
+
assert.ok(greet, "expected the 'greet' root at query offset 0");
|
|
75
|
+
assert.ok(redCircle, "expected the 'red circle' binding root");
|
|
76
|
+
|
|
77
|
+
const guide = gistOf(m, q);
|
|
78
|
+
const primary = enc("PRIMARYANCHORTEXT");
|
|
79
|
+
// forest[0] is never independently projected (see reasoning.ts: it is
|
|
80
|
+
// treated as already primary's own source) — a placeholder keeps BOTH
|
|
81
|
+
// real roots in `rest`, where this filter actually applies.
|
|
82
|
+
const placeholder = { ...greet, start: q.length, end: q.length };
|
|
83
|
+
const pre = {
|
|
84
|
+
attention: async () => ({
|
|
85
|
+
roots: [placeholder, greet, redCircle],
|
|
86
|
+
ranked: [placeholder, greet, redCircle],
|
|
87
|
+
}),
|
|
88
|
+
guide,
|
|
89
|
+
};
|
|
90
|
+
const out = dec(await fuseAttention(m, q, primary, pre));
|
|
91
|
+
assert.ok(
|
|
92
|
+
!out.includes("reply-greet"),
|
|
93
|
+
`a root whose continuation is already answered in the query must not fuse in, got "${out}"`,
|
|
94
|
+
);
|
|
95
|
+
assert.ok(
|
|
96
|
+
out.includes("answer alpha"),
|
|
97
|
+
`a genuine further topic with no embedded answer must still fuse in, got "${out}"`,
|
|
98
|
+
);
|
|
99
|
+
await m.store.close();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("fuseAttention: ordinary multi-topic fusion (no embedded answers) is completely unaffected", async () => {
|
|
103
|
+
const m = mk(1);
|
|
104
|
+
await m.ingest(CORPUS);
|
|
105
|
+
const q = enc("red then circle");
|
|
106
|
+
const roots = await m.climbAttention(q, 24);
|
|
107
|
+
assert.equal(roots.length, 1, "expected the single joint binding root");
|
|
108
|
+
// Force a second, independent (unclimbed-style) inclusion path by
|
|
109
|
+
// reusing the SAME root twice at different synthetic positions, neither
|
|
110
|
+
// of which has an embedded answer — a pure regression check that the
|
|
111
|
+
// new filter doesn't fire when there is nothing to catch.
|
|
112
|
+
const guide = gistOf(m, q);
|
|
113
|
+
const primary = enc("PRIMARYANCHORTEXT");
|
|
114
|
+
const placeholder = { ...roots[0], start: q.length, end: q.length };
|
|
115
|
+
const pre = {
|
|
116
|
+
attention: async () => ({
|
|
117
|
+
roots: [placeholder, roots[0]],
|
|
118
|
+
ranked: [placeholder, roots[0]],
|
|
119
|
+
}),
|
|
120
|
+
guide,
|
|
121
|
+
};
|
|
122
|
+
const out = dec(await fuseAttention(m, q, primary, pre));
|
|
123
|
+
assert.ok(
|
|
124
|
+
out.includes("answer alpha"),
|
|
125
|
+
`an ordinary further topic must still fuse in, got "${out}"`,
|
|
126
|
+
);
|
|
127
|
+
await m.store.close();
|
|
128
|
+
});
|