@hviana/sema 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -7
- package/dist/src/alu/src/index.d.ts +1 -1
- package/dist/src/alu/src/index.js +1 -1
- package/dist/src/alu/src/parser.js +2 -6
- package/dist/src/alu/src/resonance.d.ts +13 -0
- package/dist/src/alu/src/resonance.js +41 -0
- package/dist/src/alu/test/alu.test.js +39 -0
- package/dist/src/bytes.d.ts +6 -2
- package/dist/src/bytes.js +10 -4
- package/dist/src/canon.js +44 -0
- package/dist/src/geometry.d.ts +19 -1
- package/dist/src/geometry.js +125 -141
- package/dist/src/meter.d.ts +20 -0
- package/dist/src/meter.js +21 -1
- package/dist/src/mind/articulation.js +14 -1
- package/dist/src/mind/attention.d.ts +12 -0
- package/dist/src/mind/attention.js +44 -16
- package/dist/src/mind/bridge.js +3 -3
- package/dist/src/mind/derivation.d.ts +40 -0
- package/dist/src/mind/derivation.js +34 -0
- package/dist/src/mind/graph-search.d.ts +89 -15
- package/dist/src/mind/graph-search.js +345 -174
- package/dist/src/mind/learning.js +1 -1
- package/dist/src/mind/mechanisms/cover.d.ts +19 -3
- package/dist/src/mind/mechanisms/cover.js +101 -58
- package/dist/src/mind/mechanisms/recall.js +0 -1
- package/dist/src/mind/mind.js +2 -2
- package/dist/src/mind/pipeline.d.ts +5 -1
- package/dist/src/mind/pipeline.js +175 -87
- package/dist/src/mind/primitives.d.ts +25 -5
- package/dist/src/mind/primitives.js +107 -44
- package/dist/src/mind/reasoning.d.ts +18 -4
- package/dist/src/mind/reasoning.js +445 -321
- package/dist/src/mind/recognition.js +29 -13
- package/dist/src/mind/resonance.js +1 -11
- package/dist/src/mind/traverse.d.ts +3 -3
- package/dist/src/mind/traverse.js +3 -3
- package/dist/src/mind/types.d.ts +7 -1
- package/dist/src/store-sqlite.d.ts +25 -0
- package/dist/src/store-sqlite.js +89 -1
- package/dist/src/store.d.ts +48 -4
- package/dist/src/store.js +86 -6
- package/docs/INDEX.md +18 -18
- package/docs/INVARIANTS.md +16 -16
- package/docs/architecture/bounded-reads.md +1 -1
- package/docs/architecture/caches.md +5 -4
- package/docs/architecture/closure.md +45 -5
- package/docs/architecture/cost-model.md +16 -0
- package/docs/architecture/factored-machinery.md +14 -13
- package/docs/architecture/fold-contract.md +51 -1
- package/docs/architecture/mechanism-market.md +21 -0
- package/docs/architecture/memoization.md +3 -3
- package/docs/architecture/meter.md +2 -1
- package/docs/architecture/saturation.md +12 -0
- package/docs/architecture/store.md +25 -2
- package/docs/failures/tempting-but-wrong.md +13 -2
- package/docs/harness/gates.md +12 -10
- package/docs/mechanisms/cover.md +23 -6
- package/jsr.json +1 -1
- package/package.json +1 -1
- package/src/alu/README.md +10 -2
- package/src/alu/src/index.ts +1 -0
- package/src/alu/src/parser.ts +6 -6
- package/src/alu/src/resonance.ts +42 -0
- package/src/alu/test/alu.test.ts +40 -0
- package/src/bytes.ts +13 -3
- package/src/canon.ts +40 -0
- package/src/geometry.ts +183 -154
- package/src/meter.ts +21 -1
- package/src/mind/articulation.ts +14 -2
- package/src/mind/attention.ts +47 -25
- package/src/mind/bridge.ts +3 -3
- package/src/mind/derivation.ts +77 -0
- package/src/mind/graph-search.ts +449 -221
- package/src/mind/learning.ts +1 -7
- package/src/mind/match.ts +1 -2
- package/src/mind/mechanisms/cast.ts +1 -2
- package/src/mind/mechanisms/cover.ts +149 -84
- package/src/mind/mechanisms/extraction.ts +1 -2
- package/src/mind/mechanisms/prefix-completion.ts +1 -1
- package/src/mind/mechanisms/recall.ts +1 -3
- package/src/mind/mechanisms/reference.ts +1 -1
- package/src/mind/mind.ts +5 -30
- package/src/mind/pipeline.ts +206 -102
- package/src/mind/primitives.ts +119 -43
- package/src/mind/reasoning.ts +558 -413
- package/src/mind/recognition.ts +24 -9
- package/src/mind/resonance.ts +2 -16
- package/src/mind/trace.ts +1 -1
- package/src/mind/traverse.ts +3 -3
- package/src/mind/types.ts +9 -11
- package/src/store-sqlite.ts +92 -1
- package/src/store.ts +113 -7
- package/test/105-derive-through-reports-its-refusal.test.mjs +8 -5
- package/test/106-the-join-fires.test.mjs +21 -0
- package/test/111-the-cover-assembly-is-counted.test.mjs +8 -5
- package/test/128-the-leads-somewhere-pair-agrees.test.mjs +18 -12
- package/test/136-the-two-named-limits.test.mjs +3 -2
- package/test/137-the-law-lives-once-and-below.test.mjs +21 -0
- package/test/148-exact-shortcuts-agree.test.mjs +188 -0
- package/test/149-the-closure-engine.test.mjs +138 -0
- package/test/150-the-join-is-output-sensitive.test.mjs +66 -0
- package/test/151-the-cover-pays-for-what-it-reaches.test.mjs +142 -0
- package/test/152-the-read-side-names-as-the-write-side.test.mjs +146 -0
- package/test/153-a-cheaper-bound-is-looked-at-first.test.mjs +155 -0
- package/test/24-generalization.test.mjs +32 -0
- package/test/36-bloom.test.mjs +53 -0
- package/test/37-cluster-dispersion-fusion.test.mjs +75 -0
- package/test/48-recognise-turn-connective.test.mjs +3 -2
- package/test/55-cost-meter.test.mjs +4 -4
- package/test/90-connector-read-cap.test.mjs +7 -7
package/src/mind/recognition.ts
CHANGED
|
@@ -11,13 +11,13 @@ import {
|
|
|
11
11
|
canonResolve,
|
|
12
12
|
foldTree,
|
|
13
13
|
gistOf,
|
|
14
|
-
latin1Key,
|
|
15
14
|
perceive,
|
|
16
15
|
resolve,
|
|
17
16
|
} from "./primitives.js";
|
|
18
17
|
import { atomIsHub, bearsEdge, corpusN, leadsSomewhere } from "./traverse.js";
|
|
19
18
|
import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
|
|
20
19
|
import { canonHash } from "../canon.js";
|
|
20
|
+
import { latin1 } from "../bytes.js";
|
|
21
21
|
import { isChunk, type Sema } from "../sema.js";
|
|
22
22
|
import type { Leaf, Site } from "./graph-search.js";
|
|
23
23
|
|
|
@@ -91,7 +91,7 @@ export function recognise(
|
|
|
91
91
|
// not silent), so it is emitted here directly rather than only inside
|
|
92
92
|
// recogniseImpl.
|
|
93
93
|
if (ctx.recogniseMemo) {
|
|
94
|
-
const key =
|
|
94
|
+
const key = latin1(bytes);
|
|
95
95
|
const hit = ctx.recogniseMemo.get(key);
|
|
96
96
|
if (hit !== undefined) {
|
|
97
97
|
if (ctx.meter) ctx.meter.recogniseHits++;
|
|
@@ -495,8 +495,20 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
|
495
495
|
// encoding is the identity, so the span's bytes ARE the branch key.
|
|
496
496
|
// `subarray` is a view — this allocates nothing per probe, and the
|
|
497
497
|
// bloom filter answers the misses without touching the database.
|
|
498
|
+
//
|
|
499
|
+
// Through ONE span prober (Store.flatSpans), because hashing the key was
|
|
500
|
+
// itself the quadratic: every probe hashed its span from the start, and
|
|
501
|
+
// the interior pass below sweeps up to `reach` ends past each endpoint —
|
|
502
|
+
// O(n · reach²) bytes hashed. Measured on the 31.7M-node store, one
|
|
503
|
+
// composition-regime response (#97 of the battery) probed 2,079,400
|
|
504
|
+
// spans and hashed 251,660,406 bytes for them. The prober extends each
|
|
505
|
+
// start's hash instead, so the same probes, answered identically, cost
|
|
506
|
+
// O(n · reach).
|
|
507
|
+
const spans = store.flatSpans?.(bytes) ?? null;
|
|
498
508
|
const flatProbe = (start: number, end: number): number | null =>
|
|
499
|
-
|
|
509
|
+
spans !== null
|
|
510
|
+
? spans(start, end)
|
|
511
|
+
: store.findFlatBranch
|
|
500
512
|
? store.findFlatBranch(bytes.subarray(start, end))
|
|
501
513
|
: store.findBranch(allLeafIds.slice(start, end));
|
|
502
514
|
// THE TWO ROUTES COST DIFFERENT THINGS, SO THEY ARE PRICED SEPARATELY.
|
|
@@ -635,11 +647,15 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
|
635
647
|
// simpler form won on that measurement.
|
|
636
648
|
const reach = chainReach(W) * W * W + 2 * radius;
|
|
637
649
|
if (ctx.meter) ctx.meter.recogniseInteriorGaps += ordered.length;
|
|
650
|
+
// Each end pairs only with the starts in [end − reach, end − W], in
|
|
651
|
+
// ascending order — the pairs, and the order the budget is spent in,
|
|
652
|
+
// of the all-pairs scan, without its O(n²) enumeration.
|
|
653
|
+
let lo = 0;
|
|
638
654
|
for (const end of ordered) {
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
const
|
|
642
|
-
if (
|
|
655
|
+
while (ordered[lo] < end - reach) lo++;
|
|
656
|
+
for (let i = lo; i < ordered.length; i++) {
|
|
657
|
+
const start = ordered[i];
|
|
658
|
+
if (end - start < W) break;
|
|
643
659
|
if (ctx.meter) ctx.meter.recogniseInteriorPairs++;
|
|
644
660
|
spend(start, end);
|
|
645
661
|
}
|
|
@@ -679,8 +695,7 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
|
|
|
679
695
|
// arithmetic, not evidence. Removing it wholesale was measured and
|
|
680
696
|
// REVERTED: it also drops legitimate multi-byte chains (the 12-byte
|
|
681
697
|
// "Eiffel Tower" site vanished with it). The premise is wrong but the
|
|
682
|
-
// trust it stood in for is real;
|
|
683
|
-
// See bench/README.md.
|
|
698
|
+
// trust it stood in for is real; the replacement signal follows.
|
|
684
699
|
//
|
|
685
700
|
// THE REPLACEMENT SIGNAL (2026-08-13): `leadsSomewhere` on the BYTE-EXACT
|
|
686
701
|
// branch the chain already found. The blanket off-boundary suppression is
|
package/src/mind/resonance.ts
CHANGED
|
@@ -3,12 +3,11 @@
|
|
|
3
3
|
// Address → Resonate → filter by Traverse/Read predicates → transform.
|
|
4
4
|
// Used by bridge, recallByResonance, pivotInto, meaningOf.
|
|
5
5
|
// (The graded locate() matcher formerly here lives in match.ts.)
|
|
6
|
-
import { rItem
|
|
7
|
-
import { decodeText } from "./rationale.js";
|
|
6
|
+
import { rItem } from "./trace.js";
|
|
8
7
|
|
|
9
8
|
import { cosine, Vec } from "../vec.js";
|
|
10
9
|
import { mergeThreshold } from "../geometry.js";
|
|
11
|
-
import { concat2, concatBytes, indexOf } from "../bytes.js";
|
|
10
|
+
import { concat2, concatBytes, indexOf, latin1 } from "../bytes.js";
|
|
12
11
|
import type { MindContext } from "./types.js";
|
|
13
12
|
import { gistOf, read, resolve, walkTree } from "./primitives.js";
|
|
14
13
|
import { perceive } from "./primitives.js";
|
|
@@ -17,8 +16,6 @@ import {
|
|
|
17
16
|
cachedRead,
|
|
18
17
|
type Junction,
|
|
19
18
|
junctionContainers,
|
|
20
|
-
junctionContainersFrom,
|
|
21
|
-
junctionSeeds,
|
|
22
19
|
junctionSynonyms,
|
|
23
20
|
walkCache,
|
|
24
21
|
} from "./junction.js";
|
|
@@ -128,17 +125,6 @@ function junctionEdges(
|
|
|
128
125
|
return out;
|
|
129
126
|
}
|
|
130
127
|
|
|
131
|
-
/** A byte string as a string, ONE code unit per byte — injective, so it is
|
|
132
|
-
* safe to build a cache key from. Chunked to keep the spread within the
|
|
133
|
-
* engine's argument limit on long contexts. */
|
|
134
|
-
function latin1(b: Uint8Array): string {
|
|
135
|
-
let s = "";
|
|
136
|
-
for (let i = 0; i < b.length; i += 4096) {
|
|
137
|
-
s += String.fromCharCode(...b.subarray(i, i + 4096));
|
|
138
|
-
}
|
|
139
|
-
return s;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
128
|
/** Per-response memo of bridge results, keyed by the response's lifecycle
|
|
143
129
|
* object (ctx.climbMemo — created fresh by respond() and nulled after, so
|
|
144
130
|
* entries can never outlive the read-only window they are valid in). The
|
package/src/mind/trace.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import type { MindContext } from "./types.js";
|
|
8
8
|
import { read } from "./primitives.js";
|
|
9
|
-
import type {
|
|
9
|
+
import type { DerivationStep } from "./graph-search.js";
|
|
10
10
|
import { decodeText } from "./rationale.js";
|
|
11
11
|
import type { RationaleItem } from "./rationale.js";
|
|
12
12
|
|
package/src/mind/traverse.ts
CHANGED
|
@@ -488,9 +488,9 @@ export function bearsEdge(ctx: MindContext, id: number): boolean {
|
|
|
488
488
|
return cachedHasNext(ctx, id, getStructCache(ctx));
|
|
489
489
|
}
|
|
490
490
|
|
|
491
|
-
/** Whether a node LEADS SOMEWHERE —
|
|
492
|
-
*
|
|
493
|
-
* that
|
|
491
|
+
/** Whether a node LEADS SOMEWHERE — the store's admission predicate
|
|
492
|
+
* ({@link Store.leadsSomewhere}: edge or halo) with its edge tier memoised for
|
|
493
|
+
* the response. Recognition filters sites with it (cover.md): a form that
|
|
494
494
|
* leads nowhere contributes nothing to any derivation. Runs once per candidate
|
|
495
495
|
* span on the recognition hot path — `hasNext` is cached per response (the same
|
|
496
496
|
* flat-branch ids are probed across prefix variants by canonicalChunkId).
|
package/src/mind/types.ts
CHANGED
|
@@ -10,15 +10,7 @@ import type { Space } from "../sema.js";
|
|
|
10
10
|
import type { Alphabet } from "../alphabet.js";
|
|
11
11
|
import type { MindConfig } from "../config.js";
|
|
12
12
|
import type { Meter } from "../meter.js";
|
|
13
|
-
import type {
|
|
14
|
-
ComputedResult,
|
|
15
|
-
DerivationItem,
|
|
16
|
-
DerivationStep,
|
|
17
|
-
GraphSearch,
|
|
18
|
-
Leaf,
|
|
19
|
-
Seg,
|
|
20
|
-
Site,
|
|
21
|
-
} from "./graph-search.js";
|
|
13
|
+
import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
|
|
22
14
|
import type { Rationale } from "./rationale.js";
|
|
23
15
|
import type { ContentFold, Grid } from "../geometry.js";
|
|
24
16
|
|
|
@@ -33,7 +25,7 @@ export interface DepositCacheEntry {
|
|
|
33
25
|
/** The plain content fold's reusable segment state. */
|
|
34
26
|
content: ContentFold;
|
|
35
27
|
}
|
|
36
|
-
import { bytesEqual, concatBytes
|
|
28
|
+
import { bytesEqual, concatBytes } from "../bytes.js";
|
|
37
29
|
import { restates } from "./derivation.js";
|
|
38
30
|
import { dominates } from "../geometry.js";
|
|
39
31
|
|
|
@@ -199,7 +191,13 @@ export interface Attention {
|
|
|
199
191
|
* a genuine further topic is named in its own distinctive wording
|
|
200
192
|
* somewhere the query's scaffolding does not reach, always a SEPARATE
|
|
201
193
|
* cluster from whatever else corroborates it. See
|
|
202
|
-
* test/37-cluster-dispersion-fusion.test.mjs.
|
|
194
|
+
* test/37-cluster-dispersion-fusion.test.mjs.
|
|
195
|
+
*
|
|
196
|
+
* Read from the VOTES, this is a lossy witness: a region votes once, for its
|
|
197
|
+
* top anchor, so a place can be lost to a tie or won through an accident.
|
|
198
|
+
* Fusion therefore also asks the root's CONTEXT the same question at window
|
|
199
|
+
* scale (reasoning.ts `sharedPlaces`) and trusts a root that either reading
|
|
200
|
+
* finds in two places. */
|
|
203
201
|
clusters: number;
|
|
204
202
|
}
|
|
205
203
|
|
package/src/store-sqlite.ts
CHANGED
|
@@ -146,6 +146,17 @@ CREATE TABLE IF NOT EXISTS canon (
|
|
|
146
146
|
id INTEGER NOT NULL,
|
|
147
147
|
PRIMARY KEY (h, id)
|
|
148
148
|
) WITHOUT ROWID;
|
|
149
|
+
-- The canon index's negative filter, persisted so an open does not rescan the
|
|
150
|
+
-- h column (seconds on a trained store). Written in the SAME transaction as
|
|
151
|
+
-- the canon rows it covers, stamped with the meta 'canon.upto' of that commit;
|
|
152
|
+
-- a stamp that disagrees with the meta (rows a writer added without it) makes
|
|
153
|
+
-- the filter stale, and it is rebuilt from the table instead.
|
|
154
|
+
CREATE TABLE IF NOT EXISTS canon_bloom (
|
|
155
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
156
|
+
bits BLOB NOT NULL,
|
|
157
|
+
n INTEGER NOT NULL,
|
|
158
|
+
upto TEXT NOT NULL
|
|
159
|
+
);
|
|
149
160
|
-- CONSTITUENT SKETCH (Store.sketchGet/sketchPut): the bottom-k minimal
|
|
150
161
|
-- constituents of a node's subtree, k = √D, chosen by identity hash. The blob is
|
|
151
162
|
-- a packed int32 little-endian run, already in hash order; an EMPTY blob is a
|
|
@@ -251,6 +262,20 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
251
262
|
private _bloom: NodeBloom | null = null;
|
|
252
263
|
/** Dedup probes answered by the filter alone this session (observability). */
|
|
253
264
|
bloomSkips = 0;
|
|
265
|
+
/** The same negative filter over the CANON index's key hashes. Recognition's
|
|
266
|
+
* canonical admission and the join's canonical entity scan ask `canonFind`
|
|
267
|
+
* once per probed span, and almost every answer is "no such key" — on an
|
|
268
|
+
* index that is often EMPTY (it is built only by `buildCanonIndex`).
|
|
269
|
+
* Loaded on first use from `canon_bloom` when its stamp matches the meta,
|
|
270
|
+
* else built by one sequential scan of the h column; kept exact on
|
|
271
|
+
* `canonAdd` (rebuilt bigger from the table, uncommitted rows included,
|
|
272
|
+
* when growth saturates it) and persisted with the commit that wrote the
|
|
273
|
+
* rows. The canon table is never deleted from, so the filter can never
|
|
274
|
+
* hold a false negative: a miss it reports is a miss the query would have
|
|
275
|
+
* returned. */
|
|
276
|
+
private _canonBloom: NodeBloom | null = null;
|
|
277
|
+
/** The in-memory canon filter differs from the persisted one. */
|
|
278
|
+
private _canonBloomDirty = false;
|
|
254
279
|
|
|
255
280
|
private _insertNode: any = null;
|
|
256
281
|
private _insertKid: any = null;
|
|
@@ -509,7 +534,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
509
534
|
}
|
|
510
535
|
|
|
511
536
|
protected _dbClose(): void {
|
|
512
|
-
if (this.sqlite)
|
|
537
|
+
if (this.sqlite) {
|
|
538
|
+
this._bloomPersist();
|
|
539
|
+
this._canonBloomPersist();
|
|
540
|
+
}
|
|
513
541
|
if (this.content) {
|
|
514
542
|
this.content.close();
|
|
515
543
|
this.content = null;
|
|
@@ -534,6 +562,8 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
534
562
|
|
|
535
563
|
protected _dbCommitTx(): void {
|
|
536
564
|
if (!this._inTx || !this.sqlite) return;
|
|
565
|
+
// The canon filter commits WITH the rows it covers — see canon_bloom.
|
|
566
|
+
this._canonBloomPersist();
|
|
537
567
|
this._inTx = false; // clear first so a throw can't wedge us mid-commit
|
|
538
568
|
this.sqlite.exec("COMMIT");
|
|
539
569
|
}
|
|
@@ -615,6 +645,15 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
615
645
|
return row ? row.id : null;
|
|
616
646
|
}
|
|
617
647
|
|
|
648
|
+
/** The node filter alone: never a false negative (every inserted hash is
|
|
649
|
+
* added before any probe can see it), so `false` is exact. */
|
|
650
|
+
protected override _dbFlatMayExist(h: number, bytes: Uint8Array): boolean {
|
|
651
|
+
if (this._bloom === null) return super._dbFlatMayExist(h, bytes);
|
|
652
|
+
if (this._bloom.mightContain(h)) return true;
|
|
653
|
+
this.bloomSkips++;
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
|
|
618
657
|
protected _dbFindBranchByKids(
|
|
619
658
|
h: number,
|
|
620
659
|
packed: Uint8Array,
|
|
@@ -1053,9 +1092,61 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
1053
1092
|
// bulk index build coalesces instead of paying autocommit per row.
|
|
1054
1093
|
this._dbBeginTx();
|
|
1055
1094
|
this._insCanon.run(h, id);
|
|
1095
|
+
const b = this._canonFilter();
|
|
1096
|
+
b.add(h);
|
|
1097
|
+
// Rebuilt bigger at once, from the table as THIS connection sees it — the
|
|
1098
|
+
// persisted filter cannot stand in, it lacks this transaction's rows.
|
|
1099
|
+
if (b.saturated) this._canonBloom = this._canonScan();
|
|
1100
|
+
this._canonBloomDirty = true;
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
/** The canon filter: the persisted one when its stamp matches the meta,
|
|
1104
|
+
* else built from the index as it stands. */
|
|
1105
|
+
private _canonFilter(): NodeBloom {
|
|
1106
|
+
if (this._canonBloom === null) {
|
|
1107
|
+
const row = this.sqlite!.prepare(
|
|
1108
|
+
"SELECT bits, n, upto FROM canon_bloom WHERE id = 1",
|
|
1109
|
+
).get() as { bits: Uint8Array; n: number; upto: string } | undefined;
|
|
1110
|
+
if (row !== undefined && row.upto === this._canonStamp()) {
|
|
1111
|
+
const b = new NodeBloom(31 - Math.clz32(row.bits.length * 8));
|
|
1112
|
+
b.bits.set(row.bits);
|
|
1113
|
+
b.n = row.n;
|
|
1114
|
+
this._canonBloom = b;
|
|
1115
|
+
} else {
|
|
1116
|
+
this._canonBloom = this._canonScan();
|
|
1117
|
+
this._canonBloomDirty = true;
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
return this._canonBloom;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
private _canonScan(): NodeBloom {
|
|
1124
|
+
const b = new NodeBloom(bloomLog2For(this.canonCount()));
|
|
1125
|
+
const scan = this.sqlite!.prepare("SELECT h FROM canon");
|
|
1126
|
+
scan.setReturnArrays(true);
|
|
1127
|
+
for (const r of scan.iterate() as IterableIterator<[number]>) b.add(r[0]);
|
|
1128
|
+
return b;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
/** What a persisted canon filter is stamped with: the incremental build's
|
|
1132
|
+
* cursor, which every canon writer advances in the transaction it writes. */
|
|
1133
|
+
private _canonStamp(): string {
|
|
1134
|
+
return this._dbGetMeta("canon.upto") ?? "";
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
private _canonBloomPersist(): void {
|
|
1138
|
+
const b = this._canonBloom;
|
|
1139
|
+
if (b === null || !this._canonBloomDirty || !this.sqlite) return;
|
|
1140
|
+
this.sqlite.prepare(
|
|
1141
|
+
"INSERT INTO canon_bloom (id, bits, n, upto) VALUES (1, ?, ?, ?) " +
|
|
1142
|
+
"ON CONFLICT(id) DO UPDATE SET bits = excluded.bits, " +
|
|
1143
|
+
"n = excluded.n, upto = excluded.upto",
|
|
1144
|
+
).run(b.bits, b.n, this._canonStamp());
|
|
1145
|
+
this._canonBloomDirty = false;
|
|
1056
1146
|
}
|
|
1057
1147
|
|
|
1058
1148
|
canonFind(h: number): number[] {
|
|
1149
|
+
if (!this._canonFilter().mightContain(h)) return [];
|
|
1059
1150
|
if (!this._selCanon) {
|
|
1060
1151
|
this._selCanon = this.sqlite!.prepare(
|
|
1061
1152
|
"SELECT id FROM canon WHERE h = ?",
|
package/src/store.ts
CHANGED
|
@@ -29,9 +29,10 @@
|
|
|
29
29
|
// essentials required for SQLite + VectorDatabase communication.
|
|
30
30
|
|
|
31
31
|
import { addInto, copy, dot, normalize, Vec } from "./vec.js";
|
|
32
|
-
import {
|
|
32
|
+
import type { StoreConfig } from "./config.js";
|
|
33
33
|
import { identityBar } from "./geometry.js";
|
|
34
34
|
import type { Meter } from "./meter.js";
|
|
35
|
+
import { latin1 } from "./bytes.js";
|
|
35
36
|
|
|
36
37
|
/** A node id: a dense, non-negative integer assigned in creation order. */
|
|
37
38
|
export type NodeId = number;
|
|
@@ -340,6 +341,20 @@ export interface Store {
|
|
|
340
341
|
* bytes — the allocation-free probe span scanners use. Optional: a store
|
|
341
342
|
* without it is simply probed through `findBranch`. */
|
|
342
343
|
findFlatBranch?(bytes: Uint8Array): NodeId | null;
|
|
344
|
+
/** Whether a flat branch with these bytes MAY exist. `false` is EXACT — no
|
|
345
|
+
* such node exists, and a {@link findFlatBranch} would return null; `true`
|
|
346
|
+
* means only that a lookup is needed. The existence half of the probe,
|
|
347
|
+
* without its verification: a caller that will resolve the bytes anyway
|
|
348
|
+
* (and so verify them) asks this to refuse a miss without paying for the
|
|
349
|
+
* lookup a hit would repeat. Optional, like the probe it answers for. */
|
|
350
|
+
flatBranchMayExist?(bytes: Uint8Array): boolean;
|
|
351
|
+
/** A {@link findFlatBranch} over spans of ONE buffer: `probe(start, end)`
|
|
352
|
+
* answers exactly `findFlatBranch(bytes.subarray(start, end))`, but a span's
|
|
353
|
+
* content hash extends the one its start was last probed at, so a scanner
|
|
354
|
+
* that sweeps ends upward per start pays O(1) per probe instead of
|
|
355
|
+
* O(span). The buffer must not change while the prober is used.
|
|
356
|
+
* Optional, like the probe it accelerates. */
|
|
357
|
+
flatSpans?(bytes: Uint8Array): (start: number, end: number) => NodeId | null;
|
|
343
358
|
/** The branch nodes that list `id` among their children — the reverse of
|
|
344
359
|
* `get(id).kids`. Lets the structural DAG be climbed upward, from a
|
|
345
360
|
* recognised fragment to the larger learned forms that contain it. */
|
|
@@ -499,6 +514,12 @@ export interface Store {
|
|
|
499
514
|
* the search's fuse guard) must ask this instead: one row read, a mass
|
|
500
515
|
* compare, no decode. */
|
|
501
516
|
hasHalo(id: NodeId): boolean;
|
|
517
|
+
/** THE ADMISSION PREDICATE — whether `id` LEADS SOMEWHERE: it bears a
|
|
518
|
+
* continuation edge ({@link hasNext}) or a halo ({@link hasHalo}). A form
|
|
519
|
+
* that leads nowhere contributes nothing to any derivation. This is the ONE
|
|
520
|
+
* raw definition; `traverse.ts`'s `leadsSomewhere` is the same predicate with
|
|
521
|
+
* its edge tier memoised for the response. Two point probes at most. */
|
|
522
|
+
leadsSomewhere(id: NodeId): boolean;
|
|
502
523
|
/** How many episode signatures were poured into `id`'s halo — the DIRECT
|
|
503
524
|
* measure of distributional evidence (each training pair pours once, so
|
|
504
525
|
* repetition counts, unlike {@link prevCount}, which counts DISTINCT
|
|
@@ -662,11 +683,14 @@ export function unpackKids(blob: Uint8Array): NodeId[] {
|
|
|
662
683
|
|
|
663
684
|
/** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
|
|
664
685
|
* on. Collisions are resolved by verifying the stored blob, never trusted. */
|
|
686
|
+
const FNV_OFFSET = 0x811c9dc5 >>> 0;
|
|
687
|
+
const FNV_PRIME = 0x01000193;
|
|
688
|
+
|
|
665
689
|
function hashOf(bytes: Uint8Array): number {
|
|
666
|
-
let h =
|
|
690
|
+
let h = FNV_OFFSET;
|
|
667
691
|
for (let i = 0; i < bytes.length; i++) {
|
|
668
692
|
h ^= bytes[i];
|
|
669
|
-
h = Math.imul(h,
|
|
693
|
+
h = Math.imul(h, FNV_PRIME) >>> 0;
|
|
670
694
|
}
|
|
671
695
|
return h >>> 0;
|
|
672
696
|
}
|
|
@@ -957,6 +981,9 @@ export abstract class AbstractStore implements Store {
|
|
|
957
981
|
/** Exact-content dedup: content-key → node id. Intrinsic compression. */
|
|
958
982
|
protected readonly _leafKey: BoundedMap<string, NodeId>;
|
|
959
983
|
protected readonly _branchKey: BoundedMap<string, NodeId>;
|
|
984
|
+
/** {@link findFlatBranch}'s hits, keyed by the bytes themselves (latin1 —
|
|
985
|
+
* exact, unlike the hash keys above). */
|
|
986
|
+
protected readonly _flatKey: BoundedMap<string, NodeId>;
|
|
960
987
|
/** Reconstructed-bytes read cache (regenerable), keyed by node id. */
|
|
961
988
|
protected readonly _bytesCache: BoundedMap<NodeId, Uint8Array>;
|
|
962
989
|
/** contentLen memo — content is immutable, so entries never invalidate. */
|
|
@@ -1063,6 +1090,12 @@ export abstract class AbstractStore implements Store {
|
|
|
1063
1090
|
this.compactEveryNWrites = config.compactEveryNWrites;
|
|
1064
1091
|
this._leafKey = new BoundedMap(config.dedupCacheMax);
|
|
1065
1092
|
this._branchKey = new BoundedMap(config.dedupCacheMax);
|
|
1093
|
+
this._flatKey = new BoundedMap(
|
|
1094
|
+
config.dedupCacheMax,
|
|
1095
|
+
undefined,
|
|
1096
|
+
"lru",
|
|
1097
|
+
"clock",
|
|
1098
|
+
);
|
|
1066
1099
|
this._bytesCache = new BoundedMap(
|
|
1067
1100
|
config.bytesCacheMax,
|
|
1068
1101
|
(v) => v.byteLength,
|
|
@@ -1400,12 +1433,80 @@ export abstract class AbstractStore implements Store {
|
|
|
1400
1433
|
* answers most of those with no I/O at all, so the allocations dominated.
|
|
1401
1434
|
*
|
|
1402
1435
|
* Pass a subarray: it is a view, so a caller scanning spans of a query
|
|
1403
|
-
* allocates nothing per probe.
|
|
1404
|
-
*
|
|
1405
|
-
*
|
|
1436
|
+
* allocates nothing per probe that misses. HITS ARE MEMOIZED, MISSES ARE
|
|
1437
|
+
* NOT: the negative filter answers first, so a span that is not stored
|
|
1438
|
+
* builds no key; one that may be pays a key and then usually skips the
|
|
1439
|
+
* lookup — the spans the identity fold names are segments, a few bytes
|
|
1440
|
+
* each, asked over and over by every span that contains them. A node, once
|
|
1441
|
+
* minted, keeps its bytes and its id, so a cached hit never goes stale. */
|
|
1406
1442
|
findFlatBranch(bytes: Uint8Array): NodeId | null {
|
|
1443
|
+
return this._findFlatHashed(hashOf(bytes), bytes);
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
/** {@link Store.flatSpans}. Each start keeps the hash of the span it was
|
|
1447
|
+
* last probed to, plus the hash one byte short of it; a probe ending at or
|
|
1448
|
+
* past that end extends it, one ending one byte short of it reuses the
|
|
1449
|
+
* second, and any other probe hashes from the start. `hashOf` is FNV-1a,
|
|
1450
|
+
* a left fold over the bytes, so an extended hash IS the span's hash —
|
|
1451
|
+
* same key, same filter answer, same lookup. */
|
|
1452
|
+
flatSpans(bytes: Uint8Array): (start: number, end: number) => NodeId | null {
|
|
1453
|
+
const n = bytes.length;
|
|
1454
|
+
const runEnd = new Int32Array(n + 1).fill(-1);
|
|
1455
|
+
const runH = new Uint32Array(n + 1);
|
|
1456
|
+
const shortH = new Uint32Array(n + 1);
|
|
1457
|
+
return (start, end) => {
|
|
1458
|
+
let h: number;
|
|
1459
|
+
let from: number;
|
|
1460
|
+
const e0 = runEnd[start];
|
|
1461
|
+
if (e0 >= 0 && e0 <= end) {
|
|
1462
|
+
h = runH[start];
|
|
1463
|
+
from = e0;
|
|
1464
|
+
} else if (e0 >= 0 && e0 - 1 === end && end > start) {
|
|
1465
|
+
h = shortH[start];
|
|
1466
|
+
from = end;
|
|
1467
|
+
} else {
|
|
1468
|
+
h = FNV_OFFSET;
|
|
1469
|
+
from = start;
|
|
1470
|
+
}
|
|
1471
|
+
let prev = h;
|
|
1472
|
+
for (let i = from; i < end; i++) {
|
|
1473
|
+
prev = h;
|
|
1474
|
+
h = Math.imul(h ^ bytes[i], FNV_PRIME) >>> 0;
|
|
1475
|
+
}
|
|
1476
|
+
if (from < end) {
|
|
1477
|
+
runEnd[start] = end;
|
|
1478
|
+
runH[start] = h;
|
|
1479
|
+
shortH[start] = prev;
|
|
1480
|
+
}
|
|
1481
|
+
return this._findFlatHashed(h, bytes.subarray(start, end));
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/** {@link findFlatBranch} past the hash: `h` must be `hashOf(bytes)`. */
|
|
1486
|
+
private _findFlatHashed(h: number, bytes: Uint8Array): NodeId | null {
|
|
1407
1487
|
if (this.meter) this.meter.branchLookups++;
|
|
1408
|
-
|
|
1488
|
+
if (!this._dbFlatMayExist(h, bytes)) return null;
|
|
1489
|
+
const key = bytes.length <= DEDUP_KEY_MAX ? latin1(bytes) : null;
|
|
1490
|
+
if (key !== null) {
|
|
1491
|
+
const hit = this._flatKey.get(key);
|
|
1492
|
+
if (hit !== undefined) return hit;
|
|
1493
|
+
}
|
|
1494
|
+
const id = this._dbFindBranchByLeaf(h, bytes);
|
|
1495
|
+
if (id !== null && key !== null) this._flatKey.set(key, id);
|
|
1496
|
+
return id;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
/** {@link Store.flatBranchMayExist} — the backend's negative filter when it
|
|
1500
|
+
* keeps one ({@link _dbFlatMayExist}), else the lookup itself. */
|
|
1501
|
+
flatBranchMayExist(bytes: Uint8Array): boolean {
|
|
1502
|
+
return this._dbFlatMayExist(hashOf(bytes), bytes);
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
/** Default: no filter, so anything MAY exist and the lookup decides. A
|
|
1506
|
+
* backend with a negative filter over node hashes overrides this to answer
|
|
1507
|
+
* from the filter alone. */
|
|
1508
|
+
protected _dbFlatMayExist(_h: number, _bytes: Uint8Array): boolean {
|
|
1509
|
+
return true;
|
|
1409
1510
|
}
|
|
1410
1511
|
|
|
1411
1512
|
findBranch(kids: NodeId[]): NodeId | null {
|
|
@@ -2310,6 +2411,11 @@ export abstract class AbstractStore implements Store {
|
|
|
2310
2411
|
return r !== null && r.mass >= this.minHaloMass;
|
|
2311
2412
|
}
|
|
2312
2413
|
|
|
2414
|
+
/** {@link Store.leadsSomewhere} — edge first (the cheaper, commoner probe). */
|
|
2415
|
+
leadsSomewhere(id: NodeId): boolean {
|
|
2416
|
+
return this.hasNext(id) || this.hasHalo(id);
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2313
2419
|
async pourHalo(id: NodeId, add: Vec): Promise<void> {
|
|
2314
2420
|
await this._ensureReady();
|
|
2315
2421
|
// A node with a halo is a genuine resonance target — the consensus climb
|
|
@@ -52,10 +52,13 @@ const misses = (steps) =>
|
|
|
52
52
|
|
|
53
53
|
test("the join's refusal is reported, naming the candidate and tail it tried", async () => {
|
|
54
54
|
const mind = await chain();
|
|
55
|
-
// The THREE-relation query: the
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
55
|
+
// The THREE-relation query: the chain joins through to Stockholm, and on the
|
|
56
|
+
// way the fact it stands on ("…Gustaf Molander is Sweden.") offers its other
|
|
57
|
+
// entity, "gustaf molander", with the remaining tail " capital" — a key no
|
|
58
|
+
// deposit learnt, so that join is refused, and that refusal is observable.
|
|
59
|
+
// A join is priced only for a fact a derivation STANDS ON (graph-search.ts,
|
|
60
|
+
// solve), so the refusal reported is that fact's — not one from a fact the
|
|
61
|
+
// exploration merely reached.
|
|
59
62
|
const steps = [];
|
|
60
63
|
await mind.respond("eva director country capital", (s) => steps.push(s));
|
|
61
64
|
const got = misses(steps);
|
|
@@ -63,7 +66,7 @@ test("the join's refusal is reported, naming the candidate and tail it tried", a
|
|
|
63
66
|
const named = got.some((s) => {
|
|
64
67
|
const parts = (s.inputs ?? []).map((i) => String(i.text));
|
|
65
68
|
return parts.some((t) => t.includes("gustaf molander")) &&
|
|
66
|
-
parts.some((t) => t.includes("
|
|
69
|
+
parts.some((t) => t.includes("capital"));
|
|
67
70
|
});
|
|
68
71
|
assert.ok(named, "the report must name the candidate and the tail it tried");
|
|
69
72
|
// …and WHERE the candidate came from: a refusal that names only bytes leaves
|
|
@@ -92,3 +92,24 @@ test("a query with no tail is answered directly, not by a join", async () => {
|
|
|
92
92
|
);
|
|
93
93
|
await mind.store.close();
|
|
94
94
|
});
|
|
95
|
+
|
|
96
|
+
test("the asker's wording that already holds the answer's form is not spliced into it", async () => {
|
|
97
|
+
// Articulation revoices an answer form in the asker's words when the two
|
|
98
|
+
// keep the same company (halo). `eva` and `eva director` keep the SAME
|
|
99
|
+
// company here — both lead to the same fact — but the asker's wording holds
|
|
100
|
+
// the form plus a further word: splicing it in is not a re-voicing, it adds
|
|
101
|
+
// the asker's other words to the answer. The DAG cannot see the containment
|
|
102
|
+
// (the fold cuts `eva director` as `eva d|irector`), so it is read off the
|
|
103
|
+
// bytes. Observed before: "The director of eva director is Gustaf Molander."
|
|
104
|
+
const lower = "The director of eva is Gustaf Molander.";
|
|
105
|
+
const store = new SQliteStore({ path: ":memory:" });
|
|
106
|
+
const mind = new Mind({ seed: 7, store });
|
|
107
|
+
await mind.ingest([
|
|
108
|
+
["eva", lower],
|
|
109
|
+
["eva director", lower],
|
|
110
|
+
["gustaf molander", F2],
|
|
111
|
+
["gustaf molander country", F2],
|
|
112
|
+
]);
|
|
113
|
+
assert.equal(text(await mind.respond("eva director")).trim(), lower);
|
|
114
|
+
await mind.store.close();
|
|
115
|
+
});
|
|
@@ -46,15 +46,18 @@ test("the pairwise assembly is counted, with no rationale attached", async () =>
|
|
|
46
46
|
});
|
|
47
47
|
|
|
48
48
|
test("the n-ary interior's allowance is counted", async () => {
|
|
49
|
-
// Three sites in one query: the n-ary path passes an
|
|
50
|
-
// with the intermediate answers' bytes — the quantity
|
|
51
|
-
// hub query's answers.
|
|
49
|
+
// Three touching sites in one query, each answered: the n-ary path passes an
|
|
50
|
+
// allowance that grows with the intermediate answers' bytes — the quantity
|
|
51
|
+
// that would grow with a hub query's answers. The connectors are bridged
|
|
52
|
+
// only for the pairs the search REACHES (ConnectorLicence), so the lightest
|
|
53
|
+
// cover must stand on all three answers: a query recognised whole (an
|
|
54
|
+
// `ab cd` learnt as its own context) never reaches its parts' pair.
|
|
52
55
|
const mind = await fixture([
|
|
53
56
|
["ab", "x"],
|
|
54
57
|
["cd", "y"],
|
|
55
|
-
["
|
|
58
|
+
["ef", "z"],
|
|
56
59
|
]);
|
|
57
|
-
await mind.respondText("
|
|
60
|
+
await mind.respondText("abcdef");
|
|
58
61
|
const c = mind.lastCost?.counters ?? {};
|
|
59
62
|
assert.ok(
|
|
60
63
|
(c.coverAllowanceBytes ?? 0) > 0,
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
// 128-the-leads-somewhere-pair-agrees.test.mjs — THE PAIR, PINNED
|
|
2
2
|
//
|
|
3
|
-
// "Does this node lead somewhere?"
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// through the observable surface only:
|
|
9
|
-
// `
|
|
3
|
+
// "Does this node lead somewhere?" has ONE raw definition — the store's
|
|
4
|
+
// `leadsSomewhere` (`hasNext || hasHalo`) — and ONE memoised reading of it,
|
|
5
|
+
// `traverse.ts`'s `leadsSomewhere` (`cachedHasNext || hasHalo`), which the Mind
|
|
6
|
+
// lends to the search. The callers that used to spell the pair out by hand
|
|
7
|
+
// (`primitives.ts`, the search's bare-host fallback) now ask the store. This
|
|
8
|
+
// pins the AGREEMENT, node by node, through the observable surface only:
|
|
9
|
+
// `mind.leadsSomewhere(id)` against `store.leadsSomewhere(id)`, and both against
|
|
10
|
+
// the two probes the definition is made of.
|
|
10
11
|
//
|
|
11
12
|
// It is NOT a claim that the two are the same function: one reads a memo, the
|
|
12
13
|
// other reads the store. The claim is that the memoised reading and the store
|
|
13
|
-
// reading agree — what
|
|
14
|
-
//
|
|
14
|
+
// reading agree — what would break first if `cachedHasNext` ever outlived a
|
|
15
|
+
// store write.
|
|
15
16
|
//
|
|
16
17
|
// THE REACH OF THIS PIN, said as it is: halos are poured on the WRITE side, so
|
|
17
18
|
// a synthetic corpus of this shape carries NONE (measured: 0 nodes with a halo
|
|
@@ -49,16 +50,21 @@ test("the two homes of 'leads somewhere' agree, node by node", async () => {
|
|
|
49
50
|
let withHalo = 0;
|
|
50
51
|
for (let id = 0; id < total; id++) {
|
|
51
52
|
const named = m.leadsSomewhere(id);
|
|
52
|
-
const
|
|
53
|
+
const stored = m.store.leadsSomewhere(id);
|
|
53
54
|
const hasEdge = m.store.hasNext(id);
|
|
54
55
|
const hasHalo = m.store.hasHalo(id);
|
|
55
56
|
if (hasEdge) withEdge++;
|
|
56
57
|
if (hasHalo) withHalo++;
|
|
58
|
+
assert.equal(
|
|
59
|
+
stored,
|
|
60
|
+
hasEdge || hasHalo,
|
|
61
|
+
`node ${id}: the store's definition must be its two probes`,
|
|
62
|
+
);
|
|
57
63
|
assert.equal(
|
|
58
64
|
named,
|
|
59
|
-
|
|
65
|
+
stored,
|
|
60
66
|
`node ${id}: traverse.leadsSomewhere says ${named}, ` +
|
|
61
|
-
`
|
|
67
|
+
`the store says ${stored} (hasNext=${hasEdge}, hasHalo=${hasHalo})`,
|
|
62
68
|
);
|
|
63
69
|
}
|
|
64
70
|
|
|
@@ -77,8 +77,9 @@ test("136.1 the fuse gate asks the LAW — one condition, one home, one less sum
|
|
|
77
77
|
);
|
|
78
78
|
assert.match(
|
|
79
79
|
src,
|
|
80
|
-
/
|
|
81
|
-
"the fuse gate must ask the law's `closed
|
|
80
|
+
/engages: \(d: DerivationState\) => !closed\(d\)/,
|
|
81
|
+
"the fuse gate must ask the law's `closed` of the state fusion is offered " +
|
|
82
|
+
"against (the fusion layer engages only while it is open) — the condition the state " +
|
|
82
83
|
"already carries. If it went back to summing `unaccounted(explained)`, that " +
|
|
83
84
|
"is a second reading of one condition AND an extra sum per response: 136.3 " +
|
|
84
85
|
"measures the equivalence, so re-read it before restoring the total",
|