@hviana/sema 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/train_base.js +42 -9
- package/dist/src/alu/src/parser.d.ts +9 -0
- package/dist/src/alu/src/parser.js +110 -2
- package/dist/src/canon.d.ts +26 -0
- package/dist/src/canon.js +57 -0
- package/dist/src/geometry.d.ts +13 -2
- package/dist/src/geometry.js +101 -2
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/mind/attention.js +11 -7
- package/dist/src/mind/learning.js +33 -1
- package/dist/src/mind/match.d.ts +4 -2
- package/dist/src/mind/match.js +30 -6
- package/dist/src/mind/mechanisms/cast.js +18 -4
- package/dist/src/mind/mechanisms/confluence.js +13 -1
- package/dist/src/mind/mechanisms/recall.js +115 -31
- package/dist/src/mind/mind.d.ts +138 -12
- package/dist/src/mind/mind.js +284 -22
- package/dist/src/mind/primitives.d.ts +22 -2
- package/dist/src/mind/primitives.js +95 -6
- package/dist/src/mind/recognition.js +31 -8
- package/dist/src/mind/traverse.d.ts +13 -0
- package/dist/src/mind/traverse.js +41 -0
- package/dist/src/mind/types.d.ts +33 -21
- package/dist/src/store-sqlite.d.ts +10 -0
- package/dist/src/store-sqlite.js +58 -0
- package/dist/src/store.d.ts +23 -0
- package/dist/src/store.js +13 -2
- package/example/train_base.ts +49 -9
- package/index.html +65 -0
- package/package.json +1 -1
- package/src/alu/src/parser.ts +105 -4
- package/src/canon.ts +65 -0
- package/src/geometry.ts +105 -1
- package/src/index.ts +1 -0
- package/src/mind/attention.ts +11 -6
- package/src/mind/learning.ts +39 -1
- package/src/mind/match.ts +29 -6
- package/src/mind/mechanisms/cast.ts +21 -6
- package/src/mind/mechanisms/confluence.ts +15 -1
- package/src/mind/mechanisms/recall.ts +132 -41
- package/src/mind/mind.ts +396 -22
- package/src/mind/primitives.ts +109 -7
- package/src/mind/recognition.ts +36 -8
- package/src/mind/traverse.ts +47 -0
- package/src/mind/types.ts +30 -21
- package/src/store-sqlite.ts +76 -0
- package/src/store.ts +46 -2
- package/test/13-conversation.test.mjs +240 -20
- package/test/35-prefix-edge.test.mjs +86 -0
|
@@ -16,6 +16,19 @@ export declare function edgeAncestors(ctx: MindContext, id: number, contextCount
|
|
|
16
16
|
export declare function nextOf(ctx: MindContext, id: number): number[];
|
|
17
17
|
/** Convenience: reverse edges of a node. */
|
|
18
18
|
export declare function prevOf(ctx: MindContext, id: number): number[];
|
|
19
|
+
/** The uniform-expectation floor on a byte atom's corpus commonality: N
|
|
20
|
+
* learnt contexts, each at least one perception chunk of up to W of the 256
|
|
21
|
+
* possible byte values, contain a given atom in ≥ N·W/256 contexts on
|
|
22
|
+
* average. An atom's TRUE containment is unmeasurable (atoms carry no
|
|
23
|
+
* kid/contain links by construction), so this floor is the honest stand-in:
|
|
24
|
+
* derived entirely from the corpus scale N, the perception window W, and
|
|
25
|
+
* the alphabet size — never tuned. */
|
|
26
|
+
export declare function atomReach(ctx: MindContext, contextCount: number): number;
|
|
27
|
+
/** Whether a byte atom is a hub at this corpus scale — its commonality floor
|
|
28
|
+
* {@link atomReach} exceeds the hub bound √N. Below it (small stores) an
|
|
29
|
+
* atom votes and is recognised exactly as any stored form; above it the
|
|
30
|
+
* alphabet is scaffolding everywhere and abstains. */
|
|
31
|
+
export declare function atomIsHub(ctx: MindContext, contextCount: number): boolean;
|
|
19
32
|
/** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
|
|
20
33
|
* The admission predicate recognition filters sites with (HOW_IT_WORKS
|
|
21
34
|
* §15.3): a form that leads nowhere contributes nothing to any derivation.
|
|
@@ -70,6 +70,29 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
|
|
|
70
70
|
const hit = memo?.get(id);
|
|
71
71
|
if (hit !== undefined)
|
|
72
72
|
return hit;
|
|
73
|
+
// BYTE-ATOM COMMONALITY. A single-byte leaf (implicit negative id) has no
|
|
74
|
+
// structural parents BY CONSTRUCTION — atoms are never linked into the kid
|
|
75
|
+
// or contain tables — so this climb cannot observe its containment at all.
|
|
76
|
+
// The walk below would see only the atom's own direct edges and report
|
|
77
|
+
// contextsReached ≈ 1, turning the MOST common content in the store into
|
|
78
|
+
// the MOST discriminative voter (observed on a 325K-context store: every
|
|
79
|
+
// recognised single-letter site voted full ln N for the one fact whose
|
|
80
|
+
// continuation is that letter, and their pooled sum out-voted every
|
|
81
|
+
// genuine anchor). An unmeasurable containment must not default to
|
|
82
|
+
// "maximally rare": it is bounded below by the uniform expectation over
|
|
83
|
+
// the byte alphabet — N contexts, each at least one chunk of up to W of
|
|
84
|
+
// the 256 possible atoms, reach ≥ N·W/256 contexts per atom on average
|
|
85
|
+
// (see {@link atomReach}). When that floor itself exceeds the hub bound
|
|
86
|
+
// √N the atom is a hub at this corpus scale and the climb abstains
|
|
87
|
+
// (saturated) — the atom's own edges remain fully traversable (tier-0
|
|
88
|
+
// exact recall, chooseNext, project); only its say as a consensus voter
|
|
89
|
+
// is withdrawn. On a small store the floor stays ≤ √N and the atom
|
|
90
|
+
// climbs exactly as before, so single-letter facts keep working.
|
|
91
|
+
if (id < 0 && atomIsHub(ctx, contextCount)) {
|
|
92
|
+
const reach = { roots: [], contextsReached: 0, saturated: true };
|
|
93
|
+
memo?.set(id, reach);
|
|
94
|
+
return reach;
|
|
95
|
+
}
|
|
73
96
|
const bound = Math.ceil(Math.sqrt(contextCount));
|
|
74
97
|
const roots = [];
|
|
75
98
|
const seen = new Set([id]);
|
|
@@ -214,6 +237,24 @@ export function nextOf(ctx, id) {
|
|
|
214
237
|
export function prevOf(ctx, id) {
|
|
215
238
|
return ctx.store.prev(id);
|
|
216
239
|
}
|
|
240
|
+
/** The uniform-expectation floor on a byte atom's corpus commonality: N
|
|
241
|
+
* learnt contexts, each at least one perception chunk of up to W of the 256
|
|
242
|
+
* possible byte values, contain a given atom in ≥ N·W/256 contexts on
|
|
243
|
+
* average. An atom's TRUE containment is unmeasurable (atoms carry no
|
|
244
|
+
* kid/contain links by construction), so this floor is the honest stand-in:
|
|
245
|
+
* derived entirely from the corpus scale N, the perception window W, and
|
|
246
|
+
* the alphabet size — never tuned. */
|
|
247
|
+
export function atomReach(ctx, contextCount) {
|
|
248
|
+
return Math.max(1, Math.ceil((contextCount * ctx.space.maxGroup) / 256));
|
|
249
|
+
}
|
|
250
|
+
/** Whether a byte atom is a hub at this corpus scale — its commonality floor
|
|
251
|
+
* {@link atomReach} exceeds the hub bound √N. Below it (small stores) an
|
|
252
|
+
* atom votes and is recognised exactly as any stored form; above it the
|
|
253
|
+
* alphabet is scaffolding everywhere and abstains. */
|
|
254
|
+
export function atomIsHub(ctx, contextCount) {
|
|
255
|
+
return atomReach(ctx, contextCount) >
|
|
256
|
+
Math.ceil(Math.sqrt(Math.max(2, contextCount)));
|
|
257
|
+
}
|
|
217
258
|
/** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
|
|
218
259
|
* The admission predicate recognition filters sites with (HOW_IT_WORKS
|
|
219
260
|
* §15.3): a form that leads nowhere contributes nothing to any derivation.
|
package/dist/src/mind/types.d.ts
CHANGED
|
@@ -105,28 +105,40 @@ export interface MindContext extends GraphSearchHost {
|
|
|
105
105
|
cfg: MindConfig;
|
|
106
106
|
search: GraphSearch;
|
|
107
107
|
trace: Rationale | null;
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
|
|
115
|
-
/** Per-response memo of
|
|
116
|
-
*
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
*
|
|
108
|
+
/** The content canonicalizer for THIS response, or null — injected by the
|
|
109
|
+
* modality entry point (respondText passes the text canonicalizer; a
|
|
110
|
+
* binary respond passes none). Resolution uses it as a fallback: when
|
|
111
|
+
* the exact content-addressed lookup misses, the span's canonical key is
|
|
112
|
+
* probed against the store's canon index (see src/canon.ts). The core
|
|
113
|
+
* never inspects what the equivalence IS. */
|
|
114
|
+
canon: ((bytes: Uint8Array) => Uint8Array) | null;
|
|
115
|
+
/** Per-response memo of canonical-fallback resolutions, keyed by the
|
|
116
|
+
* span's latin1 content key. Null outside respond(). */
|
|
117
|
+
canonMemo: Map<string, number | null> | null;
|
|
118
|
+
/** Memo of the consensus climb — content-keyed (latin1) so results
|
|
119
|
+
* persist across conversation turns where the same byte spans recur.
|
|
120
|
+
* Null outside respond(); during respondTurn() the conversation's
|
|
121
|
+
* persistent map is swapped in. */
|
|
122
|
+
climbMemo: Map<string, Map<string, AttentionRead>> | null;
|
|
123
|
+
/** Memo of {@link recognise} — content-keyed (latin1) so recognised
|
|
124
|
+
* forms carry forward across conversation turns. Bypassed while a
|
|
125
|
+
* trace is attached. Null outside respond(). */
|
|
126
|
+
recogniseMemo: Map<string, Recognition> | null;
|
|
127
|
+
/** Memo of {@link perceive} — content-keyed (latin1). The general
|
|
128
|
+
* cache the result-level memos each partially compensate for. NOT
|
|
129
|
+
* bypassed under trace — perception emits no rationale steps.
|
|
130
|
+
* Null outside respond(). */
|
|
129
131
|
perceiveMemo: Map<string, Sema> | null;
|
|
132
|
+
/** Subtree-resolution cache: Sema node → its store id and byte length.
|
|
133
|
+
* Populated by {@link foldTree} during inference; checked before
|
|
134
|
+
* walking children. When a conversation's pyramid reuses prefix
|
|
135
|
+
* subtrees, this cache lets {@link recognise} skip them entirely —
|
|
136
|
+
* O(suffix) instead of O(context). Mind-lifetime (WeakMap keys are
|
|
137
|
+
* the Sema objects the pyramid keeps alive). */
|
|
138
|
+
_resolvedSubtrees: WeakMap<Sema, {
|
|
139
|
+
id: number;
|
|
140
|
+
len: number;
|
|
141
|
+
}> | null;
|
|
130
142
|
_edgeGuide: Vec | null;
|
|
131
143
|
_edgeChoice: Map<number, number>;
|
|
132
144
|
_prevSeen: Set<number> | null;
|
|
@@ -43,6 +43,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
43
43
|
private _selPrev;
|
|
44
44
|
private _setMeta;
|
|
45
45
|
private _getMeta;
|
|
46
|
+
private _insCanon;
|
|
47
|
+
private _selCanon;
|
|
48
|
+
private _cntCanon;
|
|
49
|
+
private _selContentFrom;
|
|
46
50
|
private _delMeta;
|
|
47
51
|
private _insSnapshot;
|
|
48
52
|
private _selSnapshot;
|
|
@@ -130,6 +134,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
130
134
|
protected _dbGetMeta(key: string): string | null;
|
|
131
135
|
protected _dbSetMeta(key: string, val: string): void;
|
|
132
136
|
protected _dbDeleteMeta(key: string): void;
|
|
137
|
+
canonAdd(h: number, id: number): void;
|
|
138
|
+
canonFind(h: number): number[];
|
|
139
|
+
canonCount(): number;
|
|
140
|
+
eachContent(cb: (id: number, bytes: Uint8Array) => void, fromId?: number): void;
|
|
133
141
|
protected _dbSaveSnapshot(bytes: Uint8Array): void;
|
|
134
142
|
protected _dbLoadSnapshot(): Uint8Array | null;
|
|
135
143
|
protected _vecContentUpsert(entries: Array<{
|
|
@@ -144,6 +152,7 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
144
152
|
protected _vecContentSize(): number;
|
|
145
153
|
protected _vecContentLastReads(): number;
|
|
146
154
|
protected _vecContentPhysicalSize(): number;
|
|
155
|
+
protected _vecContentClusterCount(): number;
|
|
147
156
|
protected _vecContentCompact(): void;
|
|
148
157
|
protected _vecContentDeleteMany(ids: NodeId[]): void;
|
|
149
158
|
protected _vecContentEntriesSince(after: number): IterableIterator<{
|
|
@@ -164,6 +173,7 @@ export declare class SQliteStore extends AbstractStore implements Store {
|
|
|
164
173
|
}>;
|
|
165
174
|
protected _vecHaloSize(): number;
|
|
166
175
|
protected _vecHaloPhysicalSize(): number;
|
|
176
|
+
protected _vecHaloClusterCount(): number;
|
|
167
177
|
protected _vecHaloCompact(): void;
|
|
168
178
|
/** Pre-fill both vector indices' RAM caches with sequential scans (up to
|
|
169
179
|
* their budget caps) — seconds of streaming instead of the minutes of
|
package/dist/src/store-sqlite.js
CHANGED
|
@@ -112,6 +112,18 @@ CREATE TABLE IF NOT EXISTS contain (
|
|
|
112
112
|
id INTEGER PRIMARY KEY,
|
|
113
113
|
parents BLOB NOT NULL
|
|
114
114
|
);
|
|
115
|
+
-- CANONICAL-FORM index (Store.canonAdd/canonFind): h is the 32-bit hash of a
|
|
116
|
+
-- node's CANONICAL content key (the modality's canonicalizer output — case-
|
|
117
|
+
-- folded, whitespace-collapsed text, etc.; the store never sees the key
|
|
118
|
+
-- itself, only its hash). Same hash-then-verify discipline as idx_node_h:
|
|
119
|
+
-- the caller re-canonicalizes each candidate's bytes before trusting it, so
|
|
120
|
+
-- a collision costs a read, never a wrong id. WITHOUT ROWID on (h, id):
|
|
121
|
+
-- the composite key IS the lookup index and gives free dedup.
|
|
122
|
+
CREATE TABLE IF NOT EXISTS canon (
|
|
123
|
+
h INTEGER NOT NULL,
|
|
124
|
+
id INTEGER NOT NULL,
|
|
125
|
+
PRIMARY KEY (h, id)
|
|
126
|
+
) WITHOUT ROWID;
|
|
115
127
|
CREATE TABLE IF NOT EXISTS snapshot (
|
|
116
128
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
117
129
|
data BLOB NOT NULL
|
|
@@ -212,6 +224,10 @@ export class SQliteStore extends AbstractStore {
|
|
|
212
224
|
_selPrev = null;
|
|
213
225
|
_setMeta = null;
|
|
214
226
|
_getMeta = null;
|
|
227
|
+
_insCanon = null;
|
|
228
|
+
_selCanon = null;
|
|
229
|
+
_cntCanon = null;
|
|
230
|
+
_selContentFrom = null;
|
|
215
231
|
_delMeta = null;
|
|
216
232
|
_insSnapshot = null;
|
|
217
233
|
_selSnapshot = null;
|
|
@@ -797,6 +813,42 @@ export class SQliteStore extends AbstractStore {
|
|
|
797
813
|
}
|
|
798
814
|
this._delMeta.run(key);
|
|
799
815
|
}
|
|
816
|
+
// -- Canonical-form index (Store optional capability) --
|
|
817
|
+
canonAdd(h, id) {
|
|
818
|
+
if (!this._insCanon) {
|
|
819
|
+
this._insCanon = this.sqlite.prepare("INSERT OR IGNORE INTO canon (h, id) VALUES (?, ?)");
|
|
820
|
+
}
|
|
821
|
+
// Join the deferred write transaction (committed by flush/commit), so a
|
|
822
|
+
// bulk index build coalesces instead of paying autocommit per row.
|
|
823
|
+
this._dbBeginTx();
|
|
824
|
+
this._insCanon.run(h, id);
|
|
825
|
+
}
|
|
826
|
+
canonFind(h) {
|
|
827
|
+
if (!this._selCanon) {
|
|
828
|
+
this._selCanon = this.sqlite.prepare("SELECT id FROM canon WHERE h = ?");
|
|
829
|
+
}
|
|
830
|
+
return this._selCanon.all(h).map((r) => r.id);
|
|
831
|
+
}
|
|
832
|
+
canonCount() {
|
|
833
|
+
if (!this._cntCanon) {
|
|
834
|
+
this._cntCanon = this.sqlite.prepare("SELECT count(*) AS c FROM canon");
|
|
835
|
+
}
|
|
836
|
+
return this._cntCanon.get().c;
|
|
837
|
+
}
|
|
838
|
+
eachContent(cb, fromId = 0) {
|
|
839
|
+
if (!this._selContentFrom) {
|
|
840
|
+
// Content-bearing nodes are FLAT branches: leaf present, kids an
|
|
841
|
+
// empty (zero-length) blob — the same population PART 2 of the
|
|
842
|
+
// diagnostics calls "distinct content spans".
|
|
843
|
+
this._selContentFrom = this.sqlite.prepare("SELECT id, leaf FROM node " +
|
|
844
|
+
"WHERE id >= ? AND leaf IS NOT NULL " +
|
|
845
|
+
"AND (kids IS NULL OR length(kids) = 0)");
|
|
846
|
+
}
|
|
847
|
+
for (const row of this._selContentFrom.iterate(fromId)) {
|
|
848
|
+
const r = row;
|
|
849
|
+
cb(r.id, new Uint8Array(r.leaf));
|
|
850
|
+
}
|
|
851
|
+
}
|
|
800
852
|
// -- Snapshot --
|
|
801
853
|
_dbSaveSnapshot(bytes) {
|
|
802
854
|
if (!this._insSnapshot) {
|
|
@@ -835,6 +887,9 @@ export class SQliteStore extends AbstractStore {
|
|
|
835
887
|
_vecContentPhysicalSize() {
|
|
836
888
|
return this.content ? this.content.physicalSize : 0;
|
|
837
889
|
}
|
|
890
|
+
_vecContentClusterCount() {
|
|
891
|
+
return this.content ? this.content.clusterCount : 0;
|
|
892
|
+
}
|
|
838
893
|
_vecContentCompact() {
|
|
839
894
|
this.content.compact();
|
|
840
895
|
}
|
|
@@ -869,6 +924,9 @@ export class SQliteStore extends AbstractStore {
|
|
|
869
924
|
_vecHaloPhysicalSize() {
|
|
870
925
|
return this.halos ? this.halos.physicalSize : 0;
|
|
871
926
|
}
|
|
927
|
+
_vecHaloClusterCount() {
|
|
928
|
+
return this.halos ? this.halos.clusterCount : 0;
|
|
929
|
+
}
|
|
872
930
|
_vecHaloCompact() {
|
|
873
931
|
this.halos.compact();
|
|
874
932
|
}
|
package/dist/src/store.d.ts
CHANGED
|
@@ -240,6 +240,18 @@ export interface Store {
|
|
|
240
240
|
* repetition, but repetition outranks insertion-order accident. 0 when
|
|
241
241
|
* the node has no halo row. */
|
|
242
242
|
haloMass(id: NodeId): number;
|
|
243
|
+
/** Record that node `id`'s canonical key hashes to `h`. Idempotent. */
|
|
244
|
+
canonAdd?(h: number, id: NodeId): void;
|
|
245
|
+
/** All candidate node ids whose canonical key hashes to `h` (collisions
|
|
246
|
+
* included — the caller verifies). */
|
|
247
|
+
canonFind?(h: number): NodeId[];
|
|
248
|
+
/** Number of (h, id) rows in the canon index — 0 means never built. */
|
|
249
|
+
canonCount?(): number;
|
|
250
|
+
/** Visit every content-bearing node (flat branch: `leaf` present, no
|
|
251
|
+
* kids) — the population a canonical index is built over. `fromId`
|
|
252
|
+
* restricts the scan to ids ≥ fromId, so an index refresh after further
|
|
253
|
+
* training only visits the new rows. */
|
|
254
|
+
eachContent?(cb: (id: NodeId, bytes: Uint8Array) => void, fromId?: NodeId): void;
|
|
243
255
|
size(): Promise<number>;
|
|
244
256
|
saveSnapshot(bytes: Uint8Array): Promise<void>;
|
|
245
257
|
loadSnapshot(): Promise<Uint8Array | null>;
|
|
@@ -336,6 +348,7 @@ export declare abstract class AbstractStore implements Store {
|
|
|
336
348
|
protected abstract _vecContentSize(): number;
|
|
337
349
|
protected abstract _vecContentLastReads(): number;
|
|
338
350
|
protected abstract _vecContentPhysicalSize(): number;
|
|
351
|
+
protected abstract _vecContentClusterCount(): number;
|
|
339
352
|
protected abstract _vecContentCompact(): void;
|
|
340
353
|
/** Live content-index entries whose INTERNAL id is > `after`, as
|
|
341
354
|
* {ext, internal} pairs in internal-id order. Internal ids are monotone
|
|
@@ -370,7 +383,17 @@ export declare abstract class AbstractStore implements Store {
|
|
|
370
383
|
* the tombstone-ratio compaction trigger compares physical size against. */
|
|
371
384
|
protected abstract _vecHaloSize(): number;
|
|
372
385
|
protected abstract _vecHaloPhysicalSize(): number;
|
|
386
|
+
protected abstract _vecHaloClusterCount(): number;
|
|
373
387
|
protected abstract _vecHaloCompact(): void;
|
|
388
|
+
/** Derived query breadth for a partitioned index of C clusters: probe √C
|
|
389
|
+
* of them (the same √-of-the-population convention as the hub bound √N).
|
|
390
|
+
* The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
|
|
391
|
+
* ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
|
|
392
|
+
* collection outgrows it: at 4,270 clusters the default 64 probed 16
|
|
393
|
+
* clusters (0.4%), and an exact stored match of a query routinely sat in
|
|
394
|
+
* an unprobed cluster — recall silently degraded as the store grew. The
|
|
395
|
+
* configured efSearch remains the floor for small collections. */
|
|
396
|
+
protected efFor(clusterCount: number): number;
|
|
374
397
|
protected _D: number;
|
|
375
398
|
protected _maxGroup: number;
|
|
376
399
|
protected readonly minHaloMass: number;
|
package/dist/src/store.js
CHANGED
|
@@ -359,6 +359,17 @@ function haloDecode(blob, D) {
|
|
|
359
359
|
* protected abstract methods that talk to the actual storage backend.
|
|
360
360
|
*/
|
|
361
361
|
export class AbstractStore {
|
|
362
|
+
/** Derived query breadth for a partitioned index of C clusters: probe √C
|
|
363
|
+
* of them (the same √-of-the-population convention as the hub bound √N).
|
|
364
|
+
* The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
|
|
365
|
+
* ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
|
|
366
|
+
* collection outgrows it: at 4,270 clusters the default 64 probed 16
|
|
367
|
+
* clusters (0.4%), and an exact stored match of a query routinely sat in
|
|
368
|
+
* an unprobed cluster — recall silently degraded as the store grew. The
|
|
369
|
+
* configured efSearch remains the floor for small collections. */
|
|
370
|
+
efFor(clusterCount) {
|
|
371
|
+
return Math.max(this.efSearch, 4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))));
|
|
372
|
+
}
|
|
362
373
|
// ── Config ─────────────────────────────────────────────────────────────
|
|
363
374
|
_D;
|
|
364
375
|
_maxGroup;
|
|
@@ -1144,7 +1155,7 @@ export class AbstractStore {
|
|
|
1144
1155
|
if (hit !== undefined)
|
|
1145
1156
|
return hit;
|
|
1146
1157
|
}
|
|
1147
|
-
const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.
|
|
1158
|
+
const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecContentClusterCount()));
|
|
1148
1159
|
const out = [];
|
|
1149
1160
|
for (const r of results) {
|
|
1150
1161
|
const id = r.id;
|
|
@@ -1459,7 +1470,7 @@ export class AbstractStore {
|
|
|
1459
1470
|
if (hit !== undefined)
|
|
1460
1471
|
return hit;
|
|
1461
1472
|
}
|
|
1462
|
-
const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.
|
|
1473
|
+
const results = this._vecHaloQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecHaloClusterCount()));
|
|
1463
1474
|
const out = [];
|
|
1464
1475
|
for (const r of results) {
|
|
1465
1476
|
const id = r.id;
|
package/example/train_base.ts
CHANGED
|
@@ -177,8 +177,11 @@ const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir of *.zip
|
|
|
177
177
|
const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
|
|
178
178
|
const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
|
|
179
179
|
const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel refresh cadence
|
|
180
|
-
// Index maintenance at checkpoints: compact (remove garbage)
|
|
181
|
-
// gaps)
|
|
180
|
+
// Index maintenance at checkpoints: compact (remove garbage), repair (fill
|
|
181
|
+
// gaps), then refresh the canonical-form index (equivalence-class resolution —
|
|
182
|
+
// src/canon.ts). All three are idempotent batch operations (the canon build is
|
|
183
|
+
// additionally incremental via the store's `canon.upto` cursor);
|
|
184
|
+
// INDEX_MAINTENANCE=0 disables.
|
|
182
185
|
const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
|
|
183
186
|
const DOWNLOAD_TRIES = 5;
|
|
184
187
|
// In-progress downloads are written to a sibling "<dest>.part" and atomically
|
|
@@ -1823,11 +1826,25 @@ async function main(): Promise<void> {
|
|
|
1823
1826
|
|
|
1824
1827
|
const checkpoint = () => mind.save();
|
|
1825
1828
|
|
|
1826
|
-
/** Run index maintenance: compact (remove garbage)
|
|
1827
|
-
*
|
|
1828
|
-
*
|
|
1829
|
-
*
|
|
1830
|
-
*
|
|
1829
|
+
/** Run index maintenance: compact (remove garbage), repair (fill gaps),
|
|
1830
|
+
* then refresh the canonical-form index (see below). All three are
|
|
1831
|
+
* idempotent — running twice produces the same result as once.
|
|
1832
|
+
* Compaction frees index space first; repair then adds back every
|
|
1833
|
+
* edge/halo-bearing node whose gist was evicted from the pending cache
|
|
1834
|
+
* before it reached the content index, completing the coverage that
|
|
1835
|
+
* incremental promotion alone cannot guarantee.
|
|
1836
|
+
*
|
|
1837
|
+
* repair runs with minParents = 0, NOT the library default of 2. The
|
|
1838
|
+
* default repairs only structural BRIDGES (≥2 parents), but this
|
|
1839
|
+
* trainer's fact deposits also leave answer-side DEPOSIT ROOTS with 0
|
|
1840
|
+
* structural parents ("The capital of France is Paris." as the dst of a
|
|
1841
|
+
* Q→A edge is a root of its own tree, contained in nothing). Those are
|
|
1842
|
+
* resonance targets recall depends on — a trained store shipped without
|
|
1843
|
+
* them cannot ground statement-shaped queries against its own answers
|
|
1844
|
+
* (observed: 33 such roots missing after a full curriculum, including
|
|
1845
|
+
* high-traffic conversation replies). minParents = 0 admits every
|
|
1846
|
+
* edge/halo bearer; the candidate set is still corpus-of-experiences-
|
|
1847
|
+
* sized, so the pass stays cheap.
|
|
1831
1848
|
*
|
|
1832
1849
|
* Logs the number of entries removed/added so a run that silently degrades
|
|
1833
1850
|
* (growing compaction count, or repair never recovering anything) is
|
|
@@ -1849,10 +1866,12 @@ async function main(): Promise<void> {
|
|
|
1849
1866
|
);
|
|
1850
1867
|
}
|
|
1851
1868
|
try {
|
|
1852
|
-
const added = await mind.repairContentIndex();
|
|
1869
|
+
const added = await mind.repairContentIndex(0);
|
|
1853
1870
|
if (added > 0) {
|
|
1854
1871
|
progress.log(
|
|
1855
|
-
` ${GRN}index repair: added ${
|
|
1872
|
+
` ${GRN}index repair: added ${
|
|
1873
|
+
int(added)
|
|
1874
|
+
} missing resonance targets${R}`,
|
|
1856
1875
|
);
|
|
1857
1876
|
}
|
|
1858
1877
|
} catch (err) {
|
|
@@ -1862,6 +1881,27 @@ async function main(): Promise<void> {
|
|
|
1862
1881
|
}`,
|
|
1863
1882
|
);
|
|
1864
1883
|
}
|
|
1884
|
+
// Canonical-form index (src/canon.ts): lets resolution find stored forms
|
|
1885
|
+
// across surface variation (case, width, whitespace). Incremental and
|
|
1886
|
+
// idempotent by construction — the `canon.upto` meta cursor scans only
|
|
1887
|
+
// nodes newer than the last pass, and the (h, id) primary key ignores
|
|
1888
|
+
// re-inserted rows — so it composes with the resume model exactly like
|
|
1889
|
+
// compact/repair: every checkpoint (and finish) leaves the index
|
|
1890
|
+
// covering all content trained so far.
|
|
1891
|
+
try {
|
|
1892
|
+
const added = await mind.buildCanonIndex();
|
|
1893
|
+
if (added > 0) {
|
|
1894
|
+
progress.log(
|
|
1895
|
+
` ${GRN}canon index: added ${int(added)} canonical-form entries${R}`,
|
|
1896
|
+
);
|
|
1897
|
+
}
|
|
1898
|
+
} catch (err) {
|
|
1899
|
+
progress.log(
|
|
1900
|
+
` ${YEL}⚠ canon index build failed${R}: ${
|
|
1901
|
+
err instanceof Error ? err.message : String(err)
|
|
1902
|
+
}`,
|
|
1903
|
+
);
|
|
1904
|
+
}
|
|
1865
1905
|
};
|
|
1866
1906
|
|
|
1867
1907
|
// The checkpoint recall is a best-effort diagnostic. It is time-bounded so a
|
package/index.html
CHANGED
|
@@ -52,6 +52,10 @@
|
|
|
52
52
|
z-index: 0;
|
|
53
53
|
overflow: hidden;
|
|
54
54
|
background: var(--bg);
|
|
55
|
+
/* own stacking context + GPU layer: keeps the blend-mode orbs from
|
|
56
|
+
forcing full-page re-rasterization every frame on mobile */
|
|
57
|
+
isolation: isolate;
|
|
58
|
+
transform: translateZ(0);
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
/* — colour weather: two huge drifting orbs + aurora sweep — */
|
|
@@ -60,6 +64,7 @@
|
|
|
60
64
|
border-radius: 50%;
|
|
61
65
|
filter: blur(70px);
|
|
62
66
|
mix-blend-mode: screen;
|
|
67
|
+
will-change: transform;
|
|
63
68
|
}
|
|
64
69
|
.orb.a {
|
|
65
70
|
width: 75vmax;
|
|
@@ -130,6 +135,7 @@
|
|
|
130
135
|
transparent 360deg
|
|
131
136
|
);
|
|
132
137
|
animation: spin 70s linear infinite;
|
|
138
|
+
will-change: transform;
|
|
133
139
|
}
|
|
134
140
|
@keyframes spin {
|
|
135
141
|
to {
|
|
@@ -158,6 +164,7 @@
|
|
|
158
164
|
);
|
|
159
165
|
animation: drift 50s linear infinite alternate;
|
|
160
166
|
opacity: 0.8;
|
|
167
|
+
will-change: transform;
|
|
161
168
|
}
|
|
162
169
|
@keyframes drift {
|
|
163
170
|
to {
|
|
@@ -180,10 +187,12 @@
|
|
|
180
187
|
.layerA {
|
|
181
188
|
animation: sway 30s ease-in-out infinite alternate;
|
|
182
189
|
transform-origin: 50% 50%;
|
|
190
|
+
will-change: transform;
|
|
183
191
|
}
|
|
184
192
|
.layerB {
|
|
185
193
|
animation: sway 44s ease-in-out infinite alternate-reverse;
|
|
186
194
|
transform-origin: 50% 50%;
|
|
195
|
+
will-change: transform;
|
|
187
196
|
}
|
|
188
197
|
@keyframes sway {
|
|
189
198
|
from {
|
|
@@ -341,6 +350,7 @@
|
|
|
341
350
|
opacity: 0.5;
|
|
342
351
|
animation: spin 40s linear infinite;
|
|
343
352
|
filter: drop-shadow(0 0 30px rgba(57, 230, 255, 0.3));
|
|
353
|
+
will-change: transform;
|
|
344
354
|
}
|
|
345
355
|
.hexring::before {
|
|
346
356
|
content: "";
|
|
@@ -373,6 +383,7 @@
|
|
|
373
383
|
transparent 70%
|
|
374
384
|
);
|
|
375
385
|
animation: breath 7s ease-in-out infinite;
|
|
386
|
+
will-change: transform, opacity;
|
|
376
387
|
}
|
|
377
388
|
@keyframes breath {
|
|
378
389
|
0%, 100% {
|
|
@@ -423,6 +434,11 @@
|
|
|
423
434
|
filter: drop-shadow(0 0 22px rgba(57, 230, 255, 0.35)) drop-shadow(
|
|
424
435
|
0 8px 60px rgba(168, 120, 255, 0.25)
|
|
425
436
|
);
|
|
437
|
+
/* gradient-clipped text flickers on iOS while its background animates
|
|
438
|
+
unless the element sits on its own composited layer */
|
|
439
|
+
-webkit-backface-visibility: hidden;
|
|
440
|
+
backface-visibility: hidden;
|
|
441
|
+
transform: translateZ(0);
|
|
426
442
|
}
|
|
427
443
|
@keyframes sheen {
|
|
428
444
|
0%, 100% {
|
|
@@ -450,6 +466,7 @@
|
|
|
450
466
|
color: transparent;
|
|
451
467
|
-webkit-text-stroke: 1px rgba(57, 230, 255, 0.22);
|
|
452
468
|
animation: echo 8s ease-in-out infinite;
|
|
469
|
+
will-change: transform, opacity;
|
|
453
470
|
}
|
|
454
471
|
@keyframes echo {
|
|
455
472
|
0%, 100% {
|
|
@@ -594,6 +611,7 @@
|
|
|
594
611
|
white-space: nowrap;
|
|
595
612
|
width: max-content;
|
|
596
613
|
animation: scroll 34s linear infinite;
|
|
614
|
+
will-change: transform;
|
|
597
615
|
}
|
|
598
616
|
.ticker span {
|
|
599
617
|
padding: 0.72em 0;
|
|
@@ -652,6 +670,53 @@
|
|
|
652
670
|
}
|
|
653
671
|
}
|
|
654
672
|
|
|
673
|
+
/* ——————————————————— mobile / touch performance ——————————————————
|
|
674
|
+
Phones can't sustain the per-frame repaint work: SVG dash/scale
|
|
675
|
+
animations and background-position sheens repaint every frame, and
|
|
676
|
+
each pass runs through blur/drop-shadow filters. Strip the filters
|
|
677
|
+
and the repaint-driven animations; keep every compositor-friendly
|
|
678
|
+
transform/opacity animation so the page stays alive. */
|
|
679
|
+
@media (max-width: 820px), (pointer: coarse) {
|
|
680
|
+
.orb {
|
|
681
|
+
filter: blur(40px);
|
|
682
|
+
}
|
|
683
|
+
.aurora {
|
|
684
|
+
animation: none;
|
|
685
|
+
}
|
|
686
|
+
.grain {
|
|
687
|
+
display: none;
|
|
688
|
+
}
|
|
689
|
+
.wire,
|
|
690
|
+
.wire.v,
|
|
691
|
+
.wire.m,
|
|
692
|
+
.wire.g {
|
|
693
|
+
filter: none;
|
|
694
|
+
animation: none;
|
|
695
|
+
stroke-dasharray: none;
|
|
696
|
+
stroke-width: 1.2;
|
|
697
|
+
opacity: 0.75;
|
|
698
|
+
}
|
|
699
|
+
.layerA,
|
|
700
|
+
.layerB {
|
|
701
|
+
animation: none;
|
|
702
|
+
}
|
|
703
|
+
.ring {
|
|
704
|
+
animation-duration: 7s;
|
|
705
|
+
}
|
|
706
|
+
.hexring {
|
|
707
|
+
filter: none;
|
|
708
|
+
}
|
|
709
|
+
h1 {
|
|
710
|
+
animation: up 1s 0.2s cubic-bezier(0.2, 0.7, 0.2, 1) both;
|
|
711
|
+
background-position: 35% 50%;
|
|
712
|
+
filter: drop-shadow(0 0 22px rgba(57, 230, 255, 0.35));
|
|
713
|
+
}
|
|
714
|
+
.btn.primary {
|
|
715
|
+
animation: none;
|
|
716
|
+
background-position: 50% 50%;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
655
720
|
/* ————————————————————————— reduced motion ———————————————————————— */
|
|
656
721
|
@media (prefers-reduced-motion: reduce) {
|
|
657
722
|
*, *::before, *::after {
|