@avantf/dsh-mem 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +60 -4
- package/lib/engine/mem/db/dao/shared.d.ts +153 -0
- package/lib/engine/mem/eval/index.d.ts +4 -0
- package/lib/engine/mem/index.d.ts +1 -0
- package/lib/engine/mem/runtime.d.ts +2 -2
- package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
- package/lib/engine/mem/store/hybrid.d.ts +50 -0
- package/lib/engine/mem/store/knowledge.d.ts +116 -12
- package/lib/engine/mem/store/legs.d.ts +171 -0
- package/lib/engine/mem/store/lexical.d.ts +8 -0
- package/lib/engine/mem/store/memory.d.ts +84 -72
- package/lib/engine/mem/store/vector_repair.d.ts +248 -0
- package/lib/engine/mem-contract/tools.d.ts +4 -0
- package/lib/engine/mem-contract/types.d.ts +37 -6
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/index.js +1370 -538
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +22 -18
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
|
|
3
|
+
* leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
|
|
4
|
+
* `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
|
|
5
|
+
* scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
|
|
6
|
+
* check's error and the Map insertion order live here once.
|
|
7
|
+
*
|
|
8
|
+
* `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
|
|
9
|
+
* a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
|
|
10
|
+
* `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
|
|
11
|
+
* empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
|
|
12
|
+
* paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
|
|
13
|
+
* from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
|
|
14
|
+
* probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
|
|
15
|
+
* at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
|
|
16
|
+
*
|
|
17
|
+
* WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
|
|
18
|
+
* signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
|
|
19
|
+
* counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
|
|
20
|
+
* quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
|
|
21
|
+
* deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
|
|
22
|
+
* floor functions only ever REMOVE entries, so the raw set is the honest measurement.
|
|
23
|
+
*
|
|
24
|
+
* WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
|
|
25
|
+
* `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
|
|
26
|
+
* {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
|
|
27
|
+
* the absence is structural, not something a caller can forget to set.
|
|
28
|
+
*
|
|
29
|
+
* WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
|
|
30
|
+
* table, and a template-method base class would be a structural rewrite of two hub files. This is
|
|
31
|
+
* the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
|
|
32
|
+
* be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
|
|
33
|
+
* third store in `test/legs.spec.ts`).
|
|
34
|
+
*
|
|
35
|
+
* CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
|
|
36
|
+
* (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
|
|
37
|
+
* legs, so both stores get the same awaiting and isolation by construction.
|
|
38
|
+
*
|
|
39
|
+
* @module store/legs
|
|
40
|
+
*/
|
|
41
|
+
import { type FloorLeg } from './floors.js';
|
|
42
|
+
import { type HybridContext, type HybridLeg } from './hybrid.js';
|
|
43
|
+
import { type FtsTokenizer } from '../db/tokenizer.js';
|
|
44
|
+
/**
|
|
45
|
+
* The legs {@link LegRunner.scored} may wrap: those whose raw score has an absolute scale a floor
|
|
46
|
+
* can cut. `semantic` is excluded on purpose — it shares `applyScoreFloor` but must not carry the
|
|
47
|
+
* cap signal, and it is reached through {@link LegRunner.semantic} (see the module comment).
|
|
48
|
+
*/
|
|
49
|
+
export type ScoredLeg = Exclude<FloorLeg, 'semantic'>;
|
|
50
|
+
/** Build the leg runner for one query. `legCap` is the cap every non-semantic leg is measured against. */
|
|
51
|
+
export interface LegRunner {
|
|
52
|
+
/** The per-leg row cap in force for this query (see `hybrid.ts`'s `legCapFor`). */
|
|
53
|
+
readonly cap: number;
|
|
54
|
+
/**
|
|
55
|
+
* Wrap raw scores that have NO floor of their own (memory's HRR probe and time-window scan): the
|
|
56
|
+
* cap signal is attached and nothing else — no `leg` label, no `droppedByFloor`, exactly as those
|
|
57
|
+
* legs have always shipped (they share the entity leg's weight and floor report).
|
|
58
|
+
*/
|
|
59
|
+
plain(scores: Map<number, number>, weight: number): HybridLeg;
|
|
60
|
+
/**
|
|
61
|
+
* A score-floored leg: `applyScoreFloor` on the raw map, then the cap measured on the RAW map. The
|
|
62
|
+
* floored map is what the leg ships; `raw` is only ever read for the cap.
|
|
63
|
+
*/
|
|
64
|
+
scored(name: ScoredLeg, raw: Map<number, number>, weight: number, floor: number): HybridLeg;
|
|
65
|
+
/**
|
|
66
|
+
* The FTS leg: the per-row term floor (`applyTermFloor`) needs the candidates' texts, read by the
|
|
67
|
+
* store for the whole raw set (one batched query). The cap is measured on the raw map.
|
|
68
|
+
*/
|
|
69
|
+
term(name: 'fts', raw: Map<number, number>, texts: Map<number, string>, query: string, weight: number, floor: number): HybridLeg;
|
|
70
|
+
/**
|
|
71
|
+
* The semantic leg: score-floored like {@link scored}, but deliberately WITHOUT a `capped` key
|
|
72
|
+
* (see the module comment). The async model encode itself happens at the store, which is the only
|
|
73
|
+
* place that knows its backend and filters.
|
|
74
|
+
*/
|
|
75
|
+
semantic(raw: Map<number, number>, weight: number, floor: number): HybridLeg;
|
|
76
|
+
/**
|
|
77
|
+
* The fallback for a DOWN semantic backend: an EMPTY leg at the SAME weight, reporting zero floor
|
|
78
|
+
* drops. Equal weight, not zero: the leg is absent, the weight contract (`RecallResult.weights`)
|
|
79
|
+
* must not move to a fourth value, and the orchestrator already rebalanced the weights for
|
|
80
|
+
* `semAvail === false` (`DEGRADED_WEIGHTS`).
|
|
81
|
+
*/
|
|
82
|
+
semanticOff(weight: number): HybridLeg;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Build the leg runner for one query.
|
|
86
|
+
*
|
|
87
|
+
* The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
|
|
88
|
+
* index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
|
|
89
|
+
* self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
|
|
90
|
+
* evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
|
|
91
|
+
* same wrapper, the same floor order and the same down-semantic fallback.
|
|
92
|
+
*/
|
|
93
|
+
export declare function makeLegRunner(ctx: Pick<HybridContext, 'legCap'>): LegRunner;
|
|
94
|
+
/** The per-store pieces of the FTS leg — see {@link ftsLeg}. `Scope` is opaque to the flow. */
|
|
95
|
+
export interface FtsLegOptions<Scope> {
|
|
96
|
+
query: string;
|
|
97
|
+
/** The per-leg row cap (the store's `legCap`), handed to whichever DAO statement runs. */
|
|
98
|
+
cap: number;
|
|
99
|
+
/** The store's scope shape (`{category, source}` / `{domain, source}`), passed straight through. */
|
|
100
|
+
scope: Scope;
|
|
101
|
+
/** The table's own tokenizer, for the `MATCH` build (`buildFtsQuery`). */
|
|
102
|
+
tokenizer: FtsTokenizer;
|
|
103
|
+
/** The FTS `MATCH` statement; its `rank` is bm25 (negative = better). */
|
|
104
|
+
search: (ftsQuery: string, scope: Scope, limit: number) => readonly {
|
|
105
|
+
id: number;
|
|
106
|
+
rank: number;
|
|
107
|
+
}[];
|
|
108
|
+
/** The short-query LIKE statement; its `rank` counts the terms the row contains (higher = better). */
|
|
109
|
+
substringSearch: (terms: readonly string[], scope: Scope, limit: number) => readonly {
|
|
110
|
+
id: number;
|
|
111
|
+
rank: number;
|
|
112
|
+
}[];
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
|
|
116
|
+
*
|
|
117
|
+
* The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
|
|
118
|
+
* bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
|
|
119
|
+
* order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
|
|
120
|
+
*
|
|
121
|
+
* - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
|
|
122
|
+
* stays better everywhere downstream;
|
|
123
|
+
* - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
|
|
124
|
+
* on the same "higher = better" scale;
|
|
125
|
+
* - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
|
|
126
|
+
* 3-char run) — the leg is empty by construction, not by a query that matched nothing.
|
|
127
|
+
*
|
|
128
|
+
* THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
|
|
129
|
+
* trigram index can express (measured: every term in the FTS table is 3 characters), which used to
|
|
130
|
+
* leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
|
|
131
|
+
* predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
|
|
132
|
+
* is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
|
|
133
|
+
*/
|
|
134
|
+
export declare function ftsLeg<Scope>(opts: FtsLegOptions<Scope>): Map<number, number>;
|
|
135
|
+
/** The per-store pieces of the semantic leg — see {@link semanticLeg}. */
|
|
136
|
+
export interface SemanticLegOptions {
|
|
137
|
+
query: string;
|
|
138
|
+
/** The requested leg size; the pool is over-fetched to `max(50, k)` below. */
|
|
139
|
+
k: number;
|
|
140
|
+
/** A caller-encoded vector for this exact query; skips {@link encode}. */
|
|
141
|
+
queryVector?: Float32Array;
|
|
142
|
+
/** Publishes the vector actually used, so the relaxed retry reuses it instead of re-encoding. */
|
|
143
|
+
onVector?: (vec: Float32Array) => void;
|
|
144
|
+
encode: (query: string) => Promise<Float32Array>;
|
|
145
|
+
/** The vstore's width, checked against the vector before it is scored. */
|
|
146
|
+
dim: number;
|
|
147
|
+
topk: (vec: Float32Array, k: number) => readonly {
|
|
148
|
+
id: number;
|
|
149
|
+
score: number;
|
|
150
|
+
}[];
|
|
151
|
+
/**
|
|
152
|
+
* The store's eligibility filter over the pool ids.
|
|
153
|
+
*
|
|
154
|
+
* `undefined` = **keep everything** — knowledge's "no domain/source asked" short-circuit stays a
|
|
155
|
+
* short-circuit (no lookup runs at all). A returned set is the allow-list; the flow still maps the
|
|
156
|
+
* pool in ITS order, so ids/scores/insertion order are the vstore's.
|
|
157
|
+
*/
|
|
158
|
+
filterTopk: (ids: readonly number[]) => ReadonlySet<number> | undefined;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
|
|
162
|
+
*
|
|
163
|
+
* The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
|
|
164
|
+
* `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
|
|
165
|
+
* to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
|
|
166
|
+
*
|
|
167
|
+
* Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
|
|
168
|
+
* rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
|
|
169
|
+
* empty-pool early return, and the returned map's insertion order (the vstore's pool order).
|
|
170
|
+
*/
|
|
171
|
+
export declare function semanticLeg(opts: SemanticLegOptions): Promise<Map<number, number>>;
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
* path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
|
|
41
41
|
* (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
|
|
42
42
|
*/
|
|
43
|
+
import { type FtsTokenizer } from '../db/tokenizer.js';
|
|
43
44
|
/** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
|
|
44
45
|
export interface LexicalProbe {
|
|
45
46
|
terms: number;
|
|
@@ -109,3 +110,10 @@ export declare function looksRelevant(probe: LexicalProbe): boolean;
|
|
|
109
110
|
* path is synchronous); the default keeps an exact count for the calibration recorded above.
|
|
110
111
|
*/
|
|
111
112
|
export declare function probeTerms(text: string, holds: (term: string) => boolean, stopAt?: number): LexicalProbe;
|
|
113
|
+
/**
|
|
114
|
+
* The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
|
|
115
|
+
* expression for one term (the tokenizer is a property of the store's FTS table) and ask that
|
|
116
|
+
* table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
|
|
117
|
+
* one part that has to differ, which is now a single `holds` closure per store.
|
|
118
|
+
*/
|
|
119
|
+
export declare function probeStoreTerms(text: string, tokenizer: FtsTokenizer, holds: (ftsQuery: string) => boolean, stopAt?: number): LexicalProbe;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixStoreReport } from '../../mem-contract/index.js';
|
|
2
2
|
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
|
+
import { ENTITY_SWEEP_BATCH, type EntitySweepReport } from './entity_sweep.js';
|
|
4
|
+
import { DEFAULT_VECTOR_MIGRATION_BATCH, type VectorRepairTarget } from './vector_repair.js';
|
|
3
5
|
import type { Db } from '../db/conn.js';
|
|
4
6
|
import { type LexicalProbe } from './lexical.js';
|
|
5
7
|
import { type FloorLeg } from './floors.js';
|
|
@@ -15,35 +17,22 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
|
|
|
15
17
|
/**
|
|
16
18
|
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
17
19
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
* Re-exported from the shared sweep (`store/entity_sweep.ts`), which owns the loop and therefore the
|
|
21
|
+
* default: the public name has to keep working for the CLI (`cli/src/index.ts` passes it explicitly)
|
|
22
|
+
* and the plugin's heartbeat, but a second literal here would be a second answer to "how big is one
|
|
23
|
+
* pass".
|
|
22
24
|
*/
|
|
23
|
-
export
|
|
25
|
+
export { ENTITY_SWEEP_BATCH };
|
|
24
26
|
/**
|
|
25
27
|
* Rows one background vector-space migration slice may drop + re-encode.
|
|
26
28
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
|
|
31
|
-
* whatever the caller needs.
|
|
29
|
+
* Re-exported from the shared repair flow (`store/vector_repair.ts`), which owns the loop and
|
|
30
|
+
* therefore the default: the public name has to keep working for callers that pass it explicitly,
|
|
31
|
+
* but a second literal here would be a second answer to "how big is one slice".
|
|
32
32
|
*/
|
|
33
|
-
export
|
|
34
|
-
/** What one derived-state sweep pass did (
|
|
35
|
-
export
|
|
36
|
-
/**
|
|
37
|
-
* Facts this pass RE-EXTRACTED, i.e. its batch size when it filled it. Counts rows VISITED and
|
|
38
|
-
* re-stamped, not rows whose extraction output differs — the stamp is what makes the corpus
|
|
39
|
-
* current, so a row re-extracted to the same names still counts (and is not rebuilt again).
|
|
40
|
-
*/
|
|
41
|
-
rebuilt: number;
|
|
42
|
-
/** Facts still written by older rules after this pass. */
|
|
43
|
-
deferred: number;
|
|
44
|
-
/** Another pass was already running, so this one did nothing (`deferred` is still current). */
|
|
45
|
-
skipped: boolean;
|
|
46
|
-
}
|
|
33
|
+
export { DEFAULT_VECTOR_MIGRATION_BATCH };
|
|
34
|
+
/** What one derived-state sweep pass did — re-exported from the shared loop (`store/entity_sweep.ts`). */
|
|
35
|
+
export type { EntitySweepReport };
|
|
47
36
|
/** What one bounded conflict-drain pass did (see `MemoryStore.drainConflicts`). */
|
|
48
37
|
export interface ConflictDrainReport {
|
|
49
38
|
/** Pairs newly logged by this pass. */
|
|
@@ -177,8 +166,14 @@ export declare class MemoryStore {
|
|
|
177
166
|
* and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
|
|
178
167
|
*/
|
|
179
168
|
private readonly ftsTokenizer;
|
|
180
|
-
/**
|
|
181
|
-
|
|
169
|
+
/**
|
|
170
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
171
|
+
*
|
|
172
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
|
|
173
|
+
* the rule where the loop is (the periodic callers can overlap), without a mutable field that two
|
|
174
|
+
* stores would each have to remember to hold.
|
|
175
|
+
*/
|
|
176
|
+
private readonly entitySweepGuard;
|
|
182
177
|
/**
|
|
183
178
|
* Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
|
|
184
179
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
@@ -202,13 +197,19 @@ export declare class MemoryStore {
|
|
|
202
197
|
*/
|
|
203
198
|
vectorSpace(): string;
|
|
204
199
|
/**
|
|
205
|
-
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs
|
|
206
|
-
*
|
|
207
|
-
*
|
|
200
|
+
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
|
|
201
|
+
* belongs to another vector space.
|
|
202
|
+
*
|
|
203
|
+
* Without this the semantic path is silently empty after every process restart (the DB is the
|
|
204
|
+
* authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
|
|
205
|
+
* archived facts are filtered out of every retrieval path anyway, so loading them would only grow
|
|
206
|
+
* the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
208
207
|
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*
|
|
208
|
+
* The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
|
|
209
|
+
* step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
|
|
210
|
+
* another space is still RANKED (the reload cannot tell the difference), so a silent model swap
|
|
211
|
+
* would mix two spaces in one index — acting on it is the manual fix's job for the operator path
|
|
212
|
+
* and the bounded background migration's for the automatic one.
|
|
212
213
|
*/
|
|
213
214
|
private reloadIndex;
|
|
214
215
|
/**
|
|
@@ -244,6 +245,14 @@ export declare class MemoryStore {
|
|
|
244
245
|
* stamped the marker anyway — a permanent, silent exemption).
|
|
245
246
|
*/
|
|
246
247
|
private detectContradictions;
|
|
248
|
+
/**
|
|
249
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
250
|
+
* call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
|
|
251
|
+
* backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
|
|
252
|
+
* (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
|
|
253
|
+
*/
|
|
254
|
+
private vectorWriteTarget;
|
|
255
|
+
private warnIndexFailure;
|
|
247
256
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
248
257
|
private maybeIndexSemantic;
|
|
249
258
|
update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
@@ -381,54 +390,60 @@ export declare class MemoryStore {
|
|
|
381
390
|
*/
|
|
382
391
|
private classifyVectors;
|
|
383
392
|
/**
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
394
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
395
|
+
* `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
|
|
396
|
+
* for both stores.
|
|
397
|
+
*/
|
|
398
|
+
vectorRepairTarget(): VectorRepairTarget;
|
|
399
|
+
/**
|
|
400
|
+
* {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
|
|
401
|
+
* (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
|
|
402
|
+
*/
|
|
403
|
+
private classifyRepairVectors;
|
|
404
|
+
/** Put usable persisted vectors missing from the live index back into it (no model needed). */
|
|
405
|
+
private reindexUsableVectors;
|
|
406
|
+
/**
|
|
407
|
+
* Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
|
|
408
|
+
* pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
|
|
409
|
+
* the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
|
|
410
|
+
* one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
|
|
411
|
+
*
|
|
412
|
+
* `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
|
|
413
|
+
* the embedding leg was checked against is gone, so the check has to run again.
|
|
414
|
+
*/
|
|
415
|
+
private dropRepairVectors;
|
|
416
|
+
/**
|
|
417
|
+
* Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
|
|
393
418
|
*
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
397
|
-
* `migrateVectorsBatch` passes the migration's batch size.
|
|
419
|
+
* Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
|
|
420
|
+
* the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
|
|
421
|
+
* ran has not been checked.
|
|
398
422
|
*/
|
|
399
|
-
private
|
|
423
|
+
private reencodeMissingVectors;
|
|
400
424
|
/**
|
|
401
425
|
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
402
426
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
403
|
-
* the preview also reports `would_warm` (see the contract's `
|
|
427
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
404
428
|
*/
|
|
405
|
-
vectorsFix(dryRun?: boolean): Promise<
|
|
429
|
+
vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
|
|
406
430
|
/**
|
|
407
431
|
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
408
432
|
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
409
433
|
* background migration ({@link migrateVectors}) and its tests drive.
|
|
410
|
-
*
|
|
411
|
-
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
412
|
-
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
413
434
|
*/
|
|
414
435
|
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
415
436
|
/**
|
|
416
437
|
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
417
|
-
* `shouldStop()` turns true (plugin unmount / process shutdown).
|
|
418
|
-
*
|
|
419
|
-
*
|
|
420
|
-
* blocks the event loop for longer than one batch.
|
|
421
|
-
*
|
|
422
|
-
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
423
|
-
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
438
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
|
|
439
|
+
* switch, the no-progress rule and the store-named logging all live in the shared flow
|
|
440
|
+
* (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
|
|
424
441
|
*/
|
|
425
442
|
migrateVectors(opts?: {
|
|
426
443
|
batchSize?: number;
|
|
427
444
|
shouldStop?: () => boolean;
|
|
428
445
|
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
429
446
|
}): Promise<VectorMigrationOutcome>;
|
|
430
|
-
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
431
|
-
private migrationRemaining;
|
|
432
447
|
/**
|
|
433
448
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
434
449
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
@@ -466,14 +481,10 @@ export declare class MemoryStore {
|
|
|
466
481
|
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
467
482
|
* every selected row's `content` before rebuilding the first one.
|
|
468
483
|
*
|
|
469
|
-
* The in-flight guard
|
|
470
|
-
*
|
|
471
|
-
*
|
|
472
|
-
*
|
|
473
|
-
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
474
|
-
*
|
|
475
|
-
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
476
|
-
* caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
|
|
484
|
+
* The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
|
|
485
|
+
* live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
|
|
486
|
+
* entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
|
|
487
|
+
* selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
|
|
477
488
|
*/
|
|
478
489
|
reindexEntities(budget?: number): Promise<EntitySweepReport>;
|
|
479
490
|
/**
|
|
@@ -483,8 +494,6 @@ export declare class MemoryStore {
|
|
|
483
494
|
* `ContradictionsDao.resolveForFact`).
|
|
484
495
|
*/
|
|
485
496
|
private retireConflicts;
|
|
486
|
-
/** Drop vectors of archived/purged facts from the live index (batched). */
|
|
487
|
-
private evictVectors;
|
|
488
497
|
/**
|
|
489
498
|
* Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
|
|
490
499
|
* count. Archived (incl. superseded) facts are excluded — their entities are not
|
|
@@ -581,7 +590,10 @@ export declare class MemoryStore {
|
|
|
581
590
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
582
591
|
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
583
592
|
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
584
|
-
* instead of re-encoding (performance review §7.7 / P8).
|
|
593
|
+
* instead of re-encoding (performance review §7.7 / P8). The flow is shared
|
|
594
|
+
* (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
|
|
595
|
+
* notion of status/category/source, so candidates go through the DB and archived/purged facts,
|
|
596
|
+
* other categories and other sources never leak into the leg.
|
|
585
597
|
*/
|
|
586
598
|
private semanticPath;
|
|
587
599
|
/**
|