@avantf/dsh-mem 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/lib/client.js +1 -0
  2. package/lib/engine/mem/db/dao/chunks.d.ts +60 -4
  3. package/lib/engine/mem/db/dao/shared.d.ts +153 -0
  4. package/lib/engine/mem/dispatch.d.ts +19 -1
  5. package/lib/engine/mem/eval/index.d.ts +4 -0
  6. package/lib/engine/mem/index.d.ts +2 -0
  7. package/lib/engine/mem/runtime.d.ts +2 -2
  8. package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
  9. package/lib/engine/mem/store/hybrid.d.ts +50 -0
  10. package/lib/engine/mem/store/knowledge.d.ts +116 -12
  11. package/lib/engine/mem/store/legs.d.ts +171 -0
  12. package/lib/engine/mem/store/lexical.d.ts +8 -0
  13. package/lib/engine/mem/store/memory.d.ts +84 -72
  14. package/lib/engine/mem/store/vector_repair.d.ts +248 -0
  15. package/lib/engine/mem/trustHeartbeat.d.ts +47 -0
  16. package/lib/engine/mem-contract/tools.d.ts +4 -0
  17. package/lib/engine/mem-contract/types.d.ts +37 -6
  18. package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
  19. package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
  20. package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
  21. package/lib/index.js +1653 -703
  22. package/lib/interface-version.json +1 -1
  23. package/lib/types/envinit-bootstrap.d.ts +1 -1
  24. package/lib/types/envinit.d.ts +0 -3
  25. package/lib/types/envinit.d.ts.map +1 -1
  26. package/lib/types/index.d.ts.map +1 -1
  27. package/lib/types/interface_gate.d.ts +70 -0
  28. package/lib/types/interface_gate.d.ts.map +1 -1
  29. package/lib/types/provision.d.ts +6 -0
  30. package/lib/types/provision.d.ts.map +1 -1
  31. package/lib/types/remote.d.ts.map +1 -1
  32. package/lib/types/vectorMigration.d.ts +22 -18
  33. package/lib/types/vectorMigration.d.ts.map +1 -1
  34. package/package.json +5 -5
@@ -1,8 +1,9 @@
1
- import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport } from '../../mem-contract/index.js';
1
+ import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsFixStoreReport } from '../../mem-contract/index.js';
2
2
  import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
3
3
  import { type LexicalProbe } from './lexical.js';
4
4
  import { type FloorLeg } from './floors.js';
5
5
  import { type HybridResult } from './hybrid.js';
6
+ import { type VectorRepairTarget } from './vector_repair.js';
6
7
  /** Options for {@link KnowledgeStore.search}. */
7
8
  export interface KnowledgeSearchOptions {
8
9
  domain?: string;
@@ -110,6 +111,15 @@ export declare class KnowledgeStore {
110
111
  private readonly docFiles;
111
112
  /** > 0 while a batch (directory import) is writing: commits and baseline refreshes wait. */
112
113
  private deferCorpusWrites;
114
+ /**
115
+ * Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
116
+ *
117
+ * The guard itself lives in that module; this is only the per-store identity it keys on. It exists
118
+ * as a field — rather than the adapter object, which is built per call — because the guard must
119
+ * survive between two overlapping calls: `reindex` can be reached from a tool call and the
120
+ * settings-page button at once, and both would select the same stale chunks.
121
+ */
122
+ private readonly entitySweepGuard;
113
123
  /**
114
124
  * Per-file stamps (`path → mtimeMs:size`) and the file-set they were taken from, so a reconcile
115
125
  * can ask "what changed?" for the price of one `stat` per document instead of reading and
@@ -132,13 +142,18 @@ export declare class KnowledgeStore {
132
142
  /** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
133
143
  domainConfigPath: string, logger?: AvantfLogger | undefined);
134
144
  /**
135
- * Rebuild the in-memory vstore from persisted chunk vectors.
145
+ * Rebuild the in-memory vstore from persisted chunk vectors, and say loudly what belongs to
146
+ * another vector space.
147
+ *
148
+ * Also detects a VECTOR-SPACE change, which the reload itself cannot see: the reload checks the
149
+ * width, and a different model at the same width produces dim-valid vectors that are then ranked
150
+ * against the current query's — two spaces mixed in one index, with no error anywhere.
136
151
  *
137
- * Also detects a VECTOR-SPACE change, which the reload itself cannot see: `reloadVectorIndex`
138
- * checks the width, and a different model at the same width produces dim-valid vectors that are
139
- * then ranked against the current query's — two spaces mixed in one index, with no error anywhere.
140
- * The memory store always said something about this; the knowledge store did not, so a model swap
141
- * degraded every semantic query here until someone happened to run `kb_reindex`.
152
+ * The reload, the count and the warning are the SHARED open-time step (`store/vector_repair.ts`),
153
+ * which is what turned the knowledge store from "warns about a swapped model but never repairs it"
154
+ * into the same automatic path memory has always had; this store's half is the row read. The
155
+ * manual entry stays `vectors --fix` (the vector-only repair); `kb_reindex` remains the full
156
+ * FTS + entities + vectors re-entry.
142
157
  */
143
158
  private reloadIndex;
144
159
  /**
@@ -243,8 +258,6 @@ export declare class KnowledgeStore {
243
258
  warnings?: string[]): Promise<IngestResult>;
244
259
  /** Materialize one document's editable copy; returns the `file` / `file_error` result fields. */
245
260
  private writeManagedFile;
246
- /** Batched eviction from the live vector index (removeMany when the backend supports it). */
247
- private evictVectors;
248
261
  /** The resolved ingestion boundary (see `ingest_guard.ts`). */
249
262
  private get ingestLimits();
250
263
  /**
@@ -288,7 +301,8 @@ export declare class KnowledgeStore {
288
301
  */
289
302
  private writeReusedVectors;
290
303
  /**
291
- * Encode texts ONE AT A TIME, writing every `WRITE_BATCH` vectors in one transaction.
304
+ * This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
305
+ * call, the live index and the `WRITE_BATCH` transaction this store measured its way into.
292
306
  *
293
307
  * The model call is deliberately NOT batched, and that is a MEASURED decision, not an omission:
294
308
  * a batched forward pass pads each item to the longest in its group, and real chunks span 53–550
@@ -299,7 +313,17 @@ export declare class KnowledgeStore {
299
313
  * length-sorted) for callers whose texts suit it.
300
314
  *
301
315
  * What DID pay off is the write side: one commit per `WRITE_BATCH` chunks instead of one per
302
- * chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await`.
316
+ * chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await` — that
317
+ * is the `writeBatch` below.
318
+ */
319
+ private encodeWriteTarget;
320
+ /**
321
+ * Encode texts one at a time and persist them in `WRITE_BATCH` transactions.
322
+ *
323
+ * A failed chunk is skipped, not thrown: the rows are already committed and their old vectors
324
+ * evicted by the ingest path, so a throw here would leave the caller with a failed ingest and NO
325
+ * COUNT of what was skipped. The shortfall travels back to the tool result
326
+ * (`IngestResult.vectors_failed`) instead of living only in a log line.
303
327
  */
304
328
  private encodeAndStore;
305
329
  /**
@@ -309,6 +333,59 @@ export declare class KnowledgeStore {
309
333
  * of silently mixing two spaces. Public so diagnostics and tests can name the space.
310
334
  */
311
335
  vectorSpace(): string;
336
+ /**
337
+ * Cheapest possible DETECTION of "the embedding space changed under this store": counts of
338
+ * persisted chunk vectors whose width or recorded space id no longer matches the one this store
339
+ * writes into. Reads blob LENGTHS only — never decodes — so the startup warning and the background
340
+ * drive can afford it at any corpus size.
341
+ */
342
+ vectorSpaceHealth(): VectorSpaceHealth;
343
+ /** {@link vectorSpaceHealth} plus `missing`, the third term of the repair flow's `remaining`. */
344
+ private vectorSpaceCounts;
345
+ /** Async bootstrap the semantic backend (the shared flow's explicit-fix warmup). */
346
+ warmupSemantic(): Promise<boolean>;
347
+ /**
348
+ * This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
349
+ * steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
350
+ * `semantic.auto_migrate`, the dry-run split — lives there once, for both stores.
351
+ */
352
+ vectorRepairTarget(): VectorRepairTarget;
353
+ /**
354
+ * The chunks the semantic leg cannot use (no vector / wrong width / another space). One SQL
355
+ * aggregate over blob lengths; `dropIds` is empty because knowledge overwrites in place.
356
+ */
357
+ private classifyRepairVectors;
358
+ /**
359
+ * Encode + persist up to `limit` chunks the semantic leg cannot use, bounded and ordered by id.
360
+ *
361
+ * It does NOT re-extract entities or rebuild FTS: that is `kb_reindex`'s job, and the whole point
362
+ * of the automatic repair is that the vector half can run on its own — which is exactly the
363
+ * objection the "known boundary" in DESIGN §20 used to raise against a background run.
364
+ */
365
+ private reencodeStaleChunks;
366
+ /**
367
+ * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
368
+ * `dry_run` previews every count without writing — and without loading the model, which is why
369
+ * the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
370
+ */
371
+ vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
372
+ /**
373
+ * ONE bounded slice of the vector-space migration: at most `batchSize` chunks are re-encoded, so
374
+ * the caller keeps control of the event loop between slices. This is the unit the background
375
+ * migration ({@link migrateVectors}) and its tests drive.
376
+ */
377
+ migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
378
+ /**
379
+ * Drive the bounded background migration for this store until it is current, the model is
380
+ * unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the
381
+ * `semantic.auto_migrate` switch, the no-progress rule and the store-named logging all live in the
382
+ * shared flow (`store/vector_repair.ts`); this is the one-store entry.
383
+ */
384
+ migrateVectors(opts?: {
385
+ batchSize?: number;
386
+ shouldStop?: () => boolean;
387
+ onProgress?: (progress: VectorMigrationOutcome) => void;
388
+ }): Promise<VectorMigrationOutcome>;
312
389
  /**
313
390
  * A stored vector is reusable when it exists, matches the space, and its text is unchanged.
314
391
  *
@@ -319,8 +396,28 @@ export declare class KnowledgeStore {
319
396
  * the store would keep serving a vector for text it no longer holds.
320
397
  */
321
398
  private vectorReusable;
322
- /** One transaction replacing the entity rows of the given chunks (shared by ingest + reindex). */
399
+ /**
400
+ * Extract one chunk's entity names, bounded by `MAX_ENTITIES_PER_CHUNK`.
401
+ *
402
+ * The bound is here, not in either caller: it is a property of what `chunk_entities` may hold for
403
+ * one chunk, and the ingest path and the sweep must not disagree about it.
404
+ */
405
+ private extractRowEntities;
406
+ /** One transaction replacing the entity rows of the given chunks (the ingest path). */
323
407
  private replaceChunkEntities;
408
+ /**
409
+ * The per-row primitive of the shared entity sweep (`store/entity_sweep.ts`).
410
+ *
411
+ * Replaces ONE chunk's entity rows and stamps that same row with `ENTITY_EXTRACTOR_VERSION`, in one
412
+ * transaction. Knowledge's domain difference from memory is exactly this: memory's transaction also
413
+ * moves triples, the HRR bundle and the conflict queue, while a chunk has none of those — its
414
+ * derived state is the entity names alone.
415
+ *
416
+ * Per row, not per batch, because the sweep's contract is that a failure leaves THAT row stale and
417
+ * retryable: inside a batched transaction one bad extraction would roll back the whole batch, and
418
+ * rows it never reached would look stamped.
419
+ */
420
+ private replaceRowEntities;
324
421
  /**
325
422
  * Ingest local files/directories. Every path goes through the SAME boundary as
326
423
  * `ingestUri` — a bare `kb_import` on `~/.ssh` would otherwise be the exact
@@ -459,6 +556,13 @@ export declare class KnowledgeStore {
459
556
  * re-encoding (performance review §7.7 / P8).
460
557
  */
461
558
  private semanticPath;
559
+ /**
560
+ * The semantic leg's scope filter, as a `filterTopk` primitive: `undefined` when NO domain/source
561
+ * was asked means "keep the whole pool" — the short-circuit is preserved, so no `chunks.meta`
562
+ * lookup runs at all. Otherwise one batched lookup, and rows whose domain/source do not match (or
563
+ * whose chunk is gone) are dropped.
564
+ */
565
+ private scopeFilter;
462
566
  /**
463
567
  * Sync relevance probe for the plugin's conditional hint (DESIGN §12). Same contract as
464
568
  * `MemoryStore.lexicalProbe` — see `store/lexical.ts`.
@@ -0,0 +1,171 @@
1
+ /**
2
+ * The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
3
+ * leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
4
+ * `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
5
+ * scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
6
+ * check's error and the Map insertion order live here once.
7
+ *
8
+ * `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
9
+ * a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
10
+ * `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
11
+ * empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
12
+ * paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
13
+ * from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
14
+ * probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
15
+ * at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
16
+ *
17
+ * WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
18
+ * signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
19
+ * counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
20
+ * quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
21
+ * deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
22
+ * floor functions only ever REMOVE entries, so the raw set is the honest measurement.
23
+ *
24
+ * WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
25
+ * `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
26
+ * {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
27
+ * the absence is structural, not something a caller can forget to set.
28
+ *
29
+ * WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
30
+ * table, and a template-method base class would be a structural rewrite of two hub files. This is
31
+ * the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
32
+ * be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
33
+ * third store in `test/legs.spec.ts`).
34
+ *
35
+ * CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
36
+ * (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
37
+ * legs, so both stores get the same awaiting and isolation by construction.
38
+ *
39
+ * @module store/legs
40
+ */
41
+ import { type FloorLeg } from './floors.js';
42
+ import { type HybridContext, type HybridLeg } from './hybrid.js';
43
+ import { type FtsTokenizer } from '../db/tokenizer.js';
44
+ /**
45
+ * The legs {@link LegRunner.scored} may wrap: those whose raw score has an absolute scale a floor
46
+ * can cut. `semantic` is excluded on purpose — it shares `applyScoreFloor` but must not carry the
47
+ * cap signal, and it is reached through {@link LegRunner.semantic} (see the module comment).
48
+ */
49
+ export type ScoredLeg = Exclude<FloorLeg, 'semantic'>;
50
+ /** Build the leg runner for one query. `legCap` is the cap every non-semantic leg is measured against. */
51
+ export interface LegRunner {
52
+ /** The per-leg row cap in force for this query (see `hybrid.ts`'s `legCapFor`). */
53
+ readonly cap: number;
54
+ /**
55
+ * Wrap raw scores that have NO floor of their own (memory's HRR probe and time-window scan): the
56
+ * cap signal is attached and nothing else — no `leg` label, no `droppedByFloor`, exactly as those
57
+ * legs have always shipped (they share the entity leg's weight and floor report).
58
+ */
59
+ plain(scores: Map<number, number>, weight: number): HybridLeg;
60
+ /**
61
+ * A score-floored leg: `applyScoreFloor` on the raw map, then the cap measured on the RAW map. The
62
+ * floored map is what the leg ships; `raw` is only ever read for the cap.
63
+ */
64
+ scored(name: ScoredLeg, raw: Map<number, number>, weight: number, floor: number): HybridLeg;
65
+ /**
66
+ * The FTS leg: the per-row term floor (`applyTermFloor`) needs the candidates' texts, read by the
67
+ * store for the whole raw set (one batched query). The cap is measured on the raw map.
68
+ */
69
+ term(name: 'fts', raw: Map<number, number>, texts: Map<number, string>, query: string, weight: number, floor: number): HybridLeg;
70
+ /**
71
+ * The semantic leg: score-floored like {@link scored}, but deliberately WITHOUT a `capped` key
72
+ * (see the module comment). The async model encode itself happens at the store, which is the only
73
+ * place that knows its backend and filters.
74
+ */
75
+ semantic(raw: Map<number, number>, weight: number, floor: number): HybridLeg;
76
+ /**
77
+ * The fallback for a DOWN semantic backend: an EMPTY leg at the SAME weight, reporting zero floor
78
+ * drops. Equal weight, not zero: the leg is absent, the weight contract (`RecallResult.weights`)
79
+ * must not move to a fourth value, and the orchestrator already rebalanced the weights for
80
+ * `semAvail === false` (`DEGRADED_WEIGHTS`).
81
+ */
82
+ semanticOff(weight: number): HybridLeg;
83
+ }
84
+ /**
85
+ * Build the leg runner for one query.
86
+ *
87
+ * The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
88
+ * index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
89
+ * self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
90
+ * evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
91
+ * same wrapper, the same floor order and the same down-semantic fallback.
92
+ */
93
+ export declare function makeLegRunner(ctx: Pick<HybridContext, 'legCap'>): LegRunner;
94
+ /** The per-store pieces of the FTS leg — see {@link ftsLeg}. `Scope` is opaque to the flow. */
95
+ export interface FtsLegOptions<Scope> {
96
+ query: string;
97
+ /** The per-leg row cap (the store's `legCap`), handed to whichever DAO statement runs. */
98
+ cap: number;
99
+ /** The store's scope shape (`{category, source}` / `{domain, source}`), passed straight through. */
100
+ scope: Scope;
101
+ /** The table's own tokenizer, for the `MATCH` build (`buildFtsQuery`). */
102
+ tokenizer: FtsTokenizer;
103
+ /** The FTS `MATCH` statement; its `rank` is bm25 (negative = better). */
104
+ search: (ftsQuery: string, scope: Scope, limit: number) => readonly {
105
+ id: number;
106
+ rank: number;
107
+ }[];
108
+ /** The short-query LIKE statement; its `rank` counts the terms the row contains (higher = better). */
109
+ substringSearch: (terms: readonly string[], scope: Scope, limit: number) => readonly {
110
+ id: number;
111
+ rank: number;
112
+ }[];
113
+ }
114
+ /**
115
+ * The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
116
+ *
117
+ * The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
118
+ * bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
119
+ * order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
120
+ *
121
+ * - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
122
+ * stays better everywhere downstream;
123
+ * - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
124
+ * on the same "higher = better" scale;
125
+ * - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
126
+ * 3-char run) — the leg is empty by construction, not by a query that matched nothing.
127
+ *
128
+ * THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
129
+ * trigram index can express (measured: every term in the FTS table is 3 characters), which used to
130
+ * leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
131
+ * predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
132
+ * is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
133
+ */
134
+ export declare function ftsLeg<Scope>(opts: FtsLegOptions<Scope>): Map<number, number>;
135
+ /** The per-store pieces of the semantic leg — see {@link semanticLeg}. */
136
+ export interface SemanticLegOptions {
137
+ query: string;
138
+ /** The requested leg size; the pool is over-fetched to `max(50, k)` below. */
139
+ k: number;
140
+ /** A caller-encoded vector for this exact query; skips {@link encode}. */
141
+ queryVector?: Float32Array;
142
+ /** Publishes the vector actually used, so the relaxed retry reuses it instead of re-encoding. */
143
+ onVector?: (vec: Float32Array) => void;
144
+ encode: (query: string) => Promise<Float32Array>;
145
+ /** The vstore's width, checked against the vector before it is scored. */
146
+ dim: number;
147
+ topk: (vec: Float32Array, k: number) => readonly {
148
+ id: number;
149
+ score: number;
150
+ }[];
151
+ /**
152
+ * The store's eligibility filter over the pool ids.
153
+ *
154
+ * `undefined` = **keep everything** — knowledge's "no domain/source asked" short-circuit stays a
155
+ * short-circuit (no lookup runs at all). A returned set is the allow-list; the flow still maps the
156
+ * pool in ITS order, so ids/scores/insertion order are the vstore's.
157
+ */
158
+ filterTopk: (ids: readonly number[]) => ReadonlySet<number> | undefined;
159
+ }
160
+ /**
161
+ * The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
162
+ *
163
+ * The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
164
+ * `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
165
+ * to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
166
+ *
167
+ * Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
168
+ * rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
169
+ * empty-pool early return, and the returned map's insertion order (the vstore's pool order).
170
+ */
171
+ export declare function semanticLeg(opts: SemanticLegOptions): Promise<Map<number, number>>;
@@ -40,6 +40,7 @@
40
40
  * path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
41
41
  * (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
42
42
  */
43
+ import { type FtsTokenizer } from '../db/tokenizer.js';
43
44
  /** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
44
45
  export interface LexicalProbe {
45
46
  terms: number;
@@ -109,3 +110,10 @@ export declare function looksRelevant(probe: LexicalProbe): boolean;
109
110
  * path is synchronous); the default keeps an exact count for the calibration recorded above.
110
111
  */
111
112
  export declare function probeTerms(text: string, holds: (term: string) => boolean, stopAt?: number): LexicalProbe;
113
+ /**
114
+ * The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
115
+ * expression for one term (the tokenizer is a property of the store's FTS table) and ask that
116
+ * table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
117
+ * one part that has to differ, which is now a single `holds` closure per store.
118
+ */
119
+ export declare function probeStoreTerms(text: string, tokenizer: FtsTokenizer, holds: (ftsQuery: string) => boolean, stopAt?: number): LexicalProbe;
@@ -1,5 +1,7 @@
1
- import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
1
+ import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixStoreReport } from '../../mem-contract/index.js';
2
2
  import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
3
+ import { ENTITY_SWEEP_BATCH, type EntitySweepReport } from './entity_sweep.js';
4
+ import { DEFAULT_VECTOR_MIGRATION_BATCH, type VectorRepairTarget } from './vector_repair.js';
3
5
  import type { Db } from '../db/conn.js';
4
6
  import { type LexicalProbe } from './lexical.js';
5
7
  import { type FloorLeg } from './floors.js';
@@ -15,35 +17,22 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
15
17
  /**
16
18
  * Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
17
19
  *
18
- * Bounds MEMORY, not only mission: the pass selects `content` for every row it takes, so an
19
- * "unbounded" pass (the CLI used to ask for `Number.MAX_SAFE_INTEGER`) builds an array of every
20
- * stale fact's text before rebuilding the first one. Callers that want the corpus drained loop
21
- * until `deferred === 0` instead.
20
+ * Re-exported from the shared sweep (`store/entity_sweep.ts`), which owns the loop and therefore the
21
+ * default: the public name has to keep working for the CLI (`cli/src/index.ts` passes it explicitly)
22
+ * and the plugin's heartbeat, but a second literal here would be a second answer to "how big is one
23
+ * pass".
22
24
  */
23
- export declare const ENTITY_SWEEP_BATCH = 2000;
25
+ export { ENTITY_SWEEP_BATCH };
24
26
  /**
25
27
  * Rows one background vector-space migration slice may drop + re-encode.
26
28
  *
27
- * Small on purpose: the point of batching is that a query arriving mid-migration waits for at most
28
- * one ONNX forward pass batch, not for a whole corpus. At ~30 ms per bge-base-zh encode a batch of 16
29
- * is well under a second of contested model time, and the event loop is yielded between batches
30
- * (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
31
- * whatever the caller needs.
29
+ * Re-exported from the shared repair flow (`store/vector_repair.ts`), which owns the loop and
30
+ * therefore the default: the public name has to keep working for callers that pass it explicitly,
31
+ * but a second literal here would be a second answer to "how big is one slice".
32
32
  */
33
- export declare const DEFAULT_VECTOR_MIGRATION_BATCH = 16;
34
- /** What one derived-state sweep pass did (see `MemoryStore.reindexEntities`). */
35
- export interface EntitySweepReport {
36
- /**
37
- * Facts this pass RE-EXTRACTED, i.e. its batch size when it filled it. Counts rows VISITED and
38
- * re-stamped, not rows whose extraction output differs — the stamp is what makes the corpus
39
- * current, so a row re-extracted to the same names still counts (and is not rebuilt again).
40
- */
41
- rebuilt: number;
42
- /** Facts still written by older rules after this pass. */
43
- deferred: number;
44
- /** Another pass was already running, so this one did nothing (`deferred` is still current). */
45
- skipped: boolean;
46
- }
33
+ export { DEFAULT_VECTOR_MIGRATION_BATCH };
34
+ /** What one derived-state sweep pass did — re-exported from the shared loop (`store/entity_sweep.ts`). */
35
+ export type { EntitySweepReport };
47
36
  /** What one bounded conflict-drain pass did (see `MemoryStore.drainConflicts`). */
48
37
  export interface ConflictDrainReport {
49
38
  /** Pairs newly logged by this pass. */
@@ -177,8 +166,14 @@ export declare class MemoryStore {
177
166
  * and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
178
167
  */
179
168
  private readonly ftsTokenizer;
180
- /** In-flight guard for `reindexEntities` (see there: the periodic callers can overlap). */
181
- private entitySweepInFlight;
169
+ /**
170
+ * Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
171
+ *
172
+ * The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
173
+ * the rule where the loop is (the periodic callers can overlap), without a mutable field that two
174
+ * stores would each have to remember to hold.
175
+ */
176
+ private readonly entitySweepGuard;
182
177
  /**
183
178
  * Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
184
179
  * derived state: nothing is lost when it resets, and the queue itself is in the database.
@@ -202,13 +197,19 @@ export declare class MemoryStore {
202
197
  */
203
198
  vectorSpace(): string;
204
199
  /**
205
- * Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs.
206
- * Without this the semantic path is silently empty after every process restart
207
- * (the DB is the authoritative vector store; the vstore is a derived read model).
200
+ * Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
201
+ * belongs to another vector space.
202
+ *
203
+ * Without this the semantic path is silently empty after every process restart (the DB is the
204
+ * authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
205
+ * archived facts are filtered out of every retrieval path anyway, so loading them would only grow
206
+ * the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
208
207
  *
209
- * Only ACTIVE facts are indexed: archived facts are filtered out of every
210
- * retrieval path anyway, so loading them would only grow the index forever.
211
- * `archive()` evicts and `restore()` re-adds, keeping the invariant.
208
+ * The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
209
+ * step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
210
+ * another space is still RANKED (the reload cannot tell the difference), so a silent model swap
211
+ * would mix two spaces in one index — acting on it is the manual fix's job for the operator path
212
+ * and the bounded background migration's for the automatic one.
212
213
  */
213
214
  private reloadIndex;
214
215
  /**
@@ -244,6 +245,14 @@ export declare class MemoryStore {
244
245
  * stamped the marker anyway — a permanent, silent exemption).
245
246
  */
246
247
  private detectContradictions;
248
+ /**
249
+ * This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
250
+ * call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
251
+ * backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
252
+ * (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
253
+ */
254
+ private vectorWriteTarget;
255
+ private warnIndexFailure;
247
256
  /** Best-effort semantic indexing: encode the fact and add it to the vstore. */
248
257
  private maybeIndexSemantic;
249
258
  update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
@@ -381,54 +390,60 @@ export declare class MemoryStore {
381
390
  */
382
391
  private classifyVectors;
383
392
  /**
384
- * Repair the semantic read model, in ONE bounded slice or in full:
385
- * (a) reload persisted, dim-valid vectors missing from the live index
386
- * (restart / index loss — needs no model),
387
- * (b) drop persisted vectors whose dim no longer matches `semantic.dim` so a
388
- * re-encode can replace them (never `add()` a stale vector — the store
389
- * throws on dim mismatch),
390
- * (c) encode active facts that have no vector yet (needs the semantic backend),
391
- * (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
392
- * adoption of a store predating the space id).
393
+ * This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
394
+ * steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
395
+ * `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
396
+ * for both stores.
397
+ */
398
+ vectorRepairTarget(): VectorRepairTarget;
399
+ /**
400
+ * {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
401
+ * (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
402
+ */
403
+ private classifyRepairVectors;
404
+ /** Put usable persisted vectors missing from the live index back into it (no model needed). */
405
+ private reindexUsableVectors;
406
+ /**
407
+ * Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
408
+ * pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
409
+ * the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
410
+ * one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
411
+ *
412
+ * `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
413
+ * the embedding leg was checked against is gone, so the check has to run again.
414
+ */
415
+ private dropRepairVectors;
416
+ /**
417
+ * Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
393
418
  *
394
- * `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
395
- * dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
396
- * yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
397
- * `migrateVectorsBatch` passes the migration's batch size.
419
+ * Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
420
+ * the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
421
+ * ran has not been checked.
398
422
  */
399
- private repairVectors;
423
+ private reencodeMissingVectors;
400
424
  /**
401
425
  * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
402
426
  * `dry_run` previews every count without writing — and without loading the model, which is why
403
- * the preview also reports `would_warm` (see the contract's `VectorsFixReport`).
427
+ * the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
404
428
  */
405
- vectorsFix(dryRun?: boolean): Promise<VectorsFixReport>;
429
+ vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
406
430
  /**
407
431
  * ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
408
432
  * re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
409
433
  * background migration ({@link migrateVectors}) and its tests drive.
410
- *
411
- * Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
412
- * stale from the persisted width/space, so a restart simply continues where the last slice stopped.
413
434
  */
414
435
  migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
415
436
  /**
416
437
  * Drive the bounded background migration until the store is current, the model is unavailable, or
417
- * `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE never rejects: it is
418
- * logged and reported as `remaining > 0`, which is what makes the next heartbeat a retry (the caller
419
- * still owns the outer catch for a closed database, e.g. a shutdown racing the first read). Never
420
- * blocks the event loop for longer than one batch.
421
- *
422
- * Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
423
- * on background re-encoding can leave the loud warning and the manual entry as the only path.
438
+ * `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
439
+ * switch, the no-progress rule and the store-named logging all live in the shared flow
440
+ * (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
424
441
  */
425
442
  migrateVectors(opts?: {
426
443
  batchSize?: number;
427
444
  shouldStop?: () => boolean;
428
445
  onProgress?: (progress: VectorMigrationOutcome) => void;
429
446
  }): Promise<VectorMigrationOutcome>;
430
- /** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
431
- private migrationRemaining;
432
447
  /**
433
448
  * One lifecycle pass, owned by the store so the DB writes and the live-index
434
449
  * invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
@@ -466,14 +481,10 @@ export declare class MemoryStore {
466
481
  * loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
467
482
  * every selected row's `content` before rebuilding the first one.
468
483
  *
469
- * The in-flight guard is here rather than at the callers because the callers CAN overlap: the
470
- * plugin's heartbeat and `maintenance` are both periodic, and `staleEntityRows` is ORDERED, so
471
- * two concurrent passes would select the same rows and re-tag them twice. Overlap is not
472
- * corruption (each fact is stamped in its own transaction) — it is duplicated mission in a
473
- * single-threaded process. A pass that finds one running reports `skipped` and does nothing.
474
- *
475
- * `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
476
- * caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
484
+ * The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
485
+ * live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
486
+ * entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
487
+ * selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
477
488
  */
478
489
  reindexEntities(budget?: number): Promise<EntitySweepReport>;
479
490
  /**
@@ -483,8 +494,6 @@ export declare class MemoryStore {
483
494
  * `ContradictionsDao.resolveForFact`).
484
495
  */
485
496
  private retireConflicts;
486
- /** Drop vectors of archived/purged facts from the live index (batched). */
487
- private evictVectors;
488
497
  /**
489
498
  * Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
490
499
  * count. Archived (incl. superseded) facts are excluded — their entities are not
@@ -581,7 +590,10 @@ export declare class MemoryStore {
581
590
  * Semantic leg. `queryVector` short-circuits the encode when the caller already
582
591
  * encoded this exact query (the cross-store router does, for both stores at once);
583
592
  * `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
584
- * instead of re-encoding (performance review §7.7 / P8).
593
+ * instead of re-encoding (performance review §7.7 / P8). The flow is shared
594
+ * (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
595
+ * notion of status/category/source, so candidates go through the DB and archived/purged facts,
596
+ * other categories and other sources never leak into the leg.
585
597
  */
586
598
  private semanticPath;
587
599
  /**