@avantf/dsh-mem 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +60 -4
- package/lib/engine/mem/db/dao/shared.d.ts +153 -0
- package/lib/engine/mem/dispatch.d.ts +19 -1
- package/lib/engine/mem/eval/index.d.ts +4 -0
- package/lib/engine/mem/index.d.ts +2 -0
- package/lib/engine/mem/runtime.d.ts +2 -2
- package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
- package/lib/engine/mem/store/hybrid.d.ts +50 -0
- package/lib/engine/mem/store/knowledge.d.ts +116 -12
- package/lib/engine/mem/store/legs.d.ts +171 -0
- package/lib/engine/mem/store/lexical.d.ts +8 -0
- package/lib/engine/mem/store/memory.d.ts +84 -72
- package/lib/engine/mem/store/vector_repair.d.ts +248 -0
- package/lib/engine/mem/trustHeartbeat.d.ts +47 -0
- package/lib/engine/mem-contract/tools.d.ts +4 -0
- package/lib/engine/mem-contract/types.d.ts +37 -6
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/index.js +1653 -703
- package/lib/interface-version.json +1 -1
- package/lib/types/envinit-bootstrap.d.ts +1 -1
- package/lib/types/envinit.d.ts +0 -3
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/interface_gate.d.ts +70 -0
- package/lib/types/interface_gate.d.ts.map +1 -1
- package/lib/types/provision.d.ts +6 -0
- package/lib/types/provision.d.ts.map +1 -1
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +22 -18
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport } from '../../mem-contract/index.js';
|
|
1
|
+
import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsFixStoreReport } from '../../mem-contract/index.js';
|
|
2
2
|
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
3
|
import { type LexicalProbe } from './lexical.js';
|
|
4
4
|
import { type FloorLeg } from './floors.js';
|
|
5
5
|
import { type HybridResult } from './hybrid.js';
|
|
6
|
+
import { type VectorRepairTarget } from './vector_repair.js';
|
|
6
7
|
/** Options for {@link KnowledgeStore.search}. */
|
|
7
8
|
export interface KnowledgeSearchOptions {
|
|
8
9
|
domain?: string;
|
|
@@ -110,6 +111,15 @@ export declare class KnowledgeStore {
|
|
|
110
111
|
private readonly docFiles;
|
|
111
112
|
/** > 0 while a batch (directory import) is writing: commits and baseline refreshes wait. */
|
|
112
113
|
private deferCorpusWrites;
|
|
114
|
+
/**
|
|
115
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
116
|
+
*
|
|
117
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It exists
|
|
118
|
+
* as a field — rather than the adapter object, which is built per call — because the guard must
|
|
119
|
+
* survive between two overlapping calls: `reindex` can be reached from a tool call and the
|
|
120
|
+
* settings-page button at once, and both would select the same stale chunks.
|
|
121
|
+
*/
|
|
122
|
+
private readonly entitySweepGuard;
|
|
113
123
|
/**
|
|
114
124
|
* Per-file stamps (`path → mtimeMs:size`) and the file-set they were taken from, so a reconcile
|
|
115
125
|
* can ask "what changed?" for the price of one `stat` per document instead of reading and
|
|
@@ -132,13 +142,18 @@ export declare class KnowledgeStore {
|
|
|
132
142
|
/** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
|
|
133
143
|
domainConfigPath: string, logger?: AvantfLogger | undefined);
|
|
134
144
|
/**
|
|
135
|
-
* Rebuild the in-memory vstore from persisted chunk vectors
|
|
145
|
+
* Rebuild the in-memory vstore from persisted chunk vectors, and say loudly what belongs to
|
|
146
|
+
* another vector space.
|
|
147
|
+
*
|
|
148
|
+
* Also detects a VECTOR-SPACE change, which the reload itself cannot see: the reload checks the
|
|
149
|
+
* width, and a different model at the same width produces dim-valid vectors that are then ranked
|
|
150
|
+
* against the current query's — two spaces mixed in one index, with no error anywhere.
|
|
136
151
|
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
152
|
+
* The reload, the count and the warning are the SHARED open-time step (`store/vector_repair.ts`),
|
|
153
|
+
* which is what turned the knowledge store from "warns about a swapped model but never repairs it"
|
|
154
|
+
* into the same automatic path memory has always had; this store's half is the row read. The
|
|
155
|
+
* manual entry stays `vectors --fix` (the vector-only repair); `kb_reindex` remains the full
|
|
156
|
+
* FTS + entities + vectors re-entry.
|
|
142
157
|
*/
|
|
143
158
|
private reloadIndex;
|
|
144
159
|
/**
|
|
@@ -243,8 +258,6 @@ export declare class KnowledgeStore {
|
|
|
243
258
|
warnings?: string[]): Promise<IngestResult>;
|
|
244
259
|
/** Materialize one document's editable copy; returns the `file` / `file_error` result fields. */
|
|
245
260
|
private writeManagedFile;
|
|
246
|
-
/** Batched eviction from the live vector index (removeMany when the backend supports it). */
|
|
247
|
-
private evictVectors;
|
|
248
261
|
/** The resolved ingestion boundary (see `ingest_guard.ts`). */
|
|
249
262
|
private get ingestLimits();
|
|
250
263
|
/**
|
|
@@ -288,7 +301,8 @@ export declare class KnowledgeStore {
|
|
|
288
301
|
*/
|
|
289
302
|
private writeReusedVectors;
|
|
290
303
|
/**
|
|
291
|
-
*
|
|
304
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
305
|
+
* call, the live index and the `WRITE_BATCH` transaction this store measured its way into.
|
|
292
306
|
*
|
|
293
307
|
* The model call is deliberately NOT batched, and that is a MEASURED decision, not an omission:
|
|
294
308
|
* a batched forward pass pads each item to the longest in its group, and real chunks span 53–550
|
|
@@ -299,7 +313,17 @@ export declare class KnowledgeStore {
|
|
|
299
313
|
* length-sorted) for callers whose texts suit it.
|
|
300
314
|
*
|
|
301
315
|
* What DID pay off is the write side: one commit per `WRITE_BATCH` chunks instead of one per
|
|
302
|
-
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await
|
|
316
|
+
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await` — that
|
|
317
|
+
* is the `writeBatch` below.
|
|
318
|
+
*/
|
|
319
|
+
private encodeWriteTarget;
|
|
320
|
+
/**
|
|
321
|
+
* Encode texts one at a time and persist them in `WRITE_BATCH` transactions.
|
|
322
|
+
*
|
|
323
|
+
* A failed chunk is skipped, not thrown: the rows are already committed and their old vectors
|
|
324
|
+
* evicted by the ingest path, so a throw here would leave the caller with a failed ingest and NO
|
|
325
|
+
* COUNT of what was skipped. The shortfall travels back to the tool result
|
|
326
|
+
* (`IngestResult.vectors_failed`) instead of living only in a log line.
|
|
303
327
|
*/
|
|
304
328
|
private encodeAndStore;
|
|
305
329
|
/**
|
|
@@ -309,6 +333,59 @@ export declare class KnowledgeStore {
|
|
|
309
333
|
* of silently mixing two spaces. Public so diagnostics and tests can name the space.
|
|
310
334
|
*/
|
|
311
335
|
vectorSpace(): string;
|
|
336
|
+
/**
|
|
337
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of
|
|
338
|
+
* persisted chunk vectors whose width or recorded space id no longer matches the one this store
|
|
339
|
+
* writes into. Reads blob LENGTHS only — never decodes — so the startup warning and the background
|
|
340
|
+
* drive can afford it at any corpus size.
|
|
341
|
+
*/
|
|
342
|
+
vectorSpaceHealth(): VectorSpaceHealth;
|
|
343
|
+
/** {@link vectorSpaceHealth} plus `missing`, the third term of the repair flow's `remaining`. */
|
|
344
|
+
private vectorSpaceCounts;
|
|
345
|
+
/** Async bootstrap the semantic backend (the shared flow's explicit-fix warmup). */
|
|
346
|
+
warmupSemantic(): Promise<boolean>;
|
|
347
|
+
/**
|
|
348
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
349
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
350
|
+
* `semantic.auto_migrate`, the dry-run split — lives there once, for both stores.
|
|
351
|
+
*/
|
|
352
|
+
vectorRepairTarget(): VectorRepairTarget;
|
|
353
|
+
/**
|
|
354
|
+
* The chunks the semantic leg cannot use (no vector / wrong width / another space). One SQL
|
|
355
|
+
* aggregate over blob lengths; `dropIds` is empty because knowledge overwrites in place.
|
|
356
|
+
*/
|
|
357
|
+
private classifyRepairVectors;
|
|
358
|
+
/**
|
|
359
|
+
* Encode + persist up to `limit` chunks the semantic leg cannot use, bounded and ordered by id.
|
|
360
|
+
*
|
|
361
|
+
* It does NOT re-extract entities or rebuild FTS: that is `kb_reindex`'s job, and the whole point
|
|
362
|
+
* of the automatic repair is that the vector half can run on its own — which is exactly the
|
|
363
|
+
* objection the "known boundary" in DESIGN §20 used to raise against a background run.
|
|
364
|
+
*/
|
|
365
|
+
private reencodeStaleChunks;
|
|
366
|
+
/**
|
|
367
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
368
|
+
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
369
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
370
|
+
*/
|
|
371
|
+
vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
|
|
372
|
+
/**
|
|
373
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` chunks are re-encoded, so
|
|
374
|
+
* the caller keeps control of the event loop between slices. This is the unit the background
|
|
375
|
+
* migration ({@link migrateVectors}) and its tests drive.
|
|
376
|
+
*/
|
|
377
|
+
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
378
|
+
/**
|
|
379
|
+
* Drive the bounded background migration for this store until it is current, the model is
|
|
380
|
+
* unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the
|
|
381
|
+
* `semantic.auto_migrate` switch, the no-progress rule and the store-named logging all live in the
|
|
382
|
+
* shared flow (`store/vector_repair.ts`); this is the one-store entry.
|
|
383
|
+
*/
|
|
384
|
+
migrateVectors(opts?: {
|
|
385
|
+
batchSize?: number;
|
|
386
|
+
shouldStop?: () => boolean;
|
|
387
|
+
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
388
|
+
}): Promise<VectorMigrationOutcome>;
|
|
312
389
|
/**
|
|
313
390
|
* A stored vector is reusable when it exists, matches the space, and its text is unchanged.
|
|
314
391
|
*
|
|
@@ -319,8 +396,28 @@ export declare class KnowledgeStore {
|
|
|
319
396
|
* the store would keep serving a vector for text it no longer holds.
|
|
320
397
|
*/
|
|
321
398
|
private vectorReusable;
|
|
322
|
-
/**
|
|
399
|
+
/**
|
|
400
|
+
* Extract one chunk's entity names, bounded by `MAX_ENTITIES_PER_CHUNK`.
|
|
401
|
+
*
|
|
402
|
+
* The bound is here, not in either caller: it is a property of what `chunk_entities` may hold for
|
|
403
|
+
* one chunk, and the ingest path and the sweep must not disagree about it.
|
|
404
|
+
*/
|
|
405
|
+
private extractRowEntities;
|
|
406
|
+
/** One transaction replacing the entity rows of the given chunks (the ingest path). */
|
|
323
407
|
private replaceChunkEntities;
|
|
408
|
+
/**
|
|
409
|
+
* The per-row primitive of the shared entity sweep (`store/entity_sweep.ts`).
|
|
410
|
+
*
|
|
411
|
+
* Replaces ONE chunk's entity rows and stamps that same row with `ENTITY_EXTRACTOR_VERSION`, in one
|
|
412
|
+
* transaction. Knowledge's domain difference from memory is exactly this: memory's transaction also
|
|
413
|
+
* moves triples, the HRR bundle and the conflict queue, while a chunk has none of those — its
|
|
414
|
+
* derived state is the entity names alone.
|
|
415
|
+
*
|
|
416
|
+
* Per row, not per batch, because the sweep's contract is that a failure leaves THAT row stale and
|
|
417
|
+
* retryable: inside a batched transaction one bad extraction would roll back the whole batch, and
|
|
418
|
+
* rows it never reached would look stamped.
|
|
419
|
+
*/
|
|
420
|
+
private replaceRowEntities;
|
|
324
421
|
/**
|
|
325
422
|
* Ingest local files/directories. Every path goes through the SAME boundary as
|
|
326
423
|
* `ingestUri` — a bare `kb_import` on `~/.ssh` would otherwise be the exact
|
|
@@ -459,6 +556,13 @@ export declare class KnowledgeStore {
|
|
|
459
556
|
* re-encoding (performance review §7.7 / P8).
|
|
460
557
|
*/
|
|
461
558
|
private semanticPath;
|
|
559
|
+
/**
|
|
560
|
+
* The semantic leg's scope filter, as a `filterTopk` primitive: `undefined` when NO domain/source
|
|
561
|
+
* was asked means "keep the whole pool" — the short-circuit is preserved, so no `chunks.meta`
|
|
562
|
+
* lookup runs at all. Otherwise one batched lookup, and rows whose domain/source do not match (or
|
|
563
|
+
* whose chunk is gone) are dropped.
|
|
564
|
+
*/
|
|
565
|
+
private scopeFilter;
|
|
462
566
|
/**
|
|
463
567
|
* Sync relevance probe for the plugin's conditional hint (DESIGN §12). Same contract as
|
|
464
568
|
* `MemoryStore.lexicalProbe` — see `store/lexical.ts`.
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
|
|
3
|
+
* leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
|
|
4
|
+
* `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
|
|
5
|
+
* scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
|
|
6
|
+
* check's error and the Map insertion order live here once.
|
|
7
|
+
*
|
|
8
|
+
* `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
|
|
9
|
+
* a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
|
|
10
|
+
* `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
|
|
11
|
+
* empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
|
|
12
|
+
* paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
|
|
13
|
+
* from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
|
|
14
|
+
* probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
|
|
15
|
+
* at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
|
|
16
|
+
*
|
|
17
|
+
* WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
|
|
18
|
+
* signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
|
|
19
|
+
* counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
|
|
20
|
+
* quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
|
|
21
|
+
* deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
|
|
22
|
+
* floor functions only ever REMOVE entries, so the raw set is the honest measurement.
|
|
23
|
+
*
|
|
24
|
+
* WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
|
|
25
|
+
* `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
|
|
26
|
+
* {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
|
|
27
|
+
* the absence is structural, not something a caller can forget to set.
|
|
28
|
+
*
|
|
29
|
+
* WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
|
|
30
|
+
* table, and a template-method base class would be a structural rewrite of two hub files. This is
|
|
31
|
+
* the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
|
|
32
|
+
* be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
|
|
33
|
+
* third store in `test/legs.spec.ts`).
|
|
34
|
+
*
|
|
35
|
+
* CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
|
|
36
|
+
* (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
|
|
37
|
+
* legs, so both stores get the same awaiting and isolation by construction.
|
|
38
|
+
*
|
|
39
|
+
* @module store/legs
|
|
40
|
+
*/
|
|
41
|
+
import { type FloorLeg } from './floors.js';
|
|
42
|
+
import { type HybridContext, type HybridLeg } from './hybrid.js';
|
|
43
|
+
import { type FtsTokenizer } from '../db/tokenizer.js';
|
|
44
|
+
/**
|
|
45
|
+
* The legs {@link LegRunner.scored} may wrap: those whose raw score has an absolute scale a floor
|
|
46
|
+
* can cut. `semantic` is excluded on purpose — it shares `applyScoreFloor` but must not carry the
|
|
47
|
+
* cap signal, and it is reached through {@link LegRunner.semantic} (see the module comment).
|
|
48
|
+
*/
|
|
49
|
+
export type ScoredLeg = Exclude<FloorLeg, 'semantic'>;
|
|
50
|
+
/** Build the leg runner for one query. `legCap` is the cap every non-semantic leg is measured against. */
|
|
51
|
+
export interface LegRunner {
|
|
52
|
+
/** The per-leg row cap in force for this query (see `hybrid.ts`'s `legCapFor`). */
|
|
53
|
+
readonly cap: number;
|
|
54
|
+
/**
|
|
55
|
+
* Wrap raw scores that have NO floor of their own (memory's HRR probe and time-window scan): the
|
|
56
|
+
* cap signal is attached and nothing else — no `leg` label, no `droppedByFloor`, exactly as those
|
|
57
|
+
* legs have always shipped (they share the entity leg's weight and floor report).
|
|
58
|
+
*/
|
|
59
|
+
plain(scores: Map<number, number>, weight: number): HybridLeg;
|
|
60
|
+
/**
|
|
61
|
+
* A score-floored leg: `applyScoreFloor` on the raw map, then the cap measured on the RAW map. The
|
|
62
|
+
* floored map is what the leg ships; `raw` is only ever read for the cap.
|
|
63
|
+
*/
|
|
64
|
+
scored(name: ScoredLeg, raw: Map<number, number>, weight: number, floor: number): HybridLeg;
|
|
65
|
+
/**
|
|
66
|
+
* The FTS leg: the per-row term floor (`applyTermFloor`) needs the candidates' texts, read by the
|
|
67
|
+
* store for the whole raw set (one batched query). The cap is measured on the raw map.
|
|
68
|
+
*/
|
|
69
|
+
term(name: 'fts', raw: Map<number, number>, texts: Map<number, string>, query: string, weight: number, floor: number): HybridLeg;
|
|
70
|
+
/**
|
|
71
|
+
* The semantic leg: score-floored like {@link scored}, but deliberately WITHOUT a `capped` key
|
|
72
|
+
* (see the module comment). The async model encode itself happens at the store, which is the only
|
|
73
|
+
* place that knows its backend and filters.
|
|
74
|
+
*/
|
|
75
|
+
semantic(raw: Map<number, number>, weight: number, floor: number): HybridLeg;
|
|
76
|
+
/**
|
|
77
|
+
* The fallback for a DOWN semantic backend: an EMPTY leg at the SAME weight, reporting zero floor
|
|
78
|
+
* drops. Equal weight, not zero: the leg is absent, the weight contract (`RecallResult.weights`)
|
|
79
|
+
* must not move to a fourth value, and the orchestrator already rebalanced the weights for
|
|
80
|
+
* `semAvail === false` (`DEGRADED_WEIGHTS`).
|
|
81
|
+
*/
|
|
82
|
+
semanticOff(weight: number): HybridLeg;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Build the leg runner for one query.
|
|
86
|
+
*
|
|
87
|
+
* The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
|
|
88
|
+
* index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
|
|
89
|
+
* self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
|
|
90
|
+
* evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
|
|
91
|
+
* same wrapper, the same floor order and the same down-semantic fallback.
|
|
92
|
+
*/
|
|
93
|
+
export declare function makeLegRunner(ctx: Pick<HybridContext, 'legCap'>): LegRunner;
|
|
94
|
+
/** The per-store pieces of the FTS leg — see {@link ftsLeg}. `Scope` is opaque to the flow. */
|
|
95
|
+
export interface FtsLegOptions<Scope> {
|
|
96
|
+
query: string;
|
|
97
|
+
/** The per-leg row cap (the store's `legCap`), handed to whichever DAO statement runs. */
|
|
98
|
+
cap: number;
|
|
99
|
+
/** The store's scope shape (`{category, source}` / `{domain, source}`), passed straight through. */
|
|
100
|
+
scope: Scope;
|
|
101
|
+
/** The table's own tokenizer, for the `MATCH` build (`buildFtsQuery`). */
|
|
102
|
+
tokenizer: FtsTokenizer;
|
|
103
|
+
/** The FTS `MATCH` statement; its `rank` is bm25 (negative = better). */
|
|
104
|
+
search: (ftsQuery: string, scope: Scope, limit: number) => readonly {
|
|
105
|
+
id: number;
|
|
106
|
+
rank: number;
|
|
107
|
+
}[];
|
|
108
|
+
/** The short-query LIKE statement; its `rank` counts the terms the row contains (higher = better). */
|
|
109
|
+
substringSearch: (terms: readonly string[], scope: Scope, limit: number) => readonly {
|
|
110
|
+
id: number;
|
|
111
|
+
rank: number;
|
|
112
|
+
}[];
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
|
|
116
|
+
*
|
|
117
|
+
* The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
|
|
118
|
+
* bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
|
|
119
|
+
* order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
|
|
120
|
+
*
|
|
121
|
+
* - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
|
|
122
|
+
* stays better everywhere downstream;
|
|
123
|
+
* - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
|
|
124
|
+
* on the same "higher = better" scale;
|
|
125
|
+
* - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
|
|
126
|
+
* 3-char run) — the leg is empty by construction, not by a query that matched nothing.
|
|
127
|
+
*
|
|
128
|
+
* THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
|
|
129
|
+
* trigram index can express (measured: every term in the FTS table is 3 characters), which used to
|
|
130
|
+
* leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
|
|
131
|
+
* predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
|
|
132
|
+
* is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
|
|
133
|
+
*/
|
|
134
|
+
export declare function ftsLeg<Scope>(opts: FtsLegOptions<Scope>): Map<number, number>;
|
|
135
|
+
/** The per-store pieces of the semantic leg — see {@link semanticLeg}. */
|
|
136
|
+
export interface SemanticLegOptions {
|
|
137
|
+
query: string;
|
|
138
|
+
/** The requested leg size; the pool is over-fetched to `max(50, k)` below. */
|
|
139
|
+
k: number;
|
|
140
|
+
/** A caller-encoded vector for this exact query; skips {@link encode}. */
|
|
141
|
+
queryVector?: Float32Array;
|
|
142
|
+
/** Publishes the vector actually used, so the relaxed retry reuses it instead of re-encoding. */
|
|
143
|
+
onVector?: (vec: Float32Array) => void;
|
|
144
|
+
encode: (query: string) => Promise<Float32Array>;
|
|
145
|
+
/** The vstore's width, checked against the vector before it is scored. */
|
|
146
|
+
dim: number;
|
|
147
|
+
topk: (vec: Float32Array, k: number) => readonly {
|
|
148
|
+
id: number;
|
|
149
|
+
score: number;
|
|
150
|
+
}[];
|
|
151
|
+
/**
|
|
152
|
+
* The store's eligibility filter over the pool ids.
|
|
153
|
+
*
|
|
154
|
+
* `undefined` = **keep everything** — knowledge's "no domain/source asked" short-circuit stays a
|
|
155
|
+
* short-circuit (no lookup runs at all). A returned set is the allow-list; the flow still maps the
|
|
156
|
+
* pool in ITS order, so ids/scores/insertion order are the vstore's.
|
|
157
|
+
*/
|
|
158
|
+
filterTopk: (ids: readonly number[]) => ReadonlySet<number> | undefined;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
|
|
162
|
+
*
|
|
163
|
+
* The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
|
|
164
|
+
* `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
|
|
165
|
+
* to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
|
|
166
|
+
*
|
|
167
|
+
* Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
|
|
168
|
+
* rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
|
|
169
|
+
* empty-pool early return, and the returned map's insertion order (the vstore's pool order).
|
|
170
|
+
*/
|
|
171
|
+
export declare function semanticLeg(opts: SemanticLegOptions): Promise<Map<number, number>>;
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
* path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
|
|
41
41
|
* (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
|
|
42
42
|
*/
|
|
43
|
+
import { type FtsTokenizer } from '../db/tokenizer.js';
|
|
43
44
|
/** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
|
|
44
45
|
export interface LexicalProbe {
|
|
45
46
|
terms: number;
|
|
@@ -109,3 +110,10 @@ export declare function looksRelevant(probe: LexicalProbe): boolean;
|
|
|
109
110
|
* path is synchronous); the default keeps an exact count for the calibration recorded above.
|
|
110
111
|
*/
|
|
111
112
|
export declare function probeTerms(text: string, holds: (term: string) => boolean, stopAt?: number): LexicalProbe;
|
|
113
|
+
/**
|
|
114
|
+
* The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
|
|
115
|
+
* expression for one term (the tokenizer is a property of the store's FTS table) and ask that
|
|
116
|
+
* table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
|
|
117
|
+
* one part that has to differ, which is now a single `holds` closure per store.
|
|
118
|
+
*/
|
|
119
|
+
export declare function probeStoreTerms(text: string, tokenizer: FtsTokenizer, holds: (ftsQuery: string) => boolean, stopAt?: number): LexicalProbe;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixStoreReport } from '../../mem-contract/index.js';
|
|
2
2
|
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
|
+
import { ENTITY_SWEEP_BATCH, type EntitySweepReport } from './entity_sweep.js';
|
|
4
|
+
import { DEFAULT_VECTOR_MIGRATION_BATCH, type VectorRepairTarget } from './vector_repair.js';
|
|
3
5
|
import type { Db } from '../db/conn.js';
|
|
4
6
|
import { type LexicalProbe } from './lexical.js';
|
|
5
7
|
import { type FloorLeg } from './floors.js';
|
|
@@ -15,35 +17,22 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
|
|
|
15
17
|
/**
|
|
16
18
|
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
17
19
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
* Re-exported from the shared sweep (`store/entity_sweep.ts`), which owns the loop and therefore the
|
|
21
|
+
* default: the public name has to keep working for the CLI (`cli/src/index.ts` passes it explicitly)
|
|
22
|
+
* and the plugin's heartbeat, but a second literal here would be a second answer to "how big is one
|
|
23
|
+
* pass".
|
|
22
24
|
*/
|
|
23
|
-
export
|
|
25
|
+
export { ENTITY_SWEEP_BATCH };
|
|
24
26
|
/**
|
|
25
27
|
* Rows one background vector-space migration slice may drop + re-encode.
|
|
26
28
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
|
|
31
|
-
* whatever the caller needs.
|
|
29
|
+
* Re-exported from the shared repair flow (`store/vector_repair.ts`), which owns the loop and
|
|
30
|
+
* therefore the default: the public name has to keep working for callers that pass it explicitly,
|
|
31
|
+
* but a second literal here would be a second answer to "how big is one slice".
|
|
32
32
|
*/
|
|
33
|
-
export
|
|
34
|
-
/** What one derived-state sweep pass did (
|
|
35
|
-
export
|
|
36
|
-
/**
|
|
37
|
-
* Facts this pass RE-EXTRACTED, i.e. its batch size when it filled it. Counts rows VISITED and
|
|
38
|
-
* re-stamped, not rows whose extraction output differs — the stamp is what makes the corpus
|
|
39
|
-
* current, so a row re-extracted to the same names still counts (and is not rebuilt again).
|
|
40
|
-
*/
|
|
41
|
-
rebuilt: number;
|
|
42
|
-
/** Facts still written by older rules after this pass. */
|
|
43
|
-
deferred: number;
|
|
44
|
-
/** Another pass was already running, so this one did nothing (`deferred` is still current). */
|
|
45
|
-
skipped: boolean;
|
|
46
|
-
}
|
|
33
|
+
export { DEFAULT_VECTOR_MIGRATION_BATCH };
|
|
34
|
+
/** What one derived-state sweep pass did — re-exported from the shared loop (`store/entity_sweep.ts`). */
|
|
35
|
+
export type { EntitySweepReport };
|
|
47
36
|
/** What one bounded conflict-drain pass did (see `MemoryStore.drainConflicts`). */
|
|
48
37
|
export interface ConflictDrainReport {
|
|
49
38
|
/** Pairs newly logged by this pass. */
|
|
@@ -177,8 +166,14 @@ export declare class MemoryStore {
|
|
|
177
166
|
* and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
|
|
178
167
|
*/
|
|
179
168
|
private readonly ftsTokenizer;
|
|
180
|
-
/**
|
|
181
|
-
|
|
169
|
+
/**
|
|
170
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
171
|
+
*
|
|
172
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
|
|
173
|
+
* the rule where the loop is (the periodic callers can overlap), without a mutable field that two
|
|
174
|
+
* stores would each have to remember to hold.
|
|
175
|
+
*/
|
|
176
|
+
private readonly entitySweepGuard;
|
|
182
177
|
/**
|
|
183
178
|
* Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
|
|
184
179
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
@@ -202,13 +197,19 @@ export declare class MemoryStore {
|
|
|
202
197
|
*/
|
|
203
198
|
vectorSpace(): string;
|
|
204
199
|
/**
|
|
205
|
-
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs
|
|
206
|
-
*
|
|
207
|
-
*
|
|
200
|
+
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
|
|
201
|
+
* belongs to another vector space.
|
|
202
|
+
*
|
|
203
|
+
* Without this the semantic path is silently empty after every process restart (the DB is the
|
|
204
|
+
* authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
|
|
205
|
+
* archived facts are filtered out of every retrieval path anyway, so loading them would only grow
|
|
206
|
+
* the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
208
207
|
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*
|
|
208
|
+
* The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
|
|
209
|
+
* step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
|
|
210
|
+
* another space is still RANKED (the reload cannot tell the difference), so a silent model swap
|
|
211
|
+
* would mix two spaces in one index — acting on it is the manual fix's job for the operator path
|
|
212
|
+
* and the bounded background migration's for the automatic one.
|
|
212
213
|
*/
|
|
213
214
|
private reloadIndex;
|
|
214
215
|
/**
|
|
@@ -244,6 +245,14 @@ export declare class MemoryStore {
|
|
|
244
245
|
* stamped the marker anyway — a permanent, silent exemption).
|
|
245
246
|
*/
|
|
246
247
|
private detectContradictions;
|
|
248
|
+
/**
|
|
249
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
250
|
+
* call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
|
|
251
|
+
* backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
|
|
252
|
+
* (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
|
|
253
|
+
*/
|
|
254
|
+
private vectorWriteTarget;
|
|
255
|
+
private warnIndexFailure;
|
|
247
256
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
248
257
|
private maybeIndexSemantic;
|
|
249
258
|
update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
@@ -381,54 +390,60 @@ export declare class MemoryStore {
|
|
|
381
390
|
*/
|
|
382
391
|
private classifyVectors;
|
|
383
392
|
/**
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
394
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
395
|
+
* `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
|
|
396
|
+
* for both stores.
|
|
397
|
+
*/
|
|
398
|
+
vectorRepairTarget(): VectorRepairTarget;
|
|
399
|
+
/**
|
|
400
|
+
* {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
|
|
401
|
+
* (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
|
|
402
|
+
*/
|
|
403
|
+
private classifyRepairVectors;
|
|
404
|
+
/** Put usable persisted vectors missing from the live index back into it (no model needed). */
|
|
405
|
+
private reindexUsableVectors;
|
|
406
|
+
/**
|
|
407
|
+
* Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
|
|
408
|
+
* pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
|
|
409
|
+
* the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
|
|
410
|
+
* one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
|
|
411
|
+
*
|
|
412
|
+
* `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
|
|
413
|
+
* the embedding leg was checked against is gone, so the check has to run again.
|
|
414
|
+
*/
|
|
415
|
+
private dropRepairVectors;
|
|
416
|
+
/**
|
|
417
|
+
* Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
|
|
393
418
|
*
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
397
|
-
* `migrateVectorsBatch` passes the migration's batch size.
|
|
419
|
+
* Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
|
|
420
|
+
* the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
|
|
421
|
+
* ran has not been checked.
|
|
398
422
|
*/
|
|
399
|
-
private
|
|
423
|
+
private reencodeMissingVectors;
|
|
400
424
|
/**
|
|
401
425
|
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
402
426
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
403
|
-
* the preview also reports `would_warm` (see the contract's `
|
|
427
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
404
428
|
*/
|
|
405
|
-
vectorsFix(dryRun?: boolean): Promise<
|
|
429
|
+
vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
|
|
406
430
|
/**
|
|
407
431
|
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
408
432
|
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
409
433
|
* background migration ({@link migrateVectors}) and its tests drive.
|
|
410
|
-
*
|
|
411
|
-
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
412
|
-
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
413
434
|
*/
|
|
414
435
|
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
415
436
|
/**
|
|
416
437
|
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
417
|
-
* `shouldStop()` turns true (plugin unmount / process shutdown).
|
|
418
|
-
*
|
|
419
|
-
*
|
|
420
|
-
* blocks the event loop for longer than one batch.
|
|
421
|
-
*
|
|
422
|
-
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
423
|
-
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
438
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
|
|
439
|
+
* switch, the no-progress rule and the store-named logging all live in the shared flow
|
|
440
|
+
* (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
|
|
424
441
|
*/
|
|
425
442
|
migrateVectors(opts?: {
|
|
426
443
|
batchSize?: number;
|
|
427
444
|
shouldStop?: () => boolean;
|
|
428
445
|
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
429
446
|
}): Promise<VectorMigrationOutcome>;
|
|
430
|
-
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
431
|
-
private migrationRemaining;
|
|
432
447
|
/**
|
|
433
448
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
434
449
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
@@ -466,14 +481,10 @@ export declare class MemoryStore {
|
|
|
466
481
|
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
467
482
|
* every selected row's `content` before rebuilding the first one.
|
|
468
483
|
*
|
|
469
|
-
* The in-flight guard
|
|
470
|
-
*
|
|
471
|
-
*
|
|
472
|
-
*
|
|
473
|
-
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
474
|
-
*
|
|
475
|
-
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
476
|
-
* caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
|
|
484
|
+
* The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
|
|
485
|
+
* live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
|
|
486
|
+
* entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
|
|
487
|
+
* selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
|
|
477
488
|
*/
|
|
478
489
|
reindexEntities(budget?: number): Promise<EntitySweepReport>;
|
|
479
490
|
/**
|
|
@@ -483,8 +494,6 @@ export declare class MemoryStore {
|
|
|
483
494
|
* `ContradictionsDao.resolveForFact`).
|
|
484
495
|
*/
|
|
485
496
|
private retireConflicts;
|
|
486
|
-
/** Drop vectors of archived/purged facts from the live index (batched). */
|
|
487
|
-
private evictVectors;
|
|
488
497
|
/**
|
|
489
498
|
* Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
|
|
490
499
|
* count. Archived (incl. superseded) facts are excluded — their entities are not
|
|
@@ -581,7 +590,10 @@ export declare class MemoryStore {
|
|
|
581
590
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
582
591
|
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
583
592
|
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
584
|
-
* instead of re-encoding (performance review §7.7 / P8).
|
|
593
|
+
* instead of re-encoding (performance review §7.7 / P8). The flow is shared
|
|
594
|
+
* (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
|
|
595
|
+
* notion of status/category/source, so candidates go through the DB and archived/purged facts,
|
|
596
|
+
* other categories and other sources never leak into the leg.
|
|
585
597
|
*/
|
|
586
598
|
private semanticPath;
|
|
587
599
|
/**
|