@avantf/dsh-mem 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +60 -4
- package/lib/engine/mem/db/dao/shared.d.ts +153 -0
- package/lib/engine/mem/eval/index.d.ts +4 -0
- package/lib/engine/mem/index.d.ts +1 -0
- package/lib/engine/mem/runtime.d.ts +2 -2
- package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
- package/lib/engine/mem/store/hybrid.d.ts +50 -0
- package/lib/engine/mem/store/knowledge.d.ts +116 -12
- package/lib/engine/mem/store/legs.d.ts +171 -0
- package/lib/engine/mem/store/lexical.d.ts +8 -0
- package/lib/engine/mem/store/memory.d.ts +84 -72
- package/lib/engine/mem/store/vector_repair.d.ts +248 -0
- package/lib/engine/mem-contract/tools.d.ts +4 -0
- package/lib/engine/mem-contract/types.d.ts +37 -6
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/index.js +1370 -538
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +22 -18
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
package/lib/index.js
CHANGED
|
@@ -708,7 +708,8 @@ const AdminUnion = z.discriminatedUnion("action", [
|
|
|
708
708
|
z.object({ action: z.literal("vectors_diagnose") }),
|
|
709
709
|
z.object({
|
|
710
710
|
action: z.literal("vectors_fix"),
|
|
711
|
-
dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。")
|
|
711
|
+
dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。"),
|
|
712
|
+
store: z.enum(["memory", "knowledge"]).optional().describe("只修某一个库(memory / knowledge);省略=两库都修。")
|
|
712
713
|
}),
|
|
713
714
|
z.object({ action: z.literal("contradict_check") }),
|
|
714
715
|
z.object({
|
|
@@ -6870,6 +6871,18 @@ function probeTerms(text, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
|
6870
6871
|
matched
|
|
6871
6872
|
};
|
|
6872
6873
|
}
|
|
6874
|
+
/**
|
|
6875
|
+
* The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
|
|
6876
|
+
* expression for one term (the tokenizer is a property of the store's FTS table) and ask that
|
|
6877
|
+
* table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
|
|
6878
|
+
* one part that has to differ, which is now a single `holds` closure per store.
|
|
6879
|
+
*/
|
|
6880
|
+
function probeStoreTerms(text, tokenizer, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
6881
|
+
return probeTerms(text, (term) => {
|
|
6882
|
+
const fts = buildFtsQuery(term, tokenizer);
|
|
6883
|
+
return fts !== null && holds(fts);
|
|
6884
|
+
}, stopAt);
|
|
6885
|
+
}
|
|
6873
6886
|
//#endregion
|
|
6874
6887
|
//#region ../core/lib/store/self_query.js
|
|
6875
6888
|
/**
|
|
@@ -7393,6 +7406,60 @@ async function hybridSearch(deps, plan) {
|
|
|
7393
7406
|
};
|
|
7394
7407
|
}
|
|
7395
7408
|
/**
|
|
7409
|
+
* Map a store's search arguments onto {@link HybridPlan} and run {@link hybridSearch}. ONE copy.
|
|
7410
|
+
*
|
|
7411
|
+
* Before this, `MemoryStore.search` and `KnowledgeStore.search` carried the mapping verbatim side by
|
|
7412
|
+
* side: the same field list, the same conditional spreads, the same call. That list is part of the
|
|
7413
|
+
* orchestrator's contract, so an option added to one entry point and not the other would silently
|
|
7414
|
+
* make the two stores query differently — the same shape of drift this module already documents
|
|
7415
|
+
* (`NaN` limit, `over_fetch_factor`, unrecorded capped legs).
|
|
7416
|
+
*
|
|
7417
|
+
* The conditional spreads are deliberate, not cosmetic: a caller that did not pass `overFetch` /
|
|
7418
|
+
* `includeScores` / `relaxLegs` / `rewriteQuery` leaves the plan key ABSENT rather than
|
|
7419
|
+
* present-with-`undefined`. The orchestrator reads all four through `??` / `=== true` today, so the
|
|
7420
|
+
* two are indistinguishable now — but the plan is a contract, and widening it with keys the caller
|
|
7421
|
+
* never set is how "absent means default" decays into "present-undefined means set to nothing".
|
|
7422
|
+
*/
|
|
7423
|
+
async function runHybridSearch(deps, query, fields) {
|
|
7424
|
+
return hybridSearch(deps, hybridPlanOf(query, fields));
|
|
7425
|
+
}
|
|
7426
|
+
/**
|
|
7427
|
+
* The mapping itself, separated from the call so it can be pinned directly: the contract here is
|
|
7428
|
+
* the SHAPE of the plan (which keys exist), which no behavioural assertion inside `hybridSearch`
|
|
7429
|
+
* could observe once every optional field is read through `??` / `=== true`.
|
|
7430
|
+
*/
|
|
7431
|
+
function hybridPlanOf(query, fields) {
|
|
7432
|
+
return {
|
|
7433
|
+
query,
|
|
7434
|
+
limit: fields?.limit,
|
|
7435
|
+
...fields?.overFetch === void 0 ? {} : { overFetch: fields.overFetch },
|
|
7436
|
+
maxTokens: fields?.maxTokens,
|
|
7437
|
+
queryVector: fields?.queryVector,
|
|
7438
|
+
recordStats: fields?.recordStats,
|
|
7439
|
+
floors: fields?.floors,
|
|
7440
|
+
...fields?.includeScores === void 0 ? {} : { includeScores: fields.includeScores },
|
|
7441
|
+
...fields?.relaxLegs === void 0 ? {} : { relaxLegs: fields.relaxLegs },
|
|
7442
|
+
...fields?.rewriteQuery === void 0 ? {} : { rewriteQuery: fields.rewriteQuery }
|
|
7443
|
+
};
|
|
7444
|
+
}
|
|
7445
|
+
/**
|
|
7446
|
+
* Build a store's {@link HybridDeps} — the ONE construction site for the contract fields both stores
|
|
7447
|
+
* share (`kind` / `config` / `semantic` / `legs` / `texts` / `hits` / `onReturn`). The per-store half
|
|
7448
|
+
* is `parts`; the stores used to spell the whole object out twice, so a new required `HybridDeps`
|
|
7449
|
+
* field was a change to remember in two places.
|
|
7450
|
+
*/
|
|
7451
|
+
function makeHybridDeps(kind, config, semantic, parts) {
|
|
7452
|
+
return {
|
|
7453
|
+
kind,
|
|
7454
|
+
config,
|
|
7455
|
+
semantic,
|
|
7456
|
+
legs: parts.legs,
|
|
7457
|
+
texts: parts.texts,
|
|
7458
|
+
hits: parts.hits,
|
|
7459
|
+
...parts.onReturn === void 0 ? {} : { onReturn: parts.onReturn }
|
|
7460
|
+
};
|
|
7461
|
+
}
|
|
7462
|
+
/**
|
|
7396
7463
|
* P-01: put the fused per-leg evidence onto the caller-facing hits.
|
|
7397
7464
|
*
|
|
7398
7465
|
* Matched by `ref_id`, not by array index: a store's `hits` mapper may legitimately drop an entry
|
|
@@ -7496,6 +7563,266 @@ function describe(error) {
|
|
|
7496
7563
|
return error instanceof Error ? error.message : String(error);
|
|
7497
7564
|
}
|
|
7498
7565
|
//#endregion
|
|
7566
|
+
//#region ../core/lib/store/legs.js
|
|
7567
|
+
/**
|
|
7568
|
+
* The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
|
|
7569
|
+
* leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
|
|
7570
|
+
* `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
|
|
7571
|
+
* scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
|
|
7572
|
+
* check's error and the Map insertion order live here once.
|
|
7573
|
+
*
|
|
7574
|
+
* `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
|
|
7575
|
+
* a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
|
|
7576
|
+
* `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
|
|
7577
|
+
* empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
|
|
7578
|
+
* paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
|
|
7579
|
+
* from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
|
|
7580
|
+
* probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
|
|
7581
|
+
* at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
|
|
7582
|
+
*
|
|
7583
|
+
* WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
|
|
7584
|
+
* signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
|
|
7585
|
+
* counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
|
|
7586
|
+
* quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
|
|
7587
|
+
* deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
|
|
7588
|
+
* floor functions only ever REMOVE entries, so the raw set is the honest measurement.
|
|
7589
|
+
*
|
|
7590
|
+
* WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
|
|
7591
|
+
* `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
|
|
7592
|
+
* {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
|
|
7593
|
+
* the absence is structural, not something a caller can forget to set.
|
|
7594
|
+
*
|
|
7595
|
+
* WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
|
|
7596
|
+
* table, and a template-method base class would be a structural rewrite of two hub files. This is
|
|
7597
|
+
* the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
|
|
7598
|
+
* be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
|
|
7599
|
+
* third store in `test/legs.spec.ts`).
|
|
7600
|
+
*
|
|
7601
|
+
* CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
|
|
7602
|
+
* (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
|
|
7603
|
+
* legs, so both stores get the same awaiting and isolation by construction.
|
|
7604
|
+
*
|
|
7605
|
+
* @module store/legs
|
|
7606
|
+
*/
|
|
7607
|
+
/**
|
|
7608
|
+
* The one `capped` derivation (raw-set size vs cap). Kept as a named helper so the three call sites
|
|
7609
|
+
* cannot drift into measuring the floored set.
|
|
7610
|
+
*/
|
|
7611
|
+
function wasCapped(raw, cap) {
|
|
7612
|
+
return raw.size === cap;
|
|
7613
|
+
}
|
|
7614
|
+
/**
|
|
7615
|
+
* Build the leg runner for one query.
|
|
7616
|
+
*
|
|
7617
|
+
* The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
|
|
7618
|
+
* index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
|
|
7619
|
+
* self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
|
|
7620
|
+
* evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
|
|
7621
|
+
* same wrapper, the same floor order and the same down-semantic fallback.
|
|
7622
|
+
*/
|
|
7623
|
+
function makeLegRunner(ctx) {
|
|
7624
|
+
const cap = ctx.legCap;
|
|
7625
|
+
return {
|
|
7626
|
+
cap,
|
|
7627
|
+
plain(scores, weight) {
|
|
7628
|
+
return {
|
|
7629
|
+
weight,
|
|
7630
|
+
scores,
|
|
7631
|
+
capped: wasCapped(scores, cap)
|
|
7632
|
+
};
|
|
7633
|
+
},
|
|
7634
|
+
scored(name, raw, weight, floor) {
|
|
7635
|
+
const floored = applyScoreFloor(raw, floor);
|
|
7636
|
+
return {
|
|
7637
|
+
weight,
|
|
7638
|
+
scores: floored.scores,
|
|
7639
|
+
capped: wasCapped(raw, cap),
|
|
7640
|
+
leg: name,
|
|
7641
|
+
droppedByFloor: floored.dropped
|
|
7642
|
+
};
|
|
7643
|
+
},
|
|
7644
|
+
term(name, raw, texts, query, weight, floor) {
|
|
7645
|
+
const floored = applyTermFloor(raw, texts, query, floor);
|
|
7646
|
+
return {
|
|
7647
|
+
weight,
|
|
7648
|
+
scores: floored.scores,
|
|
7649
|
+
capped: wasCapped(raw, cap),
|
|
7650
|
+
leg: name,
|
|
7651
|
+
droppedByFloor: floored.dropped
|
|
7652
|
+
};
|
|
7653
|
+
},
|
|
7654
|
+
semantic(raw, weight, floor) {
|
|
7655
|
+
const floored = applyScoreFloor(raw, floor);
|
|
7656
|
+
return {
|
|
7657
|
+
weight,
|
|
7658
|
+
scores: floored.scores,
|
|
7659
|
+
leg: "semantic",
|
|
7660
|
+
droppedByFloor: floored.dropped
|
|
7661
|
+
};
|
|
7662
|
+
},
|
|
7663
|
+
semanticOff(weight) {
|
|
7664
|
+
return {
|
|
7665
|
+
weight,
|
|
7666
|
+
scores: /* @__PURE__ */ new Map(),
|
|
7667
|
+
leg: "semantic",
|
|
7668
|
+
droppedByFloor: 0
|
|
7669
|
+
};
|
|
7670
|
+
}
|
|
7671
|
+
};
|
|
7672
|
+
}
|
|
7673
|
+
/**
|
|
7674
|
+
* The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
|
|
7675
|
+
*
|
|
7676
|
+
* The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
|
|
7677
|
+
* bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
|
|
7678
|
+
* order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
|
|
7679
|
+
*
|
|
7680
|
+
* - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
|
|
7681
|
+
* stays better everywhere downstream;
|
|
7682
|
+
* - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
|
|
7683
|
+
* on the same "higher = better" scale;
|
|
7684
|
+
* - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
|
|
7685
|
+
* 3-char run) — the leg is empty by construction, not by a query that matched nothing.
|
|
7686
|
+
*
|
|
7687
|
+
* THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
|
|
7688
|
+
* trigram index can express (measured: every term in the FTS table is 3 characters), which used to
|
|
7689
|
+
* leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
|
|
7690
|
+
* predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
|
|
7691
|
+
* is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
|
|
7692
|
+
*/
|
|
7693
|
+
function ftsLeg(opts) {
|
|
7694
|
+
const ftsQuery = buildFtsQuery(opts.query, opts.tokenizer);
|
|
7695
|
+
if (ftsQuery) {
|
|
7696
|
+
const rows = opts.search(ftsQuery, opts.scope, opts.cap);
|
|
7697
|
+
return new Map(rows.map((r) => [r.id, -r.rank]));
|
|
7698
|
+
}
|
|
7699
|
+
const terms = substringTerms(opts.query);
|
|
7700
|
+
if (terms.length === 0) return /* @__PURE__ */ new Map();
|
|
7701
|
+
const rows = opts.substringSearch(terms, opts.scope, opts.cap);
|
|
7702
|
+
return new Map(rows.map((r) => [r.id, r.rank]));
|
|
7703
|
+
}
|
|
7704
|
+
/**
|
|
7705
|
+
* The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
|
|
7706
|
+
*
|
|
7707
|
+
* The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
|
|
7708
|
+
* `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
|
|
7709
|
+
* to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
|
|
7710
|
+
*
|
|
7711
|
+
* Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
|
|
7712
|
+
* rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
|
|
7713
|
+
* empty-pool early return, and the returned map's insertion order (the vstore's pool order).
|
|
7714
|
+
*/
|
|
7715
|
+
async function semanticLeg(opts) {
|
|
7716
|
+
const vec = opts.queryVector ?? await opts.encode(opts.query);
|
|
7717
|
+
if (vec.length !== opts.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${opts.dim}`);
|
|
7718
|
+
opts.onVector?.(vec);
|
|
7719
|
+
const topk = opts.topk(vec, Math.max(50, opts.k));
|
|
7720
|
+
if (!topk.length) return /* @__PURE__ */ new Map();
|
|
7721
|
+
const allowed = opts.filterTopk(topk.map((t) => t.id));
|
|
7722
|
+
const out = /* @__PURE__ */ new Map();
|
|
7723
|
+
for (const t of topk) if (allowed === void 0 || allowed.has(t.id)) out.set(t.id, t.score);
|
|
7724
|
+
return out;
|
|
7725
|
+
}
|
|
7726
|
+
//#endregion
|
|
7727
|
+
//#region ../core/lib/store/entity_sweep.js
|
|
7728
|
+
/**
|
|
7729
|
+
* The ONE entity-version sweep both stores run.
|
|
7730
|
+
*
|
|
7731
|
+
* `entities_version` is the stamp that says "these entity rows were produced by the CURRENT
|
|
7732
|
+
* extraction rules". A rule change makes every row carrying an older stamp stale, and the only way
|
|
7733
|
+
* to close that gap is to re-extract those rows' text — for memory the entity links, the triples and
|
|
7734
|
+
* the HRR bundle derived from them; for knowledge the `chunk_entities` rows. That SKELETON was
|
|
7735
|
+
* written twice and drifted apart in exactly the ways a copy drifts: memory had the in-flight guard,
|
|
7736
|
+
* the budget clamp and the `{rebuilt, deferred, skipped}` report, while knowledge had the same
|
|
7737
|
+
* selection inlined in `reindex` with no guard at all. This module is the one copy; the stores
|
|
7738
|
+
* contribute a per-aggregate adapter (which rows, how to extract, what to write) and nothing else.
|
|
7739
|
+
*
|
|
7740
|
+
* Layering, deliberate: this is an ORCHESTRATION rule, not SQL, so it lives beside `common.ts` and
|
|
7741
|
+
* `legs.ts` in `store/` — the DAO layer stays split by aggregate and keeps its own statements
|
|
7742
|
+
* (`docs/vector-repair-shared-flow.md` §2 / §7).
|
|
7743
|
+
*
|
|
7744
|
+
* What must NOT move here: the extractor, the rows a store writes, and that store's side effects.
|
|
7745
|
+
* Memory's transaction also replaces triples, moves the HRR bundle and re-queues the conflict check
|
|
7746
|
+
* (both conflict legs read what the sweep rebuilt); knowledge replaces `chunk_entities` under
|
|
7747
|
+
* `MAX_ENTITIES_PER_CHUNK`. Those are domain semantics, and folding them in would trade one
|
|
7748
|
+
* duplicated loop for two conditional branches.
|
|
7749
|
+
*
|
|
7750
|
+
* @module store/entity_sweep
|
|
7751
|
+
*/
|
|
7752
|
+
/**
|
|
7753
|
+
* Rows one sweep pass may rebuild when the caller names no budget.
|
|
7754
|
+
*
|
|
7755
|
+
* `MemoryStore.reindexEntities` has always defaulted to this, and the CLI passes it explicitly; it
|
|
7756
|
+
* lives here so the shared loop and that public re-export cannot drift. Bounds MEMORY, not only
|
|
7757
|
+
* mission: a pass selects every chosen row's text before rebuilding the first one.
|
|
7758
|
+
*/
|
|
7759
|
+
const ENTITY_SWEEP_BATCH = 2e3;
|
|
7760
|
+
/**
|
|
7761
|
+
* The module-level in-flight guard.
|
|
7762
|
+
*
|
|
7763
|
+
* Held here, not in the stores, because it protects THIS loop: two callers can overlap (the plugin's
|
|
7764
|
+
* heartbeat and `maintenance` are both periodic), and a limited ORDERED selection makes two
|
|
7765
|
+
* concurrent passes pick the same rows and re-tag them twice. Overlap is not corruption — each row is
|
|
7766
|
+
* written in its own transaction — it is duplicated mission in a single-threaded process, and the
|
|
7767
|
+
* answer is one pass reporting `skipped`. A `WeakSet` keyed by the caller's guard object keeps the
|
|
7768
|
+
* state in this module without adding a mutable field to either hub store, and without a module-level
|
|
7769
|
+
* boolean that would make two stores' sweeps block each other.
|
|
7770
|
+
*
|
|
7771
|
+
* The `guard` argument is therefore an IDENTITY, not a token: a store must pass the SAME object to
|
|
7772
|
+
* every call for its whole lifetime (both stores hold one as a private field). Two different objects
|
|
7773
|
+
* are two different sweeps and run concurrently — that per-store granularity is the point.
|
|
7774
|
+
*/
|
|
7775
|
+
const SWEEPING = /* @__PURE__ */ new WeakSet();
|
|
7776
|
+
/**
|
|
7777
|
+
* Re-extract the entity rows written by older extraction rules, ONE bounded pass.
|
|
7778
|
+
*
|
|
7779
|
+
* The contract, in order: take the guard (an overlapping pass reports `skipped` and does nothing);
|
|
7780
|
+
* read the scope's stale count and at most `maxPerPass` rows; re-extract each row through the
|
|
7781
|
+
* adapter, isolating failures; report `{ rebuilt, deferred, skipped }`.
|
|
7782
|
+
*
|
|
7783
|
+
* Not an unbounded drain, on purpose: this is reachable from a settings-page button and a periodic
|
|
7784
|
+
* heartbeat, so it must not block on a whole stale corpus. Callers that want the corpus current loop
|
|
7785
|
+
* until `deferred === 0` (the CLI does).
|
|
7786
|
+
*
|
|
7787
|
+
* A row whose extraction or write throws is skipped, not fatal: the transaction that stamps the
|
|
7788
|
+
* version never ran, so the row is still stale and the NEXT pass retries it — the same retry contract
|
|
7789
|
+
* the knowledge store used per chunk. Letting the throw escape would instead abandon the rest of the
|
|
7790
|
+
* batch with no report of what was done.
|
|
7791
|
+
*
|
|
7792
|
+
* @param guard the store's stable in-flight identity (see {@link SWEEPING})
|
|
7793
|
+
*/
|
|
7794
|
+
async function sweepVersionedEntities(target, guard, opts = {}) {
|
|
7795
|
+
const maxPerPass = Math.max(0, Math.floor(opts.budget ?? 2e3));
|
|
7796
|
+
if (SWEEPING.has(guard)) return {
|
|
7797
|
+
rebuilt: 0,
|
|
7798
|
+
deferred: target.staleEntityCount?.() ?? target.staleEntityBatch(0).total,
|
|
7799
|
+
skipped: true
|
|
7800
|
+
};
|
|
7801
|
+
SWEEPING.add(guard);
|
|
7802
|
+
try {
|
|
7803
|
+
const { total, rows } = target.staleEntityBatch(maxPerPass);
|
|
7804
|
+
let rebuilt = 0;
|
|
7805
|
+
for (const row of rows) {
|
|
7806
|
+
if (opts.shouldStop?.() === true) break;
|
|
7807
|
+
if (rebuilt >= maxPerPass) break;
|
|
7808
|
+
try {
|
|
7809
|
+
await target.extractAndWriteEntities(row);
|
|
7810
|
+
} catch (error) {
|
|
7811
|
+
target.onRowError(row, error);
|
|
7812
|
+
continue;
|
|
7813
|
+
}
|
|
7814
|
+
rebuilt += 1;
|
|
7815
|
+
}
|
|
7816
|
+
return {
|
|
7817
|
+
rebuilt,
|
|
7818
|
+
deferred: Math.max(0, total - rebuilt),
|
|
7819
|
+
skipped: false
|
|
7820
|
+
};
|
|
7821
|
+
} finally {
|
|
7822
|
+
SWEEPING.delete(guard);
|
|
7823
|
+
}
|
|
7824
|
+
}
|
|
7825
|
+
//#endregion
|
|
7499
7826
|
//#region ../core/lib/db/vectors.js
|
|
7500
7827
|
/** Little-endian Float32 ↔ BLOB helpers for persisted vectors (cross-platform stable). */
|
|
7501
7828
|
function float32ToBytes(vec) {
|
|
@@ -7687,6 +8014,280 @@ async function forEachYielding(items, visit, every = 200) {
|
|
|
7687
8014
|
if ((index + 1) % every === 0) await yieldToEventLoop();
|
|
7688
8015
|
}
|
|
7689
8016
|
}
|
|
8017
|
+
/**
|
|
8018
|
+
* Rebuild one store's live vstore from its persisted rows, count what is unusable in the current
|
|
8019
|
+
* space, and say so once — the shared "open for business" step both stores used to write twice.
|
|
8020
|
+
*
|
|
8021
|
+
* The owning store's ONLY half is the row read (`facts.activeVectorRows()` /
|
|
8022
|
+
* `chunks.vectorRows()`); the reload, the count and the loud warning are one implementation, so a
|
|
8023
|
+
* change to what counts as stale cannot land on one store only (the incident this exists for: the
|
|
8024
|
+
* memory store checked the SPACE while knowledge checked only the WIDTH, so a model swap mixed two
|
|
8025
|
+
* spaces in one index until someone happened to run `kb_reindex`).
|
|
8026
|
+
*/
|
|
8027
|
+
function loadVectorIndex(opts) {
|
|
8028
|
+
reloadVectorIndex(opts.vstore, opts.rows, opts.store);
|
|
8029
|
+
const expectedBytes = opts.vstore.dim * 4;
|
|
8030
|
+
let stale = 0;
|
|
8031
|
+
let spaceStale = 0;
|
|
8032
|
+
let legacy = 0;
|
|
8033
|
+
for (const row of opts.rows) {
|
|
8034
|
+
if (row.vec === null || row.vec.byteLength !== expectedBytes) {
|
|
8035
|
+
stale++;
|
|
8036
|
+
continue;
|
|
8037
|
+
}
|
|
8038
|
+
if (row.embedding_model !== opts.space) {
|
|
8039
|
+
spaceStale++;
|
|
8040
|
+
if (isPreFingerprintSpace(row.embedding_model)) legacy++;
|
|
8041
|
+
}
|
|
8042
|
+
}
|
|
8043
|
+
reportStaleVectors(opts.store, opts.space, {
|
|
8044
|
+
stale,
|
|
8045
|
+
space_stale: spaceStale,
|
|
8046
|
+
legacy
|
|
8047
|
+
}, opts.manualEntry);
|
|
8048
|
+
return {
|
|
8049
|
+
stale,
|
|
8050
|
+
space_stale: spaceStale,
|
|
8051
|
+
legacy
|
|
8052
|
+
};
|
|
8053
|
+
}
|
|
8054
|
+
/**
|
|
8055
|
+
* Encode rows one at a time, add each to the live index, and persist them in groups.
|
|
8056
|
+
*
|
|
8057
|
+
* The shared skeleton: serial encode (a MEASURED decision for chunks — a padded batched forward pass
|
|
8058
|
+
* is 1.79x slower on real 53–550 character chunks), `add` right after a successful encode, one
|
|
8059
|
+
* `write` call per `writeBatch` rows, per-row isolation of encode failures, per-group isolation of
|
|
8060
|
+
* write failures, and the `{ encoded, failed }` count.
|
|
8061
|
+
*
|
|
8062
|
+
* What stays with the store: the backend call, the live index, the write statement, the replay of a
|
|
8063
|
+
* failed group, and the summary log — the memory store logs per row and keeps a half-written index
|
|
8064
|
+
* (documented as the benign direction for it), the knowledge store evicts the group so both sides
|
|
8065
|
+
* agree that the vectors are missing.
|
|
8066
|
+
*/
|
|
8067
|
+
async function encodeAndPersist(rows, target) {
|
|
8068
|
+
const every = Math.max(1, Math.floor(target.writeBatch ?? 1));
|
|
8069
|
+
let encoded = 0;
|
|
8070
|
+
let failed = 0;
|
|
8071
|
+
for (let i = 0; i < rows.length; i += every) {
|
|
8072
|
+
const group = rows.slice(i, i + every);
|
|
8073
|
+
const writes = [];
|
|
8074
|
+
for (const row of group) try {
|
|
8075
|
+
const vec = await target.encode(row.text);
|
|
8076
|
+
target.add(row.id, vec);
|
|
8077
|
+
writes.push({
|
|
8078
|
+
id: row.id,
|
|
8079
|
+
bytes: float32ToBytes(vec)
|
|
8080
|
+
});
|
|
8081
|
+
} catch (error) {
|
|
8082
|
+
failed += 1;
|
|
8083
|
+
target.onRowError?.(row, error);
|
|
8084
|
+
}
|
|
8085
|
+
if (!writes.length) continue;
|
|
8086
|
+
try {
|
|
8087
|
+
target.write(writes);
|
|
8088
|
+
encoded += writes.length;
|
|
8089
|
+
} catch (error) {
|
|
8090
|
+
failed += writes.length;
|
|
8091
|
+
target.onWriteError?.(writes, error);
|
|
8092
|
+
}
|
|
8093
|
+
}
|
|
8094
|
+
return {
|
|
8095
|
+
encoded,
|
|
8096
|
+
failed
|
|
8097
|
+
};
|
|
8098
|
+
}
|
|
8099
|
+
/**
|
|
8100
|
+
* ONE bounded slice of the repair: re-add unusable-but-readable vectors that the live index lost,
|
|
8101
|
+
* drop the rows a re-encode has to replace, then encode up to `limit` missing/foreign rows.
|
|
8102
|
+
*
|
|
8103
|
+
* `limit` is what makes the SAME logic usable as a background migration: one call is a bounded unit
|
|
8104
|
+
* of work the caller can yield between. The explicit fix passes no limit (the operator action drains
|
|
8105
|
+
* everything); the background drive passes its batch size.
|
|
8106
|
+
*
|
|
8107
|
+
* The ORDER is an invariant: unusable bytes leave the table and the live index first, so a
|
|
8108
|
+
* re-encode can pick them up and a process that dies mid-migration leaves at most one batch
|
|
8109
|
+
* vector-less — exactly the `missing` rows the next run resumes on.
|
|
8110
|
+
*/
|
|
8111
|
+
async function repairVectorSlice(target, opts) {
|
|
8112
|
+
const limit = opts.limit ?? Number.POSITIVE_INFINITY;
|
|
8113
|
+
const c = target.classifyVectors(limit);
|
|
8114
|
+
const loadedNow = target.semanticAvailable();
|
|
8115
|
+
let semAvailable = loadedNow;
|
|
8116
|
+
if (!semAvailable && !opts.dryRun && target.warmup !== void 0) semAvailable = await target.warmup();
|
|
8117
|
+
const wouldWarm = !loadedNow;
|
|
8118
|
+
const remainingOf = (encoded) => Math.max(0, c.stale + c.space_stale + c.missing - encoded);
|
|
8119
|
+
const base = {
|
|
8120
|
+
stale: c.stale,
|
|
8121
|
+
space_stale: c.space_stale,
|
|
8122
|
+
missing: c.missing,
|
|
8123
|
+
semantic_available: semAvailable,
|
|
8124
|
+
would_warm: wouldWarm
|
|
8125
|
+
};
|
|
8126
|
+
if (opts.dryRun) return {
|
|
8127
|
+
report: {
|
|
8128
|
+
...base,
|
|
8129
|
+
...c.unindexed === void 0 ? {} : {
|
|
8130
|
+
unindexed: c.unindexed.length,
|
|
8131
|
+
reindexed: 0
|
|
8132
|
+
},
|
|
8133
|
+
dropped: 0,
|
|
8134
|
+
encoded: 0,
|
|
8135
|
+
failed: 0
|
|
8136
|
+
},
|
|
8137
|
+
remaining: remainingOf(0)
|
|
8138
|
+
};
|
|
8139
|
+
let reindexed = 0;
|
|
8140
|
+
if (c.unindexed !== void 0) reindexed = target.reindexUsable?.(c.unindexed.slice(0, limit)) ?? 0;
|
|
8141
|
+
let dropped = 0;
|
|
8142
|
+
if (semAvailable && c.dropIds.length > 0) dropped = target.dropVectors(c.dropIds);
|
|
8143
|
+
const { encoded, failed } = semAvailable ? await target.reencodeVectors(limit) : {
|
|
8144
|
+
encoded: 0,
|
|
8145
|
+
failed: 0
|
|
8146
|
+
};
|
|
8147
|
+
return {
|
|
8148
|
+
report: {
|
|
8149
|
+
...base,
|
|
8150
|
+
...c.unindexed === void 0 ? {} : {
|
|
8151
|
+
unindexed: c.unindexed.length,
|
|
8152
|
+
reindexed
|
|
8153
|
+
},
|
|
8154
|
+
dropped,
|
|
8155
|
+
encoded,
|
|
8156
|
+
failed
|
|
8157
|
+
},
|
|
8158
|
+
remaining: remainingOf(encoded)
|
|
8159
|
+
};
|
|
8160
|
+
}
|
|
8161
|
+
/**
|
|
8162
|
+
* ONE bounded slice of the background migration — the unit the drive (and the tests) step.
|
|
8163
|
+
*
|
|
8164
|
+
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
8165
|
+
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
8166
|
+
*/
|
|
8167
|
+
async function migrateVectorSlice(target, batchSize = 16) {
|
|
8168
|
+
const { report, remaining } = await repairVectorSlice(target, {
|
|
8169
|
+
dryRun: false,
|
|
8170
|
+
limit: Math.max(1, Math.floor(batchSize))
|
|
8171
|
+
});
|
|
8172
|
+
return {
|
|
8173
|
+
remaining,
|
|
8174
|
+
migrated: report.encoded,
|
|
8175
|
+
dropped: report.dropped,
|
|
8176
|
+
reindexed: report.reindexed ?? 0,
|
|
8177
|
+
semantic_available: report.semantic_available
|
|
8178
|
+
};
|
|
8179
|
+
}
|
|
8180
|
+
/** ACTIVE rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
8181
|
+
function migrationRemaining(target) {
|
|
8182
|
+
const health = target.vectorSpaceHealth();
|
|
8183
|
+
return health.stale + health.space_stale + target.missingVectorCount();
|
|
8184
|
+
}
|
|
8185
|
+
/** The `auto_migrate: false` warning is one per process per store, like the startup warning. */
|
|
8186
|
+
const disabledWarned = /* @__PURE__ */ new WeakSet();
|
|
8187
|
+
/**
|
|
8188
|
+
* Drive the bounded background migration for one store until it is current, the model is
|
|
8189
|
+
* unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE
|
|
8190
|
+
* never rejects: it is logged and reported as `remaining > 0`, which is what makes the next
|
|
8191
|
+
* heartbeat a retry. Never blocks the event loop for longer than one batch.
|
|
8192
|
+
*
|
|
8193
|
+
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
8194
|
+
* on background re-encoding can leave the loud warning and the manual entry as the only path. The
|
|
8195
|
+
* switch is read through the adapter so a new store cannot forget it.
|
|
8196
|
+
*/
|
|
8197
|
+
async function driveVectorRepair(target, opts = {}) {
|
|
8198
|
+
const log = opts.log ?? retrievalLogger();
|
|
8199
|
+
const stopped = () => opts.shouldStop?.() === true;
|
|
8200
|
+
const enabled = target.autoMigrate() !== false;
|
|
8201
|
+
const snapshot = () => ({
|
|
8202
|
+
enabled,
|
|
8203
|
+
remaining: migrationRemaining(target),
|
|
8204
|
+
migrated: 0,
|
|
8205
|
+
dropped: 0,
|
|
8206
|
+
reindexed: 0,
|
|
8207
|
+
semantic_available: target.semanticAvailable()
|
|
8208
|
+
});
|
|
8209
|
+
const initial = snapshot();
|
|
8210
|
+
if (stopped()) return initial;
|
|
8211
|
+
const health = target.vectorSpaceHealth();
|
|
8212
|
+
const isSpaceChange = health.stale + health.space_stale > 0;
|
|
8213
|
+
if (!enabled) {
|
|
8214
|
+
if (initial.remaining > 0 && !disabledWarned.has(target)) {
|
|
8215
|
+
disabledWarned.add(target);
|
|
8216
|
+
log.warn(`${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) ${isSpaceChange ? "belong to an older embedding space" : "have no vector at all"} and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run ${target.manualEntry} or set \`semantic.auto_migrate: true\``);
|
|
8217
|
+
}
|
|
8218
|
+
return initial;
|
|
8219
|
+
}
|
|
8220
|
+
if (initial.remaining === 0) return initial;
|
|
8221
|
+
const startedAt = Date.now();
|
|
8222
|
+
log.info(isSpaceChange ? `${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)` : `${target.store} vector migration: ${String(initial.remaining)} ACTIVE row(s) have no vector at all — encoding them in the background`);
|
|
8223
|
+
let migrated = 0;
|
|
8224
|
+
let dropped = 0;
|
|
8225
|
+
let reindexed = 0;
|
|
8226
|
+
let last = initial;
|
|
8227
|
+
for (;;) {
|
|
8228
|
+
if (stopped()) break;
|
|
8229
|
+
let step;
|
|
8230
|
+
try {
|
|
8231
|
+
step = await migrateVectorSlice(target, opts.batchSize);
|
|
8232
|
+
} catch (error) {
|
|
8233
|
+
log.warn(`${target.store} vector migration: batch failed (${describeError$2(error)}) — ${String(migrationRemaining(target))} vector(s) still cannot be used by the semantic leg; the next pass retries`);
|
|
8234
|
+
break;
|
|
8235
|
+
}
|
|
8236
|
+
migrated += step.migrated;
|
|
8237
|
+
dropped += step.dropped;
|
|
8238
|
+
reindexed += step.reindexed;
|
|
8239
|
+
last = {
|
|
8240
|
+
...step,
|
|
8241
|
+
enabled,
|
|
8242
|
+
migrated,
|
|
8243
|
+
dropped,
|
|
8244
|
+
reindexed
|
|
8245
|
+
};
|
|
8246
|
+
opts.onProgress?.(last);
|
|
8247
|
+
log.info(`${target.store} vector migration: re-encoded ${String(migrated)}/${String(initial.remaining)} (${String(step.remaining)} left)`);
|
|
8248
|
+
if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
|
|
8249
|
+
if (last.remaining === 0) break;
|
|
8250
|
+
await yieldToEventLoop();
|
|
8251
|
+
}
|
|
8252
|
+
if (last.remaining === 0) log.info(`${target.store} vector migration: complete — ${String(migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
|
|
8253
|
+
else if (!stopped()) log.warn(`${target.store} vector migration: ${String(last.remaining)} vector(s) still cannot be used by the semantic leg (the embedding model may be unavailable) — the next heartbeat retries; manual entry ${target.manualEntry}`);
|
|
8254
|
+
return last;
|
|
8255
|
+
}
|
|
8256
|
+
/**
|
|
8257
|
+
* Drive every store in order. A stop request keeps the drive from starting another store, but each
|
|
8258
|
+
* store still answers with its snapshot (so the caller can report what is left everywhere).
|
|
8259
|
+
*/
|
|
8260
|
+
async function driveVectorRepairs(targets, opts = {}) {
|
|
8261
|
+
const outcomes = [];
|
|
8262
|
+
for (const target of targets) outcomes.push(await driveVectorRepair(target, opts));
|
|
8263
|
+
return outcomes;
|
|
8264
|
+
}
|
|
8265
|
+
/**
|
|
8266
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`,
|
|
8267
|
+
* covering one or both stores.
|
|
8268
|
+
*
|
|
8269
|
+
* `store` selects a single store (omitted = both); `dry_run` previews every count without writing —
|
|
8270
|
+
* and without loading the model (so a preview also reports `would_warm`). The top-level
|
|
8271
|
+
* `semantic_available` is AND over the selected stores: `false` means at least one store's repair
|
|
8272
|
+
* could not encode anything.
|
|
8273
|
+
*/
|
|
8274
|
+
async function vectorsFix(targets, opts) {
|
|
8275
|
+
const stores = {};
|
|
8276
|
+
let semanticAvailable = true;
|
|
8277
|
+
for (const kind of ["memory", "knowledge"]) {
|
|
8278
|
+
if (opts.store !== void 0 && opts.store !== kind) continue;
|
|
8279
|
+
const target = targets[kind];
|
|
8280
|
+
if (target === void 0) continue;
|
|
8281
|
+
const { report } = await repairVectorSlice(target, { dryRun: opts.dryRun });
|
|
8282
|
+
stores[kind] = report;
|
|
8283
|
+
semanticAvailable = semanticAvailable && report.semantic_available;
|
|
8284
|
+
}
|
|
8285
|
+
return {
|
|
8286
|
+
stores,
|
|
8287
|
+
semantic_available: semanticAvailable,
|
|
8288
|
+
dry_run: opts.dryRun
|
|
8289
|
+
};
|
|
8290
|
+
}
|
|
7690
8291
|
//#endregion
|
|
7691
8292
|
//#region ../core/lib/store/secret_guard.js
|
|
7692
8293
|
/**
|
|
@@ -8460,6 +9061,124 @@ function inList(values) {
|
|
|
8460
9061
|
};
|
|
8461
9062
|
}
|
|
8462
9063
|
//#endregion
|
|
9064
|
+
//#region ../core/lib/db/dao/shared.js
|
|
9065
|
+
/**
|
|
9066
|
+
* The entity-candidate leg's ORDER contract — the tail every candidate statement ends with.
|
|
9067
|
+
*
|
|
9068
|
+
* The formula is the leg's own score (anchored Jaccard with a saturating union,
|
|
9069
|
+
* `store/entity_leg.ts`): `shared / (queryWidth + min(total - shared, widthCap))`. Ordering by it is
|
|
9070
|
+
* load-bearing, not cosmetic: `fuse` scales each leg by its maximum, and a survivor's scaled value
|
|
9071
|
+
* is unchanged only when the ordering — and therefore the maximum — survives the trim. A narrower
|
|
9072
|
+
* row sharing one entity can therefore outrank a wide one sharing two, and the cap keeps it.
|
|
9073
|
+
* `total ASC` is only a deterministic tie-break among equal ratios, and the id makes the order total.
|
|
9074
|
+
*
|
|
9075
|
+
* `idColumn` is the aggregate's own id (`fa.fact_id` / `ce.chunk_id`); that, the `total` subquery's
|
|
9076
|
+
* table and the filter predicates are the ONLY differences between the two statements.
|
|
9077
|
+
*/
|
|
9078
|
+
function entityCandidateTail(idColumn) {
|
|
9079
|
+
return `GROUP BY ${idColumn}
|
|
9080
|
+
ORDER BY (CAST(shared AS REAL) / (? + MIN(total - shared, ?))) DESC, total ASC, ${idColumn} ASC
|
|
9081
|
+
LIMIT ?`;
|
|
9082
|
+
}
|
|
9083
|
+
/**
|
|
9084
|
+
* Run an entity-candidate query: bind order, batching and union are shared; the SQL is the caller's.
|
|
9085
|
+
*
|
|
9086
|
+
* Bind order (identical in both aggregates): `<names …>, <filterParams …>, queryWidth, widthCap, limit`.
|
|
9087
|
+
*/
|
|
9088
|
+
function entityCandidates(db, q) {
|
|
9089
|
+
if (!q.names.length || q.limit <= 0) return [];
|
|
9090
|
+
const out = [];
|
|
9091
|
+
const seen = q.batchNames ? /* @__PURE__ */ new Set() : null;
|
|
9092
|
+
for (const batch of q.batchNames ? batches(q.names) : [q.names]) {
|
|
9093
|
+
const { placeholders, values } = inList(batch);
|
|
9094
|
+
const rows = db.prepare(q.sql(placeholders)).all(...values, ...q.filterParams, q.queryWidth, q.widthCap, q.limit);
|
|
9095
|
+
for (const row of rows) {
|
|
9096
|
+
if (seen !== null) {
|
|
9097
|
+
if (seen.has(row.id)) continue;
|
|
9098
|
+
seen.add(row.id);
|
|
9099
|
+
}
|
|
9100
|
+
out.push(row.id);
|
|
9101
|
+
}
|
|
9102
|
+
}
|
|
9103
|
+
return out;
|
|
9104
|
+
}
|
|
9105
|
+
/**
|
|
9106
|
+
* The batched `UPDATE … SET entities_version = ?` shared by `FactsDao` and `ChunksDao`.
|
|
9107
|
+
*
|
|
9108
|
+
* Only the aggregate's `table` / `idColumn` differ. Returns the summed `changes`; the knowledge
|
|
9109
|
+
* caller ignores it (its method's signature returns `void`), the memory caller's `number` is
|
|
9110
|
+
* unchanged.
|
|
9111
|
+
*/
|
|
9112
|
+
function setEntitiesVersionBatch(db, table, idColumn, ids, version) {
|
|
9113
|
+
let changes = 0;
|
|
9114
|
+
for (const batch of batches(ids)) {
|
|
9115
|
+
const { placeholders, values } = inList(batch);
|
|
9116
|
+
changes += db.prepare(`UPDATE ${table} SET entities_version = ? WHERE ${idColumn} IN (${placeholders})`).run(version, ...values).changes;
|
|
9117
|
+
}
|
|
9118
|
+
return changes;
|
|
9119
|
+
}
|
|
9120
|
+
/**
|
|
9121
|
+
* Document frequency — "how many rows carry each name".
|
|
9122
|
+
*
|
|
9123
|
+
* Shared by `EntitiesDao.activeDocFrequency` and `ChunksDao.docFrequency`; the only difference is
|
|
9124
|
+
* the population the caller's `where`/`from` describe (memory counts ACTIVE facts, knowledge has no
|
|
9125
|
+
* status column and counts chunks). Only the caller's QUERY names are looked up, so this is one
|
|
9126
|
+
* indexed COUNT per name rather than a corpus scan. A name absent from the map has frequency 0.
|
|
9127
|
+
*/
|
|
9128
|
+
function queryDocFrequency(db, names, q) {
|
|
9129
|
+
const out = /* @__PURE__ */ new Map();
|
|
9130
|
+
if (!names.length) return out;
|
|
9131
|
+
const { placeholders, values } = inList(names);
|
|
9132
|
+
const rows = db.prepare(`SELECT ${q.nameExpr} AS name, COUNT(*) AS df${q.from}
|
|
9133
|
+
WHERE ${q.nameExpr} IN (${placeholders})${q.where === void 0 ? "" : ` AND ${q.where}`}
|
|
9134
|
+
GROUP BY ${q.nameExpr}`).all(...values);
|
|
9135
|
+
for (const row of rows) out.set(row.name, Number(row.df));
|
|
9136
|
+
return out;
|
|
9137
|
+
}
|
|
9138
|
+
/**
|
|
9139
|
+
* The SHORT-QUERY LIKE fallback shared by `FactsDao.ftsSubstringSearch` and
|
|
9140
|
+
* `ChunksDao.ftsSubstringSearch`.
|
|
9141
|
+
*
|
|
9142
|
+
* The predicate text itself already comes from `db/tokenizer.ts#likeSubstring`; this owns the
|
|
9143
|
+
* protocol AROUND it, which both copies spelled out:
|
|
9144
|
+
*
|
|
9145
|
+
* 1. build the fragments — empty terms yield no patterns, which means "match nothing" ⇒ `[]`;
|
|
9146
|
+
* 2. `count` lands in the SELECT and `any` in the WHERE, so the SAME pattern list is bound TWICE:
|
|
9147
|
+
* once with the SELECT's placeholders, then — after the caller's filter params — with the
|
|
9148
|
+
* WHERE's;
|
|
9149
|
+
* 3. `LIMIT ?` last, defaulting to `-1` (unbounded; the store bounds the leg).
|
|
9150
|
+
*
|
|
9151
|
+
* `ORDER BY rank DESC, <id> ASC` stays in the caller's SQL: the id column is aggregate-specific and
|
|
9152
|
+
* `rank` must read the SELECT alias, so the fragments and the patterns are built once and used twice.
|
|
9153
|
+
*/
|
|
9154
|
+
function likeSubstringLeg(db, column, terms, buildSql, filterParams = [], limit) {
|
|
9155
|
+
const fragments = likeSubstring(column, terms);
|
|
9156
|
+
if (fragments.params.length === 0) return [];
|
|
9157
|
+
return db.prepare(buildSql(fragments)).all(...fragments.params, ...filterParams, ...fragments.params, limit ?? -1);
|
|
9158
|
+
}
|
|
9159
|
+
/**
|
|
9160
|
+
* Batched entity-name bags for a row-id set: `Map<rowId, names>`.
|
|
9161
|
+
*
|
|
9162
|
+
* Shared by `EntitiesDao.bagsForFacts` and `ChunksDao.entityBags` (§4.6.3): one `IN (…)` statement
|
|
9163
|
+
* per `batches()` slice, rows folded into the map in first-seen order. The SQL SHAPE is the only
|
|
9164
|
+
* aggregate-specific part (`table` / `keyColumn` / `nameColumn` / `joinEntities`); the crucial
|
|
9165
|
+
* `seedEmpty` difference above is the caller's to declare and stays observable.
|
|
9166
|
+
*/
|
|
9167
|
+
function entityBags(db, ids, q) {
|
|
9168
|
+
const out = /* @__PURE__ */ new Map();
|
|
9169
|
+
if (q.seedEmpty) for (const id of ids) out.set(id, []);
|
|
9170
|
+
for (const batch of batches(ids)) {
|
|
9171
|
+
const { placeholders, values } = inList(batch);
|
|
9172
|
+
const rows = db.prepare(`SELECT ${q.keyColumn} AS id, ${q.nameColumn} AS name FROM ${q.table}${q.joinEntities === void 0 ? "" : ` ${q.joinEntities}`} WHERE ${q.keyColumn} IN (${placeholders})`).all(...values);
|
|
9173
|
+
for (const row of rows) {
|
|
9174
|
+
const list = out.get(row.id);
|
|
9175
|
+
if (list) list.push(row.name);
|
|
9176
|
+
else out.set(row.id, [row.name]);
|
|
9177
|
+
}
|
|
9178
|
+
}
|
|
9179
|
+
return out;
|
|
9180
|
+
}
|
|
9181
|
+
//#endregion
|
|
8463
9182
|
//#region ../core/lib/db/dao/entities.js
|
|
8464
9183
|
var EntitiesDao = class {
|
|
8465
9184
|
db;
|
|
@@ -8506,17 +9225,13 @@ var EntitiesDao = class {
|
|
|
8506
9225
|
}
|
|
8507
9226
|
/** Batched entity bags for a candidate set: `Map<factId, names>` (ids with none get `[]`). */
|
|
8508
9227
|
bagsForFacts(ids) {
|
|
8509
|
-
|
|
8510
|
-
|
|
8511
|
-
|
|
8512
|
-
|
|
8513
|
-
|
|
8514
|
-
|
|
8515
|
-
|
|
8516
|
-
WHERE fe.fact_id IN (${placeholders})`).all(...values);
|
|
8517
|
-
for (const row of rows) out.get(row.fact_id)?.push(row.name);
|
|
8518
|
-
}
|
|
8519
|
-
return out;
|
|
9228
|
+
return entityBags(this.db, ids, {
|
|
9229
|
+
table: "fact_entities fe",
|
|
9230
|
+
keyColumn: "fe.fact_id",
|
|
9231
|
+
nameColumn: "e.name",
|
|
9232
|
+
joinEntities: "JOIN entities e ON e.entity_id = fe.entity_id",
|
|
9233
|
+
seedEmpty: true
|
|
9234
|
+
});
|
|
8520
9235
|
}
|
|
8521
9236
|
/** How many entities a fact is linked to — the scorer's `|A|`. */
|
|
8522
9237
|
countForFact(factId) {
|
|
@@ -8533,17 +9248,14 @@ var EntitiesDao = class {
|
|
|
8533
9248
|
* map has frequency 0, i.e. no active fact carries it and it can never be shared.
|
|
8534
9249
|
*/
|
|
8535
9250
|
activeDocFrequency(names) {
|
|
8536
|
-
|
|
8537
|
-
|
|
8538
|
-
const { placeholders, values } = inList(names);
|
|
8539
|
-
const rows = this.db.prepare(`SELECT e.name AS name, COUNT(*) AS df
|
|
9251
|
+
return queryDocFrequency(this.db, names, {
|
|
9252
|
+
from: `
|
|
8540
9253
|
FROM entities e
|
|
8541
9254
|
JOIN fact_entities fe ON fe.entity_id = e.entity_id
|
|
8542
|
-
JOIN facts fa ON fa.fact_id = fe.fact_id
|
|
8543
|
-
|
|
8544
|
-
|
|
8545
|
-
|
|
8546
|
-
return out;
|
|
9255
|
+
JOIN facts fa ON fa.fact_id = fe.fact_id`,
|
|
9256
|
+
nameExpr: "e.name",
|
|
9257
|
+
where: "fa.status = 'active'"
|
|
9258
|
+
});
|
|
8547
9259
|
}
|
|
8548
9260
|
/**
|
|
8549
9261
|
* Candidate facts for the embedding leg, narrowed by the scorer's NECESSARY conditions
|
|
@@ -8602,9 +9314,19 @@ var EntitiesDao = class {
|
|
|
8602
9314
|
* survivors, so one formula stays in charge.
|
|
8603
9315
|
*/
|
|
8604
9316
|
candidateFactsForAnyEntity(names, category, limit, queryWidth, widthCap, source) {
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
9317
|
+
return entityCandidates(this.db, {
|
|
9318
|
+
names,
|
|
9319
|
+
filterParams: [
|
|
9320
|
+
category ?? null,
|
|
9321
|
+
category ?? null,
|
|
9322
|
+
source ?? null,
|
|
9323
|
+
source ?? null
|
|
9324
|
+
],
|
|
9325
|
+
limit,
|
|
9326
|
+
queryWidth,
|
|
9327
|
+
widthCap,
|
|
9328
|
+
batchNames: false,
|
|
9329
|
+
sql: (placeholders) => `SELECT fa.fact_id AS id, COUNT(*) AS shared,
|
|
8608
9330
|
(SELECT COUNT(*) FROM fact_entities x WHERE x.fact_id = fa.fact_id) AS total
|
|
8609
9331
|
FROM facts fa
|
|
8610
9332
|
JOIN fact_entities fe ON fe.fact_id = fa.fact_id
|
|
@@ -8613,9 +9335,8 @@ var EntitiesDao = class {
|
|
|
8613
9335
|
AND (? IS NULL OR EXISTS (
|
|
8614
9336
|
SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?
|
|
8615
9337
|
))
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
LIMIT ?`).all(...values, category ?? null, category ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit).map((row) => row.id);
|
|
9338
|
+
${entityCandidateTail("fa.fact_id")}`
|
|
9339
|
+
});
|
|
8619
9340
|
}
|
|
8620
9341
|
};
|
|
8621
9342
|
//#endregion
|
|
@@ -8967,12 +9688,7 @@ var FactsDao = class FactsDao {
|
|
|
8967
9688
|
}
|
|
8968
9689
|
/** Stamp the rules that produced a batch of rows' entity/triple data. */
|
|
8969
9690
|
setEntitiesVersion(ids, version) {
|
|
8970
|
-
|
|
8971
|
-
for (const batch of batches(ids)) {
|
|
8972
|
-
const { placeholders, values } = inList(batch);
|
|
8973
|
-
changes += this.db.prepare(`UPDATE facts SET entities_version = ? WHERE fact_id IN (${placeholders})`).run(version, ...values).changes;
|
|
8974
|
-
}
|
|
8975
|
-
return changes;
|
|
9691
|
+
return setEntitiesVersionBatch(this.db, "facts", "fact_id", ids, version);
|
|
8976
9692
|
}
|
|
8977
9693
|
/**
|
|
8978
9694
|
* Replace a fact's HRR vector.
|
|
@@ -9352,14 +10068,16 @@ var FactsDao = class FactsDao {
|
|
|
9352
10068
|
* alias, so the LIKE expressions are built once and bound twice — same fragment, same order.
|
|
9353
10069
|
*/
|
|
9354
10070
|
ftsSubstringSearch(terms, category, limit, source) {
|
|
9355
|
-
const { any, count, params } = likeSubstring("fa.content", terms);
|
|
9356
|
-
if (params.length === 0) return [];
|
|
9357
10071
|
const sourceClause = source === void 0 ? "" : " AND EXISTS (SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?)";
|
|
9358
|
-
return this.db.
|
|
10072
|
+
return likeSubstringLeg(this.db, "fa.content", terms, ({ any, count }) => `SELECT fa.fact_id AS id, ${count} AS rank
|
|
9359
10073
|
FROM facts fa
|
|
9360
10074
|
WHERE fa.status = 'active' AND (? IS NULL OR fa.category = ?)${sourceClause} AND (${any})
|
|
9361
10075
|
ORDER BY rank DESC, fa.fact_id ASC
|
|
9362
|
-
LIMIT
|
|
10076
|
+
LIMIT ?`, source === void 0 ? [category ?? null, category ?? null] : [
|
|
10077
|
+
category ?? null,
|
|
10078
|
+
category ?? null,
|
|
10079
|
+
source
|
|
10080
|
+
], limit);
|
|
9363
10081
|
}
|
|
9364
10082
|
/**
|
|
9365
10083
|
* ①'s predicate: the active-day counter crossed an integer boundary since the last settle.
|
|
@@ -10441,15 +11159,6 @@ const NO_CHECK = {
|
|
|
10441
11159
|
complete: false
|
|
10442
11160
|
};
|
|
10443
11161
|
/**
|
|
10444
|
-
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
10445
|
-
*
|
|
10446
|
-
* Bounds MEMORY, not only mission: the pass selects `content` for every row it takes, so an
|
|
10447
|
-
* "unbounded" pass (the CLI used to ask for `Number.MAX_SAFE_INTEGER`) builds an array of every
|
|
10448
|
-
* stale fact's text before rebuilding the first one. Callers that want the corpus drained loop
|
|
10449
|
-
* until `deferred === 0` instead.
|
|
10450
|
-
*/
|
|
10451
|
-
const ENTITY_SWEEP_BATCH = 2e3;
|
|
10452
|
-
/**
|
|
10453
11162
|
* `avantf_stats` key holding the retrieval-health snapshot.
|
|
10454
11163
|
*
|
|
10455
11164
|
* The counters are process-wide, but the MEMORY store owns the side table, so it is the single
|
|
@@ -10495,8 +11204,14 @@ var MemoryStore = class {
|
|
|
10495
11204
|
* and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
|
|
10496
11205
|
*/
|
|
10497
11206
|
ftsTokenizer;
|
|
10498
|
-
/**
|
|
10499
|
-
|
|
11207
|
+
/**
|
|
11208
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
11209
|
+
*
|
|
11210
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
|
|
11211
|
+
* the rule where the loop is (the periodic callers can overlap), without a mutable field that two
|
|
11212
|
+
* stores would each have to remember to hold.
|
|
11213
|
+
*/
|
|
11214
|
+
entitySweepGuard = {};
|
|
10500
11215
|
/**
|
|
10501
11216
|
* Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
|
|
10502
11217
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
@@ -10536,7 +11251,7 @@ var MemoryStore = class {
|
|
|
10536
11251
|
clock,
|
|
10537
11252
|
budget: opts?.budget ?? trust.tick_max_facts
|
|
10538
11253
|
});
|
|
10539
|
-
this.
|
|
11254
|
+
evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
|
|
10540
11255
|
this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
|
|
10541
11256
|
this.db.pragma("optimize");
|
|
10542
11257
|
return result;
|
|
@@ -10550,22 +11265,32 @@ var MemoryStore = class {
|
|
|
10550
11265
|
return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
|
|
10551
11266
|
}
|
|
10552
11267
|
/**
|
|
10553
|
-
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs
|
|
10554
|
-
*
|
|
10555
|
-
* (the DB is the authoritative vector store; the vstore is a derived read model).
|
|
11268
|
+
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
|
|
11269
|
+
* belongs to another vector space.
|
|
10556
11270
|
*
|
|
10557
|
-
*
|
|
10558
|
-
*
|
|
10559
|
-
*
|
|
11271
|
+
* Without this the semantic path is silently empty after every process restart (the DB is the
|
|
11272
|
+
* authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
|
|
11273
|
+
* archived facts are filtered out of every retrieval path anyway, so loading them would only grow
|
|
11274
|
+
* the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
11275
|
+
*
|
|
11276
|
+
* The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
|
|
11277
|
+
* step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
|
|
11278
|
+
* another space is still RANKED (the reload cannot tell the difference), so a silent model swap
|
|
11279
|
+
* would mix two spaces in one index — acting on it is the manual fix's job for the operator path
|
|
11280
|
+
* and the bounded background migration's for the automatic one.
|
|
10560
11281
|
*/
|
|
10561
11282
|
reloadIndex() {
|
|
10562
|
-
|
|
10563
|
-
|
|
10564
|
-
|
|
10565
|
-
|
|
10566
|
-
|
|
10567
|
-
|
|
10568
|
-
|
|
11283
|
+
loadVectorIndex({
|
|
11284
|
+
store: "memory",
|
|
11285
|
+
vstore: this.vstore,
|
|
11286
|
+
rows: this.facts.activeVectorRows().map((r) => ({
|
|
11287
|
+
id: r.fact_id,
|
|
11288
|
+
vec: r.semantic_vector,
|
|
11289
|
+
embedding_model: r.embedding_model
|
|
11290
|
+
})),
|
|
11291
|
+
space: this.vectorSpace(),
|
|
11292
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)"
|
|
11293
|
+
});
|
|
10569
11294
|
}
|
|
10570
11295
|
/**
|
|
10571
11296
|
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
|
|
@@ -10666,21 +11391,45 @@ var MemoryStore = class {
|
|
|
10666
11391
|
};
|
|
10667
11392
|
}
|
|
10668
11393
|
}
|
|
11394
|
+
/**
|
|
11395
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
11396
|
+
* call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
|
|
11397
|
+
* backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
|
|
11398
|
+
* (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
|
|
11399
|
+
*/
|
|
11400
|
+
vectorWriteTarget(space) {
|
|
11401
|
+
return {
|
|
11402
|
+
store: "memory",
|
|
11403
|
+
space,
|
|
11404
|
+
encode: (text) => this.semantic.encode(text),
|
|
11405
|
+
add: (id, vec) => {
|
|
11406
|
+
this.vstore.add(id, vec);
|
|
11407
|
+
},
|
|
11408
|
+
write: (rows) => {
|
|
11409
|
+
for (const r of rows) this.facts.setSemanticVector(r.id, r.bytes, space, this.vstore.name);
|
|
11410
|
+
},
|
|
11411
|
+
onRowError: (row, error) => {
|
|
11412
|
+
this.warnIndexFailure(row.id, error);
|
|
11413
|
+
},
|
|
11414
|
+
onWriteError: (rows, error) => {
|
|
11415
|
+
for (const r of rows) this.warnIndexFailure(r.id, error);
|
|
11416
|
+
}
|
|
11417
|
+
};
|
|
11418
|
+
}
|
|
11419
|
+
warnIndexFailure(factId, error) {
|
|
11420
|
+
retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
|
|
11421
|
+
}
|
|
10669
11422
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
10670
11423
|
async maybeIndexSemantic(factId, content) {
|
|
10671
|
-
|
|
10672
|
-
|
|
10673
|
-
this.semantic.ensureWarm?.();
|
|
10674
|
-
return false;
|
|
10675
|
-
}
|
|
10676
|
-
const vec = await this.semantic.encode(content);
|
|
10677
|
-
this.vstore.add(factId, vec);
|
|
10678
|
-
this.facts.setSemanticVector(factId, float32ToBytes(vec), this.vectorSpace(), this.vstore.name);
|
|
10679
|
-
return true;
|
|
10680
|
-
} catch (error) {
|
|
10681
|
-
retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
|
|
11424
|
+
if (!this.semantic.isAvailable()) {
|
|
11425
|
+
this.semantic.ensureWarm?.();
|
|
10682
11426
|
return false;
|
|
10683
11427
|
}
|
|
11428
|
+
const { encoded } = await encodeAndPersist([{
|
|
11429
|
+
id: factId,
|
|
11430
|
+
text: content
|
|
11431
|
+
}], this.vectorWriteTarget(this.vectorSpace()));
|
|
11432
|
+
return encoded === 1;
|
|
10684
11433
|
}
|
|
10685
11434
|
async update(fact_id, content, category, ttlDays, opts) {
|
|
10686
11435
|
if (!this.getRow(fact_id)) throw new Error(`找不到 fact_id=${fact_id}`);
|
|
@@ -10775,7 +11524,7 @@ var MemoryStore = class {
|
|
|
10775
11524
|
this.facts.reinforceOutcome(fact_id, outcome.next, clock, outcome.bonusCount, outcome.windowAt);
|
|
10776
11525
|
return null;
|
|
10777
11526
|
}).immediate();
|
|
10778
|
-
if (evict !== null) this.
|
|
11527
|
+
if (evict !== null) evictVectors(this.vstore, [evict]);
|
|
10779
11528
|
return this.getRow(fact_id)?.helpful_count ?? 0;
|
|
10780
11529
|
}
|
|
10781
11530
|
/** Permanent-memory administration (spec §6). */
|
|
@@ -11070,168 +11819,103 @@ var MemoryStore = class {
|
|
|
11070
11819
|
};
|
|
11071
11820
|
}
|
|
11072
11821
|
/**
|
|
11073
|
-
*
|
|
11074
|
-
*
|
|
11075
|
-
*
|
|
11076
|
-
*
|
|
11077
|
-
* re-encode can replace them (never `add()` a stale vector — the store
|
|
11078
|
-
* throws on dim mismatch),
|
|
11079
|
-
* (c) encode active facts that have no vector yet (needs the semantic backend),
|
|
11080
|
-
* (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
|
|
11081
|
-
* adoption of a store predating the space id).
|
|
11082
|
-
*
|
|
11083
|
-
* `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
|
|
11084
|
-
* dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
|
|
11085
|
-
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
11086
|
-
* `migrateVectorsBatch` passes the migration's batch size.
|
|
11822
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
11823
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
11824
|
+
* `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
|
|
11825
|
+
* for both stores.
|
|
11087
11826
|
*/
|
|
11088
|
-
|
|
11089
|
-
|
|
11090
|
-
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
11100
|
-
|
|
11101
|
-
|
|
11102
|
-
reindexed: 0,
|
|
11103
|
-
dropped: 0,
|
|
11104
|
-
fixed: 0,
|
|
11105
|
-
semantic_available: semAvailable,
|
|
11106
|
-
would_warm: wouldWarm,
|
|
11107
|
-
dry_run: true
|
|
11108
|
-
},
|
|
11109
|
-
remaining: remainingOf(0)
|
|
11827
|
+
vectorRepairTarget() {
|
|
11828
|
+
return {
|
|
11829
|
+
store: "memory",
|
|
11830
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)",
|
|
11831
|
+
vectorSpace: () => this.vectorSpace(),
|
|
11832
|
+
semanticAvailable: () => this.semantic.isAvailable(),
|
|
11833
|
+
autoMigrate: () => this.config.semantic.auto_migrate,
|
|
11834
|
+
vectorSpaceHealth: () => this.vectorSpaceHealth(),
|
|
11835
|
+
missingVectorCount: () => this.facts.vectorCounts().missing,
|
|
11836
|
+
classifyVectors: (limit) => this.classifyRepairVectors(limit),
|
|
11837
|
+
reindexUsable: (rows) => this.reindexUsableVectors(rows),
|
|
11838
|
+
dropVectors: (ids) => this.dropRepairVectors(ids),
|
|
11839
|
+
reencodeVectors: (limit) => this.reencodeMissingVectors(limit),
|
|
11840
|
+
warmup: () => this.warmupSemantic()
|
|
11110
11841
|
};
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
const toDrop = [...c.staleIds, ...c.spaceStaleIds].slice(0, limit);
|
|
11119
|
-
if (toDrop.length) {
|
|
11120
|
-
const spaceStale = new Set(c.spaceStaleIds);
|
|
11121
|
-
dropped = this.facts.clearVectors(toDrop);
|
|
11122
|
-
this.evictVectors(toDrop.filter((id) => spaceStale.has(id)));
|
|
11123
|
-
}
|
|
11124
|
-
}
|
|
11125
|
-
let fixed = 0;
|
|
11126
|
-
if (semAvailable) {
|
|
11127
|
-
const missingRows = this.facts.missingVectorRows().slice(0, limit);
|
|
11128
|
-
for (const m of missingRows) {
|
|
11129
|
-
if (!await this.maybeIndexSemantic(m.fact_id, m.content)) continue;
|
|
11130
|
-
fixed++;
|
|
11131
|
-
}
|
|
11132
|
-
}
|
|
11842
|
+
}
|
|
11843
|
+
/**
|
|
11844
|
+
* {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
|
|
11845
|
+
* (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
|
|
11846
|
+
*/
|
|
11847
|
+
classifyRepairVectors(limit) {
|
|
11848
|
+
const c = this.classifyVectors();
|
|
11133
11849
|
return {
|
|
11134
|
-
|
|
11135
|
-
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
11139
|
-
|
|
11140
|
-
|
|
11141
|
-
|
|
11142
|
-
|
|
11143
|
-
|
|
11144
|
-
|
|
11145
|
-
|
|
11146
|
-
|
|
11850
|
+
stale: c.staleIds.length,
|
|
11851
|
+
space_stale: c.spaceStaleIds.length,
|
|
11852
|
+
missing: c.missing,
|
|
11853
|
+
dropIds: [...c.staleIds, ...c.spaceStaleIds].slice(0, limit),
|
|
11854
|
+
unindexed: c.unindexed
|
|
11855
|
+
};
|
|
11856
|
+
}
|
|
11857
|
+
/** Put usable persisted vectors missing from the live index back into it (no model needed). */
|
|
11858
|
+
reindexUsableVectors(rows) {
|
|
11859
|
+
for (const u of rows) this.vstore.add(u.id, u.vec);
|
|
11860
|
+
return rows.length;
|
|
11861
|
+
}
|
|
11862
|
+
/**
|
|
11863
|
+
* Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
|
|
11864
|
+
* pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
|
|
11865
|
+
* the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
|
|
11866
|
+
* one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
|
|
11867
|
+
*
|
|
11868
|
+
* `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
|
|
11869
|
+
* the embedding leg was checked against is gone, so the check has to run again.
|
|
11870
|
+
*/
|
|
11871
|
+
dropRepairVectors(ids) {
|
|
11872
|
+
if (!ids.length) return 0;
|
|
11873
|
+
const dropped = this.facts.clearVectors(ids);
|
|
11874
|
+
evictVectors(this.vstore, ids);
|
|
11875
|
+
return dropped;
|
|
11876
|
+
}
|
|
11877
|
+
/**
|
|
11878
|
+
* Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
|
|
11879
|
+
*
|
|
11880
|
+
* Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
|
|
11881
|
+
* the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
|
|
11882
|
+
* ran has not been checked.
|
|
11883
|
+
*/
|
|
11884
|
+
async reencodeMissingVectors(limit) {
|
|
11885
|
+
const rows = this.facts.missingVectorRows().slice(0, limit).map((m) => ({
|
|
11886
|
+
id: m.fact_id,
|
|
11887
|
+
text: m.content
|
|
11888
|
+
}));
|
|
11889
|
+
if (!rows.length) return {
|
|
11890
|
+
encoded: 0,
|
|
11891
|
+
failed: 0
|
|
11147
11892
|
};
|
|
11893
|
+
return await encodeAndPersist(rows, this.vectorWriteTarget(this.vectorSpace()));
|
|
11148
11894
|
}
|
|
11149
11895
|
/**
|
|
11150
11896
|
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
11151
11897
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
11152
|
-
* the preview also reports `would_warm` (see the contract's `
|
|
11898
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
11153
11899
|
*/
|
|
11154
11900
|
async vectorsFix(dryRun = false) {
|
|
11155
|
-
return (await this.
|
|
11901
|
+
return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
|
|
11156
11902
|
}
|
|
11157
11903
|
/**
|
|
11158
11904
|
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
11159
11905
|
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
11160
11906
|
* background migration ({@link migrateVectors}) and its tests drive.
|
|
11161
|
-
*
|
|
11162
|
-
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
11163
|
-
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
11164
11907
|
*/
|
|
11165
11908
|
async migrateVectorsBatch(batchSize = 16) {
|
|
11166
|
-
|
|
11167
|
-
const { report, remaining } = await this.repairVectors({
|
|
11168
|
-
dryRun: false,
|
|
11169
|
-
limit: size
|
|
11170
|
-
});
|
|
11171
|
-
return {
|
|
11172
|
-
remaining,
|
|
11173
|
-
migrated: report.fixed,
|
|
11174
|
-
dropped: report.dropped,
|
|
11175
|
-
reindexed: report.reindexed,
|
|
11176
|
-
semantic_available: report.semantic_available
|
|
11177
|
-
};
|
|
11909
|
+
return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
|
|
11178
11910
|
}
|
|
11179
11911
|
/**
|
|
11180
11912
|
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
11181
|
-
* `shouldStop()` turns true (plugin unmount / process shutdown).
|
|
11182
|
-
*
|
|
11183
|
-
*
|
|
11184
|
-
* blocks the event loop for longer than one batch.
|
|
11185
|
-
*
|
|
11186
|
-
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
11187
|
-
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
11913
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
|
|
11914
|
+
* switch, the no-progress rule and the store-named logging all live in the shared flow
|
|
11915
|
+
* (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
|
|
11188
11916
|
*/
|
|
11189
11917
|
async migrateVectors(opts = {}) {
|
|
11190
|
-
|
|
11191
|
-
const snapshot = () => ({
|
|
11192
|
-
enabled,
|
|
11193
|
-
remaining: this.migrationRemaining(),
|
|
11194
|
-
migrated: 0,
|
|
11195
|
-
dropped: 0,
|
|
11196
|
-
reindexed: 0,
|
|
11197
|
-
semantic_available: this.semantic.isAvailable()
|
|
11198
|
-
});
|
|
11199
|
-
if (!enabled) return snapshot();
|
|
11200
|
-
let migrated = 0;
|
|
11201
|
-
let dropped = 0;
|
|
11202
|
-
let reindexed = 0;
|
|
11203
|
-
let last = snapshot();
|
|
11204
|
-
if (last.remaining === 0) return last;
|
|
11205
|
-
for (;;) {
|
|
11206
|
-
if (opts.shouldStop?.() === true) break;
|
|
11207
|
-
let step;
|
|
11208
|
-
try {
|
|
11209
|
-
step = await this.migrateVectorsBatch(opts.batchSize);
|
|
11210
|
-
} catch (error) {
|
|
11211
|
-
retrievalLogger().warn(`memory vector migration: batch failed (${describeError$2(error)}) — ${String(this.migrationRemaining())} vector(s) still belong to an older embedding space; the next pass retries`);
|
|
11212
|
-
break;
|
|
11213
|
-
}
|
|
11214
|
-
migrated += step.migrated;
|
|
11215
|
-
dropped += step.dropped;
|
|
11216
|
-
reindexed += step.reindexed;
|
|
11217
|
-
last = {
|
|
11218
|
-
...step,
|
|
11219
|
-
enabled,
|
|
11220
|
-
migrated,
|
|
11221
|
-
dropped,
|
|
11222
|
-
reindexed
|
|
11223
|
-
};
|
|
11224
|
-
opts.onProgress?.(last);
|
|
11225
|
-
if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
|
|
11226
|
-
if (last.remaining === 0) break;
|
|
11227
|
-
await yieldToEventLoop();
|
|
11228
|
-
}
|
|
11229
|
-
return last;
|
|
11230
|
-
}
|
|
11231
|
-
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
11232
|
-
migrationRemaining() {
|
|
11233
|
-
const health = this.vectorSpaceHealth();
|
|
11234
|
-
return health.stale + health.space_stale + this.facts.vectorCounts().missing;
|
|
11918
|
+
return await driveVectorRepair(this.vectorRepairTarget(), opts);
|
|
11235
11919
|
}
|
|
11236
11920
|
/**
|
|
11237
11921
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
@@ -11246,7 +11930,7 @@ var MemoryStore = class {
|
|
|
11246
11930
|
*/
|
|
11247
11931
|
async maintenance() {
|
|
11248
11932
|
const result = runMaintenance(this.db, this.config, { budget: 0 });
|
|
11249
|
-
this.
|
|
11933
|
+
evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
|
|
11250
11934
|
this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
|
|
11251
11935
|
this.vstore.compact?.();
|
|
11252
11936
|
this.db.pragma("optimize");
|
|
@@ -11279,25 +11963,19 @@ var MemoryStore = class {
|
|
|
11279
11963
|
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
11280
11964
|
* every selected row's `content` before rebuilding the first one.
|
|
11281
11965
|
*
|
|
11282
|
-
* The in-flight guard
|
|
11283
|
-
*
|
|
11284
|
-
*
|
|
11285
|
-
*
|
|
11286
|
-
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
11287
|
-
*
|
|
11288
|
-
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
11289
|
-
* caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
|
|
11966
|
+
* The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
|
|
11967
|
+
* live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
|
|
11968
|
+
* entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
|
|
11969
|
+
* selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
|
|
11290
11970
|
*/
|
|
11291
11971
|
async reindexEntities(budget = ENTITY_SWEEP_BATCH) {
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
const rows = this.facts.staleEntityRows(1, Math.max(0, Math.floor(budget)));
|
|
11300
|
-
for (const row of rows) {
|
|
11972
|
+
return await sweepVersionedEntities({
|
|
11973
|
+
staleEntityBatch: (limit) => ({
|
|
11974
|
+
total: this.facts.countStaleEntities(1),
|
|
11975
|
+
rows: this.facts.staleEntityRows(1, limit)
|
|
11976
|
+
}),
|
|
11977
|
+
staleEntityCount: () => this.facts.countStaleEntities(1),
|
|
11978
|
+
extractAndWriteEntities: async (row) => {
|
|
11301
11979
|
const tokens = await tagText(row.content);
|
|
11302
11980
|
const entityWrites = normalizeEntityWrites(entitiesFromTokens(tokens, row.content));
|
|
11303
11981
|
const entities = entityWrites.map((e) => e.name);
|
|
@@ -11312,15 +11990,11 @@ var MemoryStore = class {
|
|
|
11312
11990
|
this.facts.setEntitiesVersion([row.fact_id], 1);
|
|
11313
11991
|
this.facts.requeueConflictCheck([row.fact_id]);
|
|
11314
11992
|
})();
|
|
11993
|
+
},
|
|
11994
|
+
onRowError: (row, error) => {
|
|
11995
|
+
retrievalLogger().warn(`memory entity sweep: fact ${String(row.fact_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next pass retries it`);
|
|
11315
11996
|
}
|
|
11316
|
-
|
|
11317
|
-
rebuilt: rows.length,
|
|
11318
|
-
deferred: this.facts.countStaleEntities(1),
|
|
11319
|
-
skipped: false
|
|
11320
|
-
};
|
|
11321
|
-
} finally {
|
|
11322
|
-
this.entitySweepInFlight = false;
|
|
11323
|
-
}
|
|
11997
|
+
}, this.entitySweepGuard, { budget });
|
|
11324
11998
|
}
|
|
11325
11999
|
/**
|
|
11326
12000
|
* A fact left the ACTIVE corpus: retire the open conflicts that name it, so
|
|
@@ -11331,10 +12005,6 @@ var MemoryStore = class {
|
|
|
11331
12005
|
retireConflicts(factIds) {
|
|
11332
12006
|
this.contradictions.resolveForFacts(factIds);
|
|
11333
12007
|
}
|
|
11334
|
-
/** Drop vectors of archived/purged facts from the live index (batched). */
|
|
11335
|
-
evictVectors(factIds) {
|
|
11336
|
-
evictVectors(this.vstore, factIds);
|
|
11337
|
-
}
|
|
11338
12008
|
/**
|
|
11339
12009
|
* Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
|
|
11340
12010
|
* count. Archived (incl. superseded) facts are excluded — their entities are not
|
|
@@ -11382,17 +12052,7 @@ var MemoryStore = class {
|
|
|
11382
12052
|
}
|
|
11383
12053
|
/** Hybrid search over the memory corpus (semantic + FTS + entity Jaccard [+ HRR probe] via fusion). */
|
|
11384
12054
|
async search(input) {
|
|
11385
|
-
const result = await
|
|
11386
|
-
query: input.query,
|
|
11387
|
-
limit: input.limit,
|
|
11388
|
-
maxTokens: input.maxTokens,
|
|
11389
|
-
queryVector: input.queryVector,
|
|
11390
|
-
recordStats: input.recordStats,
|
|
11391
|
-
floors: input.floors,
|
|
11392
|
-
...input.includeScores === void 0 ? {} : { includeScores: input.includeScores },
|
|
11393
|
-
...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs },
|
|
11394
|
-
...input.rewriteQuery === void 0 ? {} : { rewriteQuery: input.rewriteQuery }
|
|
11395
|
-
});
|
|
12055
|
+
const result = await runHybridSearch(this.hybridDeps(input), input.query, input);
|
|
11396
12056
|
return {
|
|
11397
12057
|
hits: result.hits,
|
|
11398
12058
|
degraded: result.degraded,
|
|
@@ -11407,10 +12067,7 @@ var MemoryStore = class {
|
|
|
11407
12067
|
* fact→hit mapping, and what to do with the hits the caller actually receives.
|
|
11408
12068
|
*/
|
|
11409
12069
|
hybridDeps(input) {
|
|
11410
|
-
return {
|
|
11411
|
-
kind: "memory",
|
|
11412
|
-
config: this.config,
|
|
11413
|
-
semantic: this.semantic,
|
|
12070
|
+
return makeHybridDeps("memory", this.config, this.semantic, {
|
|
11414
12071
|
legs: (ctx) => this.searchLegs(input, ctx),
|
|
11415
12072
|
texts: (ids) => this.loadTexts(ids),
|
|
11416
12073
|
hits: (ranked, texts) => {
|
|
@@ -11420,7 +12077,7 @@ var MemoryStore = class {
|
|
|
11420
12077
|
onReturn: (kept) => {
|
|
11421
12078
|
if (input.track !== false) this.reinforce(kept.map((h) => h.ref_id));
|
|
11422
12079
|
}
|
|
11423
|
-
};
|
|
12080
|
+
});
|
|
11424
12081
|
}
|
|
11425
12082
|
/**
|
|
11426
12083
|
* The memory legs: semantic, FTS, entity Jaccard, and — for `recall.probe` — the HRR probe.
|
|
@@ -11442,49 +12099,18 @@ var MemoryStore = class {
|
|
|
11442
12099
|
const qEntities = Array.from(new Set((await extractEntities(ctx.query)).map((e) => e.name)));
|
|
11443
12100
|
const anchors = qEntities.length > 0 ? selectAnchors(qEntities, this.entities.activeDocFrequency(qEntities), this.facts.countActive()) : [];
|
|
11444
12101
|
const candidates = anchors.length > 0 ? this.entities.candidateFactsForAnyEntity(anchors, input.category, ctx.legCap, qEntities.length, 3, input.source) : [];
|
|
11445
|
-
|
|
11446
|
-
* Wrap one leg's raw scores. `capped` must be measured on the RAW set (before the relevance
|
|
11447
|
-
* floor): the floor removes the tail anyway, and deriving the flag from the floored size would
|
|
11448
|
-
* erase the "this leg was cut at legCap" signal exactly when it bound.
|
|
11449
|
-
*/
|
|
11450
|
-
const leg = (scores, weight, raw) => ({
|
|
11451
|
-
weight,
|
|
11452
|
-
scores,
|
|
11453
|
-
capped: (raw ?? scores).size === ctx.legCap
|
|
11454
|
-
});
|
|
12102
|
+
const runner = makeLegRunner(ctx);
|
|
11455
12103
|
const ftsRaw = this.ftsPath(ctx.query, input.category, ctx.legCap, input.source);
|
|
11456
|
-
const
|
|
12104
|
+
const ftsLeg = runner.term("fts", ftsRaw, this.loadTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts);
|
|
11457
12105
|
const jaccardRaw = this.jaccardPath(anchors, qEntities.length, candidates);
|
|
11458
|
-
const
|
|
11459
|
-
const jaccardLeg = {
|
|
11460
|
-
...leg(jaccardFloored.scores, ctx.weights.jaccard, jaccardRaw),
|
|
11461
|
-
leg: "jaccard",
|
|
11462
|
-
droppedByFloor: jaccardFloored.dropped
|
|
11463
|
-
};
|
|
12106
|
+
const jaccardLeg = runner.scored("jaccard", jaccardRaw, ctx.weights.jaccard, ctx.floors.jaccard);
|
|
11464
12107
|
const legs = [
|
|
11465
|
-
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) =>
|
|
11466
|
-
const floored = applyScoreFloor(raw, ctx.floors.semantic);
|
|
11467
|
-
return {
|
|
11468
|
-
weight: ctx.weights.semantic,
|
|
11469
|
-
scores: floored.scores,
|
|
11470
|
-
leg: "semantic",
|
|
11471
|
-
droppedByFloor: floored.dropped
|
|
11472
|
-
};
|
|
11473
|
-
}) : {
|
|
11474
|
-
weight: ctx.weights.semantic,
|
|
11475
|
-
scores: /* @__PURE__ */ new Map(),
|
|
11476
|
-
leg: "semantic",
|
|
11477
|
-
droppedByFloor: 0
|
|
11478
|
-
},
|
|
12108
|
+
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
|
|
11479
12109
|
Promise.resolve(jaccardLeg),
|
|
11480
|
-
|
|
11481
|
-
...leg(ftsFloored.scores, ctx.weights.fts, ftsRaw),
|
|
11482
|
-
leg: "fts",
|
|
11483
|
-
droppedByFloor: ftsFloored.dropped
|
|
11484
|
-
}
|
|
12110
|
+
ftsLeg
|
|
11485
12111
|
];
|
|
11486
|
-
if (input.includeHrr) legs.push(Promise.resolve(
|
|
11487
|
-
if (this.config.retriever.time_window) legs.push(Promise.resolve(
|
|
12112
|
+
if (input.includeHrr) legs.push(Promise.resolve(runner.plain(this.hrrPath(ctx.query, qEntities, [...jaccardLeg.scores.keys()], input.category, ctx.legCap, candidates.length === 0, input.source), ctx.weights.jaccard)));
|
|
12113
|
+
if (this.config.retriever.time_window) legs.push(Promise.resolve(runner.plain(this.timeWindowPath(ctx.query, input.category, ctx.legCap, input.source), ctx.weights.jaccard)));
|
|
11488
12114
|
return legs;
|
|
11489
12115
|
}
|
|
11490
12116
|
/**
|
|
@@ -11609,18 +12235,22 @@ var MemoryStore = class {
|
|
|
11609
12235
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
11610
12236
|
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
11611
12237
|
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
11612
|
-
* instead of re-encoding (performance review §7.7 / P8).
|
|
12238
|
+
* instead of re-encoding (performance review §7.7 / P8). The flow is shared
|
|
12239
|
+
* (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
|
|
12240
|
+
* notion of status/category/source, so candidates go through the DB and archived/purged facts,
|
|
12241
|
+
* other categories and other sources never leak into the leg.
|
|
11613
12242
|
*/
|
|
11614
12243
|
async semanticPath(query, category, k, queryVector, onVector, source) {
|
|
11615
|
-
|
|
11616
|
-
|
|
11617
|
-
|
|
11618
|
-
|
|
11619
|
-
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
12244
|
+
return semanticLeg({
|
|
12245
|
+
query,
|
|
12246
|
+
k,
|
|
12247
|
+
queryVector,
|
|
12248
|
+
onVector,
|
|
12249
|
+
encode: (q) => this.semantic.encode(q),
|
|
12250
|
+
dim: this.vstore.dim,
|
|
12251
|
+
topk: (vec, n) => this.vstore.topk(vec, n),
|
|
12252
|
+
filterTopk: (ids) => new Set(this.facts.activeIdsIn(ids, category, source))
|
|
12253
|
+
});
|
|
11624
12254
|
}
|
|
11625
12255
|
/**
|
|
11626
12256
|
* HRR entity-probe path: phase similarity between the query's entity bundle and
|
|
@@ -11689,21 +12319,20 @@ var MemoryStore = class {
|
|
|
11689
12319
|
* semantic, why it must be synchronous, and how the bar itself was measured.
|
|
11690
12320
|
*/
|
|
11691
12321
|
lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
|
|
11692
|
-
return
|
|
11693
|
-
const fts = buildFtsQuery(term, this.ftsTokenizer);
|
|
11694
|
-
return fts !== null && this.facts.ftsSearch(fts, void 0, 1).length > 0;
|
|
11695
|
-
}, stopAt);
|
|
12322
|
+
return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.facts.ftsSearch(fts, void 0, 1).length > 0, stopAt);
|
|
11696
12323
|
}
|
|
11697
12324
|
ftsPath(query, category, cap, source) {
|
|
11698
|
-
|
|
11699
|
-
|
|
11700
|
-
|
|
11701
|
-
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
12325
|
+
return ftsLeg({
|
|
12326
|
+
query,
|
|
12327
|
+
cap,
|
|
12328
|
+
scope: {
|
|
12329
|
+
category,
|
|
12330
|
+
source
|
|
12331
|
+
},
|
|
12332
|
+
tokenizer: this.ftsTokenizer,
|
|
12333
|
+
search: (ftsQuery, scope, limit) => this.facts.ftsSearch(ftsQuery, scope.category, limit, scope.source),
|
|
12334
|
+
substringSearch: (terms, scope, limit) => this.facts.ftsSubstringSearch(terms, scope.category, limit, scope.source)
|
|
12335
|
+
});
|
|
11707
12336
|
}
|
|
11708
12337
|
/**
|
|
11709
12338
|
* Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
|
|
@@ -11810,7 +12439,7 @@ var MemoryStore = class {
|
|
|
11810
12439
|
supersededId
|
|
11811
12440
|
};
|
|
11812
12441
|
})();
|
|
11813
|
-
if (supersededId !== null) this.
|
|
12442
|
+
if (supersededId !== null) evictVectors(this.vstore, [supersededId]);
|
|
11814
12443
|
return result;
|
|
11815
12444
|
}
|
|
11816
12445
|
/**
|
|
@@ -12179,10 +12808,7 @@ var ChunksDao = class {
|
|
|
12179
12808
|
}
|
|
12180
12809
|
/** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
|
|
12181
12810
|
setEntitiesVersion(ids, version) {
|
|
12182
|
-
|
|
12183
|
-
const { placeholders, values } = inList(batch);
|
|
12184
|
-
this.db.prepare(`UPDATE doc_chunks SET entities_version = ? WHERE chunk_id IN (${placeholders})`).run(version, ...values);
|
|
12185
|
-
}
|
|
12811
|
+
setEntitiesVersionBatch(this.db, "doc_chunks", "chunk_id", ids, version);
|
|
12186
12812
|
}
|
|
12187
12813
|
/** Every persisted chunk vector, for rebuilding the in-memory index at open. */
|
|
12188
12814
|
/**
|
|
@@ -12195,14 +12821,68 @@ var ChunksDao = class {
|
|
|
12195
12821
|
vectorRows() {
|
|
12196
12822
|
return this.db.prepare("SELECT chunk_id AS id, semantic_vector AS vec, embedding_model AS embedding_model FROM doc_chunks WHERE semantic_vector IS NOT NULL").all();
|
|
12197
12823
|
}
|
|
12824
|
+
/**
|
|
12825
|
+
* Vector-space health of the WHOLE chunk corpus, by kind, WITHOUT decoding a single blob.
|
|
12826
|
+
*
|
|
12827
|
+
* The same two counts the open-time warning reports (`stale` = wrong width, `space_stale` = same
|
|
12828
|
+
* width but another recorded space — `embedding_model IS NOT` is null-safe, so a row with no
|
|
12829
|
+
* recorded space counts as foreign exactly like the JS comparison in `reloadIndex`) plus `missing`
|
|
12830
|
+
* (no vector at all), which is the third reason the semantic leg cannot use a chunk.
|
|
12831
|
+
*
|
|
12832
|
+
* This is what lets the shared repair flow read "what is stale" cheaply per slice instead of
|
|
12833
|
+
* re-scanning and hashing the corpus (`corpusState` reads every text; a batched migration must
|
|
12834
|
+
* not).
|
|
12835
|
+
*/
|
|
12836
|
+
vectorSpaceCounts(space, expectedBytes) {
|
|
12837
|
+
const row = this.db.prepare(`SELECT
|
|
12838
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NULL THEN 1 ELSE 0 END), 0) AS missing,
|
|
12839
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) != :bytes THEN 1 ELSE 0 END), 0) AS stale,
|
|
12840
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) = :bytes AND embedding_model IS NOT :space THEN 1 ELSE 0 END), 0) AS space_stale
|
|
12841
|
+
FROM doc_chunks`).get({
|
|
12842
|
+
bytes: expectedBytes,
|
|
12843
|
+
space
|
|
12844
|
+
});
|
|
12845
|
+
return {
|
|
12846
|
+
stale: Number(row?.stale ?? 0),
|
|
12847
|
+
space_stale: Number(row?.space_stale ?? 0),
|
|
12848
|
+
missing: Number(row?.missing ?? 0)
|
|
12849
|
+
};
|
|
12850
|
+
}
|
|
12851
|
+
/** Chunks with no persisted vector — the repair flow's `missing` term in `remaining`. */
|
|
12852
|
+
countMissingVectors() {
|
|
12853
|
+
return Number(this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks WHERE semantic_vector IS NULL").get()?.n ?? 0);
|
|
12854
|
+
}
|
|
12855
|
+
/**
|
|
12856
|
+
* Chunks the semantic leg cannot use (no vector / wrong width / another space), bounded and
|
|
12857
|
+
* ordered by id — the repair flow's encode source.
|
|
12858
|
+
*
|
|
12859
|
+
* The content-hash check `vectorReusable` adds is deliberately NOT here: a chunk whose vector is
|
|
12860
|
+
* in the current space is usable even if its text was edited behind the store's back, and that
|
|
12861
|
+
* (rarer) case belongs to `kb_reindex`, which re-derives every leg. Repairing the SPACE must not
|
|
12862
|
+
* pay a jieba/hash pass over the corpus to find rows the space change did not touch.
|
|
12863
|
+
*/
|
|
12864
|
+
staleVectorRows(space, expectedBytes, limit) {
|
|
12865
|
+
const bound = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : -1;
|
|
12866
|
+
return this.db.prepare(`SELECT chunk_id AS chunk_id, text AS text FROM doc_chunks
|
|
12867
|
+
WHERE semantic_vector IS NULL OR length(semantic_vector) != :bytes OR embedding_model IS NOT :space
|
|
12868
|
+
ORDER BY chunk_id ASC LIMIT :limit`).all({
|
|
12869
|
+
bytes: expectedBytes,
|
|
12870
|
+
space,
|
|
12871
|
+
limit: bound
|
|
12872
|
+
});
|
|
12873
|
+
}
|
|
12198
12874
|
count() {
|
|
12199
12875
|
return this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks").get().n;
|
|
12200
12876
|
}
|
|
12201
12877
|
/**
|
|
12202
|
-
* Replace the entity rows of the given chunks in one transaction
|
|
12203
|
-
*
|
|
12878
|
+
* Replace the entity rows of the given chunks in one transaction.
|
|
12879
|
+
*
|
|
12880
|
+
* `replace` is REQUIRED, not defaulted: `true` clears the old rows first (the entity sweep, where
|
|
12881
|
+
* the extractor may have changed), `false` is the ingest path, where the chunk ids are freshly
|
|
12882
|
+
* inserted and an absent old row set. A default would let one caller silently inherit the other's
|
|
12883
|
+
* semantics — an extra `DELETE` per chunk on ingest, or an append-on-top-of-stale sweep.
|
|
12204
12884
|
*/
|
|
12205
|
-
replaceEntities(rows, replace
|
|
12885
|
+
replaceEntities(rows, replace) {
|
|
12206
12886
|
const del = this.db.prepare("DELETE FROM chunk_entities WHERE chunk_id = ?");
|
|
12207
12887
|
const ins = this.db.prepare("INSERT OR IGNORE INTO chunk_entities (chunk_id, name) VALUES (?, ?)");
|
|
12208
12888
|
this.db.transaction(() => {
|
|
@@ -12214,17 +12894,12 @@ var ChunksDao = class {
|
|
|
12214
12894
|
}
|
|
12215
12895
|
/** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
|
|
12216
12896
|
entityBags(ids) {
|
|
12217
|
-
|
|
12218
|
-
|
|
12219
|
-
|
|
12220
|
-
|
|
12221
|
-
|
|
12222
|
-
|
|
12223
|
-
if (list) list.push(r.name);
|
|
12224
|
-
else out.set(r.chunk_id, [r.name]);
|
|
12225
|
-
}
|
|
12226
|
-
}
|
|
12227
|
-
return out;
|
|
12897
|
+
return entityBags(this.db, ids, {
|
|
12898
|
+
table: "chunk_entities",
|
|
12899
|
+
keyColumn: "chunk_id",
|
|
12900
|
+
nameColumn: "name",
|
|
12901
|
+
seedEmpty: false
|
|
12902
|
+
});
|
|
12228
12903
|
}
|
|
12229
12904
|
/** Chunk body + owning-document identity for the fused candidate ids. */
|
|
12230
12905
|
hits(ids) {
|
|
@@ -12264,13 +12939,48 @@ var ChunksDao = class {
|
|
|
12264
12939
|
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12265
12940
|
WHERE (? IS NULL OR d.domain = ?)`).all(domain ?? null, domain ?? null);
|
|
12266
12941
|
}
|
|
12942
|
+
/**
|
|
12943
|
+
* Chunk rows whose `chunk_entities` were produced by OLDER extraction rules — the entity sweep's
|
|
12944
|
+
* selection, bounded and ordered by id.
|
|
12945
|
+
*
|
|
12946
|
+
* `COALESCE(entities_version, 0) < :version`, not `entities_version != :version`: the column is
|
|
12947
|
+
* NULLABLE on purpose (a row written before the derived-state columns existed must look stale
|
|
12948
|
+
* exactly once and be adopted by the first rebuild — see `ChunkStateRow`), and a bare `<` would
|
|
12949
|
+
* evaluate to NULL for those rows and let them hide from the sweep forever.
|
|
12950
|
+
*
|
|
12951
|
+
* The predicate is index-served in id order (`doc_chunks` is keyed by `chunk_id`), so the LIMIT
|
|
12952
|
+
* bounds what is READ, not only what is rebuilt — the same property `FactsDao.staleEntityRows`
|
|
12953
|
+
* documents for the memory side.
|
|
12954
|
+
*/
|
|
12955
|
+
chunkEntityRows(version, limit, domain) {
|
|
12956
|
+
return this.db.prepare(`SELECT dc.chunk_id AS chunk_id, dc.text AS text
|
|
12957
|
+
FROM doc_chunks dc
|
|
12958
|
+
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12959
|
+
WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)
|
|
12960
|
+
ORDER BY dc.chunk_id ASC LIMIT :limit`).all({
|
|
12961
|
+
version,
|
|
12962
|
+
limit,
|
|
12963
|
+
domain: domain ?? null
|
|
12964
|
+
});
|
|
12965
|
+
}
|
|
12966
|
+
/** How many chunks still carry entity rows from older rules — the sweep's `deferred` source. */
|
|
12967
|
+
countStaleEntityChunks(version, domain) {
|
|
12968
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n
|
|
12969
|
+
FROM doc_chunks dc
|
|
12970
|
+
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12971
|
+
WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)`).get({
|
|
12972
|
+
version,
|
|
12973
|
+
domain: domain ?? null
|
|
12974
|
+
})?.n ?? 0);
|
|
12975
|
+
}
|
|
12267
12976
|
/** Rebuild the FTS index from the content table (write-path maintenance). */
|
|
12268
12977
|
rebuildFts() {
|
|
12269
12978
|
this.db.exec("INSERT INTO doc_chunks_fts(doc_chunks_fts) VALUES('rebuild')");
|
|
12270
12979
|
}
|
|
12271
12980
|
/** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
|
|
12272
12981
|
/**
|
|
12273
|
-
* Entity-overlap candidates,
|
|
12982
|
+
* Entity-overlap candidates, highest-score first and CAPPED (see the memory store's sibling;
|
|
12983
|
+
* the score/order tail itself lives in `./shared.js#entityCandidateTail`).
|
|
12274
12984
|
*
|
|
12275
12985
|
* `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
|
|
12276
12986
|
* `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
|
|
@@ -12280,11 +12990,19 @@ var ChunksDao = class {
|
|
|
12280
12990
|
* of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
|
|
12281
12991
|
*/
|
|
12282
12992
|
candidatesByEntityNames(names, domain, source, limit, queryWidth, widthCap) {
|
|
12283
|
-
|
|
12284
|
-
|
|
12285
|
-
|
|
12286
|
-
|
|
12287
|
-
|
|
12993
|
+
return entityCandidates(this.db, {
|
|
12994
|
+
names,
|
|
12995
|
+
filterParams: [
|
|
12996
|
+
domain ?? null,
|
|
12997
|
+
domain ?? null,
|
|
12998
|
+
source ?? null,
|
|
12999
|
+
source ?? null
|
|
13000
|
+
],
|
|
13001
|
+
limit,
|
|
13002
|
+
queryWidth,
|
|
13003
|
+
widthCap,
|
|
13004
|
+
batchNames: true,
|
|
13005
|
+
sql: (placeholders) => `SELECT ce.chunk_id AS id, COUNT(*) AS shared,
|
|
12288
13006
|
(SELECT COUNT(*) FROM chunk_entities x WHERE x.chunk_id = ce.chunk_id) AS total
|
|
12289
13007
|
FROM chunk_entities ce
|
|
12290
13008
|
JOIN doc_chunks dc ON dc.chunk_id = ce.chunk_id
|
|
@@ -12295,12 +13013,8 @@ var ChunksDao = class {
|
|
|
12295
13013
|
-- removed document's chunks out of a result is the JOIN itself.
|
|
12296
13014
|
WHERE ce.name IN (${placeholders})
|
|
12297
13015
|
AND (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?)
|
|
12298
|
-
|
|
12299
|
-
|
|
12300
|
-
LIMIT ?`).all(...values, domain ?? null, domain ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit);
|
|
12301
|
-
for (const r of rows) out.add(r.chunk_id);
|
|
12302
|
-
}
|
|
12303
|
-
return [...out];
|
|
13016
|
+
${entityCandidateTail("ce.chunk_id")}`
|
|
13017
|
+
});
|
|
12304
13018
|
}
|
|
12305
13019
|
/**
|
|
12306
13020
|
* How many chunks carry each name — the knowledge side of the entity leg's document frequency.
|
|
@@ -12310,12 +13024,10 @@ var ChunksDao = class {
|
|
|
12310
13024
|
* all chunks, which is the population the leg ranks.
|
|
12311
13025
|
*/
|
|
12312
13026
|
docFrequency(names) {
|
|
12313
|
-
|
|
12314
|
-
|
|
12315
|
-
|
|
12316
|
-
|
|
12317
|
-
for (const row of rows) out.set(row.name, Number(row.df));
|
|
12318
|
-
return out;
|
|
13027
|
+
return queryDocFrequency(this.db, names, {
|
|
13028
|
+
from: " FROM chunk_entities",
|
|
13029
|
+
nameExpr: "name"
|
|
13030
|
+
});
|
|
12319
13031
|
}
|
|
12320
13032
|
/** FTS leg (`bm25` rank; negated by the caller so higher = better). */
|
|
12321
13033
|
ftsSearch(ftsQuery, domain, source, limit) {
|
|
@@ -12335,14 +13047,17 @@ var ChunksDao = class {
|
|
|
12335
13047
|
* the domain/source filter lives.
|
|
12336
13048
|
*/
|
|
12337
13049
|
ftsSubstringSearch(terms, domain, source, limit) {
|
|
12338
|
-
|
|
12339
|
-
if (params.length === 0) return [];
|
|
12340
|
-
return this.db.prepare(`SELECT dc.chunk_id AS id, ${count} AS rank
|
|
13050
|
+
return likeSubstringLeg(this.db, "dc.text", terms, ({ any, count }) => `SELECT dc.chunk_id AS id, ${count} AS rank
|
|
12341
13051
|
FROM doc_chunks dc
|
|
12342
13052
|
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12343
13053
|
WHERE (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?) AND (${any})
|
|
12344
13054
|
ORDER BY rank DESC, dc.chunk_id ASC
|
|
12345
|
-
LIMIT
|
|
13055
|
+
LIMIT ?`, [
|
|
13056
|
+
domain ?? null,
|
|
13057
|
+
domain ?? null,
|
|
13058
|
+
source ?? null,
|
|
13059
|
+
source ?? null
|
|
13060
|
+
], limit);
|
|
12346
13061
|
}
|
|
12347
13062
|
};
|
|
12348
13063
|
//#endregion
|
|
@@ -13649,6 +14364,15 @@ var KnowledgeStore = class {
|
|
|
13649
14364
|
/** > 0 while a batch (directory import) is writing: commits and baseline refreshes wait. */
|
|
13650
14365
|
deferCorpusWrites = 0;
|
|
13651
14366
|
/**
|
|
14367
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
14368
|
+
*
|
|
14369
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It exists
|
|
14370
|
+
* as a field — rather than the adapter object, which is built per call — because the guard must
|
|
14371
|
+
* survive between two overlapping calls: `reindex` can be reached from a tool call and the
|
|
14372
|
+
* settings-page button at once, and both would select the same stale chunks.
|
|
14373
|
+
*/
|
|
14374
|
+
entitySweepGuard = {};
|
|
14375
|
+
/**
|
|
13652
14376
|
* Per-file stamps (`path → mtimeMs:size`) and the file-set they were taken from, so a reconcile
|
|
13653
14377
|
* can ask "what changed?" for the price of one `stat` per document instead of reading and
|
|
13654
14378
|
* hashing the whole corpus. `null` = no baseline yet.
|
|
@@ -13696,37 +14420,27 @@ var KnowledgeStore = class {
|
|
|
13696
14420
|
this.reloadIndex();
|
|
13697
14421
|
}
|
|
13698
14422
|
/**
|
|
13699
|
-
* Rebuild the in-memory vstore from persisted chunk vectors
|
|
14423
|
+
* Rebuild the in-memory vstore from persisted chunk vectors, and say loudly what belongs to
|
|
14424
|
+
* another vector space.
|
|
14425
|
+
*
|
|
14426
|
+
* Also detects a VECTOR-SPACE change, which the reload itself cannot see: the reload checks the
|
|
14427
|
+
* width, and a different model at the same width produces dim-valid vectors that are then ranked
|
|
14428
|
+
* against the current query's — two spaces mixed in one index, with no error anywhere.
|
|
13700
14429
|
*
|
|
13701
|
-
*
|
|
13702
|
-
*
|
|
13703
|
-
*
|
|
13704
|
-
*
|
|
13705
|
-
*
|
|
14430
|
+
* The reload, the count and the warning are the SHARED open-time step (`store/vector_repair.ts`),
|
|
14431
|
+
* which is what turned the knowledge store from "warns about a swapped model but never repairs it"
|
|
14432
|
+
* into the same automatic path memory has always had; this store's half is the row read. The
|
|
14433
|
+
* manual entry stays `vectors --fix` (the vector-only repair); `kb_reindex` remains the full
|
|
14434
|
+
* FTS + entities + vectors re-entry.
|
|
13706
14435
|
*/
|
|
13707
14436
|
reloadIndex() {
|
|
13708
|
-
|
|
13709
|
-
|
|
13710
|
-
|
|
13711
|
-
|
|
13712
|
-
|
|
13713
|
-
|
|
13714
|
-
|
|
13715
|
-
for (const row of persisted) {
|
|
13716
|
-
if (!row.vec || row.vec.byteLength !== expectedBytes) {
|
|
13717
|
-
stale++;
|
|
13718
|
-
continue;
|
|
13719
|
-
}
|
|
13720
|
-
if (row.embedding_model !== space) {
|
|
13721
|
-
spaceStale++;
|
|
13722
|
-
if (isPreFingerprintSpace(row.embedding_model)) legacy++;
|
|
13723
|
-
}
|
|
13724
|
-
}
|
|
13725
|
-
reportStaleVectors("knowledge", space, {
|
|
13726
|
-
stale,
|
|
13727
|
-
space_stale: spaceStale,
|
|
13728
|
-
legacy
|
|
13729
|
-
}, "`avantf-mem kb reindex` (MCP/plugin: `kb_reindex`)");
|
|
14437
|
+
loadVectorIndex({
|
|
14438
|
+
store: "knowledge",
|
|
14439
|
+
vstore: this.vstore,
|
|
14440
|
+
rows: this.chunks.vectorRows(),
|
|
14441
|
+
space: this.vectorSpace(),
|
|
14442
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)"
|
|
14443
|
+
});
|
|
13730
14444
|
}
|
|
13731
14445
|
/**
|
|
13732
14446
|
* The write-side domain allowlist AND the 知识页 picker's data, in one place: the configured
|
|
@@ -14005,7 +14719,7 @@ var KnowledgeStore = class {
|
|
|
14005
14719
|
reusable
|
|
14006
14720
|
};
|
|
14007
14721
|
}).immediate();
|
|
14008
|
-
this.
|
|
14722
|
+
evictVectors(this.vstore, result.oldChunkIds);
|
|
14009
14723
|
const indexed = await this.indexChunks(result.newChunkIds, result.reusable);
|
|
14010
14724
|
return {
|
|
14011
14725
|
doc_id: result.doc_id,
|
|
@@ -14042,10 +14756,6 @@ var KnowledgeStore = class {
|
|
|
14042
14756
|
return { file_error: error instanceof Error ? error.message : String(error) };
|
|
14043
14757
|
}
|
|
14044
14758
|
}
|
|
14045
|
-
/** Batched eviction from the live vector index (removeMany when the backend supports it). */
|
|
14046
|
-
evictVectors(ids) {
|
|
14047
|
-
evictVectors(this.vstore, ids);
|
|
14048
|
-
}
|
|
14049
14759
|
/** The resolved ingestion boundary (see `ingest_guard.ts`). */
|
|
14050
14760
|
get ingestLimits() {
|
|
14051
14761
|
return this.kbConfig.ingest;
|
|
@@ -14168,7 +14878,8 @@ var KnowledgeStore = class {
|
|
|
14168
14878
|
};
|
|
14169
14879
|
}
|
|
14170
14880
|
/**
|
|
14171
|
-
*
|
|
14881
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
14882
|
+
* call, the live index and the `WRITE_BATCH` transaction this store measured its way into.
|
|
14172
14883
|
*
|
|
14173
14884
|
* The model call is deliberately NOT batched, and that is a MEASURED decision, not an omission:
|
|
14174
14885
|
* a batched forward pass pads each item to the longest in its group, and real chunks span 53–550
|
|
@@ -14179,34 +14890,43 @@ var KnowledgeStore = class {
|
|
|
14179
14890
|
* length-sorted) for callers whose texts suit it.
|
|
14180
14891
|
*
|
|
14181
14892
|
* What DID pay off is the write side: one commit per `WRITE_BATCH` chunks instead of one per
|
|
14182
|
-
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await
|
|
14893
|
+
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await` — that
|
|
14894
|
+
* is the `writeBatch` below.
|
|
14183
14895
|
*/
|
|
14184
|
-
|
|
14185
|
-
|
|
14186
|
-
|
|
14187
|
-
|
|
14188
|
-
|
|
14189
|
-
|
|
14190
|
-
|
|
14191
|
-
|
|
14192
|
-
|
|
14193
|
-
writes.
|
|
14194
|
-
chunk_id:
|
|
14195
|
-
bytes:
|
|
14896
|
+
encodeWriteTarget(space) {
|
|
14897
|
+
return {
|
|
14898
|
+
store: "knowledge",
|
|
14899
|
+
space,
|
|
14900
|
+
encode: (text) => this.semantic.encode(text),
|
|
14901
|
+
add: (id, vec) => {
|
|
14902
|
+
this.vstore.add(id, vec);
|
|
14903
|
+
},
|
|
14904
|
+
write: (writes) => {
|
|
14905
|
+
this.chunks.setVectors(writes.map((w) => ({
|
|
14906
|
+
chunk_id: w.id,
|
|
14907
|
+
bytes: w.bytes,
|
|
14196
14908
|
embeddingModel: space
|
|
14197
|
-
});
|
|
14198
|
-
}
|
|
14199
|
-
|
|
14200
|
-
|
|
14201
|
-
|
|
14202
|
-
try {
|
|
14203
|
-
this.chunks.setVectors(writes);
|
|
14204
|
-
encoded += writes.length;
|
|
14205
|
-
} catch {
|
|
14206
|
-
failed += writes.length;
|
|
14207
|
-
this.evictVectors(writes.map((w) => w.chunk_id));
|
|
14909
|
+
})));
|
|
14910
|
+
},
|
|
14911
|
+
writeBatch: WRITE_BATCH,
|
|
14912
|
+
onWriteError: (writes) => {
|
|
14913
|
+
evictVectors(this.vstore, writes.map((w) => w.id));
|
|
14208
14914
|
}
|
|
14209
|
-
}
|
|
14915
|
+
};
|
|
14916
|
+
}
|
|
14917
|
+
/**
|
|
14918
|
+
* Encode texts one at a time and persist them in `WRITE_BATCH` transactions.
|
|
14919
|
+
*
|
|
14920
|
+
* A failed chunk is skipped, not thrown: the rows are already committed and their old vectors
|
|
14921
|
+
* evicted by the ingest path, so a throw here would leave the caller with a failed ingest and NO
|
|
14922
|
+
* COUNT of what was skipped. The shortfall travels back to the tool result
|
|
14923
|
+
* (`IngestResult.vectors_failed`) instead of living only in a log line.
|
|
14924
|
+
*/
|
|
14925
|
+
async encodeAndStore(rows, space) {
|
|
14926
|
+
const { encoded, failed } = await encodeAndPersist(rows.map((r) => ({
|
|
14927
|
+
id: r.chunk_id,
|
|
14928
|
+
text: r.text
|
|
14929
|
+
})), this.encodeWriteTarget(space));
|
|
14210
14930
|
if (failed > 0) retrievalLogger().warn(`kb index: ${String(failed)} of ${String(rows.length)} chunk vector(s) were not written — run kb_reindex to retry (the text/entity/FTS indexes are complete; only the semantic vectors are affected)`);
|
|
14211
14931
|
return {
|
|
14212
14932
|
encoded,
|
|
@@ -14223,6 +14943,102 @@ var KnowledgeStore = class {
|
|
|
14223
14943
|
return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
|
|
14224
14944
|
}
|
|
14225
14945
|
/**
|
|
14946
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of
|
|
14947
|
+
* persisted chunk vectors whose width or recorded space id no longer matches the one this store
|
|
14948
|
+
* writes into. Reads blob LENGTHS only — never decodes — so the startup warning and the background
|
|
14949
|
+
* drive can afford it at any corpus size.
|
|
14950
|
+
*/
|
|
14951
|
+
vectorSpaceHealth() {
|
|
14952
|
+
const c = this.vectorSpaceCounts();
|
|
14953
|
+
return {
|
|
14954
|
+
stale: c.stale,
|
|
14955
|
+
space_stale: c.space_stale
|
|
14956
|
+
};
|
|
14957
|
+
}
|
|
14958
|
+
/** {@link vectorSpaceHealth} plus `missing`, the third term of the repair flow's `remaining`. */
|
|
14959
|
+
vectorSpaceCounts() {
|
|
14960
|
+
return this.chunks.vectorSpaceCounts(this.vectorSpace(), this.vstore.dim * 4);
|
|
14961
|
+
}
|
|
14962
|
+
/** Async bootstrap the semantic backend (the shared flow's explicit-fix warmup). */
|
|
14963
|
+
async warmupSemantic() {
|
|
14964
|
+
await (this.semantic.warmUp?.() ?? Promise.resolve());
|
|
14965
|
+
return this.semantic.isAvailable();
|
|
14966
|
+
}
|
|
14967
|
+
/**
|
|
14968
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
14969
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
14970
|
+
* `semantic.auto_migrate`, the dry-run split — lives there once, for both stores.
|
|
14971
|
+
*/
|
|
14972
|
+
vectorRepairTarget() {
|
|
14973
|
+
return {
|
|
14974
|
+
store: "knowledge",
|
|
14975
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)",
|
|
14976
|
+
vectorSpace: () => this.vectorSpace(),
|
|
14977
|
+
semanticAvailable: () => this.semantic.isAvailable(),
|
|
14978
|
+
autoMigrate: () => this.config.semantic.auto_migrate,
|
|
14979
|
+
vectorSpaceHealth: () => this.vectorSpaceHealth(),
|
|
14980
|
+
missingVectorCount: () => this.chunks.countMissingVectors(),
|
|
14981
|
+
classifyVectors: () => this.classifyRepairVectors(),
|
|
14982
|
+
dropVectors: () => 0,
|
|
14983
|
+
reencodeVectors: (limit) => this.reencodeStaleChunks(limit),
|
|
14984
|
+
warmup: () => this.warmupSemantic()
|
|
14985
|
+
};
|
|
14986
|
+
}
|
|
14987
|
+
/**
|
|
14988
|
+
* The chunks the semantic leg cannot use (no vector / wrong width / another space). One SQL
|
|
14989
|
+
* aggregate over blob lengths; `dropIds` is empty because knowledge overwrites in place.
|
|
14990
|
+
*/
|
|
14991
|
+
classifyRepairVectors() {
|
|
14992
|
+
const c = this.vectorSpaceCounts();
|
|
14993
|
+
return {
|
|
14994
|
+
stale: c.stale,
|
|
14995
|
+
space_stale: c.space_stale,
|
|
14996
|
+
missing: c.missing,
|
|
14997
|
+
dropIds: []
|
|
14998
|
+
};
|
|
14999
|
+
}
|
|
15000
|
+
/**
|
|
15001
|
+
* Encode + persist up to `limit` chunks the semantic leg cannot use, bounded and ordered by id.
|
|
15002
|
+
*
|
|
15003
|
+
* It does NOT re-extract entities or rebuild FTS: that is `kb_reindex`'s job, and the whole point
|
|
15004
|
+
* of the automatic repair is that the vector half can run on its own — which is exactly the
|
|
15005
|
+
* objection the "known boundary" in DESIGN §20 used to raise against a background run.
|
|
15006
|
+
*/
|
|
15007
|
+
async reencodeStaleChunks(limit) {
|
|
15008
|
+
const space = this.vectorSpace();
|
|
15009
|
+
const rows = this.chunks.staleVectorRows(space, this.vstore.dim * 4, limit);
|
|
15010
|
+
if (!rows.length) return {
|
|
15011
|
+
encoded: 0,
|
|
15012
|
+
failed: 0
|
|
15013
|
+
};
|
|
15014
|
+
return await this.encodeAndStore(rows, space);
|
|
15015
|
+
}
|
|
15016
|
+
/**
|
|
15017
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
15018
|
+
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
15019
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
15020
|
+
*/
|
|
15021
|
+
async vectorsFix(dryRun = false) {
|
|
15022
|
+
return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
|
|
15023
|
+
}
|
|
15024
|
+
/**
|
|
15025
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` chunks are re-encoded, so
|
|
15026
|
+
* the caller keeps control of the event loop between slices. This is the unit the background
|
|
15027
|
+
* migration ({@link migrateVectors}) and its tests drive.
|
|
15028
|
+
*/
|
|
15029
|
+
async migrateVectorsBatch(batchSize = 16) {
|
|
15030
|
+
return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
|
|
15031
|
+
}
|
|
15032
|
+
/**
|
|
15033
|
+
* Drive the bounded background migration for this store until it is current, the model is
|
|
15034
|
+
* unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the
|
|
15035
|
+
* `semantic.auto_migrate` switch, the no-progress rule and the store-named logging all live in the
|
|
15036
|
+
* shared flow (`store/vector_repair.ts`); this is the one-store entry.
|
|
15037
|
+
*/
|
|
15038
|
+
async migrateVectors(opts = {}) {
|
|
15039
|
+
return await driveVectorRepair(this.vectorRepairTarget(), opts);
|
|
15040
|
+
}
|
|
15041
|
+
/**
|
|
14226
15042
|
* A stored vector is reusable when it exists, matches the space, and its text is unchanged.
|
|
14227
15043
|
*
|
|
14228
15044
|
* The three checks answer different questions, and the hash is deliberately the RAREST of them
|
|
@@ -14236,17 +15052,43 @@ var KnowledgeStore = class {
|
|
|
14236
15052
|
if (row.embedding_model !== space) return false;
|
|
14237
15053
|
return row.content_hash === contentHash(row.text);
|
|
14238
15054
|
}
|
|
14239
|
-
/**
|
|
14240
|
-
|
|
15055
|
+
/**
|
|
15056
|
+
* Extract one chunk's entity names, bounded by `MAX_ENTITIES_PER_CHUNK`.
|
|
15057
|
+
*
|
|
15058
|
+
* The bound is here, not in either caller: it is a property of what `chunk_entities` may hold for
|
|
15059
|
+
* one chunk, and the ingest path and the sweep must not disagree about it.
|
|
15060
|
+
*/
|
|
15061
|
+
async extractRowEntities(text) {
|
|
15062
|
+
return (await extractEntities(text)).map((e) => e.name).slice(0, MAX_ENTITIES_PER_CHUNK);
|
|
15063
|
+
}
|
|
15064
|
+
/** One transaction replacing the entity rows of the given chunks (the ingest path). */
|
|
15065
|
+
async replaceChunkEntities(rows) {
|
|
14241
15066
|
const extracted = [];
|
|
14242
|
-
for (const r of rows) {
|
|
14243
|
-
|
|
14244
|
-
|
|
14245
|
-
|
|
14246
|
-
|
|
14247
|
-
|
|
14248
|
-
|
|
14249
|
-
|
|
15067
|
+
for (const r of rows) extracted.push({
|
|
15068
|
+
chunk_id: r.chunk_id,
|
|
15069
|
+
names: await this.extractRowEntities(r.text)
|
|
15070
|
+
});
|
|
15071
|
+
this.chunks.replaceEntities(extracted, false);
|
|
15072
|
+
}
|
|
15073
|
+
/**
|
|
15074
|
+
* The per-row primitive of the shared entity sweep (`store/entity_sweep.ts`).
|
|
15075
|
+
*
|
|
15076
|
+
* Replaces ONE chunk's entity rows and stamps that same row with `ENTITY_EXTRACTOR_VERSION`, in one
|
|
15077
|
+
* transaction. Knowledge's domain difference from memory is exactly this: memory's transaction also
|
|
15078
|
+
* moves triples, the HRR bundle and the conflict queue, while a chunk has none of those — its
|
|
15079
|
+
* derived state is the entity names alone.
|
|
15080
|
+
*
|
|
15081
|
+
* Per row, not per batch, because the sweep's contract is that a failure leaves THAT row stale and
|
|
15082
|
+
* retryable: inside a batched transaction one bad extraction would roll back the whole batch, and
|
|
15083
|
+
* rows it never reached would look stamped.
|
|
15084
|
+
*/
|
|
15085
|
+
async replaceRowEntities(chunkId, text) {
|
|
15086
|
+
const names = await this.extractRowEntities(text);
|
|
15087
|
+
this.chunks.replaceEntities([{
|
|
15088
|
+
chunk_id: chunkId,
|
|
15089
|
+
names
|
|
15090
|
+
}], true);
|
|
15091
|
+
this.chunks.setEntitiesVersion([chunkId], 1);
|
|
14250
15092
|
}
|
|
14251
15093
|
/**
|
|
14252
15094
|
* Ingest local files/directories. Every path goes through the SAME boundary as
|
|
@@ -14330,7 +15172,7 @@ var KnowledgeStore = class {
|
|
|
14330
15172
|
const chunkIds = this.chunks.idsForDoc(docId);
|
|
14331
15173
|
const removed = this.docs.remove(docId);
|
|
14332
15174
|
if (removed) this.commitCorpus(`remove: ${doc.domain}/${doc.source}/${doc.title}`);
|
|
14333
|
-
this.
|
|
15175
|
+
evictVectors(this.vstore, chunkIds);
|
|
14334
15176
|
return removed;
|
|
14335
15177
|
}
|
|
14336
15178
|
/**
|
|
@@ -14522,12 +15364,11 @@ var KnowledgeStore = class {
|
|
|
14522
15364
|
async reindex(domain, opts) {
|
|
14523
15365
|
const dryRun = opts?.dryRun === true;
|
|
14524
15366
|
const space = this.vectorSpace();
|
|
14525
|
-
const
|
|
14526
|
-
const
|
|
14527
|
-
const staleVectors = rows.filter((r) => !this.vectorReusable(r, space));
|
|
15367
|
+
const corpus = this.chunks.corpusState(domain);
|
|
15368
|
+
const staleVectors = corpus.filter((r) => !this.vectorReusable(r, space));
|
|
14528
15369
|
const semanticAvailable = this.semantic.isAvailable();
|
|
14529
15370
|
const report = {
|
|
14530
|
-
chunks:
|
|
15371
|
+
chunks: corpus.length,
|
|
14531
15372
|
entities_rebuilt: 0,
|
|
14532
15373
|
vectors_stale: staleVectors.length,
|
|
14533
15374
|
vectors_encoded: 0,
|
|
@@ -14536,11 +15377,19 @@ var KnowledgeStore = class {
|
|
|
14536
15377
|
};
|
|
14537
15378
|
if (dryRun) return report;
|
|
14538
15379
|
this.chunks.rebuildFts();
|
|
14539
|
-
|
|
14540
|
-
|
|
14541
|
-
|
|
14542
|
-
|
|
14543
|
-
|
|
15380
|
+
report.entities_rebuilt = (await sweepVersionedEntities({
|
|
15381
|
+
staleEntityBatch: (limit) => ({
|
|
15382
|
+
total: this.chunks.countStaleEntityChunks(1, domain),
|
|
15383
|
+
rows: this.chunks.chunkEntityRows(1, limit, domain)
|
|
15384
|
+
}),
|
|
15385
|
+
staleEntityCount: () => this.chunks.countStaleEntityChunks(1, domain),
|
|
15386
|
+
extractAndWriteEntities: async (row) => {
|
|
15387
|
+
await this.replaceRowEntities(row.chunk_id, row.text);
|
|
15388
|
+
},
|
|
15389
|
+
onRowError: (row, error) => {
|
|
15390
|
+
retrievalLogger().warn(`kb entity sweep: chunk ${String(row.chunk_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next reindex retries it`);
|
|
15391
|
+
}
|
|
15392
|
+
}, this.entitySweepGuard, { budget: this.chunks.countStaleEntityChunks(1, domain) })).rebuilt;
|
|
14544
15393
|
if (!semanticAvailable) this.semantic.ensureWarm?.();
|
|
14545
15394
|
else if (staleVectors.length) report.vectors_encoded = (await this.encodeAndStore(staleVectors, space)).encoded;
|
|
14546
15395
|
return report;
|
|
@@ -14556,18 +15405,7 @@ var KnowledgeStore = class {
|
|
|
14556
15405
|
}
|
|
14557
15406
|
/** Chunk search over the knowledge corpus (semantic + FTS + entity, joint fusion). */
|
|
14558
15407
|
async search(query, opts) {
|
|
14559
|
-
const result = await
|
|
14560
|
-
query,
|
|
14561
|
-
limit: opts?.limit,
|
|
14562
|
-
overFetch: opts?.overFetch,
|
|
14563
|
-
maxTokens: opts?.maxTokens,
|
|
14564
|
-
queryVector: opts?.queryVector,
|
|
14565
|
-
recordStats: opts?.recordStats,
|
|
14566
|
-
floors: opts?.floors,
|
|
14567
|
-
...opts?.includeScores === void 0 ? {} : { includeScores: opts.includeScores },
|
|
14568
|
-
...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs },
|
|
14569
|
-
...opts?.rewriteQuery === void 0 ? {} : { rewriteQuery: opts.rewriteQuery }
|
|
14570
|
-
});
|
|
15408
|
+
const result = await runHybridSearch(this.hybridDeps(opts), query, opts);
|
|
14571
15409
|
opts?.onResult?.(result);
|
|
14572
15410
|
return result.hits;
|
|
14573
15411
|
}
|
|
@@ -14583,10 +15421,7 @@ var KnowledgeStore = class {
|
|
|
14583
15421
|
*/
|
|
14584
15422
|
hybridDeps(opts) {
|
|
14585
15423
|
const rowsById = /* @__PURE__ */ new Map();
|
|
14586
|
-
return {
|
|
14587
|
-
kind: "knowledge",
|
|
14588
|
-
config: this.config,
|
|
14589
|
-
semantic: this.semantic,
|
|
15424
|
+
return makeHybridDeps("knowledge", this.config, this.semantic, {
|
|
14590
15425
|
legs: (ctx) => this.searchLegs(opts, ctx),
|
|
14591
15426
|
texts: (ids) => {
|
|
14592
15427
|
rowsById.clear();
|
|
@@ -14612,7 +15447,7 @@ var KnowledgeStore = class {
|
|
|
14612
15447
|
}];
|
|
14613
15448
|
});
|
|
14614
15449
|
}
|
|
14615
|
-
};
|
|
15450
|
+
});
|
|
14616
15451
|
}
|
|
14617
15452
|
/**
|
|
14618
15453
|
* The knowledge legs: semantic, FTS, and entity overlap against the entities extracted AT INGEST
|
|
@@ -14622,40 +15457,12 @@ var KnowledgeStore = class {
|
|
|
14622
15457
|
* matching corpus only makes `fuse` normalize and sort it (see `MemoryStore.searchLegs`).
|
|
14623
15458
|
*/
|
|
14624
15459
|
async searchLegs(opts, ctx) {
|
|
14625
|
-
|
|
14626
|
-
* Wrap one leg's floored scores. `raw` is the PRE-floor set: `capped` must be measured there,
|
|
14627
|
-
* because the relevance floor removes the tail anyway and re-deriving the flag afterwards would
|
|
14628
|
-
* erase the "cut at legCap" signal exactly when it bound.
|
|
14629
|
-
*/
|
|
14630
|
-
const leg = (scores, weight, name, dropped, raw) => ({
|
|
14631
|
-
weight,
|
|
14632
|
-
scores,
|
|
14633
|
-
capped: (raw ?? scores).size === ctx.legCap,
|
|
14634
|
-
leg: name,
|
|
14635
|
-
droppedByFloor: dropped
|
|
14636
|
-
});
|
|
15460
|
+
const runner = makeLegRunner(ctx);
|
|
14637
15461
|
const ftsRaw = this.ftsPath(ctx.query, opts?.domain, opts?.source, ctx.legCap);
|
|
14638
|
-
const ftsFloored = applyTermFloor(ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
|
|
14639
15462
|
return [
|
|
14640
|
-
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) =>
|
|
14641
|
-
|
|
14642
|
-
|
|
14643
|
-
weight: ctx.weights.semantic,
|
|
14644
|
-
scores: floored.scores,
|
|
14645
|
-
leg: "semantic",
|
|
14646
|
-
droppedByFloor: floored.dropped
|
|
14647
|
-
};
|
|
14648
|
-
}) : {
|
|
14649
|
-
weight: ctx.weights.semantic,
|
|
14650
|
-
scores: /* @__PURE__ */ new Map(),
|
|
14651
|
-
leg: "semantic",
|
|
14652
|
-
droppedByFloor: 0
|
|
14653
|
-
},
|
|
14654
|
-
leg(ftsFloored.scores, ctx.weights.fts, "fts", ftsFloored.dropped, ftsRaw),
|
|
14655
|
-
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => {
|
|
14656
|
-
const floored = applyScoreFloor(raw, ctx.floors.jaccard);
|
|
14657
|
-
return leg(floored.scores, ctx.weights.jaccard, "jaccard", floored.dropped, raw);
|
|
14658
|
-
})
|
|
15463
|
+
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
|
|
15464
|
+
runner.term("fts", ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts),
|
|
15465
|
+
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => runner.scored("jaccard", raw, ctx.weights.jaccard, ctx.floors.jaccard))
|
|
14659
15466
|
];
|
|
14660
15467
|
}
|
|
14661
15468
|
/** The texts of a candidate set, for the per-row FTS floor (one batched query). */
|
|
@@ -14670,43 +15477,55 @@ var KnowledgeStore = class {
|
|
|
14670
15477
|
* re-encoding (performance review §7.7 / P8).
|
|
14671
15478
|
*/
|
|
14672
15479
|
async semanticPath(query, k, opts, queryVector, onVector) {
|
|
14673
|
-
|
|
14674
|
-
|
|
14675
|
-
|
|
14676
|
-
|
|
14677
|
-
|
|
14678
|
-
|
|
14679
|
-
|
|
14680
|
-
|
|
14681
|
-
|
|
14682
|
-
|
|
15480
|
+
return semanticLeg({
|
|
15481
|
+
query,
|
|
15482
|
+
k,
|
|
15483
|
+
queryVector,
|
|
15484
|
+
onVector,
|
|
15485
|
+
encode: (q) => this.semantic.encode(q),
|
|
15486
|
+
dim: this.vstore.dim,
|
|
15487
|
+
topk: (vec, n) => this.vstore.topk(vec, n),
|
|
15488
|
+
filterTopk: (ids) => this.scopeFilter(ids, opts)
|
|
15489
|
+
});
|
|
15490
|
+
}
|
|
15491
|
+
/**
|
|
15492
|
+
* The semantic leg's scope filter, as a `filterTopk` primitive: `undefined` when NO domain/source
|
|
15493
|
+
* was asked means "keep the whole pool" — the short-circuit is preserved, so no `chunks.meta`
|
|
15494
|
+
* lookup runs at all. Otherwise one batched lookup, and rows whose domain/source do not match (or
|
|
15495
|
+
* whose chunk is gone) are dropped.
|
|
15496
|
+
*/
|
|
15497
|
+
scopeFilter(ids, opts) {
|
|
15498
|
+
if (!opts?.domain && !opts?.source) return void 0;
|
|
15499
|
+
const meta = new Map(this.chunks.meta(ids).map((r) => [r.chunk_id, r]));
|
|
15500
|
+
const keep = /* @__PURE__ */ new Set();
|
|
15501
|
+
for (const id of ids) {
|
|
15502
|
+
const r = meta.get(id);
|
|
14683
15503
|
if (!r) continue;
|
|
14684
15504
|
if (opts.domain && r.domain !== opts.domain) continue;
|
|
14685
15505
|
if (opts.source && r.source !== opts.source) continue;
|
|
14686
|
-
|
|
15506
|
+
keep.add(id);
|
|
14687
15507
|
}
|
|
14688
|
-
return
|
|
15508
|
+
return keep;
|
|
14689
15509
|
}
|
|
14690
15510
|
/**
|
|
14691
15511
|
* Sync relevance probe for the plugin's conditional hint (DESIGN §12). Same contract as
|
|
14692
15512
|
* `MemoryStore.lexicalProbe` — see `store/lexical.ts`.
|
|
14693
15513
|
*/
|
|
14694
15514
|
lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
|
|
14695
|
-
return
|
|
14696
|
-
const fts = buildFtsQuery(term, this.ftsTokenizer);
|
|
14697
|
-
return fts !== null && this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0;
|
|
14698
|
-
}, stopAt);
|
|
15515
|
+
return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0, stopAt);
|
|
14699
15516
|
}
|
|
14700
15517
|
ftsPath(query, domain, source, cap) {
|
|
14701
|
-
|
|
14702
|
-
|
|
14703
|
-
|
|
14704
|
-
|
|
14705
|
-
|
|
14706
|
-
|
|
14707
|
-
|
|
14708
|
-
|
|
14709
|
-
|
|
15518
|
+
return ftsLeg({
|
|
15519
|
+
query,
|
|
15520
|
+
cap,
|
|
15521
|
+
scope: {
|
|
15522
|
+
domain,
|
|
15523
|
+
source
|
|
15524
|
+
},
|
|
15525
|
+
tokenizer: this.ftsTokenizer,
|
|
15526
|
+
search: (ftsQuery, scope, limit) => this.chunks.ftsSearch(ftsQuery, scope.domain, scope.source, limit),
|
|
15527
|
+
substringSearch: (terms, scope, limit) => this.chunks.ftsSubstringSearch(terms, scope.domain, scope.source, limit)
|
|
15528
|
+
});
|
|
14710
15529
|
}
|
|
14711
15530
|
/**
|
|
14712
15531
|
* Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan.
|
|
@@ -15079,7 +15898,13 @@ function buildRuntime(opts) {
|
|
|
15079
15898
|
case "contradict_resolve": return this.memory.resolveContradiction(req.contradiction_id, req.resolution, req.loser_fact_id);
|
|
15080
15899
|
case "maintenance": return this.memory.maintenance();
|
|
15081
15900
|
case "vectors_diagnose": return this.memory.vectorsDiagnose();
|
|
15082
|
-
case "vectors_fix": return
|
|
15901
|
+
case "vectors_fix": return vectorsFix({
|
|
15902
|
+
memory: this.memory.vectorRepairTarget(),
|
|
15903
|
+
knowledge: this.knowledge.vectorRepairTarget()
|
|
15904
|
+
}, {
|
|
15905
|
+
dryRun: req.dry_run ?? false,
|
|
15906
|
+
...req.store === void 0 ? {} : { store: req.store }
|
|
15907
|
+
});
|
|
15083
15908
|
default: return { error: `admin.${req.action} not implemented` };
|
|
15084
15909
|
}
|
|
15085
15910
|
},
|
|
@@ -15331,6 +16156,7 @@ const descriptors = [
|
|
|
15331
16156
|
offset: count.optional(),
|
|
15332
16157
|
reason: optionalText.optional(),
|
|
15333
16158
|
dry_run: bool.optional(),
|
|
16159
|
+
store: optionalText.optional(),
|
|
15334
16160
|
contradiction_id: count.optional(),
|
|
15335
16161
|
resolution: optionalText.optional(),
|
|
15336
16162
|
loser_fact_id: count.optional()
|
|
@@ -16772,37 +17598,43 @@ function openDocumentPath(documentPath, target, configured, deps = {}) {
|
|
|
16772
17598
|
}
|
|
16773
17599
|
//#endregion
|
|
16774
17600
|
//#region lib/types/vectorMigration.js
|
|
17601
|
+
/**
|
|
17602
|
+
* The plugin's lifecycle for the bounded, resumable background migration of persisted vectors into
|
|
17603
|
+
* the current embedding space.
|
|
17604
|
+
*
|
|
17605
|
+
* Changing the embedding space (default model, width, pooling, normalization, input window, weights)
|
|
17606
|
+
* is a DATA MIGRATION, not a config tweak: every persisted vector still decodes, but in another
|
|
17607
|
+
* model's coordinates, so the semantic leg must not serve it. Measured on the real library after the
|
|
17608
|
+
* 512→768 default-model swap, 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval
|
|
17609
|
+
* degraded to lexical+entity with no error anywhere — the stores' open-time warning now says so, and
|
|
17610
|
+
* this module is what repairs it without the operator having to know `vectors --fix` exists.
|
|
17611
|
+
*
|
|
17612
|
+
* The FLOW lives in the engine, once, for both stores (`@avantf/mem`'s `store/vector_repair.ts`:
|
|
17613
|
+
* batch size, event-loop yield, stop / resume / no-progress rules, the `semantic.auto_migrate`
|
|
17614
|
+
* decision and the four store-named log kinds). It is driven for BOTH stores — memory and knowledge
|
|
17615
|
+
* — in one pass, because one embedding space is configured for the whole runtime: the knowledge
|
|
17616
|
+
* store used to be left behind, silently, with only a manual `kb_reindex` able to fix it.
|
|
17617
|
+
*
|
|
17618
|
+
* This module owns only the process LIFECYCLE: one start, one stop, and "never an unhandled
|
|
17619
|
+
* rejection". It sits BEHIND the startup gate (`awaitStartupGate`, see index.ts): the model warm and
|
|
17620
|
+
* this first batch must not land in the boot window (DESIGN §20.14).
|
|
17621
|
+
*
|
|
17622
|
+
* Deliberately no state: what still needs migrating is derived from the databases on every pass —
|
|
17623
|
+
* and the adapters are created ONCE so the shared flow's once-per-process `auto_migrate` warning is
|
|
17624
|
+
* genuinely once per process, not once per heartbeat.
|
|
17625
|
+
*/
|
|
16775
17626
|
function createVectorMigration(opts) {
|
|
16776
17627
|
const { rt, logger } = opts;
|
|
17628
|
+
const targets = [rt.memory.vectorRepairTarget(), rt.knowledge.vectorRepairTarget()];
|
|
16777
17629
|
let running = false;
|
|
16778
17630
|
let stopped = false;
|
|
16779
|
-
let warnedDisabled = false;
|
|
16780
17631
|
const run = async () => {
|
|
16781
|
-
const health = rt.memory.vectorSpaceHealth();
|
|
16782
|
-
const pending = health.stale + health.space_stale;
|
|
16783
|
-
if (pending === 0) return;
|
|
16784
17632
|
if (rt.closed || !opts.isActive()) return;
|
|
16785
|
-
|
|
16786
|
-
if (!warnedDisabled) {
|
|
16787
|
-
warnedDisabled = true;
|
|
16788
|
-
logger.warn(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run \`avantf-mem vectors --fix\` (MCP/plugin: \`mem_admin vectors_fix\`) or set \`semantic.auto_migrate: true\``);
|
|
16789
|
-
}
|
|
16790
|
-
return;
|
|
16791
|
-
}
|
|
16792
|
-
const startedAt = Date.now();
|
|
16793
|
-
logger.info(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)`);
|
|
16794
|
-
const outcome = await rt.memory.migrateVectors({
|
|
17633
|
+
await driveVectorRepairs(targets, {
|
|
16795
17634
|
...opts.batchSize === void 0 ? {} : { batchSize: opts.batchSize },
|
|
16796
17635
|
shouldStop: () => stopped || rt.closed || !opts.isActive(),
|
|
16797
|
-
|
|
16798
|
-
logger.info(`vector migration: re-encoded ${String(progress.migrated)}/${String(pending)} (${String(progress.remaining)} left)`);
|
|
16799
|
-
}
|
|
17636
|
+
log: logger
|
|
16800
17637
|
});
|
|
16801
|
-
if (outcome.remaining === 0) {
|
|
16802
|
-
logger.info(`vector migration: complete — ${String(outcome.migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
|
|
16803
|
-
return;
|
|
16804
|
-
}
|
|
16805
|
-
if (!stopped && !rt.closed && opts.isActive()) logger.warn(`vector migration: ${String(outcome.remaining)} vector(s) still belong to an older embedding space (the embedding model may be unavailable) — the next heartbeat retries; manual entry \`avantf-mem vectors --fix\``);
|
|
16806
17638
|
};
|
|
16807
17639
|
return {
|
|
16808
17640
|
start() {
|