@avantf/dsh-mem 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -708,7 +708,8 @@ const AdminUnion = z.discriminatedUnion("action", [
708
708
  z.object({ action: z.literal("vectors_diagnose") }),
709
709
  z.object({
710
710
  action: z.literal("vectors_fix"),
711
- dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。")
711
+ dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。"),
712
+ store: z.enum(["memory", "knowledge"]).optional().describe("只修某一个库(memory / knowledge);省略=两库都修。")
712
713
  }),
713
714
  z.object({ action: z.literal("contradict_check") }),
714
715
  z.object({
@@ -6870,6 +6871,18 @@ function probeTerms(text, holds, stopAt = Number.POSITIVE_INFINITY) {
6870
6871
  matched
6871
6872
  };
6872
6873
  }
6874
+ /**
6875
+ * The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
6876
+ * expression for one term (the tokenizer is a property of the store's FTS table) and ask that
6877
+ * table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
6878
+ * one part that has to differ, which is now a single `holds` closure per store.
6879
+ */
6880
+ function probeStoreTerms(text, tokenizer, holds, stopAt = Number.POSITIVE_INFINITY) {
6881
+ return probeTerms(text, (term) => {
6882
+ const fts = buildFtsQuery(term, tokenizer);
6883
+ return fts !== null && holds(fts);
6884
+ }, stopAt);
6885
+ }
6873
6886
  //#endregion
6874
6887
  //#region ../core/lib/store/self_query.js
6875
6888
  /**
@@ -7393,6 +7406,60 @@ async function hybridSearch(deps, plan) {
7393
7406
  };
7394
7407
  }
7395
7408
  /**
7409
+ * Map a store's search arguments onto {@link HybridPlan} and run {@link hybridSearch}. ONE copy.
7410
+ *
7411
+ * Before this, `MemoryStore.search` and `KnowledgeStore.search` carried the mapping verbatim side by
7412
+ * side: the same field list, the same conditional spreads, the same call. That list is part of the
7413
+ * orchestrator's contract, so an option added to one entry point and not the other would silently
7414
+ * make the two stores query differently — the same shape of drift this module already documents
7415
+ * (`NaN` limit, `over_fetch_factor`, unrecorded capped legs).
7416
+ *
7417
+ * The conditional spreads are deliberate, not cosmetic: a caller that did not pass `overFetch` /
7418
+ * `includeScores` / `relaxLegs` / `rewriteQuery` leaves the plan key ABSENT rather than
7419
+ * present-with-`undefined`. The orchestrator reads all four through `??` / `=== true` today, so the
7420
+ * two are indistinguishable now — but the plan is a contract, and widening it with keys the caller
7421
+ * never set is how "absent means default" decays into "present-undefined means set to nothing".
7422
+ */
7423
+ async function runHybridSearch(deps, query, fields) {
7424
+ return hybridSearch(deps, hybridPlanOf(query, fields));
7425
+ }
7426
+ /**
7427
+ * The mapping itself, separated from the call so it can be pinned directly: the contract here is
7428
+ * the SHAPE of the plan (which keys exist), which no behavioural assertion inside `hybridSearch`
7429
+ * could observe once every optional field is read through `??` / `=== true`.
7430
+ */
7431
+ function hybridPlanOf(query, fields) {
7432
+ return {
7433
+ query,
7434
+ limit: fields?.limit,
7435
+ ...fields?.overFetch === void 0 ? {} : { overFetch: fields.overFetch },
7436
+ maxTokens: fields?.maxTokens,
7437
+ queryVector: fields?.queryVector,
7438
+ recordStats: fields?.recordStats,
7439
+ floors: fields?.floors,
7440
+ ...fields?.includeScores === void 0 ? {} : { includeScores: fields.includeScores },
7441
+ ...fields?.relaxLegs === void 0 ? {} : { relaxLegs: fields.relaxLegs },
7442
+ ...fields?.rewriteQuery === void 0 ? {} : { rewriteQuery: fields.rewriteQuery }
7443
+ };
7444
+ }
7445
+ /**
7446
+ * Build a store's {@link HybridDeps} — the ONE construction site for the contract fields both stores
7447
+ * share (`kind` / `config` / `semantic` / `legs` / `texts` / `hits` / `onReturn`). The per-store half
7448
+ * is `parts`; the stores used to spell the whole object out twice, so a new required `HybridDeps`
7449
+ * field was a change to remember in two places.
7450
+ */
7451
+ function makeHybridDeps(kind, config, semantic, parts) {
7452
+ return {
7453
+ kind,
7454
+ config,
7455
+ semantic,
7456
+ legs: parts.legs,
7457
+ texts: parts.texts,
7458
+ hits: parts.hits,
7459
+ ...parts.onReturn === void 0 ? {} : { onReturn: parts.onReturn }
7460
+ };
7461
+ }
7462
+ /**
7396
7463
  * P-01: put the fused per-leg evidence onto the caller-facing hits.
7397
7464
  *
7398
7465
  * Matched by `ref_id`, not by array index: a store's `hits` mapper may legitimately drop an entry
@@ -7496,6 +7563,266 @@ function describe(error) {
7496
7563
  return error instanceof Error ? error.message : String(error);
7497
7564
  }
7498
7565
  //#endregion
7566
+ //#region ../core/lib/store/legs.js
7567
+ /**
7568
+ * The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
7569
+ * leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
7570
+ * `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
7571
+ * scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
7572
+ * check's error and the Map insertion order live here once.
7573
+ *
7574
+ * `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
7575
+ * a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
7576
+ * `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
7577
+ * empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
7578
+ * paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
7579
+ * from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
7580
+ * probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
7581
+ * at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
7582
+ *
7583
+ * WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
7584
+ * signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
7585
+ * counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
7586
+ * quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
7587
+ * deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
7588
+ * floor functions only ever REMOVE entries, so the raw set is the honest measurement.
7589
+ *
7590
+ * WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
7591
+ * `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
7592
+ * {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
7593
+ * the absence is structural, not something a caller can forget to set.
7594
+ *
7595
+ * WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
7596
+ * table, and a template-method base class would be a structural rewrite of two hub files. This is
7597
+ * the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
7598
+ * be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
7599
+ * third store in `test/legs.spec.ts`).
7600
+ *
7601
+ * CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
7602
+ * (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
7603
+ * legs, so both stores get the same awaiting and isolation by construction.
7604
+ *
7605
+ * @module store/legs
7606
+ */
7607
+ /**
7608
+ * The one `capped` derivation (raw-set size vs cap). Kept as a named helper so the three call sites
7609
+ * cannot drift into measuring the floored set.
7610
+ */
7611
+ function wasCapped(raw, cap) {
7612
+ return raw.size === cap;
7613
+ }
7614
+ /**
7615
+ * Build the leg runner for one query.
7616
+ *
7617
+ * The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
7618
+ * index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
7619
+ * self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
7620
+ * evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
7621
+ * same wrapper, the same floor order and the same down-semantic fallback.
7622
+ */
7623
+ function makeLegRunner(ctx) {
7624
+ const cap = ctx.legCap;
7625
+ return {
7626
+ cap,
7627
+ plain(scores, weight) {
7628
+ return {
7629
+ weight,
7630
+ scores,
7631
+ capped: wasCapped(scores, cap)
7632
+ };
7633
+ },
7634
+ scored(name, raw, weight, floor) {
7635
+ const floored = applyScoreFloor(raw, floor);
7636
+ return {
7637
+ weight,
7638
+ scores: floored.scores,
7639
+ capped: wasCapped(raw, cap),
7640
+ leg: name,
7641
+ droppedByFloor: floored.dropped
7642
+ };
7643
+ },
7644
+ term(name, raw, texts, query, weight, floor) {
7645
+ const floored = applyTermFloor(raw, texts, query, floor);
7646
+ return {
7647
+ weight,
7648
+ scores: floored.scores,
7649
+ capped: wasCapped(raw, cap),
7650
+ leg: name,
7651
+ droppedByFloor: floored.dropped
7652
+ };
7653
+ },
7654
+ semantic(raw, weight, floor) {
7655
+ const floored = applyScoreFloor(raw, floor);
7656
+ return {
7657
+ weight,
7658
+ scores: floored.scores,
7659
+ leg: "semantic",
7660
+ droppedByFloor: floored.dropped
7661
+ };
7662
+ },
7663
+ semanticOff(weight) {
7664
+ return {
7665
+ weight,
7666
+ scores: /* @__PURE__ */ new Map(),
7667
+ leg: "semantic",
7668
+ droppedByFloor: 0
7669
+ };
7670
+ }
7671
+ };
7672
+ }
7673
+ /**
7674
+ * The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
7675
+ *
7676
+ * The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
7677
+ * bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
7678
+ * order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
7679
+ *
7680
+ * - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
7681
+ * stays better everywhere downstream;
7682
+ * - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
7683
+ * on the same "higher = better" scale;
7684
+ * - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
7685
+ * 3-char run) — the leg is empty by construction, not by a query that matched nothing.
7686
+ *
7687
+ * THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
7688
+ * trigram index can express (measured: every term in the FTS table is 3 characters), which used to
7689
+ * leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
7690
+ * predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
7691
+ * is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
7692
+ */
7693
+ function ftsLeg(opts) {
7694
+ const ftsQuery = buildFtsQuery(opts.query, opts.tokenizer);
7695
+ if (ftsQuery) {
7696
+ const rows = opts.search(ftsQuery, opts.scope, opts.cap);
7697
+ return new Map(rows.map((r) => [r.id, -r.rank]));
7698
+ }
7699
+ const terms = substringTerms(opts.query);
7700
+ if (terms.length === 0) return /* @__PURE__ */ new Map();
7701
+ const rows = opts.substringSearch(terms, opts.scope, opts.cap);
7702
+ return new Map(rows.map((r) => [r.id, r.rank]));
7703
+ }
7704
+ /**
7705
+ * The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
7706
+ *
7707
+ * The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
7708
+ * `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
7709
+ * to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
7710
+ *
7711
+ * Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
7712
+ * rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
7713
+ * empty-pool early return, and the returned map's insertion order (the vstore's pool order).
7714
+ */
7715
+ async function semanticLeg(opts) {
7716
+ const vec = opts.queryVector ?? await opts.encode(opts.query);
7717
+ if (vec.length !== opts.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${opts.dim}`);
7718
+ opts.onVector?.(vec);
7719
+ const topk = opts.topk(vec, Math.max(50, opts.k));
7720
+ if (!topk.length) return /* @__PURE__ */ new Map();
7721
+ const allowed = opts.filterTopk(topk.map((t) => t.id));
7722
+ const out = /* @__PURE__ */ new Map();
7723
+ for (const t of topk) if (allowed === void 0 || allowed.has(t.id)) out.set(t.id, t.score);
7724
+ return out;
7725
+ }
7726
+ //#endregion
7727
+ //#region ../core/lib/store/entity_sweep.js
7728
+ /**
7729
+ * The ONE entity-version sweep both stores run.
7730
+ *
7731
+ * `entities_version` is the stamp that says "these entity rows were produced by the CURRENT
7732
+ * extraction rules". A rule change makes every row carrying an older stamp stale, and the only way
7733
+ * to close that gap is to re-extract those rows' text — for memory the entity links, the triples and
7734
+ * the HRR bundle derived from them; for knowledge the `chunk_entities` rows. That SKELETON was
7735
+ * written twice and drifted apart in exactly the ways a copy drifts: memory had the in-flight guard,
7736
+ * the budget clamp and the `{rebuilt, deferred, skipped}` report, while knowledge had the same
7737
+ * selection inlined in `reindex` with no guard at all. This module is the one copy; the stores
7738
+ * contribute a per-aggregate adapter (which rows, how to extract, what to write) and nothing else.
7739
+ *
7740
+ * Layering, deliberate: this is an ORCHESTRATION rule, not SQL, so it lives beside `common.ts` and
7741
+ * `legs.ts` in `store/` — the DAO layer stays split by aggregate and keeps its own statements
7742
+ * (`docs/vector-repair-shared-flow.md` §2 / §7).
7743
+ *
7744
+ * What must NOT move here: the extractor, the rows a store writes, and that store's side effects.
7745
+ * Memory's transaction also replaces triples, moves the HRR bundle and re-queues the conflict check
7746
+ * (both conflict legs read what the sweep rebuilt); knowledge replaces `chunk_entities` under
7747
+ * `MAX_ENTITIES_PER_CHUNK`. Those are domain semantics, and folding them in would trade one
7748
+ * duplicated loop for two conditional branches.
7749
+ *
7750
+ * @module store/entity_sweep
7751
+ */
7752
+ /**
7753
+ * Rows one sweep pass may rebuild when the caller names no budget.
7754
+ *
7755
+ * `MemoryStore.reindexEntities` has always defaulted to this, and the CLI passes it explicitly; it
7756
+ * lives here so the shared loop and that public re-export cannot drift. Bounds MEMORY, not only
7757
+ * mission: a pass selects every chosen row's text before rebuilding the first one.
7758
+ */
7759
+ const ENTITY_SWEEP_BATCH = 2e3;
7760
+ /**
7761
+ * The module-level in-flight guard.
7762
+ *
7763
+ * Held here, not in the stores, because it protects THIS loop: two callers can overlap (the plugin's
7764
+ * heartbeat and `maintenance` are both periodic), and a limited ORDERED selection makes two
7765
+ * concurrent passes pick the same rows and re-tag them twice. Overlap is not corruption — each row is
7766
+ * written in its own transaction — it is duplicated mission in a single-threaded process, and the
7767
+ * answer is one pass reporting `skipped`. A `WeakSet` keyed by the caller's guard object keeps the
7768
+ * state in this module without adding a mutable field to either hub store, and without a module-level
7769
+ * boolean that would make two stores' sweeps block each other.
7770
+ *
7771
+ * The `guard` argument is therefore an IDENTITY, not a token: a store must pass the SAME object to
7772
+ * every call for its whole lifetime (both stores hold one as a private field). Two different objects
7773
+ * are two different sweeps and run concurrently — that per-store granularity is the point.
7774
+ */
7775
+ const SWEEPING = /* @__PURE__ */ new WeakSet();
7776
+ /**
7777
+ * Re-extract the entity rows written by older extraction rules, ONE bounded pass.
7778
+ *
7779
+ * The contract, in order: take the guard (an overlapping pass reports `skipped` and does nothing);
7780
+ * read the scope's stale count and at most `maxPerPass` rows; re-extract each row through the
7781
+ * adapter, isolating failures; report `{ rebuilt, deferred, skipped }`.
7782
+ *
7783
+ * Not an unbounded drain, on purpose: this is reachable from a settings-page button and a periodic
7784
+ * heartbeat, so it must not block on a whole stale corpus. Callers that want the corpus current loop
7785
+ * until `deferred === 0` (the CLI does).
7786
+ *
7787
+ * A row whose extraction or write throws is skipped, not fatal: the transaction that stamps the
7788
+ * version never ran, so the row is still stale and the NEXT pass retries it — the same retry contract
7789
+ * the knowledge store used per chunk. Letting the throw escape would instead abandon the rest of the
7790
+ * batch with no report of what was done.
7791
+ *
7792
+ * @param guard the store's stable in-flight identity (see {@link SWEEPING})
7793
+ */
7794
+ async function sweepVersionedEntities(target, guard, opts = {}) {
7795
+ const maxPerPass = Math.max(0, Math.floor(opts.budget ?? 2e3));
7796
+ if (SWEEPING.has(guard)) return {
7797
+ rebuilt: 0,
7798
+ deferred: target.staleEntityCount?.() ?? target.staleEntityBatch(0).total,
7799
+ skipped: true
7800
+ };
7801
+ SWEEPING.add(guard);
7802
+ try {
7803
+ const { total, rows } = target.staleEntityBatch(maxPerPass);
7804
+ let rebuilt = 0;
7805
+ for (const row of rows) {
7806
+ if (opts.shouldStop?.() === true) break;
7807
+ if (rebuilt >= maxPerPass) break;
7808
+ try {
7809
+ await target.extractAndWriteEntities(row);
7810
+ } catch (error) {
7811
+ target.onRowError(row, error);
7812
+ continue;
7813
+ }
7814
+ rebuilt += 1;
7815
+ }
7816
+ return {
7817
+ rebuilt,
7818
+ deferred: Math.max(0, total - rebuilt),
7819
+ skipped: false
7820
+ };
7821
+ } finally {
7822
+ SWEEPING.delete(guard);
7823
+ }
7824
+ }
7825
+ //#endregion
7499
7826
  //#region ../core/lib/db/vectors.js
7500
7827
  /** Little-endian Float32 ↔ BLOB helpers for persisted vectors (cross-platform stable). */
7501
7828
  function float32ToBytes(vec) {
@@ -7687,6 +8014,280 @@ async function forEachYielding(items, visit, every = 200) {
7687
8014
  if ((index + 1) % every === 0) await yieldToEventLoop();
7688
8015
  }
7689
8016
  }
8017
+ /**
8018
+ * Rebuild one store's live vstore from its persisted rows, count what is unusable in the current
8019
+ * space, and say so once — the shared "open for business" step both stores used to write twice.
8020
+ *
8021
+ * The owning store's ONLY half is the row read (`facts.activeVectorRows()` /
8022
+ * `chunks.vectorRows()`); the reload, the count and the loud warning are one implementation, so a
8023
+ * change to what counts as stale cannot land on one store only (the incident this exists for: the
8024
+ * memory store checked the SPACE while knowledge checked only the WIDTH, so a model swap mixed two
8025
+ * spaces in one index until someone happened to run `kb_reindex`).
8026
+ */
8027
+ function loadVectorIndex(opts) {
8028
+ reloadVectorIndex(opts.vstore, opts.rows, opts.store);
8029
+ const expectedBytes = opts.vstore.dim * 4;
8030
+ let stale = 0;
8031
+ let spaceStale = 0;
8032
+ let legacy = 0;
8033
+ for (const row of opts.rows) {
8034
+ if (row.vec === null || row.vec.byteLength !== expectedBytes) {
8035
+ stale++;
8036
+ continue;
8037
+ }
8038
+ if (row.embedding_model !== opts.space) {
8039
+ spaceStale++;
8040
+ if (isPreFingerprintSpace(row.embedding_model)) legacy++;
8041
+ }
8042
+ }
8043
+ reportStaleVectors(opts.store, opts.space, {
8044
+ stale,
8045
+ space_stale: spaceStale,
8046
+ legacy
8047
+ }, opts.manualEntry);
8048
+ return {
8049
+ stale,
8050
+ space_stale: spaceStale,
8051
+ legacy
8052
+ };
8053
+ }
8054
+ /**
8055
+ * Encode rows one at a time, add each to the live index, and persist them in groups.
8056
+ *
8057
+ * The shared skeleton: serial encode (a MEASURED decision for chunks — a padded batched forward pass
8058
+ * is 1.79x slower on real 53–550 character chunks), `add` right after a successful encode, one
8059
+ * `write` call per `writeBatch` rows, per-row isolation of encode failures, per-group isolation of
8060
+ * write failures, and the `{ encoded, failed }` count.
8061
+ *
8062
+ * What stays with the store: the backend call, the live index, the write statement, the replay of a
8063
+ * failed group, and the summary log — the memory store logs per row and keeps a half-written index
8064
+ * (documented as the benign direction for it), the knowledge store evicts the group so both sides
8065
+ * agree that the vectors are missing.
8066
+ */
8067
+ async function encodeAndPersist(rows, target) {
8068
+ const every = Math.max(1, Math.floor(target.writeBatch ?? 1));
8069
+ let encoded = 0;
8070
+ let failed = 0;
8071
+ for (let i = 0; i < rows.length; i += every) {
8072
+ const group = rows.slice(i, i + every);
8073
+ const writes = [];
8074
+ for (const row of group) try {
8075
+ const vec = await target.encode(row.text);
8076
+ target.add(row.id, vec);
8077
+ writes.push({
8078
+ id: row.id,
8079
+ bytes: float32ToBytes(vec)
8080
+ });
8081
+ } catch (error) {
8082
+ failed += 1;
8083
+ target.onRowError?.(row, error);
8084
+ }
8085
+ if (!writes.length) continue;
8086
+ try {
8087
+ target.write(writes);
8088
+ encoded += writes.length;
8089
+ } catch (error) {
8090
+ failed += writes.length;
8091
+ target.onWriteError?.(writes, error);
8092
+ }
8093
+ }
8094
+ return {
8095
+ encoded,
8096
+ failed
8097
+ };
8098
+ }
8099
+ /**
8100
+ * ONE bounded slice of the repair: re-add unusable-but-readable vectors that the live index lost,
8101
+ * drop the rows a re-encode has to replace, then encode up to `limit` missing/foreign rows.
8102
+ *
8103
+ * `limit` is what makes the SAME logic usable as a background migration: one call is a bounded unit
8104
+ * of work the caller can yield between. The explicit fix passes no limit (the operator action drains
8105
+ * everything); the background drive passes its batch size.
8106
+ *
8107
+ * The ORDER is an invariant: unusable bytes leave the table and the live index first, so a
8108
+ * re-encode can pick them up and a process that dies mid-migration leaves at most one batch
8109
+ * vector-less — exactly the `missing` rows the next run resumes on.
8110
+ */
8111
+ async function repairVectorSlice(target, opts) {
8112
+ const limit = opts.limit ?? Number.POSITIVE_INFINITY;
8113
+ const c = target.classifyVectors(limit);
8114
+ const loadedNow = target.semanticAvailable();
8115
+ let semAvailable = loadedNow;
8116
+ if (!semAvailable && !opts.dryRun && target.warmup !== void 0) semAvailable = await target.warmup();
8117
+ const wouldWarm = !loadedNow;
8118
+ const remainingOf = (encoded) => Math.max(0, c.stale + c.space_stale + c.missing - encoded);
8119
+ const base = {
8120
+ stale: c.stale,
8121
+ space_stale: c.space_stale,
8122
+ missing: c.missing,
8123
+ semantic_available: semAvailable,
8124
+ would_warm: wouldWarm
8125
+ };
8126
+ if (opts.dryRun) return {
8127
+ report: {
8128
+ ...base,
8129
+ ...c.unindexed === void 0 ? {} : {
8130
+ unindexed: c.unindexed.length,
8131
+ reindexed: 0
8132
+ },
8133
+ dropped: 0,
8134
+ encoded: 0,
8135
+ failed: 0
8136
+ },
8137
+ remaining: remainingOf(0)
8138
+ };
8139
+ let reindexed = 0;
8140
+ if (c.unindexed !== void 0) reindexed = target.reindexUsable?.(c.unindexed.slice(0, limit)) ?? 0;
8141
+ let dropped = 0;
8142
+ if (semAvailable && c.dropIds.length > 0) dropped = target.dropVectors(c.dropIds);
8143
+ const { encoded, failed } = semAvailable ? await target.reencodeVectors(limit) : {
8144
+ encoded: 0,
8145
+ failed: 0
8146
+ };
8147
+ return {
8148
+ report: {
8149
+ ...base,
8150
+ ...c.unindexed === void 0 ? {} : {
8151
+ unindexed: c.unindexed.length,
8152
+ reindexed
8153
+ },
8154
+ dropped,
8155
+ encoded,
8156
+ failed
8157
+ },
8158
+ remaining: remainingOf(encoded)
8159
+ };
8160
+ }
8161
+ /**
8162
+ * ONE bounded slice of the background migration — the unit the drive (and the tests) step.
8163
+ *
8164
+ * Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
8165
+ * stale from the persisted width/space, so a restart simply continues where the last slice stopped.
8166
+ */
8167
+ async function migrateVectorSlice(target, batchSize = 16) {
8168
+ const { report, remaining } = await repairVectorSlice(target, {
8169
+ dryRun: false,
8170
+ limit: Math.max(1, Math.floor(batchSize))
8171
+ });
8172
+ return {
8173
+ remaining,
8174
+ migrated: report.encoded,
8175
+ dropped: report.dropped,
8176
+ reindexed: report.reindexed ?? 0,
8177
+ semantic_available: report.semantic_available
8178
+ };
8179
+ }
8180
+ /** ACTIVE rows the semantic leg still cannot use (old width, old space, or no vector at all). */
8181
+ function migrationRemaining(target) {
8182
+ const health = target.vectorSpaceHealth();
8183
+ return health.stale + health.space_stale + target.missingVectorCount();
8184
+ }
8185
+ /** The `auto_migrate: false` warning is one per process per store, like the startup warning. */
8186
+ const disabledWarned = /* @__PURE__ */ new WeakSet();
8187
+ /**
8188
+ * Drive the bounded background migration for one store until it is current, the model is
8189
+ * unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE
8190
+ * never rejects: it is logged and reported as `remaining > 0`, which is what makes the next
8191
+ * heartbeat a retry. Never blocks the event loop for longer than one batch.
8192
+ *
8193
+ * Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
8194
+ * on background re-encoding can leave the loud warning and the manual entry as the only path. The
8195
+ * switch is read through the adapter so a new store cannot forget it.
8196
+ */
8197
+ async function driveVectorRepair(target, opts = {}) {
8198
+ const log = opts.log ?? retrievalLogger();
8199
+ const stopped = () => opts.shouldStop?.() === true;
8200
+ const enabled = target.autoMigrate() !== false;
8201
+ const snapshot = () => ({
8202
+ enabled,
8203
+ remaining: migrationRemaining(target),
8204
+ migrated: 0,
8205
+ dropped: 0,
8206
+ reindexed: 0,
8207
+ semantic_available: target.semanticAvailable()
8208
+ });
8209
+ const initial = snapshot();
8210
+ if (stopped()) return initial;
8211
+ const health = target.vectorSpaceHealth();
8212
+ const isSpaceChange = health.stale + health.space_stale > 0;
8213
+ if (!enabled) {
8214
+ if (initial.remaining > 0 && !disabledWarned.has(target)) {
8215
+ disabledWarned.add(target);
8216
+ log.warn(`${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) ${isSpaceChange ? "belong to an older embedding space" : "have no vector at all"} and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run ${target.manualEntry} or set \`semantic.auto_migrate: true\``);
8217
+ }
8218
+ return initial;
8219
+ }
8220
+ if (initial.remaining === 0) return initial;
8221
+ const startedAt = Date.now();
8222
+ log.info(isSpaceChange ? `${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)` : `${target.store} vector migration: ${String(initial.remaining)} ACTIVE row(s) have no vector at all — encoding them in the background`);
8223
+ let migrated = 0;
8224
+ let dropped = 0;
8225
+ let reindexed = 0;
8226
+ let last = initial;
8227
+ for (;;) {
8228
+ if (stopped()) break;
8229
+ let step;
8230
+ try {
8231
+ step = await migrateVectorSlice(target, opts.batchSize);
8232
+ } catch (error) {
8233
+ log.warn(`${target.store} vector migration: batch failed (${describeError$2(error)}) — ${String(migrationRemaining(target))} vector(s) still cannot be used by the semantic leg; the next pass retries`);
8234
+ break;
8235
+ }
8236
+ migrated += step.migrated;
8237
+ dropped += step.dropped;
8238
+ reindexed += step.reindexed;
8239
+ last = {
8240
+ ...step,
8241
+ enabled,
8242
+ migrated,
8243
+ dropped,
8244
+ reindexed
8245
+ };
8246
+ opts.onProgress?.(last);
8247
+ log.info(`${target.store} vector migration: re-encoded ${String(migrated)}/${String(initial.remaining)} (${String(step.remaining)} left)`);
8248
+ if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
8249
+ if (last.remaining === 0) break;
8250
+ await yieldToEventLoop();
8251
+ }
8252
+ if (last.remaining === 0) log.info(`${target.store} vector migration: complete — ${String(migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
8253
+ else if (!stopped()) log.warn(`${target.store} vector migration: ${String(last.remaining)} vector(s) still cannot be used by the semantic leg (the embedding model may be unavailable) — the next heartbeat retries; manual entry ${target.manualEntry}`);
8254
+ return last;
8255
+ }
8256
+ /**
8257
+ * Drive every store in order. A stop request keeps the drive from starting another store, but each
8258
+ * store still answers with its snapshot (so the caller can report what is left everywhere).
8259
+ */
8260
+ async function driveVectorRepairs(targets, opts = {}) {
8261
+ const outcomes = [];
8262
+ for (const target of targets) outcomes.push(await driveVectorRepair(target, opts));
8263
+ return outcomes;
8264
+ }
8265
+ /**
8266
+ * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`,
8267
+ * covering one or both stores.
8268
+ *
8269
+ * `store` selects a single store (omitted = both); `dry_run` previews every count without writing —
8270
+ * and without loading the model (so a preview also reports `would_warm`). The top-level
8271
+ * `semantic_available` is AND over the selected stores: `false` means at least one store's repair
8272
+ * could not encode anything.
8273
+ */
8274
+ async function vectorsFix(targets, opts) {
8275
+ const stores = {};
8276
+ let semanticAvailable = true;
8277
+ for (const kind of ["memory", "knowledge"]) {
8278
+ if (opts.store !== void 0 && opts.store !== kind) continue;
8279
+ const target = targets[kind];
8280
+ if (target === void 0) continue;
8281
+ const { report } = await repairVectorSlice(target, { dryRun: opts.dryRun });
8282
+ stores[kind] = report;
8283
+ semanticAvailable = semanticAvailable && report.semantic_available;
8284
+ }
8285
+ return {
8286
+ stores,
8287
+ semantic_available: semanticAvailable,
8288
+ dry_run: opts.dryRun
8289
+ };
8290
+ }
7690
8291
  //#endregion
7691
8292
  //#region ../core/lib/store/secret_guard.js
7692
8293
  /**
@@ -8460,6 +9061,124 @@ function inList(values) {
8460
9061
  };
8461
9062
  }
8462
9063
  //#endregion
9064
+ //#region ../core/lib/db/dao/shared.js
9065
+ /**
9066
+ * The entity-candidate leg's ORDER contract — the tail every candidate statement ends with.
9067
+ *
9068
+ * The formula is the leg's own score (anchored Jaccard with a saturating union,
9069
+ * `store/entity_leg.ts`): `shared / (queryWidth + min(total - shared, widthCap))`. Ordering by it is
9070
+ * load-bearing, not cosmetic: `fuse` scales each leg by its maximum, and a survivor's scaled value
9071
+ * is unchanged only when the ordering — and therefore the maximum — survives the trim. A narrower
9072
+ * row sharing one entity can therefore outrank a wide one sharing two, and the cap keeps it.
9073
+ * `total ASC` is only a deterministic tie-break among equal ratios, and the id makes the order total.
9074
+ *
9075
+ * `idColumn` is the aggregate's own id (`fa.fact_id` / `ce.chunk_id`); that, the `total` subquery's
9076
+ * table and the filter predicates are the ONLY differences between the two statements.
9077
+ */
9078
+ function entityCandidateTail(idColumn) {
9079
+ return `GROUP BY ${idColumn}
9080
+ ORDER BY (CAST(shared AS REAL) / (? + MIN(total - shared, ?))) DESC, total ASC, ${idColumn} ASC
9081
+ LIMIT ?`;
9082
+ }
9083
+ /**
9084
+ * Run an entity-candidate query: bind order, batching and union are shared; the SQL is the caller's.
9085
+ *
9086
+ * Bind order (identical in both aggregates): `<names …>, <filterParams …>, queryWidth, widthCap, limit`.
9087
+ */
9088
+ function entityCandidates(db, q) {
9089
+ if (!q.names.length || q.limit <= 0) return [];
9090
+ const out = [];
9091
+ const seen = q.batchNames ? /* @__PURE__ */ new Set() : null;
9092
+ for (const batch of q.batchNames ? batches(q.names) : [q.names]) {
9093
+ const { placeholders, values } = inList(batch);
9094
+ const rows = db.prepare(q.sql(placeholders)).all(...values, ...q.filterParams, q.queryWidth, q.widthCap, q.limit);
9095
+ for (const row of rows) {
9096
+ if (seen !== null) {
9097
+ if (seen.has(row.id)) continue;
9098
+ seen.add(row.id);
9099
+ }
9100
+ out.push(row.id);
9101
+ }
9102
+ }
9103
+ return out;
9104
+ }
9105
+ /**
9106
+ * The batched `UPDATE … SET entities_version = ?` shared by `FactsDao` and `ChunksDao`.
9107
+ *
9108
+ * Only the aggregate's `table` / `idColumn` differ. Returns the summed `changes`; the knowledge
9109
+ * caller ignores it (its method's signature returns `void`), the memory caller's `number` is
9110
+ * unchanged.
9111
+ */
9112
+ function setEntitiesVersionBatch(db, table, idColumn, ids, version) {
9113
+ let changes = 0;
9114
+ for (const batch of batches(ids)) {
9115
+ const { placeholders, values } = inList(batch);
9116
+ changes += db.prepare(`UPDATE ${table} SET entities_version = ? WHERE ${idColumn} IN (${placeholders})`).run(version, ...values).changes;
9117
+ }
9118
+ return changes;
9119
+ }
9120
+ /**
9121
+ * Document frequency — "how many rows carry each name".
9122
+ *
9123
+ * Shared by `EntitiesDao.activeDocFrequency` and `ChunksDao.docFrequency`; the only difference is
9124
+ * the population the caller's `where`/`from` describe (memory counts ACTIVE facts, knowledge has no
9125
+ * status column and counts chunks). Only the caller's QUERY names are looked up, so this is one
9126
+ * indexed COUNT per name rather than a corpus scan. A name absent from the map has frequency 0.
9127
+ */
9128
+ function queryDocFrequency(db, names, q) {
9129
+ const out = /* @__PURE__ */ new Map();
9130
+ if (!names.length) return out;
9131
+ const { placeholders, values } = inList(names);
9132
+ const rows = db.prepare(`SELECT ${q.nameExpr} AS name, COUNT(*) AS df${q.from}
9133
+ WHERE ${q.nameExpr} IN (${placeholders})${q.where === void 0 ? "" : ` AND ${q.where}`}
9134
+ GROUP BY ${q.nameExpr}`).all(...values);
9135
+ for (const row of rows) out.set(row.name, Number(row.df));
9136
+ return out;
9137
+ }
9138
+ /**
9139
+ * The SHORT-QUERY LIKE fallback shared by `FactsDao.ftsSubstringSearch` and
9140
+ * `ChunksDao.ftsSubstringSearch`.
9141
+ *
9142
+ * The predicate text itself already comes from `db/tokenizer.ts#likeSubstring`; this owns the
9143
+ * protocol AROUND it, which both copies spelled out:
9144
+ *
9145
+ * 1. build the fragments — empty terms yield no patterns, which means "match nothing" ⇒ `[]`;
9146
+ * 2. `count` lands in the SELECT and `any` in the WHERE, so the SAME pattern list is bound TWICE:
9147
+ * once with the SELECT's placeholders, then — after the caller's filter params — with the
9148
+ * WHERE's;
9149
+ * 3. `LIMIT ?` last, defaulting to `-1` (unbounded; the store bounds the leg).
9150
+ *
9151
+ * `ORDER BY rank DESC, <id> ASC` stays in the caller's SQL: the id column is aggregate-specific and
9152
+ * `rank` must read the SELECT alias, so the fragments and the patterns are built once and used twice.
9153
+ */
9154
+ function likeSubstringLeg(db, column, terms, buildSql, filterParams = [], limit) {
9155
+ const fragments = likeSubstring(column, terms);
9156
+ if (fragments.params.length === 0) return [];
9157
+ return db.prepare(buildSql(fragments)).all(...fragments.params, ...filterParams, ...fragments.params, limit ?? -1);
9158
+ }
9159
+ /**
9160
+ * Batched entity-name bags for a row-id set: `Map<rowId, names>`.
9161
+ *
9162
+ * Shared by `EntitiesDao.bagsForFacts` and `ChunksDao.entityBags` (§4.6.3): one `IN (…)` statement
9163
+ * per `batches()` slice, rows folded into the map in first-seen order. The SQL SHAPE is the only
9164
+ * aggregate-specific part (`table` / `keyColumn` / `nameColumn` / `joinEntities`); the crucial
9165
+ * `seedEmpty` difference above is the caller's to declare and stays observable.
9166
+ */
9167
+ function entityBags(db, ids, q) {
9168
+ const out = /* @__PURE__ */ new Map();
9169
+ if (q.seedEmpty) for (const id of ids) out.set(id, []);
9170
+ for (const batch of batches(ids)) {
9171
+ const { placeholders, values } = inList(batch);
9172
+ const rows = db.prepare(`SELECT ${q.keyColumn} AS id, ${q.nameColumn} AS name FROM ${q.table}${q.joinEntities === void 0 ? "" : ` ${q.joinEntities}`} WHERE ${q.keyColumn} IN (${placeholders})`).all(...values);
9173
+ for (const row of rows) {
9174
+ const list = out.get(row.id);
9175
+ if (list) list.push(row.name);
9176
+ else out.set(row.id, [row.name]);
9177
+ }
9178
+ }
9179
+ return out;
9180
+ }
9181
+ //#endregion
8463
9182
  //#region ../core/lib/db/dao/entities.js
8464
9183
  var EntitiesDao = class {
8465
9184
  db;
@@ -8506,17 +9225,13 @@ var EntitiesDao = class {
8506
9225
  }
8507
9226
  /** Batched entity bags for a candidate set: `Map<factId, names>` (ids with none get `[]`). */
8508
9227
  bagsForFacts(ids) {
8509
- const out = /* @__PURE__ */ new Map();
8510
- if (!ids.length) return out;
8511
- for (const id of ids) out.set(id, []);
8512
- for (const batch of batches(ids)) {
8513
- const { placeholders, values } = inList(batch);
8514
- const rows = this.db.prepare(`SELECT fe.fact_id AS fact_id, e.name AS name FROM fact_entities fe
8515
- JOIN entities e ON e.entity_id = fe.entity_id
8516
- WHERE fe.fact_id IN (${placeholders})`).all(...values);
8517
- for (const row of rows) out.get(row.fact_id)?.push(row.name);
8518
- }
8519
- return out;
9228
+ return entityBags(this.db, ids, {
9229
+ table: "fact_entities fe",
9230
+ keyColumn: "fe.fact_id",
9231
+ nameColumn: "e.name",
9232
+ joinEntities: "JOIN entities e ON e.entity_id = fe.entity_id",
9233
+ seedEmpty: true
9234
+ });
8520
9235
  }
8521
9236
  /** How many entities a fact is linked to — the scorer's `|A|`. */
8522
9237
  countForFact(factId) {
@@ -8533,17 +9248,14 @@ var EntitiesDao = class {
8533
9248
  * map has frequency 0, i.e. no active fact carries it and it can never be shared.
8534
9249
  */
8535
9250
  activeDocFrequency(names) {
8536
- const out = /* @__PURE__ */ new Map();
8537
- if (!names.length) return out;
8538
- const { placeholders, values } = inList(names);
8539
- const rows = this.db.prepare(`SELECT e.name AS name, COUNT(*) AS df
9251
+ return queryDocFrequency(this.db, names, {
9252
+ from: `
8540
9253
  FROM entities e
8541
9254
  JOIN fact_entities fe ON fe.entity_id = e.entity_id
8542
- JOIN facts fa ON fa.fact_id = fe.fact_id
8543
- WHERE e.name IN (${placeholders}) AND fa.status = 'active'
8544
- GROUP BY e.name`).all(...values);
8545
- for (const row of rows) out.set(row.name, Number(row.df));
8546
- return out;
9255
+ JOIN facts fa ON fa.fact_id = fe.fact_id`,
9256
+ nameExpr: "e.name",
9257
+ where: "fa.status = 'active'"
9258
+ });
8547
9259
  }
8548
9260
  /**
8549
9261
  * Candidate facts for the embedding leg, narrowed by the scorer's NECESSARY conditions
@@ -8602,9 +9314,19 @@ var EntitiesDao = class {
8602
9314
  * survivors, so one formula stays in charge.
8603
9315
  */
8604
9316
  candidateFactsForAnyEntity(names, category, limit, queryWidth, widthCap, source) {
8605
- if (!names.length || limit <= 0) return [];
8606
- const { placeholders, values } = inList(names);
8607
- return this.db.prepare(`SELECT fa.fact_id AS id, COUNT(*) AS shared,
9317
+ return entityCandidates(this.db, {
9318
+ names,
9319
+ filterParams: [
9320
+ category ?? null,
9321
+ category ?? null,
9322
+ source ?? null,
9323
+ source ?? null
9324
+ ],
9325
+ limit,
9326
+ queryWidth,
9327
+ widthCap,
9328
+ batchNames: false,
9329
+ sql: (placeholders) => `SELECT fa.fact_id AS id, COUNT(*) AS shared,
8608
9330
  (SELECT COUNT(*) FROM fact_entities x WHERE x.fact_id = fa.fact_id) AS total
8609
9331
  FROM facts fa
8610
9332
  JOIN fact_entities fe ON fe.fact_id = fa.fact_id
@@ -8613,9 +9335,8 @@ var EntitiesDao = class {
8613
9335
  AND (? IS NULL OR EXISTS (
8614
9336
  SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?
8615
9337
  ))
8616
- GROUP BY fa.fact_id
8617
- ORDER BY (CAST(shared AS REAL) / (? + MIN(total - shared, ?))) DESC, total ASC, fa.fact_id ASC
8618
- LIMIT ?`).all(...values, category ?? null, category ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit).map((row) => row.id);
9338
+ ${entityCandidateTail("fa.fact_id")}`
9339
+ });
8619
9340
  }
8620
9341
  };
8621
9342
  //#endregion
@@ -8967,12 +9688,7 @@ var FactsDao = class FactsDao {
8967
9688
  }
8968
9689
  /** Stamp the rules that produced a batch of rows' entity/triple data. */
8969
9690
  setEntitiesVersion(ids, version) {
8970
- let changes = 0;
8971
- for (const batch of batches(ids)) {
8972
- const { placeholders, values } = inList(batch);
8973
- changes += this.db.prepare(`UPDATE facts SET entities_version = ? WHERE fact_id IN (${placeholders})`).run(version, ...values).changes;
8974
- }
8975
- return changes;
9691
+ return setEntitiesVersionBatch(this.db, "facts", "fact_id", ids, version);
8976
9692
  }
8977
9693
  /**
8978
9694
  * Replace a fact's HRR vector.
@@ -9352,14 +10068,16 @@ var FactsDao = class FactsDao {
9352
10068
  * alias, so the LIKE expressions are built once and bound twice — same fragment, same order.
9353
10069
  */
9354
10070
  ftsSubstringSearch(terms, category, limit, source) {
9355
- const { any, count, params } = likeSubstring("fa.content", terms);
9356
- if (params.length === 0) return [];
9357
10071
  const sourceClause = source === void 0 ? "" : " AND EXISTS (SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?)";
9358
- return this.db.prepare(`SELECT fa.fact_id AS id, ${count} AS rank
10072
+ return likeSubstringLeg(this.db, "fa.content", terms, ({ any, count }) => `SELECT fa.fact_id AS id, ${count} AS rank
9359
10073
  FROM facts fa
9360
10074
  WHERE fa.status = 'active' AND (? IS NULL OR fa.category = ?)${sourceClause} AND (${any})
9361
10075
  ORDER BY rank DESC, fa.fact_id ASC
9362
- LIMIT ?`).all(...params, category ?? null, category ?? null, ...source === void 0 ? [] : [source], ...params, limit ?? -1);
10076
+ LIMIT ?`, source === void 0 ? [category ?? null, category ?? null] : [
10077
+ category ?? null,
10078
+ category ?? null,
10079
+ source
10080
+ ], limit);
9363
10081
  }
9364
10082
  /**
9365
10083
  * ①'s predicate: the active-day counter crossed an integer boundary since the last settle.
@@ -10441,15 +11159,6 @@ const NO_CHECK = {
10441
11159
  complete: false
10442
11160
  };
10443
11161
  /**
10444
- * Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
10445
- *
10446
- * Bounds MEMORY, not only mission: the pass selects `content` for every row it takes, so an
10447
- * "unbounded" pass (the CLI used to ask for `Number.MAX_SAFE_INTEGER`) builds an array of every
10448
- * stale fact's text before rebuilding the first one. Callers that want the corpus drained loop
10449
- * until `deferred === 0` instead.
10450
- */
10451
- const ENTITY_SWEEP_BATCH = 2e3;
10452
- /**
10453
11162
  * `avantf_stats` key holding the retrieval-health snapshot.
10454
11163
  *
10455
11164
  * The counters are process-wide, but the MEMORY store owns the side table, so it is the single
@@ -10495,8 +11204,14 @@ var MemoryStore = class {
10495
11204
  * and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
10496
11205
  */
10497
11206
  ftsTokenizer;
10498
- /** In-flight guard for `reindexEntities` (see there: the periodic callers can overlap). */
10499
- entitySweepInFlight = false;
11207
+ /**
11208
+ * Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
11209
+ *
11210
+ * The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
11211
+ * the rule where the loop is (the periodic callers can overlap), without a mutable field that two
11212
+ * stores would each have to remember to hold.
11213
+ */
11214
+ entitySweepGuard = {};
10500
11215
  /**
10501
11216
  * Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
10502
11217
  * derived state: nothing is lost when it resets, and the queue itself is in the database.
@@ -10536,7 +11251,7 @@ var MemoryStore = class {
10536
11251
  clock,
10537
11252
  budget: opts?.budget ?? trust.tick_max_facts
10538
11253
  });
10539
- this.evictVectors([...result.archived_ids, ...result.purged_ids]);
11254
+ evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
10540
11255
  this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
10541
11256
  this.db.pragma("optimize");
10542
11257
  return result;
@@ -10550,22 +11265,32 @@ var MemoryStore = class {
10550
11265
  return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
10551
11266
  }
10552
11267
  /**
10553
- * Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs.
10554
- * Without this the semantic path is silently empty after every process restart
10555
- * (the DB is the authoritative vector store; the vstore is a derived read model).
11268
+ * Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
11269
+ * belongs to another vector space.
10556
11270
  *
10557
- * Only ACTIVE facts are indexed: archived facts are filtered out of every
10558
- * retrieval path anyway, so loading them would only grow the index forever.
10559
- * `archive()` evicts and `restore()` re-adds, keeping the invariant.
11271
+ * Without this the semantic path is silently empty after every process restart (the DB is the
11272
+ * authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
11273
+ * archived facts are filtered out of every retrieval path anyway, so loading them would only grow
11274
+ * the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
11275
+ *
11276
+ * The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
11277
+ * step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
11278
+ * another space is still RANKED (the reload cannot tell the difference), so a silent model swap
11279
+ * would mix two spaces in one index — acting on it is the manual fix's job for the operator path
11280
+ * and the bounded background migration's for the automatic one.
10560
11281
  */
10561
11282
  reloadIndex() {
10562
- const space = this.vectorSpace();
10563
- const rows = this.facts.activeVectorRows().map((r) => ({
10564
- id: r.fact_id,
10565
- vec: r.semantic_vector
10566
- }));
10567
- reloadVectorIndex(this.vstore, rows, "memory");
10568
- reportStaleVectors("memory", space, this.vectorSpaceCounts(), "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)");
11283
+ loadVectorIndex({
11284
+ store: "memory",
11285
+ vstore: this.vstore,
11286
+ rows: this.facts.activeVectorRows().map((r) => ({
11287
+ id: r.fact_id,
11288
+ vec: r.semantic_vector,
11289
+ embedding_model: r.embedding_model
11290
+ })),
11291
+ space: this.vectorSpace(),
11292
+ manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)"
11293
+ });
10569
11294
  }
10570
11295
  /**
10571
11296
  * Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
@@ -10666,21 +11391,45 @@ var MemoryStore = class {
10666
11391
  };
10667
11392
  }
10668
11393
  }
11394
+ /**
11395
+ * This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
11396
+ * call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
11397
+ * backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
11398
+ * (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
11399
+ */
11400
+ vectorWriteTarget(space) {
11401
+ return {
11402
+ store: "memory",
11403
+ space,
11404
+ encode: (text) => this.semantic.encode(text),
11405
+ add: (id, vec) => {
11406
+ this.vstore.add(id, vec);
11407
+ },
11408
+ write: (rows) => {
11409
+ for (const r of rows) this.facts.setSemanticVector(r.id, r.bytes, space, this.vstore.name);
11410
+ },
11411
+ onRowError: (row, error) => {
11412
+ this.warnIndexFailure(row.id, error);
11413
+ },
11414
+ onWriteError: (rows, error) => {
11415
+ for (const r of rows) this.warnIndexFailure(r.id, error);
11416
+ }
11417
+ };
11418
+ }
11419
+ warnIndexFailure(factId, error) {
11420
+ retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
11421
+ }
10669
11422
  /** Best-effort semantic indexing: encode the fact and add it to the vstore. */
10670
11423
  async maybeIndexSemantic(factId, content) {
10671
- try {
10672
- if (!this.semantic.isAvailable()) {
10673
- this.semantic.ensureWarm?.();
10674
- return false;
10675
- }
10676
- const vec = await this.semantic.encode(content);
10677
- this.vstore.add(factId, vec);
10678
- this.facts.setSemanticVector(factId, float32ToBytes(vec), this.vectorSpace(), this.vstore.name);
10679
- return true;
10680
- } catch (error) {
10681
- retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
11424
+ if (!this.semantic.isAvailable()) {
11425
+ this.semantic.ensureWarm?.();
10682
11426
  return false;
10683
11427
  }
11428
+ const { encoded } = await encodeAndPersist([{
11429
+ id: factId,
11430
+ text: content
11431
+ }], this.vectorWriteTarget(this.vectorSpace()));
11432
+ return encoded === 1;
10684
11433
  }
10685
11434
  async update(fact_id, content, category, ttlDays, opts) {
10686
11435
  if (!this.getRow(fact_id)) throw new Error(`找不到 fact_id=${fact_id}`);
@@ -10775,7 +11524,7 @@ var MemoryStore = class {
10775
11524
  this.facts.reinforceOutcome(fact_id, outcome.next, clock, outcome.bonusCount, outcome.windowAt);
10776
11525
  return null;
10777
11526
  }).immediate();
10778
- if (evict !== null) this.evictVectors([evict]);
11527
+ if (evict !== null) evictVectors(this.vstore, [evict]);
10779
11528
  return this.getRow(fact_id)?.helpful_count ?? 0;
10780
11529
  }
10781
11530
  /** Permanent-memory administration (spec §6). */
@@ -11070,168 +11819,103 @@ var MemoryStore = class {
11070
11819
  };
11071
11820
  }
11072
11821
  /**
11073
- * Repair the semantic read model, in ONE bounded slice or in full:
11074
- * (a) reload persisted, dim-valid vectors missing from the live index
11075
- * (restart / index loss — needs no model),
11076
- * (b) drop persisted vectors whose dim no longer matches `semantic.dim` so a
11077
- * re-encode can replace them (never `add()` a stale vector — the store
11078
- * throws on dim mismatch),
11079
- * (c) encode active facts that have no vector yet (needs the semantic backend),
11080
- * (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
11081
- * adoption of a store predating the space id).
11082
- *
11083
- * `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
11084
- * dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
11085
- * yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
11086
- * `migrateVectorsBatch` passes the migration's batch size.
11822
+ * This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
11823
+ * steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
11824
+ * `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
11825
+ * for both stores.
11087
11826
  */
11088
- async repairVectors(opts) {
11089
- const limit = opts.limit ?? Number.POSITIVE_INFINITY;
11090
- const c = this.classifyVectors();
11091
- const loadedNow = this.semantic.isAvailable();
11092
- let semAvailable = loadedNow;
11093
- if (!semAvailable && !opts.dryRun) semAvailable = await this.warmupSemantic();
11094
- const wouldWarm = !loadedNow;
11095
- const remainingOf = (fixed) => Math.max(0, c.staleIds.length + c.spaceStaleIds.length + c.missing - fixed);
11096
- if (opts.dryRun) return {
11097
- report: {
11098
- missing: c.missing,
11099
- stale: c.staleIds.length,
11100
- space_stale: c.spaceStaleIds.length,
11101
- unindexed: c.unindexed.length,
11102
- reindexed: 0,
11103
- dropped: 0,
11104
- fixed: 0,
11105
- semantic_available: semAvailable,
11106
- would_warm: wouldWarm,
11107
- dry_run: true
11108
- },
11109
- remaining: remainingOf(0)
11827
+ vectorRepairTarget() {
11828
+ return {
11829
+ store: "memory",
11830
+ manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)",
11831
+ vectorSpace: () => this.vectorSpace(),
11832
+ semanticAvailable: () => this.semantic.isAvailable(),
11833
+ autoMigrate: () => this.config.semantic.auto_migrate,
11834
+ vectorSpaceHealth: () => this.vectorSpaceHealth(),
11835
+ missingVectorCount: () => this.facts.vectorCounts().missing,
11836
+ classifyVectors: (limit) => this.classifyRepairVectors(limit),
11837
+ reindexUsable: (rows) => this.reindexUsableVectors(rows),
11838
+ dropVectors: (ids) => this.dropRepairVectors(ids),
11839
+ reencodeVectors: (limit) => this.reencodeMissingVectors(limit),
11840
+ warmup: () => this.warmupSemantic()
11110
11841
  };
11111
- let reindexed = 0;
11112
- for (const u of c.unindexed.slice(0, limit)) {
11113
- this.vstore.add(u.id, u.vec);
11114
- reindexed++;
11115
- }
11116
- let dropped = 0;
11117
- if (semAvailable) {
11118
- const toDrop = [...c.staleIds, ...c.spaceStaleIds].slice(0, limit);
11119
- if (toDrop.length) {
11120
- const spaceStale = new Set(c.spaceStaleIds);
11121
- dropped = this.facts.clearVectors(toDrop);
11122
- this.evictVectors(toDrop.filter((id) => spaceStale.has(id)));
11123
- }
11124
- }
11125
- let fixed = 0;
11126
- if (semAvailable) {
11127
- const missingRows = this.facts.missingVectorRows().slice(0, limit);
11128
- for (const m of missingRows) {
11129
- if (!await this.maybeIndexSemantic(m.fact_id, m.content)) continue;
11130
- fixed++;
11131
- }
11132
- }
11842
+ }
11843
+ /**
11844
+ * {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
11845
+ * (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
11846
+ */
11847
+ classifyRepairVectors(limit) {
11848
+ const c = this.classifyVectors();
11133
11849
  return {
11134
- report: {
11135
- missing: c.missing,
11136
- stale: c.staleIds.length,
11137
- space_stale: c.spaceStaleIds.length,
11138
- unindexed: c.unindexed.length,
11139
- reindexed,
11140
- dropped,
11141
- fixed,
11142
- semantic_available: semAvailable,
11143
- would_warm: wouldWarm,
11144
- dry_run: false
11145
- },
11146
- remaining: remainingOf(fixed)
11850
+ stale: c.staleIds.length,
11851
+ space_stale: c.spaceStaleIds.length,
11852
+ missing: c.missing,
11853
+ dropIds: [...c.staleIds, ...c.spaceStaleIds].slice(0, limit),
11854
+ unindexed: c.unindexed
11855
+ };
11856
+ }
11857
+ /** Put usable persisted vectors missing from the live index back into it (no model needed). */
11858
+ reindexUsableVectors(rows) {
11859
+ for (const u of rows) this.vstore.add(u.id, u.vec);
11860
+ return rows.length;
11861
+ }
11862
+ /**
11863
+ * Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
11864
+ * pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
11865
+ * the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
11866
+ * one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
11867
+ *
11868
+ * `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
11869
+ * the embedding leg was checked against is gone, so the check has to run again.
11870
+ */
11871
+ dropRepairVectors(ids) {
11872
+ if (!ids.length) return 0;
11873
+ const dropped = this.facts.clearVectors(ids);
11874
+ evictVectors(this.vstore, ids);
11875
+ return dropped;
11876
+ }
11877
+ /**
11878
+ * Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
11879
+ *
11880
+ * Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
11881
+ * the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
11882
+ * ran has not been checked.
11883
+ */
11884
+ async reencodeMissingVectors(limit) {
11885
+ const rows = this.facts.missingVectorRows().slice(0, limit).map((m) => ({
11886
+ id: m.fact_id,
11887
+ text: m.content
11888
+ }));
11889
+ if (!rows.length) return {
11890
+ encoded: 0,
11891
+ failed: 0
11147
11892
  };
11893
+ return await encodeAndPersist(rows, this.vectorWriteTarget(this.vectorSpace()));
11148
11894
  }
11149
11895
  /**
11150
11896
  * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
11151
11897
  * `dry_run` previews every count without writing — and without loading the model, which is why
11152
- * the preview also reports `would_warm` (see the contract's `VectorsFixReport`).
11898
+ * the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
11153
11899
  */
11154
11900
  async vectorsFix(dryRun = false) {
11155
- return (await this.repairVectors({ dryRun })).report;
11901
+ return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
11156
11902
  }
11157
11903
  /**
11158
11904
  * ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
11159
11905
  * re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
11160
11906
  * background migration ({@link migrateVectors}) and its tests drive.
11161
- *
11162
- * Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
11163
- * stale from the persisted width/space, so a restart simply continues where the last slice stopped.
11164
11907
  */
11165
11908
  async migrateVectorsBatch(batchSize = 16) {
11166
- const size = Math.max(1, Math.floor(batchSize));
11167
- const { report, remaining } = await this.repairVectors({
11168
- dryRun: false,
11169
- limit: size
11170
- });
11171
- return {
11172
- remaining,
11173
- migrated: report.fixed,
11174
- dropped: report.dropped,
11175
- reindexed: report.reindexed,
11176
- semantic_available: report.semantic_available
11177
- };
11909
+ return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
11178
11910
  }
11179
11911
  /**
11180
11912
  * Drive the bounded background migration until the store is current, the model is unavailable, or
11181
- * `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE never rejects: it is
11182
- * logged and reported as `remaining > 0`, which is what makes the next heartbeat a retry (the caller
11183
- * still owns the outer catch for a closed database, e.g. a shutdown racing the first read). Never
11184
- * blocks the event loop for longer than one batch.
11185
- *
11186
- * Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
11187
- * on background re-encoding can leave the loud warning and the manual entry as the only path.
11913
+ * `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
11914
+ * switch, the no-progress rule and the store-named logging all live in the shared flow
11915
+ * (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
11188
11916
  */
11189
11917
  async migrateVectors(opts = {}) {
11190
- const enabled = this.config.semantic.auto_migrate !== false;
11191
- const snapshot = () => ({
11192
- enabled,
11193
- remaining: this.migrationRemaining(),
11194
- migrated: 0,
11195
- dropped: 0,
11196
- reindexed: 0,
11197
- semantic_available: this.semantic.isAvailable()
11198
- });
11199
- if (!enabled) return snapshot();
11200
- let migrated = 0;
11201
- let dropped = 0;
11202
- let reindexed = 0;
11203
- let last = snapshot();
11204
- if (last.remaining === 0) return last;
11205
- for (;;) {
11206
- if (opts.shouldStop?.() === true) break;
11207
- let step;
11208
- try {
11209
- step = await this.migrateVectorsBatch(opts.batchSize);
11210
- } catch (error) {
11211
- retrievalLogger().warn(`memory vector migration: batch failed (${describeError$2(error)}) — ${String(this.migrationRemaining())} vector(s) still belong to an older embedding space; the next pass retries`);
11212
- break;
11213
- }
11214
- migrated += step.migrated;
11215
- dropped += step.dropped;
11216
- reindexed += step.reindexed;
11217
- last = {
11218
- ...step,
11219
- enabled,
11220
- migrated,
11221
- dropped,
11222
- reindexed
11223
- };
11224
- opts.onProgress?.(last);
11225
- if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
11226
- if (last.remaining === 0) break;
11227
- await yieldToEventLoop();
11228
- }
11229
- return last;
11230
- }
11231
- /** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
11232
- migrationRemaining() {
11233
- const health = this.vectorSpaceHealth();
11234
- return health.stale + health.space_stale + this.facts.vectorCounts().missing;
11918
+ return await driveVectorRepair(this.vectorRepairTarget(), opts);
11235
11919
  }
11236
11920
  /**
11237
11921
  * One lifecycle pass, owned by the store so the DB writes and the live-index
@@ -11246,7 +11930,7 @@ var MemoryStore = class {
11246
11930
  */
11247
11931
  async maintenance() {
11248
11932
  const result = runMaintenance(this.db, this.config, { budget: 0 });
11249
- this.evictVectors([...result.archived_ids, ...result.purged_ids]);
11933
+ evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
11250
11934
  this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
11251
11935
  this.vstore.compact?.();
11252
11936
  this.db.pragma("optimize");
@@ -11279,25 +11963,19 @@ var MemoryStore = class {
11279
11963
  * loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
11280
11964
  * every selected row's `content` before rebuilding the first one.
11281
11965
  *
11282
- * The in-flight guard is here rather than at the callers because the callers CAN overlap: the
11283
- * plugin's heartbeat and `maintenance` are both periodic, and `staleEntityRows` is ORDERED, so
11284
- * two concurrent passes would select the same rows and re-tag them twice. Overlap is not
11285
- * corruption (each fact is stamped in its own transaction) — it is duplicated mission in a
11286
- * single-threaded process. A pass that finds one running reports `skipped` and does nothing.
11287
- *
11288
- * `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
11289
- * caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
11966
+ * The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
11967
+ * live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
11968
+ * entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
11969
+ * selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
11290
11970
  */
11291
11971
  async reindexEntities(budget = ENTITY_SWEEP_BATCH) {
11292
- if (this.entitySweepInFlight) return {
11293
- rebuilt: 0,
11294
- deferred: this.facts.countStaleEntities(1),
11295
- skipped: true
11296
- };
11297
- this.entitySweepInFlight = true;
11298
- try {
11299
- const rows = this.facts.staleEntityRows(1, Math.max(0, Math.floor(budget)));
11300
- for (const row of rows) {
11972
+ return await sweepVersionedEntities({
11973
+ staleEntityBatch: (limit) => ({
11974
+ total: this.facts.countStaleEntities(1),
11975
+ rows: this.facts.staleEntityRows(1, limit)
11976
+ }),
11977
+ staleEntityCount: () => this.facts.countStaleEntities(1),
11978
+ extractAndWriteEntities: async (row) => {
11301
11979
  const tokens = await tagText(row.content);
11302
11980
  const entityWrites = normalizeEntityWrites(entitiesFromTokens(tokens, row.content));
11303
11981
  const entities = entityWrites.map((e) => e.name);
@@ -11312,15 +11990,11 @@ var MemoryStore = class {
11312
11990
  this.facts.setEntitiesVersion([row.fact_id], 1);
11313
11991
  this.facts.requeueConflictCheck([row.fact_id]);
11314
11992
  })();
11993
+ },
11994
+ onRowError: (row, error) => {
11995
+ retrievalLogger().warn(`memory entity sweep: fact ${String(row.fact_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next pass retries it`);
11315
11996
  }
11316
- return {
11317
- rebuilt: rows.length,
11318
- deferred: this.facts.countStaleEntities(1),
11319
- skipped: false
11320
- };
11321
- } finally {
11322
- this.entitySweepInFlight = false;
11323
- }
11997
+ }, this.entitySweepGuard, { budget });
11324
11998
  }
11325
11999
  /**
11326
12000
  * A fact left the ACTIVE corpus: retire the open conflicts that name it, so
@@ -11331,10 +12005,6 @@ var MemoryStore = class {
11331
12005
  retireConflicts(factIds) {
11332
12006
  this.contradictions.resolveForFacts(factIds);
11333
12007
  }
11334
- /** Drop vectors of archived/purged facts from the live index (batched). */
11335
- evictVectors(factIds) {
11336
- evictVectors(this.vstore, factIds);
11337
- }
11338
12008
  /**
11339
12009
  * Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
11340
12010
  * count. Archived (incl. superseded) facts are excluded — their entities are not
@@ -11382,17 +12052,7 @@ var MemoryStore = class {
11382
12052
  }
11383
12053
  /** Hybrid search over the memory corpus (semantic + FTS + entity Jaccard [+ HRR probe] via fusion). */
11384
12054
  async search(input) {
11385
- const result = await hybridSearch(this.hybridDeps(input), {
11386
- query: input.query,
11387
- limit: input.limit,
11388
- maxTokens: input.maxTokens,
11389
- queryVector: input.queryVector,
11390
- recordStats: input.recordStats,
11391
- floors: input.floors,
11392
- ...input.includeScores === void 0 ? {} : { includeScores: input.includeScores },
11393
- ...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs },
11394
- ...input.rewriteQuery === void 0 ? {} : { rewriteQuery: input.rewriteQuery }
11395
- });
12055
+ const result = await runHybridSearch(this.hybridDeps(input), input.query, input);
11396
12056
  return {
11397
12057
  hits: result.hits,
11398
12058
  degraded: result.degraded,
@@ -11407,10 +12067,7 @@ var MemoryStore = class {
11407
12067
  * fact→hit mapping, and what to do with the hits the caller actually receives.
11408
12068
  */
11409
12069
  hybridDeps(input) {
11410
- return {
11411
- kind: "memory",
11412
- config: this.config,
11413
- semantic: this.semantic,
12070
+ return makeHybridDeps("memory", this.config, this.semantic, {
11414
12071
  legs: (ctx) => this.searchLegs(input, ctx),
11415
12072
  texts: (ids) => this.loadTexts(ids),
11416
12073
  hits: (ranked, texts) => {
@@ -11420,7 +12077,7 @@ var MemoryStore = class {
11420
12077
  onReturn: (kept) => {
11421
12078
  if (input.track !== false) this.reinforce(kept.map((h) => h.ref_id));
11422
12079
  }
11423
- };
12080
+ });
11424
12081
  }
11425
12082
  /**
11426
12083
  * The memory legs: semantic, FTS, entity Jaccard, and — for `recall.probe` — the HRR probe.
@@ -11442,49 +12099,18 @@ var MemoryStore = class {
11442
12099
  const qEntities = Array.from(new Set((await extractEntities(ctx.query)).map((e) => e.name)));
11443
12100
  const anchors = qEntities.length > 0 ? selectAnchors(qEntities, this.entities.activeDocFrequency(qEntities), this.facts.countActive()) : [];
11444
12101
  const candidates = anchors.length > 0 ? this.entities.candidateFactsForAnyEntity(anchors, input.category, ctx.legCap, qEntities.length, 3, input.source) : [];
11445
- /**
11446
- * Wrap one leg's raw scores. `capped` must be measured on the RAW set (before the relevance
11447
- * floor): the floor removes the tail anyway, and deriving the flag from the floored size would
11448
- * erase the "this leg was cut at legCap" signal exactly when it bound.
11449
- */
11450
- const leg = (scores, weight, raw) => ({
11451
- weight,
11452
- scores,
11453
- capped: (raw ?? scores).size === ctx.legCap
11454
- });
12102
+ const runner = makeLegRunner(ctx);
11455
12103
  const ftsRaw = this.ftsPath(ctx.query, input.category, ctx.legCap, input.source);
11456
- const ftsFloored = applyTermFloor(ftsRaw, this.loadTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
12104
+ const ftsLeg = runner.term("fts", ftsRaw, this.loadTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts);
11457
12105
  const jaccardRaw = this.jaccardPath(anchors, qEntities.length, candidates);
11458
- const jaccardFloored = applyScoreFloor(jaccardRaw, ctx.floors.jaccard);
11459
- const jaccardLeg = {
11460
- ...leg(jaccardFloored.scores, ctx.weights.jaccard, jaccardRaw),
11461
- leg: "jaccard",
11462
- droppedByFloor: jaccardFloored.dropped
11463
- };
12106
+ const jaccardLeg = runner.scored("jaccard", jaccardRaw, ctx.weights.jaccard, ctx.floors.jaccard);
11464
12107
  const legs = [
11465
- ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) => {
11466
- const floored = applyScoreFloor(raw, ctx.floors.semantic);
11467
- return {
11468
- weight: ctx.weights.semantic,
11469
- scores: floored.scores,
11470
- leg: "semantic",
11471
- droppedByFloor: floored.dropped
11472
- };
11473
- }) : {
11474
- weight: ctx.weights.semantic,
11475
- scores: /* @__PURE__ */ new Map(),
11476
- leg: "semantic",
11477
- droppedByFloor: 0
11478
- },
12108
+ ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
11479
12109
  Promise.resolve(jaccardLeg),
11480
- {
11481
- ...leg(ftsFloored.scores, ctx.weights.fts, ftsRaw),
11482
- leg: "fts",
11483
- droppedByFloor: ftsFloored.dropped
11484
- }
12110
+ ftsLeg
11485
12111
  ];
11486
- if (input.includeHrr) legs.push(Promise.resolve(leg(this.hrrPath(ctx.query, qEntities, [...jaccardFloored.scores.keys()], input.category, ctx.legCap, candidates.length === 0, input.source), ctx.weights.jaccard)));
11487
- if (this.config.retriever.time_window) legs.push(Promise.resolve(leg(this.timeWindowPath(ctx.query, input.category, ctx.legCap, input.source), ctx.weights.jaccard)));
12112
+ if (input.includeHrr) legs.push(Promise.resolve(runner.plain(this.hrrPath(ctx.query, qEntities, [...jaccardLeg.scores.keys()], input.category, ctx.legCap, candidates.length === 0, input.source), ctx.weights.jaccard)));
12113
+ if (this.config.retriever.time_window) legs.push(Promise.resolve(runner.plain(this.timeWindowPath(ctx.query, input.category, ctx.legCap, input.source), ctx.weights.jaccard)));
11488
12114
  return legs;
11489
12115
  }
11490
12116
  /**
@@ -11609,18 +12235,22 @@ var MemoryStore = class {
11609
12235
  * Semantic leg. `queryVector` short-circuits the encode when the caller already
11610
12236
  * encoded this exact query (the cross-store router does, for both stores at once);
11611
12237
  * `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
11612
- * instead of re-encoding (performance review §7.7 / P8).
12238
+ * instead of re-encoding (performance review §7.7 / P8). The flow is shared
12239
+ * (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
12240
+ * notion of status/category/source, so candidates go through the DB and archived/purged facts,
12241
+ * other categories and other sources never leak into the leg.
11613
12242
  */
11614
12243
  async semanticPath(query, category, k, queryVector, onVector, source) {
11615
- const vec = queryVector ?? await this.semantic.encode(query);
11616
- if (vec.length !== this.vstore.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${this.vstore.dim}`);
11617
- onVector?.(vec);
11618
- const topk = this.vstore.topk(vec, Math.max(50, k));
11619
- if (!topk.length) return /* @__PURE__ */ new Map();
11620
- const allowed = new Set(this.facts.activeIdsIn(topk.map((t) => t.id), category, source));
11621
- const out = /* @__PURE__ */ new Map();
11622
- for (const t of topk) if (allowed.has(t.id)) out.set(t.id, t.score);
11623
- return out;
12244
+ return semanticLeg({
12245
+ query,
12246
+ k,
12247
+ queryVector,
12248
+ onVector,
12249
+ encode: (q) => this.semantic.encode(q),
12250
+ dim: this.vstore.dim,
12251
+ topk: (vec, n) => this.vstore.topk(vec, n),
12252
+ filterTopk: (ids) => new Set(this.facts.activeIdsIn(ids, category, source))
12253
+ });
11624
12254
  }
11625
12255
  /**
11626
12256
  * HRR entity-probe path: phase similarity between the query's entity bundle and
@@ -11689,21 +12319,20 @@ var MemoryStore = class {
11689
12319
  * semantic, why it must be synchronous, and how the bar itself was measured.
11690
12320
  */
11691
12321
  lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
11692
- return probeTerms(text, (term) => {
11693
- const fts = buildFtsQuery(term, this.ftsTokenizer);
11694
- return fts !== null && this.facts.ftsSearch(fts, void 0, 1).length > 0;
11695
- }, stopAt);
12322
+ return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.facts.ftsSearch(fts, void 0, 1).length > 0, stopAt);
11696
12323
  }
11697
12324
  ftsPath(query, category, cap, source) {
11698
- const ftsQuery = buildFtsQuery(query, this.ftsTokenizer);
11699
- if (ftsQuery) {
11700
- const rows = this.facts.ftsSearch(ftsQuery, category, cap, source);
11701
- return new Map(rows.map((r) => [r.id, -r.rank]));
11702
- }
11703
- const terms = substringTerms(query);
11704
- if (terms.length === 0) return /* @__PURE__ */ new Map();
11705
- const rows = this.facts.ftsSubstringSearch(terms, category, cap, source);
11706
- return new Map(rows.map((r) => [r.id, r.rank]));
12325
+ return ftsLeg({
12326
+ query,
12327
+ cap,
12328
+ scope: {
12329
+ category,
12330
+ source
12331
+ },
12332
+ tokenizer: this.ftsTokenizer,
12333
+ search: (ftsQuery, scope, limit) => this.facts.ftsSearch(ftsQuery, scope.category, limit, scope.source),
12334
+ substringSearch: (terms, scope, limit) => this.facts.ftsSubstringSearch(terms, scope.category, limit, scope.source)
12335
+ });
11707
12336
  }
11708
12337
  /**
11709
12338
  * Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
@@ -11810,7 +12439,7 @@ var MemoryStore = class {
11810
12439
  supersededId
11811
12440
  };
11812
12441
  })();
11813
- if (supersededId !== null) this.evictVectors([supersededId]);
12442
+ if (supersededId !== null) evictVectors(this.vstore, [supersededId]);
11814
12443
  return result;
11815
12444
  }
11816
12445
  /**
@@ -12179,10 +12808,7 @@ var ChunksDao = class {
12179
12808
  }
12180
12809
  /** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
12181
12810
  setEntitiesVersion(ids, version) {
12182
- for (const batch of batches(ids)) {
12183
- const { placeholders, values } = inList(batch);
12184
- this.db.prepare(`UPDATE doc_chunks SET entities_version = ? WHERE chunk_id IN (${placeholders})`).run(version, ...values);
12185
- }
12811
+ setEntitiesVersionBatch(this.db, "doc_chunks", "chunk_id", ids, version);
12186
12812
  }
12187
12813
  /** Every persisted chunk vector, for rebuilding the in-memory index at open. */
12188
12814
  /**
@@ -12195,14 +12821,68 @@ var ChunksDao = class {
12195
12821
  vectorRows() {
12196
12822
  return this.db.prepare("SELECT chunk_id AS id, semantic_vector AS vec, embedding_model AS embedding_model FROM doc_chunks WHERE semantic_vector IS NOT NULL").all();
12197
12823
  }
12824
+ /**
12825
+ * Vector-space health of the WHOLE chunk corpus, by kind, WITHOUT decoding a single blob.
12826
+ *
12827
+ * The same two counts the open-time warning reports (`stale` = wrong width, `space_stale` = same
12828
+ * width but another recorded space — `embedding_model IS NOT` is null-safe, so a row with no
12829
+ * recorded space counts as foreign exactly like the JS comparison in `reloadIndex`) plus `missing`
12830
+ * (no vector at all), which is the third reason the semantic leg cannot use a chunk.
12831
+ *
12832
+ * This is what lets the shared repair flow read "what is stale" cheaply per slice instead of
12833
+ * re-scanning and hashing the corpus (`corpusState` reads every text; a batched migration must
12834
+ * not).
12835
+ */
12836
+ vectorSpaceCounts(space, expectedBytes) {
12837
+ const row = this.db.prepare(`SELECT
12838
+ COALESCE(SUM(CASE WHEN semantic_vector IS NULL THEN 1 ELSE 0 END), 0) AS missing,
12839
+ COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) != :bytes THEN 1 ELSE 0 END), 0) AS stale,
12840
+ COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) = :bytes AND embedding_model IS NOT :space THEN 1 ELSE 0 END), 0) AS space_stale
12841
+ FROM doc_chunks`).get({
12842
+ bytes: expectedBytes,
12843
+ space
12844
+ });
12845
+ return {
12846
+ stale: Number(row?.stale ?? 0),
12847
+ space_stale: Number(row?.space_stale ?? 0),
12848
+ missing: Number(row?.missing ?? 0)
12849
+ };
12850
+ }
12851
+ /** Chunks with no persisted vector — the repair flow's `missing` term in `remaining`. */
12852
+ countMissingVectors() {
12853
+ return Number(this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks WHERE semantic_vector IS NULL").get()?.n ?? 0);
12854
+ }
12855
+ /**
12856
+ * Chunks the semantic leg cannot use (no vector / wrong width / another space), bounded and
12857
+ * ordered by id — the repair flow's encode source.
12858
+ *
12859
+ * The content-hash check `vectorReusable` adds is deliberately NOT here: a chunk whose vector is
12860
+ * in the current space is usable even if its text was edited behind the store's back, and that
12861
+ * (rarer) case belongs to `kb_reindex`, which re-derives every leg. Repairing the SPACE must not
12862
+ * pay a jieba/hash pass over the corpus to find rows the space change did not touch.
12863
+ */
12864
+ staleVectorRows(space, expectedBytes, limit) {
12865
+ const bound = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : -1;
12866
+ return this.db.prepare(`SELECT chunk_id AS chunk_id, text AS text FROM doc_chunks
12867
+ WHERE semantic_vector IS NULL OR length(semantic_vector) != :bytes OR embedding_model IS NOT :space
12868
+ ORDER BY chunk_id ASC LIMIT :limit`).all({
12869
+ bytes: expectedBytes,
12870
+ space,
12871
+ limit: bound
12872
+ });
12873
+ }
12198
12874
  count() {
12199
12875
  return this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks").get().n;
12200
12876
  }
12201
12877
  /**
12202
- * Replace the entity rows of the given chunks in one transaction (`replace` clears the
12203
- * old rows first — used by reindex, where the extractor may have changed).
12878
+ * Replace the entity rows of the given chunks in one transaction.
12879
+ *
12880
+ * `replace` is REQUIRED, not defaulted: `true` clears the old rows first (the entity sweep, where
12881
+ * the extractor may have changed), `false` is the ingest path, where the chunk ids are freshly
12882
+ * inserted and an absent old row set. A default would let one caller silently inherit the other's
12883
+ * semantics — an extra `DELETE` per chunk on ingest, or an append-on-top-of-stale sweep.
12204
12884
  */
12205
- replaceEntities(rows, replace = false) {
12885
+ replaceEntities(rows, replace) {
12206
12886
  const del = this.db.prepare("DELETE FROM chunk_entities WHERE chunk_id = ?");
12207
12887
  const ins = this.db.prepare("INSERT OR IGNORE INTO chunk_entities (chunk_id, name) VALUES (?, ?)");
12208
12888
  this.db.transaction(() => {
@@ -12214,17 +12894,12 @@ var ChunksDao = class {
12214
12894
  }
12215
12895
  /** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
12216
12896
  entityBags(ids) {
12217
- const out = /* @__PURE__ */ new Map();
12218
- for (const batch of batches(ids)) {
12219
- const { placeholders, values } = inList(batch);
12220
- const rows = this.db.prepare(`SELECT chunk_id, name FROM chunk_entities WHERE chunk_id IN (${placeholders})`).all(...values);
12221
- for (const r of rows) {
12222
- const list = out.get(r.chunk_id);
12223
- if (list) list.push(r.name);
12224
- else out.set(r.chunk_id, [r.name]);
12225
- }
12226
- }
12227
- return out;
12897
+ return entityBags(this.db, ids, {
12898
+ table: "chunk_entities",
12899
+ keyColumn: "chunk_id",
12900
+ nameColumn: "name",
12901
+ seedEmpty: false
12902
+ });
12228
12903
  }
12229
12904
  /** Chunk body + owning-document identity for the fused candidate ids. */
12230
12905
  hits(ids) {
@@ -12264,13 +12939,48 @@ var ChunksDao = class {
12264
12939
  JOIN documents d ON d.doc_id = dc.doc_id
12265
12940
  WHERE (? IS NULL OR d.domain = ?)`).all(domain ?? null, domain ?? null);
12266
12941
  }
12942
+ /**
12943
+ * Chunk rows whose `chunk_entities` were produced by OLDER extraction rules — the entity sweep's
12944
+ * selection, bounded and ordered by id.
12945
+ *
12946
+ * `COALESCE(entities_version, 0) < :version`, not `entities_version != :version`: the column is
12947
+ * NULLABLE on purpose (a row written before the derived-state columns existed must look stale
12948
+ * exactly once and be adopted by the first rebuild — see `ChunkStateRow`), and a bare `<` would
12949
+ * evaluate to NULL for those rows and let them hide from the sweep forever.
12950
+ *
12951
+ * The predicate is index-served in id order (`doc_chunks` is keyed by `chunk_id`), so the LIMIT
12952
+ * bounds what is READ, not only what is rebuilt — the same property `FactsDao.staleEntityRows`
12953
+ * documents for the memory side.
12954
+ */
12955
+ chunkEntityRows(version, limit, domain) {
12956
+ return this.db.prepare(`SELECT dc.chunk_id AS chunk_id, dc.text AS text
12957
+ FROM doc_chunks dc
12958
+ JOIN documents d ON d.doc_id = dc.doc_id
12959
+ WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)
12960
+ ORDER BY dc.chunk_id ASC LIMIT :limit`).all({
12961
+ version,
12962
+ limit,
12963
+ domain: domain ?? null
12964
+ });
12965
+ }
12966
+ /** How many chunks still carry entity rows from older rules — the sweep's `deferred` source. */
12967
+ countStaleEntityChunks(version, domain) {
12968
+ return Number(this.db.prepare(`SELECT COUNT(*) AS n
12969
+ FROM doc_chunks dc
12970
+ JOIN documents d ON d.doc_id = dc.doc_id
12971
+ WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)`).get({
12972
+ version,
12973
+ domain: domain ?? null
12974
+ })?.n ?? 0);
12975
+ }
12267
12976
  /** Rebuild the FTS index from the content table (write-path maintenance). */
12268
12977
  rebuildFts() {
12269
12978
  this.db.exec("INSERT INTO doc_chunks_fts(doc_chunks_fts) VALUES('rebuild')");
12270
12979
  }
12271
12980
  /** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
12272
12981
  /**
12273
- * Entity-overlap candidates, most-shared first and CAPPED (see the memory store's sibling).
12982
+ * Entity-overlap candidates, highest-score first and CAPPED (see the memory store's sibling;
12983
+ * the score/order tail itself lives in `./shared.js#entityCandidateTail`).
12274
12984
  *
12275
12985
  * `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
12276
12986
  * `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
@@ -12280,11 +12990,19 @@ var ChunksDao = class {
12280
12990
  * of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
12281
12991
  */
12282
12992
  candidatesByEntityNames(names, domain, source, limit, queryWidth, widthCap) {
12283
- if (limit <= 0) return [];
12284
- const out = /* @__PURE__ */ new Set();
12285
- for (const batch of batches(names)) {
12286
- const { placeholders, values } = inList(batch);
12287
- const rows = this.db.prepare(`SELECT ce.chunk_id AS chunk_id, COUNT(*) AS shared,
12993
+ return entityCandidates(this.db, {
12994
+ names,
12995
+ filterParams: [
12996
+ domain ?? null,
12997
+ domain ?? null,
12998
+ source ?? null,
12999
+ source ?? null
13000
+ ],
13001
+ limit,
13002
+ queryWidth,
13003
+ widthCap,
13004
+ batchNames: true,
13005
+ sql: (placeholders) => `SELECT ce.chunk_id AS id, COUNT(*) AS shared,
12288
13006
  (SELECT COUNT(*) FROM chunk_entities x WHERE x.chunk_id = ce.chunk_id) AS total
12289
13007
  FROM chunk_entities ce
12290
13008
  JOIN doc_chunks dc ON dc.chunk_id = ce.chunk_id
@@ -12295,12 +13013,8 @@ var ChunksDao = class {
12295
13013
  -- removed document's chunks out of a result is the JOIN itself.
12296
13014
  WHERE ce.name IN (${placeholders})
12297
13015
  AND (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?)
12298
- GROUP BY ce.chunk_id
12299
- ORDER BY (CAST(shared AS REAL) / (? + MIN(total - shared, ?))) DESC, total ASC, ce.chunk_id ASC
12300
- LIMIT ?`).all(...values, domain ?? null, domain ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit);
12301
- for (const r of rows) out.add(r.chunk_id);
12302
- }
12303
- return [...out];
13016
+ ${entityCandidateTail("ce.chunk_id")}`
13017
+ });
12304
13018
  }
12305
13019
  /**
12306
13020
  * How many chunks carry each name — the knowledge side of the entity leg's document frequency.
@@ -12310,12 +13024,10 @@ var ChunksDao = class {
12310
13024
  * all chunks, which is the population the leg ranks.
12311
13025
  */
12312
13026
  docFrequency(names) {
12313
- const out = /* @__PURE__ */ new Map();
12314
- if (!names.length) return out;
12315
- const { placeholders, values } = inList(names);
12316
- const rows = this.db.prepare(`SELECT name, COUNT(*) AS df FROM chunk_entities WHERE name IN (${placeholders}) GROUP BY name`).all(...values);
12317
- for (const row of rows) out.set(row.name, Number(row.df));
12318
- return out;
13027
+ return queryDocFrequency(this.db, names, {
13028
+ from: " FROM chunk_entities",
13029
+ nameExpr: "name"
13030
+ });
12319
13031
  }
12320
13032
  /** FTS leg (`bm25` rank; negated by the caller so higher = better). */
12321
13033
  ftsSearch(ftsQuery, domain, source, limit) {
@@ -12335,14 +13047,17 @@ var ChunksDao = class {
12335
13047
  * the domain/source filter lives.
12336
13048
  */
12337
13049
  ftsSubstringSearch(terms, domain, source, limit) {
12338
- const { any, count, params } = likeSubstring("dc.text", terms);
12339
- if (params.length === 0) return [];
12340
- return this.db.prepare(`SELECT dc.chunk_id AS id, ${count} AS rank
13050
+ return likeSubstringLeg(this.db, "dc.text", terms, ({ any, count }) => `SELECT dc.chunk_id AS id, ${count} AS rank
12341
13051
  FROM doc_chunks dc
12342
13052
  JOIN documents d ON d.doc_id = dc.doc_id
12343
13053
  WHERE (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?) AND (${any})
12344
13054
  ORDER BY rank DESC, dc.chunk_id ASC
12345
- LIMIT ?`).all(...params, domain ?? null, domain ?? null, source ?? null, source ?? null, ...params, limit ?? -1);
13055
+ LIMIT ?`, [
13056
+ domain ?? null,
13057
+ domain ?? null,
13058
+ source ?? null,
13059
+ source ?? null
13060
+ ], limit);
12346
13061
  }
12347
13062
  };
12348
13063
  //#endregion
@@ -13649,6 +14364,15 @@ var KnowledgeStore = class {
13649
14364
  /** > 0 while a batch (directory import) is writing: commits and baseline refreshes wait. */
13650
14365
  deferCorpusWrites = 0;
13651
14366
  /**
14367
+ * Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
14368
+ *
14369
+ * The guard itself lives in that module; this is only the per-store identity it keys on. It exists
14370
+ * as a field — rather than the adapter object, which is built per call — because the guard must
14371
+ * survive between two overlapping calls: `reindex` can be reached from a tool call and the
14372
+ * settings-page button at once, and both would select the same stale chunks.
14373
+ */
14374
+ entitySweepGuard = {};
14375
+ /**
13652
14376
  * Per-file stamps (`path → mtimeMs:size`) and the file-set they were taken from, so a reconcile
13653
14377
  * can ask "what changed?" for the price of one `stat` per document instead of reading and
13654
14378
  * hashing the whole corpus. `null` = no baseline yet.
@@ -13696,37 +14420,27 @@ var KnowledgeStore = class {
13696
14420
  this.reloadIndex();
13697
14421
  }
13698
14422
  /**
13699
- * Rebuild the in-memory vstore from persisted chunk vectors.
14423
+ * Rebuild the in-memory vstore from persisted chunk vectors, and say loudly what belongs to
14424
+ * another vector space.
14425
+ *
14426
+ * Also detects a VECTOR-SPACE change, which the reload itself cannot see: the reload checks the
14427
+ * width, and a different model at the same width produces dim-valid vectors that are then ranked
14428
+ * against the current query's — two spaces mixed in one index, with no error anywhere.
13700
14429
  *
13701
- * Also detects a VECTOR-SPACE change, which the reload itself cannot see: `reloadVectorIndex`
13702
- * checks the width, and a different model at the same width produces dim-valid vectors that are
13703
- * then ranked against the current query's — two spaces mixed in one index, with no error anywhere.
13704
- * The memory store always said something about this; the knowledge store did not, so a model swap
13705
- * degraded every semantic query here until someone happened to run `kb_reindex`.
14430
+ * The reload, the count and the warning are the SHARED open-time step (`store/vector_repair.ts`),
14431
+ * which is what turned the knowledge store from "warns about a swapped model but never repairs it"
14432
+ * into the same automatic path memory has always had; this store's half is the row read. The
14433
+ * manual entry stays `vectors --fix` (the vector-only repair); `kb_reindex` remains the full
14434
+ * FTS + entities + vectors re-entry.
13706
14435
  */
13707
14436
  reloadIndex() {
13708
- const space = this.vectorSpace();
13709
- const persisted = this.chunks.vectorRows();
13710
- reloadVectorIndex(this.vstore, persisted, "knowledge");
13711
- let stale = 0;
13712
- let spaceStale = 0;
13713
- let legacy = 0;
13714
- const expectedBytes = this.vstore.dim * 4;
13715
- for (const row of persisted) {
13716
- if (!row.vec || row.vec.byteLength !== expectedBytes) {
13717
- stale++;
13718
- continue;
13719
- }
13720
- if (row.embedding_model !== space) {
13721
- spaceStale++;
13722
- if (isPreFingerprintSpace(row.embedding_model)) legacy++;
13723
- }
13724
- }
13725
- reportStaleVectors("knowledge", space, {
13726
- stale,
13727
- space_stale: spaceStale,
13728
- legacy
13729
- }, "`avantf-mem kb reindex` (MCP/plugin: `kb_reindex`)");
14437
+ loadVectorIndex({
14438
+ store: "knowledge",
14439
+ vstore: this.vstore,
14440
+ rows: this.chunks.vectorRows(),
14441
+ space: this.vectorSpace(),
14442
+ manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)"
14443
+ });
13730
14444
  }
13731
14445
  /**
13732
14446
  * The write-side domain allowlist AND the 知识页 picker's data, in one place: the configured
@@ -14005,7 +14719,7 @@ var KnowledgeStore = class {
14005
14719
  reusable
14006
14720
  };
14007
14721
  }).immediate();
14008
- this.evictVectors(result.oldChunkIds);
14722
+ evictVectors(this.vstore, result.oldChunkIds);
14009
14723
  const indexed = await this.indexChunks(result.newChunkIds, result.reusable);
14010
14724
  return {
14011
14725
  doc_id: result.doc_id,
@@ -14042,10 +14756,6 @@ var KnowledgeStore = class {
14042
14756
  return { file_error: error instanceof Error ? error.message : String(error) };
14043
14757
  }
14044
14758
  }
14045
- /** Batched eviction from the live vector index (removeMany when the backend supports it). */
14046
- evictVectors(ids) {
14047
- evictVectors(this.vstore, ids);
14048
- }
14049
14759
  /** The resolved ingestion boundary (see `ingest_guard.ts`). */
14050
14760
  get ingestLimits() {
14051
14761
  return this.kbConfig.ingest;
@@ -14168,7 +14878,8 @@ var KnowledgeStore = class {
14168
14878
  };
14169
14879
  }
14170
14880
  /**
14171
- * Encode texts ONE AT A TIME, writing every `WRITE_BATCH` vectors in one transaction.
14881
+ * This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
14882
+ * call, the live index and the `WRITE_BATCH` transaction this store measured its way into.
14172
14883
  *
14173
14884
  * The model call is deliberately NOT batched, and that is a MEASURED decision, not an omission:
14174
14885
  * a batched forward pass pads each item to the longest in its group, and real chunks span 53–550
@@ -14179,34 +14890,43 @@ var KnowledgeStore = class {
14179
14890
  * length-sorted) for callers whose texts suit it.
14180
14891
  *
14181
14892
  * What DID pay off is the write side: one commit per `WRITE_BATCH` chunks instead of one per
14182
- * chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await`.
14893
+ * chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await` — that
14894
+ * is the `writeBatch` below.
14183
14895
  */
14184
- async encodeAndStore(rows, space) {
14185
- let encoded = 0;
14186
- let failed = 0;
14187
- for (let i = 0; i < rows.length; i += WRITE_BATCH) {
14188
- const group = rows.slice(i, i + WRITE_BATCH);
14189
- const writes = [];
14190
- for (const r of group) try {
14191
- const vec = await this.semantic.encode(r.text);
14192
- this.vstore.add(r.chunk_id, vec);
14193
- writes.push({
14194
- chunk_id: r.chunk_id,
14195
- bytes: float32ToBytes(vec),
14896
+ encodeWriteTarget(space) {
14897
+ return {
14898
+ store: "knowledge",
14899
+ space,
14900
+ encode: (text) => this.semantic.encode(text),
14901
+ add: (id, vec) => {
14902
+ this.vstore.add(id, vec);
14903
+ },
14904
+ write: (writes) => {
14905
+ this.chunks.setVectors(writes.map((w) => ({
14906
+ chunk_id: w.id,
14907
+ bytes: w.bytes,
14196
14908
  embeddingModel: space
14197
- });
14198
- } catch {
14199
- failed += 1;
14200
- }
14201
- if (!writes.length) continue;
14202
- try {
14203
- this.chunks.setVectors(writes);
14204
- encoded += writes.length;
14205
- } catch {
14206
- failed += writes.length;
14207
- this.evictVectors(writes.map((w) => w.chunk_id));
14909
+ })));
14910
+ },
14911
+ writeBatch: WRITE_BATCH,
14912
+ onWriteError: (writes) => {
14913
+ evictVectors(this.vstore, writes.map((w) => w.id));
14208
14914
  }
14209
- }
14915
+ };
14916
+ }
14917
+ /**
14918
+ * Encode texts one at a time and persist them in `WRITE_BATCH` transactions.
14919
+ *
14920
+ * A failed chunk is skipped, not thrown: the rows are already committed and their old vectors
14921
+ * evicted by the ingest path, so a throw here would leave the caller with a failed ingest and NO
14922
+ * COUNT of what was skipped. The shortfall travels back to the tool result
14923
+ * (`IngestResult.vectors_failed`) instead of living only in a log line.
14924
+ */
14925
+ async encodeAndStore(rows, space) {
14926
+ const { encoded, failed } = await encodeAndPersist(rows.map((r) => ({
14927
+ id: r.chunk_id,
14928
+ text: r.text
14929
+ })), this.encodeWriteTarget(space));
14210
14930
  if (failed > 0) retrievalLogger().warn(`kb index: ${String(failed)} of ${String(rows.length)} chunk vector(s) were not written — run kb_reindex to retry (the text/entity/FTS indexes are complete; only the semantic vectors are affected)`);
14211
14931
  return {
14212
14932
  encoded,
@@ -14223,6 +14943,102 @@ var KnowledgeStore = class {
14223
14943
  return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
14224
14944
  }
14225
14945
  /**
14946
+ * Cheapest possible DETECTION of "the embedding space changed under this store": counts of
14947
+ * persisted chunk vectors whose width or recorded space id no longer matches the one this store
14948
+ * writes into. Reads blob LENGTHS only — never decodes — so the startup warning and the background
14949
+ * drive can afford it at any corpus size.
14950
+ */
14951
+ vectorSpaceHealth() {
14952
+ const c = this.vectorSpaceCounts();
14953
+ return {
14954
+ stale: c.stale,
14955
+ space_stale: c.space_stale
14956
+ };
14957
+ }
14958
+ /** {@link vectorSpaceHealth} plus `missing`, the third term of the repair flow's `remaining`. */
14959
+ vectorSpaceCounts() {
14960
+ return this.chunks.vectorSpaceCounts(this.vectorSpace(), this.vstore.dim * 4);
14961
+ }
14962
+ /** Async bootstrap the semantic backend (the shared flow's explicit-fix warmup). */
14963
+ async warmupSemantic() {
14964
+ await (this.semantic.warmUp?.() ?? Promise.resolve());
14965
+ return this.semantic.isAvailable();
14966
+ }
14967
+ /**
14968
+ * This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
14969
+ * steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
14970
+ * `semantic.auto_migrate`, the dry-run split — lives there once, for both stores.
14971
+ */
14972
+ vectorRepairTarget() {
14973
+ return {
14974
+ store: "knowledge",
14975
+ manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)",
14976
+ vectorSpace: () => this.vectorSpace(),
14977
+ semanticAvailable: () => this.semantic.isAvailable(),
14978
+ autoMigrate: () => this.config.semantic.auto_migrate,
14979
+ vectorSpaceHealth: () => this.vectorSpaceHealth(),
14980
+ missingVectorCount: () => this.chunks.countMissingVectors(),
14981
+ classifyVectors: () => this.classifyRepairVectors(),
14982
+ dropVectors: () => 0,
14983
+ reencodeVectors: (limit) => this.reencodeStaleChunks(limit),
14984
+ warmup: () => this.warmupSemantic()
14985
+ };
14986
+ }
14987
+ /**
14988
+ * The chunks the semantic leg cannot use (no vector / wrong width / another space). One SQL
14989
+ * aggregate over blob lengths; `dropIds` is empty because knowledge overwrites in place.
14990
+ */
14991
+ classifyRepairVectors() {
14992
+ const c = this.vectorSpaceCounts();
14993
+ return {
14994
+ stale: c.stale,
14995
+ space_stale: c.space_stale,
14996
+ missing: c.missing,
14997
+ dropIds: []
14998
+ };
14999
+ }
15000
+ /**
15001
+ * Encode + persist up to `limit` chunks the semantic leg cannot use, bounded and ordered by id.
15002
+ *
15003
+ * It does NOT re-extract entities or rebuild FTS: that is `kb_reindex`'s job, and the whole point
15004
+ * of the automatic repair is that the vector half can run on its own — which is exactly the
15005
+ * objection the "known boundary" in DESIGN §20 used to raise against a background run.
15006
+ */
15007
+ async reencodeStaleChunks(limit) {
15008
+ const space = this.vectorSpace();
15009
+ const rows = this.chunks.staleVectorRows(space, this.vstore.dim * 4, limit);
15010
+ if (!rows.length) return {
15011
+ encoded: 0,
15012
+ failed: 0
15013
+ };
15014
+ return await this.encodeAndStore(rows, space);
15015
+ }
15016
+ /**
15017
+ * The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
15018
+ * `dry_run` previews every count without writing — and without loading the model, which is why
15019
+ * the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
15020
+ */
15021
+ async vectorsFix(dryRun = false) {
15022
+ return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
15023
+ }
15024
+ /**
15025
+ * ONE bounded slice of the vector-space migration: at most `batchSize` chunks are re-encoded, so
15026
+ * the caller keeps control of the event loop between slices. This is the unit the background
15027
+ * migration ({@link migrateVectors}) and its tests drive.
15028
+ */
15029
+ async migrateVectorsBatch(batchSize = 16) {
15030
+ return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
15031
+ }
15032
+ /**
15033
+ * Drive the bounded background migration for this store until it is current, the model is
15034
+ * unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the
15035
+ * `semantic.auto_migrate` switch, the no-progress rule and the store-named logging all live in the
15036
+ * shared flow (`store/vector_repair.ts`); this is the one-store entry.
15037
+ */
15038
+ async migrateVectors(opts = {}) {
15039
+ return await driveVectorRepair(this.vectorRepairTarget(), opts);
15040
+ }
15041
+ /**
14226
15042
  * A stored vector is reusable when it exists, matches the space, and its text is unchanged.
14227
15043
  *
14228
15044
  * The three checks answer different questions, and the hash is deliberately the RAREST of them
@@ -14236,17 +15052,43 @@ var KnowledgeStore = class {
14236
15052
  if (row.embedding_model !== space) return false;
14237
15053
  return row.content_hash === contentHash(row.text);
14238
15054
  }
14239
- /** One transaction replacing the entity rows of the given chunks (shared by ingest + reindex). */
14240
- async replaceChunkEntities(rows, replace = false) {
15055
+ /**
15056
+ * Extract one chunk's entity names, bounded by `MAX_ENTITIES_PER_CHUNK`.
15057
+ *
15058
+ * The bound is here, not in either caller: it is a property of what `chunk_entities` may hold for
15059
+ * one chunk, and the ingest path and the sweep must not disagree about it.
15060
+ */
15061
+ async extractRowEntities(text) {
15062
+ return (await extractEntities(text)).map((e) => e.name).slice(0, MAX_ENTITIES_PER_CHUNK);
15063
+ }
15064
+ /** One transaction replacing the entity rows of the given chunks (the ingest path). */
15065
+ async replaceChunkEntities(rows) {
14241
15066
  const extracted = [];
14242
- for (const r of rows) {
14243
- const names = (await extractEntities(r.text)).map((e) => e.name).slice(0, MAX_ENTITIES_PER_CHUNK);
14244
- extracted.push({
14245
- chunk_id: r.chunk_id,
14246
- names
14247
- });
14248
- }
14249
- this.chunks.replaceEntities(extracted, replace);
15067
+ for (const r of rows) extracted.push({
15068
+ chunk_id: r.chunk_id,
15069
+ names: await this.extractRowEntities(r.text)
15070
+ });
15071
+ this.chunks.replaceEntities(extracted, false);
15072
+ }
15073
+ /**
15074
+ * The per-row primitive of the shared entity sweep (`store/entity_sweep.ts`).
15075
+ *
15076
+ * Replaces ONE chunk's entity rows and stamps that same row with `ENTITY_EXTRACTOR_VERSION`, in one
15077
+ * transaction. Knowledge's domain difference from memory is exactly this: memory's transaction also
15078
+ * moves triples, the HRR bundle and the conflict queue, while a chunk has none of those — its
15079
+ * derived state is the entity names alone.
15080
+ *
15081
+ * Per row, not per batch, because the sweep's contract is that a failure leaves THAT row stale and
15082
+ * retryable: inside a batched transaction one bad extraction would roll back the whole batch, and
15083
+ * rows it never reached would look stamped.
15084
+ */
15085
+ async replaceRowEntities(chunkId, text) {
15086
+ const names = await this.extractRowEntities(text);
15087
+ this.chunks.replaceEntities([{
15088
+ chunk_id: chunkId,
15089
+ names
15090
+ }], true);
15091
+ this.chunks.setEntitiesVersion([chunkId], 1);
14250
15092
  }
14251
15093
  /**
14252
15094
  * Ingest local files/directories. Every path goes through the SAME boundary as
@@ -14330,7 +15172,7 @@ var KnowledgeStore = class {
14330
15172
  const chunkIds = this.chunks.idsForDoc(docId);
14331
15173
  const removed = this.docs.remove(docId);
14332
15174
  if (removed) this.commitCorpus(`remove: ${doc.domain}/${doc.source}/${doc.title}`);
14333
- this.evictVectors(chunkIds);
15175
+ evictVectors(this.vstore, chunkIds);
14334
15176
  return removed;
14335
15177
  }
14336
15178
  /**
@@ -14522,12 +15364,11 @@ var KnowledgeStore = class {
14522
15364
  async reindex(domain, opts) {
14523
15365
  const dryRun = opts?.dryRun === true;
14524
15366
  const space = this.vectorSpace();
14525
- const rows = this.chunks.corpusState(domain);
14526
- const staleEntities = rows.filter((r) => r.entities_version !== 1);
14527
- const staleVectors = rows.filter((r) => !this.vectorReusable(r, space));
15367
+ const corpus = this.chunks.corpusState(domain);
15368
+ const staleVectors = corpus.filter((r) => !this.vectorReusable(r, space));
14528
15369
  const semanticAvailable = this.semantic.isAvailable();
14529
15370
  const report = {
14530
- chunks: rows.length,
15371
+ chunks: corpus.length,
14531
15372
  entities_rebuilt: 0,
14532
15373
  vectors_stale: staleVectors.length,
14533
15374
  vectors_encoded: 0,
@@ -14536,11 +15377,19 @@ var KnowledgeStore = class {
14536
15377
  };
14537
15378
  if (dryRun) return report;
14538
15379
  this.chunks.rebuildFts();
14539
- if (staleEntities.length) {
14540
- await this.replaceChunkEntities(staleEntities, true);
14541
- this.chunks.setEntitiesVersion(staleEntities.map((r) => r.chunk_id), 1);
14542
- report.entities_rebuilt = staleEntities.length;
14543
- }
15380
+ report.entities_rebuilt = (await sweepVersionedEntities({
15381
+ staleEntityBatch: (limit) => ({
15382
+ total: this.chunks.countStaleEntityChunks(1, domain),
15383
+ rows: this.chunks.chunkEntityRows(1, limit, domain)
15384
+ }),
15385
+ staleEntityCount: () => this.chunks.countStaleEntityChunks(1, domain),
15386
+ extractAndWriteEntities: async (row) => {
15387
+ await this.replaceRowEntities(row.chunk_id, row.text);
15388
+ },
15389
+ onRowError: (row, error) => {
15390
+ retrievalLogger().warn(`kb entity sweep: chunk ${String(row.chunk_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next reindex retries it`);
15391
+ }
15392
+ }, this.entitySweepGuard, { budget: this.chunks.countStaleEntityChunks(1, domain) })).rebuilt;
14544
15393
  if (!semanticAvailable) this.semantic.ensureWarm?.();
14545
15394
  else if (staleVectors.length) report.vectors_encoded = (await this.encodeAndStore(staleVectors, space)).encoded;
14546
15395
  return report;
@@ -14556,18 +15405,7 @@ var KnowledgeStore = class {
14556
15405
  }
14557
15406
  /** Chunk search over the knowledge corpus (semantic + FTS + entity, joint fusion). */
14558
15407
  async search(query, opts) {
14559
- const result = await hybridSearch(this.hybridDeps(opts), {
14560
- query,
14561
- limit: opts?.limit,
14562
- overFetch: opts?.overFetch,
14563
- maxTokens: opts?.maxTokens,
14564
- queryVector: opts?.queryVector,
14565
- recordStats: opts?.recordStats,
14566
- floors: opts?.floors,
14567
- ...opts?.includeScores === void 0 ? {} : { includeScores: opts.includeScores },
14568
- ...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs },
14569
- ...opts?.rewriteQuery === void 0 ? {} : { rewriteQuery: opts.rewriteQuery }
14570
- });
15408
+ const result = await runHybridSearch(this.hybridDeps(opts), query, opts);
14571
15409
  opts?.onResult?.(result);
14572
15410
  return result.hits;
14573
15411
  }
@@ -14583,10 +15421,7 @@ var KnowledgeStore = class {
14583
15421
  */
14584
15422
  hybridDeps(opts) {
14585
15423
  const rowsById = /* @__PURE__ */ new Map();
14586
- return {
14587
- kind: "knowledge",
14588
- config: this.config,
14589
- semantic: this.semantic,
15424
+ return makeHybridDeps("knowledge", this.config, this.semantic, {
14590
15425
  legs: (ctx) => this.searchLegs(opts, ctx),
14591
15426
  texts: (ids) => {
14592
15427
  rowsById.clear();
@@ -14612,7 +15447,7 @@ var KnowledgeStore = class {
14612
15447
  }];
14613
15448
  });
14614
15449
  }
14615
- };
15450
+ });
14616
15451
  }
14617
15452
  /**
14618
15453
  * The knowledge legs: semantic, FTS, and entity overlap against the entities extracted AT INGEST
@@ -14622,40 +15457,12 @@ var KnowledgeStore = class {
14622
15457
  * matching corpus only makes `fuse` normalize and sort it (see `MemoryStore.searchLegs`).
14623
15458
  */
14624
15459
  async searchLegs(opts, ctx) {
14625
- /**
14626
- * Wrap one leg's floored scores. `raw` is the PRE-floor set: `capped` must be measured there,
14627
- * because the relevance floor removes the tail anyway and re-deriving the flag afterwards would
14628
- * erase the "cut at legCap" signal exactly when it bound.
14629
- */
14630
- const leg = (scores, weight, name, dropped, raw) => ({
14631
- weight,
14632
- scores,
14633
- capped: (raw ?? scores).size === ctx.legCap,
14634
- leg: name,
14635
- droppedByFloor: dropped
14636
- });
15460
+ const runner = makeLegRunner(ctx);
14637
15461
  const ftsRaw = this.ftsPath(ctx.query, opts?.domain, opts?.source, ctx.legCap);
14638
- const ftsFloored = applyTermFloor(ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
14639
15462
  return [
14640
- ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) => {
14641
- const floored = applyScoreFloor(raw, ctx.floors.semantic);
14642
- return {
14643
- weight: ctx.weights.semantic,
14644
- scores: floored.scores,
14645
- leg: "semantic",
14646
- droppedByFloor: floored.dropped
14647
- };
14648
- }) : {
14649
- weight: ctx.weights.semantic,
14650
- scores: /* @__PURE__ */ new Map(),
14651
- leg: "semantic",
14652
- droppedByFloor: 0
14653
- },
14654
- leg(ftsFloored.scores, ctx.weights.fts, "fts", ftsFloored.dropped, ftsRaw),
14655
- this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => {
14656
- const floored = applyScoreFloor(raw, ctx.floors.jaccard);
14657
- return leg(floored.scores, ctx.weights.jaccard, "jaccard", floored.dropped, raw);
14658
- })
15463
+ ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
15464
+ runner.term("fts", ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts),
15465
+ this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => runner.scored("jaccard", raw, ctx.weights.jaccard, ctx.floors.jaccard))
14659
15466
  ];
14660
15467
  }
14661
15468
  /** The texts of a candidate set, for the per-row FTS floor (one batched query). */
@@ -14670,43 +15477,55 @@ var KnowledgeStore = class {
14670
15477
  * re-encoding (performance review §7.7 / P8).
14671
15478
  */
14672
15479
  async semanticPath(query, k, opts, queryVector, onVector) {
14673
- const vec = queryVector ?? await this.semantic.encode(query);
14674
- if (vec.length !== this.vstore.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${this.vstore.dim}`);
14675
- onVector?.(vec);
14676
- const topk = this.vstore.topk(vec, Math.max(50, k));
14677
- if (!topk.length) return /* @__PURE__ */ new Map();
14678
- if (!opts?.domain && !opts?.source) return new Map(topk.map((t) => [t.id, t.score]));
14679
- const meta = new Map(this.chunks.meta(topk.map((t) => t.id)).map((r) => [r.chunk_id, r]));
14680
- const out = /* @__PURE__ */ new Map();
14681
- for (const t of topk) {
14682
- const r = meta.get(t.id);
15480
+ return semanticLeg({
15481
+ query,
15482
+ k,
15483
+ queryVector,
15484
+ onVector,
15485
+ encode: (q) => this.semantic.encode(q),
15486
+ dim: this.vstore.dim,
15487
+ topk: (vec, n) => this.vstore.topk(vec, n),
15488
+ filterTopk: (ids) => this.scopeFilter(ids, opts)
15489
+ });
15490
+ }
15491
+ /**
15492
+ * The semantic leg's scope filter, as a `filterTopk` primitive: `undefined` when NO domain/source
15493
+ * was asked means "keep the whole pool" — the short-circuit is preserved, so no `chunks.meta`
15494
+ * lookup runs at all. Otherwise one batched lookup, and rows whose domain/source do not match (or
15495
+ * whose chunk is gone) are dropped.
15496
+ */
15497
+ scopeFilter(ids, opts) {
15498
+ if (!opts?.domain && !opts?.source) return void 0;
15499
+ const meta = new Map(this.chunks.meta(ids).map((r) => [r.chunk_id, r]));
15500
+ const keep = /* @__PURE__ */ new Set();
15501
+ for (const id of ids) {
15502
+ const r = meta.get(id);
14683
15503
  if (!r) continue;
14684
15504
  if (opts.domain && r.domain !== opts.domain) continue;
14685
15505
  if (opts.source && r.source !== opts.source) continue;
14686
- out.set(t.id, t.score);
15506
+ keep.add(id);
14687
15507
  }
14688
- return out;
15508
+ return keep;
14689
15509
  }
14690
15510
  /**
14691
15511
  * Sync relevance probe for the plugin's conditional hint (DESIGN §12). Same contract as
14692
15512
  * `MemoryStore.lexicalProbe` — see `store/lexical.ts`.
14693
15513
  */
14694
15514
  lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
14695
- return probeTerms(text, (term) => {
14696
- const fts = buildFtsQuery(term, this.ftsTokenizer);
14697
- return fts !== null && this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0;
14698
- }, stopAt);
15515
+ return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0, stopAt);
14699
15516
  }
14700
15517
  ftsPath(query, domain, source, cap) {
14701
- const ftsQuery = buildFtsQuery(query, this.ftsTokenizer);
14702
- if (ftsQuery) {
14703
- const rows = this.chunks.ftsSearch(ftsQuery, domain, source, cap);
14704
- return new Map(rows.map((r) => [r.id, -r.rank]));
14705
- }
14706
- const terms = substringTerms(query);
14707
- if (terms.length === 0) return /* @__PURE__ */ new Map();
14708
- const rows = this.chunks.ftsSubstringSearch(terms, domain, source, cap);
14709
- return new Map(rows.map((r) => [r.id, r.rank]));
15518
+ return ftsLeg({
15519
+ query,
15520
+ cap,
15521
+ scope: {
15522
+ domain,
15523
+ source
15524
+ },
15525
+ tokenizer: this.ftsTokenizer,
15526
+ search: (ftsQuery, scope, limit) => this.chunks.ftsSearch(ftsQuery, scope.domain, scope.source, limit),
15527
+ substringSearch: (terms, scope, limit) => this.chunks.ftsSubstringSearch(terms, scope.domain, scope.source, limit)
15528
+ });
14710
15529
  }
14711
15530
  /**
14712
15531
  * Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan.
@@ -15079,7 +15898,13 @@ function buildRuntime(opts) {
15079
15898
  case "contradict_resolve": return this.memory.resolveContradiction(req.contradiction_id, req.resolution, req.loser_fact_id);
15080
15899
  case "maintenance": return this.memory.maintenance();
15081
15900
  case "vectors_diagnose": return this.memory.vectorsDiagnose();
15082
- case "vectors_fix": return this.memory.vectorsFix(req.dry_run ?? false);
15901
+ case "vectors_fix": return vectorsFix({
15902
+ memory: this.memory.vectorRepairTarget(),
15903
+ knowledge: this.knowledge.vectorRepairTarget()
15904
+ }, {
15905
+ dryRun: req.dry_run ?? false,
15906
+ ...req.store === void 0 ? {} : { store: req.store }
15907
+ });
15083
15908
  default: return { error: `admin.${req.action} not implemented` };
15084
15909
  }
15085
15910
  },
@@ -15331,6 +16156,7 @@ const descriptors = [
15331
16156
  offset: count.optional(),
15332
16157
  reason: optionalText.optional(),
15333
16158
  dry_run: bool.optional(),
16159
+ store: optionalText.optional(),
15334
16160
  contradiction_id: count.optional(),
15335
16161
  resolution: optionalText.optional(),
15336
16162
  loser_fact_id: count.optional()
@@ -16772,37 +17598,43 @@ function openDocumentPath(documentPath, target, configured, deps = {}) {
16772
17598
  }
16773
17599
  //#endregion
16774
17600
  //#region lib/types/vectorMigration.js
17601
+ /**
17602
+ * The plugin's lifecycle for the bounded, resumable background migration of persisted vectors into
17603
+ * the current embedding space.
17604
+ *
17605
+ * Changing the embedding space (default model, width, pooling, normalization, input window, weights)
17606
+ * is a DATA MIGRATION, not a config tweak: every persisted vector still decodes, but in another
17607
+ * model's coordinates, so the semantic leg must not serve it. Measured on the real library after the
17608
+ * 512→768 default-model swap, 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval
17609
+ * degraded to lexical+entity with no error anywhere — the stores' open-time warning now says so, and
17610
+ * this module is what repairs it without the operator having to know `vectors --fix` exists.
17611
+ *
17612
+ * The FLOW lives in the engine, once, for both stores (`@avantf/mem`'s `store/vector_repair.ts`:
17613
+ * batch size, event-loop yield, stop / resume / no-progress rules, the `semantic.auto_migrate`
17614
+ * decision and the four store-named log kinds). It is driven for BOTH stores — memory and knowledge
17615
+ * — in one pass, because one embedding space is configured for the whole runtime: the knowledge
17616
+ * store used to be left behind, silently, with only a manual `kb_reindex` able to fix it.
17617
+ *
17618
+ * This module owns only the process LIFECYCLE: one start, one stop, and "never an unhandled
17619
+ * rejection". It sits BEHIND the startup gate (`awaitStartupGate`, see index.ts): the model warm and
17620
+ * this first batch must not land in the boot window (DESIGN §20.14).
17621
+ *
17622
+ * Deliberately no state: what still needs migrating is derived from the databases on every pass —
17623
+ * and the adapters are created ONCE so the shared flow's once-per-process `auto_migrate` warning is
17624
+ * genuinely once per process, not once per heartbeat.
17625
+ */
16775
17626
  function createVectorMigration(opts) {
16776
17627
  const { rt, logger } = opts;
17628
+ const targets = [rt.memory.vectorRepairTarget(), rt.knowledge.vectorRepairTarget()];
16777
17629
  let running = false;
16778
17630
  let stopped = false;
16779
- let warnedDisabled = false;
16780
17631
  const run = async () => {
16781
- const health = rt.memory.vectorSpaceHealth();
16782
- const pending = health.stale + health.space_stale;
16783
- if (pending === 0) return;
16784
17632
  if (rt.closed || !opts.isActive()) return;
16785
- if (rt.config.common.semantic.auto_migrate === false) {
16786
- if (!warnedDisabled) {
16787
- warnedDisabled = true;
16788
- logger.warn(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run \`avantf-mem vectors --fix\` (MCP/plugin: \`mem_admin vectors_fix\`) or set \`semantic.auto_migrate: true\``);
16789
- }
16790
- return;
16791
- }
16792
- const startedAt = Date.now();
16793
- logger.info(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)`);
16794
- const outcome = await rt.memory.migrateVectors({
17633
+ await driveVectorRepairs(targets, {
16795
17634
  ...opts.batchSize === void 0 ? {} : { batchSize: opts.batchSize },
16796
17635
  shouldStop: () => stopped || rt.closed || !opts.isActive(),
16797
- onProgress: (progress) => {
16798
- logger.info(`vector migration: re-encoded ${String(progress.migrated)}/${String(pending)} (${String(progress.remaining)} left)`);
16799
- }
17636
+ log: logger
16800
17637
  });
16801
- if (outcome.remaining === 0) {
16802
- logger.info(`vector migration: complete — ${String(outcome.migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
16803
- return;
16804
- }
16805
- if (!stopped && !rt.closed && opts.isActive()) logger.warn(`vector migration: ${String(outcome.remaining)} vector(s) still belong to an older embedding space (the embedding model may be unavailable) — the next heartbeat retries; manual entry \`avantf-mem vectors --fix\``);
16806
17638
  };
16807
17639
  return {
16808
17640
  start() {