@avantf/dsh-mem 0.4.1 → 0.6.0

package/lib/engine/mem/store/floors.d.ts

@@ -5,9 +5,9 @@
  * MAXIMUM, so every leg's head is 1.0 by construction and the fused number is only comparable
  * WITHIN one query. A cutoff on the fused score would therefore mean a different thing per query
  * (and would move whenever a leg's head changed). The legs' raw numbers do have an absolute scale —
- * cosine for the semantic leg, the Jaccard ratio for the entity leg, "distinct query terms hit" for
- * FTS — so that is where the floors live. The side benefit is that flooring a leg's tail cannot
- * move its maximum, so the cap-invariance §20.17 relies on is untouched.
+ * cosine for the semantic leg, anchored Jaccard for the entity leg (`store/entity_leg.ts`),
+ * "distinct query terms hit" for FTS — so that is where the floors live. The side benefit is that
+ * flooring a leg's tail cannot move its maximum, so the cap-invariance §20.17 relies on is untouched.
  *
  * THREE FLOORS, ONE SHAPE. `min_semantic_similarity` / `min_fts_terms` / `min_jaccard`; each `0`
  * means "this leg is not gated", and a candidate EQUAL to its floor is KEPT (only strictly-lower
@@ -49,10 +49,25 @@
  * of its own is bounded instead: it costs one extra query, only when the strict answer was empty,
  * and it can still report "nothing relevant" for a genuinely unrelated question.
  *
- * FTS REACHABILITY. `min_fts_terms` counts distinct query terms a row hits, but a query can only
- * produce so many terms (`relevanceTerms`: latin words ≥5 chars + CJK 3-grams) — a 3-character CJK
- * query yields exactly ONE trigram, so a configured bar of 2 is unreachable and that leg is empty
- * for it by construction. The effective bar is therefore `min(configured, termCount)`.
+ * FTS REACHABILITY, AND THE SHORT-QUERY FALLBACK. `min_fts_terms` counts distinct query terms a row
+ * hits, but a query can only produce so many terms — a 3-character CJK query yields exactly ONE
+ * trigram, so a configured bar of 2 is unreachable and that leg is empty for it by construction. The
+ * effective bar is therefore `min(configured, termCount)`, and `termCount` is that of the text being
+ * GRADED — with 方案 A's augmentation the original and the rewrite are different texts with
+ * different counts, so each resolves its own bar (`store/hybrid.ts`). Resolving the rewrite from the
+ * original's count is the defect that let a single incidental trigram pass a bar it could not reach
+ * (2026-10-04).
+ *
+ * That clamp still leaves the shape with NO term at all: a 2-char CJK query (`buildFtsQuery` returns
+ * `null` — the trigram index holds no term under three characters, measured on the live store). The
+ * term set is therefore {@link gradedTerms}: when the query yields no index-expressible term, the
+ * 2-char CJK runs themselves become the terms, and the legs search/grade them as SUBSTRINGS
+ * (`content LIKE '%…%'`, the finer predicate an FTS5 trigram table still exposes; see
+ * `store/lexical.ts`). The clamp stays the same rule and stays honest — one substring term is
+ * `min(configured, 1)`, and {@link applyTermFloor} re-derives it from the query it is grading, so
+ * the reported number is the applied number. The guard against the noise a 2-char term invites is
+ * the term itself: the row must CONTAIN the run contiguously (not merely share a trigram), and a
+ * multi-run query must contain `min(configured, runs)` of them.
  *
  * @module store/floors
  */
@@ -82,16 +97,20 @@ export type FloorLeg = 'semantic' | 'fts' | 'jaccard';
  *  - fts: NOT a constant — {@link looseTermFloor}, half the query's distinct terms with a floor of
  *    1. `1` here is the "no terms to scale by" baseline (the previously constant value); a query
  *    that cannot produce terms is not graded at all, and one that produces a single term (every
- *    3-char CJK query) is already graded against that one term by the reachability clamp.
- *  - jaccard **0.15** (positive — the previous `0` was the ungated hole this closes). Measured, not
- *    guessed, on two corpora: (a) the live store, 64 active facts, the common-entity questions a
- *    real user asks about this repo ("插件怎么安装" / "任务怎么拆分" / "知识库在哪里" …) put their
- *    TOP non-answering entity candidate at Jaccard 0.0909 and most at 0.02–0.07, so 0.15 removes that
- *    whole tail; (b) the gold-labelled 35-query eval set, where the LOWEST Jaccard any answering pair
- *    reached is 0.25 (「老王喜欢什么风格」) — a bottom line above 0.25 starts dropping gold pairs,
- *    while 0.15 filters none. The entity leg cannot fully separate the two classes on short queries
- *    (a one-entity query against a one-entity fact is 1.0 by construction, and non-gold pairs reach
- *    0.667), so this is deliberately a "no incidental small overlap" line, not a relevance classifier.
+ *    3-char CJK query, and every 2-char one through the substring fallback) is already graded
+ *    against that one term by the reachability clamp.
+ *  - jaccard **0.15** (positive — the previous `0` was the ungated hole this closes). The metric was
+ *    recalibrated on 2026-10-04 (`store/entity_leg.ts`): the leg still scores a Jaccard ratio, but
+ *    over the query's ANCHOR entities and with the fact's extra-entity term SATURATED at
+ *    `ENTITY_UNION_CAP`. 0.15 on that unit means "shares at least one discriminative entity" for any
+ *    query with up to six anchors (`1/6 = 0.167 > 0.15`), i.e. the bottom line that cannot be lowered
+ *    to "any incidental overlap" without admitting a length-dependent fraction. The old Jaccard
+ *    calibration, kept for the record: (a) on the live store the top non-answering candidate for
+ *    common-entity questions sat at 0.0909 and most at 0.02–0.07; (b) the lowest Jaccard any
+ *    answering pair on the gold 35-query set reached was 0.25. The entity leg cannot separate the
+ *    classes on short queries by itself, so this is deliberately a "no incidental small overlap"
+ *    line, not a relevance classifier — see `store/entity_leg.ts` for why the fact's own width was
+ *    saturated rather than used as the separator.
  */
 export declare const LOOSE_FLOORS: RetrievalFloors;
 /**
@@ -112,21 +131,29 @@ export declare function emptyFloorDrops(): RetrievalFloorDrops;
  * The FTS bar actually applied to a query carrying `termCount` distinct terms.
  *
  * A query cannot be graded against more terms than it has (see the module comment: a 3-char CJK
- * query has one trigram), so the configured bar is CLAMPED to the reachable maximum. `termCount`
- * `0`/omitted leaves the value alone — such a query is not graded at all by {@link applyTermFloor},
- * and changing the REPORTED value there would misdescribe what the caller is reading. `0` stays `0`
- * (the operator's off switch).
- *
- * This is the ONE place the clamp lives; `resolveFloors` (what the result reports) and
- * `applyTermFloor` (what the leg does) both call it, so the number the caller reads is the number
- * the leg used.
+ * query has one trigram, and a 2-char one has one substring term), so the configured bar is CLAMPED
+ * to the reachable maximum. `termCount` `0`/omitted leaves the value alone — such a query is not
+ * graded at all by {@link applyTermFloor}, and changing the REPORTED value there would misdescribe
+ * what the caller is reading. `0` stays `0` (the operator's off switch).
+ *
+ * This is the ONE place the clamp lives; `resolveFloors` (what the result reports / a run is handed)
+ * and `applyTermFloor` (what the leg does) both call it, so the number a run reads is the number its
+ * leg used. It is a `min`, i.e. one-directional: it can LOWER an unreachable configured bar but never
+ * RAISE a bar inherited from another text, which is why the caller must resolve per graded text
+ * rather than reusing one text's resolution for another.
  */
 export declare function effectiveTermFloor(floor: number, termCount?: number): number;
 /** The per-query floor resolution inputs beyond the configuration. */
 export interface FloorResolution {
     /** `loose` lowers each configured floor (see {@link LOOSE_FLOORS}); absent means `strict`. */
     profile?: FloorProfile;
-    /** This query's distinct term count (`relevanceTerms(query).length`), for the FTS clamp. */
+    /**
+     * The distinct term count of the text being GRADED (`gradedTerms(text).length`), for the FTS
+     * clamp. It must describe the query the legs will actually run on: under 方案 A's augmentation the
+     * rewrite is a DIFFERENT text with its own term count, and resolving both runs from the original's
+     * count is the defect that let one incidental trigram through (see `store/hybrid.ts`). Callers with
+     * several graded texts resolve once per text.
+     */
     termCount?: number;
     /**
      * Which legs a `'loose'` profile may lower, from {@link droppedLegs} of the pass that made the
@@ -170,16 +197,21 @@ export declare function applyScoreFloor(scores: Map<number, number>, floor: numb
  *
  * Substring containment, case-insensitively: FTS5's trigram tokenizer matches a quoted phrase by
  * consecutive trigrams and is ASCII-case-insensitive, which is the same predicate as "the term
- * occurs in the text". `relevanceTerms` lowercases its latin words, so the comparison has to.
+ * occurs in the text" — and the 2-char fallback terms ARE substrings by construction
+ * (`store/lexical.ts`), so one predicate covers both. `relevanceTerms` lowercases its latin words,
+ * so the comparison has to.
  */
 export declare function countMatchedTerms(text: string, terms: readonly string[]): number;
 /**
  * Apply the per-row FTS floor to one FTS leg's scores.
  *
- * The terms come from `relevanceTerms(query)` (latin words ≥ 5 chars + every CJK 3-gram) — the same
- * tokenisation the conditional-hint probe uses, so "relevant enough" means one thing in this repo.
- * A query with NO such term is not graded at all: there is nothing to count, and dropping every row
- * on a technicality would turn a 3–4 char latin query (`buildFtsQuery` still emits it) into silence.
+ * The terms come from `gradedTerms(query)` — latin words ≥ 5 chars + every CJK 3-gram, and (only
+ * when the query produces no such term) the 2-char CJK runs as substrings. The indexed terms are
+ * the same tokenisation the conditional-hint probe uses, so "relevant enough" means one thing in
+ * this repo; the fallback exists because the hint path may stay silent while the LEG must not
+ * (`store/lexical.ts`). A query with NO such term is not graded at all: there is nothing to count,
+ * and dropping every row on a technicality would turn a 3–4 char latin query (`buildFtsQuery` still
+ * emits it) into silence.
  *
  * The bar itself is {@link effectiveTermFloor}: a query that can only produce one term (a 3-char CJK
  * query is one trigram) is measured against that one term, not against a configured 2 it could never

package/lib/engine/mem/store/hybrid.d.ts

@@ -3,18 +3,16 @@
  *
  * `MemoryStore.search` and `KnowledgeStore.search` used to carry parallel copies of this flow
  * (~80 lines each: limit normalization → weights → over-fetch → leg cap → fuse → live filter →
- * rerank → slice → output budget → health event). The legs themselves are legitimately
- * store-specific — different tables, different filters (`category` vs `domain`/`source`), and only
- * memory has the HRR probe — but the ORCHESTRATION is one algorithm, and two copies of it drifted:
+ * slice → output budget → health event). The legs themselves are legitimately store-specific —
+ * different tables, different filters (`category` vs `domain`/`source`), and only memory has the
+ * HRR probe — but the ORCHESTRATION is one algorithm, and two copies of it drifted:
  *
  *   - the `NaN` limit guard existed only in memory, so a non-finite limit reached knowledge's SQL
  *     `LIMIT` and took the whole cross-store query down with it (`Promise.all` has no per-leg catch);
  *   - `retriever.over_fetch_factor` was read only by memory, while knowledge hardcoded `limit * 3`,
  *     so one config knob silently meant two different things;
  *   - `recordLegCapped()` was called only by memory, so a capped knowledge leg was unobservable —
- *     exactly the blindness DESIGN §20.17 says the counter exists to remove;
- *   - the reranker's used/fallback flags were a helper in memory and an inline expression in
- *     knowledge.
+ *     exactly the blindness DESIGN §20.17 says the counter exists to remove.
  *
  * Fixing those one at a time would leave the next drift a matter of when. The stores now supply
  * their legs and their hit mapping; everything that can drift lives here once.
@@ -26,7 +24,7 @@
  * @module store/hybrid
  */
 import { type Config, type FloorProfile, type RecallResult, type RetrievalFloorDrops, type RetrievalFloors } from '../../mem-contract/index.js';
-import { type BudgetInput, type Hit, type Reranker, type SemanticBackend } from '../../mem-retrieval/index.js';
+import { type BudgetInput, type Hit, type SemanticBackend } from '../../mem-retrieval/index.js';
 import { type FloorLeg } from './floors.js';
 /** The `limit` a caller gets when it passes nothing usable. */
 export declare const DEFAULT_SEARCH_LIMIT = 10;
@@ -63,8 +61,12 @@ export interface HybridLeg {
     capped?: boolean;
     /**
      * Which floor governs this leg, for {@link HybridResult.dropped_by_floor}'s per-leg breakdown.
-     * The HRR probe shares the Jaccard floor; naming it separately keeps a narrowed candidate set
-     * attributable without double-counting the Jaccard leg's own drops.
+     * The HRR probe shares the entity (`jaccard`) floor; naming it separately keeps a narrowed
+     * candidate set attributable without double-counting the entity leg's own drops.
+     *
+     * The `jaccard` KEY is historical: the entity leg's metric is now anchored Jaccard with a
+     * saturating union, whose unit (and the `min_jaccard` floor on it) is defined in
+     * `store/entity_leg.ts` — read that before re-calibrating the knob.
      */
     leg?: keyof RetrievalFloorDrops;
     /** Candidates this leg removed because they fell below its floor. */
@@ -113,7 +115,6 @@ export interface HybridDeps<H> {
     kind: 'memory' | 'knowledge';
     config: Config;
     semantic: SemanticBackend;
-    reranker: Reranker;
     /**
      * The legs to fuse, as promises (or values). Awaited concurrently; a rejection costs that leg
      * only. The async setup a store needs before it can build its legs — memory extracts the query's
@@ -121,7 +122,7 @@ export interface HybridDeps<H> {
      * body, before the array is returned.
      */
     legs(ctx: HybridContext): Promise<readonly (HybridLeg | Promise<HybridLeg>)[]>;
-    /** Text for the fused ids: the live filter, the reranker's input, and the hit bodies. */
+    /** Text for the fused ids: the live filter and the hit bodies. */
     texts(ids: number[]): Map<number, string>;
     /** Build the caller-facing hits from the ranked, live, already-sliced ids. */
     hits(ranked: readonly Hit[], texts: Map<number, string>): H[];
@@ -177,6 +178,12 @@ export interface HybridPlan {
      * `queries` counts questions instead of legs (DESIGN §20.5).
      */
     recordStats?: boolean;
+    /**
+     * P-01: attach per-leg evidence to the returned hits (`RecallHit.scores` / `final`). Default
+     * false, and that default is load-bearing: the evidence is explanation only, so a caller that did
+     * not ask must not pay for it in tokens (`fuse` skips building the arrays entirely).
+     */
+    includeScores?: boolean;
 }
 /** A hit after the output budget: `text` may have been shortened, and `truncated` says so. */
 export type Budgeted<T> = T & {
@@ -207,17 +214,6 @@ export interface HybridResult<H> {
  * rather than per store, because the health counter compares against whatever this returns.
  */
 export declare function legCapFor(config: Config, overFetch: number): number;
-/**
- * Whether the configured reranker is actually in use — the two flags a retrieval event carries.
- *
- * "Configured" is not "used": a reranker that is selected but unavailable falls back to the fused
- * order, and counting that as a rerank would make the counter lie in exactly the case it exists to
- * expose (the reference implementation leaves `rerank_fallback` unfeedable).
- */
-export declare function rerankState(reranker: Reranker): {
-    used: boolean;
-    fallback: boolean;
-};
 /**
  * Bound a ranked hit list to a token budget, marking what was shortened.
  *
@@ -233,9 +229,8 @@ export declare function applyOutputBudget<H extends BudgetInput>(config: Config,
 /**
  * Run one hybrid search.
  *
- * The order of the last three steps is load-bearing: rerank the over-fetched pool, THEN slice to
- * `limit`, THEN apply the output budget, and only then tell the store what was returned. Slicing
- * before the rerank would rank an arbitrary subset; budgeting before the slice would spend the
- * budget on hits that are about to be dropped.
+ * The order of the last steps is load-bearing: slice the fused (over-fetched) pool to `limit`,
+ * THEN apply the output budget, and only then tell the store what was returned. Budgeting before
+ * the slice would spend the budget on hits that are about to be dropped.
  */
 export declare function hybridSearch<H extends BudgetInput>(deps: HybridDeps<H>, plan: HybridPlan): Promise<HybridResult<H>>;

package/lib/engine/mem/store/knowledge.d.ts

@@ -1,5 +1,5 @@
 import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport } from '../../mem-contract/index.js';
-import type { Reranker, SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
+import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
 import { type LexicalProbe } from './lexical.js';
 import { type FloorLeg } from './floors.js';
 import { type HybridResult } from './hybrid.js';
@@ -43,6 +43,8 @@ export interface KnowledgeSearchOptions {
      * store-level "last result" field would race two concurrent queries.
      */
     onResult?: (result: HybridResult<RecallHit>) => void;
+    /** P-01: attach per-leg evidence (`scores` / `final`) to every hit. Default false. */
+    includeScores?: boolean;
 }
 /**
  * One identity a write will land on: the title it will use, the resolved local path it would read
@@ -87,7 +89,6 @@ export type IngestMode = 'add' | 'replace';
 export declare class KnowledgeStore {
     private readonly semantic;
     private readonly vstore;
-    private readonly reranker;
     /** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
     private readonly domainConfigPath;
     private readonly logger?;
@@ -127,7 +128,7 @@ export declare class KnowledgeStore {
     private readonly allowedDomains;
     /** Whether the config allowlist is active (non-empty); `false` = `domains: []` = no restriction. */
     private readonly domainsRestricted;
-    constructor(dbPath: string, config: Config, kbConfig: KnowledgeConfig, semantic: SemanticBackend, vstore: VectorStore, reranker: Reranker, 
+    constructor(dbPath: string, config: Config, kbConfig: KnowledgeConfig, semantic: SemanticBackend, vstore: VectorStore, 
     /** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
     domainConfigPath: string, logger?: AvantfLogger | undefined);
     /**
@@ -302,11 +303,12 @@ export declare class KnowledgeStore {
      */
     private encodeAndStore;
     /**
-     * Identity of the vector space this store currently writes into (DESIGN §20): a change of
-     * backend, model or width makes every stored chunk vector stale, and the whole point of
-     * recording it is that such a change is DETECTED instead of silently mixing two spaces.
+     * Identity of the vector space this store currently writes into (DESIGN §20): the representation
+     * fingerprint — backend, model, width AND the backend's declared pooling/normalization/input
+     * window/model revision — so any representation change makes stored chunk vectors stale instead
+     * of silently mixing two spaces. Public so diagnostics and tests can name the space.
      */
-    private vectorSpace;
+    vectorSpace(): string;
     /**
      * A stored vector is reusable when it exists, matches the space, and its text is unchanged.
      *
@@ -434,7 +436,7 @@ export declare class KnowledgeStore {
      * and the chunk→hit mapping.
      *
      * The limit normalization, the over-fetch factor, the leg cap, the capped-leg counter, the
-     * degraded weights, the rerank and the output budget all live there now. They used to be copied
+     * degraded weights and the output budget all live there now. They used to be copied
      * here, and every fix landed on the memory side only — a `NaN` limit reached this store's SQL and
      * took the whole cross-store query down with it, `retriever.over_fetch_factor` meant something
      * different per store, and a capped knowledge leg was invisible to the health counters.
@@ -463,7 +465,14 @@ export declare class KnowledgeStore {
      */
     lexicalProbe(text: string, stopAt?: number): LexicalProbe;
     private ftsPath;
-    /** Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan. */
+    /**
+     * Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan.
+     *
+     * Same metric as the memory store's sibling (`store/entity_leg.ts` carries the measurement and
+     * the reasoning): anchored Jaccard with a saturating union over the query's rare entities, so a
+     * wide chunk cannot dilute the denominator and a corpus-wide word cannot drag the whole knowledge
+     * base in.
+     */
     private jaccardPath;
 }
 export interface Chunk {

package/lib/engine/mem/store/lexical.d.ts

@@ -25,6 +25,20 @@
  * `matched >= 2`, with the margin between 2 and 1 rather than a tuned threshold inside a
  * continuum. A single-word query (`cgroup`) scores 1 and stays silent on purpose: one term is not
  * evidence.
+ *
+ * THE SHORT-CJK FALLBACK (and why the hint probe does NOT use it). The trigram index has no term
+ * shorter than three characters — measured on the live store: `fts5vocab` over `facts_fts` holds
+ * 27906 distinct terms and EVERY one is 3 characters (and a 2-char `MATCH` returns nothing, even as
+ * a prefix query). So a 2-char CJK query produces no FTS term at all and that leg is empty by
+ * construction. The FTS LEG therefore falls back to the one finer predicate the FTS5 trigram TABLE
+ * does expose: `content LIKE '%…%'`, which matches a 2-char run by scanning (no index can serve a
+ * 2-char pattern). See {@link substringTerms} / {@link gradedTerms}.
+ *
+ * The conditional-hint probe deliberately keeps the index-expressible terms only: it is
+ * SYNCHRONOUS and runs on every step's prompt assembly, so it must stay a bounded number of
+ * `LIMIT 1` index lookups — an O(corpus) scan per 2-char term would put a full scan on the prompt
+ * path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
+ * (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
  */
 /** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
 export interface LexicalProbe {
@@ -46,6 +60,36 @@ export type RelevanceHit = boolean;
  * caller runs one `LIMIT 1` MATCH per term and counts the hits.
  */
 export declare function relevanceTerms(text: string, cap?: number): string[];
+/**
+ * The terms the FTS leg falls back to when the trigram index can express NONE of the query.
+ *
+ * A 2-char CJK run IS the finer granularity the FTS5 trigram TABLE can still answer — not as an
+ * indexed token (the vocabulary holds 3-character terms only, measured; a 2-char `MATCH`, quoted or
+ * prefixed, returns zero rows) but as the substring predicate FTS5 exposes on a trigram table
+ * (`content LIKE '%…%'`, which falls back to a scan for patterns under three characters). One-char
+ * runs are deliberately NOT included: a single character is not evidence (the same reason
+ * {@link looksRelevant} needs two terms), and it would make the leg match a large share of any
+ * corpus. Latin runs are not included either — `buildFtsQuery` already admits latin tokens from
+ * three characters, and the module's own floor (5, `MIN_LATIN`) is a separate, measured decision.
+ *
+ * PRECISION GUARD. The fallback is graded by the SAME per-row floor as the indexed path and by the
+ * reachability clamp: for a single 2-char term the effective bar is 1, and the term is a substring,
+ * so "the row contains the query's characters CONTIGUOUSLY" is what admits it — not "shares one
+ * incidental trigram". A two-run query ('李娜 张伟') produces two substring terms and is judged
+ * against `min(configured, 2)`, i.e. both runs must occur in the row.
+ */
+export declare function substringTerms(text: string, cap?: number): string[];
+/**
+ * The term set the FTS leg is SEARCHED and GRADED with — the one definition of "the query's terms"
+ * for that leg (`store/floors.ts` grades with it, `store/hybrid.ts` resolves the reachability clamp
+ * from it, and the stores build their MATCH / substring probe from it).
+ *
+ * The fallback is whole-query, not per run: any text that yields even one index-expressible term is
+ * returned UNCHANGED from {@link relevanceTerms}. That is what keeps every long query bit-identical
+ * (a mixed text like `缓存失效 李娜` keeps only its trigrams, exactly as before) and confines the
+ * new behaviour to the shape that had NO leg at all.
+ */
+export declare function gradedTerms(text: string, cap?: number): string[];
 /**
  * The gate itself: is this store worth telling the model about?
  *