@avantf/dsh-mem 0.4.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +19 -1
  2. package/lib/client.js +40 -9
  3. package/lib/engine/mem/config/config_files.d.ts +1 -1
  4. package/lib/engine/mem/db/dao/chunks.d.ts +20 -1
  5. package/lib/engine/mem/db/dao/entities.d.ts +35 -11
  6. package/lib/engine/mem/db/dao/fact_sources.d.ts +53 -0
  7. package/lib/engine/mem/db/dao/facts.d.ts +127 -12
  8. package/lib/engine/mem/db/invariants.d.ts +26 -0
  9. package/lib/engine/mem/db/tokenizer.d.ts +23 -0
  10. package/lib/engine/mem/db/vectors.d.ts +40 -8
  11. package/lib/engine/mem/entities/extract.d.ts +1 -1
  12. package/lib/engine/mem/eval/derive.d.ts +120 -0
  13. package/lib/engine/mem/eval/derived_runner.d.ts +88 -0
  14. package/lib/engine/mem/eval/metrics.d.ts +24 -0
  15. package/lib/engine/mem/eval/runner.d.ts +8 -1
  16. package/lib/engine/mem/index.d.ts +1 -1
  17. package/lib/engine/mem/modelBootstrap.d.ts +2 -2
  18. package/lib/engine/mem/runtime.d.ts +1 -1
  19. package/lib/engine/mem/store/common.d.ts +33 -11
  20. package/lib/engine/mem/store/entity_leg.d.ts +95 -0
  21. package/lib/engine/mem/store/floors.d.ts +63 -31
  22. package/lib/engine/mem/store/hybrid.d.ts +21 -26
  23. package/lib/engine/mem/store/knowledge.d.ts +18 -9
  24. package/lib/engine/mem/store/lexical.d.ts +44 -0
  25. package/lib/engine/mem/store/memory.d.ts +151 -24
  26. package/lib/engine/mem/store/secret_guard.d.ts +34 -0
  27. package/lib/engine/mem/store/time_window.d.ts +74 -0
  28. package/lib/engine/mem-contract/config.d.ts +9 -16
  29. package/lib/engine/mem-contract/family.d.ts +1 -1
  30. package/lib/engine/mem-contract/tools.d.ts +10 -0
  31. package/lib/engine/mem-contract/types.d.ts +164 -9
  32. package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +37 -7
  33. package/lib/engine/mem-retrieval/fusion.d.ts +14 -1
  34. package/lib/engine/mem-retrieval/index.d.ts +1 -3
  35. package/lib/engine/mem-retrieval/interfaces.d.ts +38 -14
  36. package/lib/engine/mem-retrieval/registry.d.ts +2 -4
  37. package/lib/engine/mem-retrieval/representation.d.ts +50 -0
  38. package/lib/engine/mem-retrieval/stats.d.ts +1 -3
  39. package/lib/index.js +2003 -565
  40. package/lib/types/client/index.d.ts.map +1 -1
  41. package/lib/types/index.d.ts +1 -1
  42. package/lib/types/index.d.ts.map +1 -1
  43. package/lib/types/prompt.d.ts +7 -0
  44. package/lib/types/prompt.d.ts.map +1 -1
  45. package/lib/types/remote.d.ts +12 -6
  46. package/lib/types/remote.d.ts.map +1 -1
  47. package/lib/types/vectorMigration.d.ts +44 -0
  48. package/lib/types/vectorMigration.d.ts.map +1 -0
  49. package/package.json +5 -5
  50. package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
  51. package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
  52. package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
@@ -59,3 +59,26 @@ export declare const MAX_FTS_PHRASES = 64;
59
59
  * once when the tokenizer fallback landed).
60
60
  */
61
61
  export declare function buildFtsQuery(query: string, tokenizer?: FtsTokenizer): string | null;
62
+ /**
63
+ * The `LIKE` predicate that matches `column` against every term as a contiguous SUBSTRING.
64
+ *
65
+ * This is the SHORT-QUERY FALLBACK's search primitive: the trigram index holds no term under three
66
+ * characters (measured on the live store: `fts5vocab` over `facts_fts` = 27906 terms, all length 3),
67
+ * and a 2-char `MATCH` returns zero rows — quoted or prefixed. An FTS5 trigram table does still
68
+ * answer a substring `LIKE`, but only by SCANNING when the pattern is shorter than three characters
69
+ * (SQLite documents the index as usable from three), so callers must bound the result themselves.
70
+ * See `store/lexical.ts` for the precision guard and why only 2-char CJK runs reach here.
71
+ *
72
+ * `ESCAPE` is not decoration: the terms are the caller's text, so a `%` or `_` in one would silently
73
+ * turn a literal query into a wildcard. The fragment is built once and shared by BOTH stores' DAOs,
74
+ * the way {@link buildFtsQuery} is — the two copies of that rule had already drifted once.
75
+ *
76
+ * @returns `any` / `count` (SQL expressions over the SAME bind order) and the patterns. `count` is
77
+ * the number of terms the row contains, i.e. the fallback leg's relevance score; `any` is its
78
+ * `WHERE` clause. Empty terms are dropped, so a caller never gets a `%%` that matches everything.
79
+ */
80
+ export declare function likeSubstring(column: string, terms: readonly string[]): {
81
+ any: string;
82
+ count: string;
83
+ params: string[];
84
+ };
@@ -3,18 +3,44 @@ import type { VectorStore } from '../../mem-retrieval/index.js';
3
3
  export declare function float32ToBytes(vec: Float32Array): Buffer;
4
4
  export declare function bytesToFloat32(buf: Uint8Array | null | undefined): Float32Array | null;
5
5
  /**
6
- * Identity of a VECTOR SPACE: which backend, which model, how wide.
6
+ * Identity of a VECTOR SPACE — the REPRESENTATION FINGERPRINT.
7
7
  *
8
8
  * Persisted beside every vector (`facts.embedding_model`, `doc_chunks.embedding_model`) so
9
- * "these vectors came from a different model" is detectable. It used to record only the
10
- * backend name, which made a same-width model swap (bge-small-zh → bge-m3, both 512d)
11
- * indistinguishable from "already encoded": the two spaces then shared one index and were
12
- * ranked against each other.
9
+ * "these vectors came from a different representation" is detectable. It used to record only the
10
+ * backend name, then `backend/model/dim`; both were blind to the rest of what decides which
11
+ * coordinates a vector lives in — pooling, normalization, the truncation window, and the weights
12
+ * behind an unchanged repo name. The space id now embeds:
13
+ *
14
+ * - {@link VECTOR_SPACE_FORMAT}, a format version. Bumping it is a deliberate ONE-TIME full
15
+ * re-encode of every existing library (the old ids simply are not equal to any new id), which
16
+ * is exactly why the stale-vector warning has to explain that case instead of calling it a
17
+ * model swap (see `reportStaleVectors`);
18
+ * - the backend, model and width, as before;
19
+ * - the backend's declared representation (`representationKey`): pooling / normalize /
20
+ * max_input_tokens / model revision. The revision is omitted when the family sidecar could not
21
+ * be read — a documented degradation, never a crash.
13
22
  *
14
23
  * The vector STORE is deliberately not part of the identity: numpy vs hnswlib indexes the
15
24
  * same vectors, so switching it must not trigger re-encoding (that is `reloadIndex`).
16
25
  */
17
- export declare function vectorSpaceId(backend: string, model: string, dim: number): string;
26
+ export declare const VECTOR_SPACE_FORMAT = 2;
27
+ /** `v<format>/` — the prefix every current-format space id starts with (and old ids do not). */
28
+ export declare const VECTOR_SPACE_FORMAT_PREFIX: string;
29
+ /**
30
+ * @param backend - the `SemanticBackend` name.
31
+ * @param model - the configured model id.
32
+ * @param dim - the configured width.
33
+ * @param representation - the normalized representation key (`representationKey`). Omitted by
34
+ * callers with no backend in hand (benchmarks, the pure id tests); such an id is still
35
+ * format-prefixed, so it can never equal a pre-fingerprint id.
36
+ */
37
+ export declare function vectorSpaceId(backend: string, model: string, dim: number, representation?: string): string;
38
+ /**
39
+ * `true` when a recorded space id predates the representation fingerprint (`v2/…`), or comes from a
40
+ * format this build does not know. The distinction is what lets the warning explain a one-time
41
+ * format upgrade separately from "another model's vectors are in this index".
42
+ */
43
+ export declare function isPreFingerprintSpace(recorded: string | null | undefined): boolean;
18
44
  /**
19
45
  * Rebuild a live vstore from persisted vector BLOBs — the restart-recovery path
20
46
  * shared by both stores (the two copies had already drifted: memory warned on
@@ -23,13 +49,19 @@ export declare function vectorSpaceId(backend: string, model: string, dim: numbe
23
49
  * The rows come from the owning store's DAO (DESIGN §19): this helper decodes and
24
50
  * filters them, it never runs SQL itself.
25
51
  *
52
+ * A row whose width does not fit the store is skipped (never `add()`ed — the store
53
+ * would throw). The OWNING store reports that condition itself, as one actionable
54
+ * line with the count, the reason and the manual re-encode entry (`reportStaleVectors`),
55
+ * so this helper stays quiet: two warnings for one condition read as two problems.
56
+ *
26
57
  * @param rows - `id` (fact/chunk id) + `vec` (BLOB), as read back from the store.
27
- * @param label - store name used in the dim-mismatch warning.
58
+ * @param _label - store name; unused here (the owning store owns the warning), kept so call sites read
59
+ * as "which store is reloading".
28
60
  */
29
61
  export declare function reloadVectorIndex(vstore: VectorStore, rows: readonly {
30
62
  id: number;
31
63
  vec: Uint8Array | null;
32
- }[], label: string): {
64
+ }[], _label: string): {
33
65
  loaded: number;
34
66
  skipped: number;
35
67
  };
@@ -6,7 +6,7 @@
6
6
  * Note: nodejieba's `tag()` returns `{ word, tag }` objects, and its POS flag set
7
7
  * differs from Python jieba (e.g. proper nouns → `nr`, 名动词 → `vn`).
8
8
  */
9
- interface ExtractedEntity {
9
+ export interface ExtractedEntity {
10
10
  name: string;
11
11
  type: string;
12
12
  method: string;
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Deterministic query-derivation engine for the P-02 regression net.
3
+ *
4
+ * The shipped 41-query fixture is frozen to the last bit, so it cannot grow to cover a new
5
+ * pathology without re-freezing seven numbers. This engine derives NEW queries from a corpus
6
+ * instead, with no model and no clock: the same facts + seed always produce byte-identical cases.
7
+ *
8
+ * What it derives, per gold fact:
9
+ * - one query assembled from the gold's SHARED literal fragments plus a bridge fragment the gold
10
+ * does NOT contain (so the query is never answered by simply echoing the gold's own text);
11
+ * - one planted, unrelated, LONG carrier fact that literally contains the bridge (requirement ②);
12
+ * - the collision rate of the query against the non-gold facts;
13
+ * - the counterfactual arm (requirement ③): removing the carrier must put the gold back at rank 1.
14
+ *
15
+ * The two guards:
16
+ * - `no_time_word`: an emitted query may not contain a time expression (a temporal query belongs to
17
+ * a different leg, and would confound "did the ranking change?" with "did the clock move").
18
+ * Offending candidates are DROPPED and counted, not silently kept.
19
+ * - `irrelevant`: a negative-control query that shares no fragment with any fact; a retriever that
20
+ * answers it is returning something for nothing.
21
+ *
22
+ * Requirement ① (a corpus with the real store's length distribution) is served by
23
+ * {@link synthesizeCorpus}, whose output mirrors the measured profile in
24
+ * `docs/bench/BASELINE_REPORT.md` §0.1 (9 / 243 / 313 / 409 / 882 characters); {@link analyzeCorpus}
25
+ * reports how close a corpus actually is so the caller can assert it rather than trust it.
26
+ */
27
+ export interface LengthProfile {
28
+ min: number;
29
+ p25: number;
30
+ median: number;
31
+ p75: number;
32
+ max: number;
33
+ }
34
+ /**
35
+ * The measured ACTIVE-corpus shape on 2026-10-06 (`docs/bench/BASELINE_REPORT.md` §0.1:
36
+ * 86 active, min/p25/median/p75/max = 9/243/313/409/882 characters). Frozen here so a fixture can
37
+ * be checked against the real store's shape instead of against a guess.
38
+ */
39
+ export declare const REAL_LENGTH_PROFILE: LengthProfile;
40
+ export interface CorpusAnalysis {
41
+ n: number;
42
+ length_profile: LengthProfile;
43
+ /** Fraction of facts whose length falls inside the real p25..p75 band. */
44
+ in_band_rate: number;
45
+ }
46
+ export declare function lengthProfileOf(lengths: readonly number[]): LengthProfile;
47
+ export declare function analyzeCorpus(facts: readonly string[]): CorpusAnalysis;
48
+ /**
49
+ * The literal unit of collision: latin/digit words of length ≥ 2, and CJK bigrams.
50
+ *
51
+ * Bigrams are the CJK unit on purpose — it is what a trigram FTS index cannot fully guard, and it
52
+ * is exactly the granularity of the 2026-10-04 self-referential ranking incident (`用户是`, a
53
+ * fragment an unrelated note happened to contain).
54
+ */
55
+ export declare function fragmentsOf(text: string): string[];
56
+ export declare function timeWordsIn(text: string): string[];
57
+ export type GuardKind = 'none' | 'no_time_word' | 'irrelevant';
58
+ export interface DerivedQuery {
59
+ query: string;
60
+ k: number;
61
+ /** Index of the gold fact in `setup_facts`, or null for a negative-control query. */
62
+ gold_index: number | null;
63
+ must_include: number[];
64
+ must_exclude: number[];
65
+ guard: GuardKind;
66
+ /** The literal fragments the query is built from. */
67
+ fragments: string[];
68
+ /** Fraction of the NON-gold facts that literally share at least one query fragment. */
69
+ collision_rate: number;
70
+ /** Index of the planted fragment carrier in `setup_facts`, when this query has one. */
71
+ carrier_index: number | null;
72
+ }
73
+ export interface CollisionSelfCheck {
74
+ query_index: number;
75
+ carrier_index: number;
76
+ gold_index: number;
77
+ /** The fixture is only meaningful if the carrier actually outranks the gold in the base arm. */
78
+ base_carrier_before_gold: boolean;
79
+ /** …and if removing it puts the gold first (the counterfactual arm). */
80
+ counterfactual_gold_top1: boolean;
81
+ }
82
+ export interface DerivedCase {
83
+ id: string;
84
+ setup_facts: string[];
85
+ queries: DerivedQuery[];
86
+ analysis: CorpusAnalysis;
87
+ collision: CollisionSelfCheck | null;
88
+ guards: {
89
+ /** Candidates dropped because they contained a time expression (cumulative across the suite). */
90
+ no_time_word_dropped: number;
91
+ /** Negative-control queries emitted. */
92
+ irrelevant_queries: number;
93
+ };
94
+ }
95
+ export interface DeriveOptions {
96
+ seed?: number;
97
+ /** Maximum number of gold facts to derive a case from. */
98
+ maxGolds?: number;
99
+ /** Shared fragments folded into one query. */
100
+ fragmentsPerQuery?: number;
101
+ /** Target character length of a planted carrier (default: the real store's p75). */
102
+ carrierLength?: number;
103
+ }
104
+ /**
105
+ * Derive one case per shared-fragment gold fact (deterministic order, capped by `maxGolds`).
106
+ *
107
+ * The bridge fragment is a NONCE because the engine may not call a model: a real paraphrase's bridge
108
+ * (`我是谁?` → `用户是谁`, introducing `用户是`) cannot be produced without one. The nonce plays the
109
+ * same mechanical role — a literal the gold does not contain and an unrelated fact does — and the
110
+ * point of the fixture is the collision arithmetic, which is identical.
111
+ */
112
+ export declare function deriveCases(facts: readonly string[], opts?: DeriveOptions): DerivedCase[];
113
+ /**
114
+ * A deterministic corpus whose length distribution mirrors the real store's (requirement ①).
115
+ *
116
+ * `n` short-to-long facts are generated by padding a short seed sentence with the same unrelated
117
+ * filler the carriers use; the caller is expected to assert the result with {@link analyzeCorpus}
118
+ * rather than trust the generator.
119
+ */
120
+ export declare function synthesizeCorpus(n: number, seed?: number): string[];
@@ -0,0 +1,88 @@
1
+ /**
2
+ * The two arms the P-02 regression net runs, over the derived cases from `derive.ts`.
3
+ *
4
+ * - `stubRank` / `runDerivedCaseStub`: a DETERMINISTIC, model-free retriever that scores a fact by
5
+ * how many of the query's literal fragments it contains. It exists to prove the FIXTURE, not the
6
+ * product: a planted carrier that literally contains the bridge must outrank the gold in the base
7
+ * arm, and removing it must put the gold back at rank 1. That is the "counterfactual self-check"
8
+ * the collision sensitivity requirement asks for, and it needs no model to be meaningful.
9
+ * - `runDerivedCaseReal`: the same cases driven through a real runtime (`remember` + `recall`), for
10
+ * the nightly job. It is duck-typed so this module does not depend on the runtime.
11
+ *
12
+ * The two arms share ONE outcome shape, so a report can say which arm produced which numbers.
13
+ */
14
+ import { type DerivedCase, type GuardKind } from './derive.js';
15
+ export interface RankedFacts {
16
+ /** Fact indices, best first. */
17
+ ranked: number[];
18
+ /** The stub score of each ranked fact (parallel to `ranked`). */
19
+ scores: number[];
20
+ }
21
+ /**
22
+ * Deterministic lexical-overlap stub: score = number of the query's distinct fragments the fact
23
+ * contains literally. Raw count, deliberately NOT length-normalized — the pathology under test is
24
+ * "a long unrelated note that happens to contain the bridge out-scores the short gold", which a
25
+ * per-character normalization would erase.
26
+ */
27
+ export declare function stubRank(query: string, facts: readonly string[], k?: number): RankedFacts;
28
+ export interface DerivedQueryOutcome {
29
+ query: string;
30
+ guard: GuardKind;
31
+ ranked: number[];
32
+ scores: number[];
33
+ gold_rank: number | null;
34
+ carrier_rank: number | null;
35
+ must_include_ok: boolean;
36
+ must_exclude_ok: boolean;
37
+ /** The fixture's own recorded collision rate (fraction of non-gold facts sharing a fragment). */
38
+ collision_rate: number;
39
+ }
40
+ export interface CollisionCheckOutcome {
41
+ /** `true` when the planted carrier really does outrank the gold in the base arm. */
42
+ base_carrier_before_gold: boolean;
43
+ /** `true` when removing the carrier puts the gold first (the counterfactual arm). */
44
+ counterfactual_gold_top1: boolean;
45
+ }
46
+ export interface DerivedCaseOutcome {
47
+ id: string;
48
+ /** Arm label so a merged report cannot confuse stub numbers with real ones. */
49
+ arm: 'stub' | 'real';
50
+ queries: DerivedQueryOutcome[];
51
+ collision_self_check: CollisionCheckOutcome | null;
52
+ }
53
+ export declare function runDerivedCaseStub(c: DerivedCase): DerivedCaseOutcome;
54
+ export interface DerivedSuiteOutcome {
55
+ arm: 'stub';
56
+ cases: DerivedCaseOutcome[];
57
+ all_collision_self_checks_pass: boolean;
58
+ /** A retriever that emits any query containing a time expression has broken the guard. */
59
+ any_time_word_query: boolean;
60
+ /** Every `irrelevant` negative-control query must come back empty. */
61
+ all_irrelevant_empty: boolean;
62
+ }
63
+ export declare function runDerivedSuiteStub(cases: readonly DerivedCase[]): DerivedSuiteOutcome;
64
+ /** The slice of the runtime the real arm needs. Duck-typed on purpose (no runtime import). */
65
+ export interface DerivedRuntimePort {
66
+ remember(req: {
67
+ action: 'add';
68
+ content: string;
69
+ }): Promise<{
70
+ fact_id: number;
71
+ }>;
72
+ recall(req: {
73
+ action: 'search';
74
+ query: string;
75
+ limit?: number;
76
+ }): Promise<{
77
+ hits: {
78
+ ref_id: number;
79
+ }[];
80
+ }>;
81
+ }
82
+ /** Seed a case's facts and return the fact ids in `setup_facts` order. */
83
+ export declare function seedDerivedFacts(rt: DerivedRuntimePort, facts: readonly string[]): Promise<number[]>;
84
+ /**
85
+ * Drive one derived case through a real runtime. The `k` of each query is honored with `limit`
86
+ * (the derived queries use k=3), and the returned order is the store's own.
87
+ */
88
+ export declare function runDerivedCaseReal(rt: DerivedRuntimePort, c: DerivedCase): Promise<DerivedCaseOutcome>;
@@ -23,3 +23,27 @@ export interface AggregateMetrics {
23
23
  must_exclude_pass_rate: number;
24
24
  }
25
25
  export declare function aggregate(perQuery: PerQueryMetrics[]): AggregateMetrics;
26
+ /** Discounted cumulative gain at k (binary relevance). */
27
+ export declare function dcgAtK(actual: number[], expected: number[], k: number): number;
28
+ /**
29
+ * nDCG@k (binary relevance).
30
+ *
31
+ * 0 when there is nothing relevant to find (and when the ideal ranking is empty), never NaN — the
32
+ * per-query value is averaged and a NaN would poison the whole aggregate.
33
+ */
34
+ export declare function ndcgAtK(actual: number[], expected: number[], k: number): number;
35
+ /** How many relevant results appear in the first `n` slots. */
36
+ export declare function relevantInTop(actual: number[], expected: number[], n: number): number;
37
+ /** The window "top-3 relevant count" is measured over. */
38
+ export declare const TOP_N = 3;
39
+ export interface RankingMetrics {
40
+ n_queries: number;
41
+ mean_ndcg_at_k: number;
42
+ /** Relevant results inside the top {@link TOP_N}, summed over queries. */
43
+ top3_relevant_total: number;
44
+ /** Mean relevant results inside the top {@link TOP_N}. */
45
+ mean_top3_relevant: number;
46
+ /** Fraction of queries with at least one relevant result in the top {@link TOP_N}. */
47
+ top3_hit_rate: number;
48
+ }
49
+ export declare function aggregateRanking(perQuery: PerQueryMetrics[]): RankingMetrics;
@@ -1,10 +1,17 @@
1
- import { type PerQueryMetrics, type AggregateMetrics } from './metrics.js';
1
+ import { type PerQueryMetrics, type AggregateMetrics, type RankingMetrics } from './metrics.js';
2
2
  import type { EvalCase } from './loader.js';
3
3
  /** A retrieve function over the case's facts, returning indices in relevance order. */
4
4
  type RetrieveFn = (query: string, k: number, facts: string[]) => Promise<number[]>;
5
5
  interface EvalReport {
6
6
  perQuery: PerQueryMetrics[];
7
7
  summary: AggregateMetrics;
8
+ /**
9
+ * P-02 ranking-quality metrics. A SIBLING of `summary`, never a key inside it: `eval_zh.spec.ts`
10
+ * asserts `summary` with `toEqual` over exactly seven keys, so a new key there would turn a
11
+ * frozen-number regression into a key-set failure. `perQuery` is unchanged, so the bench
12
+ * harness's `(ids, scores)` fingerprint over it cannot move.
13
+ */
14
+ ranking: RankingMetrics;
8
15
  }
9
16
  export declare function evaluateCases(cases: EvalCase[], retrieve: RetrieveFn): Promise<EvalReport>;
10
17
  export {};
@@ -24,4 +24,4 @@ export * from './store/hybrid.js';
24
24
  export * from './store/source_picker.js';
25
25
  export * from './store/lexical.js';
26
26
  export * from './store/floors.js';
27
- export { registerSemanticBackend, registerReranker, registerVectorStore, type Reranker, type SemanticBackend, type VectorStore, } from '../mem-retrieval/index.js';
27
+ export { registerSemanticBackend, registerVectorStore, type SemanticBackend, type VectorStore, } from '../mem-retrieval/index.js';
@@ -40,8 +40,8 @@ export interface WarmOptions {
40
40
  deferTokenizerUntilIdle?: boolean;
41
41
  }
42
42
  /**
43
- * Fire-and-forget model bootstrap for DSH startup. Configures the semantic /
44
- * rerank model env (mirror + cache) and triggers the model download/load
43
+ * Fire-and-forget model bootstrap for DSH startup. Configures the semantic
44
+ * model env (mirror + cache) and triggers the model download/load
45
45
  * asynchronously, without blocking the plugin's `apply`. Returns the availability
46
46
  * of the semantic path once the download/load completes.
47
47
  *
@@ -20,7 +20,7 @@ export interface RuntimeOptions {
20
20
  semantic?: SemanticBackend;
21
21
  /**
22
22
  * The family framework's managed roots, as the built-in defaults for `tools.dir` and
23
- * `semantic.cache_dir` / `rerank.cache_dir` (see `LoadConfigOptions.managedRoots`).
23
+ * `semantic.cache_dir` (see `LoadConfigOptions.managedRoots`).
24
24
  *
25
25
  * The DSH plugin passes `<family home>/tools` and `<family home>/models` while
26
26
  * `@avantf/dsh-plugin-base` owns the fetching; the CLI and the MCP server pass nothing, because they
@@ -4,7 +4,7 @@
4
4
  * Small on purpose: `MemoryStore` and `KnowledgeStore` are separate stores over separate databases
5
5
  * and should stay that way (AGENTS.md). What they must NOT have is two copies of the same
6
6
  * three-line helper, because a fix then lands in one of them — which is exactly what happened to the
7
- * retrieval orchestration (see `store/hybrid.ts`) and to the foreign-vector-space warning below: the
7
+ * retrieval orchestration (see `store/hybrid.ts`) and to the stale-vector warning below: the
8
8
  * memory store checked whether its persisted vectors came from the CURRENT vector space and told the
9
9
  * operator how to fix it, while the knowledge store checked only the width, so a model swap left two
10
10
  * spaces silently mixed in one index until someone happened to run `kb_reindex`.
@@ -28,10 +28,25 @@ export declare function normalizeWrites(values: readonly string[]): string[];
28
28
  /**
29
29
  * Identity of the vector space a store currently writes into.
30
30
  *
31
- * A change of backend, model or width makes every stored vector stale, and the point of recording
32
- * the identity is that such a change is DETECTED instead of silently mixing two spaces in one index.
31
+ * A change of backend, model, width — or of anything else that decides which coordinates a vector
32
+ * lives in (pooling, normalization, the input window, the model revision behind an unchanged repo
33
+ * name) — makes every stored vector stale. The point of recording the identity is that such a
34
+ * change is DETECTED instead of silently mixing two spaces in one index. The backend declares its
35
+ * representation (`SemanticBackend.representation`); one that does not is fingerprinted as
36
+ * `rep=undeclared`, which is honest rather than a claim nobody verified.
33
37
  */
34
38
  export declare function vectorSpaceOf(semantic: SemanticBackend, model: string): string;
39
+ /** The two contract counts plus the internal "predates the representation fingerprint" split. */
40
+ export interface StaleVectorCounts {
41
+ stale: number;
42
+ space_stale: number;
43
+ /**
44
+ * Of {@link space_stale}, how many were written BEFORE the representation fingerprint (`v2/…`).
45
+ * The count is what makes the warning able to say "this is a one-time format upgrade" instead of
46
+ * blaming a model swap the operator never made.
47
+ */
48
+ legacy?: number;
49
+ }
35
50
  /**
36
51
  * Drop ids from the vector index, in one batch when the backend supports it.
37
52
  *
@@ -40,19 +55,26 @@ export declare function vectorSpaceOf(semantic: SemanticBackend, model: string):
40
55
  */
41
56
  export declare function evictVectors(vstore: VectorStore, ids: readonly number[]): void;
42
57
  /**
43
- * Tell the operator that persisted vectors came from ANOTHER vector space.
58
+ * Tell the operator that persisted vectors came from ANOTHER vector space — loudly, and with the
59
+ * manual remedy.
44
60
  *
45
- * A dim-valid vector from a different space is still RANKED — the reload cannot tell the difference
46
- * — so a silent model swap would mix two spaces in one index and degrade every semantic query
47
- * without an error anywhere. Said once per process; acting on it is the operator's choice because
48
- * re-encoding is a real cost (and they should see the count first).
61
+ * Changing the embedding space (model, width, pooling, normalization) is a DATA MIGRATION: the
62
+ * persisted bytes stay valid-looking but are no longer comparable with the current query vector, so
63
+ * the semantic leg must not serve them. Measured on the real library after the 512→768 default-model
64
+ * swap: 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval silently degraded to
65
+ * lexical+entity (every probe query answered with the same unrelated short fact), with no error
66
+ * anywhere. The count, the reason and the ONE manual entry have to be visible at start.
67
+ *
68
+ * Both halves are reported in one message because they are one condition seen two ways: a changed
69
+ * WIDTH (`stale`, whose bytes cannot even be decoded into the current store) and a changed SPACE at
70
+ * the same width (`space_stale`, whose bytes decode but rank in another model's coordinates).
49
71
  *
50
72
  * @param kind store label for the message (`memory` / `knowledge`)
51
73
  * @param space the vector space the store writes into now
52
- * @param foreign how many persisted vectors were written in a different one
53
- * @param fix the action that re-encodes them, as the operator would type it
74
+ * @param counts how many persisted vectors are unusable in the current space, by kind
75
+ * @param fix the manual entry that re-encodes them, as the operator would type it
54
76
  */
55
- export declare function reportForeignVectors(kind: string, space: string, foreign: number, fix: string): void;
77
+ export declare function reportStaleVectors(kind: string, space: string, counts: StaleVectorCounts, fix: string): void;
56
78
  /**
57
79
  * Hand the event loop back between two synchronous batches.
58
80
  *
@@ -0,0 +1,95 @@
1
+ /**
2
+ * The entity leg's metric: ANCHORED JACCARD WITH A SATURATING UNION.
3
+ *
4
+ * WHY THE OLD METRIC FAILED SHORT QUERIES. The leg scored the plain Jaccard ratio
5
+ * `|Q ∩ F| / |Q ∪ F|`. Both stores are written by the SAME entity extractor, which keeps every
6
+ * noun-ish token, so a real memory fact carries ~31 entity names (measured on the live store:
7
+ * min 3 / median 31 / max 66; `scripts/ENTITY_TOKENIZER_VALUE.md`). A query extracts one or two.
8
+ * Even a PERFECT 1-entity match is then `1 / (1 + 31 - 1) ≈ 0.03`, so with `min_jaccard` at any
9
+ * useful value the leg is unreachable for a short query BY CONSTRUCTION — measured: 1-entity
10
+ * queries could clear 0.2 against 2 of 80 facts. Extracting MORE entities only grows the
11
+ * denominator (the tokenizer direction was measured and rejected), so the fix has to be the
12
+ * metric, not the extractor.
13
+ *
14
+ * THE METRIC. `A` is the query's ANCHOR entities (below). The score keeps the Jaccard form — so a
15
+ * fact that is "about" the query outranks one that merely contains it — but the fact's own width
16
+ * enters through a SATURATING term:
17
+ *
18
+ * score(A, W, F) = |A ∩ F| / ( W + min(|F \ A|, ENTITY_UNION_CAP) )
19
+ *
20
+ * The query side is `W`, the query's FULL entity count, not `|A|`: the Jaccard denominator is the
21
+ * union, and the query's own width belongs in it whether or not each entity turned out to be
22
+ * discriminative (a question that names five things asks for more than one that names one). When
23
+ * every query entity is an anchor (the frozen eval set) and the fact is no wider than
24
+ * `W + ENTITY_UNION_CAP`, this IS the plain Jaccard ratio `|Q ∩ F| / |Q ∪ F|` — those facts keep
25
+ * their EXACT old score and ordering (verified bit-for-bit on all four eval arms). Past the cap the
26
+ * denominator stops growing, so a wide fact that genuinely mentions a query's rare entity is
27
+ * REACHABLE instead of being divided into oblivion. The cap is not a free parameter: the floor is
28
+ * 0.2 and a two-entity query sharing ONE entity must still clear it (`1 / (2 + C) >= 0.2 ⇒ C <= 3`),
29
+ * so `C = 3` is the largest fact penalty that keeps that shape.
30
+ *
31
+ * WHY ANCHORS (IDF FILTERING). Removing the width blow-up alone would let a corpus-wide word
32
+ * (`用户` / `插件` / `任务` …) admit EVERY fact that mentions it — "drag the whole store in". The
33
+ * extracted bags are dominated by such words, so the precision lever is rarity: an entity is an
34
+ * ANCHOR when its document frequency over the active corpus is at most {@link anchorCeiling}. Only
35
+ * anchors can produce candidates or contribute to the numerator; a query whose entities are ALL
36
+ * generic therefore yields no entity evidence and the other legs answer it. That is deliberate
37
+ * (silent beats noisy), and the FTS leg's 2-char substring fallback (§20.20⑦) serves the 2-char
38
+ * shape.
39
+ *
40
+ * WHAT `W` COUNTS, AND WHY IT IS NOT `|A|`. `W` is the query's FULL entity count — every entity the
41
+ * extractor returned, generic or absent included. A Jaccard denominator is the union of what the
42
+ * query asked for, and the old metric counted all of it; keeping `W` there is what makes a narrow
43
+ * fact's score IDENTICAL to the old ratio, so the frozen eval and every narrow real fact keep their
44
+ * exact numbers and ordering. Dropping the absent entities from `W` was measured and moved the live
45
+ * eval (`CI 用哪个 Python 版本` lost rank 1): a query that names things the corpus does not carry
46
+ * really is asking for more, and the floor is calibrated against that.
47
+ *
48
+ * THE CEILING SCALES WITH THE CORPUS, WITH A FLOOR. A fixed count would mean different things on a
49
+ * 3-fact eval case and a 300k-fact store, and a purely relative one would call every entity
50
+ * "generic" on a tiny corpus (`ceil(0.2 * 3) = 1` filters any name that appears twice — which is
51
+ * every name in a 3-fact relation case). {@link ANCHOR_MIN} forgives small corpora; the relative
52
+ * term is what bites on a real store. Both are calibrated, not guessed: DESIGN §20.19.
53
+ *
54
+ * @module store/entity_leg
55
+ */
56
+ /** An entity is an anchor at or below `max(ANCHOR_MIN, ceil(ANCHOR_RATE × activeFacts))`. */
57
+ export declare const ANCHOR_RATE = 0.2;
58
+ /**
59
+ * Small-corpus forgiveness, and its bound.
60
+ *
61
+ * Without it, `ceil(0.2 × 3) = 1` would call every name appearing twice "generic" — which is every
62
+ * name in a 3-fact relation case — and the eval set would lose its entity leg entirely. This is
63
+ * measured, not guessed: at `2` the eval corpus calls `Python` (carried by all three facts of its
64
+ * case) generic, and `CI 用哪个 Python 版本` loses its answer's entity evidence to the FTS leg —
65
+ * the live arm regressed `must_exclude`, the degraded arm lost rank 1 on that query. `4` keeps a
66
+ * name a small corpus repeats; the relative term is what bites on a real store (a 3-fact case
67
+ * filters nothing, by design).
68
+ */
69
+ export declare const ANCHOR_MIN = 4;
70
+ /**
71
+ * How many entities a fact may carry beyond the query's anchors before the width penalty stops
72
+ * growing (see the module doc: derived from the 0.2 floor and the 2-anchor reachability
73
+ * requirement, `1 / (2 + C) >= 0.2`).
74
+ */
75
+ export declare const ENTITY_UNION_CAP = 3;
76
+ /** The document-frequency ceiling for a corpus of `corpus` active facts/chunks. */
77
+ export declare function anchorCeiling(corpus: number): number;
78
+ /**
79
+ * The query entities that may serve as evidence.
80
+ *
81
+ * `df` is the DOCUMENT FREQUENCY of each query entity over the active corpus (how many facts /
82
+ * chunks carry it). An entity is kept when it occurs at least once and at most
83
+ * {@link anchorCeiling} times; the returned order is the input order (the caller only needs the
84
+ * set, and a stable order keeps the SQL/scoring deterministic).
85
+ */
86
+ export declare function selectAnchors(queryEntities: Iterable<string>, df: ReadonlyMap<string, number>, corpus: number): string[];
87
+ /**
88
+ * The leg's raw score, in [0, 1] — see the module doc for the formula and the constants.
89
+ *
90
+ * `0` for an empty anchor set (no discriminative query entity) and for "shares none". `queryWidth`
91
+ * is the query's full entity count (`W`), which keeps the narrow-fact case identical to the old
92
+ * Jaccard ratio; the fact's width only matters up to {@link ENTITY_UNION_CAP} extra entities, so a
93
+ * wide bag cannot divide a short query into nothing while a narrow one still ranks above a wide one.
94
+ */
95
+ export declare function anchoredOverlap(anchors: readonly string[], queryWidth: number, factEntities: ReadonlySet<string>): number;