@avantf/dsh-mem 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/lib/client.js +38 -9
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/chunks.d.ts +20 -1
- package/lib/engine/mem/db/dao/entities.d.ts +35 -11
- package/lib/engine/mem/db/dao/fact_sources.d.ts +53 -0
- package/lib/engine/mem/db/dao/facts.d.ts +115 -12
- package/lib/engine/mem/db/invariants.d.ts +26 -0
- package/lib/engine/mem/db/tokenizer.d.ts +23 -0
- package/lib/engine/mem/db/vectors.d.ts +32 -6
- package/lib/engine/mem/entities/extract.d.ts +1 -1
- package/lib/engine/mem/eval/derive.d.ts +120 -0
- package/lib/engine/mem/eval/derived_runner.d.ts +88 -0
- package/lib/engine/mem/eval/metrics.d.ts +24 -0
- package/lib/engine/mem/eval/runner.d.ts +8 -1
- package/lib/engine/mem/index.d.ts +1 -1
- package/lib/engine/mem/modelBootstrap.d.ts +2 -2
- package/lib/engine/mem/runtime.d.ts +1 -1
- package/lib/engine/mem/store/common.d.ts +18 -6
- package/lib/engine/mem/store/entity_leg.d.ts +95 -0
- package/lib/engine/mem/store/floors.d.ts +51 -31
- package/lib/engine/mem/store/hybrid.d.ts +21 -26
- package/lib/engine/mem/store/knowledge.d.ts +18 -9
- package/lib/engine/mem/store/lexical.d.ts +44 -0
- package/lib/engine/mem/store/memory.d.ts +94 -21
- package/lib/engine/mem/store/secret_guard.d.ts +34 -0
- package/lib/engine/mem/store/time_window.d.ts +74 -0
- package/lib/engine/mem-contract/config.d.ts +8 -16
- package/lib/engine/mem-contract/family.d.ts +1 -1
- package/lib/engine/mem-contract/tools.d.ts +10 -0
- package/lib/engine/mem-contract/types.d.ts +114 -9
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +37 -7
- package/lib/engine/mem-retrieval/fusion.d.ts +14 -1
- package/lib/engine/mem-retrieval/index.d.ts +1 -3
- package/lib/engine/mem-retrieval/interfaces.d.ts +38 -14
- package/lib/engine/mem-retrieval/registry.d.ts +2 -4
- package/lib/engine/mem-retrieval/representation.d.ts +50 -0
- package/lib/engine/mem-retrieval/stats.d.ts +1 -3
- package/lib/index.js +1680 -512
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +7 -0
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/remote.d.ts +12 -6
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
- package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
|
@@ -3,18 +3,16 @@
|
|
|
3
3
|
*
|
|
4
4
|
* `MemoryStore.search` and `KnowledgeStore.search` used to carry parallel copies of this flow
|
|
5
5
|
* (~80 lines each: limit normalization → weights → over-fetch → leg cap → fuse → live filter →
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* slice → output budget → health event). The legs themselves are legitimately store-specific —
|
|
7
|
+
* different tables, different filters (`category` vs `domain`/`source`), and only memory has the
|
|
8
|
+
* HRR probe — but the ORCHESTRATION is one algorithm, and two copies of it drifted:
|
|
9
9
|
*
|
|
10
10
|
* - the `NaN` limit guard existed only in memory, so a non-finite limit reached knowledge's SQL
|
|
11
11
|
* `LIMIT` and took the whole cross-store query down with it (`Promise.all` has no per-leg catch);
|
|
12
12
|
* - `retriever.over_fetch_factor` was read only by memory, while knowledge hardcoded `limit * 3`,
|
|
13
13
|
* so one config knob silently meant two different things;
|
|
14
14
|
* - `recordLegCapped()` was called only by memory, so a capped knowledge leg was unobservable —
|
|
15
|
-
* exactly the blindness DESIGN §20.17 says the counter exists to remove
|
|
16
|
-
* - the reranker's used/fallback flags were a helper in memory and an inline expression in
|
|
17
|
-
* knowledge.
|
|
15
|
+
* exactly the blindness DESIGN §20.17 says the counter exists to remove.
|
|
18
16
|
*
|
|
19
17
|
* Fixing those one at a time would leave the next drift a matter of when. The stores now supply
|
|
20
18
|
* their legs and their hit mapping; everything that can drift lives here once.
|
|
@@ -26,7 +24,7 @@
|
|
|
26
24
|
* @module store/hybrid
|
|
27
25
|
*/
|
|
28
26
|
import { type Config, type FloorProfile, type RecallResult, type RetrievalFloorDrops, type RetrievalFloors } from '../../mem-contract/index.js';
|
|
29
|
-
import { type BudgetInput, type Hit, type
|
|
27
|
+
import { type BudgetInput, type Hit, type SemanticBackend } from '../../mem-retrieval/index.js';
|
|
30
28
|
import { type FloorLeg } from './floors.js';
|
|
31
29
|
/** The `limit` a caller gets when it passes nothing usable. */
|
|
32
30
|
export declare const DEFAULT_SEARCH_LIMIT = 10;
|
|
@@ -63,8 +61,12 @@ export interface HybridLeg {
|
|
|
63
61
|
capped?: boolean;
|
|
64
62
|
/**
|
|
65
63
|
* Which floor governs this leg, for {@link HybridResult.dropped_by_floor}'s per-leg breakdown.
|
|
66
|
-
* The HRR probe shares the
|
|
67
|
-
* attributable without double-counting the
|
|
64
|
+
* The HRR probe shares the entity (`jaccard`) floor; naming it separately keeps a narrowed
|
|
65
|
+
* candidate set attributable without double-counting the entity leg's own drops.
|
|
66
|
+
*
|
|
67
|
+
* The `jaccard` KEY is historical: the entity leg's metric is now anchored Jaccard with a
|
|
68
|
+
* saturating union, whose unit (and the `min_jaccard` floor on it) is defined in
|
|
69
|
+
* `store/entity_leg.ts` — read that before re-calibrating the knob.
|
|
68
70
|
*/
|
|
69
71
|
leg?: keyof RetrievalFloorDrops;
|
|
70
72
|
/** Candidates this leg removed because they fell below its floor. */
|
|
@@ -113,7 +115,6 @@ export interface HybridDeps<H> {
|
|
|
113
115
|
kind: 'memory' | 'knowledge';
|
|
114
116
|
config: Config;
|
|
115
117
|
semantic: SemanticBackend;
|
|
116
|
-
reranker: Reranker;
|
|
117
118
|
/**
|
|
118
119
|
* The legs to fuse, as promises (or values). Awaited concurrently; a rejection costs that leg
|
|
119
120
|
* only. The async setup a store needs before it can build its legs — memory extracts the query's
|
|
@@ -121,7 +122,7 @@ export interface HybridDeps<H> {
|
|
|
121
122
|
* body, before the array is returned.
|
|
122
123
|
*/
|
|
123
124
|
legs(ctx: HybridContext): Promise<readonly (HybridLeg | Promise<HybridLeg>)[]>;
|
|
124
|
-
/** Text for the fused ids: the live filter
|
|
125
|
+
/** Text for the fused ids: the live filter and the hit bodies. */
|
|
125
126
|
texts(ids: number[]): Map<number, string>;
|
|
126
127
|
/** Build the caller-facing hits from the ranked, live, already-sliced ids. */
|
|
127
128
|
hits(ranked: readonly Hit[], texts: Map<number, string>): H[];
|
|
@@ -177,6 +178,12 @@ export interface HybridPlan {
|
|
|
177
178
|
* `queries` counts questions instead of legs (DESIGN §20.5).
|
|
178
179
|
*/
|
|
179
180
|
recordStats?: boolean;
|
|
181
|
+
/**
|
|
182
|
+
* P-01: attach per-leg evidence to the returned hits (`RecallHit.scores` / `final`). Default
|
|
183
|
+
* false, and that default is load-bearing: the evidence is explanation only, so a caller that did
|
|
184
|
+
* not ask must not pay for it in tokens (`fuse` skips building the arrays entirely).
|
|
185
|
+
*/
|
|
186
|
+
includeScores?: boolean;
|
|
180
187
|
}
|
|
181
188
|
/** A hit after the output budget: `text` may have been shortened, and `truncated` says so. */
|
|
182
189
|
export type Budgeted<T> = T & {
|
|
@@ -207,17 +214,6 @@ export interface HybridResult<H> {
|
|
|
207
214
|
* rather than per store, because the health counter compares against whatever this returns.
|
|
208
215
|
*/
|
|
209
216
|
export declare function legCapFor(config: Config, overFetch: number): number;
|
|
210
|
-
/**
|
|
211
|
-
* Whether the configured reranker is actually in use — the two flags a retrieval event carries.
|
|
212
|
-
*
|
|
213
|
-
* "Configured" is not "used": a reranker that is selected but unavailable falls back to the fused
|
|
214
|
-
* order, and counting that as a rerank would make the counter lie in exactly the case it exists to
|
|
215
|
-
* expose (the reference implementation leaves `rerank_fallback` unfeedable).
|
|
216
|
-
*/
|
|
217
|
-
export declare function rerankState(reranker: Reranker): {
|
|
218
|
-
used: boolean;
|
|
219
|
-
fallback: boolean;
|
|
220
|
-
};
|
|
221
217
|
/**
|
|
222
218
|
* Bound a ranked hit list to a token budget, marking what was shortened.
|
|
223
219
|
*
|
|
@@ -233,9 +229,8 @@ export declare function applyOutputBudget<H extends BudgetInput>(config: Config,
|
|
|
233
229
|
/**
|
|
234
230
|
* Run one hybrid search.
|
|
235
231
|
*
|
|
236
|
-
* The order of the last
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
* budget on hits that are about to be dropped.
|
|
232
|
+
* The order of the last steps is load-bearing: slice the fused (over-fetched) pool to `limit`,
|
|
233
|
+
* THEN apply the output budget, and only then tell the store what was returned. Budgeting before
|
|
234
|
+
* the slice would spend the budget on hits that are about to be dropped.
|
|
240
235
|
*/
|
|
241
236
|
export declare function hybridSearch<H extends BudgetInput>(deps: HybridDeps<H>, plan: HybridPlan): Promise<HybridResult<H>>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type AvantfLogger, type Config, type DocumentDetail, type DocumentSummary, type DomainCatalog, type FloorProfile, type ImportResult, type IngestResult, type KbConflictReport, type KbIngestConflict, type KbSyncReport, type KnowledgeConfig, type RecallHit, type ReindexReport } from '../../mem-contract/index.js';
|
|
2
|
-
import type {
|
|
2
|
+
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
3
|
import { type LexicalProbe } from './lexical.js';
|
|
4
4
|
import { type FloorLeg } from './floors.js';
|
|
5
5
|
import { type HybridResult } from './hybrid.js';
|
|
@@ -43,6 +43,8 @@ export interface KnowledgeSearchOptions {
|
|
|
43
43
|
* store-level "last result" field would race two concurrent queries.
|
|
44
44
|
*/
|
|
45
45
|
onResult?: (result: HybridResult<RecallHit>) => void;
|
|
46
|
+
/** P-01: attach per-leg evidence (`scores` / `final`) to every hit. Default false. */
|
|
47
|
+
includeScores?: boolean;
|
|
46
48
|
}
|
|
47
49
|
/**
|
|
48
50
|
* One identity a write will land on: the title it will use, the resolved local path it would read
|
|
@@ -87,7 +89,6 @@ export type IngestMode = 'add' | 'replace';
|
|
|
87
89
|
export declare class KnowledgeStore {
|
|
88
90
|
private readonly semantic;
|
|
89
91
|
private readonly vstore;
|
|
90
|
-
private readonly reranker;
|
|
91
92
|
/** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
|
|
92
93
|
private readonly domainConfigPath;
|
|
93
94
|
private readonly logger?;
|
|
@@ -127,7 +128,7 @@ export declare class KnowledgeStore {
|
|
|
127
128
|
private readonly allowedDomains;
|
|
128
129
|
/** Whether the config allowlist is active (non-empty); `false` = `domains: []` = no restriction. */
|
|
129
130
|
private readonly domainsRestricted;
|
|
130
|
-
constructor(dbPath: string, config: Config, kbConfig: KnowledgeConfig, semantic: SemanticBackend, vstore: VectorStore,
|
|
131
|
+
constructor(dbPath: string, config: Config, kbConfig: KnowledgeConfig, semantic: SemanticBackend, vstore: VectorStore,
|
|
131
132
|
/** The store config file `addDomain` writes the allowlist back to (`knowledgeConfigPath(home)`). */
|
|
132
133
|
domainConfigPath: string, logger?: AvantfLogger | undefined);
|
|
133
134
|
/**
|
|
@@ -302,11 +303,12 @@ export declare class KnowledgeStore {
|
|
|
302
303
|
*/
|
|
303
304
|
private encodeAndStore;
|
|
304
305
|
/**
|
|
305
|
-
* Identity of the vector space this store currently writes into (DESIGN §20):
|
|
306
|
-
* backend, model
|
|
307
|
-
*
|
|
306
|
+
* Identity of the vector space this store currently writes into (DESIGN §20): the representation
|
|
307
|
+
* fingerprint — backend, model, width AND the backend's declared pooling/normalization/input
|
|
308
|
+
* window/model revision — so any representation change makes stored chunk vectors stale instead
|
|
309
|
+
* of silently mixing two spaces. Public so diagnostics and tests can name the space.
|
|
308
310
|
*/
|
|
309
|
-
|
|
311
|
+
vectorSpace(): string;
|
|
310
312
|
/**
|
|
311
313
|
* A stored vector is reusable when it exists, matches the space, and its text is unchanged.
|
|
312
314
|
*
|
|
@@ -434,7 +436,7 @@ export declare class KnowledgeStore {
|
|
|
434
436
|
* and the chunk→hit mapping.
|
|
435
437
|
*
|
|
436
438
|
* The limit normalization, the over-fetch factor, the leg cap, the capped-leg counter, the
|
|
437
|
-
* degraded weights
|
|
439
|
+
* degraded weights and the output budget all live there now. They used to be copied
|
|
438
440
|
* here, and every fix landed on the memory side only — a `NaN` limit reached this store's SQL and
|
|
439
441
|
* took the whole cross-store query down with it, `retriever.over_fetch_factor` meant something
|
|
440
442
|
* different per store, and a capped knowledge leg was invisible to the health counters.
|
|
@@ -463,7 +465,14 @@ export declare class KnowledgeStore {
|
|
|
463
465
|
*/
|
|
464
466
|
lexicalProbe(text: string, stopAt?: number): LexicalProbe;
|
|
465
467
|
private ftsPath;
|
|
466
|
-
/**
|
|
468
|
+
/**
|
|
469
|
+
* Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan.
|
|
470
|
+
*
|
|
471
|
+
* Same metric as the memory store's sibling (`store/entity_leg.ts` carries the measurement and
|
|
472
|
+
* the reasoning): anchored Jaccard with a saturating union over the query's rare entities, so a
|
|
473
|
+
* wide chunk cannot dilute the denominator and a corpus-wide word cannot drag the whole knowledge
|
|
474
|
+
* base in.
|
|
475
|
+
*/
|
|
467
476
|
private jaccardPath;
|
|
468
477
|
}
|
|
469
478
|
export interface Chunk {
|
|
@@ -25,6 +25,20 @@
|
|
|
25
25
|
* `matched >= 2`, with the margin between 2 and 1 rather than a tuned threshold inside a
|
|
26
26
|
* continuum. A single-word query (`cgroup`) scores 1 and stays silent on purpose: one term is not
|
|
27
27
|
* evidence.
|
|
28
|
+
*
|
|
29
|
+
* THE SHORT-CJK FALLBACK (and why the hint probe does NOT use it). The trigram index has no term
|
|
30
|
+
* shorter than three characters — measured on the live store: `fts5vocab` over `facts_fts` holds
|
|
31
|
+
* 27906 distinct terms and EVERY one is 3 characters (and a 2-char `MATCH` returns nothing, even as
|
|
32
|
+
* a prefix query). So a 2-char CJK query produces no FTS term at all and that leg is empty by
|
|
33
|
+
* construction. The FTS LEG therefore falls back to the one finer predicate the FTS5 trigram TABLE
|
|
34
|
+
* does expose: `content LIKE '%…%'`, which matches a 2-char run by scanning (no index can serve a
|
|
35
|
+
* 2-char pattern). See {@link substringTerms} / {@link gradedTerms}.
|
|
36
|
+
*
|
|
37
|
+
* The conditional-hint probe deliberately keeps the index-expressible terms only: it is
|
|
38
|
+
* SYNCHRONOUS and runs on every step's prompt assembly, so it must stay a bounded number of
|
|
39
|
+
* `LIMIT 1` index lookups — an O(corpus) scan per 2-char term would put a full scan on the prompt
|
|
40
|
+
* path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
|
|
41
|
+
* (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
|
|
28
42
|
*/
|
|
29
43
|
/** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
|
|
30
44
|
export interface LexicalProbe {
|
|
@@ -46,6 +60,36 @@ export type RelevanceHit = boolean;
|
|
|
46
60
|
* caller runs one `LIMIT 1` MATCH per term and counts the hits.
|
|
47
61
|
*/
|
|
48
62
|
export declare function relevanceTerms(text: string, cap?: number): string[];
|
|
63
|
+
/**
|
|
64
|
+
* The terms the FTS leg falls back to when the trigram index can express NONE of the query.
|
|
65
|
+
*
|
|
66
|
+
* A 2-char CJK run IS the finer granularity the FTS5 trigram TABLE can still answer — not as an
|
|
67
|
+
* indexed token (the vocabulary holds 3-character terms only, measured; a 2-char `MATCH`, quoted or
|
|
68
|
+
* prefixed, returns zero rows) but as the substring predicate FTS5 exposes on a trigram table
|
|
69
|
+
* (`content LIKE '%…%'`, which falls back to a scan for patterns under three characters). One-char
|
|
70
|
+
* runs are deliberately NOT included: a single character is not evidence (the same reason
|
|
71
|
+
* {@link looksRelevant} needs two terms), and it would make the leg match a large share of any
|
|
72
|
+
* corpus. Latin runs are not included either — `buildFtsQuery` already admits latin tokens from
|
|
73
|
+
* three characters, and the module's own floor (5, `MIN_LATIN`) is a separate, measured decision.
|
|
74
|
+
*
|
|
75
|
+
* PRECISION GUARD. The fallback is graded by the SAME per-row floor as the indexed path and by the
|
|
76
|
+
* reachability clamp: for a single 2-char term the effective bar is 1, and the term is a substring,
|
|
77
|
+
* so "the row contains the query's characters CONTIGUOUSLY" is what admits it — not "shares one
|
|
78
|
+
* incidental trigram". A two-run query ('李娜 张伟') produces two substring terms and is judged
|
|
79
|
+
* against `min(configured, 2)`, i.e. both runs must occur in the row.
|
|
80
|
+
*/
|
|
81
|
+
export declare function substringTerms(text: string, cap?: number): string[];
|
|
82
|
+
/**
|
|
83
|
+
* The term set the FTS leg is SEARCHED and GRADED with — the one definition of "the query's terms"
|
|
84
|
+
* for that leg (`store/floors.ts` grades with it, `store/hybrid.ts` resolves the reachability clamp
|
|
85
|
+
* from it, and the stores build their MATCH / substring probe from it).
|
|
86
|
+
*
|
|
87
|
+
* The fallback is whole-query, not per run: any text that yields even one index-expressible term is
|
|
88
|
+
* returned UNCHANGED from {@link relevanceTerms}. That is what keeps every long query bit-identical
|
|
89
|
+
* (a mixed text like `缓存失效 李娜` keeps only its trigrams, exactly as before) and confines the
|
|
90
|
+
* new behaviour to the shape that had NO leg at all.
|
|
91
|
+
*/
|
|
92
|
+
export declare function gradedTerms(text: string, cap?: number): string[];
|
|
49
93
|
/**
|
|
50
94
|
* The gate itself: is this store worth telling the model about?
|
|
51
95
|
*
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type TrustDiagnostic, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
2
|
-
import type {
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
2
|
+
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
3
|
import type { Db } from '../db/conn.js';
|
|
4
4
|
import { type LexicalProbe } from './lexical.js';
|
|
5
5
|
import { type FloorLeg } from './floors.js';
|
|
@@ -88,6 +88,13 @@ export interface ContradictionResolution {
|
|
|
88
88
|
export interface SearchInput {
|
|
89
89
|
query: string;
|
|
90
90
|
category?: string;
|
|
91
|
+
/**
|
|
92
|
+
* P-08: restrict every leg to facts whose `fact_sources.ref` equals this value. Applied INSIDE
|
|
93
|
+
* each leg's SQL (an `EXISTS` predicate, before its `LIMIT cap`), never as a post-fusion filter:
|
|
94
|
+
* pruning after fusion would return fewer than `limit` and would make the relaxed-floor retry
|
|
95
|
+
* mistake a source-empty result for "the floors were too strict".
|
|
96
|
+
*/
|
|
97
|
+
source?: string;
|
|
91
98
|
limit?: number;
|
|
92
99
|
/** Also fuse the HRR entity-probe path (used by `recall.probe`). */
|
|
93
100
|
includeHrr?: boolean;
|
|
@@ -134,17 +141,34 @@ export interface SearchInput {
|
|
|
134
141
|
* `track`, which is about the dormancy clock — a caller may want one without the other.
|
|
135
142
|
*/
|
|
136
143
|
recordStats?: boolean;
|
|
144
|
+
/** P-01: attach per-leg evidence (`scores` / `final`) to the returned hits. Default false. */
|
|
145
|
+
includeScores?: boolean;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* P-08 / P-13 write-side extras on `add` / `update`.
|
|
149
|
+
*
|
|
150
|
+
* All three are optional and "omitted" has one meaning everywhere: do not write (a new row's
|
|
151
|
+
* NULL/1 default, or the existing row's value on a duplicate) — the store never invents a source,
|
|
152
|
+
* an event time or an end date.
|
|
153
|
+
*/
|
|
154
|
+
export interface FactWriteOptions {
|
|
155
|
+
/** P-08: `fact_sources` provenance. `kind:` prefix selects the kind, else `manual`. */
|
|
156
|
+
sourceRef?: string;
|
|
157
|
+
/** P-13: event time → `valid_from`. */
|
|
158
|
+
eventDate?: string;
|
|
159
|
+
/** P-13: known end → `valid_to`, WITHOUT archiving the row. */
|
|
160
|
+
validUntil?: string;
|
|
137
161
|
}
|
|
138
162
|
export declare class MemoryStore {
|
|
139
163
|
private readonly db;
|
|
140
164
|
private readonly config;
|
|
141
165
|
private readonly semantic;
|
|
142
166
|
private readonly vstore;
|
|
143
|
-
private readonly reranker;
|
|
144
167
|
private readonly contradictions;
|
|
145
168
|
private readonly facts;
|
|
146
169
|
private readonly entities;
|
|
147
170
|
private readonly triples;
|
|
171
|
+
private readonly sources;
|
|
148
172
|
/**
|
|
149
173
|
* The tokenizer `facts_fts` was ACTUALLY built with, read from the database at open.
|
|
150
174
|
*
|
|
@@ -160,7 +184,7 @@ export declare class MemoryStore {
|
|
|
160
184
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
161
185
|
*/
|
|
162
186
|
private conflictDrainCursor;
|
|
163
|
-
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
187
|
+
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
164
188
|
/** The store's database path, used ONLY to name the vector index snapshot beside it. */
|
|
165
189
|
dbPath?: string);
|
|
166
190
|
/**
|
|
@@ -171,8 +195,12 @@ export declare class MemoryStore {
|
|
|
171
195
|
trustTick(opts?: {
|
|
172
196
|
budget?: number | null;
|
|
173
197
|
}): TickResult;
|
|
174
|
-
/**
|
|
175
|
-
|
|
198
|
+
/**
|
|
199
|
+
* Identity of the vector space this store currently writes into — the REPRESENTATION fingerprint
|
|
200
|
+
* (`vectorSpaceId` + the backend's declared pooling/normalize/window/revision), not just the
|
|
201
|
+
* model name. Public so diagnostics and tests can name the space the store would record.
|
|
202
|
+
*/
|
|
203
|
+
vectorSpace(): string;
|
|
176
204
|
/**
|
|
177
205
|
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs.
|
|
178
206
|
* Without this the semantic path is silently empty after every process restart
|
|
@@ -193,7 +221,13 @@ export declare class MemoryStore {
|
|
|
193
221
|
* is the read model those surfaces use instead of the full diagnose pass.
|
|
194
222
|
*/
|
|
195
223
|
vectorSpaceHealth(): VectorSpaceHealth;
|
|
196
|
-
|
|
224
|
+
/**
|
|
225
|
+
* {@link vectorSpaceHealth} plus the internal `legacy` split (how many `space_stale` rows predate
|
|
226
|
+
* the representation fingerprint). Kept off the contract type so the public detection surface —
|
|
227
|
+
* and every existing assertion on it — is unchanged; only the warning's wording uses it.
|
|
228
|
+
*/
|
|
229
|
+
private vectorSpaceCounts;
|
|
230
|
+
add(content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
197
231
|
/**
|
|
198
232
|
* Write-path contradiction detection (DESIGN §11): check the fact that was just
|
|
199
233
|
* written against the corpus, then report its FULL open-conflict set.
|
|
@@ -212,7 +246,7 @@ export declare class MemoryStore {
|
|
|
212
246
|
private detectContradictions;
|
|
213
247
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
214
248
|
private maybeIndexSemantic;
|
|
215
|
-
update(fact_id: number, content: string, category?: string, ttlDays?: number): Promise<RememberResult>;
|
|
249
|
+
update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
216
250
|
remove(fact_id: number, reason?: string): boolean;
|
|
217
251
|
archive(fact_id: number, reason?: string): boolean;
|
|
218
252
|
/**
|
|
@@ -250,7 +284,17 @@ export declare class MemoryStore {
|
|
|
250
284
|
* @returns the page, the page size (`count`), the matching total, and whether
|
|
251
285
|
* more rows remain after this page.
|
|
252
286
|
*/
|
|
253
|
-
list(category?: string, status?: string, limit?: number, offset?: number
|
|
287
|
+
list(category?: string, status?: string, limit?: number, offset?: number,
|
|
288
|
+
/** P-08: only facts whose `fact_sources.ref` equals this value (reverse provenance lookup). */
|
|
289
|
+
source?: string): FactPage;
|
|
290
|
+
/**
|
|
291
|
+
* P-08 coverage over the ACTIVE corpus, for `admin stats`: the share of facts that can answer
|
|
292
|
+
* "why do I remember this". Reported from day one so the field cannot quietly become a second
|
|
293
|
+
* `mirror_source` (filled but meaningless).
|
|
294
|
+
*/
|
|
295
|
+
sourceCoverage(): SourceCoverage;
|
|
296
|
+
/** P-13 coverage over the ACTIVE corpus: the share of facts carrying an event time. */
|
|
297
|
+
validityCoverage(): ValidityCoverage;
|
|
254
298
|
countByStatus(): {
|
|
255
299
|
active: number;
|
|
256
300
|
archived: number;
|
|
@@ -474,20 +518,32 @@ export declare class MemoryStore {
|
|
|
474
518
|
* rescale the survivors. That covers the Jaccard and FTS legs. It does NOT cover the HRR leg, whose
|
|
475
519
|
* candidates arrive in the JACCARD leg's order (or by recency in its fallback), so its cap can
|
|
476
520
|
* remove its own highest scorer and `v / max` really does move. Accepted — the cap bounds per-query
|
|
477
|
-
* mission
|
|
478
|
-
*
|
|
521
|
+
* mission — but do not read these scores as comparable across different cap/over-fetch settings;
|
|
522
|
+
* see `docs/PROVENANCE_REVIEW.md` N5 for the missing differential.
|
|
479
523
|
*/
|
|
480
524
|
private searchLegs;
|
|
481
525
|
/**
|
|
482
|
-
*
|
|
483
|
-
*
|
|
484
|
-
*
|
|
485
|
-
*
|
|
526
|
+
* P-11: active facts whose EVENT time (`valid_from`) falls in the query's own time window.
|
|
527
|
+
*
|
|
528
|
+
* THREE DELIBERATE PROPERTIES, each of which the acceptance criteria name:
|
|
529
|
+
*
|
|
530
|
+
* 1. **Only `valid_from`.** A fact with no event time is not a candidate, and there is no
|
|
531
|
+
* fallback to `created_at`: mixing a write clock into the event-time axis would make the leg
|
|
532
|
+
* "work" on a corpus where it measured zero effect and would answer "上个月做了什么" with
|
|
533
|
+
* facts merely WRITTEN last month.
|
|
534
|
+
* 2. **A leg, not a filter.** The window only ever ADDS candidates at a constant score; it never
|
|
535
|
+
* removes one another leg (or the FTS/entity/semantic path) produced. A hard pre-filter would
|
|
536
|
+
* drop gold facts whose event time is outside the phrase's window but whose text is the answer.
|
|
537
|
+
* 3. **Constant score, capped.** Every in-window fact scores 1, so `fuse`'s per-leg max-scaling
|
|
538
|
+
* is cap-INVARIANT for this leg (any survivor IS the maximum) and the ranking WITHIN the
|
|
539
|
+
* window stays the other legs' business. The constant is why "which of last month's facts is
|
|
540
|
+
* the answer" is not decided by a date.
|
|
541
|
+
*
|
|
542
|
+
* Cost: one non-indexable scan (the date prefix is computed, so no index can serve it) of the
|
|
543
|
+
* active corpus, capped at `legCap`. It runs only when `retriever.time_window` is on — the whole
|
|
544
|
+
* point of the default-OFF switch.
|
|
486
545
|
*/
|
|
487
|
-
|
|
488
|
-
used: boolean;
|
|
489
|
-
fallback: boolean;
|
|
490
|
-
};
|
|
546
|
+
private timeWindowPath;
|
|
491
547
|
/**
|
|
492
548
|
* The one fact→RecallHit mapper (kind/source_ref/entities shape lives here ONLY;
|
|
493
549
|
* chain/reason/ask/search all used to carry byte-identical copies, three of them
|
|
@@ -553,8 +609,15 @@ export declare class MemoryStore {
|
|
|
553
609
|
lexicalProbe(text: string, stopAt?: number): LexicalProbe;
|
|
554
610
|
private ftsPath;
|
|
555
611
|
/**
|
|
556
|
-
*
|
|
557
|
-
* sharing at least one query entity, pre-ranked by how many they share and already capped.
|
|
612
|
+
* Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
|
|
613
|
+
* sharing at least one ANCHOR query entity, pre-ranked by how many they share and already capped.
|
|
614
|
+
*
|
|
615
|
+
* The score is anchored Jaccard with a saturating union —
|
|
616
|
+
* `|anchors ∩ F| / (queryWidth + min(|F \ anchors|, ENTITY_UNION_CAP))`; `store/entity_leg.ts`
|
|
617
|
+
* carries why the old `|Q ∩ F| / |Q ∪ F|` was structurally unreachable for short queries, and why
|
|
618
|
+
* the generic-word filter keeps the wider denominator from dragging the corpus in. The fact set is
|
|
619
|
+
* read per candidate and the score is recomputed here even though the DAO ordered by it, so one
|
|
620
|
+
* formula stays in charge (the DAO's `total` tie-break only orders equal scores).
|
|
558
621
|
*/
|
|
559
622
|
private jaccardPath;
|
|
560
623
|
/**
|
|
@@ -569,6 +632,16 @@ export declare class MemoryStore {
|
|
|
569
632
|
* defaults (`general` / `0`).
|
|
570
633
|
*/
|
|
571
634
|
private persistFact;
|
|
635
|
+
/**
|
|
636
|
+
* P-08: land the provenance rows for one write.
|
|
637
|
+
*
|
|
638
|
+
* Rules, in order: an explicit `source_ref` is recorded (classified by its `kind:` prefix); on a
|
|
639
|
+
* REVISION with none given, the replaced row's sources are copied forward (it is the same logical
|
|
640
|
+
* fact, and the new row would otherwise become untraceable); on a plain duplicate, nothing is
|
|
641
|
+
* added (its sources are already there). Nothing is ever invented — an omitted source on a first
|
|
642
|
+
* add leaves the fact with no `fact_sources` row at all, which is what the stats coverage reports.
|
|
643
|
+
*/
|
|
644
|
+
private recordSources;
|
|
572
645
|
/**
|
|
573
646
|
* Link the revision chain and archive the superseded fact — on both the fresh-insert
|
|
574
647
|
* and duplicate-content branches.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-side secret / PII shape guard (P-03) — a PURE function, no store, no IO.
|
|
3
|
+
*
|
|
4
|
+
* The memory store is a long-lived, backed-up, sometimes-remote copy of what the model chose to
|
|
5
|
+
* write down, so a plaintext credential that reaches it is hard to retract. This guard refuses a
|
|
6
|
+
* fact whose TEXT matches a HIGH-CONFIDENCE credential/PII shape. It is deliberately narrow: only
|
|
7
|
+
* shapes that a false positive is unlikely to produce, because a guard that refuses legitimate
|
|
8
|
+
* notes is worse than no guard — the model learns to stop recording.
|
|
9
|
+
*
|
|
10
|
+
* What it does NOT do, by design:
|
|
11
|
+
* - no rewriting (a silently redacted fact is a lie the caller cannot see);
|
|
12
|
+
* - no scanning of what is already stored (writes only, never a retroactive sweep);
|
|
13
|
+
* - no knowledge-base coverage (`kb_add` ingests user-provided documents and is out of scope);
|
|
14
|
+
* - no config/override path (the refusal carries its own remedy: record WHERE the secret lives).
|
|
15
|
+
*
|
|
16
|
+
* The refusal message is the interface: it names the shape and tells the caller what to write
|
|
17
|
+
* instead. The plugin turns a thrown store error into a tool error through its existing `wireErr`
|
|
18
|
+
* path, so no new field and no new error surface is introduced.
|
|
19
|
+
*/
|
|
20
|
+
export type SecretKind = 'provider-key' | 'private-key' | 'connection-string' | 'card-number' | 'national-id';
|
|
21
|
+
export interface SecretShape {
|
|
22
|
+
kind: SecretKind;
|
|
23
|
+
/** Human-readable name of the shape, safe to put in an error (never the matched value). */
|
|
24
|
+
label: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The guard. Returns the matched SHAPE (never the matched text) or `null`.
|
|
28
|
+
*
|
|
29
|
+
* Order is by specificity: a PEM block or a known prefix is a stronger signal than a bare number,
|
|
30
|
+
* so a fact that carries both is reported as the credential it obviously is.
|
|
31
|
+
*/
|
|
32
|
+
export declare function detectSecretShape(text: string): SecretShape | null;
|
|
33
|
+
/** The refusal the caller sees — names the shape and says what to write instead. */
|
|
34
|
+
export declare function secretRefusalMessage(shape: SecretShape): string;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* P-11 · the deterministic Chinese time-expression parser behind the (default-OFF) time-window leg.
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS. One pure function that turns a query's own text into ONE half-open window
|
|
5
|
+
* `[start, end)` in epoch milliseconds, plus the `rule` that produced it. It has no clock of its
|
|
6
|
+
* own — `now` is a parameter (default `new Date()`), so a caller can pin it ("pinned_now") and get
|
|
7
|
+
* bit-identical answers from any machine. No third-party date library, no locale data: only `Date`
|
|
8
|
+
* and regexes.
|
|
9
|
+
*
|
|
10
|
+
* WHY THE LEG NEEDS IT AT ALL. "上个月做了什么" carries its evidence in the TIME EXPRESSION, not in
|
|
11
|
+
* the words: the fact that answers it reads "完成了支付网关的重构" and shares no query term. No
|
|
12
|
+
* lexical, entity or semantic leg can see that; the window is the only usable signal.
|
|
13
|
+
*
|
|
14
|
+
* PARITY WITH THE BENCHMARK HARNESS. The rules, boundaries and `rule` names mirror
|
|
15
|
+
* `scripts/spikes/bench-r4-lib.mjs`'s `parseTimeWindow` verbatim (that is the copy the A2 derived
|
|
16
|
+
* "time" shape and the round-4/round-6 cards already use to pick their windows). The benchmark
|
|
17
|
+
* harness is NOT modified, so a production/harness disagreement would show up as an unreproducible
|
|
18
|
+
* measurement — keep the two in step. The ONE intended addition here is Chinese numerals in
|
|
19
|
+
* `最近N天` ("最近三天"), which the spike's `\d+` cannot express.
|
|
20
|
+
*
|
|
21
|
+
* WHAT IT DELIBERATELY DOES NOT PARSE. Quarter words (`上季度`, `第三季度`) and anything else outside
|
|
22
|
+
* the table below return `undefined` rather than a guess: a wrong window would silently boost the
|
|
23
|
+
* wrong facts, and "no window" only leaves the other legs exactly as they were.
|
|
24
|
+
*
|
|
25
|
+
* @module store/time_window
|
|
26
|
+
*/
|
|
27
|
+
/** Milliseconds in a day. Rolling windows use it the way the spike does (no DST in the zh corpus). */
|
|
28
|
+
export declare const DAY_MS = 86400000;
|
|
29
|
+
/** One parsed window. `end` is EXCLUSIVE. */
|
|
30
|
+
export interface TimeWindow {
|
|
31
|
+
/** Inclusive start, epoch ms. */
|
|
32
|
+
start: number;
|
|
33
|
+
/** Exclusive end, epoch ms. */
|
|
34
|
+
end: number;
|
|
35
|
+
/** The spike's rule label (also what the benchmark reports as the query's time shape). */
|
|
36
|
+
rule: string;
|
|
37
|
+
/** The literal span that matched, verbatim. */
|
|
38
|
+
matched: string;
|
|
39
|
+
}
|
|
40
|
+
/** Inclusive local-day bounds, the form the SQL predicate compares `valid_from`'s date prefix with. */
|
|
41
|
+
export interface DayBounds {
|
|
42
|
+
from: string;
|
|
43
|
+
to: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse an arabic or Chinese cardinal (`3`, `三`, `十`, `十三`, `二十一`) for the `最近N天` window.
|
|
47
|
+
*
|
|
48
|
+
* Capped at what a day count can plausibly be: values above 99 are treated as unparseable rather
|
|
49
|
+
* than silently producing a decade-long window. Returns `undefined` when the text is not a number
|
|
50
|
+
* this grammar knows, so the caller can fall through to the next rule.
|
|
51
|
+
*/
|
|
52
|
+
export declare function parseCount(text: string): number | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* The first time window in `text`, or `undefined` when the text carries no expression from the
|
|
55
|
+
* table. Ordered ABSOLUTE before RELATIVE (the spike's order): a text naming a concrete date must
|
|
56
|
+
* not be re-read as a relative one.
|
|
57
|
+
*/
|
|
58
|
+
export declare function parseTimeWindow(text: string, now?: Date): TimeWindow | undefined;
|
|
59
|
+
/** Local `YYYY-MM-DD` for an epoch-ms instant (the form `valid_from`'s prefix is compared against). */
|
|
60
|
+
export declare function localDayString(ms: number): string;
|
|
61
|
+
/**
|
|
62
|
+
* The INCLUSIVE local-day bounds of a window, for a `substr(valid_from,1,10) BETWEEN ? AND ?`
|
|
63
|
+
* predicate.
|
|
64
|
+
*
|
|
65
|
+
* `end` is exclusive in epoch terms, and an event time is stored at day granularity (`2026-09-15`)
|
|
66
|
+
* or finer; `end - 1ms` therefore names the last DAY the window can contain. That is what makes
|
|
67
|
+
* `最近三天` include today (`end` is "now", so `end - 1ms` is still today) while `上个月` stops at
|
|
68
|
+
* the month's last day (`end` is the 1st, so `end - 1ms` is the last day of the previous month)
|
|
69
|
+
* instead of leaking the 1st of the current one.
|
|
70
|
+
*
|
|
71
|
+
* An empty or inverted window yields `from > to`, and the SQL `BETWEEN` then matches nothing —
|
|
72
|
+
* the honest answer for "最近0天".
|
|
73
|
+
*/
|
|
74
|
+
export declare function windowDayBounds(window: TimeWindow): DayBounds;
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
/**
|
|
3
|
-
* Input window of the SHIPPED
|
|
4
|
-
* `Xenova/bge-
|
|
5
|
-
*
|
|
6
|
-
* the embedder changed; only its vector width did, 512 → 768.)
|
|
3
|
+
* Input window of the SHIPPED model: `Xenova/bge-base-zh-v1.5` reads 512 tokens. (The previous
|
|
4
|
+
* default embedder, `Xenova/bge-small-zh-v1.5`, shares the same 512-token window — this constant
|
|
5
|
+
* did NOT move when the embedder changed; only its vector width did, 512 → 768.)
|
|
7
6
|
*
|
|
8
|
-
* Exported because more than one default derives from it: the embedder's fallback window
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Exported because more than one default derives from it: the embedder's fallback window and
|
|
8
|
+
* `knowledge.chunk_size` below. transformers.js truncates past the window SILENTLY, so every
|
|
9
|
+
* derivation has to agree on the number.
|
|
11
10
|
*/
|
|
12
11
|
export declare const DEFAULT_MODEL_WINDOW_TOKENS = 512;
|
|
13
12
|
/** The merged runtime config shared by memory and knowledge stores. */
|
|
@@ -23,14 +22,6 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
23
22
|
auto_download: z.ZodDefault<z.ZodBoolean>;
|
|
24
23
|
auto_migrate: z.ZodDefault<z.ZodBoolean>;
|
|
25
24
|
}, z.core.$strip>>;
|
|
26
|
-
rerank: z.ZodPrefault<z.ZodObject<{
|
|
27
|
-
backend: z.ZodDefault<z.ZodString>;
|
|
28
|
-
local_model: z.ZodDefault<z.ZodString>;
|
|
29
|
-
max_input_tokens: z.ZodDefault<z.ZodNumber>;
|
|
30
|
-
mirror: z.ZodDefault<z.ZodString>;
|
|
31
|
-
cache_dir: z.ZodDefault<z.ZodString>;
|
|
32
|
-
auto_download: z.ZodDefault<z.ZodBoolean>;
|
|
33
|
-
}, z.core.$strip>>;
|
|
34
25
|
vectorStore: z.ZodPrefault<z.ZodObject<{
|
|
35
26
|
backend: z.ZodDefault<z.ZodString>;
|
|
36
27
|
auto_thresholds: z.ZodDefault<z.ZodObject<{
|
|
@@ -48,6 +39,7 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
48
39
|
min_fts_terms: z.ZodDefault<z.ZodNumber>;
|
|
49
40
|
min_jaccard: z.ZodDefault<z.ZodNumber>;
|
|
50
41
|
leg_cap: z.ZodDefault<z.ZodNumber>;
|
|
42
|
+
time_window: z.ZodDefault<z.ZodBoolean>;
|
|
51
43
|
}, z.core.$strip>>;
|
|
52
44
|
lifecycle: z.ZodPrefault<z.ZodObject<{
|
|
53
45
|
purge_after_archived_days: z.ZodDefault<z.ZodNumber>;
|
|
@@ -120,8 +112,8 @@ export declare const KnowledgeConfigSchema: z.ZodObject<{
|
|
|
120
112
|
chunk_size: z.ZodDefault<z.ZodNumber>;
|
|
121
113
|
chunk_overlap: z.ZodDefault<z.ZodNumber>;
|
|
122
114
|
source_priority: z.ZodDefault<z.ZodEnum<{
|
|
123
|
-
none: "none";
|
|
124
115
|
workspace: "workspace";
|
|
116
|
+
none: "none";
|
|
125
117
|
}>>;
|
|
126
118
|
ingest: z.ZodPrefault<z.ZodObject<{
|
|
127
119
|
local_roots: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* `@avantf/dsh-plugin-base` (the family's environment framework) installs everything it manages under a
|
|
6
6
|
* single root: `$AVANTF_HOME` when set, else `~/.avantf/env`. `tools` holds external binaries
|
|
7
|
-
* (pandoc), `models` the downloaded embedding
|
|
7
|
+
* (pandoc), `models` the downloaded embedding weights in the flat layout the transformers.js
|
|
8
8
|
* cache reads. The engine's built-in defaults point HERE, not at the pre-framework
|
|
9
9
|
* `~/.avantf/{tools,models}`: those legacy directories are no longer a fallback anywhere, so a
|
|
10
10
|
* machine that never creates them loses nothing, and no process resolves a path that only missions
|