@avantf/dsh-mem 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/lib/client.js +40 -9
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/chunks.d.ts +20 -1
- package/lib/engine/mem/db/dao/entities.d.ts +35 -11
- package/lib/engine/mem/db/dao/fact_sources.d.ts +53 -0
- package/lib/engine/mem/db/dao/facts.d.ts +127 -12
- package/lib/engine/mem/db/invariants.d.ts +26 -0
- package/lib/engine/mem/db/tokenizer.d.ts +23 -0
- package/lib/engine/mem/db/vectors.d.ts +40 -8
- package/lib/engine/mem/entities/extract.d.ts +1 -1
- package/lib/engine/mem/eval/derive.d.ts +120 -0
- package/lib/engine/mem/eval/derived_runner.d.ts +88 -0
- package/lib/engine/mem/eval/metrics.d.ts +24 -0
- package/lib/engine/mem/eval/runner.d.ts +8 -1
- package/lib/engine/mem/index.d.ts +1 -1
- package/lib/engine/mem/modelBootstrap.d.ts +2 -2
- package/lib/engine/mem/runtime.d.ts +1 -1
- package/lib/engine/mem/store/common.d.ts +33 -11
- package/lib/engine/mem/store/entity_leg.d.ts +95 -0
- package/lib/engine/mem/store/floors.d.ts +63 -31
- package/lib/engine/mem/store/hybrid.d.ts +21 -26
- package/lib/engine/mem/store/knowledge.d.ts +18 -9
- package/lib/engine/mem/store/lexical.d.ts +44 -0
- package/lib/engine/mem/store/memory.d.ts +151 -24
- package/lib/engine/mem/store/secret_guard.d.ts +34 -0
- package/lib/engine/mem/store/time_window.d.ts +74 -0
- package/lib/engine/mem-contract/config.d.ts +9 -16
- package/lib/engine/mem-contract/family.d.ts +1 -1
- package/lib/engine/mem-contract/tools.d.ts +10 -0
- package/lib/engine/mem-contract/types.d.ts +164 -9
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +37 -7
- package/lib/engine/mem-retrieval/fusion.d.ts +14 -1
- package/lib/engine/mem-retrieval/index.d.ts +1 -3
- package/lib/engine/mem-retrieval/interfaces.d.ts +38 -14
- package/lib/engine/mem-retrieval/registry.d.ts +2 -4
- package/lib/engine/mem-retrieval/representation.d.ts +50 -0
- package/lib/engine/mem-retrieval/stats.d.ts +1 -3
- package/lib/index.js +2003 -565
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +7 -0
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/remote.d.ts +12 -6
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +44 -0
- package/lib/types/vectorMigration.d.ts.map +1 -0
- package/package.json +5 -5
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
- package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type TrustDiagnostic, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
2
|
-
import type {
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixReport } from '../../mem-contract/index.js';
|
|
2
|
+
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
3
|
import type { Db } from '../db/conn.js';
|
|
4
4
|
import { type LexicalProbe } from './lexical.js';
|
|
5
5
|
import { type FloorLeg } from './floors.js';
|
|
@@ -21,6 +21,16 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
|
|
|
21
21
|
* until `deferred === 0` instead.
|
|
22
22
|
*/
|
|
23
23
|
export declare const ENTITY_SWEEP_BATCH = 2000;
|
|
24
|
+
/**
|
|
25
|
+
* Rows one background vector-space migration slice may drop + re-encode.
|
|
26
|
+
*
|
|
27
|
+
* Small on purpose: the point of batching is that a query arriving mid-migration waits for at most
|
|
28
|
+
* one ONNX forward pass batch, not for a whole corpus. At ~30 ms per bge-base-zh encode a batch of 16
|
|
29
|
+
* is well under a second of contested model time, and the event loop is yielded between batches
|
|
30
|
+
* (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
|
|
31
|
+
* whatever the caller needs.
|
|
32
|
+
*/
|
|
33
|
+
export declare const DEFAULT_VECTOR_MIGRATION_BATCH = 16;
|
|
24
34
|
/** What one derived-state sweep pass did (see `MemoryStore.reindexEntities`). */
|
|
25
35
|
export interface EntitySweepReport {
|
|
26
36
|
/**
|
|
@@ -78,6 +88,13 @@ export interface ContradictionResolution {
|
|
|
78
88
|
export interface SearchInput {
|
|
79
89
|
query: string;
|
|
80
90
|
category?: string;
|
|
91
|
+
/**
|
|
92
|
+
* P-08: restrict every leg to facts whose `fact_sources.ref` equals this value. Applied INSIDE
|
|
93
|
+
* each leg's SQL (an `EXISTS` predicate, before its `LIMIT cap`), never as a post-fusion filter:
|
|
94
|
+
* pruning after fusion would return fewer than `limit` and would make the relaxed-floor retry
|
|
95
|
+
* mistake a source-empty result for "the floors were too strict".
|
|
96
|
+
*/
|
|
97
|
+
source?: string;
|
|
81
98
|
limit?: number;
|
|
82
99
|
/** Also fuse the HRR entity-probe path (used by `recall.probe`). */
|
|
83
100
|
includeHrr?: boolean;
|
|
@@ -124,17 +141,34 @@ export interface SearchInput {
|
|
|
124
141
|
* `track`, which is about the dormancy clock — a caller may want one without the other.
|
|
125
142
|
*/
|
|
126
143
|
recordStats?: boolean;
|
|
144
|
+
/** P-01: attach per-leg evidence (`scores` / `final`) to the returned hits. Default false. */
|
|
145
|
+
includeScores?: boolean;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* P-08 / P-13 write-side extras on `add` / `update`.
|
|
149
|
+
*
|
|
150
|
+
* All three are optional and "omitted" has one meaning everywhere: do not write (a new row's
|
|
151
|
+
* NULL/1 default, or the existing row's value on a duplicate) — the store never invents a source,
|
|
152
|
+
* an event time or an end date.
|
|
153
|
+
*/
|
|
154
|
+
export interface FactWriteOptions {
|
|
155
|
+
/** P-08: `fact_sources` provenance. `kind:` prefix selects the kind, else `manual`. */
|
|
156
|
+
sourceRef?: string;
|
|
157
|
+
/** P-13: event time → `valid_from`. */
|
|
158
|
+
eventDate?: string;
|
|
159
|
+
/** P-13: known end → `valid_to`, WITHOUT archiving the row. */
|
|
160
|
+
validUntil?: string;
|
|
127
161
|
}
|
|
128
162
|
export declare class MemoryStore {
|
|
129
163
|
private readonly db;
|
|
130
164
|
private readonly config;
|
|
131
165
|
private readonly semantic;
|
|
132
166
|
private readonly vstore;
|
|
133
|
-
private readonly reranker;
|
|
134
167
|
private readonly contradictions;
|
|
135
168
|
private readonly facts;
|
|
136
169
|
private readonly entities;
|
|
137
170
|
private readonly triples;
|
|
171
|
+
private readonly sources;
|
|
138
172
|
/**
|
|
139
173
|
* The tokenizer `facts_fts` was ACTUALLY built with, read from the database at open.
|
|
140
174
|
*
|
|
@@ -150,7 +184,7 @@ export declare class MemoryStore {
|
|
|
150
184
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
151
185
|
*/
|
|
152
186
|
private conflictDrainCursor;
|
|
153
|
-
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
187
|
+
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
154
188
|
/** The store's database path, used ONLY to name the vector index snapshot beside it. */
|
|
155
189
|
dbPath?: string);
|
|
156
190
|
/**
|
|
@@ -161,8 +195,12 @@ export declare class MemoryStore {
|
|
|
161
195
|
trustTick(opts?: {
|
|
162
196
|
budget?: number | null;
|
|
163
197
|
}): TickResult;
|
|
164
|
-
/**
|
|
165
|
-
|
|
198
|
+
/**
|
|
199
|
+
* Identity of the vector space this store currently writes into — the REPRESENTATION fingerprint
|
|
200
|
+
* (`vectorSpaceId` + the backend's declared pooling/normalize/window/revision), not just the
|
|
201
|
+
* model name. Public so diagnostics and tests can name the space the store would record.
|
|
202
|
+
*/
|
|
203
|
+
vectorSpace(): string;
|
|
166
204
|
/**
|
|
167
205
|
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs.
|
|
168
206
|
* Without this the semantic path is silently empty after every process restart
|
|
@@ -173,7 +211,23 @@ export declare class MemoryStore {
|
|
|
173
211
|
* `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
174
212
|
*/
|
|
175
213
|
private reloadIndex;
|
|
176
|
-
|
|
214
|
+
/**
|
|
215
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
|
|
216
|
+
* persisted vectors whose width or recorded space id no longer matches the one this store writes
|
|
217
|
+
* into. Reads blob LENGTHS only — never decodes — so the `/mem` status surface and the startup
|
|
218
|
+
* warning can afford it at any corpus size.
|
|
219
|
+
*
|
|
220
|
+
* The counts are the same two `VectorsDiagnostic` already separates (`stale` / `space_stale`); this
|
|
221
|
+
* is the read model those surfaces use instead of the full diagnose pass.
|
|
222
|
+
*/
|
|
223
|
+
vectorSpaceHealth(): VectorSpaceHealth;
|
|
224
|
+
/**
|
|
225
|
+
* {@link vectorSpaceHealth} plus the internal `legacy` split (how many `space_stale` rows predate
|
|
226
|
+
* the representation fingerprint). Kept off the contract type so the public detection surface —
|
|
227
|
+
* and every existing assertion on it — is unchanged; only the warning's wording uses it.
|
|
228
|
+
*/
|
|
229
|
+
private vectorSpaceCounts;
|
|
230
|
+
add(content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
177
231
|
/**
|
|
178
232
|
* Write-path contradiction detection (DESIGN §11): check the fact that was just
|
|
179
233
|
* written against the corpus, then report its FULL open-conflict set.
|
|
@@ -192,7 +246,7 @@ export declare class MemoryStore {
|
|
|
192
246
|
private detectContradictions;
|
|
193
247
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
194
248
|
private maybeIndexSemantic;
|
|
195
|
-
update(fact_id: number, content: string, category?: string, ttlDays?: number): Promise<RememberResult>;
|
|
249
|
+
update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
196
250
|
remove(fact_id: number, reason?: string): boolean;
|
|
197
251
|
archive(fact_id: number, reason?: string): boolean;
|
|
198
252
|
/**
|
|
@@ -230,7 +284,17 @@ export declare class MemoryStore {
|
|
|
230
284
|
* @returns the page, the page size (`count`), the matching total, and whether
|
|
231
285
|
* more rows remain after this page.
|
|
232
286
|
*/
|
|
233
|
-
list(category?: string, status?: string, limit?: number, offset?: number
|
|
287
|
+
list(category?: string, status?: string, limit?: number, offset?: number,
|
|
288
|
+
/** P-08: only facts whose `fact_sources.ref` equals this value (reverse provenance lookup). */
|
|
289
|
+
source?: string): FactPage;
|
|
290
|
+
/**
|
|
291
|
+
* P-08 coverage over the ACTIVE corpus, for `admin stats`: the share of facts that can answer
|
|
292
|
+
* "why do I remember this". Reported from day one so the field cannot quietly become a second
|
|
293
|
+
* `mirror_source` (filled but meaningless).
|
|
294
|
+
*/
|
|
295
|
+
sourceCoverage(): SourceCoverage;
|
|
296
|
+
/** P-13 coverage over the ACTIVE corpus: the share of facts carrying an event time. */
|
|
297
|
+
validityCoverage(): ValidityCoverage;
|
|
234
298
|
countByStatus(): {
|
|
235
299
|
active: number;
|
|
236
300
|
archived: number;
|
|
@@ -317,7 +381,7 @@ export declare class MemoryStore {
|
|
|
317
381
|
*/
|
|
318
382
|
private classifyVectors;
|
|
319
383
|
/**
|
|
320
|
-
* Repair the semantic read model:
|
|
384
|
+
* Repair the semantic read model, in ONE bounded slice or in full:
|
|
321
385
|
* (a) reload persisted, dim-valid vectors missing from the live index
|
|
322
386
|
* (restart / index loss — needs no model),
|
|
323
387
|
* (b) drop persisted vectors whose dim no longer matches `semantic.dim` so a
|
|
@@ -325,12 +389,46 @@ export declare class MemoryStore {
|
|
|
325
389
|
* throws on dim mismatch),
|
|
326
390
|
* (c) encode active facts that have no vector yet (needs the semantic backend),
|
|
327
391
|
* (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
|
|
328
|
-
* adoption of a store predating the space id).
|
|
329
|
-
*
|
|
392
|
+
* adoption of a store predating the space id).
|
|
393
|
+
*
|
|
394
|
+
* `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
|
|
395
|
+
* dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
|
|
396
|
+
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
397
|
+
* `migrateVectorsBatch` passes the migration's batch size.
|
|
398
|
+
*/
|
|
399
|
+
private repairVectors;
|
|
400
|
+
/**
|
|
401
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
330
402
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
331
403
|
* the preview also reports `would_warm` (see the contract's `VectorsFixReport`).
|
|
332
404
|
*/
|
|
333
405
|
vectorsFix(dryRun?: boolean): Promise<VectorsFixReport>;
|
|
406
|
+
/**
|
|
407
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
408
|
+
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
409
|
+
* background migration ({@link migrateVectors}) and its tests drive.
|
|
410
|
+
*
|
|
411
|
+
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
412
|
+
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
413
|
+
*/
|
|
414
|
+
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
415
|
+
/**
|
|
416
|
+
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
417
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE never rejects: it is
|
|
418
|
+
* logged and reported as `remaining > 0`, which is what makes the next heartbeat a retry (the caller
|
|
419
|
+
* still owns the outer catch for a closed database, e.g. a shutdown racing the first read). Never
|
|
420
|
+
* blocks the event loop for longer than one batch.
|
|
421
|
+
*
|
|
422
|
+
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
423
|
+
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
424
|
+
*/
|
|
425
|
+
migrateVectors(opts?: {
|
|
426
|
+
batchSize?: number;
|
|
427
|
+
shouldStop?: () => boolean;
|
|
428
|
+
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
429
|
+
}): Promise<VectorMigrationOutcome>;
|
|
430
|
+
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
431
|
+
private migrationRemaining;
|
|
334
432
|
/**
|
|
335
433
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
336
434
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
@@ -420,20 +518,32 @@ export declare class MemoryStore {
|
|
|
420
518
|
* rescale the survivors. That covers the Jaccard and FTS legs. It does NOT cover the HRR leg, whose
|
|
421
519
|
* candidates arrive in the JACCARD leg's order (or by recency in its fallback), so its cap can
|
|
422
520
|
* remove its own highest scorer and `v / max` really does move. Accepted — the cap bounds per-query
|
|
423
|
-
* mission
|
|
424
|
-
*
|
|
521
|
+
* mission — but do not read these scores as comparable across different cap/over-fetch settings;
|
|
522
|
+
* see `docs/PROVENANCE_REVIEW.md` N5 for the missing differential.
|
|
425
523
|
*/
|
|
426
524
|
private searchLegs;
|
|
427
525
|
/**
|
|
428
|
-
*
|
|
429
|
-
*
|
|
430
|
-
*
|
|
431
|
-
*
|
|
526
|
+
* P-11: active facts whose EVENT time (`valid_from`) falls in the query's own time window.
|
|
527
|
+
*
|
|
528
|
+
* THREE DELIBERATE PROPERTIES, each of which the acceptance criteria name:
|
|
529
|
+
*
|
|
530
|
+
* 1. **Only `valid_from`.** A fact with no event time is not a candidate, and there is no
|
|
531
|
+
* fallback to `created_at`: mixing a write clock into the event-time axis would make the leg
|
|
532
|
+
* "work" on a corpus where it measured zero effect and would answer "上个月做了什么" with
|
|
533
|
+
* facts merely WRITTEN last month.
|
|
534
|
+
* 2. **A leg, not a filter.** The window only ever ADDS candidates at a constant score; it never
|
|
535
|
+
* removes one another leg (or the FTS/entity/semantic path) produced. A hard pre-filter would
|
|
536
|
+
* drop gold facts whose event time is outside the phrase's window but whose text is the answer.
|
|
537
|
+
* 3. **Constant score, capped.** Every in-window fact scores 1, so `fuse`'s per-leg max-scaling
|
|
538
|
+
* is cap-INVARIANT for this leg (any survivor IS the maximum) and the ranking WITHIN the
|
|
539
|
+
* window stays the other legs' business. The constant is why "which of last month's facts is
|
|
540
|
+
* the answer" is not decided by a date.
|
|
541
|
+
*
|
|
542
|
+
* Cost: one non-indexable scan (the date prefix is computed, so no index can serve it) of the
|
|
543
|
+
* active corpus, capped at `legCap`. It runs only when `retriever.time_window` is on — the whole
|
|
544
|
+
* point of the default-OFF switch.
|
|
432
545
|
*/
|
|
433
|
-
|
|
434
|
-
used: boolean;
|
|
435
|
-
fallback: boolean;
|
|
436
|
-
};
|
|
546
|
+
private timeWindowPath;
|
|
437
547
|
/**
|
|
438
548
|
* The one fact→RecallHit mapper (kind/source_ref/entities shape lives here ONLY;
|
|
439
549
|
* chain/reason/ask/search all used to carry byte-identical copies, three of them
|
|
@@ -499,8 +609,15 @@ export declare class MemoryStore {
|
|
|
499
609
|
lexicalProbe(text: string, stopAt?: number): LexicalProbe;
|
|
500
610
|
private ftsPath;
|
|
501
611
|
/**
|
|
502
|
-
*
|
|
503
|
-
* sharing at least one query entity, pre-ranked by how many they share and already capped.
|
|
612
|
+
* Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
|
|
613
|
+
* sharing at least one ANCHOR query entity, pre-ranked by how many they share and already capped.
|
|
614
|
+
*
|
|
615
|
+
* The score is anchored Jaccard with a saturating union —
|
|
616
|
+
* `|anchors ∩ F| / (queryWidth + min(|F \ anchors|, ENTITY_UNION_CAP))`; `store/entity_leg.ts`
|
|
617
|
+
* carries why the old `|Q ∩ F| / |Q ∪ F|` was structurally unreachable for short queries, and why
|
|
618
|
+
* the generic-word filter keeps the wider denominator from dragging the corpus in. The fact set is
|
|
619
|
+
* read per candidate and the score is recomputed here even though the DAO ordered by it, so one
|
|
620
|
+
* formula stays in charge (the DAO's `total` tie-break only orders equal scores).
|
|
504
621
|
*/
|
|
505
622
|
private jaccardPath;
|
|
506
623
|
/**
|
|
@@ -515,6 +632,16 @@ export declare class MemoryStore {
|
|
|
515
632
|
* defaults (`general` / `0`).
|
|
516
633
|
*/
|
|
517
634
|
private persistFact;
|
|
635
|
+
/**
|
|
636
|
+
* P-08: land the provenance rows for one write.
|
|
637
|
+
*
|
|
638
|
+
* Rules, in order: an explicit `source_ref` is recorded (classified by its `kind:` prefix); on a
|
|
639
|
+
* REVISION with none given, the replaced row's sources are copied forward (it is the same logical
|
|
640
|
+
* fact, and the new row would otherwise become untraceable); on a plain duplicate, nothing is
|
|
641
|
+
* added (its sources are already there). Nothing is ever invented — an omitted source on a first
|
|
642
|
+
* add leaves the fact with no `fact_sources` row at all, which is what the stats coverage reports.
|
|
643
|
+
*/
|
|
644
|
+
private recordSources;
|
|
518
645
|
/**
|
|
519
646
|
* Link the revision chain and archive the superseded fact — on both the fresh-insert
|
|
520
647
|
* and duplicate-content branches.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-side secret / PII shape guard (P-03) — a PURE function, no store, no IO.
|
|
3
|
+
*
|
|
4
|
+
* The memory store is a long-lived, backed-up, sometimes-remote copy of what the model chose to
|
|
5
|
+
* write down, so a plaintext credential that reaches it is hard to retract. This guard refuses a
|
|
6
|
+
* fact whose TEXT matches a HIGH-CONFIDENCE credential/PII shape. It is deliberately narrow: only
|
|
7
|
+
* shapes that a false positive is unlikely to produce, because a guard that refuses legitimate
|
|
8
|
+
* notes is worse than no guard — the model learns to stop recording.
|
|
9
|
+
*
|
|
10
|
+
* What it does NOT do, by design:
|
|
11
|
+
* - no rewriting (a silently redacted fact is a lie the caller cannot see);
|
|
12
|
+
* - no scanning of what is already stored (writes only, never a retroactive sweep);
|
|
13
|
+
* - no knowledge-base coverage (`kb_add` ingests user-provided documents and is out of scope);
|
|
14
|
+
* - no config/override path (the refusal carries its own remedy: record WHERE the secret lives).
|
|
15
|
+
*
|
|
16
|
+
* The refusal message is the interface: it names the shape and tells the caller what to write
|
|
17
|
+
* instead. The plugin turns a thrown store error into a tool error through its existing `wireErr`
|
|
18
|
+
* path, so no new field and no new error surface is introduced.
|
|
19
|
+
*/
|
|
20
|
+
export type SecretKind = 'provider-key' | 'private-key' | 'connection-string' | 'card-number' | 'national-id';
|
|
21
|
+
export interface SecretShape {
|
|
22
|
+
kind: SecretKind;
|
|
23
|
+
/** Human-readable name of the shape, safe to put in an error (never the matched value). */
|
|
24
|
+
label: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The guard. Returns the matched SHAPE (never the matched text) or `null`.
|
|
28
|
+
*
|
|
29
|
+
* Order is by specificity: a PEM block or a known prefix is a stronger signal than a bare number,
|
|
30
|
+
* so a fact that carries both is reported as the credential it obviously is.
|
|
31
|
+
*/
|
|
32
|
+
export declare function detectSecretShape(text: string): SecretShape | null;
|
|
33
|
+
/** The refusal the caller sees — names the shape and says what to write instead. */
|
|
34
|
+
export declare function secretRefusalMessage(shape: SecretShape): string;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* P-11 · the deterministic Chinese time-expression parser behind the (default-OFF) time-window leg.
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS. One pure function that turns a query's own text into ONE half-open window
|
|
5
|
+
* `[start, end)` in epoch milliseconds, plus the `rule` that produced it. It has no clock of its
|
|
6
|
+
* own — `now` is a parameter (default `new Date()`), so a caller can pin it ("pinned_now") and get
|
|
7
|
+
* bit-identical answers from any machine. No third-party date library, no locale data: only `Date`
|
|
8
|
+
* and regexes.
|
|
9
|
+
*
|
|
10
|
+
* WHY THE LEG NEEDS IT AT ALL. "上个月做了什么" carries its evidence in the TIME EXPRESSION, not in
|
|
11
|
+
* the words: the fact that answers it reads "完成了支付网关的重构" and shares no query term. No
|
|
12
|
+
* lexical, entity or semantic leg can see that; the window is the only usable signal.
|
|
13
|
+
*
|
|
14
|
+
* PARITY WITH THE BENCHMARK HARNESS. The rules, boundaries and `rule` names mirror
|
|
15
|
+
* `scripts/spikes/bench-r4-lib.mjs`'s `parseTimeWindow` verbatim (that is the copy the A2 derived
|
|
16
|
+
* "time" shape and the round-4/round-6 cards already use to pick their windows). The benchmark
|
|
17
|
+
* harness is NOT modified, so a production/harness disagreement would show up as an unreproducible
|
|
18
|
+
* measurement — keep the two in step. The ONE intended addition here is Chinese numerals in
|
|
19
|
+
* `最近N天` ("最近三天"), which the spike's `\d+` cannot express.
|
|
20
|
+
*
|
|
21
|
+
* WHAT IT DELIBERATELY DOES NOT PARSE. Quarter words (`上季度`, `第三季度`) and anything else outside
|
|
22
|
+
* the table below return `undefined` rather than a guess: a wrong window would silently boost the
|
|
23
|
+
* wrong facts, and "no window" only leaves the other legs exactly as they were.
|
|
24
|
+
*
|
|
25
|
+
* @module store/time_window
|
|
26
|
+
*/
|
|
27
|
+
/** Milliseconds in a day. Rolling windows use it the way the spike does (no DST in the zh corpus). */
|
|
28
|
+
export declare const DAY_MS = 86400000;
|
|
29
|
+
/** One parsed window. `end` is EXCLUSIVE. */
|
|
30
|
+
export interface TimeWindow {
|
|
31
|
+
/** Inclusive start, epoch ms. */
|
|
32
|
+
start: number;
|
|
33
|
+
/** Exclusive end, epoch ms. */
|
|
34
|
+
end: number;
|
|
35
|
+
/** The spike's rule label (also what the benchmark reports as the query's time shape). */
|
|
36
|
+
rule: string;
|
|
37
|
+
/** The literal span that matched, verbatim. */
|
|
38
|
+
matched: string;
|
|
39
|
+
}
|
|
40
|
+
/** Inclusive local-day bounds, the form the SQL predicate compares `valid_from`'s date prefix with. */
|
|
41
|
+
export interface DayBounds {
|
|
42
|
+
from: string;
|
|
43
|
+
to: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse an arabic or Chinese cardinal (`3`, `三`, `十`, `十三`, `二十一`) for the `最近N天` window.
|
|
47
|
+
*
|
|
48
|
+
* Capped at what a day count can plausibly be: values above 99 are treated as unparseable rather
|
|
49
|
+
* than silently producing a decade-long window. Returns `undefined` when the text is not a number
|
|
50
|
+
* this grammar knows, so the caller can fall through to the next rule.
|
|
51
|
+
*/
|
|
52
|
+
export declare function parseCount(text: string): number | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* The first time window in `text`, or `undefined` when the text carries no expression from the
|
|
55
|
+
* table. Ordered ABSOLUTE before RELATIVE (the spike's order): a text naming a concrete date must
|
|
56
|
+
* not be re-read as a relative one.
|
|
57
|
+
*/
|
|
58
|
+
export declare function parseTimeWindow(text: string, now?: Date): TimeWindow | undefined;
|
|
59
|
+
/** Local `YYYY-MM-DD` for an epoch-ms instant (the form `valid_from`'s prefix is compared against). */
|
|
60
|
+
export declare function localDayString(ms: number): string;
|
|
61
|
+
/**
|
|
62
|
+
* The INCLUSIVE local-day bounds of a window, for a `substr(valid_from,1,10) BETWEEN ? AND ?`
|
|
63
|
+
* predicate.
|
|
64
|
+
*
|
|
65
|
+
* `end` is exclusive in epoch terms, and an event time is stored at day granularity (`2026-09-15`)
|
|
66
|
+
* or finer; `end - 1ms` therefore names the last DAY the window can contain. That is what makes
|
|
67
|
+
* `最近三天` include today (`end` is "now", so `end - 1ms` is still today) while `上个月` stops at
|
|
68
|
+
* the month's last day (`end` is the 1st, so `end - 1ms` is the last day of the previous month)
|
|
69
|
+
* instead of leaking the 1st of the current one.
|
|
70
|
+
*
|
|
71
|
+
* An empty or inverted window yields `from > to`, and the SQL `BETWEEN` then matches nothing —
|
|
72
|
+
* the honest answer for "最近0天".
|
|
73
|
+
*/
|
|
74
|
+
export declare function windowDayBounds(window: TimeWindow): DayBounds;
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
/**
|
|
3
|
-
* Input window of the SHIPPED
|
|
4
|
-
* `Xenova/bge-
|
|
5
|
-
*
|
|
6
|
-
* the embedder changed; only its vector width did, 512 → 768.)
|
|
3
|
+
* Input window of the SHIPPED model: `Xenova/bge-base-zh-v1.5` reads 512 tokens. (The previous
|
|
4
|
+
* default embedder, `Xenova/bge-small-zh-v1.5`, shares the same 512-token window — this constant
|
|
5
|
+
* did NOT move when the embedder changed; only its vector width did, 512 → 768.)
|
|
7
6
|
*
|
|
8
|
-
* Exported because more than one default derives from it: the embedder's fallback window
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Exported because more than one default derives from it: the embedder's fallback window and
|
|
8
|
+
* `knowledge.chunk_size` below. transformers.js truncates past the window SILENTLY, so every
|
|
9
|
+
* derivation has to agree on the number.
|
|
11
10
|
*/
|
|
12
11
|
export declare const DEFAULT_MODEL_WINDOW_TOKENS = 512;
|
|
13
12
|
/** The merged runtime config shared by memory and knowledge stores. */
|
|
@@ -21,14 +20,7 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
21
20
|
mirror: z.ZodDefault<z.ZodString>;
|
|
22
21
|
cache_dir: z.ZodDefault<z.ZodString>;
|
|
23
22
|
auto_download: z.ZodDefault<z.ZodBoolean>;
|
|
24
|
-
|
|
25
|
-
rerank: z.ZodPrefault<z.ZodObject<{
|
|
26
|
-
backend: z.ZodDefault<z.ZodString>;
|
|
27
|
-
local_model: z.ZodDefault<z.ZodString>;
|
|
28
|
-
max_input_tokens: z.ZodDefault<z.ZodNumber>;
|
|
29
|
-
mirror: z.ZodDefault<z.ZodString>;
|
|
30
|
-
cache_dir: z.ZodDefault<z.ZodString>;
|
|
31
|
-
auto_download: z.ZodDefault<z.ZodBoolean>;
|
|
23
|
+
auto_migrate: z.ZodDefault<z.ZodBoolean>;
|
|
32
24
|
}, z.core.$strip>>;
|
|
33
25
|
vectorStore: z.ZodPrefault<z.ZodObject<{
|
|
34
26
|
backend: z.ZodDefault<z.ZodString>;
|
|
@@ -47,6 +39,7 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
47
39
|
min_fts_terms: z.ZodDefault<z.ZodNumber>;
|
|
48
40
|
min_jaccard: z.ZodDefault<z.ZodNumber>;
|
|
49
41
|
leg_cap: z.ZodDefault<z.ZodNumber>;
|
|
42
|
+
time_window: z.ZodDefault<z.ZodBoolean>;
|
|
50
43
|
}, z.core.$strip>>;
|
|
51
44
|
lifecycle: z.ZodPrefault<z.ZodObject<{
|
|
52
45
|
purge_after_archived_days: z.ZodDefault<z.ZodNumber>;
|
|
@@ -119,8 +112,8 @@ export declare const KnowledgeConfigSchema: z.ZodObject<{
|
|
|
119
112
|
chunk_size: z.ZodDefault<z.ZodNumber>;
|
|
120
113
|
chunk_overlap: z.ZodDefault<z.ZodNumber>;
|
|
121
114
|
source_priority: z.ZodDefault<z.ZodEnum<{
|
|
122
|
-
none: "none";
|
|
123
115
|
workspace: "workspace";
|
|
116
|
+
none: "none";
|
|
124
117
|
}>>;
|
|
125
118
|
ingest: z.ZodPrefault<z.ZodObject<{
|
|
126
119
|
local_roots: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* `@avantf/dsh-plugin-base` (the family's environment framework) installs everything it manages under a
|
|
6
6
|
* single root: `$AVANTF_HOME` when set, else `~/.avantf/env`. `tools` holds external binaries
|
|
7
|
-
* (pandoc), `models` the downloaded embedding
|
|
7
|
+
* (pandoc), `models` the downloaded embedding weights in the flat layout the transformers.js
|
|
8
8
|
* cache reads. The engine's built-in defaults point HERE, not at the pre-framework
|
|
9
9
|
* `~/.avantf/{tools,models}`: those legacy directories are no longer a fallback anywhere, so a
|
|
10
10
|
* machine that never creates them loses nothing, and no process resolves a path that only missions
|
|
@@ -54,12 +54,18 @@ export declare const RememberUnion: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
54
54
|
content: z.ZodString;
|
|
55
55
|
category: z.ZodOptional<z.ZodString>;
|
|
56
56
|
ttl_days: z.ZodOptional<z.ZodNumber>;
|
|
57
|
+
source_ref: z.ZodOptional<z.ZodString>;
|
|
58
|
+
event_date: z.ZodOptional<z.ZodString>;
|
|
59
|
+
valid_until: z.ZodOptional<z.ZodString>;
|
|
57
60
|
}, z.core.$strip>, z.ZodObject<{
|
|
58
61
|
action: z.ZodLiteral<"update">;
|
|
59
62
|
fact_id: z.ZodNumber;
|
|
60
63
|
content: z.ZodString;
|
|
61
64
|
category: z.ZodOptional<z.ZodString>;
|
|
62
65
|
ttl_days: z.ZodOptional<z.ZodNumber>;
|
|
66
|
+
source_ref: z.ZodOptional<z.ZodString>;
|
|
67
|
+
event_date: z.ZodOptional<z.ZodString>;
|
|
68
|
+
valid_until: z.ZodOptional<z.ZodString>;
|
|
63
69
|
}, z.core.$strip>, z.ZodObject<{
|
|
64
70
|
action: z.ZodLiteral<"remove">;
|
|
65
71
|
fact_id: z.ZodNumber;
|
|
@@ -77,12 +83,14 @@ export declare const RecallUnion: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
77
83
|
action: z.ZodLiteral<"search">;
|
|
78
84
|
query: z.ZodString;
|
|
79
85
|
category: z.ZodOptional<z.ZodString>;
|
|
86
|
+
source: z.ZodOptional<z.ZodString>;
|
|
80
87
|
limit: z.ZodOptional<z.ZodNumber>;
|
|
81
88
|
max_tokens: z.ZodOptional<z.ZodNumber>;
|
|
82
89
|
floors: z.ZodOptional<z.ZodEnum<{
|
|
83
90
|
strict: "strict";
|
|
84
91
|
loose: "loose";
|
|
85
92
|
}>>;
|
|
93
|
+
include_scores: z.ZodOptional<z.ZodBoolean>;
|
|
86
94
|
}, z.core.$strip>, z.ZodObject<{
|
|
87
95
|
action: z.ZodLiteral<"ask">;
|
|
88
96
|
subj: z.ZodOptional<z.ZodString>;
|
|
@@ -127,6 +135,7 @@ export declare const AdminUnion: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
127
135
|
active: "active";
|
|
128
136
|
archived: "archived";
|
|
129
137
|
}>>;
|
|
138
|
+
source: z.ZodOptional<z.ZodString>;
|
|
130
139
|
limit: z.ZodOptional<z.ZodNumber>;
|
|
131
140
|
offset: z.ZodOptional<z.ZodNumber>;
|
|
132
141
|
}, z.core.$strip>, z.ZodObject<{
|
|
@@ -220,6 +229,7 @@ export declare const QueryUnion: z.ZodObject<{
|
|
|
220
229
|
strict: "strict";
|
|
221
230
|
loose: "loose";
|
|
222
231
|
}>>;
|
|
232
|
+
include_scores: z.ZodOptional<z.ZodBoolean>;
|
|
223
233
|
}, z.core.$strip>;
|
|
224
234
|
export type QueryRequest = z.infer<typeof QueryUnion>;
|
|
225
235
|
/** Transport-agnostic tool spec: name / key / description / zod input. */
|