@avantf/dsh-mem 0.4.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/lib/client.js +39 -9
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/chunks.d.ts +80 -5
- package/lib/engine/mem/db/dao/entities.d.ts +35 -11
- package/lib/engine/mem/db/dao/fact_sources.d.ts +53 -0
- package/lib/engine/mem/db/dao/facts.d.ts +115 -12
- package/lib/engine/mem/db/dao/shared.d.ts +153 -0
- package/lib/engine/mem/db/invariants.d.ts +26 -0
- package/lib/engine/mem/db/tokenizer.d.ts +23 -0
- package/lib/engine/mem/db/vectors.d.ts +32 -6
- package/lib/engine/mem/entities/extract.d.ts +1 -1
- package/lib/engine/mem/eval/derive.d.ts +120 -0
- package/lib/engine/mem/eval/derived_runner.d.ts +88 -0
- package/lib/engine/mem/eval/index.d.ts +4 -0
- package/lib/engine/mem/eval/metrics.d.ts +24 -0
- package/lib/engine/mem/eval/runner.d.ts +8 -1
- package/lib/engine/mem/index.d.ts +2 -1
- package/lib/engine/mem/modelBootstrap.d.ts +2 -2
- package/lib/engine/mem/runtime.d.ts +3 -3
- package/lib/engine/mem/store/common.d.ts +18 -6
- package/lib/engine/mem/store/entity_leg.d.ts +95 -0
- package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
- package/lib/engine/mem/store/floors.d.ts +51 -31
- package/lib/engine/mem/store/hybrid.d.ts +71 -26
- package/lib/engine/mem/store/knowledge.d.ts +134 -21
- package/lib/engine/mem/store/legs.d.ts +171 -0
- package/lib/engine/mem/store/lexical.d.ts +52 -0
- package/lib/engine/mem/store/memory.d.ts +177 -92
- package/lib/engine/mem/store/secret_guard.d.ts +34 -0
- package/lib/engine/mem/store/time_window.d.ts +74 -0
- package/lib/engine/mem/store/vector_repair.d.ts +248 -0
- package/lib/engine/mem-contract/config.d.ts +8 -16
- package/lib/engine/mem-contract/family.d.ts +1 -1
- package/lib/engine/mem-contract/tools.d.ts +14 -0
- package/lib/engine/mem-contract/types.d.ts +151 -15
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +37 -7
- package/lib/engine/mem-retrieval/fusion.d.ts +14 -1
- package/lib/engine/mem-retrieval/index.d.ts +1 -3
- package/lib/engine/mem-retrieval/interfaces.d.ts +38 -14
- package/lib/engine/mem-retrieval/registry.d.ts +2 -4
- package/lib/engine/mem-retrieval/representation.d.ts +50 -0
- package/lib/engine/mem-retrieval/stats.d.ts +1 -3
- package/lib/index.js +3008 -1008
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +7 -0
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/remote.d.ts +12 -6
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +22 -18
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type TrustDiagnostic, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type
|
|
2
|
-
import type {
|
|
1
|
+
import { type Config, type ContradictionRecord, type FactDetail, type FactPage, type FloorProfile, type RecallResult, type RememberResult, type SourceCoverage, type TrustDiagnostic, type ValidityCoverage, type VectorMigrationOutcome, type VectorMigrationProgress, type VectorSpaceHealth, type VectorsDiagnostic, type VectorsFixStoreReport } from '../../mem-contract/index.js';
|
|
2
|
+
import type { SemanticBackend, VectorStore } from '../../mem-retrieval/index.js';
|
|
3
|
+
import { ENTITY_SWEEP_BATCH, type EntitySweepReport } from './entity_sweep.js';
|
|
4
|
+
import { DEFAULT_VECTOR_MIGRATION_BATCH, type VectorRepairTarget } from './vector_repair.js';
|
|
3
5
|
import type { Db } from '../db/conn.js';
|
|
4
6
|
import { type LexicalProbe } from './lexical.js';
|
|
5
7
|
import { type FloorLeg } from './floors.js';
|
|
@@ -15,35 +17,22 @@ export type { DetectedContradiction, RememberResult as AddResult } from '../../m
|
|
|
15
17
|
/**
|
|
16
18
|
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
17
19
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
20
|
+
* Re-exported from the shared sweep (`store/entity_sweep.ts`), which owns the loop and therefore the
|
|
21
|
+
* default: the public name has to keep working for the CLI (`cli/src/index.ts` passes it explicitly)
|
|
22
|
+
* and the plugin's heartbeat, but a second literal here would be a second answer to "how big is one
|
|
23
|
+
* pass".
|
|
22
24
|
*/
|
|
23
|
-
export
|
|
25
|
+
export { ENTITY_SWEEP_BATCH };
|
|
24
26
|
/**
|
|
25
27
|
* Rows one background vector-space migration slice may drop + re-encode.
|
|
26
28
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* (`yieldToEventLoop`). The value is a default, not a promise — `migrateVectorsBatch(batchSize)` takes
|
|
31
|
-
* whatever the caller needs.
|
|
29
|
+
* Re-exported from the shared repair flow (`store/vector_repair.ts`), which owns the loop and
|
|
30
|
+
* therefore the default: the public name has to keep working for callers that pass it explicitly,
|
|
31
|
+
* but a second literal here would be a second answer to "how big is one slice".
|
|
32
32
|
*/
|
|
33
|
-
export
|
|
34
|
-
/** What one derived-state sweep pass did (
|
|
35
|
-
export
|
|
36
|
-
/**
|
|
37
|
-
* Facts this pass RE-EXTRACTED, i.e. its batch size when it filled it. Counts rows VISITED and
|
|
38
|
-
* re-stamped, not rows whose extraction output differs — the stamp is what makes the corpus
|
|
39
|
-
* current, so a row re-extracted to the same names still counts (and is not rebuilt again).
|
|
40
|
-
*/
|
|
41
|
-
rebuilt: number;
|
|
42
|
-
/** Facts still written by older rules after this pass. */
|
|
43
|
-
deferred: number;
|
|
44
|
-
/** Another pass was already running, so this one did nothing (`deferred` is still current). */
|
|
45
|
-
skipped: boolean;
|
|
46
|
-
}
|
|
33
|
+
export { DEFAULT_VECTOR_MIGRATION_BATCH };
|
|
34
|
+
/** What one derived-state sweep pass did — re-exported from the shared loop (`store/entity_sweep.ts`). */
|
|
35
|
+
export type { EntitySweepReport };
|
|
47
36
|
/** What one bounded conflict-drain pass did (see `MemoryStore.drainConflicts`). */
|
|
48
37
|
export interface ConflictDrainReport {
|
|
49
38
|
/** Pairs newly logged by this pass. */
|
|
@@ -88,6 +77,13 @@ export interface ContradictionResolution {
|
|
|
88
77
|
export interface SearchInput {
|
|
89
78
|
query: string;
|
|
90
79
|
category?: string;
|
|
80
|
+
/**
|
|
81
|
+
* P-08: restrict every leg to facts whose `fact_sources.ref` equals this value. Applied INSIDE
|
|
82
|
+
* each leg's SQL (an `EXISTS` predicate, before its `LIMIT cap`), never as a post-fusion filter:
|
|
83
|
+
* pruning after fusion would return fewer than `limit` and would make the relaxed-floor retry
|
|
84
|
+
* mistake a source-empty result for "the floors were too strict".
|
|
85
|
+
*/
|
|
86
|
+
source?: string;
|
|
91
87
|
limit?: number;
|
|
92
88
|
/** Also fuse the HRR entity-probe path (used by `recall.probe`). */
|
|
93
89
|
includeHrr?: boolean;
|
|
@@ -134,17 +130,34 @@ export interface SearchInput {
|
|
|
134
130
|
* `track`, which is about the dormancy clock — a caller may want one without the other.
|
|
135
131
|
*/
|
|
136
132
|
recordStats?: boolean;
|
|
133
|
+
/** P-01: attach per-leg evidence (`scores` / `final`) to the returned hits. Default false. */
|
|
134
|
+
includeScores?: boolean;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* P-08 / P-13 write-side extras on `add` / `update`.
|
|
138
|
+
*
|
|
139
|
+
* All three are optional and "omitted" has one meaning everywhere: do not write (a new row's
|
|
140
|
+
* NULL/1 default, or the existing row's value on a duplicate) — the store never invents a source,
|
|
141
|
+
* an event time or an end date.
|
|
142
|
+
*/
|
|
143
|
+
export interface FactWriteOptions {
|
|
144
|
+
/** P-08: `fact_sources` provenance. `kind:` prefix selects the kind, else `manual`. */
|
|
145
|
+
sourceRef?: string;
|
|
146
|
+
/** P-13: event time → `valid_from`. */
|
|
147
|
+
eventDate?: string;
|
|
148
|
+
/** P-13: known end → `valid_to`, WITHOUT archiving the row. */
|
|
149
|
+
validUntil?: string;
|
|
137
150
|
}
|
|
138
151
|
export declare class MemoryStore {
|
|
139
152
|
private readonly db;
|
|
140
153
|
private readonly config;
|
|
141
154
|
private readonly semantic;
|
|
142
155
|
private readonly vstore;
|
|
143
|
-
private readonly reranker;
|
|
144
156
|
private readonly contradictions;
|
|
145
157
|
private readonly facts;
|
|
146
158
|
private readonly entities;
|
|
147
159
|
private readonly triples;
|
|
160
|
+
private readonly sources;
|
|
148
161
|
/**
|
|
149
162
|
* The tokenizer `facts_fts` was ACTUALLY built with, read from the database at open.
|
|
150
163
|
*
|
|
@@ -153,14 +166,20 @@ export declare class MemoryStore {
|
|
|
153
166
|
* and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
|
|
154
167
|
*/
|
|
155
168
|
private readonly ftsTokenizer;
|
|
156
|
-
/**
|
|
157
|
-
|
|
169
|
+
/**
|
|
170
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
171
|
+
*
|
|
172
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
|
|
173
|
+
* the rule where the loop is (the periodic callers can overlap), without a mutable field that two
|
|
174
|
+
* stores would each have to remember to hold.
|
|
175
|
+
*/
|
|
176
|
+
private readonly entitySweepGuard;
|
|
158
177
|
/**
|
|
159
178
|
* Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
|
|
160
179
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
161
180
|
*/
|
|
162
181
|
private conflictDrainCursor;
|
|
163
|
-
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
182
|
+
constructor(db: Db, config: Config, semantic: SemanticBackend, vstore: VectorStore,
|
|
164
183
|
/** The store's database path, used ONLY to name the vector index snapshot beside it. */
|
|
165
184
|
dbPath?: string);
|
|
166
185
|
/**
|
|
@@ -171,16 +190,26 @@ export declare class MemoryStore {
|
|
|
171
190
|
trustTick(opts?: {
|
|
172
191
|
budget?: number | null;
|
|
173
192
|
}): TickResult;
|
|
174
|
-
/** Identity of the vector space this store currently writes into (`vectorSpaceId`). */
|
|
175
|
-
private vectorSpace;
|
|
176
193
|
/**
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
194
|
+
* Identity of the vector space this store currently writes into — the REPRESENTATION fingerprint
|
|
195
|
+
* (`vectorSpaceId` + the backend's declared pooling/normalize/window/revision), not just the
|
|
196
|
+
* model name. Public so diagnostics and tests can name the space the store would record.
|
|
197
|
+
*/
|
|
198
|
+
vectorSpace(): string;
|
|
199
|
+
/**
|
|
200
|
+
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
|
|
201
|
+
* belongs to another vector space.
|
|
202
|
+
*
|
|
203
|
+
* Without this the semantic path is silently empty after every process restart (the DB is the
|
|
204
|
+
* authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
|
|
205
|
+
* archived facts are filtered out of every retrieval path anyway, so loading them would only grow
|
|
206
|
+
* the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
180
207
|
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
*
|
|
208
|
+
* The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
|
|
209
|
+
* step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
|
|
210
|
+
* another space is still RANKED (the reload cannot tell the difference), so a silent model swap
|
|
211
|
+
* would mix two spaces in one index — acting on it is the manual fix's job for the operator path
|
|
212
|
+
* and the bounded background migration's for the automatic one.
|
|
184
213
|
*/
|
|
185
214
|
private reloadIndex;
|
|
186
215
|
/**
|
|
@@ -193,7 +222,13 @@ export declare class MemoryStore {
|
|
|
193
222
|
* is the read model those surfaces use instead of the full diagnose pass.
|
|
194
223
|
*/
|
|
195
224
|
vectorSpaceHealth(): VectorSpaceHealth;
|
|
196
|
-
|
|
225
|
+
/**
|
|
226
|
+
* {@link vectorSpaceHealth} plus the internal `legacy` split (how many `space_stale` rows predate
|
|
227
|
+
* the representation fingerprint). Kept off the contract type so the public detection surface —
|
|
228
|
+
* and every existing assertion on it — is unchanged; only the warning's wording uses it.
|
|
229
|
+
*/
|
|
230
|
+
private vectorSpaceCounts;
|
|
231
|
+
add(content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
197
232
|
/**
|
|
198
233
|
* Write-path contradiction detection (DESIGN §11): check the fact that was just
|
|
199
234
|
* written against the corpus, then report its FULL open-conflict set.
|
|
@@ -210,9 +245,17 @@ export declare class MemoryStore {
|
|
|
210
245
|
* stamped the marker anyway — a permanent, silent exemption).
|
|
211
246
|
*/
|
|
212
247
|
private detectContradictions;
|
|
248
|
+
/**
|
|
249
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
250
|
+
* call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
|
|
251
|
+
* backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
|
|
252
|
+
* (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
|
|
253
|
+
*/
|
|
254
|
+
private vectorWriteTarget;
|
|
255
|
+
private warnIndexFailure;
|
|
213
256
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
214
257
|
private maybeIndexSemantic;
|
|
215
|
-
update(fact_id: number, content: string, category?: string, ttlDays?: number): Promise<RememberResult>;
|
|
258
|
+
update(fact_id: number, content: string, category?: string, ttlDays?: number, opts?: FactWriteOptions): Promise<RememberResult>;
|
|
216
259
|
remove(fact_id: number, reason?: string): boolean;
|
|
217
260
|
archive(fact_id: number, reason?: string): boolean;
|
|
218
261
|
/**
|
|
@@ -250,7 +293,17 @@ export declare class MemoryStore {
|
|
|
250
293
|
* @returns the page, the page size (`count`), the matching total, and whether
|
|
251
294
|
* more rows remain after this page.
|
|
252
295
|
*/
|
|
253
|
-
list(category?: string, status?: string, limit?: number, offset?: number
|
|
296
|
+
list(category?: string, status?: string, limit?: number, offset?: number,
|
|
297
|
+
/** P-08: only facts whose `fact_sources.ref` equals this value (reverse provenance lookup). */
|
|
298
|
+
source?: string): FactPage;
|
|
299
|
+
/**
|
|
300
|
+
* P-08 coverage over the ACTIVE corpus, for `admin stats`: the share of facts that can answer
|
|
301
|
+
* "why do I remember this". Reported from day one so the field cannot quietly become a second
|
|
302
|
+
* `mirror_source` (filled but meaningless).
|
|
303
|
+
*/
|
|
304
|
+
sourceCoverage(): SourceCoverage;
|
|
305
|
+
/** P-13 coverage over the ACTIVE corpus: the share of facts carrying an event time. */
|
|
306
|
+
validityCoverage(): ValidityCoverage;
|
|
254
307
|
countByStatus(): {
|
|
255
308
|
active: number;
|
|
256
309
|
archived: number;
|
|
@@ -337,54 +390,60 @@ export declare class MemoryStore {
|
|
|
337
390
|
*/
|
|
338
391
|
private classifyVectors;
|
|
339
392
|
/**
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
*
|
|
348
|
-
*
|
|
393
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
394
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
395
|
+
* `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
|
|
396
|
+
* for both stores.
|
|
397
|
+
*/
|
|
398
|
+
vectorRepairTarget(): VectorRepairTarget;
|
|
399
|
+
/**
|
|
400
|
+
* {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
|
|
401
|
+
* (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
|
|
402
|
+
*/
|
|
403
|
+
private classifyRepairVectors;
|
|
404
|
+
/** Put usable persisted vectors missing from the live index back into it (no model needed). */
|
|
405
|
+
private reindexUsableVectors;
|
|
406
|
+
/**
|
|
407
|
+
* Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
|
|
408
|
+
* pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
|
|
409
|
+
* the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
|
|
410
|
+
* one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
|
|
349
411
|
*
|
|
350
|
-
* `
|
|
351
|
-
*
|
|
352
|
-
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
353
|
-
* `migrateVectorsBatch` passes the migration's batch size.
|
|
412
|
+
* `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
|
|
413
|
+
* the embedding leg was checked against is gone, so the check has to run again.
|
|
354
414
|
*/
|
|
355
|
-
private
|
|
415
|
+
private dropRepairVectors;
|
|
416
|
+
/**
|
|
417
|
+
* Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
|
|
418
|
+
*
|
|
419
|
+
* Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
|
|
420
|
+
* the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
|
|
421
|
+
* ran has not been checked.
|
|
422
|
+
*/
|
|
423
|
+
private reencodeMissingVectors;
|
|
356
424
|
/**
|
|
357
425
|
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
358
426
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
359
|
-
* the preview also reports `would_warm` (see the contract's `
|
|
427
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
360
428
|
*/
|
|
361
|
-
vectorsFix(dryRun?: boolean): Promise<
|
|
429
|
+
vectorsFix(dryRun?: boolean): Promise<VectorsFixStoreReport>;
|
|
362
430
|
/**
|
|
363
431
|
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
364
432
|
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
365
433
|
* background migration ({@link migrateVectors}) and its tests drive.
|
|
366
|
-
*
|
|
367
|
-
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
368
|
-
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
369
434
|
*/
|
|
370
435
|
migrateVectorsBatch(batchSize?: number): Promise<VectorMigrationProgress>;
|
|
371
436
|
/**
|
|
372
437
|
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
373
|
-
* `shouldStop()` turns true (plugin unmount / process shutdown).
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
* blocks the event loop for longer than one batch.
|
|
377
|
-
*
|
|
378
|
-
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
379
|
-
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
438
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
|
|
439
|
+
* switch, the no-progress rule and the store-named logging all live in the shared flow
|
|
440
|
+
* (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
|
|
380
441
|
*/
|
|
381
442
|
migrateVectors(opts?: {
|
|
382
443
|
batchSize?: number;
|
|
383
444
|
shouldStop?: () => boolean;
|
|
384
445
|
onProgress?: (progress: VectorMigrationOutcome) => void;
|
|
385
446
|
}): Promise<VectorMigrationOutcome>;
|
|
386
|
-
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
387
|
-
private migrationRemaining;
|
|
388
447
|
/**
|
|
389
448
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
390
449
|
* invariant (ACTIVE vectors only) are maintained in ONE place — the runtime no
|
|
@@ -422,14 +481,10 @@ export declare class MemoryStore {
|
|
|
422
481
|
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
423
482
|
* every selected row's `content` before rebuilding the first one.
|
|
424
483
|
*
|
|
425
|
-
* The in-flight guard
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
429
|
-
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
430
|
-
*
|
|
431
|
-
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
432
|
-
* caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
|
|
484
|
+
* The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
|
|
485
|
+
* live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
|
|
486
|
+
* entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
|
|
487
|
+
* selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
|
|
433
488
|
*/
|
|
434
489
|
reindexEntities(budget?: number): Promise<EntitySweepReport>;
|
|
435
490
|
/**
|
|
@@ -439,8 +494,6 @@ export declare class MemoryStore {
|
|
|
439
494
|
* `ContradictionsDao.resolveForFact`).
|
|
440
495
|
*/
|
|
441
496
|
private retireConflicts;
|
|
442
|
-
/** Drop vectors of archived/purged facts from the live index (batched). */
|
|
443
|
-
private evictVectors;
|
|
444
497
|
/**
|
|
445
498
|
* Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
|
|
446
499
|
* count. Archived (incl. superseded) facts are excluded — their entities are not
|
|
@@ -474,20 +527,32 @@ export declare class MemoryStore {
|
|
|
474
527
|
* rescale the survivors. That covers the Jaccard and FTS legs. It does NOT cover the HRR leg, whose
|
|
475
528
|
* candidates arrive in the JACCARD leg's order (or by recency in its fallback), so its cap can
|
|
476
529
|
* remove its own highest scorer and `v / max` really does move. Accepted — the cap bounds per-query
|
|
477
|
-
* mission
|
|
478
|
-
*
|
|
530
|
+
* mission — but do not read these scores as comparable across different cap/over-fetch settings;
|
|
531
|
+
* see `docs/PROVENANCE_REVIEW.md` N5 for the missing differential.
|
|
479
532
|
*/
|
|
480
533
|
private searchLegs;
|
|
481
534
|
/**
|
|
482
|
-
*
|
|
483
|
-
*
|
|
484
|
-
*
|
|
485
|
-
*
|
|
535
|
+
* P-11: active facts whose EVENT time (`valid_from`) falls in the query's own time window.
|
|
536
|
+
*
|
|
537
|
+
* THREE DELIBERATE PROPERTIES, each of which the acceptance criteria name:
|
|
538
|
+
*
|
|
539
|
+
* 1. **Only `valid_from`.** A fact with no event time is not a candidate, and there is no
|
|
540
|
+
* fallback to `created_at`: mixing a write clock into the event-time axis would make the leg
|
|
541
|
+
* "work" on a corpus where it measured zero effect and would answer "上个月做了什么" with
|
|
542
|
+
* facts merely WRITTEN last month.
|
|
543
|
+
* 2. **A leg, not a filter.** The window only ever ADDS candidates at a constant score; it never
|
|
544
|
+
* removes one another leg (or the FTS/entity/semantic path) produced. A hard pre-filter would
|
|
545
|
+
* drop gold facts whose event time is outside the phrase's window but whose text is the answer.
|
|
546
|
+
* 3. **Constant score, capped.** Every in-window fact scores 1, so `fuse`'s per-leg max-scaling
|
|
547
|
+
* is cap-INVARIANT for this leg (any survivor IS the maximum) and the ranking WITHIN the
|
|
548
|
+
* window stays the other legs' business. The constant is why "which of last month's facts is
|
|
549
|
+
* the answer" is not decided by a date.
|
|
550
|
+
*
|
|
551
|
+
* Cost: one non-indexable scan (the date prefix is computed, so no index can serve it) of the
|
|
552
|
+
* active corpus, capped at `legCap`. It runs only when `retriever.time_window` is on — the whole
|
|
553
|
+
* point of the default-OFF switch.
|
|
486
554
|
*/
|
|
487
|
-
|
|
488
|
-
used: boolean;
|
|
489
|
-
fallback: boolean;
|
|
490
|
-
};
|
|
555
|
+
private timeWindowPath;
|
|
491
556
|
/**
|
|
492
557
|
* The one fact→RecallHit mapper (kind/source_ref/entities shape lives here ONLY;
|
|
493
558
|
* chain/reason/ask/search all used to carry byte-identical copies, three of them
|
|
@@ -525,7 +590,10 @@ export declare class MemoryStore {
|
|
|
525
590
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
526
591
|
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
527
592
|
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
528
|
-
* instead of re-encoding (performance review §7.7 / P8).
|
|
593
|
+
* instead of re-encoding (performance review §7.7 / P8). The flow is shared
|
|
594
|
+
* (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
|
|
595
|
+
* notion of status/category/source, so candidates go through the DB and archived/purged facts,
|
|
596
|
+
* other categories and other sources never leak into the leg.
|
|
529
597
|
*/
|
|
530
598
|
private semanticPath;
|
|
531
599
|
/**
|
|
@@ -553,8 +621,15 @@ export declare class MemoryStore {
|
|
|
553
621
|
lexicalProbe(text: string, stopAt?: number): LexicalProbe;
|
|
554
622
|
private ftsPath;
|
|
555
623
|
/**
|
|
556
|
-
*
|
|
557
|
-
* sharing at least one query entity, pre-ranked by how many they share and already capped.
|
|
624
|
+
* Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
|
|
625
|
+
* sharing at least one ANCHOR query entity, pre-ranked by how many they share and already capped.
|
|
626
|
+
*
|
|
627
|
+
* The score is anchored Jaccard with a saturating union —
|
|
628
|
+
* `|anchors ∩ F| / (queryWidth + min(|F \ anchors|, ENTITY_UNION_CAP))`; `store/entity_leg.ts`
|
|
629
|
+
* carries why the old `|Q ∩ F| / |Q ∪ F|` was structurally unreachable for short queries, and why
|
|
630
|
+
* the generic-word filter keeps the wider denominator from dragging the corpus in. The fact set is
|
|
631
|
+
* read per candidate and the score is recomputed here even though the DAO ordered by it, so one
|
|
632
|
+
* formula stays in charge (the DAO's `total` tie-break only orders equal scores).
|
|
558
633
|
*/
|
|
559
634
|
private jaccardPath;
|
|
560
635
|
/**
|
|
@@ -569,6 +644,16 @@ export declare class MemoryStore {
|
|
|
569
644
|
* defaults (`general` / `0`).
|
|
570
645
|
*/
|
|
571
646
|
private persistFact;
|
|
647
|
+
/**
|
|
648
|
+
* P-08: land the provenance rows for one write.
|
|
649
|
+
*
|
|
650
|
+
* Rules, in order: an explicit `source_ref` is recorded (classified by its `kind:` prefix); on a
|
|
651
|
+
* REVISION with none given, the replaced row's sources are copied forward (it is the same logical
|
|
652
|
+
* fact, and the new row would otherwise become untraceable); on a plain duplicate, nothing is
|
|
653
|
+
* added (its sources are already there). Nothing is ever invented — an omitted source on a first
|
|
654
|
+
* add leaves the fact with no `fact_sources` row at all, which is what the stats coverage reports.
|
|
655
|
+
*/
|
|
656
|
+
private recordSources;
|
|
572
657
|
/**
|
|
573
658
|
* Link the revision chain and archive the superseded fact — on both the fresh-insert
|
|
574
659
|
* and duplicate-content branches.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-side secret / PII shape guard (P-03) — a PURE function, no store, no IO.
|
|
3
|
+
*
|
|
4
|
+
* The memory store is a long-lived, backed-up, sometimes-remote copy of what the model chose to
|
|
5
|
+
* write down, so a plaintext credential that reaches it is hard to retract. This guard refuses a
|
|
6
|
+
* fact whose TEXT matches a HIGH-CONFIDENCE credential/PII shape. It is deliberately narrow: only
|
|
7
|
+
* shapes that a false positive is unlikely to produce, because a guard that refuses legitimate
|
|
8
|
+
* notes is worse than no guard — the model learns to stop recording.
|
|
9
|
+
*
|
|
10
|
+
* What it does NOT do, by design:
|
|
11
|
+
* - no rewriting (a silently redacted fact is a lie the caller cannot see);
|
|
12
|
+
* - no scanning of what is already stored (writes only, never a retroactive sweep);
|
|
13
|
+
* - no knowledge-base coverage (`kb_add` ingests user-provided documents and is out of scope);
|
|
14
|
+
* - no config/override path (the refusal carries its own remedy: record WHERE the secret lives).
|
|
15
|
+
*
|
|
16
|
+
* The refusal message is the interface: it names the shape and tells the caller what to write
|
|
17
|
+
* instead. The plugin turns a thrown store error into a tool error through its existing `wireErr`
|
|
18
|
+
* path, so no new field and no new error surface is introduced.
|
|
19
|
+
*/
|
|
20
|
+
export type SecretKind = 'provider-key' | 'private-key' | 'connection-string' | 'card-number' | 'national-id';
|
|
21
|
+
export interface SecretShape {
|
|
22
|
+
kind: SecretKind;
|
|
23
|
+
/** Human-readable name of the shape, safe to put in an error (never the matched value). */
|
|
24
|
+
label: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The guard. Returns the matched SHAPE (never the matched text) or `null`.
|
|
28
|
+
*
|
|
29
|
+
* Order is by specificity: a PEM block or a known prefix is a stronger signal than a bare number,
|
|
30
|
+
* so a fact that carries both is reported as the credential it obviously is.
|
|
31
|
+
*/
|
|
32
|
+
export declare function detectSecretShape(text: string): SecretShape | null;
|
|
33
|
+
/** The refusal the caller sees — names the shape and says what to write instead. */
|
|
34
|
+
export declare function secretRefusalMessage(shape: SecretShape): string;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* P-11 · the deterministic Chinese time-expression parser behind the (default-OFF) time-window leg.
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS. One pure function that turns a query's own text into ONE half-open window
|
|
5
|
+
* `[start, end)` in epoch milliseconds, plus the `rule` that produced it. It has no clock of its
|
|
6
|
+
* own — `now` is a parameter (default `new Date()`), so a caller can pin it ("pinned_now") and get
|
|
7
|
+
* bit-identical answers from any machine. No third-party date library, no locale data: only `Date`
|
|
8
|
+
* and regexes.
|
|
9
|
+
*
|
|
10
|
+
* WHY THE LEG NEEDS IT AT ALL. "上个月做了什么" carries its evidence in the TIME EXPRESSION, not in
|
|
11
|
+
* the words: the fact that answers it reads "完成了支付网关的重构" and shares no query term. No
|
|
12
|
+
* lexical, entity or semantic leg can see that; the window is the only usable signal.
|
|
13
|
+
*
|
|
14
|
+
* PARITY WITH THE BENCHMARK HARNESS. The rules, boundaries and `rule` names mirror
|
|
15
|
+
* `scripts/spikes/bench-r4-lib.mjs`'s `parseTimeWindow` verbatim (that is the copy the A2 derived
|
|
16
|
+
* "time" shape and the round-4/round-6 cards already use to pick their windows). The benchmark
|
|
17
|
+
* harness is NOT modified, so a production/harness disagreement would show up as an unreproducible
|
|
18
|
+
* measurement — keep the two in step. The ONE intended addition here is Chinese numerals in
|
|
19
|
+
* `最近N天` ("最近三天"), which the spike's `\d+` cannot express.
|
|
20
|
+
*
|
|
21
|
+
* WHAT IT DELIBERATELY DOES NOT PARSE. Quarter words (`上季度`, `第三季度`) and anything else outside
|
|
22
|
+
* the table below return `undefined` rather than a guess: a wrong window would silently boost the
|
|
23
|
+
* wrong facts, and "no window" only leaves the other legs exactly as they were.
|
|
24
|
+
*
|
|
25
|
+
* @module store/time_window
|
|
26
|
+
*/
|
|
27
|
+
/** Milliseconds in a day. Rolling windows use it the way the spike does (no DST in the zh corpus). */
|
|
28
|
+
export declare const DAY_MS = 86400000;
|
|
29
|
+
/** One parsed window. `end` is EXCLUSIVE. */
|
|
30
|
+
export interface TimeWindow {
|
|
31
|
+
/** Inclusive start, epoch ms. */
|
|
32
|
+
start: number;
|
|
33
|
+
/** Exclusive end, epoch ms. */
|
|
34
|
+
end: number;
|
|
35
|
+
/** The spike's rule label (also what the benchmark reports as the query's time shape). */
|
|
36
|
+
rule: string;
|
|
37
|
+
/** The literal span that matched, verbatim. */
|
|
38
|
+
matched: string;
|
|
39
|
+
}
|
|
40
|
+
/** Inclusive local-day bounds, the form the SQL predicate compares `valid_from`'s date prefix with. */
|
|
41
|
+
export interface DayBounds {
|
|
42
|
+
from: string;
|
|
43
|
+
to: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse an arabic or Chinese cardinal (`3`, `三`, `十`, `十三`, `二十一`) for the `最近N天` window.
|
|
47
|
+
*
|
|
48
|
+
* Capped at what a day count can plausibly be: values above 99 are treated as unparseable rather
|
|
49
|
+
* than silently producing a decade-long window. Returns `undefined` when the text is not a number
|
|
50
|
+
* this grammar knows, so the caller can fall through to the next rule.
|
|
51
|
+
*/
|
|
52
|
+
export declare function parseCount(text: string): number | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* The first time window in `text`, or `undefined` when the text carries no expression from the
|
|
55
|
+
* table. Ordered ABSOLUTE before RELATIVE (the spike's order): a text naming a concrete date must
|
|
56
|
+
* not be re-read as a relative one.
|
|
57
|
+
*/
|
|
58
|
+
export declare function parseTimeWindow(text: string, now?: Date): TimeWindow | undefined;
|
|
59
|
+
/** Local `YYYY-MM-DD` for an epoch-ms instant (the form `valid_from`'s prefix is compared against). */
|
|
60
|
+
export declare function localDayString(ms: number): string;
|
|
61
|
+
/**
|
|
62
|
+
* The INCLUSIVE local-day bounds of a window, for a `substr(valid_from,1,10) BETWEEN ? AND ?`
|
|
63
|
+
* predicate.
|
|
64
|
+
*
|
|
65
|
+
* `end` is exclusive in epoch terms, and an event time is stored at day granularity (`2026-09-15`)
|
|
66
|
+
* or finer; `end - 1ms` therefore names the last DAY the window can contain. That is what makes
|
|
67
|
+
* `最近三天` include today (`end` is "now", so `end - 1ms` is still today) while `上个月` stops at
|
|
68
|
+
* the month's last day (`end` is the 1st, so `end - 1ms` is the last day of the previous month)
|
|
69
|
+
* instead of leaking the 1st of the current one.
|
|
70
|
+
*
|
|
71
|
+
* An empty or inverted window yields `from > to`, and the SQL `BETWEEN` then matches nothing —
|
|
72
|
+
* the honest answer for "最近0天".
|
|
73
|
+
*/
|
|
74
|
+
export declare function windowDayBounds(window: TimeWindow): DayBounds;
|