@avantf/dsh-mem 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -28
- package/cordis.patch.yml +17 -0
- package/lib/client.js +155 -14
- package/lib/dsh-build.json +5 -5
- package/lib/engine/mem/config/config_files.d.ts +26 -0
- package/lib/engine/mem/config/domains.d.ts +20 -0
- package/lib/engine/mem/config/loader.d.ts +35 -0
- package/lib/engine/mem/config/paths.d.ts +53 -0
- package/lib/engine/mem/db/chunk.d.ts +26 -0
- package/lib/engine/mem/db/conn.d.ts +13 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +150 -0
- package/lib/engine/mem/db/dao/contradictions.d.ts +98 -0
- package/lib/engine/mem/db/dao/documents.d.ts +51 -0
- package/lib/engine/mem/db/dao/entities.d.ts +67 -0
- package/lib/engine/mem/db/dao/facts.d.ts +472 -0
- package/lib/engine/mem/db/dao/stats.d.ts +14 -0
- package/lib/engine/mem/db/dao/triples.d.ts +80 -0
- package/lib/engine/mem/db/hash.d.ts +2 -0
- package/lib/engine/mem/db/knowledge.d.ts +17 -0
- package/lib/engine/mem/db/migrations.d.ts +71 -0
- package/lib/engine/mem/db/port.d.ts +57 -0
- package/lib/engine/mem/db/schema.d.ts +26 -0
- package/lib/engine/mem/db/sqlite.d.ts +89 -0
- package/lib/engine/mem/db/store.d.ts +35 -0
- package/lib/engine/mem/db/tokenizer.d.ts +61 -0
- package/lib/engine/mem/db/vectors.d.ts +50 -0
- package/lib/engine/mem/dispatch.d.ts +51 -0
- package/lib/engine/mem/entities/extract.d.ts +84 -0
- package/lib/engine/mem/eval/loader.d.ts +19 -0
- package/lib/engine/mem/eval/metrics.d.ts +25 -0
- package/lib/engine/mem/eval/runner.d.ts +10 -0
- package/lib/engine/mem/eval/write_metrics.d.ts +88 -0
- package/lib/engine/mem/git.d.ts +73 -0
- package/lib/engine/mem/hrr/algebra.d.ts +30 -0
- package/lib/engine/mem/hrr/atoms.d.ts +22 -0
- package/lib/engine/mem/hrr/encode.d.ts +32 -0
- package/lib/engine/mem/hrr/index.d.ts +3 -0
- package/lib/engine/mem/index.d.ts +27 -0
- package/lib/engine/mem/lifecycle/contradiction.d.ts +156 -0
- package/lib/engine/mem/lifecycle/index.d.ts +5 -0
- package/lib/engine/mem/lifecycle/maintenance.d.ts +48 -0
- package/lib/engine/mem/lifecycle/presence.d.ts +29 -0
- package/lib/engine/mem/lifecycle/tick.d.ts +43 -0
- package/lib/engine/mem/lifecycle/trust.d.ts +90 -0
- package/lib/engine/mem/modelBootstrap.d.ts +207 -0
- package/lib/engine/mem/router.d.ts +51 -0
- package/lib/engine/mem/runtime.d.ts +232 -0
- package/lib/engine/mem/store/common.d.ts +82 -0
- package/lib/engine/mem/store/doc_files.d.ts +125 -0
- package/lib/engine/mem/store/document_text.d.ts +47 -0
- package/lib/engine/mem/store/floors.d.ts +191 -0
- package/lib/engine/mem/store/hybrid.d.ts +232 -0
- package/lib/engine/mem/store/ingest_guard.d.ts +25 -0
- package/lib/engine/mem/store/knowledge.d.ts +483 -0
- package/lib/engine/mem/store/lexical.d.ts +67 -0
- package/lib/engine/mem/store/memory.d.ts +543 -0
- package/lib/engine/mem/store/source_picker.d.ts +43 -0
- package/lib/engine/mem-contract/config.d.ts +129 -0
- package/lib/engine/mem-contract/env.d.ts +31 -0
- package/lib/engine/mem-contract/family.d.ts +25 -0
- package/lib/engine/mem-contract/index.d.ts +8 -0
- package/lib/engine/mem-contract/log.d.ts +29 -0
- package/lib/engine/mem-contract/remote.d.ts +80 -0
- package/lib/engine/mem-contract/text.d.ts +92 -0
- package/lib/engine/mem-contract/tools.d.ts +284 -0
- package/lib/engine/mem-contract/types.d.ts +615 -0
- package/lib/engine/mem-retrieval/adapters/auto_vstore.d.ts +47 -0
- package/lib/engine/mem-retrieval/adapters/hnswlib.d.ts +174 -0
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +121 -0
- package/lib/engine/mem-retrieval/adapters/local_numpy.d.ts +28 -0
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/adapters/warm_gate.d.ts +46 -0
- package/lib/engine/mem-retrieval/budget.d.ts +30 -0
- package/lib/engine/mem-retrieval/fusion.d.ts +53 -0
- package/lib/engine/mem-retrieval/index.d.ts +14 -0
- package/lib/engine/mem-retrieval/interfaces.d.ts +82 -0
- package/lib/engine/mem-retrieval/log.d.ts +11 -0
- package/lib/engine/mem-retrieval/registry.d.ts +23 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/engine/mem-retrieval/stats.d.ts +64 -0
- package/lib/engine/mem-retrieval/text_budget.d.ts +52 -0
- package/lib/index.js +1773 -473
- package/lib/interface-version.json +2 -2
- package/lib/types/client/domains.d.ts +5 -4
- package/lib/types/client/domains.d.ts.map +1 -1
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/client/ingest.d.ts +1 -1
- package/lib/types/client/wire.d.ts +36 -0
- package/lib/types/client/wire.d.ts.map +1 -0
- package/lib/types/envinit-bootstrap.d.ts +3 -2
- package/lib/types/envinit.d.ts +2 -19
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/hints.d.ts +1 -1
- package/lib/types/index.d.ts +8 -3
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/open.d.ts +61 -11
- package/lib/types/open.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +19 -1
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/provision.d.ts +1 -1
- package/lib/types/provision.d.ts.map +1 -1
- package/lib/types/reconcile.d.ts +64 -0
- package/lib/types/reconcile.d.ts.map +1 -0
- package/lib/types/remote.d.ts +53 -0
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/render.d.ts +35 -0
- package/lib/types/render.d.ts.map +1 -0
- package/lib/types/tool_schema.d.ts +3 -13
- package/lib/types/tool_schema.d.ts.map +1 -1
- package/package.json +27 -18
- package/lib/client.js.map +0 -1
- package/lib/types/client/domains.js +0 -25
- package/lib/types/client/domains.js.map +0 -1
- package/lib/types/client/index.js +0 -1371
- package/lib/types/client/index.js.map +0 -1
- package/lib/types/client/ingest.js +0 -30
- package/lib/types/client/ingest.js.map +0 -1
- package/lib/types/client/paging.js +0 -36
- package/lib/types/client/paging.js.map +0 -1
- package/lib/types/client/summarize.js +0 -79
- package/lib/types/client/summarize.js.map +0 -1
- package/lib/types/data_home.js +0 -26
- package/lib/types/data_home.js.map +0 -1
- package/lib/types/envinit-bootstrap.js +0 -333
- package/lib/types/envinit.js +0 -402
- package/lib/types/envinit.js.map +0 -1
- package/lib/types/hints.js +0 -53
- package/lib/types/hints.js.map +0 -1
- package/lib/types/index.js +0 -755
- package/lib/types/index.js.map +0 -1
- package/lib/types/inject.js +0 -14
- package/lib/types/inject.js.map +0 -1
- package/lib/types/interface_gate.js +0 -84
- package/lib/types/interface_gate.js.map +0 -1
- package/lib/types/open.js +0 -133
- package/lib/types/open.js.map +0 -1
- package/lib/types/prompt.js +0 -148
- package/lib/types/prompt.js.map +0 -1
- package/lib/types/provision.js +0 -177
- package/lib/types/provision.js.map +0 -1
- package/lib/types/remote.js +0 -247
- package/lib/types/remote.js.map +0 -1
- package/lib/types/tool_schema.js +0 -149
- package/lib/types/tool_schema.js.map +0 -1
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { AvantfLogger } from '../mem-contract/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* A reusable git wrapper for the stores' on-disk artifacts.
|
|
4
|
+
*
|
|
5
|
+
* It is deliberately root-agnostic: the knowledge corpus (`knowledge.docs.dir`, the default root)
|
|
6
|
+
* is one use, and "share a whole store" is another — point `root` at `~/.avantf/knowledge` or
|
|
7
|
+
* `~/.avantf` and pass an `ignore` list so the databases stay out of the history. The caller owns
|
|
8
|
+
* WHAT is versioned; this owns HOW, and nothing else in the codebase needs to know git exists.
|
|
9
|
+
*
|
|
10
|
+
* THREE PROPERTIES ARE LOAD-BEARING, and they are why this is not just `execFileSync` at each
|
|
11
|
+
* call site:
|
|
12
|
+
*
|
|
13
|
+
* - **It never throws.** Every caller writes here AFTER its real mission is durable (the row is
|
|
14
|
+
* committed, the document is indexed). A missing `git`, a locked index, or an unwritable repo
|
|
15
|
+
* must not turn a successful operation into a failed one — so failures are warned and swallowed,
|
|
16
|
+
* the same rule `IngestResult.file_error` follows for the managed file itself.
|
|
17
|
+
* - **It is always recognisable as automatic, and never invents an identity.** Every commit
|
|
18
|
+
* carries the `Automatic: avantf-mem` trailer. When the machine has an identity the commit is
|
|
19
|
+
* authored with it (the repo is the user's, and they may commit in it by hand — attributing
|
|
20
|
+
* their machine's mission to a bot by default would be the wrong kind of surprise); when it has
|
|
21
|
+
* NONE, the repo gets a LOCAL bot identity rather than failing or writing to the user's GLOBAL
|
|
22
|
+
* config, which is what lets the commit happen at all.
|
|
23
|
+
* - **It never commits an empty change.** `git status --porcelain` gates the commit, because
|
|
24
|
+
* `git add` + `git commit` with nothing staged fails, and a store write that changed nothing
|
|
25
|
+
* (a re-ingest of identical text) is common.
|
|
26
|
+
*/
|
|
27
|
+
type GitMode = 'auto' | 'off';
|
|
28
|
+
interface GitRepoOptions {
|
|
29
|
+
/** The directory this repo tracks. `init`/`commit` no-op when it does not exist. */
|
|
30
|
+
root: string;
|
|
31
|
+
/** `auto` (default) = create the repo on first commit; `off` = never invoke git at all. */
|
|
32
|
+
mode?: GitMode;
|
|
33
|
+
/**
|
|
34
|
+
* Patterns written to `.gitignore` when the repo is CREATED. An existing `.gitignore` is left
|
|
35
|
+
* alone. Only matters when `root` is broader than a text directory — e.g. pointing at a store
|
|
36
|
+
* directory brings the SQLite files with it, and those must never enter the history.
|
|
37
|
+
*/
|
|
38
|
+
ignore?: readonly string[];
|
|
39
|
+
/** Committer used when the machine has no identity. Defaults to the avantf bot. */
|
|
40
|
+
identity?: {
|
|
41
|
+
name: string;
|
|
42
|
+
email: string;
|
|
43
|
+
};
|
|
44
|
+
logger?: AvantfLogger;
|
|
45
|
+
timeoutMs?: number;
|
|
46
|
+
}
|
|
47
|
+
export declare class GitRepo {
|
|
48
|
+
readonly root: string;
|
|
49
|
+
private readonly mode;
|
|
50
|
+
private readonly ignore;
|
|
51
|
+
private readonly identity;
|
|
52
|
+
private readonly logger?;
|
|
53
|
+
private readonly timeoutMs;
|
|
54
|
+
constructor(options: GitRepoOptions);
|
|
55
|
+
/** Whether git is in play at all (a disabled repo does nothing, silently and by design). */
|
|
56
|
+
get enabled(): boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Create the repository if it is not one yet (writing `.gitignore` when it does). Idempotent and
|
|
59
|
+
* never throws; returns whether a repository exists afterwards.
|
|
60
|
+
*/
|
|
61
|
+
init(): boolean;
|
|
62
|
+
/** Stage and commit whatever changed under `root`. No-op when nothing did. Never throws. */
|
|
63
|
+
commit(message: string): void;
|
|
64
|
+
/** Commit subjects, newest first. `[]` when git is off, absent, or the repo has no commits. */
|
|
65
|
+
history(limit?: number): string[];
|
|
66
|
+
/** Current branch, or `null` when unavailable (never throws — this is diagnostics only). */
|
|
67
|
+
branch(): string | null;
|
|
68
|
+
private ensureIdentity;
|
|
69
|
+
private writeIgnore;
|
|
70
|
+
private warn;
|
|
71
|
+
private git;
|
|
72
|
+
}
|
|
73
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare class ShapeMismatchError extends Error {
|
|
2
|
+
}
|
|
3
|
+
/** bind(a, b) = (a + b) mod 2π — compose two phase vectors. */
|
|
4
|
+
export declare function bind(a: Float64Array, b: Float64Array): Float64Array;
|
|
5
|
+
/** bundle(*vectors) — circular-mean superposition `angle(Σ exp(i·v)) mod 2π`. */
|
|
6
|
+
export declare function bundle(...vectors: Float64Array[]): Float64Array;
|
|
7
|
+
/**
|
|
8
|
+
* Cosine-like phase similarity in [0, 1]: `(sum cos(a-b) + dim) / (2*dim)`.
|
|
9
|
+
*
|
|
10
|
+
* THE BASELINE IS 0.5, AND THAT SHAPES HOW THE LEG CONTRIBUTES. Two unrelated random bundles score
|
|
11
|
+
* 0.5 in expectation, and at dim=1024 the spread around it is tiny — measured over 200 unrelated
|
|
12
|
+
* pairs against five different probes: mean 0.499–0.503, sd 0.0105–0.0111, and the BEST of the 200
|
|
13
|
+
* reached only 0.525–0.536. A bundle scores 1.0 against itself, so the range that matters in
|
|
14
|
+
* practice is [~0.5, 1.0].
|
|
15
|
+
*
|
|
16
|
+
* Consequence at the fusion step: `scaleByMax` divides the leg by its own maximum, so the weakest
|
|
17
|
+
* unrelated candidate still keeps 0.88–0.90 of the strongest one's contribution. Within a set of
|
|
18
|
+
* candidates that do NOT share the probe's entities, this leg is therefore close to a CONSTANT boost
|
|
19
|
+
* rather than a ranking signal — which is benign in `MemoryStore.hrrPath`'s normal path (every
|
|
20
|
+
* candidate there shares an entity by construction, and sharing moves the score well above 0.5) and
|
|
21
|
+
* NOT benign in its fallback path, where the scored set is "the `cap` most recent facts" and most of
|
|
22
|
+
* them share nothing.
|
|
23
|
+
*
|
|
24
|
+
* Stated rather than fixed: subtracting the baseline (`max(0, (sim - 0.5) * 2)`) would make the leg
|
|
25
|
+
* discriminative, but it changes ranking, and the differential that would justify it is the same one
|
|
26
|
+
* `docs/PROVENANCE_REVIEW.md` N5 records as missing — the frozen eval set exercises `recall.search`,
|
|
27
|
+
* and this leg only runs for `recall.probe`, so no existing gate would move either way. Pinned by
|
|
28
|
+
* `test/hrr.spec.ts` so the change has to be made deliberately, with a probe-shaped eval beside it.
|
|
29
|
+
*/
|
|
30
|
+
export declare function phaseSimilarity(a: Float64Array, b: Float64Array): number;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** 2π (float64, cross-platform stable). */
|
|
2
|
+
export declare const TWO_PI: number;
|
|
3
|
+
export declare class AtomDimensionError extends Error {
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Deterministic phase vector derived from `name` (Plate-style HRR atom).
|
|
7
|
+
* SHA-256(name) → rolling SHA-256 blocks → uint64 LE → phase in [0, 2π).
|
|
8
|
+
* Same (name, dim) yields identical vectors across environments.
|
|
9
|
+
*
|
|
10
|
+
* The returned array is SHARED with the memo (treat it as immutable — callers only read it).
|
|
11
|
+
*/
|
|
12
|
+
export declare function atom(name: string, dim?: number): Float64Array;
|
|
13
|
+
/** Batch-derive atom vectors; returns `names.length` rows of dim. */
|
|
14
|
+
export declare function atomBatch(names: string[], dim?: number): Float64Array[];
|
|
15
|
+
/**
|
|
16
|
+
* Whether `vec` is a valid phase vector (Float64Array, even positive length, finite and in [0, 2π)).
|
|
17
|
+
*
|
|
18
|
+
* The finiteness check is not redundant with the range check: every comparison against NaN is false,
|
|
19
|
+
* so `NaN < 0 || NaN >= TWO_PI` passes and a corrupt blob decoded to NaN used to read as a valid
|
|
20
|
+
* atom — then `phaseSimilarity` returned NaN and poisoned the fused ranking (see `scaleByMax`).
|
|
21
|
+
*/
|
|
22
|
+
export declare function isValidAtom(vec: Float64Array): boolean;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encode a fact's entity set as an HRR phase vector (composition via bundle of
|
|
3
|
+
* entity atoms). Deterministic for a given entity set, enabling the HRR probe
|
|
4
|
+
* (phase similarity) used for entity-level recall of facts whose entities were
|
|
5
|
+
* not captured by the entity table.
|
|
6
|
+
*/
|
|
7
|
+
export declare function encodeHrrEntityVector(entityNames: string[], dim?: number): Float64Array;
|
|
8
|
+
/**
|
|
9
|
+
* Serialize a phase vector to bytes (for SQLite BLOB).
|
|
10
|
+
*
|
|
11
|
+
* Written as explicit little-endian, matching `float32ToBytes` — the layout is then
|
|
12
|
+
* a property of the format rather than of the host's byte order.
|
|
13
|
+
*/
|
|
14
|
+
export declare function hrrToBytes(vec: Float64Array): Buffer;
|
|
15
|
+
/**
|
|
16
|
+
* Deserialize bytes to a phase vector, or `null` when the bytes are not one.
|
|
17
|
+
*
|
|
18
|
+
* Reads through a `DataView` instead of `new Float64Array(buf.buffer, buf.byteOffset, …)`:
|
|
19
|
+
* that constructor requires an 8-byte-ALIGNED `byteOffset`, and nothing about a SQLite
|
|
20
|
+
* BLOB promises one (the driver happens to hand back offset-0 views today, and the adapter passes
|
|
21
|
+
* that offset through unchanged, but the
|
|
22
|
+
* callers here also pass plain `Uint8Array`s — a subarray or a pooled `Buffer` slice
|
|
23
|
+
* would throw a RangeError from deep inside retrieval). A trailing partial element is
|
|
24
|
+
* ignored, as before.
|
|
25
|
+
*
|
|
26
|
+
* A NON-FINITE phase is rejected here rather than scored: every range check in
|
|
27
|
+
* {@link isValidAtom} compares false against NaN, so a corrupt blob would otherwise sail through
|
|
28
|
+
* the caller's length check, make `phaseSimilarity` return NaN, and poison the whole fused ranking
|
|
29
|
+
* (one NaN in a sort comparator leaves the ORDER undefined, not just one score wrong). Callers
|
|
30
|
+
* already treat `null` as "skip this row".
|
|
31
|
+
*/
|
|
32
|
+
export declare function hrrFromBytes(buf: Buffer | Uint8Array): Float64Array | null;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The engine's public surface.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately narrow, and deliberately explicit about what is NOT here. This used to be `export *`
|
|
5
|
+
* over twenty modules, which published the database connection and schema, the migration list, the
|
|
6
|
+
* HRR algebra, the entity extractor, the eval runner and the git wrapper to every consumer — none of
|
|
7
|
+
* which any consumer used (the DSH plugin, the MCP server and the CLI import ten symbols between
|
|
8
|
+
* them). A wide surface is not a harmless convenience: anything exported is a promise, and the
|
|
9
|
+
* internals move (a DAO signature, a migration step, a tokenizer convention) on the assumption that
|
|
10
|
+
* only the stores call them.
|
|
11
|
+
*
|
|
12
|
+
* Everything else is still importable BY MODULE PATH, which is what this package's own specs and the
|
|
13
|
+
* `scripts/bench-*.mjs` benchmarks do — reaching into the engine is a choice a caller has to spell
|
|
14
|
+
* out, rather than something the front door hands them.
|
|
15
|
+
*/
|
|
16
|
+
export * from './config/paths.js';
|
|
17
|
+
export * from './config/loader.js';
|
|
18
|
+
export * from './modelBootstrap.js';
|
|
19
|
+
export * from './dispatch.js';
|
|
20
|
+
export * from './runtime.js';
|
|
21
|
+
export * from './store/memory.js';
|
|
22
|
+
export * from './store/knowledge.js';
|
|
23
|
+
export * from './store/hybrid.js';
|
|
24
|
+
export * from './store/source_picker.js';
|
|
25
|
+
export * from './store/lexical.js';
|
|
26
|
+
export * from './store/floors.js';
|
|
27
|
+
export { registerSemanticBackend, registerReranker, registerVectorStore, type Reranker, type SemanticBackend, type VectorStore, } from '../mem-retrieval/index.js';
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { MAX_REPORTED_CONFLICTS, type ContradictionRecord } from '../../mem-contract/index.js';
|
|
2
|
+
import type { Db } from '../db/conn.js';
|
|
3
|
+
import { type ConflictPair, type StoredConflict } from '../db/dao/contradictions.js';
|
|
4
|
+
export { polarityLookupSql, sameSubjPredLookupSql } from '../db/dao/triples.js';
|
|
5
|
+
/** Re-exported for `@avantf/mem` consumers; the value lives in the contract (the tool description states it). */
|
|
6
|
+
export { MAX_REPORTED_CONFLICTS };
|
|
7
|
+
/**
|
|
8
|
+
* Embedding-fallback contradiction score: `entity_overlap * semantic_sim`, only
|
|
9
|
+
* when all hold: min entity count, entity overlap floor, and cos sim in
|
|
10
|
+
* [sim_min, sim_dup_max]. `sim > sim_dup_max` is a near-duplicate → 0.
|
|
11
|
+
* Returns [0,1]; 0 means "not a contradiction candidate".
|
|
12
|
+
*/
|
|
13
|
+
export declare function detectContradictionEmbedding(entitiesA: Iterable<string>, entitiesB: Iterable<string>, vecA: Float32Array, vecB: Float32Array, opts?: {
|
|
14
|
+
minEntities?: number;
|
|
15
|
+
overlapMin?: number;
|
|
16
|
+
simMin?: number;
|
|
17
|
+
simDupMax?: number;
|
|
18
|
+
}): number;
|
|
19
|
+
interface ContradictionSig {
|
|
20
|
+
fact_a: number;
|
|
21
|
+
fact_b: number;
|
|
22
|
+
score: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* What one detection pass could and could not do.
|
|
26
|
+
*
|
|
27
|
+
* `logged` alone is not enough to decide whether a fact's check is FINISHED: it says "no pair was
|
|
28
|
+
* recorded", which is also the answer for a fact whose embedding leg never ran because its vector
|
|
29
|
+
* was not in the live index (another space, another process's write, dropped while the index was
|
|
30
|
+
* rebuilt). Stamping `conflict_checked` on that answer is a permanent, silent exemption — the very
|
|
31
|
+
* failure the durable queue exists to prevent. So the pass reports the two separately.
|
|
32
|
+
*/
|
|
33
|
+
export interface CheckResult {
|
|
34
|
+
/** Pairs this pass inserted (`maybeLog` skips the ones already open). */
|
|
35
|
+
logged: Sig[];
|
|
36
|
+
/**
|
|
37
|
+
* Facts whose EMBEDDING leg is COMPLETE — a vector was available for them, so nothing about
|
|
38
|
+
* their check depends on a later run. The store stamps exactly these; anything absent stays
|
|
39
|
+
* queued and `conflict_pending` keeps reporting it.
|
|
40
|
+
*/
|
|
41
|
+
complete: number[];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Contradiction detector: the structural signals (always, no vectors) then the embedding
|
|
45
|
+
* fallback when vectors are available. Pairs are normalized (min,max) and deduped.
|
|
46
|
+
*
|
|
47
|
+
* STATELESS about what is pending. It used to hold a Set of changed fact ids and retry them, which
|
|
48
|
+
* meant "awaiting the embedder" was lost on every restart, could grow without bound, and needed a
|
|
49
|
+
* bound plus a dropped-count to stay honest. `facts.conflict_checked` now records it durably, so
|
|
50
|
+
* this class only ever answers "what do these facts conflict with" (`checkOne` / `checkMany`) and
|
|
51
|
+
* the STORE owns the queue (see `MemoryStore.checkContradictions`).
|
|
52
|
+
*/
|
|
53
|
+
export declare class ContradictDetector {
|
|
54
|
+
private readonly db;
|
|
55
|
+
private readonly threshold;
|
|
56
|
+
private readonly fetchVectors;
|
|
57
|
+
private readonly fetchEntities;
|
|
58
|
+
/** Optional batched entity fetch — one query for the whole candidate set instead of N+1. */
|
|
59
|
+
private readonly fetchEntitiesBatch?;
|
|
60
|
+
private readonly triples;
|
|
61
|
+
private readonly repo;
|
|
62
|
+
private readonly entities;
|
|
63
|
+
private readonly facts;
|
|
64
|
+
constructor(db: Db, threshold?: number, fetchVectors?: (ids: number[]) => Map<number, Float32Array>, fetchEntities?: (id: number) => string[],
|
|
65
|
+
/** Optional batched entity fetch — one query for the whole candidate set instead of N+1. */
|
|
66
|
+
fetchEntitiesBatch?: ((ids: number[]) => Map<number, string[]>) | undefined);
|
|
67
|
+
/**
|
|
68
|
+
* Check ONE fact against the corpus — the WRITE-path entry.
|
|
69
|
+
*
|
|
70
|
+
* A fact is checked as it is written, so the open-contradiction list is truthful
|
|
71
|
+
* without anyone remembering to run a sweep (the UI used to read an always-empty
|
|
72
|
+
* log as "no contradictions").
|
|
73
|
+
*
|
|
74
|
+
* It leaves the pending set itself (see {@link detect}): the id stays queued only while
|
|
75
|
+
* the embedding leg could not run — i.e. no vector in the live index — so a later
|
|
76
|
+
* {@link checkMany} can retry exactly those. Re-checking a fact that was already scored
|
|
77
|
+
* with its vector would repeat an identical pass, and doing that for every write is what
|
|
78
|
+
* made the queue grow without bound.
|
|
79
|
+
*/
|
|
80
|
+
checkOne(factId: number): CheckResult;
|
|
81
|
+
/**
|
|
82
|
+
* A fact left the active corpus: retire the open conflicts that name it (see
|
|
83
|
+
* {@link ContradictionsDao.resolveForFact}). Returns how many rows were closed.
|
|
84
|
+
*/
|
|
85
|
+
resolveForFact(factId: number): number;
|
|
86
|
+
/**
|
|
87
|
+
* The lifecycle tick's bulk form: close the open conflicts of every id in one pass.
|
|
88
|
+
*
|
|
89
|
+
* The per-id loop cost O(archived × log) — measured 6.3 s for 999 ids at 99k open pairs, inside
|
|
90
|
+
* the tick's single IMMEDIATE transaction, i.e. holding the write lock for that whole time.
|
|
91
|
+
*/
|
|
92
|
+
resolveForFacts(factIds: readonly number[]): number;
|
|
93
|
+
/** Every OPEN conflict naming `factId` — what a writer is told about (see the repo). */
|
|
94
|
+
openConflictsFor(factId: number): ConflictPair[];
|
|
95
|
+
/** One stored conflict by id (adjudication needs the pair and its state). */
|
|
96
|
+
get(id: number): StoredConflict | undefined;
|
|
97
|
+
/** Record a verdict on a conflict WITHOUT touching the facts (see `resolveContradiction`). */
|
|
98
|
+
resolve(id: number, resolution: 'true_positive' | 'false_positive', loserFactId?: number): void;
|
|
99
|
+
/** List conflicts for the admin surface (`resolved` 0 = open). */
|
|
100
|
+
list(resolved?: number, category?: string, threshold?: number, limit?: number): ContradictionRecord[];
|
|
101
|
+
/** Drain the pending set: the incremental sweep (`admin contradict_check`). */
|
|
102
|
+
/**
|
|
103
|
+
* Check a LIST of facts — the store's durable drain supplies it (`contradict_check`).
|
|
104
|
+
*
|
|
105
|
+
* Only ACTIVE ids participate (design: 新变更 × 全量 active). A fact that is no longer active
|
|
106
|
+
* can never be checked again — its open rows were retired when it left the corpus — so it is
|
|
107
|
+
* filtered out here. It needs no stamp either: the queue predicate is `status = 'active'`.
|
|
108
|
+
*/
|
|
109
|
+
checkMany(factIds: readonly number[]): CheckResult;
|
|
110
|
+
/**
|
|
111
|
+
* Both signal families over the given (already filtered to ACTIVE) facts.
|
|
112
|
+
*
|
|
113
|
+
* `suppressed` decides how the already-logged set is read: a single-fact check only needs that
|
|
114
|
+
* fact's pairs, while the sweep reads the whole log once. Both are correct because `maybeLog` is
|
|
115
|
+
* only ever called with one side being a CHANGED id.
|
|
116
|
+
*/
|
|
117
|
+
private detect;
|
|
118
|
+
/** Structural signals — pure SQL, no vectors needed. */
|
|
119
|
+
private structuralPass;
|
|
120
|
+
/** ACTIVE subset of `ids` (batched: the pending set grows with the corpus). */
|
|
121
|
+
private filterActive;
|
|
122
|
+
/**
|
|
123
|
+
* Active facts that could POSSIBLY conflict with `factId` through the embedding leg.
|
|
124
|
+
*
|
|
125
|
+
* The scorer is `overlap × cosine` with `overlap = |A∩B| / |A∪B|`, gated by
|
|
126
|
+
* `overlap ≥ EMBED_OVERLAP_MIN`, `sim ≤ EMBED_SIM_DUP_MAX` and `|·| ≥ EMBED_MIN_ENTITIES`,
|
|
127
|
+
* and the pair is only logged when the score reaches `threshold`. So a qualifying pair
|
|
128
|
+
* must have
|
|
129
|
+
*
|
|
130
|
+
* p := max(EMBED_OVERLAP_MIN, threshold / EMBED_SIM_DUP_MAX) (score ≤ overlap × simDupMax)
|
|
131
|
+
* |A∩B| ≥ p·|A| because |A∪B| ≥ |A|
|
|
132
|
+
* |B| ≤ |A|/p because |A∩B| ≤ |A| and |A∩B| ≥ p·|B|
|
|
133
|
+
*
|
|
134
|
+
* Both are NECESSARY conditions, so filtering on them drops no pair that could ever be
|
|
135
|
+
* logged — it only shrinks the set the vector fetch and the pairwise cosine then pay for.
|
|
136
|
+
* On a hub entity (one entity shared by 30k facts, checked fact has two) the shared-entity
|
|
137
|
+
* floor alone removes the whole hub, because 1 shared entity cannot reach `p·|A| = 2`.
|
|
138
|
+
*/
|
|
139
|
+
private findEntitySharing;
|
|
140
|
+
/**
|
|
141
|
+
* Score every changed fact against its candidate set.
|
|
142
|
+
*
|
|
143
|
+
* Returns the ids whose OWN vector this pass had in hand — the embedding leg RAN for those, so
|
|
144
|
+
* they are the only ones whose check is complete. Note what this is NOT: `logged.length`. A
|
|
145
|
+
* fact can be fully checked and log nothing (no candidate reached the threshold), and a fact
|
|
146
|
+
* whose vector is missing was never checked at all yet logs nothing either. Only the vector
|
|
147
|
+
* table separates the two, which is why the pass returns it instead of the caller re-deriving
|
|
148
|
+
* it from the database (`semantic_vector IS NOT NULL` is a different question — see `CheckResult`).
|
|
149
|
+
*
|
|
150
|
+
* The entity floor counts as complete: `detectContradictionEmbedding` returns 0 below
|
|
151
|
+
* `EMBED_MIN_ENTITIES` whatever the vectors are, so no later pass could score differently.
|
|
152
|
+
*/
|
|
153
|
+
private embeddingPass;
|
|
154
|
+
private maybeLog;
|
|
155
|
+
}
|
|
156
|
+
export type Sig = ContradictionSig;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The manual `admin maintenance` entry — now a thin shell over the automatic tick
|
|
3
|
+
* (spec docs/TRUST_MODEL.md §6.1 / M3b).
|
|
4
|
+
*
|
|
5
|
+
* Surface stability (D10): the original report keys are all preserved
|
|
6
|
+
* (`decayed / archived_ttl / archived_age / purged / purged_ids / archived_ids`)
|
|
7
|
+
* and the new detail is purely additive. `archived_age` now means "non-TTL
|
|
8
|
+
* automatic archival" = `forgot + idle`.
|
|
9
|
+
*
|
|
10
|
+
* The `maintenance` action forces a FULL pass (no settle budget); the automatic
|
|
11
|
+
* startup/heartbeat passes use `tick_max_facts`.
|
|
12
|
+
*/
|
|
13
|
+
import type { Config } from '../../mem-contract/index.js';
|
|
14
|
+
import type { Db } from '../db/conn.js';
|
|
15
|
+
export interface MaintenanceResult {
|
|
16
|
+
/** Rows whose trust was materialized by the settle sweep (see tick.ts). */
|
|
17
|
+
decayed: number;
|
|
18
|
+
archived_ttl: number;
|
|
19
|
+
/** Non-TTL automatic archival = `forgot + idle` (kept name, widened meaning). */
|
|
20
|
+
archived_age: number;
|
|
21
|
+
purged: number;
|
|
22
|
+
/** Fact ids physically deleted by the purge — callers evict their vectors from the live index. */
|
|
23
|
+
purged_ids: number[];
|
|
24
|
+
/** Fact ids archived by this pass — the live index tracks ACTIVE facts only. */
|
|
25
|
+
archived_ids: number[];
|
|
26
|
+
settled: number;
|
|
27
|
+
clock: number;
|
|
28
|
+
archived_forgot: number;
|
|
29
|
+
archived_idle: number;
|
|
30
|
+
/** Rows still pending settle because the budget ran out (always 0 for a forced pass). */
|
|
31
|
+
skipped: number;
|
|
32
|
+
/** Rows still due for TTL/forgot/idle because the budget ran out (0 for a forced pass). */
|
|
33
|
+
archived_deferred: number;
|
|
34
|
+
/** Rows still due for purge when the budget ran out (0 for a forced pass). */
|
|
35
|
+
purged_deferred: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Run one lifecycle pass now.
|
|
39
|
+
*
|
|
40
|
+
* @param db - memory DB.
|
|
41
|
+
* @param cfg - resolved config.
|
|
42
|
+
* @param opts.clock - active-day clock to settle at (defaults to the stored clock).
|
|
43
|
+
* @param opts.budget - settle budget; `0` (default here) = full pass.
|
|
44
|
+
*/
|
|
45
|
+
export declare function runMaintenance(db: Db, cfg: Config, opts?: {
|
|
46
|
+
clock?: number;
|
|
47
|
+
budget?: number | null;
|
|
48
|
+
}): MaintenanceResult;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ACTIVE-DAY clock (spec docs/TRUST_MODEL.md §2.1).
|
|
3
|
+
*
|
|
4
|
+
* `clock` counts days the system was actually present (startup + heartbeat), with
|
|
5
|
+
* each presence contributing at most `presence.gap_cap_days` days — so a 90-day
|
|
6
|
+
* shutdown ages memories by 1 day, while an always-on system advances 1:1 with the
|
|
7
|
+
* calendar. State lives in the existing `avantf_stats` key/value table.
|
|
8
|
+
*
|
|
9
|
+
* The read-modify-write runs inside an IMMEDIATE transaction: two processes starting
|
|
10
|
+
* together serialize, and the second one sees the fresh `last_seen` (gap ≈ 0), so
|
|
11
|
+
* the clock is never advanced twice.
|
|
12
|
+
*/
|
|
13
|
+
import type { Db } from '../db/conn.js';
|
|
14
|
+
/** Current active-day count (0 when never initialized). */
|
|
15
|
+
export declare function readClock(db: Db): number;
|
|
16
|
+
/**
|
|
17
|
+
* Initialize/repair the clock. If the meta row was lost while facts exist, the
|
|
18
|
+
* clock must not trail their `settle_clock`, or those facts would never settle
|
|
19
|
+
* again (spec §2.1 self-heal).
|
|
20
|
+
*/
|
|
21
|
+
export declare function initClock(db: Db): number;
|
|
22
|
+
/** Advance the clock for one presence; returns the new clock and what was counted. */
|
|
23
|
+
export declare function advancePresence(db: Db, opts?: {
|
|
24
|
+
nowMs?: number;
|
|
25
|
+
gapCapDays?: number;
|
|
26
|
+
}): {
|
|
27
|
+
clock: number;
|
|
28
|
+
counted: number;
|
|
29
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One trust/lifecycle tick (spec docs/TRUST_MODEL.md §4) — five ordered statements:
|
|
3
|
+
*
|
|
4
|
+
* ① settle — materialize decay for every row whose ACTIVE DAY changed
|
|
5
|
+
* ② TTL — calendar expiry (explicit instruction; pinned is not exempt)
|
|
6
|
+
* ③ forgot — trust ≤ forget_threshold after settling (pinned exempt)
|
|
7
|
+
* ④ idle — calendar fallback: unused for `idle_calendar_days`
|
|
8
|
+
* ⑤ purge — physical delete of archived rows past the window
|
|
9
|
+
*
|
|
10
|
+
* EVERY step is budgeted by the same `tick_max_facts`, not just ①.
|
|
11
|
+
*
|
|
12
|
+
* It used to say "only ① is budgeted; ②③④⑤ only ever touch critical rows", and that reading is
|
|
13
|
+
* wrong in the direction that matters: ②③④⑤ each run inside the tick's ONE `IMMEDIATE` transaction,
|
|
14
|
+
* which holds the write lock for its whole duration, so an unbounded archive or purge pass over a
|
|
15
|
+
* large corpus blocks every other process (and the startup path, which is where the tick runs) for
|
|
16
|
+
* as long as it takes. Bounding each step is what keeps that lock short; `archived_deferred` and
|
|
17
|
+
* `purged_deferred` report what a bounded pass left for the next beat, so a backlog is visible
|
|
18
|
+
* instead of silent. The whole pass is one transaction, and the archived/purged ids are returned so
|
|
19
|
+
* the caller can evict them from the live vector index.
|
|
20
|
+
*/
|
|
21
|
+
import type { Config } from '../../mem-contract/index.js';
|
|
22
|
+
import type { Db } from '../db/conn.js';
|
|
23
|
+
export interface TickResult {
|
|
24
|
+
clock: number;
|
|
25
|
+
/** Rows materialized by ① (equals "trust actually changed" — see the note in runTrustTick). */
|
|
26
|
+
settled: number;
|
|
27
|
+
archived_ttl: number;
|
|
28
|
+
archived_forgot: number;
|
|
29
|
+
archived_idle: number;
|
|
30
|
+
purged: number;
|
|
31
|
+
purged_ids: number[];
|
|
32
|
+
archived_ids: number[];
|
|
33
|
+
/** Rows still awaiting ① when the budget ran out (0 for a full pass). */
|
|
34
|
+
skipped: number;
|
|
35
|
+
/** Rows still due for ②③④ when the budget ran out (0 when the pass finished them). */
|
|
36
|
+
archived_deferred: number;
|
|
37
|
+
/** Rows still due for ⑤ when the budget ran out (0 when the pass finished them). */
|
|
38
|
+
purged_deferred: number;
|
|
39
|
+
}
|
|
40
|
+
export declare function runTrustTick(db: Db, cfg: Config, opts: {
|
|
41
|
+
clock: number;
|
|
42
|
+
budget?: number | null;
|
|
43
|
+
}): TickResult;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Trust & forgetting math — PURE functions, no DB and no clock of their own
|
|
3
|
+
* (`clock` / `nowMs` are always injected, so every rule is unit-testable).
|
|
4
|
+
*
|
|
5
|
+
* Spec: docs/TRUST_MODEL.md §2. Key rules encoded here:
|
|
6
|
+
* - `eff = pinned ? 1 : clamp(trust − step × (clock − settle_clock), 0, 1)` (§2.2)
|
|
7
|
+
* - recall only ever RAISES trust (`max(eff, …)`, R2) and never pins (D11)
|
|
8
|
+
* - zero-gain recalls neither consume quota nor stamp `last_reinforced_at` (R10)
|
|
9
|
+
* - explicit feedback can pin (snap to 1.0) or forget immediately (§2.4)
|
|
10
|
+
* - `enabled=false` degrades `eff()` to the stored value (D12)
|
|
11
|
+
*/
|
|
12
|
+
import type { Config } from '../../mem-contract/index.js';
|
|
13
|
+
export declare const DAY_MS = 86400000;
|
|
14
|
+
/** Guard for "did the number actually move" comparisons. */
|
|
15
|
+
export declare const EPSILON = 1e-9;
|
|
16
|
+
type TrustConfig = Config['trust'];
|
|
17
|
+
/** The facts-row subset the trust math needs. */
|
|
18
|
+
export interface TrustRow {
|
|
19
|
+
trust_score: number;
|
|
20
|
+
settle_clock: number;
|
|
21
|
+
pinned: number | boolean;
|
|
22
|
+
bonus_count: number | null;
|
|
23
|
+
bonus_window_at: string | null;
|
|
24
|
+
status: string;
|
|
25
|
+
}
|
|
26
|
+
export declare function clamp01(v: number): number;
|
|
27
|
+
export declare function isPinned(row: Pick<TrustRow, 'pinned'>): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* `'YYYY-MM-DD HH:MM:SS'` (SQLite UTC) or ISO → epoch ms; null/garbage → null.
|
|
30
|
+
*
|
|
31
|
+
* A stamp with NO zone designator is read as UTC, not as local time. `Date.parse` treats a zone-less
|
|
32
|
+
* ISO string as LOCAL, and everything in this store is written in UTC (`CURRENT_TIMESTAMP`, and
|
|
33
|
+
* {@link formatUtcTs}), so handing one over as-is would shift every quota window and every decay
|
|
34
|
+
* measurement by the host's UTC offset — a fact reinforced at 23:00 UTC+8 would look reinforced 8
|
|
35
|
+
* hours earlier, and the 24h cap would open or close at the wrong moment. An explicit `Z` or
|
|
36
|
+
* `±HH:MM` is left alone.
|
|
37
|
+
*/
|
|
38
|
+
export declare function parseUtcTs(ts: string | null | undefined): number | null;
|
|
39
|
+
/** Epoch ms → `'YYYY-MM-DD HH:MM:SS'` (the SQLite `CURRENT_TIMESTAMP` shape, UTC). */
|
|
40
|
+
export declare function formatUtcTs(ms: number): string;
|
|
41
|
+
/** eff(f) — the trust a caller should act on right now (spec §2.2). */
|
|
42
|
+
export declare function effectiveTrust(row: TrustRow, clock: number, cfg: TrustConfig, enabled?: boolean): number;
|
|
43
|
+
/**
|
|
44
|
+
* What a caller should SEE for a row — the R11 display rule (spec §6):
|
|
45
|
+
* `active` → `eff()`; **non-active → the stored value verbatim** (the decay
|
|
46
|
+
* formula keeps running mathematically for archived rows, so the projection must
|
|
47
|
+
* not use it), and `enabled=false` → stored value (D12).
|
|
48
|
+
*/
|
|
49
|
+
export declare function displayTrust(row: TrustRow, clock: number, cfg: TrustConfig, enabled?: boolean): number;
|
|
50
|
+
/**
|
|
51
|
+
* Remaining active days: `eff / step`. `null` for pinned rows, non-active rows and
|
|
52
|
+
* when trust is disabled — the contract's `remaining_days` (spec §6 / R11).
|
|
53
|
+
*/
|
|
54
|
+
export declare function remainingDays(row: TrustRow, clock: number, cfg: TrustConfig, enabled?: boolean): number | null;
|
|
55
|
+
/** Everything a recall write needs to persist (spec §2.3). */
|
|
56
|
+
export interface RecallOutcome {
|
|
57
|
+
/** `trust_score` after settling the decay at `clock`. */
|
|
58
|
+
settled: number;
|
|
59
|
+
/** Final `trust_score` (equals `settled` when no bonus was granted). */
|
|
60
|
+
next: number;
|
|
61
|
+
granted: boolean;
|
|
62
|
+
bonusCount: number;
|
|
63
|
+
windowAt: string;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Apply one recall to a fact: settle, then grant a bonus inside the daily quota.
|
|
67
|
+
* Returns `next === settled` when the quota is exhausted, the ceiling is reached,
|
|
68
|
+
* the fact is pinned, or the gain would be zero (R2/R10/D11).
|
|
69
|
+
*/
|
|
70
|
+
export declare function grantRecallBonus(row: TrustRow, clock: number, nowMs: number, cfg: TrustConfig, enabled?: boolean): RecallOutcome;
|
|
71
|
+
/** Everything a feedback write needs to persist (spec §2.4). */
|
|
72
|
+
interface FeedbackOutcome {
|
|
73
|
+
settled: number;
|
|
74
|
+
next: number;
|
|
75
|
+
pin: boolean;
|
|
76
|
+
forget: boolean;
|
|
77
|
+
/** true when the row was left untouched (pinned / archived). */
|
|
78
|
+
untouched: boolean;
|
|
79
|
+
bonusCount: number;
|
|
80
|
+
windowAt: string;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Apply one explicit `helpful` (+1) / `unhelpful` (-1) step.
|
|
84
|
+
*
|
|
85
|
+
* Pinned and archived rows are untouched (R7): only `helpful_count` changes for
|
|
86
|
+
* them, so a pinned fact can never end up storing 0.95 while displaying 1.0.
|
|
87
|
+
* When `feedback_daily_cap > 0` feedback shares the per-fact 24h window counter.
|
|
88
|
+
*/
|
|
89
|
+
export declare function applyFeedbackDelta(row: TrustRow, clock: number, delta: number, nowMs: number, cfg: TrustConfig, enabled?: boolean): FeedbackOutcome;
|
|
90
|
+
export {};
|