@avantf/dsh-mem 0.3.1 → 0.4.0

package/lib/engine/mem/config/paths.d.ts

@@ -0,0 +1,53 @@
+import type { Config } from '../../mem-contract/index.js';
+import { expandHome } from '../../mem-contract/index.js';
+/** Expand a leading `~/` to the user home directory — single source in the contract. */
+export { expandHome };
+/**
+ * Resolve the data home (default `~/.avantf`).
+ *
+ * Precedence, per DESIGN §3 and the family layering (root `AGENTS.md` 「边界与路径」: built-in →
+ * profile → store → ENV → explicit): an EXPLICIT caller value (⑤) → `AVANTF_HOME` (④) → the
+ * configured `common.dataHome` (②) → `~/.avantf`. The environment is a deployment override and
+ * outranks the config file.
+ *
+ * `configured` cannot stand in for layer ⑤: the schema gives it a default, so it always holds a
+ * value and would make an explicit argument indistinguishable from "nothing was configured" — which
+ * is exactly how the env var used to win over a caller's own `dataHome`.
+ *
+ * The slots are NAMED for the same reason the base kit's copy is: the two implementations used to
+ * disagree about their POSITIONAL order (`(explicit, env, common)` here, `(common, explicit)`…
+ * whichever way a reader guessed), so "which slot does the profile's `dataHome` belong in?" was
+ * answerable only by reading the call site. The base kit implements the same rule with the same named
+ * shape, and `packages/plugin/test/family_pin.spec.ts` pins the two implementations together.
+ */
+export declare function resolveDataHome(input: {
+    /** Layer ②: the configured `common.dataHome`. */
+    readonly common: Pick<Config, 'dataHome'>;
+    /** Layer ⑤: an explicit caller value (the CLI's `--data-home`). */
+    readonly explicit?: string;
+}): string;
+/**
+ * Every config file lives in ONE directory, beside the prompt files.
+ *
+ * The layering is unchanged (① built-in → ② common → ③ store → ④ env → ⑤ explicit); only the
+ * locations moved, so all of a deployment's editable text — `configs/*.yaml` and `prompts/*.md` —
+ * is in one place instead of scattered through `memory/` and `knowledge/` next to the databases.
+ */
+export declare function configsDir(home: string): string;
+export declare function commonConfigPath(home: string): string;
+export declare function memoryDir(home: string): string;
+export declare function knowledgeDir(home: string): string;
+export declare function memoryConfigPath(home: string): string;
+export declare function knowledgeConfigPath(home: string): string;
+export declare function memoryDbPath(home: string, override?: string): string;
+export declare function knowledgeDbPath(home: string, override?: string): string;
+/**
+ * The managed document directory (default `<home>/knowledge/docs`).
+ *
+ * One editable `.md` per document, so the corpus can be maintained with ordinary tools. It sits
+ * BESIDE the database rather than replacing it: the file holds the body, the database holds the
+ * identity, chunks and indexes derived from it.
+ */
+export declare function knowledgeDocsDir(home: string, override?: string): string;
+/** Ensure the memory/knowledge layout exists under `home`. */
+export declare function ensureDataLayout(home: string): void;

package/lib/engine/mem/db/chunk.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Batch an id list for a SQL `IN (...)` clause.
+ *
+ * SQLite caps the number of bound parameters in one statement (SQLITE_MAX_VARIABLE_NUMBER,
+ * 32 766 in the bundled build). A query whose id list can grow with the CORPUS rather than
+ * with the request — `entityBags` on the write-path contradiction check, `filterActive` on
+ * the catch-up sweep — will therefore hit "too many SQL variables" on a large store. The
+ * write path swallows that as best-effort, so the symptom is a detection feature that
+ * silently stops running instead of an error anyone notices.
+ */
+/** Split `items` into consecutive batches of at most `size` (an empty list yields none). */
+export declare function batches<T>(items: readonly T[], size?: number): T[][];
+/** A fixed-width `IN (…)` fragment plus the exactly-matching bound values. */
+export interface InList<T> {
+    /** The `?,?,…` body to splice into `IN (${placeholders})` (or the literal `NULL` for an empty list). */
+    placeholders: string;
+    /** The values to bind — the input, padded to the rung with repeats of its last element. */
+    values: T[];
+}
+/**
+ * Prepare a value list for an `IN (…)` clause at a ladder-rounded, cache-stable width.
+ *
+ * `count <= 0` yields `NULL` (matches nothing) rather than `IN ()`, a syntax error — callers guard
+ * empty lists anyway; this is belt-and-braces.
+ */
+export declare function inList<T>(values: readonly T[]): InList<T>;

package/lib/engine/mem/db/conn.d.ts

@@ -0,0 +1,13 @@
+import { type OpenedStore, type StoreSchema } from './store.js';
+import { type FtsTokenizer } from './tokenizer.js';
+export type { Db, DbStatement, DbRunResult } from './port.js';
+export type { FtsTokenizer };
+/**
+ * Ordered schema steps. Step 1 is the base schema and MUST stay idempotent: a database
+ * created before this mechanism existed reports `user_version = 0` while already holding it.
+ */
+export declare const MEMORY_SCHEMA: StoreSchema;
+/** Open (or create) the memory DB, bringing its schema up to the current version. */
+export declare function openMemoryDb(path: string): OpenedStore['db'];
+/** Open and report which migrations ran (tests, and any explicit upgrade path). */
+export declare function openMemoryStore(path: string): OpenedStore;

package/lib/engine/mem/db/dao/chunks.d.ts

@@ -0,0 +1,150 @@
+/**
+ * `doc_chunks` + `chunk_entities` + the `doc_chunks_fts` index — the knowledge read model.
+ *
+ * Every id list here is batched (`batches()`): candidate sets are derived from the CORPUS
+ * (entity overlap, semantic top-k), not from the request, so a plain `IN (…)` would trip
+ * SQLite's bind-parameter cap on a large store. That failure is silent on the search path
+ * (it just drops a leg), which is exactly what batching prevents.
+ */
+import type { DocumentChunk } from '../../../mem-contract/index.js';
+import type { Db } from '../port.js';
+/** One chunk to persist, in document order (`idx` is the position, not the array index). */
+interface ChunkInsert {
+    idx: number;
+    text: string;
+    headingsPath: string;
+    sourceRef: string;
+    charStart: number;
+    charEnd: number;
+}
+/**
+ * A chunk plus the state of everything DERIVED from it (DESIGN §20).
+ *
+ * `content_hash` says which text the derivations were built from, `embedding_model` which
+ * vector space they live in, `entities_version` which extraction rules produced the entity
+ * rows. `reindex` decides per row what actually needs redoing from these; without them it has
+ * to assume every row is stale and pay for the whole corpus.
+ */
+export interface ChunkStateRow {
+    chunk_id: number;
+    text: string;
+    content_hash: string | null;
+    embedding_model: string | null;
+    entities_version: number | null;
+    /** `1` when a non-NULL vector blob is stored (SQLite has no boolean). */
+    has_vector: number;
+}
+/** A persisted chunk with the entity names extracted at ingest time. */
+interface ChunkEntityRow {
+    chunk_id: number;
+    names: readonly string[];
+}
+/** A search hit row: chunk body plus the owning document's identity. */
+export interface ChunkHitRow {
+    chunk_id: number;
+    text: string;
+    source_ref: string;
+    domain: string;
+    source: string;
+    created_at: string | null;
+    updated_at: string | null;
+}
+export declare class ChunksDao {
+    private readonly db;
+    constructor(db: Db);
+    idsForDoc(docId: number): number[];
+    /** Re-ingest replaces the chunk set wholesale (entities/FTS follow via cascade/triggers). */
+    deleteForDoc(docId: number): void;
+    /** Insert the doc's chunks in one statement per chunk; returns the new ids in order. */
+    insertMany(docId: number, chunks: readonly ChunkInsert[]): number[];
+    /** Chunk bodies by id (entity extraction + vector encoding after ingest). */
+    texts(ids: readonly number[]): {
+        chunk_id: number;
+        text: string;
+    }[];
+    /** Chunk rows for the catalog detail view, in document order (`DocumentDetail.chunks`). */
+    rowsForDoc(docId: number): DocumentChunk[];
+    /** Persist a chunk vector together with the vector space it belongs to. */
+    setVector(chunkId: number, bytes: Buffer, embeddingModel: string): void;
+    /**
+     * The doc's chunks WITH their stored vectors — the reuse index for a re-ingest.
+     *
+     * A re-ingest deletes and re-inserts every chunk (new ids), so without this the only way to keep a
+     * vector is to re-encode the same text; `vectorReusable` already knows how to tell "unchanged", but
+     * only `reindex` could use it. Read BEFORE the replace, inside its transaction.
+     */
+    vectorStateForDoc(docId: number): (ChunkStateRow & {
+        vec: Buffer | null;
+    })[];
+    /**
+     * Write a batch of vectors in ONE transaction.
+     *
+     * `setVector` is autocommit: each chunk costs a commit (measured 14.2 µs/row against 2.2 µs in a
+     * transaction, and on a WAL database every commit appends its dirty pages). Callers encode in
+     * batches anyway, so this is the same loop with the boundary moved out — and the transaction
+     * NEVER spans an `await` (the caller encodes first, then calls this).
+     */
+    setVectors(rows: readonly {
+        chunk_id: number;
+        bytes: Buffer;
+        embeddingModel: string;
+    }[]): void;
+    /** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
+    setEntitiesVersion(ids: readonly number[], version: number): void;
+    /** Every persisted chunk vector, for rebuilding the in-memory index at open. */
+    /**
+     * Every persisted chunk vector, with the space it was written in.
+     *
+     * `embedding_model` is selected too (not just the bytes) so a reload can DETECT a model swap: a
+     * dim-valid vector from another space is still ranked, and without this the two spaces would mix
+     * in one index silently — the memory side had the check, the knowledge side did not.
+     */
+    vectorRows(): {
+        id: number;
+        vec: Buffer;
+        embedding_model: string | null;
+    }[];
+    count(): number;
+    /**
+     * Replace the entity rows of the given chunks in one transaction (`replace` clears the
+     * old rows first — used by reindex, where the extractor may have changed).
+     */
+    replaceEntities(rows: readonly ChunkEntityRow[], replace?: boolean): void;
+    /** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
+    entityBags(ids: readonly number[]): Map<number, string[]>;
+    /** Chunk body + owning-document identity for the fused candidate ids. */
+    hits(ids: readonly number[]): ChunkHitRow[];
+    /** `(chunk_id, domain, source)` for the semantic top-k, to apply the domain/source filter. */
+    meta(ids: readonly number[]): {
+        chunk_id: number;
+        domain: string;
+        source: string;
+    }[];
+    /**
+     * Corpus rows for the rebuild legs, optionally scoped to one domain, together with the state
+     * of their derivations. This is the `reindex` source: the caller compares `content_hash` /
+     * `embedding_model` / `entities_version` against what it is about to produce and only pays
+     * for the rows that are actually out of date (DESIGN §20).
+     */
+    corpusState(domain?: string): ChunkStateRow[];
+    /** Rebuild the FTS index from the content table (write-path maintenance). */
+    rebuildFts(): void;
+    /** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
+    /**
+     * Entity-overlap candidates, most-shared first and CAPPED (see the memory store's sibling).
+     *
+     * `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
+     * `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
+     * caller is expected to enforce the real `cap` on the union AFTER scoring it (it re-computes the
+     * true Jaccard in JS): a union of per-batch tops is not a top-N, and a leg that returns more than
+     * the cap makes `scores.size === legCap` — the "was this leg trimmed" signal — report the opposite
+     * of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
+     */
+    candidatesByEntityNames(names: readonly string[], domain: string | undefined, source: string | undefined, limit: number): number[];
+    /** FTS leg (`bm25` rank; negated by the caller so higher = better). */
+    ftsSearch(ftsQuery: string, domain?: string, source?: string, limit?: number): {
+        id: number;
+        rank: number;
+    }[];
+}
+export {};

package/lib/engine/mem/db/dao/contradictions.d.ts

@@ -0,0 +1,98 @@
+/**
+ * `contradiction_log` — the open/resolved conflict pairs.
+ *
+ * The detector decides WHAT conflicts; this DAO owns how a pair is recorded, deduped (the
+ * partial unique index makes `log` idempotent across processes), retired, and listed. Both
+ * sides of every read are filtered to ACTIVE facts: the corpus is "changed × active", so a
+ * pair whose other side left it is no longer actionable (see DESIGN §11).
+ */
+import { type ContradictionRecord } from '../../../mem-contract/index.js';
+import type { Db } from '../port.js';
+export interface ConflictPair {
+    /** Row id of the pair — the handle `contradict_resolve` takes. */
+    contradiction_id: number;
+    fact_a: number;
+    fact_b: number;
+    score: number;
+}
+/** A row of `contradiction_log` as stored: the pair plus its adjudication state. */
+export interface StoredConflict {
+    id: number;
+    fact_a: number;
+    fact_b: number;
+    score: number;
+    /** 0 = open, 1 = closed (see `resolution` / `loser_fact_id` for how). */
+    resolved: number;
+    resolution: string | null;
+    loser_fact_id: number | null;
+    /** `auto` (a fact left the corpus) or `verdict` (explicit adjudication). */
+    resolved_by: string | null;
+}
+export declare class ContradictionsDao {
+    private readonly db;
+    constructor(db: Db);
+    /**
+     * Pairs the detector must NOT log again, normalized `min|max` — open pairs plus pairs closed
+     * by an explicit VERDICT.
+     *
+     * The distinction matters in both directions: a pair retired because a fact left the active
+     * corpus must be loggable again after `restore` (that is the documented archive → restore →
+     * re-detect cycle), while a pair someone adjudicated as `false_positive` — or as a
+     * `true_positive` where both statements stay — must not be resurrected by a later sweep just
+     * because a pending id was still queued.
+     */
+    suppressedPairs(): Set<string>;
+    /**
+     * Suppressed pairs that NAME `factId` — the write path's version of {@link suppressedPairs}.
+     *
+     * A single fact's check can only ever consult pairs it is part of (`maybeLog(fid, other, …)`),
+     * so reading the whole log was pure waste: measured 60.8 ms per write at 100k open pairs, while
+     * this answers in 0.06 ms. Both branches are index seeks on the plain `fact_a`/`fact_b` indexes
+     * (the partial UNIQUE index cannot serve the `resolved_by` disjunct).
+     */
+    suppressedPairsFor(factId: number): Set<string>;
+    log(factA: number, factB: number, score: number): void;
+    resolve(id: number, resolution: 'true_positive' | 'false_positive', loserFactId?: number, resolvedBy?: 'auto' | 'verdict'): void;
+    /**
+     * One stored row by id — what an adjudication needs before it decides anything: which two
+     * facts the pair names (so a "loser" can be validated against it) and whether it is still
+     * open.
+     */
+    getById(id: number): StoredConflict | undefined;
+    /**
+     * Close every OPEN conflict that involves `factId`.
+     *
+     * Called when a fact leaves the ACTIVE corpus (archived / superseded / forgotten).
+     * Two things depend on it:
+     *  - the conflict is no longer actionable, and `list` would otherwise keep showing a
+     *    row whose `content_a/content_b` describe a revision that is already archived;
+     *  - `update` archives the old revision and writes a NEW fact, so without this every
+     *    edit of a conflicting fact appended another permanent open row for the same
+     *    logical conflict — and nothing in the API can retire one (there is no
+     *    `resolve` action on any tool/CLI/MCP surface).
+     *
+     * The archived fact is recorded as the loser: it is the revision that left the store.
+     */
+    resolveForFact(factId: number, resolution?: 'true_positive' | 'false_positive'): number;
+    /**
+     * {@link resolveForFact} for a BATCH of ids leaving the corpus — the lifecycle tick path.
+     *
+     * The per-id loop cost O(archived × log) plus one statement compile per id: measured 6.3 s for
+     * 999 ids at 99k open pairs. Two statements PER BATCH (one per side) do the same mission: each is
+     * an index seek on the plain `fact_a`/`fact_b` index, and the side that matched supplies the
+     * loser (the fact that actually left the corpus), so the recorded verdict is unchanged.
+     */
+    resolveForFacts(factIds: readonly number[], resolution?: 'true_positive' | 'false_positive'): number;
+    list(resolved?: number, category?: string, threshold?: number, limit?: number): ContradictionRecord[];
+    /**
+     * Every OPEN conflict that names `factId`, most severe first, with BOTH sides still active.
+     *
+     * The detector's own `logged` list carries only what THIS pass inserted (`maybeLog` skips an
+     * already-open pair), while a caller asking "what conflicts with the fact I just wrote?"
+     * needs the whole open set: an update that merges into an existing fact changes state
+     * without inserting a row for a conflict that was already open, and reporting nothing there
+     * is a false all-clear.
+     */
+    openConflictsFor(factId: number, limit?: number): ConflictPair[];
+    private normalize;
+}

package/lib/engine/mem/db/dao/documents.d.ts

@@ -0,0 +1,51 @@
+/**
+ * `documents` — the knowledge corpus identity table (`domain → source → title`).
+ *
+ * Identity is the `(domain, source, title)` unique key, so `upsert` is `INSERT OR IGNORE`
+ * followed by a lookup: a re-ingest of the same document must land on the SAME `doc_id`
+ * (chunks are replaced under it), never create a second row.
+ */
+import type { Db } from '../port.js';
+import type { DocumentRecord, DocumentSummary } from '../../../mem-contract/index.js';
+/**
+ * The stored document rows. The PAYLOAD shapes live in the contract (`kb list`/`kb detail` return
+ * them verbatim, and the settings page imports them), so these are aliases rather than a second
+ * declaration — a column added here without the contract would be a type error, not a silent
+ * extra field on the wire.
+ */
+type DocumentRow = DocumentSummary;
+type DocumentDetailRow = DocumentRecord;
+export declare class DocumentsDao {
+    private readonly db;
+    constructor(db: Db);
+    /** Idempotent upsert by the unique key; returns the existing or newly created id. */
+    upsert(domain: string, source: string, title: string, sourceUri?: string): number;
+    /**
+     * One row by the unique identity `(domain, source, title)`, or `null` when no such document
+     * exists. Walks the `UNIQUE(domain, source, title)` index — the add-only guard's existence test.
+     */
+    find(domain: string, source: string, title: string): DocumentRow | null;
+    /**
+     * Bump `updated_at` and record the URI that was actually read. `COALESCE` keeps the
+     * previous URI when this ingest did not come from one (plain text has none).
+     */
+    touch(docId: number, sourceUri?: string): void;
+    /**
+     * Catalog page: newest first, optionally scoped to one domain and/or source.
+     *
+     * `limit`/`offset` page it for the UI (which renders one page at a time); omitting `limit` keeps
+     * the historical "everything" behavior the CLI and MCP surfaces expect.
+     */
+    list(domain?: string, source?: string, limit?: number, offset?: number): DocumentRow[];
+    get(docId: number): DocumentDetailRow | null;
+    /**
+     * The distinct domains the library already holds, sorted. This is the "already in the library"
+     * half of the domain allowlist: a name in use must stay usable (and selectable) even when the
+     * configured allowlist would not have accepted it as a NEW domain.
+     */
+    domains(): string[];
+    /** `true` when a document was actually deleted (cascades chunks/entities/FTS). */
+    remove(docId: number): boolean;
+    count(): number;
+}
+export {};

package/lib/engine/mem/db/dao/entities.d.ts

@@ -0,0 +1,67 @@
+/**
+ * `entities` + `fact_entities` — the entity graph behind the jaccard / related / reason
+ * paths and the contradiction detector's candidate narrowing.
+ *
+ * Every id list that can grow with the CORPUS goes through `batches()`: a single `IN (…)`
+ * would exceed SQLite's bind-parameter cap and, on the write path, silently disable
+ * contradiction detection (see `db/chunk.ts`).
+ */
+import type { Db } from '../port.js';
+export declare class EntitiesDao {
+    private readonly db;
+    constructor(db: Db);
+    /**
+     * Ensure every name exists and link the fact to them.
+     *
+     * `INSERT OR IGNORE` + the `name` uniqueness make this idempotent, which matters because
+     * both `add` and the revive branch of `persistFact` call it for the same fact.
+     */
+    linkFact(factId: number, names: readonly string[]): void;
+    /**
+     * Drop one fact's links, leaving the shared `entities` vocabulary alone.
+     *
+     * `linkFact` is additive (`INSERT OR IGNORE`) — correct for the write path, where a fact's links
+     * are written once — so RE-EXTRACTION has to clear first or the old names would survive alongside
+     * the new ones. Orphaned `entities` rows are harmless: the entity leg joins `fact_entities`, so
+     * an unlinked name is unreachable, and a later fact using it reuses the row.
+     */
+    unlinkFact(factId: number): void;
+    /** One fact's entity names (detail / contradiction scoring). */
+    namesForFact(factId: number): string[];
+    /** Batched entity bags for a candidate set: `Map<factId, names>` (ids with none get `[]`). */
+    bagsForFacts(ids: readonly number[]): Map<number, string[]>;
+    /** How many entities a fact is linked to — the scorer's `|A|`. */
+    countForFact(factId: number): number;
+    /**
+     * Candidate facts for the embedding leg, narrowed by the scorer's NECESSARY conditions
+     * (see {@link ContradictDetector}): at least `minShared` shared entities and at most
+     * `maxEntities` of their own, both active and never the checked fact itself.
+     *
+     * `INDEXED BY` pins the join order to mine → other → facts. Without statistics SQLite
+     * drives from `facts` by status instead (a corpus-sized scan: measured 115 ms with zero
+     * candidates on a 300k-fact store, versus 0.13 ms here), and it only collects statistics on
+     * the lifecycle tick — so a store that grew since startup would charge every write for the
+     * whole corpus. The hint makes the plan independent of ANALYZE timing.
+     */
+    candidateFacts(factId: number, minShared: number, maxEntities: number): number[];
+    /** Active facts linked to `name`, optionally limited to one category (`related` leg). */
+    activeFactsForEntity(name: string, category?: string): number[];
+    /** Active facts linked to ALL of `names` (AND-join, `reason` leg). */
+    activeFactsForAllEntities(names: readonly string[]): number[];
+    /** Active facts sharing ANY of `names`, in one category (`jaccard` leg). */
+    /**
+     * Candidate facts for an entity-Jaccard query, ordered by the ACTUAL Jaccard and CAPPED.
+     *
+     * The cap is the whole point: this leg used to return every fact sharing ANY query entity —
+     * unbounded in the corpus (measured 6600 rows for a common phrase at 33k facts, and the fusion
+     * step then normalized and sorted all of them).
+     *
+     * Ordering by the shared COUNT alone (the first version of this cap) was WRONG: the leg's score
+     * is `shared / union`, so a fact sharing 3 of 10 entities (0.30) outranked one sharing 2 of 2
+     * (0.67) and pushed it out of the cap — measured: dropping the higher-scoring document. The
+     * ratio is computed in SQL instead: `union = |q| + |f| - shared`, and `|f|` is one indexed COUNT
+     * per candidate (the `fact_entities` primary key covers it). The caller still recomputes the same
+     * ratio in JS for the survivors, which is cheap (<= the cap) and keeps one formula in charge.
+     */
+    candidateFactsForAnyEntity(names: readonly string[], category: string | undefined, limit: number): number[];
+}