@avantf/dsh-mem 0.6.4 → 0.8.0

package/lib/engine/mem/facts_io.d.ts

@@ -0,0 +1,328 @@
+import { EXPORT_STATUSES, type ExportStatus, type MemoryKind, type RememberRequest } from '../mem-contract/index.js';
+/**
+ * The runtime slice these flows actually touch.
+ *
+ * Narrow on purpose: export reads two config fields, import/merge also call the store's `add` path.
+ * Keeping the parameter structural (rather than the whole `AvantfRuntime`) is what lets the admin
+ * dispatcher — an object literal whose `closed` getter is attached only at return time — pass
+ * itself in without a cast, and it documents that these flows cannot reach any other store face.
+ */
+export interface FactsIoRuntime {
+    config: {
+        home: string;
+        memory: {
+            db: {
+                path: string;
+            };
+        };
+    };
+    remember(req: RememberRequest): Promise<unknown>;
+}
+/** The document's discriminator. A file without it is not an export of this tool. */
+export declare const FACTS_EXPORT_FORMAT = "avantf-mem-facts";
+/**
+ * The version THIS build writes: 5 (v4's `title` plus the note-only `relations`).
+ *
+ * Versions are additive by design: v2/v3/v4/v5 each add OPTIONAL fields, so an older document is the
+ * same shape minus the new keys. A version outside {@link FACTS_EXPORT_VERSIONS} must be refused,
+ * never guessed at.
+ */
+export declare const FACTS_EXPORT_VERSION = 5;
+/**
+ * A version this build can READ. v1–v4 documents are what users already have on disk, and a v5
+ * document adds only the OPTIONAL `relations` field — so all five parse into the same type.
+ */
+export declare const FACTS_EXPORT_VERSIONS: readonly [1, 2, 3, 4, 5];
+/** The version literal, so a parsed document can say which field set it came from. */
+export type FactsExportVersion = (typeof FACTS_EXPORT_VERSIONS)[number];
+/**
+ * `--status` values accepted by `export` (the whole DB, or one status).
+ *
+ * Re-exported from the contract, where the admin action and the CLI flag read the same set. The
+ * engine's public face keeps the name its callers already import.
+ */
+export { EXPORT_STATUSES, type ExportStatus };
+/** One `fact_sources` row, copied verbatim. */
+export interface ExportedFactSource {
+    kind: string;
+    ref: string;
+}
+/**
+ * One fact, as it stands in the source database.
+ *
+ * Every field is part of the documented format even when v1 import ignores it — the point of the
+ * document is that a later restore does not have to invent what was never written down.
+ */
+export interface ExportedFact {
+    /** Source-local row id. For reports/debugging ONLY; content is the identity. */
+    source_id: number;
+    content: string;
+    /** The memory layer (v3). ABSENT on a v1/v2 document, which carries `category` instead. */
+    kind?: MemoryKind;
+    /**
+     * v4: the journal title (JOURNAL_TITLE_SEARCH §7). ABSENT on a v1/v2/v3 document — those rows had
+     * no title column at all, and an import restores them as NULL rather than inventing one.
+     */
+    title?: string | null;
+    /** The old free-form label, kept for audit only; `null` when the row predates the migration. */
+    legacy_category?: string | null;
+    /**
+     * v1/v2 ONLY: the old free-form category. Read (and mapped) for backward compatibility, never
+     * written by this build — an export from a v10 source database still carries it, and a v1/v2
+     * document on disk is exactly that shape.
+     */
+    category?: string;
+    status: string;
+    created_at: string | null;
+    updated_at: string | null;
+    ttl_days: number;
+    pinned: boolean;
+    pinned_at: string | null;
+    trust_score: number;
+    assert_count: number;
+    helpful_count: number;
+    last_reinforced_at: string | null;
+    sources: ExportedFactSource[];
+    valid_from: string | null;
+    valid_to: string | null;
+    supersedes_source_id: number | null;
+    /** `null` for an active row; the archive bookkeeping for an archived one. */
+    archive: {
+        at: string | null;
+        reason: string | null;
+    } | null;
+    /** Active-day clock at the last trust settle — the decay baseline. v1 had no such column. */
+    settle_clock?: number | null;
+    /** Reinforcement events inside the current 24h window. */
+    bonus_count?: number | null;
+    /** Calendar start of that window. */
+    bonus_window_at?: string | null;
+    /** How many times this fact was RETURNED by a retrieval leg (local usage stat, still stored). */
+    retrieval_count?: number | null;
+    last_retrieved_at?: string | null;
+    /** Mirror bookkeeping (a fact copied between libraries); informational, restored verbatim. */
+    mirror_source?: string | null;
+    mirror_target?: string | null;
+    /**
+     * The DERIVED-STATE stamps. Exported for completeness/diagnostics, but restore deliberately does
+     * not trust them: it stamps `entities_version = 0` (so the entity sweep rebuilds) and
+     * `conflict_checked = 1` (no corpus-wide conflict re-scan after a rebuild).
+     */
+    entities_version?: number | null;
+    conflict_checked?: number | null;
+    /**
+     * v5: the relations the CALLER stated on this note (`edges.source = 'explicit'`), in insertion
+     * order. ABSENT on a v1–v4 document (those rows were written before the field existed) and on
+     * every `journal` row — a journal refuses relations by contract, so the key there would assert a
+     * relation set a journal cannot have.
+     *
+     * These are NOT the extracted (`svo`) edges: those are re-derived from `content`, while an
+     * explicit relation is the one graph fact the text does not contain. That is why it travels.
+     */
+    relations?: ExportedRelation[];
+}
+/** One caller-stated relation inside an export document (v5). */
+export interface ExportedRelation {
+    subject: string;
+    predicate: string;
+    object: string;
+}
+/** The single JSON document `export` writes (and `import`/`restore` read). */
+export interface FactsExportDocument {
+    format: typeof FACTS_EXPORT_FORMAT;
+    /** The version the document was WRITTEN as (1, 2 or 3); this build writes 3. */
+    version: FactsExportVersion;
+    exported_at: string;
+    source: {
+        data_home: string;
+        db: string;
+        /** The SOURCE database's whole-DB status counts, independent of `--status`. */
+        counts: {
+            active: number;
+            archived: number;
+        };
+    };
+    facts: ExportedFact[];
+}
+/** How v1 import may treat the provenance embedded in the document. */
+export interface ImportOptions {
+    /** Plan only: no write, and the report gets `would_import`. */
+    dryRun?: boolean;
+    /** Fallback provenance when the document has none (the CLI passes `import:<file>`). */
+    sourceRef?: string;
+    /**
+     * Use `sources[0].ref` when present (the default, and the import contract).
+     *
+     * `merge` turns this OFF: its documented provenance is `merge:<absolute source path>` for every
+     * imported row, so a merge must not start attributing rows to whatever the other host recorded.
+     */
+    preferEmbeddedSources?: boolean;
+}
+/** One row the store refused; the rest of the import continues. */
+export interface ImportFailure {
+    /** The document's `source_id`, or `null` when the row was too malformed to have one. */
+    source_id: number | null;
+    error: string;
+}
+/** Source id → target id for one fact this invocation actually CREATED. */
+export interface ImportNewId {
+    source_id: number | null;
+    new_fact_id: number;
+}
+/**
+ * Counts of values v1 could not carry, over the facts this invocation imported (or, for a dry run,
+ * planned to import). `1` per fact that actually carried the value; `0` when there was nothing to
+ * lose. See the module doc for why these are losses rather than silent successes.
+ */
+export interface MetadataDropped {
+    timestamps: number;
+    trust: number;
+    assert_count: number;
+    helpful_count: number;
+    pinned: number;
+    supersedes: number;
+}
+/** The single JSON object `import` prints (`file` is added by the CLI, which knows the argument). */
+export interface ImportReport {
+    /** The input form this run detected: an export document. */
+    mode: 'json';
+    version: number;
+    total: number;
+    imported: number;
+    /** Present on `--dry-run` only; the number of facts that WOULD be written. */
+    would_import?: number;
+    skipped_existing: number;
+    skipped_archived: number;
+    failed: ImportFailure[];
+    new_ids: ImportNewId[];
+    metadata_dropped: MetadataDropped;
+}
+/** The db→db variant's report. Field names kept from the original `merge` output. */
+export interface MergeReport {
+    /** The input form this run detected: a SQLite source database. */
+    mode: 'db';
+    source: string;
+    source_active: number;
+    added: number;
+    would_add: number;
+    skipped_existing: number;
+    skipped_archived: number;
+    failed: {
+        source_fact_id: number | null;
+        error: string;
+    }[];
+    new_ids: {
+        source_fact_id: number | null;
+        new_fact_id: number;
+    }[];
+}
+export interface MergeOptions {
+    /** Provenance for rows without an embedded source; default `merge:<absolute source path>`. */
+    sourceRef?: string;
+    /** Plan only: no model warmup, no write, `added` stays 0. */
+    dryRun?: boolean;
+}
+/**
+ * Export THIS runtime's memory facts. Read-only: the store's own connection is never asked to write
+ * (and this command needs no embedding model — the rows are already there).
+ */
+export declare function exportFacts(rt: FactsIoRuntime, options?: {
+    status?: ExportStatus;
+}): FactsExportDocument;
+/**
+ * The compression containers `export` can write. `none` is plain minified JSON (the format's
+ * canonical text, and what stdout always carries).
+ */
+export type FactsExportCodec = 'brotli' | 'gzip' | 'none';
+/**
+ * The one serialization. MINIFIED by default: the document is a machine artifact that gets
+ * compressed, and indentation only makes the uncompressed fallback bigger. `pretty` exists for
+ * debugging (a human reading a diff), never as the default — and it is the ONLY mode that appends
+ * the trailing newline that used to be unconditional.
+ */
+export declare function serializeFactsExport(doc: FactsExportDocument, options?: {
+    pretty?: boolean;
+}): string;
+/**
+ * Serialize + compress in one step. gzip uses level 9; brotli uses quality 11 and tells the encoder
+ * the input size, which is the smallest output either library produces for a one-shot export.
+ *
+ * `codec` defaults to `none` so a caller that says nothing gets the canonical minified JSON.
+ */
+export declare function encodeFactsExport(doc: FactsExportDocument, options?: {
+    codec?: FactsExportCodec;
+    pretty?: boolean;
+}): Buffer;
+/** The document an encoded file carries, plus the container it actually turned out to be. */
+export interface DecodedFactsExport {
+    doc: FactsExportDocument;
+    codec: FactsExportCodec;
+}
+/**
+ * Whether these bytes are a SQLite database, decided by the FILE HEADER.
+ *
+ * `mem_admin import` takes either an export document or another host's `memory.db`, and the only
+ * honest discriminator is the content: the export containers (plain / gzip / brotli) never start with
+ * the SQLite header, and a `.db` file renamed to `.json` still is one. Kept here, beside
+ * {@link decodeFactsExport}, so both content sniffers live together.
+ */
+export declare function isSqliteDatabase(bytes: Buffer): boolean;
+/**
+ * The same check for a PATH, reading only the 16-byte header.
+ *
+ * A `memory.db` can be hundreds of MiB; deciding which flow to take must not read it into memory
+ * first. The JSON path still reads the whole document (it has to decode it), but the db path hands the
+ * path straight to the read-only opener.
+ */
+export declare function isSqliteDatabaseFile(path: string): boolean;
+/**
+ * Decode one export file from its BYTES. The file name is deliberately not an input: gzip has a
+ * magic header, brotli does not, and a pre-compression export is plain JSON — so the only reliable
+ * discriminator is the content itself (a brotli file renamed to `.json` still imports, and a plain
+ * JSON file renamed to `.json.br` does too).
+ *
+ * Order, and why: gzip magic → gunzip (a corrupt gzip is refused as gzip, not silently "JSON");
+ * otherwise UTF-8 JSON (the backward-compatible path for every old pretty file); only when JSON
+ * parsing fails, brotli (its decoder rejects non-brotli input, so this cannot shadow a JSON
+ * document). A document that IS JSON but has the wrong format/version throws its own error right
+ * away — {@link parseFactsExport} is the format gate, not a fallback.
+ */
+export declare function decodeFactsExport(bytes: Buffer): DecodedFactsExport;
+/**
+ * Resolve the container for a file export: an explicit flag wins, then the extension (`.br` /
+ * `.gz`), then plain minified JSON. Shared by the CLI and `/mem` so the two surfaces cannot
+ * disagree about what `--out backup.json.gz` means.
+ */
+export declare function codecForExportPath(path: string, explicit?: FactsExportCodec): FactsExportCodec;
+/**
+ * Parse and validate an export document TEXT (the stdin entry point). Top-level format errors are
+ * FATAL (the caller must write nothing); a malformed individual fact is NOT — {@link importFacts}
+ * collects it in `failed` and keeps going. Restore is STRICTER by construction: it validates every
+ * row before opening its transaction, because skipping a row would produce a different history.
+ */
+export declare function parseFactsExport(text: string): FactsExportDocument;
+/**
+ * Import one export document into `rt`'s memory store.
+ *
+ * Order of decisions per fact: archived → skipped_archived (v1 never revives); content already in
+ * the target (or planned by an earlier row of this same document) → skipped_existing; otherwise the
+ * store's add path runs. A row the store refuses is collected in `failed` and the loop continues.
+ *
+ * A v5 row's `relations` travel through that same add path, so the imported fact gets its declared
+ * relations as `source: 'explicit'` edges (normalized, name-gated, four-tuple deduped against the
+ * extracted ones). Two consequences worth knowing: a row that is SKIPPED as already-present keeps
+ * the target's own relation set (this is a content union, not a merge of graphs), and a `journal`
+ * row carrying relations is refused per row with {@link RELATIONS_NEED_NOTE} — that combination is
+ * unrepresentable, so it must be reported rather than silently stripped.
+ */
+export declare function importFacts(rt: FactsIoRuntime, doc: FactsExportDocument, options?: ImportOptions): Promise<ImportReport>;
+/**
+ * `merge` — the db→document→import form of the same union.
+ *
+ * The source is another host's file: opened read-only, never migrated, never written. The document is
+ * built in memory and handed to {@link importFacts}, so merge and `import` share one per-fact add
+ * path (and one failure-collection rule). Its provenance stays `merge:<absolute source path>` even
+ * when the source rows carry their own sources — that is the documented merge contract and the
+ * reason {@link ImportOptions.preferEmbeddedSources} can be turned off.
+ */
+export declare function mergeDb(rt: FactsIoRuntime, dbPath: string, options?: MergeOptions): Promise<MergeReport>;

package/lib/engine/mem/index.d.ts

@@ -19,11 +19,19 @@ export * from './modelBootstrap.js';
 export * from './dispatch.js';
 export * from './trustHeartbeat.js';
 export * from './runtime.js';
+export * from './storage/index.js';
+export * from './interfaces/index.js';
+export { isSqliteStorageBackend, sqliteDriver, SqliteStorageBackend, SQLITE_CAPABILITIES } from './dbdrivers/sqlite/index.js';
+export { FAKE_CAPABILITIES, FakeStorageBackend, fakeDriver, isFakeStorageBackend, MockRemoteTransport, type MockRemoteTurnaround, type StorageTransport, type TransportRequestRecord, } from './dbdrivers/fake/index.js';
 export * from './store/memory.js';
-export * from './store/knowledge.js';
+export { normalizeWrite } from './store/common.js';
+export * from './facts_io.js';
+export * from './maintenance_paths.js';
+export * from './store/facts_restore.js';
 export * from './store/vector_repair.js';
 export * from './store/hybrid.js';
-export * from './store/source_picker.js';
 export * from './store/lexical.js';
 export * from './store/floors.js';
 export { registerSemanticBackend, registerVectorStore, type SemanticBackend, type VectorStore, } from '../mem-retrieval/index.js';
+export * from './store/relation_backfill.js';
+export { extractRelationCandidates, normalizeRelation, evaluateRelation, type RelationCandidate, type RelationEvaluation, type RelationExtractor, } from './relations/index.js';

package/lib/engine/mem/interfaces/capabilities.d.ts

@@ -0,0 +1,30 @@
+/**
+ * What a storage driver DECLARES it can do (docs/DB_DRIVER_LAYOUT.md §二).
+ *
+ * Capabilities are how the HAL stays honest about optional behaviour without a second interface per
+ * driver: a backend that cannot do full-text search says so once, here, instead of the store probing
+ * it with a query that fails. Adding a flag is a MINOR contract change (an old driver that does not
+ * know the flag simply does not declare it; the host must tolerate unknown flags).
+ *
+ * @module interfaces/capabilities
+ */
+export interface StorageCapabilities {
+    /** Async unit-of-work transactions (`LifecycleService.transaction`). */
+    transactions: boolean;
+    /** Content full-text search (`SearchIndex.searchContent`). */
+    fullText: boolean;
+    /** Title search (`SearchIndex.searchTitles` + the exact-title face). */
+    titles: boolean;
+    /** The relation graph (entities / triples / edges / graph reader). */
+    relations: boolean;
+    /**
+     * Persisted vector BLOBS (`facts.semantic_vector` / `hrr_vector`) and their inventory reads. The
+     * live vector INDEX is a retrieval concern resolved above the storage layer; this flag is about
+     * storing and reading the rows a driver owns.
+     */
+    vectors: boolean;
+    /** The backend applies its own schema migrations on open (`MigrationRunner`). */
+    migrations: boolean;
+    /** The contract is async (always `true` for contract major 1 — see `interfaces/version.ts`). */
+    async: true;
+}

package/lib/engine/mem/interfaces/common.d.ts

@@ -0,0 +1,11 @@
+/**
+ * The storage HAL's async shape (DB-2).
+ *
+ * Every port method that touches stored state returns this: the in-process SQLite driver fulfils the
+ * contract by returning a plain value (it has no I/O to wait for), while a remote driver returns a
+ * promise. Callers ALWAYS `await`, so no store, no retrieval leg and no lifecycle caller has to know
+ * which kind of driver it holds — and a new driver never rewrites a call site.
+ *
+ * @module interfaces/common
+ */
+export type Awaitable<T> = T | Promise<T>;

package/lib/engine/mem/interfaces/errors.d.ts

@@ -0,0 +1,65 @@
+/**
+ * The storage error boundary (DB-3).
+ *
+ * SQLite reports failures as a driver error object with a numeric `errcode`; PostgreSQL has SQLSTATE,
+ * an HTTP backend has status codes, and none of those belong above the port. The store has exactly
+ * three things it can usefully do with a failed read/write: retry (the backend is temporarily
+ * unavailable), report a conflict (a uniqueness/constraint rule refused the write), or fail (anything
+ * else). This module names those three, and `db/sqlite.ts` — the one place that sees a driver error —
+ * maps into them.
+ *
+ * WHAT THIS IS NOT: a replacement for the domain refusals. `TITLE_IMMUTABLE`, `RELATIONS_NEED_NOTE`
+ * and the other `Error`s the store throws are decisions about the caller's request, not backend
+ * failures; they stay plain and unchanged. Only errors that come out of the database layer are
+ * classified here.
+ *
+ * The messages are preserved verbatim (the classification is metadata), so every existing caller and
+ * test that asserts on a driver message keeps working.
+ *
+ * @module interfaces/errors
+ */
+/** What a caller can do about a failed storage operation. */
+export type StorageFailureKind = 'unavailable' | 'constraint' | 'corrupt' | 'unknown';
+/** A database-layer failure, classified so the boundary does not leak a driver's error shape. */
+export declare class StorageError extends Error {
+    readonly kind: StorageFailureKind;
+    constructor(message: string, kind: StorageFailureKind, options?: {
+        code?: string | number;
+        cause?: unknown;
+    });
+    /** The driver's own code, kept for a log line — never branched on above the adapter. */
+    readonly code?: string | number;
+    /** Whether the same call may succeed if repeated later (a lock timeout, a restarting server). */
+    get retryable(): boolean;
+}
+/** The backend is temporarily unable to serve (SQLite: `SQLITE_BUSY`/`SQLITE_LOCKED`). */
+export declare class StorageUnavailableError extends StorageError {
+    constructor(message: string, options?: {
+        code?: string | number;
+        cause?: unknown;
+    });
+}
+/** The backend refused the write because of one of its integrity rules. */
+export declare class StorageConstraintError extends StorageError {
+    constructor(message: string, options?: {
+        code?: string | number;
+        cause?: unknown;
+    });
+}
+/** The stored data cannot be read (SQLite: `SQLITE_CORRUPT`/`SQLITE_NOTADB`). */
+export declare class StorageCorruptError extends StorageError {
+    constructor(message: string, options?: {
+        code?: string | number;
+        cause?: unknown;
+    });
+}
+/** Whether a caught value is already one of ours (a second mapping must not wrap it again). */
+export declare function isStorageError(error: unknown): error is StorageError;
+/**
+ * Classify a driver failure into the storage vocabulary.
+ *
+ * Takes `unknown` because that is what a `catch` binds; anything already classified is returned
+ * untouched (mapping is idempotent), and a non-error value becomes a generic `StorageError` carrying
+ * its string form, so callers never have to re-check.
+ */
+export declare function mapStorageError(error: unknown): StorageError;

package/lib/engine/mem/interfaces/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * The storage HAL contract (docs/DB_DRIVER_LAYOUT.md).
+ *
+ * Nothing in this directory may import `dbdrivers/`, `db/dao` or `node:sqlite`: these files are the
+ * vocabulary a driver implements and the store consumes, and they have to stay loadable in a host
+ * that has no SQLite at all.
+ *
+ * @module interfaces
+ */
+export * from './common.js';
+export * from './version.js';
+export * from './capabilities.js';
+export * from './migrations.js';
+export * from './repositories.js';
+export * from './search_index.js';
+export * from './errors.js';
+export * from './storage_backend.js';
+export * from './storage_driver.js';

package/lib/engine/mem/interfaces/migrations.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Schema migration state, as the HAL sees it (DB-3).
+ *
+ * A driver keeps its own schema state (SQLite's `PRAGMA user_version`, a remote backend's version
+ * table or a service endpoint). The host needs two things from it and nothing more: what this open
+ * applied, and a way to RE-READ the current version without applying anything (the hook the
+ * migration/import tooling needs).
+ *
+ * @module interfaces/migrations
+ */
+import type { Awaitable } from './common.js';
+export interface MigrationRunner {
+    /** The version the backend reported before this open. */
+    readonly from: number;
+    /** The version this build brings (or brought) the backend to. */
+    readonly to: number;
+    /** The steps this open actually applied, in order (empty = the backend was already current). */
+    readonly applied: readonly number[];
+    /** Re-read the backend's current schema version, applying nothing. */
+    current(): Awaitable<number>;
+}