@avantf/dsh-mem 0.3.1 → 0.4.0

package/lib/engine/mem-contract/config.d.ts

@@ -0,0 +1,129 @@
+import { z } from 'zod';
+/**
+ * Input window of the SHIPPED models: `Xenova/bge-small-zh-v1.5` (embedder) and
+ * `Xenova/bge-reranker-base` (reranker) both read 512 tokens.
+ *
+ * Exported because more than one default derives from it: the embedder's fallback window, the
+ * reranker's pair budget (retrieval-core), and `knowledge.chunk_size` below. transformers.js
+ * truncates past the window SILENTLY, so every derivation has to agree on the number.
+ */
+export declare const DEFAULT_MODEL_WINDOW_TOKENS = 512;
+/** The merged runtime config shared by memory and knowledge stores. */
+export declare const ConfigSchema: z.ZodObject<{
+    dataHome: z.ZodDefault<z.ZodString>;
+    semantic: z.ZodPrefault<z.ZodObject<{
+        backend: z.ZodDefault<z.ZodString>;
+        local_model: z.ZodDefault<z.ZodString>;
+        dim: z.ZodDefault<z.ZodNumber>;
+        max_input_tokens: z.ZodDefault<z.ZodNumber>;
+        mirror: z.ZodDefault<z.ZodString>;
+        cache_dir: z.ZodDefault<z.ZodString>;
+        auto_download: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strip>>;
+    rerank: z.ZodPrefault<z.ZodObject<{
+        backend: z.ZodDefault<z.ZodString>;
+        local_model: z.ZodDefault<z.ZodString>;
+        max_input_tokens: z.ZodDefault<z.ZodNumber>;
+        mirror: z.ZodDefault<z.ZodString>;
+        cache_dir: z.ZodDefault<z.ZodString>;
+        auto_download: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strip>>;
+    vectorStore: z.ZodPrefault<z.ZodObject<{
+        backend: z.ZodDefault<z.ZodString>;
+        auto_thresholds: z.ZodDefault<z.ZodObject<{
+            hnswlib: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strip>>;
+        hnswlib_ef_search: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>>;
+    retriever: z.ZodPrefault<z.ZodObject<{
+        weight_semantic: z.ZodDefault<z.ZodNumber>;
+        weight_fts: z.ZodDefault<z.ZodNumber>;
+        weight_jaccard: z.ZodDefault<z.ZodNumber>;
+        max_output_tokens: z.ZodDefault<z.ZodNumber>;
+        over_fetch_factor: z.ZodDefault<z.ZodNumber>;
+        min_semantic_similarity: z.ZodDefault<z.ZodNumber>;
+        min_fts_terms: z.ZodDefault<z.ZodNumber>;
+        min_jaccard: z.ZodDefault<z.ZodNumber>;
+        leg_cap: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>>;
+    lifecycle: z.ZodPrefault<z.ZodObject<{
+        purge_after_archived_days: z.ZodDefault<z.ZodNumber>;
+        contradiction_threshold: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>>;
+    trust: z.ZodPrefault<z.ZodObject<{
+        enabled: z.ZodDefault<z.ZodBoolean>;
+        start: z.ZodDefault<z.ZodNumber>;
+        decay_per_day: z.ZodDefault<z.ZodNumber>;
+        forget_threshold: z.ZodDefault<z.ZodNumber>;
+        permanent_threshold: z.ZodDefault<z.ZodNumber>;
+        recall_floor: z.ZodDefault<z.ZodNumber>;
+        recall_delta: z.ZodDefault<z.ZodNumber>;
+        recall_daily_cap: z.ZodDefault<z.ZodNumber>;
+        recall_marginal_decay: z.ZodDefault<z.ZodNumber>;
+        recall_ceiling: z.ZodDefault<z.ZodNumber>;
+        feedback_delta: z.ZodDefault<z.ZodNumber>;
+        feedback_daily_cap: z.ZodDefault<z.ZodNumber>;
+        inherit_trust_on_update: z.ZodDefault<z.ZodBoolean>;
+        purge_skips_pinned: z.ZodDefault<z.ZodBoolean>;
+        idle_calendar_days: z.ZodDefault<z.ZodNumber>;
+        tick_max_facts: z.ZodDefault<z.ZodNumber>;
+        presence: z.ZodPrefault<z.ZodObject<{
+            mode: z.ZodDefault<z.ZodEnum<{
+                process: "process";
+            }>>;
+            gap_cap_days: z.ZodDefault<z.ZodNumber>;
+            heartbeat_minutes: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    tools: z.ZodPrefault<z.ZodObject<{
+        dir: z.ZodDefault<z.ZodString>;
+        mirror: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        auto_install: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type Config = z.infer<typeof ConfigSchema>;
+/** Built-in defaults — the lowest precedence layer. */
+export declare const defaultConfig: Config;
+/**
+ * Memory store-specific config. Overrides on top of the common Config.
+ * Note: `db.path` is the single authoritative path for the memory DB.
+ */
+export declare const MemoryConfigSchema: z.ZodObject<{
+    db: z.ZodPrefault<z.ZodObject<{
+        path: z.ZodDefault<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type MemoryConfig = z.infer<typeof MemoryConfigSchema>;
+/** Knowledge store-specific config. */
+export declare const KnowledgeConfigSchema: z.ZodObject<{
+    db: z.ZodPrefault<z.ZodObject<{
+        path: z.ZodDefault<z.ZodString>;
+    }, z.core.$strip>>;
+    docs: z.ZodPrefault<z.ZodObject<{
+        dir: z.ZodDefault<z.ZodString>;
+    }, z.core.$strip>>;
+    git: z.ZodPrefault<z.ZodObject<{
+        mode: z.ZodDefault<z.ZodEnum<{
+            auto: "auto";
+            off: "off";
+        }>>;
+        root: z.ZodDefault<z.ZodString>;
+        ignore: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    open: z.ZodPrefault<z.ZodObject<{
+        editor: z.ZodDefault<z.ZodString>;
+    }, z.core.$strip>>;
+    domains: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    chunk_size: z.ZodDefault<z.ZodNumber>;
+    chunk_overlap: z.ZodDefault<z.ZodNumber>;
+    source_priority: z.ZodDefault<z.ZodEnum<{
+        none: "none";
+        workspace: "workspace";
+    }>>;
+    ingest: z.ZodPrefault<z.ZodObject<{
+        local_roots: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        allow_outside_workspace: z.ZodDefault<z.ZodBoolean>;
+        allow_private_network: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type KnowledgeConfig = z.infer<typeof KnowledgeConfigSchema>;

package/lib/engine/mem-contract/env.d.ts

@@ -0,0 +1,31 @@
+/** Expand `~/…` (and a BARE `~`) against the USER home directory.
+ *
+ * The bare form matters: `AVANTF_HOME=~` used to come back verbatim, i.e. as the RELATIVE path `~`,
+ * and everything built on it was written into the current working directory. The base kit's own
+ * `expandHome` always expanded it, so the two copies disagreed on exactly the input a shell makes
+ * easy to type. */
+export declare function expandHome(p: string): string;
+/** Parse a boolean env var: `0`/`false` → false, unset/empty → undefined, else true. */
+export declare function envFlag(name: string): boolean | undefined;
+/**
+ * The global model-download kill switch.
+ *
+ * TWO switches, either of which turns downloads off:
+ *
+ *   - `AVANTF_MEM_AUTO_DOWNLOAD` — this project's switch (CLI, MCP server, tests);
+ *   - `AVANTF_ENVINIT_AUTO_DOWNLOAD` — the family framework's switch, a MASTER gate: the operator
+ *     turns the whole family's provisioning off in one place, so the engine must stop fetching too
+ *     (otherwise the framework skips its item while the runtime silently downloads the same model).
+ *
+ * Deliberately honored in the config loader (env layer ④, so `rt.config` reflects reality) and in
+ * the model adapters themselves, so even a directly constructed backend (tests, third-party
+ * embedders) can never reach the network. As an ops/test kill switch it intentionally overrides the
+ * configured/passed `autoDownload`.
+ *
+ * Three-valued on purpose: either switch `0`/`false` → `false`; the project switch `1`/`true` →
+ * `true`; otherwise (including the family switch merely being `1`, which only means "the family did
+ * not disable anything") → `undefined`, so the configured value stands. The family switch is a
+ * master GATE, not a force-on: it must be able to stop the runtime, never to override
+ * `semantic.auto_download: false`.
+ */
+export declare function envAutoDownload(): boolean | undefined;

package/lib/engine/mem-contract/family.d.ts

@@ -0,0 +1,25 @@
+/**
+ * The family root and the two managed directories under it — ONE definition of a convention that
+ * four packages would otherwise each spell out.
+ *
+ * `@avantf/dsh-plugin-base` (the family's environment framework) installs everything it manages under a
+ * single root: `$AVANTF_HOME` when set, else `~/.avantf/env`. `tools` holds external binaries
+ * (pandoc), `models` the downloaded embedding/rerank weights in the flat layout the transformers.js
+ * cache reads. The engine's built-in defaults point HERE, not at the pre-framework
+ * `~/.avantf/{tools,models}`: those legacy directories are no longer a fallback anywhere, so a
+ * machine that never creates them loses nothing, and no process resolves a path that only missions
+ * because somebody left a compatibility symlink behind.
+ *
+ * `provision` mirrors {@link familyHome} (it is deliberately dependency-free, so it cannot import
+ * this module); `packages/core/test/family_paths.spec.ts` pins the two copies together so the
+ * convention cannot drift.
+ *
+ * The `~/` expansion is NOT a private copy any more: this module and `env.ts` used to carry two
+ * literally identical `expandTilde`/`expandHome` bodies in the same package, and only the exported
+ * one was pinned — so the un-pinned copy could drift silently. It now calls {@link expandHome}.
+ */
+export declare function familyHome(env?: Record<string, string | undefined>): string;
+/** The managed external-binary root (`<family root>/tools`). */
+export declare function familyToolsDir(env?: Record<string, string | undefined>): string;
+/** The managed model-cache root (`<family root>/models`). */
+export declare function familyModelsDir(env?: Record<string, string | undefined>): string;

package/lib/engine/mem-contract/index.d.ts

@@ -0,0 +1,8 @@
+export * from './types.js';
+export * from './config.js';
+export * from './family.js';
+export * from './tools.js';
+export * from './remote.js';
+export * from './log.js';
+export * from './env.js';
+export * from './text.js';

package/lib/engine/mem-contract/log.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Minimal leveled logger shared by the engine, the retrieval core, and the DSH
+ * plugin. The plugin injects the host `ctx.logger` facade so every line lands in
+ * the DSH host output; standalone runs (CLI / MCP / tests) fall back to console.
+ *
+ * The console fallback writes to STDERR on purpose: the CLI and the MCP server
+ * own stdout for JSON / JSON-RPC, so logs must never share that stream.
+ */
+/** One leveled log sink. */
+export interface AvantfLogger {
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+}
+/** Build a stderr-backed logger with a stable `[avantf-mem]` prefix. */
+export declare function createConsoleLogger(prefix?: string): AvantfLogger;
+/** The default console logger used whenever no host logger is supplied. */
+export declare const defaultLogger: AvantfLogger;
+/**
+ * Flatten an error and its `cause` chain into one log-safe line.
+ *
+ * Node's `fetch` (undici) reports any failed request as a bare
+ * `TypeError: fetch failed` and puts the actionable detail — DNS failure,
+ * connect timeout, TLS error — plus the `UND_ERR_*` code on `error.cause`.
+ * Logging `message` alone therefore hides *why* a model download failed: a
+ * poisoned DNS entry and a dead mirror both read as "fetch failed". Every
+ * adapter / bootstrap log goes through this helper so the cause survives.
+ */
+export declare function describeError(error: unknown): string;

package/lib/engine/mem-contract/remote.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Transport-shape helpers shared by the DSH Remote host gateway (@avantf/dsh-mem)
+ * and its client pages.
+ *
+ * There are TWO nested envelopes on the client↔host Remote path:
+ *
+ *  1. the **transport** envelope, added by DSH itself: every `@Remote` method call
+ *     resolves to `{ok:true, value:<host method return>}` (or `{ok:false, error}`
+ *     on a carrier/host failure) — see the harness Typert client
+ *     (`packages/api/gateway/src/client/index.ts`). The host method must NOT add
+ *     this layer itself.
+ *  2. the **application** envelope this plugin's gateway returns deliberately, so
+ *     the settings pages can show contract-validation violations instead of a raw
+ *     exception: `{ok:true, value}` / `{ok:false, error, violations}`.
+ *
+ * `unwrapRemoteEnvelope` peels both, in order, and also tolerates a bare payload
+ * (a gateway that returns raw values, or a future protocol change), so the pages
+ * never render an envelope as if it were data.
+ *
+ * The application envelope also carries the **wire revision** (`wire`, optional): the host stamps
+ * the revision of the Remote FACE it publishes onto every answer (see `@avantf/dsh-mem`'s
+ * `remote.ts` `WIRE_VERSION`), and the client half reads it back to tell a rebuilt browser bundle
+ * from the host process that `dsh web` loaded before it. It rides the envelope rather than one
+ * method's payload because this plugin has no single entry point — each tab makes its own first
+ * call — while the envelope is the one object every answer shares. Absent means "a host older
+ * than the marker": still readable, but the client may not assume a method it knows about exists.
+ */
+/** Successful application result. */
+export interface RemoteOk<T> {
+    ok: true;
+    value: T;
+    /** The publisher's wire revision, when it reports one (see the module doc). */
+    wire?: number;
+}
+/** Failed application result (validation violations are optional). */
+export interface RemoteErr {
+    ok: false;
+    error: string;
+    violations?: string[];
+    /** The publisher's wire revision, when it reports one (see the module doc). */
+    wire?: number;
+}
+export type RemoteEnvelope<T> = RemoteOk<T> | RemoteErr;
+/** Best-effort human-readable text for any error carrier (string, Error, object). */
+export declare function remoteErrorText(error: unknown): string;
+/**
+ * Decode one Remote call result into the application envelope.
+ *
+ * @param result - whatever the client transport resolved with.
+ * @returns the application payload, or the application error + violations.
+ */
+export declare function unwrapRemoteEnvelope<T>(result: unknown): RemoteEnvelope<T>;
+/** Render a violation list for display (used by the settings pages). */
+export declare function formatViolations(violations: string[] | undefined): string;
+/**
+ * The envelope every AGENT-facing tool result uses, on BOTH surfaces: the DSH
+ * `defineTool` tools (`@avantf/dsh-mem`) and the standalone MCP server
+ * (`@avantf/mem-mcp`). Keeping one shape means a model/consumer sees the same
+ * response whether it talks to the plugin or to the MCP process.
+ *
+ * Note the deliberate difference from {@link RemoteEnvelope}: the payload key is
+ * `result` (tools) vs `value` (settings-page Remote calls).
+ */
+export type ToolEnvelope<T> = {
+    ok: true;
+    result: T;
+} | {
+    ok: false;
+    error: string;
+    violations?: string[];
+};
+/** Wrap a successful tool payload (a null/undefined payload becomes explicit null). */
+export declare function toolOk<T>(result: T): ToolEnvelope<T | null>;
+/** Wrap a failed tool call (message + optional contract violations). */
+export declare function toolErr(error: unknown, violations?: string[]): ToolEnvelope<never>;
+/** Contract-validation failure message shared by both tool surfaces. */
+export declare function validationError(toolName: string, issues: readonly {
+    path: PropertyKey[];
+    message: string;
+}[]): ToolEnvelope<never>;

package/lib/engine/mem-contract/text.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Well-formedness of every string mem stores or shows a model — the mem-side MIRROR.
+ *
+ * The defect this module exists for is ONE shape: a lone surrogate — a UTF-16 code unit in the
+ * `D800–DFFF` range with no partner. A string carrying one is not a valid Unicode scalar sequence,
+ * and `JSON.stringify` happily emits it as the escape `"\ud800"`: JavaScript's own `JSON.parse`
+ * accepts that, so the defect passes every check inside this repository, while a strict parser
+ * (Rust's `serde_json`, JSON Schema validators, most non-JS consumers) rejects the whole document.
+ * The same half-code-unit also poisons FTS, embeddings and the UI. `String.prototype.toWellFormed`
+ * (Node ≥20, present in this repo's Node ≥22.15) replaces each lone surrogate with U+FFFD, which is
+ * exactly the repair wanted.
+ *
+ * ── THE ORIGINAL IS THE BASE KIT ──────────────────────────────────────────────────────────────
+ *
+ * `base/plugin-base/src/kit/wellformed.ts` owns `wellFormedText` / `wellFormedDeep` (interface v2)
+ * and is the copy ONE base release fixes for the whole family. This module is its **deliberate
+ * mirror**, in the same sense as `./family.ts` (whose original is the base's `kit/family.ts`): the
+ * engine (`@avantf/mem`) and the CLI / MCP faces have **no DSH host and never load the base**
+ * (`AGENTS.md`), so they cannot reach the original and keep a dependency-free copy as their
+ * documented degradation path. The cross-tree test
+ * `mem/packages/plugin/test/wellformed_pin.spec.ts` takes the REAL linked base and compares it with
+ * this mirror on one set of samples (the precedent is `family_pin.spec.ts`); **do not change one
+ * side without the other**.
+ *
+ * ── WHO RUNS: the base when a host installed it, the mirror otherwise ─────────────────────────
+ *
+ * This module is also the ONE place the DSH plugin can hand the base's implementation in:
+ * {@link adoptWellFormed} takes the module the bootstrap loaded and swaps the two functions for the
+ * base's, falling back per member when it does not carry them. The switch is deliberately
+ * **"does the loaded module HAVE the function"** (a `typeof` read of each member, guarded against a
+ * hostile getter) — never an interface-generation check, so a base that predates the v2 helpers
+ * (generation v1) simply keeps the mirror and the plugin still mounts completely. Nothing here can
+ * throw or refuse a mount.
+ *
+ * Two directions, two functions, deliberately NOT one:
+ *
+ *  - {@link toWellFormedText} is the WRITE side. It additionally applies `normalize('NFC')`, because
+ *    Unicode equivalence is a persistence concern: two spellings of the same text must not become
+ *    two rows, two FTS terms or two entities. Every store write and every derived string (entity
+ *    names, SPO slots) goes through it BEFORE it is persisted. **NFC is mem's policy, not the
+ *    well-formed layer's**: the base's `wellFormedText` does NOT normalize, and when it is adopted
+ *    the write wrapper here is exactly `base.wellFormedText(value).normalize('NFC')`.
+ *  - {@link toWellFormedDeep} is the READ side. It never changes anything except lone surrogates,
+ *    and it walks the result value so that data this process did not just write — a row written by
+ *    an older build or a foreign driver, a string the tokenizer derived — still serializes to JSON
+ *    every strict parser accepts. Applying NFC here would silently rewrite history for display, so
+ *    it does not.
+ *
+ * @module contract/text
+ */
+/**
+ * The ONE write-side normalization: well-formed, and NFC.
+ *
+ * Called at the store write entries (never at each individual string field), so a fix here lands
+ * everywhere at once.
+ */
+export declare function toWellFormedText(value: string): string;
+/** {@link toWellFormedText} over a batch (entity names, caller paths, …). */
+export declare function toWellFormedTexts(values: readonly string[]): string[];
+/**
+ * The read-side twin: a structurally identical value in which every string is well-formed.
+ *
+ * Delegates to {@link repairDeep}: the local {@link mirrorDeep} until a host installs the base's
+ * `wellFormedDeep` through {@link adoptWellFormed}, which is the same walk (see the module note).
+ */
+export declare function toWellFormedDeep<T>(value: T): T;
+/** The slice of a loaded base {@link adoptWellFormed} reads; both members are optional. */
+export interface WellFormedKit {
+    readonly wellFormedText?: unknown;
+    readonly wellFormedDeep?: unknown;
+}
+/** Which of the two repairs came from the kit (`false` = the mirror is in force for it). */
+export interface WellFormedSource {
+    readonly text: boolean;
+    readonly deep: boolean;
+}
+/**
+ * Prefer the loaded base's well-formed helpers; fall back to this mirror for whatever it lacks.
+ *
+ * The plugin calls this ONCE at mount with the module the bootstrap loaded (`env?.kit`). The
+ * decision per member is a `typeof` check — the v2 members are an OPTIONAL capability, and a base
+ * that predates them (interface v1) or a module with none of them leaves both repairs on the mirror.
+ * It is total: a hostile module reads as "reports none", nothing throws, and a mount is never
+ * refused or changed by the outcome.
+ *
+ * Calling it AGAIN replaces whatever was installed before, so passing `undefined` restores the
+ * mirror (the CLI / MCP faces never call it at all and keep the mirror for their whole life).
+ *
+ * @param kit - the loaded base module (or anything else; only the two members are read).
+ * @returns which repair each side came from, for the caller's log line.
+ */
+export declare function adoptWellFormed(kit: unknown): WellFormedSource;