@avantf/dsh-mem 0.8.1 → 0.10.0

package/lib/engine/mem-retrieval/adapters/auto_vstore.d.ts

@@ -39,6 +39,16 @@ export declare class AutoVectorStore implements VectorStore {
     /** Build the ANN index now if one is owed (see the class comment). */
     prepare(): void;
     attachPersistence(path: string): void;
+    /**
+     * Point the ANN side at a shared module root (the plugin's native self-heal item) — for THIS store
+     * when it already migrated, and for the one it may migrate to later.
+     *
+     * The delegate call covers the first case; the module-level root (set by
+     * `HnswlibVectorStore.useModuleDir` / `useHnswlibModuleDir`) is what a FUTURE
+     * `new HnswlibVectorStore` in `maybeUpgrade` reads. Below the threshold nothing changes: an
+     * unmigrated store keeps serving from brute force, which is the intended backend at that size.
+     */
+    useModuleDir(dir: string): void;
     flush(): void;
     /** Reclaim dead entries in the migrated backend (the numpy fallback has none). */
     compact(): void;

package/lib/engine/mem-retrieval/adapters/hnswlib.d.ts

@@ -1,4 +1,13 @@
 import type { VectorStore } from '../interfaces.js';
+/**
+ * Point the loader at a shared module root (the base `runtime-deps` handle's directory).
+ *
+ * The cache follows the ROOT: a change forgets what was loaded, because the next `require` must come
+ * from the new root (that is the whole point of setting one). It costs nothing in practice — Node's
+ * own `require` cache makes re-entering the same file cheap, and a live native index is untouched
+ * (see `HnswlibVectorStore.useModuleDir`, which only rebuilds when it has no native index at all).
+ */
+export declare function useHnswlibModuleDir(dir: string | undefined): void;
 /**
  * Search-time beam width, when the config does not say otherwise.
  *
@@ -63,6 +72,25 @@ export declare class HnswlibVectorStore implements VectorStore {
     private restores;
     private rebuilds;
     constructor(dim: number, efSearch?: number);
+    /**
+     * Build a native index at `max` capacity, or `null` when the binding cannot be loaded.
+     *
+     * One place for "make an index": the constructor, `reindex()` and `rebuild()` each used to spell
+     * the `require` out themselves, which is how a module root set after construction could never
+     * reach them. `null` is the caller's signal to serve from {@link fallback} — a missing binding
+     * degrades, it never throws.
+     */
+    private openNative;
+    /**
+     * Point this store's native side at a shared module root and, when the binding was unavailable
+     * until now, rebuild the index from the tracked vectors under it.
+     *
+     * This is the case the whole hook exists for: `mem` builds its vector store during mount, while the
+     * native self-heal item is still installing, so the constructor may have fallen back. When the item
+     * settles, the plugin calls this — and a store that already serves natively is left alone (the
+     * binding is the same one).
+     */
+    useModuleDir(dir: string): void;
     add(id: number, vec: Float32Array): void;
     /** Whether the real hnswlib native index is active (false = brute-force fallback). */
     get native(): boolean;

package/lib/engine/mem-retrieval/adapters/local_bge.d.ts

@@ -53,6 +53,16 @@ export interface ModelEnv {
      * the eager constructor warm.
      */
     deferWarm?: boolean;
+    /**
+     * Module root to load `@huggingface/transformers` from, when it is not this tree's own optional
+     * dependency. The DSH plugin sets it (through {@link LocalBgeBackend.useModuleDir}) to the base
+     * provider's shared `runtime-deps` root after the semantic self-heal item is ready.
+     *
+     * Empty/absent = the pre-existing resolution: `import('@huggingface/transformers')` from this
+     * module, i.e. the optional dependency or nothing. A root that no longer resolves falls back to
+     * that path with one warning — a deleted store must not be worse than no self-heal.
+     */
+    moduleDir?: string;
 }
 /**
  * Local BGE embedder via `@huggingface/transformers` (transformers.js, ONNX).
@@ -101,6 +111,8 @@ export declare class LocalBgeBackend implements SemanticBackend {
      * the weights). The resolved representation is CACHED, so the warning is one line per process.
      */
     representation(): SemanticRepresentation;
+    /** Point the loader at a shared runtime-dependency root before the first warmup. */
+    useModuleDir(dir: string): void;
     /**
      * Fire-and-forget retry for call sites that only observe availability. No-op
      * while the model is ready or an attempt is already in flight; otherwise
@@ -119,14 +131,44 @@ export declare class LocalBgeBackend implements SemanticBackend {
      */
     private window;
     /**
-     * Bound `text` to the window BEFORE it reaches transformers.js.
+     * The window-sized slices `text` is encoded in (see `text_budget.splitToWindows`).
+     *
+     * WHERE THE BOUNDARY CAN LIVE. The feature-extraction pipeline's `_call` accepts
+     * `{pooling, normalize, quantize, precision}` and nothing else, so `max_length` cannot be raised
+     * from the call site — past `model_max_length` transformers.js truncates and says nothing. This
+     * adapter is therefore the only place that can apply the model's own window, and it SPLITS rather
+     * than cuts: a text past the window is a document chunk (or a long fact) whose tail is content,
+     * and cutting it dropped 503 of ~1015 tokens out of the vector in production.
+     */
+    private windows;
+    /**
+     * Report an input that needed more than one window — once per process, and as an INFO line.
+     *
+     * Not a warning any more: nothing is lost (every window is pooled into the one vector), so the
+     * line is a tuning hint, not a degradation. It carries the actionable number: `chunk_size` is
+     * measured in CHARACTERS and {@link windowChars} converts the model's token window into exactly
+     * that, which is the thing an operator (or a chunker's default) has to change to keep one window
+     * per input. `semantic.max_input_tokens` is deliberately NOT suggested: `resolveWindow` clamps it
+     * to the model's declared window, so raising it cannot buy a single extra token.
+     */
+    private reportSplit;
+    /** The configured width's vector: model output padded/truncated, with the one-shot dim warning. */
+    private toVector;
+    /**
+     * ONE vector for several windows: token-weighted mean of the window vectors, re-normalized when the
+     * representation normalizes.
      *
-     * This is the only place the guard can live: the feature-extraction pipeline's `_call`
-     * accepts `{pooling, normalize, quantize, precision}` and nothing else, so `max_length`
-     * cannot be raised or lowered from the call site — past `model_max_length` the tokenizer
-     * truncates and says nothing.
+     * Weighted rather than a plain mean because the last window is usually short, and a plain mean
+     * would over-represent a fragment. Mean-of-normalized-vectors then re-normalized is the mean
+     * DIRECTION of the windows — the same coordinate space the model's own mean pooling produces, so
+     * `representationKey` (`pooling`/`normalize`/window) does not move and stored vectors stay
+     * comparable with the ones this writes.
      */
-    private bound;
+    private pool;
+    /** Encode the windows of one text in ONE model call, falling back to one call per window. */
+    private encodeWindows;
+    /** Encode one plan: a single window verbatim, several pooled. */
+    private encodePlan;
     encode(text: string): Promise<Float32Array>;
     /**
      * Encode several texts in ONE model call.

package/lib/engine/mem-retrieval/interfaces.d.ts

@@ -19,8 +19,9 @@ export interface SemanticRepresentation {
     /** Whether vectors are L2-normalized by the pipeline (cosine relies on it). */
     readonly normalize: boolean;
     /**
-     * The CONFIGURED input-token cap (`0` = auto/declared window), i.e. the truncation window the
-     * adapter bounds text to before encoding. The CONFIGURED value, not the resolved one: the
+     * The CONFIGURED input-token cap (`0` = auto/declared window), i.e. the input window the
+     * adapter bounds text to before encoding (an over-window text is split into windows and pooled —
+     * `text_budget.splitToWindows` — so the window bounds the INPUT, it no longer drops its tail). The CONFIGURED value, not the resolved one: the
      * resolved window depends on the loaded tokenizer, so it is unknown before warmup — and the
      * fingerprint must be identical before and after warmup, or a store would declare its own rows
      * stale the moment the model finished loading.
@@ -56,6 +57,15 @@ export interface SemanticBackend {
      * floor. Implementations that load lazily may omit it.
      */
     ensureWarm?(): void;
+    /**
+     * Optional: point the backend at an external module root for its native runtime dependency.
+     *
+     * The DSH plugin sets this to the base provider's shared `runtime-deps` root once that item is
+     * ready, so a package the plugin's own tree does not contain can still be loaded from the
+     * content-addressed store. Called BEFORE `warmUp()`; a backend that resolves its dependency
+     * statically may omit it, and one whose store root is absent keeps its normal resolution.
+     */
+    useModuleDir?(dir: string): void;
 }
 /** Distance/ann index over entry vectors (per-store instance). */
 export interface VectorStore {
@@ -98,6 +108,16 @@ export interface VectorStore {
      * snapshot built from another space. Best-effort: a missing/corrupt/stale file is rebuilt.
      */
     attachPersistence?(path: string): void;
+    /**
+     * Optional: point the backend at an external module root for its NATIVE dependency.
+     *
+     * The DSH plugin sets this to the base provider's shared `runtime-deps` root once the native
+     * self-heal item is ready, so `hnswlib-node` can be loaded from the content-addressed store. Unlike
+     * `SemanticBackend.useModuleDir` it may arrive AFTER construction (the item settles in the
+     * background): an implementation whose native side already failed to load must retry from the new
+     * root rather than stay on its fallback. A backend with no native dependency omits it.
+     */
+    useModuleDir?(dir: string): void;
     /**
      * Optional: persist any state that is only in memory. Called on shutdown; implementations that
      * have nothing pending may omit it.

package/lib/engine/mem-retrieval/text_budget.d.ts

@@ -28,6 +28,37 @@ export declare function truncateToTokens(text: string, maxTokens: number, opts?:
     marker?: string;
     estimate?: (text: string) => number;
 }): TruncatedText;
+/** A model-facing text sliced into window-sized pieces (see {@link splitToWindows}). */
+export interface WindowedText {
+    /** The windows, in order. Concatenated they ARE the input: nothing is dropped. */
+    windows: string[];
+    /** Estimated tokens of the whole input. */
+    tokens: number;
+    /** Estimated tokens of each window, aligned with {@link windows} (the pooling weight). */
+    windowTokens: number[];
+}
+/**
+ * Slice `text` into windows of at most `maxTokens` estimated tokens instead of truncating it.
+ *
+ * WHY THIS EXISTS. The window belongs to the loaded model and the call site cannot widen it, so a
+ * text past it has exactly one of two fates: cut (what {@link truncateToTokens} does, and what
+ * transformers.js does silently on its own) or split. A knowledge chunk of ~1000 characters against
+ * a 512-token model lost half its content from the vector that way — measured in production as
+ * "503 tokens dropped here". Splitting keeps every token in the representation; the adapter that
+ * can only return ONE vector then pools the window vectors (`LocalBgeBackend.encode`).
+ *
+ * Every returned window satisfies `estimate(window) <= maxTokens` (a one-character window is the
+ * floor; the estimator cannot exceed a limit of 1 token for a single character). Cuts land on a
+ * sentence/clause boundary when one is in the window's final quarter, so a window does not start or
+ * end mid-word.
+ *
+ * @param text - the input.
+ * @param maxTokens - the per-window budget (already net of special tokens, as the adapter computes it).
+ * @param opts - estimator override (tests use a flat one); defaults to {@link estimateTokens}.
+ */
+export declare function splitToWindows(text: string, maxTokens: number, opts?: {
+    estimate?: (text: string) => number;
+}): WindowedText;
 /**
  * Resolve the effective input window for a loaded model.
  *