@avantf/dsh-mem 0.8.0 → 0.9.0

package/lib/engine/mem-retrieval/interfaces.d.ts

@@ -19,8 +19,9 @@ export interface SemanticRepresentation {
     /** Whether vectors are L2-normalized by the pipeline (cosine relies on it). */
     readonly normalize: boolean;
     /**
-     * The CONFIGURED input-token cap (`0` = auto/declared window), i.e. the truncation window the
-     * adapter bounds text to before encoding. The CONFIGURED value, not the resolved one: the
+     * The CONFIGURED input-token cap (`0` = auto/declared window), i.e. the input window the
+     * adapter bounds text to before encoding (an over-window text is split into windows and pooled —
+     * `text_budget.splitToWindows` — so the window bounds the INPUT, it no longer drops its tail). The CONFIGURED value, not the resolved one: the
      * resolved window depends on the loaded tokenizer, so it is unknown before warmup — and the
      * fingerprint must be identical before and after warmup, or a store would declare its own rows
      * stale the moment the model finished loading.
@@ -56,6 +57,15 @@ export interface SemanticBackend {
      * floor. Implementations that load lazily may omit it.
      */
     ensureWarm?(): void;
+    /**
+     * Optional: point the backend at an external module root for its native runtime dependency.
+     *
+     * The DSH plugin sets this to the base provider's shared `runtime-deps` root once that item is
+     * ready, so a package the plugin's own tree does not contain can still be loaded from the
+     * content-addressed store. Called BEFORE `warmUp()`; a backend that resolves its dependency
+     * statically may omit it, and one whose store root is absent keeps its normal resolution.
+     */
+    useModuleDir?(dir: string): void;
 }
 /** Distance/ann index over entry vectors (per-store instance). */
 export interface VectorStore {
@@ -98,6 +108,16 @@ export interface VectorStore {
      * snapshot built from another space. Best-effort: a missing/corrupt/stale file is rebuilt.
      */
     attachPersistence?(path: string): void;
+    /**
+     * Optional: point the backend at an external module root for its NATIVE dependency.
+     *
+     * The DSH plugin sets this to the base provider's shared `runtime-deps` root once the native
+     * self-heal item is ready, so `hnswlib-node` can be loaded from the content-addressed store. Unlike
+     * `SemanticBackend.useModuleDir` it may arrive AFTER construction (the item settles in the
+     * background): an implementation whose native side already failed to load must retry from the new
+     * root rather than stay on its fallback. A backend with no native dependency omits it.
+     */
+    useModuleDir?(dir: string): void;
     /**
      * Optional: persist any state that is only in memory. Called on shutdown; implementations that
      * have nothing pending may omit it.

package/lib/engine/mem-retrieval/text_budget.d.ts

@@ -28,6 +28,37 @@ export declare function truncateToTokens(text: string, maxTokens: number, opts?:
     marker?: string;
     estimate?: (text: string) => number;
 }): TruncatedText;
+/** A model-facing text sliced into window-sized pieces (see {@link splitToWindows}). */
+export interface WindowedText {
+    /** The windows, in order. Concatenated they ARE the input: nothing is dropped. */
+    windows: string[];
+    /** Estimated tokens of the whole input. */
+    tokens: number;
+    /** Estimated tokens of each window, aligned with {@link windows} (the pooling weight). */
+    windowTokens: number[];
+}
+/**
+ * Slice `text` into windows of at most `maxTokens` estimated tokens instead of truncating it.
+ *
+ * WHY THIS EXISTS. The window belongs to the loaded model and the call site cannot widen it, so a
+ * text past it has exactly one of two fates: cut (what {@link truncateToTokens} does, and what
+ * transformers.js does silently on its own) or split. A knowledge chunk of ~1000 characters against
+ * a 512-token model lost half its content from the vector that way — measured in production as
+ * "503 tokens dropped here". Splitting keeps every token in the representation; the adapter that
+ * can only return ONE vector then pools the window vectors (`LocalBgeBackend.encode`).
+ *
+ * Every returned window satisfies `estimate(window) <= maxTokens` (a one-character window is the
+ * floor; the estimator cannot exceed a limit of 1 token for a single character). Cuts land on a
+ * sentence/clause boundary when one is in the window's final quarter, so a window does not start or
+ * end mid-word.
+ *
+ * @param text - the input.
+ * @param maxTokens - the per-window budget (already net of special tokens, as the adapter computes it).
+ * @param opts - estimator override (tests use a flat one); defaults to {@link estimateTokens}.
+ */
+export declare function splitToWindows(text: string, maxTokens: number, opts?: {
+    estimate?: (text: string) => number;
+}): WindowedText;
 /**
  * Resolve the effective input window for a loaded model.
  *