@avantf/dsh-mem 0.3.0 → 0.4.0

package/lib/engine/mem-retrieval/rerank.d.ts

@@ -0,0 +1,13 @@
+import type { Reranker } from './interfaces.js';
+/**
+ * The one rerank-and-reorder pass shared by every retrieval leg (memory search,
+ * knowledge search — the blocks were byte-identical and had already drifted once
+ * on the miss-sort sentinel).
+ *
+ * Contract: `reranker.rerank` returns candidate ids best-first; ids it omits sort
+ * last with their fused order preserved (stable sort). A disabled/unavailable
+ * reranker is a pass-through.
+ */
+export declare function rerankHits<T extends {
+    id: number;
+}>(reranker: Reranker, query: string, hits: T[], textOf: (id: number) => string): Promise<T[]>;

package/lib/engine/mem-retrieval/stats.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Process-wide retrieval health counters (DESIGN §20).
+ *
+ * What this answers, and why it is a counter rather than a query: "is the semantic leg
+ * actually live?", "how often does a query return nothing?", "how slow is retrieval?", and
+ * "did any budget truncate model-facing text?". None of those can be recovered from the
+ * database after the fact — the DB only holds what was stored, not how it was retrieved.
+ *
+ * In-process like {@link retrievalLogger}, with `snapshot()`/`restore()` so the STORE can
+ * persist the aggregate through `avantf_stats`: a restart resets counters in the reference
+ * design, which makes a long-lived degradation look like a fresh healthy process. The
+ * counters themselves stay free of any DB dependency (retrieval-core has none).
+ *
+ * The SHAPES live in `@avantf/mem-contract` (they are UI payloads — the settings page renders
+ * them), so this module only owns the counting.
+ */
+import type { RetrievalFloorDrops, RetrievalHealth, RetrievalHealthSummary } from '../mem-contract/index.js';
+export interface RetrievalEvent {
+    /** `memory` | `knowledge` | `cross` — free-form so a new store needs no schema change. */
+    kind: string;
+    results: number;
+    latencyMs: number;
+    semanticLive: boolean;
+    rerankUsed?: boolean;
+    rerankFallback?: boolean;
+    /**
+     * Per-leg candidates removed by the relevance floors before fusion. The per-leg shape is what
+     * makes an empty result readable ("the FTS floor cut 7 rows" vs "no leg had a candidate"); the
+     * aggregate counter below is what survives the process.
+     */
+    droppedByFloor?: RetrievalFloorDrops;
+}
+/** Record one completed retrieval. */
+export declare function recordRetrieval(event: RetrievalEvent): void;
+export type TruncationKind = 'embedding' | 'rerank' | 'output';
+/**
+ * Record that a retrieval LEG returned exactly its cap, i.e. that its tail was dropped.
+ *
+ * The cap is a cost bound, and `fusion` scaling by each leg's maximum means a trimmed tail cannot
+ * rescale the survivors — but "cannot distort" is not "did not bind": past the headroom a leg's
+ * weakest entries simply cannot reach the result, and nothing else reports that. `size === cap` is
+ * the same proxy the lifecycle tick uses for "the budget ran out" (a leg that finished under the
+ * cap cannot have been cut).
+ */
+export declare function recordLegCapped(): void;
+/**
+ * Record that a budget had to bound model-facing text.
+ *
+ * This counter is the point of the whole module: the pre-existing behavior truncated
+ * embedding input silently, so nothing anywhere could tell that a store had been indexed
+ * with half its text missing.
+ */
+export declare function recordTruncation(kind: TruncationKind): void;
+/** A copy safe to hand to a caller (never the live object). */
+export declare function retrievalHealth(): RetrievalHealth;
+export declare function retrievalHealthSummary(): RetrievalHealthSummary;
+/**
+ * Restore a snapshot read back from storage. Deliberately total: a snapshot written by an
+ * older build (or a corrupt row) must not break the diagnostics path, so unknown/missing
+ * numbers fall back to the empty counter instead of throwing.
+ */
+export declare function restoreRetrievalHealth(snapshot: unknown): void;
+/** Clear every counter (tests, and an explicit operator reset). */
+export declare function resetRetrievalHealth(): void;

package/lib/engine/mem-retrieval/text_budget.d.ts

@@ -0,0 +1,52 @@
+/** Window assumed when neither the config nor the loaded model declares one. */
+export declare const DEFAULT_MODEL_WINDOW = 512;
+/** Roughly how many tokens CJK text yields per character (used by the chunk-size derivation). */
+export declare const CJK_TOKENS_PER_CHAR = 1;
+/**
+ * Estimate the token count of `text`.
+ *
+ * Deliberately an OVER-estimate for mixed text: under-estimating is what silently drops
+ * content past the window, while over-estimating only truncates slightly earlier.
+ */
+export declare function estimateTokens(text: string): number;
+export interface TruncatedText {
+    /** The bounded text (equals the input when it already fit). */
+    text: string;
+    truncated: boolean;
+    /** Estimated tokens dropped, for the log line. */
+    omittedTokens: number;
+}
+/**
+ * Bound `text` to `maxTokens`, preferring a sentence/clause boundary near the cut and
+ * appending `marker` so the truncation is VISIBLE rather than silent.
+ *
+ * The marker's own cost is reserved out of the budget, and the result is verified against
+ * `maxTokens` before returning (the estimator rounds up per character class, so a naive
+ * character walk can land one token over).
+ */
+export declare function truncateToTokens(text: string, maxTokens: number, opts?: {
+    marker?: string;
+    estimate?: (text: string) => number;
+}): TruncatedText;
+/**
+ * Resolve the effective input window for a loaded model.
+ *
+ * @param configured - `0` (or negative) means "auto: use the model's declared window".
+ *                     A positive value is an operator cap and is CLAMPED to the declared
+ *                     window when the model declares one, so `max_input_tokens` can never
+ *                     push text past what the model can actually read.
+ * @param declared - `model_max_length` reported by the loaded tokenizer, if any.
+ * @param fallback - used when neither is available.
+ */
+export declare function resolveWindow(configured: number, declared: number | undefined, fallback?: number): number;
+/**
+ * Read the declared window off a transformers.js pipeline. The pipeline is typed as a bare
+ * function in the adapters, so this probes defensively and treats a sentinel as absent.
+ */
+export declare function declaredWindowOf(pipe: unknown): number | undefined;
+/**
+ * Characters of a chunk that fit the window for `window` tokens, minus the per-chunk
+ * overhead (`specialTokens` for CLS/SEP). Used to derive `knowledge.chunk_size` so the
+ * default chunking is justified by the shipped model's window instead of by feel.
+ */
+export declare function windowChars(window: number, specialTokens?: number): number;