@avantf/dsh-mem 0.3.0 → 0.4.0

package/lib/engine/mem/store/doc_files.d.ts

@@ -0,0 +1,125 @@
+/** The frontmatter a managed file carries; `content_hash` is the body AS INGESTED. */
+interface DocFileMeta {
+    doc_id: number;
+    domain: string;
+    source: string;
+    title: string;
+    source_uri: string | null;
+    ingested_at: string;
+    content_hash: string;
+    /**
+     * The `@avantf/mem-convert` converter that produced the body, when the source was converted
+     * (absent for text and PDF, and for every file written before this field existed — readers must
+     * treat it as optional, which is why it is not `null`).
+     */
+    converter?: string;
+}
+/** One managed file as the file system sees it now. */
+interface DocFileState {
+    path: string;
+    stale: boolean;
+    missing: boolean;
+}
+/** A parsed managed file: frontmatter (possibly partial — a hand-written file may omit it) + body. */
+interface ParsedDocFile {
+    meta: Partial<DocFileMeta>;
+    body: string;
+}
+/** One path segment, made safe and human-readable; `fallback` covers names that sanitize away. */
+export declare function sanitizeSegment(raw: string, fallback: string): string;
+/**
+ * A stable title for a body that arrived as TEXT (a paste), where there is no file name to borrow.
+ *
+ * Pasted text has no `source_uri` to name it after, and `source` defaults to the same value for
+ * every paste — so without this, every untitled paste into one domain collapsed onto one identity
+ * and the second ingest silently replaced the first. The rule: the first Markdown ATX heading,
+ * else the first non-empty line with its block markers stripped.
+ *
+ * Fenced code is stepped over in BOTH halves: `#include` is not a heading, and a fence opener is
+ * not a title. `MAX_SEGMENT` is the same cap the file name uses, so a long line cannot produce a
+ * title the managed path has to truncate differently. Empty input falls back to `untitled`, and the
+ * result depends only on the text (same body → same title), which is what keeps a re-paste
+ * landing on the SAME identity instead of quietly creating a sibling every time.
+ */
+export declare function deriveDocTitle(text: string): string;
+/** sha256 of a document body — the staleness key. */
+export declare function bodyHash(body: string): string;
+/**
+ * Render one managed file: frontmatter (JSON-encoded values, so any title/URI survives) + body.
+ *
+ * The body is written VERBATIM — no trailing newline is added. `content_hash` records the ingested
+ * text, and `state` compares it against the file's body, so anything this function normalizes
+ * would make every document read as edited the moment it was written.
+ *
+ * An `undefined` value is OMITTED rather than rendered: JSON.stringify(undefined) is the bare token
+ * `undefined`, which the tolerant parser would then hand back as the string "undefined". Optional
+ * keys (today: `converter`) therefore simply do not appear for the documents that lack them.
+ */
+export declare function renderDocFile(meta: DocFileMeta, body: string): string;
+/**
+ * Split one managed file into frontmatter and body.
+ *
+ * Deliberately tolerant: a file without frontmatter (or with a hand-mangled one) is ALL body —
+ * the user's content is never thrown away because a header key is missing, and the caller decides
+ * what an unknown `doc_id` means.
+ */
+export declare function parseDocFile(text: string): ParsedDocFile;
+/**
+ * The managed document directory.
+ *
+ * Paths are `<root>/<domain>/<source>/<title>.md`, sanitized per segment. Collisions resolve by
+ * asking the EXISTING file who it belongs to: a file whose frontmatter names another `doc_id` is
+ * left alone and this document takes the `<title>~<doc_id>.md` variant. Without that check two
+ * documents that sanitize to the same name would take turns clobbering each other.
+ */
+export declare class DocFiles {
+    private readonly root;
+    constructor(root: string);
+    /** The directory holding one document's file (created on write, not here). */
+    dirFor(domain: string, source: string): string;
+    /**
+     * The file NAME for a title, minus the directory: a title that is already a Markdown file name
+     * (`kb ingest --uri notes.md` names the document after the file) must not become `notes.md.md`.
+     */
+    fileName(title: string, suffix?: string): string;
+    /** The base path for a title (before collision handling). */
+    basePath(domain: string, source: string, title: string): string;
+    /**
+     * The file this document owns: the base path, or its `~<doc_id>` variant when the base path is
+     * taken by a DIFFERENT document. Purely derived — no state is written to answer this.
+     */
+    pathFor(docId: number, domain: string, source: string, title: string): string;
+    /** Write the managed file for one document (atomic: temp file + rename). Returns its path. */
+    write(meta: DocFileMeta, body: string): string;
+    /** Parse one managed file, or `null` when it does not exist. */
+    read(path: string): ParsedDocFile | null;
+    /**
+     * A cheap per-file stamp — `mtimeMs:size`, no reads — or `null` when the file is absent.
+     *
+     * This is a TRIGGER, never a judgement: same-second edits, coarse filesystem timestamps and
+     * `cp -p` can all leave it unchanged, so a matching stamp means "probably unchanged" and only
+     * `state()` (which hashes the body) decides. The point is that the common case costs one `stat`
+     * per file instead of reading and hashing every file in the corpus.
+     */
+    stamp(path: string): string | null;
+    /** Every `.md` path under the root. A directory walk WITHOUT reads — no frontmatter, no hashing. */
+    listPaths(): string[];
+    /** Is this document's file there, and does it still match what was ingested? */
+    state(docId: number, domain: string, source: string, title: string): DocFileState;
+    /** Delete this document's file. `false` when there was nothing to delete. */
+    remove(docId: number, domain: string, source: string, title: string): boolean;
+    /**
+     * Every `.md` under the root, with the `doc_id` its frontmatter names (when it names one).
+     * Used to report orphans — files left behind by a document that no longer exists.
+     *
+     * ASYNC and batched: the walk itself is an async recursive `readdir` (a `readdirSync` gathers
+     * every entry in one ~41 ms call at 10k files), and the per-file parse is a `read` — at 10k
+     * documents that is a second full read of the corpus on top of `sync`'s staleness pass, and it
+     * used to be one uninterrupted synchronous block (performance review §7.5 / P6).
+     */
+    scan(): Promise<{
+        path: string;
+        doc_id?: number;
+    }[]>;
+}
+export {};

package/lib/engine/mem/store/document_text.d.ts

@@ -0,0 +1,47 @@
+/** What the first bytes say a document is. */
+type DocumentFormat = 'pdf' | 'binary' | 'text';
+/** How text bytes were decoded. Reported so a guess is visible instead of silent. */
+type DocumentEncoding = 'utf-8' | 'utf-8-bom' | 'utf-16le' | 'utf-16be' | 'gb18030';
+/** One document's indexable text, and where it came from. */
+export interface DocumentText {
+    text: string;
+    /**
+     * `pdf` = extracted text layer; `converter` = a document format turned into Markdown (see
+     * {@link DocumentText.converter}); `text` = decoded bytes (see {@link DocumentText.encoding}).
+     */
+    via: 'text' | 'pdf' | 'converter';
+    encoding?: DocumentEncoding;
+    /** Which `@avantf/mem-convert` converter produced the body (only when `via === 'converter'`). */
+    converter?: string;
+    /** What that conversion could not carry over (dropped media, a truncated sheet, …). */
+    warnings?: string[];
+}
+/** Classify one document by its leading bytes. */
+export declare function sniffFormat(bytes: Uint8Array): {
+    format: DocumentFormat;
+    kind?: string;
+};
+/**
+ * Restore the ideographs pdfjs sometimes reports as compatibility codepoints.
+ *
+ * Some CJK PDFs extract `⽹站` (KANGXI RADICAL NET + 站) instead of `网站` — visually identical,
+ * different codepoints, so a search for the real word misses. Only those three compatibility ranges
+ * are normalized: NFKC over the whole text would also rewrite full-width punctuation (，。！？),
+ * which is real content and must survive verbatim.
+ */
+export declare function normalizeCjkCompatibility(text: string): string;
+/**
+ * The text of one document, whatever its bytes are.
+ *
+ * CONVERSION RUNS FIRST, before the magic-number sniff and therefore before the binary rejection.
+ * docx/xlsx/html/csv are converted to Markdown here, and the order is not cosmetic: docx and xlsx
+ * are ZIPs, so a sniff-first pipeline would file them under "known binary" and refuse them. A
+ * document no converter claims falls through to the text/PDF/binary paths unchanged.
+ *
+ * @param bytes - the whole document (already bounded by the byte cap).
+ * @param label - the path or URL, used verbatim in error messages and as the sniff's extension hint.
+ * @throws when the document is a binary we cannot read, is not decodable, or is a PDF without a
+ * text layer.
+ */
+export declare function documentText(bytes: Uint8Array, label: string): Promise<DocumentText>;
+export {};

package/lib/engine/mem/store/floors.d.ts

@@ -0,0 +1,191 @@
+/**
+ * The relevance floors both stores apply to their legs' OWN raw scores, before `fuse()`.
+ *
+ * WHY PRE-FUSION, AND WHY ABSOLUTE. `retrieval-core/src/fusion.ts` scales each leg by that leg's
+ * MAXIMUM, so every leg's head is 1.0 by construction and the fused number is only comparable
+ * WITHIN one query. A cutoff on the fused score would therefore mean a different thing per query
+ * (and would move whenever a leg's head changed). The legs' raw numbers do have an absolute scale —
+ * cosine for the semantic leg, the Jaccard ratio for the entity leg, "distinct query terms hit" for
+ * FTS — so that is where the floors live. The side benefit is that flooring a leg's tail cannot
+ * move its maximum, so the cap-invariance §20.17 relies on is untouched.
+ *
+ * THREE FLOORS, ONE SHAPE. `min_semantic_similarity` / `min_fts_terms` / `min_jaccard`; each `0`
+ * means "this leg is not gated", and a candidate EQUAL to its floor is KEPT (only strictly-lower
+ * candidates are dropped). The FTS floor is judged PER ROW, not per query: "the store holds this
+ * term somewhere" is the conditional-hint probe's question (`lexical.ts`), and it would let a row
+ * that shares one incidental trigram ride into the result — exactly the low-relevance candidate the
+ * floor exists to remove.
+ *
+ * DEGRADED RELAXATION. With the semantic leg down (`DEGRADED_WEIGHTS`), FTS + entity are the only
+ * evidence a query has, so a two-term bar that the semantic leg used to soften would empty whole
+ * short queries. The FTS floor's effective value drops to 1 in that mode; an explicit `0` stays 0.
+ *
+ * WHY THERE IS A RELAXED PROFILE AT ALL, AND WHY IT IS NOT "NO FLOOR". The floors are calibrated
+ * for paraphrase-style relevance, and there is a band where no threshold separates "answers the
+ * question" from "unrelated": measured on the live store, the fact answering 「我是谁」 scored cosine
+ * 0.444 while four unrelated queries topped out at 0.384 — a 0.06 margin, and the answering fact
+ * was the query's own top-1. So the answer is not a lower default (which would trade precision for
+ * the whole corpus) but a SECOND pass with an absolute bottom line ({@link LOOSE_FLOORS}), run only
+ * when the strict pass returned nothing at all AND the floors are why (`dropped_by_floor > 0`).
+ *
+ * WHICH LEGS A RELAXED PASS LOWERS, AND WHY THAT IS NOT A NO-OP IN THE REPORT. Only the legs the
+ * strict pass actually DROPPED candidates on (see {@link droppedLegs}) are lowered. Lowering a leg
+ * that dropped nothing cannot admit anything — a floor only removes entries — so the chosen
+ * candidates are identical either way; what changes is the `floors` envelope handed back, which
+ * used to claim every leg had been relaxed even when only one was ever the problem. An EXPLICIT
+ * `floors: 'loose'` request has no prior pass to read, so it lowers every gated leg — to the SAME
+ * bottom lines the automatic pass uses, so the explicit entry can never drop below them (it is not
+ * a backdoor; see {@link resolveFloors}).
+ *
+ * WHY NOT EXEMPT `pinned` FACTS FROM THE FLOORS. It was the obvious shortcut and it is wrong: the
+ * exemption only lets a fact into the fusion pool, it does not add score, and `fuse` ranks by each
+ * leg's normalized raw score before slicing to `limit`. So on a query with plenty of strong
+ * candidates the archive sits at the tail and is invisible — but on a weak (or should-be-empty)
+ * query it fills the tail, i.e. every search starts showing the same archive. The effect grows with
+ * the number of pinned facts, and on a small corpus (`overFetch = limit × 5` covers the whole store)
+ * the exemption degenerates into "every pinned fact joins every fusion". A relaxed PASS with a floor
+ * of its own is bounded instead: it costs one extra query, only when the strict answer was empty,
+ * and it can still report "nothing relevant" for a genuinely unrelated question.
+ *
+ * FTS REACHABILITY. `min_fts_terms` counts distinct query terms a row hits, but a query can only
+ * produce so many terms (`relevanceTerms`: latin words ≥5 chars + CJK 3-grams) — a 3-character CJK
+ * query yields exactly ONE trigram, so a configured bar of 2 is unreachable and that leg is empty
+ * for it by construction. The effective bar is therefore `min(configured, termCount)`.
+ *
+ * @module store/floors
+ */
+import type { FloorProfile, RetrievalFloorDrops, RetrievalFloors } from '../../mem-contract/index.js';
+/** The retriever knobs the floors read (a structural type so tests can pass a literal). */
+export interface FloorConfig {
+    min_semantic_similarity: number;
+    min_fts_terms: number;
+    min_jaccard: number;
+}
+/**
+ * The legs that own a floor. The HRR probe has no floor of its own — it shares the Jaccard one — so
+ * it never appears here; a drop on `hrr` relaxes `jaccard` (see {@link droppedLegs}).
+ */
+export type FloorLeg = 'semantic' | 'fts' | 'jaccard';
+/**
+ * What the RELAXED pass lowers each floor TO, as absolute values.
+ *
+ *  - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling (0.384) and the
+ *    pronoun-query top-1 (0.444). Both bounds are load-bearing. Above 0.384, so a genuinely
+ *    unrelated question still comes back EMPTY from the relaxed pass — the four measured samples
+ *    topped out at 0.300/0.384/0.315/0.351, and a 0.35 bar would already admit two of them. Below
+ *    0.444, so 「我是谁」 finds its fact. 0.40 keeps the larger margin on the noise side, which is
+ *    where a false positive costs the caller a wrong answer rather than a missing one.
+ *  - fts: NOT a constant — {@link looseTermFloor}, half the query's distinct terms with a floor of
+ *    1. `1` here is the "no terms to scale by" baseline (the previously constant value); a query
+ *    that cannot produce terms is not graded at all, and one that produces a single term (every
+ *    3-char CJK query) is already graded against that one term by the reachability clamp.
+ *  - jaccard **0.15** (positive — the previous `0` was the ungated hole this closes). Measured, not
+ *    guessed, on two corpora: (a) the live store, 64 active facts, the common-entity questions a
+ *    real user asks about this repo ("插件怎么安装" / "任务怎么拆分" / "知识库在哪里" …) put their
+ *    TOP non-answering entity candidate at Jaccard 0.0909 and most at 0.02–0.07, so 0.15 removes that
+ *    whole tail; (b) the gold-labelled 35-query eval set, where the LOWEST Jaccard any answering pair
+ *    reached is 0.25 (「老王喜欢什么风格」) — a bottom line above 0.25 starts dropping gold pairs,
+ *    while 0.15 filters none. The entity leg cannot fully separate the two classes on short queries
+ *    (a one-entity query against a one-entity fact is 1.0 by construction, and non-gold pairs reach
+ *    0.667), so this is deliberately a "no incidental small overlap" line, not a relevance classifier.
+ */
+export declare const LOOSE_FLOORS: RetrievalFloors;
+/**
+ * The FTS bar the RELAXED pass applies to a query carrying `termCount` distinct terms.
+ *
+ * Positive on purpose: "Relaxed" must never mean "ungated", and the old constant of `1` let a single
+ * incidental trigram become a leg's head (per-leg max normalization makes it 1.0) on any query long
+ * enough to produce several terms. Half the query's terms — `ceil(termCount / 2)`, at least 1 — keeps
+ * a real fraction of what the query asked for while still relaxing the configured bar of 2 to 1 for
+ * a two-term query. `termCount` 0/omitted keeps the baseline of 1 (such a query is not graded by
+ * {@link applyTermFloor} at all). The `min(configured, …)` in {@link resolveFloors} means this can
+ * only relax, never raise.
+ */
+export declare function looseTermFloor(termCount?: number): number;
+/** The floor every leg's drops start from. */
+export declare function emptyFloorDrops(): RetrievalFloorDrops;
+/**
+ * The FTS bar actually applied to a query carrying `termCount` distinct terms.
+ *
+ * A query cannot be graded against more terms than it has (see the module comment: a 3-char CJK
+ * query has one trigram), so the configured bar is CLAMPED to the reachable maximum. `termCount`
+ * `0`/omitted leaves the value alone — such a query is not graded at all by {@link applyTermFloor},
+ * and changing the REPORTED value there would misdescribe what the caller is reading. `0` stays `0`
+ * (the operator's off switch).
+ *
+ * This is the ONE place the clamp lives; `resolveFloors` (what the result reports) and
+ * `applyTermFloor` (what the leg does) both call it, so the number the caller reads is the number
+ * the leg used.
+ */
+export declare function effectiveTermFloor(floor: number, termCount?: number): number;
+/** The per-query floor resolution inputs beyond the configuration. */
+export interface FloorResolution {
+    /** `loose` lowers each configured floor (see {@link LOOSE_FLOORS}); absent means `strict`. */
+    profile?: FloorProfile;
+    /** This query's distinct term count (`relevanceTerms(query).length`), for the FTS clamp. */
+    termCount?: number;
+    /**
+     * Which legs a `'loose'` profile may lower, from {@link droppedLegs} of the pass that made the
+     * relaxation necessary. Omitted = every gated leg, which is what an EXPLICIT `floors: 'loose'`
+     * request means (the caller asked for the profile outright and there is no prior pass to read).
+     * Either way the target values are the same {@link LOOSE_FLOORS} / {@link looseTermFloor} bottom
+     * lines — naming fewer legs can only make the pass stricter, never looser.
+     */
+    relaxLegs?: readonly FloorLeg[];
+}
+/**
+ * The legs a pass actually dropped candidates on — the only ones a relaxed pass can help.
+ *
+ * A leg that dropped nothing has no candidate below its strict floor, so lowering that floor admits
+ * nothing new; including it in the relaxed `floors` would just misreport which leg was the problem.
+ * `hrr` maps onto `jaccard` because the probe shares the entity leg's candidate set and floor (see
+ * `HybridLeg.leg`).
+ */
+export declare function droppedLegs(drops: RetrievalFloorDrops): FloorLeg[];
+/**
+ * Resolve the effective floors for one query (see the module comment for the rules).
+ *
+ * Exported so both stores and the tests read the SAME rule rather than re-deriving it — the whole
+ * point of hosting the orchestration in one file was that a per-store copy of a rule drifts.
+ *
+ * The relaxed profile takes `min(configured, relaxed)` per leg: it can only RELAX. A leg the
+ * operator disabled (`0` = off) stays disabled, a floor the operator already set below the relaxed
+ * value is left alone, and {@link FloorResolution.relaxLegs} can only narrow WHICH legs move.
+ * "Relaxed" never means "ungated" — otherwise the pass could not answer "nothing relevant".
+ */
+export declare function resolveFloors(retriever: FloorConfig, semAvail: boolean, resolution?: FloorResolution): RetrievalFloors;
+/** `0` = off; a score EQUAL to the floor passes (the floor drops strictly-lower candidates only). */
+export declare function passesFloor(score: number, floor: number): boolean;
+/** Drop the entries of one leg's raw score map that fall below `floor`, counting them. */
+export declare function applyScoreFloor(scores: Map<number, number>, floor: number): {
+    scores: Map<number, number>;
+    dropped: number;
+};
+/**
+ * How many distinct query terms `text` carries.
+ *
+ * Substring containment, case-insensitively: FTS5's trigram tokenizer matches a quoted phrase by
+ * consecutive trigrams and is ASCII-case-insensitive, which is the same predicate as "the term
+ * occurs in the text". `relevanceTerms` lowercases its latin words, so the comparison has to.
+ */
+export declare function countMatchedTerms(text: string, terms: readonly string[]): number;
+/**
+ * Apply the per-row FTS floor to one FTS leg's scores.
+ *
+ * The terms come from `relevanceTerms(query)` (latin words ≥ 5 chars + every CJK 3-gram) — the same
+ * tokenisation the conditional-hint probe uses, so "relevant enough" means one thing in this repo.
+ * A query with NO such term is not graded at all: there is nothing to count, and dropping every row
+ * on a technicality would turn a 3–4 char latin query (`buildFtsQuery` still emits it) into silence.
+ *
+ * The bar itself is {@link effectiveTermFloor}: a query that can only produce one term (a 3-char CJK
+ * query is one trigram) is measured against that one term, not against a configured 2 it could never
+ * reach.
+ */
+export declare function applyTermFloor(scores: Map<number, number>, texts: Map<number, string>, query: string, floor: number): {
+    scores: Map<number, number>;
+    dropped: number;
+};
+/** Sum per-leg drop counts (the cross-store result merges both stores' reports). */
+export declare function mergeFloorDrops(...parts: readonly (RetrievalFloorDrops | undefined)[]): RetrievalFloorDrops;
+/** Sum every leg of one drop report (the retry decision and the health counter both read it). */
+export declare function totalFloorDrops(drops: RetrievalFloorDrops): number;

package/lib/engine/mem/store/hybrid.d.ts

@@ -0,0 +1,232 @@
+/**
+ * The retrieval orchestration BOTH stores run.
+ *
+ * `MemoryStore.search` and `KnowledgeStore.search` used to carry parallel copies of this flow
+ * (~80 lines each: limit normalization → weights → over-fetch → leg cap → fuse → live filter →
+ * rerank → slice → output budget → health event). The legs themselves are legitimately
+ * store-specific — different tables, different filters (`category` vs `domain`/`source`), and only
+ * memory has the HRR probe — but the ORCHESTRATION is one algorithm, and two copies of it drifted:
+ *
+ *   - the `NaN` limit guard existed only in memory, so a non-finite limit reached knowledge's SQL
+ *     `LIMIT` and took the whole cross-store query down with it (`Promise.all` has no per-leg catch);
+ *   - `retriever.over_fetch_factor` was read only by memory, while knowledge hardcoded `limit * 3`,
+ *     so one config knob silently meant two different things;
+ *   - `recordLegCapped()` was called only by memory, so a capped knowledge leg was unobservable —
+ *     exactly the blindness DESIGN §20.17 says the counter exists to remove;
+ *   - the reranker's used/fallback flags were a helper in memory and an inline expression in
+ *     knowledge.
+ *
+ * Fixing those one at a time would leave the next drift a matter of when. The stores now supply
+ * their legs and their hit mapping; everything that can drift lives here once.
+ *
+ * A leg that THROWS is isolated: it contributes nothing, the query still answers from the others,
+ * and the failure is logged. The read path used to have no per-leg catch at all (the write path
+ * did), so an ONNX error at inference cost the caller the FTS and entity legs too.
+ *
+ * @module store/hybrid
+ */
+import { type Config, type FloorProfile, type RecallResult, type RetrievalFloorDrops, type RetrievalFloors } from '../../mem-contract/index.js';
+import { type BudgetInput, type Hit, type Reranker, type SemanticBackend } from '../../mem-retrieval/index.js';
+import { type FloorLeg } from './floors.js';
+/** The `limit` a caller gets when it passes nothing usable. */
+export declare const DEFAULT_SEARCH_LIMIT = 10;
+/**
+ * A retrieval input the CALLER got wrong, as opposed to a backend that failed at runtime.
+ *
+ * Leg failures are isolated (see {@link runLegs}) so one dead leg cannot cost the caller the whole
+ * query — but that must not extend to a caller contract violation. A wrong-width `queryVector` means
+ * the caller encoded with a different backend than this store, so EVERY query in that session would
+ * score as garbage; answering from the other legs would hide exactly the misconfiguration the check
+ * exists to surface.
+ */
+export declare class RetrievalInputError extends Error {
+}
+/** …and the over-fetch factor when the config does not set one (matches the contract default). */
+export declare const DEFAULT_OVER_FETCH_FACTOR = 5;
+/** The three weights a `RecallResult` reports. */
+export type RetrievalWeights = RecallResult['weights'];
+/** One retrieval leg's contribution to the fusion. */
+export interface HybridLeg {
+    /** Fusion weight; the caller resolves it against `semAvail` (see {@link HybridContext}). */
+    weight: number;
+    /** candidate id → raw score, in the leg's own scale (`fuse` normalizes per leg). */
+    scores: Map<number, number>;
+    /**
+     * True when the leg was cut at {@link HybridContext.legCap}.
+     *
+     * `size === cap` is the only observable "this leg was trimmed" signal — a leg that finished under
+     * the cap cannot have been cut — and it feeds the health counters, so the headroom can be sized
+     * from data instead of from a constant nobody re-derives. Measured on the leg's RAW score set:
+     * the relevance floors may shrink a capped leg afterwards, and re-deriving this from the floored
+     * size would silently erase the cap signal.
+     */
+    capped?: boolean;
+    /**
+     * Which floor governs this leg, for {@link HybridResult.dropped_by_floor}'s per-leg breakdown.
+     * The HRR probe shares the Jaccard floor; naming it separately keeps a narrowed candidate set
+     * attributable without double-counting the Jaccard leg's own drops.
+     */
+    leg?: keyof RetrievalFloorDrops;
+    /** Candidates this leg removed because they fell below its floor. */
+    droppedByFloor?: number;
+}
+/** What the orchestrator resolved before handing control to the store's legs. */
+export interface HybridContext {
+    /** The query, trimmed. */
+    query: string;
+    /** How many hits the caller receives (already guarded against a non-finite input). */
+    limit: number;
+    /** Size of the fused pool the legs should aim to fill. */
+    overFetch: number;
+    /** Per-leg row cap. */
+    legCap: number;
+    /** Whether the semantic backend is live; false means the degraded weights are in force. */
+    semAvail: boolean;
+    /**
+     * The EFFECTIVE relevance floors for this query, already resolved for `semAvail` (see
+     * `store/floors.ts`). Handed to the store for the same reason as {@link weights}: a leg applies
+     * its own floor, and deriving the relaxed value twice is how two stores end up cutting different
+     * candidates.
+     */
+    floors: RetrievalFloors;
+    /**
+     * The fusion weights in force, already rebalanced for `semAvail`. Handed to the store because a
+     * leg's weight is part of the leg, and the degraded rebalance must be the SAME one the result
+     * reports — deriving it twice is how two stores end up reporting different numbers.
+     */
+    weights: RetrievalWeights;
+    /** A query vector the caller already encoded (the cross-store router encodes once for both stores). */
+    queryVector?: Float32Array;
+    /**
+     * Called by a store when it ENCODED this query itself (rather than reusing {@link queryVector}),
+     * so a retry pass can hand the same vector back instead of paying the model again.
+     *
+     * The single-store `search` path is the one that needs this: its caller supplies no vector, and the
+     * relaxed retry below re-ran the whole pass — including `semantic.encode(query)`. The cross-store
+     * router already passes a vector, so it is unaffected (performance review §7.7 / P8).
+     */
+    onQueryVector?(vec: Float32Array): void;
+}
+/** What a store supplies. Generic in `H`, the caller-facing hit shape. */
+export interface HybridDeps<H> {
+    /** Which health-event kind this search records. */
+    kind: 'memory' | 'knowledge';
+    config: Config;
+    semantic: SemanticBackend;
+    reranker: Reranker;
+    /**
+     * The legs to fuse, as promises (or values). Awaited concurrently; a rejection costs that leg
+     * only. The async setup a store needs before it can build its legs — memory extracts the query's
+     * entities ONCE and shares them between the Jaccard and HRR legs — happens in this method's own
+     * body, before the array is returned.
+     */
+    legs(ctx: HybridContext): Promise<readonly (HybridLeg | Promise<HybridLeg>)[]>;
+    /** Text for the fused ids: the live filter, the reranker's input, and the hit bodies. */
+    texts(ids: number[]): Map<number, string>;
+    /** Build the caller-facing hits from the ranked, live, already-sliced ids. */
+    hits(ranked: readonly Hit[], texts: Map<number, string>): H[];
+    /**
+     * Called with the hits the caller ACTUALLY receives — after the output budget, not before it.
+     *
+     * This is where a store reinforces what it returned. Reinforcing the pre-budget list used to
+     * refresh the dormancy clock (and grant the trust bonus) for hits the token budget had already
+     * dropped, i.e. rewarding facts the caller never saw; the cross-store path had already learned
+     * that lesson (R5/R21) and the single-store path had not.
+     */
+    onReturn?(kept: readonly Budgeted<H>[]): void;
+}
+/** One search request, before resolution. */
+export interface HybridPlan {
+    query: string;
+    limit?: number;
+    /** Overrides the config-derived pool size; never smaller than `limit`. */
+    overFetch?: number;
+    /** Per-call output token budget; `0` = unlimited, omitted = `retriever.max_output_tokens`. */
+    maxTokens?: number;
+    queryVector?: Float32Array;
+    /**
+     * Which relevance-floor policy governs this query (see the contract's `FLOOR_PROFILES`):
+     * omitted = the configured floors plus ONE relaxed pass when they empty the result, `'strict'` =
+     * no fallback, `'loose'` = the relaxed floors outright.
+     *
+     * The cross-store router pins a profile on every store call (`'strict'` for its first pass,
+     * `'loose'` for its second): only the MERGED result may decide whether relaxing is warranted, or a
+     * store that happens to be empty would inject its relaxed tail into another store's strict answer.
+     */
+    floors?: FloorProfile;
+    /**
+     * Which legs a `'loose'` pass may lower (see `store/floors.ts`'s `FloorResolution.relaxLegs`).
+     * Omitted on the default policy: the single-store retry reads the STRICT pass's own drop report and
+     * lowers exactly those legs. The cross-store router has no single strict pass — it pins the legs
+     * the MERGED strict pass dropped on its second call, so "only what actually dropped" holds there
+     * too (relaxing a leg that dropped nothing admits no candidate; it only misreports `floors`).
+     */
+    relaxLegs?: readonly FloorLeg[];
+    /**
+     * Emit a health event for this search (default true). The cross-store router sets it false: it
+     * fuses both stores into ONE user-facing query and records a single `kind: 'cross'` event, so
+     * `queries` counts questions instead of legs (DESIGN §20.5).
+     */
+    recordStats?: boolean;
+}
+/** A hit after the output budget: `text` may have been shortened, and `truncated` says so. */
+export type Budgeted<T> = T & {
+    truncated?: boolean;
+};
+export interface HybridResult<H> {
+    hits: Budgeted<H>[];
+    /** The semantic backend was down, so the degraded weights were used. */
+    degraded: boolean;
+    weights: RetrievalWeights;
+    /** Tokens the returned hits carry; `0` means "not computed" (an unlimited budget skips the pass). */
+    used_tokens: number;
+    /** The floors actually applied (post degraded-relaxation AND profile), for the result envelope. */
+    floors: RetrievalFloors;
+    /** How many candidates each leg dropped below its floor. */
+    dropped_by_floor: RetrievalFloorDrops;
+    /**
+     * The strict pass returned nothing while having dropped candidates, and ONE relaxed pass supplied
+     * these hits — `floors` are that pass's values. An explicit `floors: 'loose'` request does NOT set
+     * this: the caller asked for the relaxed profile, so nothing was "relaxed behind its back".
+     */
+    relaxed?: boolean;
+}
+/**
+ * The per-leg row cap: `retriever.leg_cap` when set, otherwise derived from the pool size.
+ *
+ * Derived default: 4× the pool, so each leg can fill the fused result on its own. Stated once here
+ * rather than per store, because the health counter compares against whatever this returns.
+ */
+export declare function legCapFor(config: Config, overFetch: number): number;
+/**
+ * Whether the configured reranker is actually in use — the two flags a retrieval event carries.
+ *
+ * "Configured" is not "used": a reranker that is selected but unavailable falls back to the fused
+ * order, and counting that as a rerank would make the counter lie in exactly the case it exists to
+ * expose (the reference implementation leaves `rerank_fallback` unfeedable).
+ */
+export declare function rerankState(reranker: Reranker): {
+    used: boolean;
+    fallback: boolean;
+};
+/**
+ * Bound a ranked hit list to a token budget, marking what was shortened.
+ *
+ * An unlimited budget returns the hits untouched: `fitToTokenBudget` would still walk every entry
+ * to sum `used_tokens`, and every caller discards that number (the cross-store path passes 0
+ * deliberately — it bounds the MERGED result instead). `used_tokens: 0` means "not computed", and
+ * the budget module keeps its own honest accounting for callers that ask.
+ */
+export declare function applyOutputBudget<H extends BudgetInput>(config: Config, hits: H[], perCall?: number): {
+    kept: Budgeted<H>[];
+    used_tokens: number;
+};
+/**
+ * Run one hybrid search.
+ *
+ * The order of the last three steps is load-bearing: rerank the over-fetched pool, THEN slice to
+ * `limit`, THEN apply the output budget, and only then tell the store what was returned. Slicing
+ * before the rerank would rank an arbitrary subset; budgeting before the slice would spend the
+ * budget on hits that are about to be dropped.
+ */
+export declare function hybridSearch<H extends BudgetInput>(deps: HybridDeps<H>, plan: HybridPlan): Promise<HybridResult<H>>;

package/lib/engine/mem/store/ingest_guard.d.ts

@@ -0,0 +1,25 @@
+/** The boundary knobs, as resolved from `knowledge.ingest`. */
+export interface IngestLimits {
+    /** Roots a local `source_uri` may be read from; empty ⇒ the process workspace. */
+    local_roots: readonly string[];
+    allow_outside_workspace: boolean;
+    allow_private_network: boolean;
+}
+/**
+ * Whether a URL hostname addresses loopback / private / link-local / other
+ * non-public space. Names are matched by suffix too (`*.local` mDNS, `*.internal`,
+ * `*.home.arpa`), since those resolve inside the LAN by construction.
+ */
+export declare function isPrivateHostname(hostname: string): boolean;
+/** Validate one `http(s)` source URL, refusing non-public targets unless opted in. */
+export declare function assertFetchableUrl(uri: string, limits: Pick<IngestLimits, 'allow_private_network'>): URL;
+/** The roots a local `source_uri` may be read from (realpath'd), defaulting to the workspace. */
+export declare function allowedLocalRoots(limits: Pick<IngestLimits, 'local_roots'>): string[];
+/**
+ * Resolve a local `source_uri` to the real absolute path it names, refusing one
+ * that lands outside the allowed roots.
+ *
+ * @returns the `realpath` — which is what the caller should BOTH read and store, so
+ *   the recorded `source_uri` is exactly the file that was read.
+ */
+export declare function resolveLocalSource(raw: string, limits: IngestLimits): string;