onadit 0.0.1 → 1.0.0

package/dist/src/retrieval/lexical.js

@@ -0,0 +1,379 @@
+/**
+ * BM25 lexical search over LanceDB tables.
+ *
+ * Primary path: LanceDB full-text indexes (src/storage/fts.ts), which cover
+ * every row, including rows written since the last index refresh.
+ *
+ * Fallback path, used only for a table whose indexes are not built yet (fresh
+ * install before the first maintenance run, or an interrupted build): read
+ * every matching row, tokenize, and score BM25 in JS. It covers the whole
+ * table — slower, never narrower.
+ */
+import { MultiMatchQuery } from "@lancedb/lancedb";
+import { getTable } from "../storage/tables.js";
+import { FTS_COLUMNS, FTS_FACET_COLUMNS, fullTextReady } from "../storage/fts.js";
+import { escapeLike } from "./cwd.js";
+import { escapeSql, inFilter } from "../storage/filter.js";
+import { isRiftBasicMetadata, RIFT_BASIC_METADATA_PROVIDER } from "../providers/basic-metadata-extraction.js";
+import { indexCardsEnabled, REPORT_CARD_SQL_EXCLUSION } from "../ingestion/report-cards.js";
+// --- BM25 parameters ---
+const K1 = 1.2;
+const B = 0.75;
+/**
+ * Run BM25 lexical search across tables for a text query.
+ * Returns results sorted by BM25 score descending, limited to top_k.
+ */
+export async function lexicalSearch(query, options) {
+    const tables = tablesToQuery(options.scope).filter((t) => !options.tables || options.tables.includes(t));
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0)
+        return [];
+    const allResults = [];
+    for (const tableName of tables) {
+        allResults.push(...(await searchTable(tableName, query, queryTerms, options, options.top_k, false)));
+    }
+    allResults.sort((a, b) => b.score - a.score);
+    return allResults.slice(0, options.top_k);
+}
+function tablesToQuery(scope) {
+    switch (scope) {
+        case "documents":
+        case "clients":
+        case "projects":
+            return ["structured_docs"];
+        case "conversations":
+            return ["conversations_hot", "digests"];
+        case "all":
+            return ["structured_docs", "conversations_hot", "digests"];
+    }
+}
+const READY_TTL_MS = 60_000;
+let readyCache = new WeakMap();
+/**
+ * Whether every FTS column of this table is indexed. Cached per table handle
+ * for a minute: index listing is cheap but not free, and a rebuild/swap
+ * replaces the handle anyway.
+ */
+async function indexedPathReady(tableName, table) {
+    const hit = readyCache.get(table);
+    if (hit && Date.now() - hit.at < READY_TTL_MS)
+        return hit.ready;
+    let ready = false;
+    try {
+        ready = await fullTextReady(tableName, table);
+    }
+    catch {
+        ready = false;
+    }
+    readyCache.set(table, { ready, at: Date.now() });
+    return ready;
+}
+/** Forget cached index readiness (tests, and after maintenance in-process). */
+export function resetLexicalIndexCache() {
+    readyCache = new WeakMap();
+}
+/** Search one table: indexed BM25 when available, full scan otherwise. */
+async function searchTable(tableName, query, queryTerms, options, limit, carryConversationMeta) {
+    const table = getTable(tableName);
+    let scored;
+    if (options.scanLimit === undefined && (await indexedPathReady(tableName, table))) {
+        const schemaCols = new Set((await table.schema()).fields.map((f) => f.name));
+        const indexed = FTS_COLUMNS[tableName].filter((c) => schemaCols.has(c));
+        const textCols = indexed.filter((c) => !FTS_FACET_COLUMNS.includes(c));
+        const facetCols = indexed.filter((c) => FTS_FACET_COLUMNS.includes(c));
+        const select = await textColumns(table);
+        const clauses = filterClauses(tableName, options);
+        const run = async (columns, extra) => {
+            let q = table.query().fullTextSearch(new MultiMatchQuery(query, columns)).select(select);
+            const where = [...clauses, ...extra];
+            if (where.length > 0)
+                q = q.where(where.join(" AND "));
+            return (await q.limit(limit).toArray());
+        };
+        // Facets (domain) match only on enriched rows, as the scan did: a basic
+        // import's placeholder is not a classification. Scores add per row.
+        const [textRows, facetRows] = await Promise.all([
+            run(textCols, []),
+            facetCols.length > 0 ? run(facetCols, [ENRICHED_ROWS_CLAUSE]) : Promise.resolve([]),
+        ]);
+        const byId = new Map();
+        for (const row of [...textRows, ...facetRows]) {
+            const id = row.id;
+            const prev = byId.get(id);
+            if (prev)
+                prev.score += Number(row._score ?? 0);
+            else
+                byId.set(id, { row, score: Number(row._score ?? 0) });
+        }
+        scored = [...byId.values()];
+    }
+    else {
+        const rows = await fetchRows(tableName, options, options.scanLimit);
+        if (rows.length === 0)
+            return [];
+        const { documents, termDocFreq, avgDl } = buildCorpusStats(rows, tableName, queryTerms);
+        scored = documents.map(({ row, length, termFreq }) => ({
+            row,
+            score: bm25Score(queryTerms, length, termFreq, termDocFreq, rows.length, avgDl),
+        }));
+    }
+    const results = [];
+    for (const { row, score } of scored) {
+        if (!(score > 0))
+            continue;
+        const result = {
+            id: row.id,
+            content: row.content || row.summary || "",
+            score,
+            retrieval_method: "lexical",
+            source_table: tableName,
+        };
+        const ts = row.indexed_at || row.created_at;
+        if (ts)
+            result.indexed_at = ts;
+        // `quality` drives the reranker's S_qual boost. Basic-import rows carry
+        // a placeholder ("medium"); surfacing it would bias ranking on a fake
+        // label. Leave it unset for basic rows so the reranker treats them as
+        // quality-neutral (1.0), same as enriched rows with unknown quality.
+        const q = row.quality;
+        if (q && !isRiftBasicMetadata(row.metadata_provider)) {
+            result.quality = q;
+        }
+        // Carry topics/decisions for conversation tables (used by post-filter)
+        if (carryConversationMeta &&
+            tableName !== "structured_docs" &&
+            tableName !== "structured_docs_local" &&
+            tableName !== "digests") {
+            const t = row.topics;
+            if (t)
+                result.topics = t;
+            const d = row.decisions;
+            if (d)
+                result.decisions = d;
+        }
+        results.push(result);
+    }
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, limit);
+}
+/** Rows whose facets came from real enrichment (NULL provider = legacy row, kept). */
+const ENRICHED_ROWS_CLAUSE = `(metadata_provider IS NULL OR metadata_provider != '${RIFT_BASIC_METADATA_PROVIDER}')`;
+/** WHERE clauses for scope/client/since/path filters. */
+function filterClauses(tableName, options) {
+    const clauses = [];
+    if (tableName === "structured_docs" || tableName === "structured_docs_local") {
+        if (options.scope === "documents")
+            clauses.push("source_scope = 'document'");
+        else if (options.scope === "clients")
+            clauses.push("source_scope = 'client'");
+        else if (options.scope === "projects")
+            clauses.push("source_scope = 'project'");
+        if (options.client) {
+            clauses.push(`client_name = '${escapeSql(options.client)}'`);
+        }
+        if (options.sourcePathPrefix) {
+            clauses.push(`source_path LIKE '${escapeSql(escapeLike(options.sourcePathPrefix))}%' ESCAPE '\\'`);
+        }
+        if (options.excludeSourcePathPrefix) {
+            clauses.push(`source_path NOT LIKE '${escapeSql(escapeLike(options.excludeSourcePathPrefix))}%' ESCAPE '\\'`);
+        }
+        if (options.since) {
+            clauses.push(`indexed_at >= '${escapeSql(options.since)}'`);
+        }
+        // beta.28: exclude index cards unless RIFT_INDEX_CARDS is on (default off →
+        // byte-identical to pre-card retrieval). Mirrors the vector-search guard.
+        if (!indexCardsEnabled()) {
+            clauses.push(REPORT_CARD_SQL_EXCLUSION);
+        }
+    }
+    else if (tableName === "digests") {
+        if (options.since) {
+            clauses.push(`created_at >= '${escapeSql(options.since)}'`);
+        }
+    }
+    else {
+        // conversations_hot or conversations_cold
+        if (options.conversationIds?.length) {
+            clauses.push(inFilter("id", options.conversationIds, { validateAsRowId: true }));
+        }
+        if (options.excludeConversationIds?.length) {
+            clauses.push(`NOT (${inFilter("id", options.excludeConversationIds, { validateAsRowId: true })})`);
+        }
+        if (options.since) {
+            clauses.push(`indexed_at >= '${escapeSql(options.since)}'`);
+        }
+    }
+    // Append caller-supplied extra clauses (e.g. metadata filters)
+    if (options.extraClauses) {
+        clauses.push(...options.extraClauses);
+    }
+    return clauses;
+}
+/** Every column except the vector: keyword scoring never uses embeddings. */
+async function textColumns(table) {
+    return (await table.schema()).fields
+        .map((field) => field.name)
+        .filter((name) => name !== "embedding");
+}
+/** Scan fallback: fetch rows, applying scope/client/since filters via WHERE. */
+async function fetchRows(tableName, options, limit) {
+    const table = getTable(tableName);
+    const clauses = filterClauses(tableName, options);
+    // Keyword scoring never uses vectors. Keep them out of Arrow/JS allocations
+    // while retaining every text and metadata field used below.
+    let q = table.query().select(await textColumns(table));
+    if (clauses.length > 0) {
+        q = q.where(clauses.join(" AND "));
+    }
+    if (limit !== undefined)
+        q = q.limit(limit);
+    return (await q.toArray());
+}
+/**
+ * Extract searchable text from a row.
+ * Includes content + metadata fields for proper noun matching.
+ */
+function extractText(row, tableName) {
+    const parts = [];
+    if (row.content)
+        parts.push(row.content);
+    if (row.summary)
+        parts.push(row.summary);
+    if (tableName === "structured_docs" || tableName === "structured_docs_local") {
+        if (row.source_path)
+            parts.push(row.source_path);
+        if (row.client_name)
+            parts.push(row.client_name);
+        // Parse metadata JSON for searchable fields
+        if (row.metadata && typeof row.metadata === "string") {
+            try {
+                const meta = JSON.parse(row.metadata);
+                for (const v of Object.values(meta)) {
+                    if (typeof v === "string")
+                        parts.push(v);
+                }
+            }
+            catch {
+                // ignore invalid metadata
+            }
+        }
+    }
+    else if (tableName !== "digests") {
+        // Conversation rows
+        if (row.source)
+            parts.push(row.source);
+        // `domain` is a real facet only for AI-enriched rows. A no-AI basic import
+        // stamps a generic placeholder (e.g. "tech") on every row; indexing that as
+        // searchable text makes the placeholder a junk match for the literal word.
+        // Drop it for basic rows — the real, text-derived `topics` below still index.
+        if (row.domain && !isRiftBasicMetadata(row.metadata_provider)) {
+            parts.push(row.domain);
+        }
+        // Parse JSON array fields
+        for (const field of ["topics", "decisions", "key_outputs"]) {
+            if (row[field] && typeof row[field] === "string") {
+                try {
+                    const arr = JSON.parse(row[field]);
+                    if (Array.isArray(arr))
+                        parts.push(...arr.filter((v) => typeof v === "string"));
+                }
+                catch {
+                    // ignore
+                }
+            }
+        }
+    }
+    return parts.join(" ");
+}
+/**
+ * Tokenize text into lowercase terms.
+ * Unicode-aware: keeps accented characters (é, ç, à, ü, etc.) intact
+ * so that French and other Latin-script languages are searchable.
+ * Preserves proper nouns as additional tokens (uppercase sequences, camelCase).
+ */
+export function tokenize(text) {
+    // Normalize to NFC so that decomposed accents (e + combining acute)
+    // are collapsed to precomposed form (é) before splitting. This ensures
+    // that NFD text (common in macOS filenames and pasted content) matches
+    // NFC text in the index, and vice versa.
+    const normalized = text.normalize("NFC");
+    // Split on characters that are NOT Unicode letters, digits, or underscores.
+    // \p{L} matches any Unicode letter (including accented: é, ç, ü, etc.)
+    // \p{N} matches any Unicode digit
+    const raw = normalized.split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
+    const tokens = [];
+    for (const word of raw) {
+        tokens.push(word.toLowerCase());
+        // Keep original casing as a bonus token for exact-term matching
+        if (word !== word.toLowerCase()) {
+            tokens.push(word);
+        }
+    }
+    return tokens;
+}
+/** Retain only query-term counts, not every token in every document. */
+function buildCorpusStats(rows, tableName, queryTerms) {
+    const wanted = new Set(queryTerms);
+    const termDocFreq = new Map();
+    let totalLength = 0;
+    const documents = rows.map((row) => {
+        const tokens = tokenize(extractText(row, tableName));
+        const termFreq = new Map();
+        for (const token of tokens) {
+            if (wanted.has(token)) {
+                termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
+            }
+        }
+        for (const term of termFreq.keys()) {
+            termDocFreq.set(term, (termDocFreq.get(term) ?? 0) + 1);
+        }
+        totalLength += tokens.length;
+        return { row, length: tokens.length, termFreq };
+    });
+    return { documents, termDocFreq, avgDl: totalLength / rows.length };
+}
+/** BM25 score for a single document against a query. */
+function bm25Score(queryTerms, dl, tf, termDocFreq, docCount, avgDl) {
+    if (dl === 0)
+        return 0;
+    let score = 0;
+    const seen = new Set();
+    for (const qt of queryTerms) {
+        if (seen.has(qt))
+            continue;
+        seen.add(qt);
+        const termFreq = tf.get(qt) ?? 0;
+        if (termFreq === 0)
+            continue;
+        const df = termDocFreq.get(qt) ?? 0;
+        // IDF with floor to avoid negative scores for very common terms
+        const idf = Math.max(0, Math.log((docCount - df + 0.5) / (df + 0.5) + 1));
+        const tfNorm = (termFreq * (K1 + 1)) /
+            (termFreq + K1 * (1 - B + B * (dl / avgDl)));
+        score += idf * tfNorm;
+    }
+    return score;
+}
+/**
+ * Run BM25 lexical search against specific tables (for cold-tier fallback).
+ */
+export async function lexicalSearchTables(query, tables, options) {
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0)
+        return [];
+    const allResults = [];
+    const tableOptions = {
+        scope: options.scope ?? "all",
+        top_k: options.top_k,
+        since: options.since,
+        client: options.client,
+        sourcePathPrefix: options.sourcePathPrefix,
+        extraClauses: options.extraClauses,
+    };
+    for (const tableName of tables) {
+        allResults.push(...(await searchTable(tableName, query, queryTerms, tableOptions, options.top_k, true)));
+    }
+    allResults.sort((a, b) => b.score - a.score);
+    return allResults.slice(0, options.top_k);
+}
+//# sourceMappingURL=lexical.js.map

package/dist/src/retrieval/lexical.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexical.js","sourceRoot":"","sources":["../../../src/retrieval/lexical.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,eAAe,EAAc,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAGlF,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,4BAA4B,EAAE,MAAM,2CAA2C,CAAC;AAC9G,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AAE5F,0BAA0B;AAC1B,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAcf;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,OAA6B;IAE7B,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CACrD,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,UAAU,GAAmB,EAAE,CAAC;IACtC,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CACb,GAAG,CAAC,MAAM,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CACpF,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAMD,SAAS,aAAa,CAAC,KAAkB;IACvC,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,WAAW,CAAC;QACjB,KAAK,SAAS,CAAC;QACf,KAAK,UAAU;YACb,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAC7B,KAAK,eAAe;YAClB,OAAO,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC;QAC1C,KAAK,KAAK;YACR,OAAO,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,MAAM,YAAY,GAAG,MAAM,CAAC;AAC5B,IAAI,UAAU,GAAG,IAAI,OAAO,EAAyC,CAAC;AAEtE;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAAC,SAAoB,EAAE,KAAY;IAChE,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,YAAY;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC;IAChE,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,GAAG,KAAK,CAAC;IAChB,CAAC;IACD,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,sBAAsB;IACpC,UAAU,GAAG,IAAI,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED,0EAA0E;AAC1E,KAAK,UAAU,WAAW,CACxB,SAAoB,EACpB,KAAa,EACb,UAAoB,EACpB,OAA6B,EAC7B,KAAa,EACb,qBAA8B;IAE9B,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,MAA0C,CAAC;IAC/C,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,EAAE,OAAiB,EAAE,KAAe,EAAuB,EAAE;YAC5E,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,cAAc,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzF,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC;YACrC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACvD,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAe,CAAC;QACxD,CAAC,CAAC;QACF,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;SACpF,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAC;QACjE,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC;YAC9C,MAAM,EAAE,GAAG,GAAG,CAAC,EAAY,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1B,IAAI,IAAI;gBAAE,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;;gBAC3C,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QACxF,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACrD,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;SAChF,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,MAAM,EAAE,CAAC;QACpC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YAAE,SAAS;QAC3B,MAAM,MAAM,GAAiB;YAC3B,EAAE,EAAE,GAAG,CAAC,EAAY;YACpB,OAAO,EAAG,GAAG,CAAC,OAAkB,IAAK,GAAG,CAAC,OAAkB,IAAI,EAAE;YACjE,KAAK;YACL,gBAAgB,EAAE,SAAS;YAC3B,YAAY,EAAE,SAAS;SACxB,CAAC;QACF,MAAM,EAAE,GAAI,GAAG,CAAC,UAAqB,IAAK,GAAG,CAAC,UAAqB,CAAC;QACpE,IAAI,EAAE;YAAE,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;QAC/B,wEAAwE;QACxE,sEAAsE;QACtE,sEAAsE;QACtE,qEAAqE;QACrE,MAAM,CAAC,GAAG,GAAG,CAAC,OAAiB,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,iBAAuC,CAAC,EAAE,CAAC;YAC3E,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,uEAAuE;QACvE,IACE,qBAAqB;YACrB,SAAS,KAAK,iBAAiB;YAC/B,SAAS,KAAK,uBAAuB;YACrC,SAAS,KAAK,SAAS,EACvB,CAAC;YACD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAgB,CAAC;YAC/B,IAAI,CAAC;gBAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YACzB,MAAM,CAAC,GAAG,GAAG,CAAC,SAAmB,CAAC;YAClC,IAAI,CAAC;gBAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,sFAAsF;AACtF,MAAM,oBAAoB,GAAG,uDAAuD,4BAA4B,IAAI,CAAC;AAErH,yDAAyD;AACzD,SAAS,aAAa,CAAC,SAAoB,EAAE,OAA6B;IACxE,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,SAAS,KAAK,iBAAiB,IAAI,SAAS,KAAK,uBAAuB,EAAE,CAAC;QAC7E,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW;YAAE,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;aACxE,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;aACzE,IAAI,OAAO,CAAC,KAAK,KAAK,UAAU;YAAE,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAEhF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,kBAAkB,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CACV,qBAAqB,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,gBAAgB,CACrF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,uBAAuB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CACV,yBAAyB,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,gBAAgB,CAChG,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,kBAAkB,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9D,CAAC;QAED,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;SAAM,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,kBAAkB,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,0CAA0C;QAC1C,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,eAAe,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,OAAO,CAAC,sBAAsB,EAAE,MAAM,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CACV,QAAQ,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,GAAG,CACrF,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,kBAAkB,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,WAAW,CAAC,KAAY;IACrC,OAAO,CAAC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM;SACjC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;AAC5C,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,SAAS,CACtB,SAAoB,EACpB,OAA6B,EAC7B,KAAyB;IAEzB,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAElD,4EAA4E;IAC5E,4DAA4D;IAC5D,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,KAAK,KAAK,SAAS;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAE5C,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAe,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,GAAa,EAAE,SAAoB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAiB,CAAC,CAAC;IACnD,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAiB,CAAC,CAAC;IAEnD,IAAI,SAAS,KAAK,iBAAiB,IAAI,SAAS,KAAK,uBAAuB,EAAE,CAAC;QAC7E,IAAI,GAAG,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAqB,CAAC,CAAC;QAC3D,IAAI,GAAG,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAqB,CAAC,CAAC;QAC3D,4CAA4C;QAC5C,IAAI,GAAG,CAAC,QAAQ,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpC,IAAI,OAAO,CAAC,KAAK,QAAQ;wBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,oBAAoB;QACpB,IAAI,GAAG,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAgB,CAAC,CAAC;QACjD,2EAA2E;QAC3E,4EAA4E;QAC5E,2EAA2E;QAC3E,8EAA8E;QAC9E,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,iBAAuC,CAAC,EAAE,CAAC;YACpF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAgB,CAAC,CAAC;QACnC,CAAC;QACD,0BAA0B;QAC1B,KAAK,MAAM,KAAK,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,EAAE,CAAC;YAC3D,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAW,CAAC,CAAC;oBAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;wBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;gBAC3F,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,oEAAoE;IACpE,uEAAuE;IACvE,uEAAuE;IACvE,yCAAyC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAEzC,4EAA4E;IAC5E,uEAAuE;IACvE,kCAAkC;IAClC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjE,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAChC,gEAAgE;QAChE,IAAI,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,SAAS,gBAAgB,CAAC,IAAgB,EAAE,SAAoB,EAAE,UAAoB;IACpF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACnC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC7B,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;IAClD,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;AACtE,CAAC;AAED,wDAAwD;AACxD,SAAS,SAAS,CAChB,UAAoB,EACpB,EAAU,EACV,EAAuB,EACvB,WAAgC,EAChC,QAAgB,EAChB,KAAa;IAEb,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,SAAS;QAC3B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEb,MAAM,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,QAAQ,KAAK,CAAC;YAAE,SAAS;QAE7B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACpC,gEAAgE;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CACjD,CAAC;QAEF,MAAM,MAAM,GACV,CAAC,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACrB,CAAC,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAE/C,KAAK,IAAI,GAAG,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAa,EACb,MAAmB,EACnB,OAAkI;IAElI,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,UAAU,GAAmB,EAAE,CAAC;IACtC,MAAM,YAAY,GAAG;QACnB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;QAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,YAAY,EAAE,OAAO,CAAC,YAAY;KACX,CAAC;IAC1B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CACb,GAAG,CAAC,MAAM,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CACxF,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC"}

package/dist/src/retrieval/locator-boost.d.ts

@@ -0,0 +1,37 @@
+import type { RankedResult } from "./reranker.js";
+/** Tokens >=4 chars, lowercased, stopwords dropped. */
+export declare function tokens(s: string | undefined): string[];
+/** Strip the extension and a leading `YYYY-MM-DD-` date prefix from a filename. */
+export declare function slugFromPath(sourcePath: string | undefined): string;
+/** Did the reference-lookup (locator) intent fire for this query? */
+export declare function locatorIntentFired(query: string): boolean;
+/** The matchable fields of a candidate (never its body). */
+export interface LocatorFields {
+    source_path?: string;
+    title?: string;
+}
+/**
+ * Best per-field query-token overlap ratio across filename / slug / path /
+ * title. Range [0,1]. NEVER reads body or BM25 — see THE GUARD above. A row with
+ * no path/title (a conversation) scores 0 and is never boosted.
+ */
+export declare function locatorFieldScore(query: string, fields: LocatorFields): number;
+/** Default additive weight applied to a row's [0,1] locator field score. */
+export declare const DEFAULT_LOCATOR_BOOST_WEIGHT = 0.5;
+/**
+ * Stable, bounded locator reorder. Sorts `ranked` by an effective key of
+ * `score + weight * locatorFieldScore`, but PRESERVES each row's displayed
+ * score — mirroring `demoteReports`' "reorder, never rewrite scores" contract.
+ * Ties (and zero-boost rows) keep their incoming relative order. Returns a new
+ * array; the input is untouched.
+ */
+export declare function applyLocatorBoost(query: string, ranked: readonly RankedResult[], fieldsById: ReadonlyMap<string, LocatorFields>, weight?: number): RankedResult[];
+/** Minimum field score for a report to be treated as the asked-for target. */
+export declare const DEFAULT_LOCATOR_TARGET_THRESHOLD = 0.5;
+/**
+ * Whether a row is a strong-enough field match to be the locator's target — used
+ * by the demotion exception to exempt a field-matched report from being pushed
+ * behind non-report evidence (the asker explicitly wants that report).
+ */
+export declare function isLocatorTarget(query: string, fields: LocatorFields, threshold?: number): boolean;
+//# sourceMappingURL=locator-boost.d.ts.map

package/dist/src/retrieval/locator-boost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locator-boost.d.ts","sourceRoot":"","sources":["../../../src/retrieval/locator-boost.ts"],"names":[],"mappings":"AA+BA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAUlD,uDAAuD;AACvD,wBAAgB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAMtD;AAED,mFAAmF;AACnF,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAKnE;AAaD,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAGzD;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,MAAM,CAmB9E;AAED,4EAA4E;AAC5E,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,SAAS,YAAY,EAAE,EAC/B,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,EAC9C,MAAM,GAAE,MAAqC,GAC5C,YAAY,EAAE,CAShB;AAED,8EAA8E;AAC9E,eAAO,MAAM,gCAAgC,MAAM,CAAC;AAEpD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,aAAa,EACrB,SAAS,GAAE,MAAyC,GACnD,OAAO,CAET"}

package/dist/src/retrieval/locator-boost.js

@@ -0,0 +1,129 @@
+/**
+ * Locator-relevance boost for flat `onadit_search` (POST /search), scope:"all".
+ *
+ * WHY
+ * ---
+ * The docs-vs-discussions eval (`--explain`) showed reference-lookup queries
+ * ("find the doc with the fresh-mac gate findings", "where is the onboard
+ * wizard defined") failing: the target file sits in the prod candidate pool by
+ * raw retrieval signal (fresh-mac-gate #1, onboard #3) with a maxed BM25, but
+ * the reranker discards BM25 magnitude for hybrid hits (`S_sim = raw_sim`), so
+ * the formula drops the target out of the top-10; for a report target, the
+ * post-rerank report-demotion then shoves it behind every non-report. Ranking +
+ * demotion are coupled in the shipped path, so the fix has two coupled levers.
+ *
+ * THE GUARD (load-bearing)
+ * ------------------------
+ * The boost is computed from FIELD hits only — filename / slug / path / title —
+ * and NEVER from body text or raw BM25. That is deliberate: the decoys the eval
+ * flagged (`.rift-rename-allowlist.txt`, `src/main.ts`) only matched the query
+ * in their BODY; their path/slug/title share zero tokens with the query, so a
+ * field-only boost scores them 0 and cannot promote them. Body BM25 is exactly
+ * what floated them, which is why it is excluded here.
+ *
+ * SAFETY
+ * ------
+ * Every lever fires ONLY when `locatorIntentFired(query)` is true. Discussion-
+ * intent queries (the set doc-intrusion is measured on) never trigger a marker,
+ * so these levers cannot regress doc-intrusion — they only reorder locator
+ * queries. Pure + dependency-light so the eval harness can ablate them directly.
+ */
+import path from "node:path";
+// Promoted from eval/retrieval/reorder-experiments.ts (same token rules) so the
+// shipped scorer and the eval share one tokenizer rather than forking two.
+const STOPWORDS = new Set([
+    "the", "and", "for", "with", "find", "doc", "docs", "document", "documents",
+    "file", "files", "report", "reports", "where", "what", "which", "did",
+    "write", "wrote", "about", "from", "that", "this", "our",
+]);
+/** Tokens >=4 chars, lowercased, stopwords dropped. */
+export function tokens(s) {
+    if (!s)
+        return [];
+    return s
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((t) => t.length >= 4 && !STOPWORDS.has(t));
+}
+/** Strip the extension and a leading `YYYY-MM-DD-` date prefix from a filename. */
+export function slugFromPath(sourcePath) {
+    if (!sourcePath)
+        return "";
+    const base = path.basename(sourcePath);
+    const noExt = base.replace(/\.[a-z0-9]+$/i, "");
+    return noExt.replace(/^\d{4}-\d{2}-\d{2}-/, "");
+}
+/**
+ * Reference-lookup intent markers. Plain "find the doc/file/report where X is"
+ * phrasing. Conservative on purpose: a false negative just leaves a query on the
+ * current (already-shipped) path; a false positive only reorders that one query.
+ */
+const LOCATOR_MARKERS = [
+    "find the doc", "find the report", "find the file", "the doc with",
+    "which file", "where is", "where did i write", "where did i put",
+    "write up", "write-up", "writeup", "findings", "ledger",
+];
+/** Did the reference-lookup (locator) intent fire for this query? */
+export function locatorIntentFired(query) {
+    const q = (query || "").toLowerCase();
+    return LOCATOR_MARKERS.some((m) => q.includes(m));
+}
+/**
+ * Best per-field query-token overlap ratio across filename / slug / path /
+ * title. Range [0,1]. NEVER reads body or BM25 — see THE GUARD above. A row with
+ * no path/title (a conversation) scores 0 and is never boosted.
+ */
+export function locatorFieldScore(query, fields) {
+    const qt = tokens(query);
+    if (qt.length === 0)
+        return 0;
+    const filename = fields.source_path ? path.basename(fields.source_path) : "";
+    const haystacks = [
+        filename,
+        slugFromPath(fields.source_path),
+        fields.source_path ?? "",
+        fields.title ?? "",
+    ];
+    let best = 0;
+    for (const text of haystacks) {
+        const hay = new Set(tokens(text));
+        if (hay.size === 0)
+            continue;
+        let hits = 0;
+        for (const t of qt)
+            if (hay.has(t))
+                hits++;
+        best = Math.max(best, hits / qt.length);
+    }
+    return best;
+}
+/** Default additive weight applied to a row's [0,1] locator field score. */
+export const DEFAULT_LOCATOR_BOOST_WEIGHT = 0.5;
+/**
+ * Stable, bounded locator reorder. Sorts `ranked` by an effective key of
+ * `score + weight * locatorFieldScore`, but PRESERVES each row's displayed
+ * score — mirroring `demoteReports`' "reorder, never rewrite scores" contract.
+ * Ties (and zero-boost rows) keep their incoming relative order. Returns a new
+ * array; the input is untouched.
+ */
+export function applyLocatorBoost(query, ranked, fieldsById, weight = DEFAULT_LOCATOR_BOOST_WEIGHT) {
+    return ranked
+        .map((r, i) => {
+        const f = fieldsById.get(r.id);
+        const fs = f ? locatorFieldScore(query, f) : 0;
+        return { r, i, key: r.score + weight * fs };
+    })
+        .sort((a, b) => b.key - a.key || a.i - b.i)
+        .map(({ r }) => r);
+}
+/** Minimum field score for a report to be treated as the asked-for target. */
+export const DEFAULT_LOCATOR_TARGET_THRESHOLD = 0.5;
+/**
+ * Whether a row is a strong-enough field match to be the locator's target — used
+ * by the demotion exception to exempt a field-matched report from being pushed
+ * behind non-report evidence (the asker explicitly wants that report).
+ */
+export function isLocatorTarget(query, fields, threshold = DEFAULT_LOCATOR_TARGET_THRESHOLD) {
+    return locatorFieldScore(query, fields) >= threshold;
+}
+//# sourceMappingURL=locator-boost.js.map

package/dist/src/retrieval/locator-boost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locator-boost.js","sourceRoot":"","sources":["../../../src/retrieval/locator-boost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,gFAAgF;AAChF,2EAA2E;AAC3E,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW;IAC3E,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;IACrE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;CACzD,CAAC,CAAC;AAEH,uDAAuD;AACvD,MAAM,UAAU,MAAM,CAAC,CAAqB;IAC1C,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,OAAO,CAAC;SACL,WAAW,EAAE;SACb,KAAK,CAAC,YAAY,CAAC;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,YAAY,CAAC,UAA8B;IACzD,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;AAClD,CAAC;AAED;;;;GAIG;AACH,MAAM,eAAe,GAAG;IACtB,cAAc,EAAE,iBAAiB,EAAE,eAAe,EAAE,cAAc;IAClE,YAAY,EAAE,UAAU,EAAE,mBAAmB,EAAE,iBAAiB;IAChE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ;CACxD,CAAC;AAEF,qEAAqE;AACrE,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACtC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC;AAQD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,MAAqB;IACpE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7E,MAAM,SAAS,GAAG;QAChB,QAAQ;QACR,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC;QAChC,MAAM,CAAC,WAAW,IAAI,EAAE;QACxB,MAAM,CAAC,KAAK,IAAI,EAAE;KACnB,CAAC;IACF,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,EAAE;YAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAa,EACb,MAA+B,EAC/B,UAA8C,EAC9C,SAAiB,4BAA4B;IAE7C,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,MAAM,GAAG,EAAE,EAAE,CAAC;IAC9C,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC1C,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,MAAqB,EACrB,YAAoB,gCAAgC;IAEpD,OAAO,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,CAAC;AACvD,CAAC"}

package/dist/src/retrieval/passages.d.ts

@@ -0,0 +1,24 @@
+export interface Passage {
+    text: string;
+    /** Offsets into the stored row content (UTF-16 code units, end exclusive). */
+    start: number;
+    end: number;
+    /** Distinct query terms inside this passage, out of `terms_total`. */
+    terms_matched: number;
+    terms_total: number;
+}
+export interface PassageOptions {
+    /** Max passages returned. */
+    max?: number;
+    /** Target window size in characters. */
+    windowChars?: number;
+}
+/** Lowercase + strip accents, keeping a 1:1 index mapping to the input. */
+export declare function foldForMatch(text: string): string;
+export declare function passageQueryTerms(query: string): string[];
+/**
+ * Best query-matching passages in `content`, ordered by position. Empty when
+ * no query term occurs (callers fall back to their summary/snippet).
+ */
+export declare function extractPassages(content: string, query: string, opts?: PassageOptions): Passage[];
+//# sourceMappingURL=passages.d.ts.map

package/dist/src/retrieval/passages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"passages.d.ts","sourceRoot":"","sources":["../../../src/retrieval/passages.ts"],"names":[],"mappings":"AAgBA,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAiBD,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMjD;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAazD;AAkBD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,OAAO,EAAE,CA6DpG"}