akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/indexer/search/db-search.js

@@ -24,17 +24,18 @@ import { systemErrorCode } from "../../core/system-error.js";
 import { allowsFragmentRef, defaultRendererRegistry } from "../../core/type-presentation.js";
 import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
 import { assertIndexPathReadable, closeDatabase, openExistingDatabase, } from "../../storage/repositories/index-connection.js";
-import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount, getPositiveFeedbackCountsByIds, } from "../../storage/repositories/index-entries-repository.js";
-import { getIndexedMarkdownFragment, getIndexedMarkdownFragments, searchFts, } from "../../storage/repositories/index-fts-repository.js";
+import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryCount, getPositiveFeedbackCountsByIds, } from "../../storage/repositories/index-entries-repository.js";
+import { getIndexedMarkdownFragment, getIndexedMarkdownFragments, } from "../../storage/repositories/index-fts-repository.js";
 import { getMeta } from "../../storage/repositories/index-meta-repository.js";
-import { getEmbeddingCount, searchVec } from "../../storage/repositories/index-vec-repository.js";
+import { searchUnits } from "../../storage/repositories/units-repository.js";
 import { getCurrentWorkflowScopeKey } from "../../workflows/authoring/scope-key.js";
+import { deriveObservedEmbeddingIdentity } from "../embedding-identity.js";
 import { ensureIndex } from "../ensure-index.js";
 import { collectGraphRelatedHit, loadGraphBoostContext } from "../graph/graph-boost.js";
 import { isProposedQuality } from "../passes/metadata.js";
 import { resolveProjectContext } from "../walk/project-context.js";
 import { buildLexicalQueryPlan, parseRefPrefixQuery, parseRetiredTypePrefixQuery, } from "./fts-query.js";
-import { applyRankingRules, combineSearchScores, lexicalNameMatchTier, normalizeFtsScores } from "./ranking.js";
+import { applyRankingRules, fuseByEntry, lexicalNameMatchTier } from "./ranking.js";
 import { typeBoostFor } from "./ranking-contributors.js";
 import { attachSearchHitAttribution, copySearchHitAttribution, getSearchHitAttribution } from "./search-attribution.js";
 import { enrichSearchHit } from "./search-hit-enrichers.js";
@@ -225,6 +226,39 @@ export function canonicalContentTieKey(entry) {
     const source = (body || entry.description || "").replace(/^ +| +$/g, "");
     return Buffer.from(asciiCaseFold(source), "utf8").toString("hex");
 }
+/**
+ * Priority rank for `RankedEntryInput.lexicalMatch` — lower is stronger
+ * evidence. `undefined` (a pure-semantic hit with no lexical component at
+ * all) ranks weakest, below even a relaxed OR-pool recovery.
+ *
+ * Named-mechanism fix (fix-ranking-derived-outranks-primary): the exact →
+ * prefix → relaxed tier ladder (`searchUnitsLexicalScoped` in this file) is
+ * computed and carried on every candidate as `lexicalMatch`, but nothing
+ * downstream ever CONSULTED it as ranking evidence — `fuseByEntry` scores
+ * every tier on the same `stableFtsScore` magnitude scale (deliberately, so a
+ * relaxed hit that topped up the candidate pool floors at 0.3 instead of
+ * racing on rank), and the final comparator below sorted purely by that
+ * magnitude. `stableFtsScore`'s [0.3, 0.8] compression then flattens a large
+ * raw-BM25 gap between an all-token exact match and a two-of-three relaxed
+ * match to a few thousandths (e.g. 0.7148 vs 0.7053 for a ~6x BM25 gap) — well
+ * inside the swing of any single additive ranking contributor (alias-ranking
+ * alone is +0.3) or a belief-state ceiling. So a contributor or a ceiling,
+ * neither of which is supposed to do more than nudge, ends up DECIDING an
+ * ordering that the lexical tier — which already told us conclusively that
+ * one candidate matched every query token and the other did not — should
+ * have decided.
+ *
+ * This is the same escape hatch `aNameTier === 3` below already uses for a
+ * perfect name match, generalized to the tier ladder: exact tier is stronger
+ * evidence than prefix, which is stronger than relaxed, independent of the
+ * compressed magnitude gap between them. It sits after the name-tier-3 gate
+ * (an exact full name equality is stronger evidence still) and before the
+ * score comparison it used to lose to.
+ */
+const LEXICAL_TIER_RANK = { exact: 0, prefix: 1, relaxed: 2 };
+function lexicalTierRank(tier) {
+    return tier === undefined ? 3 : LEXICAL_TIER_RANK[tier];
+}
 function buildSearchResultComparator(query) {
     const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
     const displayScore = (score) => Math.round(displaySearchScore(score) * 10000) / 10000;
@@ -237,6 +271,9 @@ function buildSearchResultComparator(query) {
             if (nameDiff !== 0)
                 return nameDiff;
         }
+        const tierDiff = lexicalTierRank(a.lexicalMatch) - lexicalTierRank(b.lexicalMatch);
+        if (tierDiff !== 0)
+            return tierDiff;
         const scoreDiff = displayScore(b.score) - displayScore(a.score);
         if (scoreDiff !== 0)
             return scoreDiff;
@@ -245,9 +282,8 @@ function buildSearchResultComparator(query) {
             return rawScoreDiff;
         // Ceiling values are intentionally allowed to demote visibility, but not
         // to erase relevance. Prefer the score before a relaxed body-only ceiling;
-        // a later belief-state ceiling has its own minScore handoff and must not
-        // overwrite this ordering evidence. Belief-only ceilings fall back to
-        // their `preCeilingScore`.
+        // a later belief-state ceiling must not overwrite this ordering evidence.
+        // Belief-only ceilings fall back to their `preCeilingScore`.
         const preCeilingRelevance = (item) => item.preRelaxedCeilingScore ?? item.preCeilingScore ?? item.score;
         const ceilingDiff = stableRankScore(preCeilingRelevance(b)) - stableRankScore(preCeilingRelevance(a));
         if (ceilingDiff !== 0)
@@ -325,35 +361,13 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             mode: "keyword",
         };
     }
-    // Start the async embedding request without awaiting, then run FTS
-    // synchronously while the HTTP/local embedding request is in-flight.
+    // Start the async embedding request without awaiting, then run the lexical
+    // units_fts query synchronously while the HTTP/local embedding request is
+    // in-flight.
     const typeFilter = searchType === "any" ? undefined : searchType;
-    const { ftsResults, embeddingScores, embedMs, mode, semanticWarning } = await collectSearchSignals(db, query, limit * 3, typeFilter, defaultExcludes, config);
+    const { embedMs, mode, semanticWarning, unitScored } = await collectSearchSignals(db, query, limit * 3, typeFilter, defaultExcludes, config);
     const tRank0 = Date.now();
-    // ── Score normalization ──────────────────────────────────────────────
-    // Stable bounded BM25 transform + cosine similarity with weighted addition
-    // (FTS 0.7, vector 0.3). The lexical transform is per-row, so widening the
-    // candidate set cannot alter a pre-existing row's base score.
-    const ftsScoreMap = normalizeFtsScores(ftsResults);
-    // Build embedding score map (cosine similarities already 0-1)
-    const embedScoreMap = new Map();
-    if (embeddingScores) {
-        for (const [id, cosine] of embeddingScores) {
-            embedScoreMap.set(id, cosine);
-        }
-    }
-    // ── Combine FTS + vector scores ──────────────────────────────────────
-    const scored = combineSearchScores({
-        ftsScoreMap,
-        embedScoreMap,
-        getEntryById: (id) => getEntryById(db, id) ?? undefined,
-        typeFilter,
-        // #627 — also exclude default-hidden types from the vector-only branch so a
-        // session asset that is a top-k vector neighbor (but not an FTS match) does
-        // not leak into default ('any') results. defaultExcludes is already []
-        // unless this is the untyped path without includeExcludedTypes.
-        excludeTypes: defaultExcludes,
-    }).filter(hasIndexedProvenance);
+    const scored = unitScored.filter(hasIndexedProvenance);
     // ── Scoring Phase ──────────────────────────────────────────────────────
     // Apply boosts as multiplicative factors (all boosts in a single phase
     // so that sort order and displayed scores are always consistent).
@@ -419,21 +433,15 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
         positiveFeedbackCounts,
         scopeKey,
     });
-    // ── minScore floor ──────────────────────────────────────────────────────
-    // Drop semantic-only hits (cosine-only, no FTS match) whose score falls
-    // below the configured floor. FTS hits and hybrid hits are always kept.
-    // Default floor: 0.2. Set search.minScore = 0 in config to disable.
-    // Judged on the PRE-ceiling score when a demoting belief state clamped the
-    // item (`preCeilingScore`): the belief ceilings can sit below this floor
-    // (archived 0.15 < 0.2), and a demotion must rank the hit last, not
-    // silently remove a result that would otherwise have listed.
-    const minScore = config.search?.minScore ?? 0.2;
-    const preFilter = minScore > 0
-        ? scored.filter((item) => item.rankingMode !== "semantic" || (item.preCeilingScore ?? item.score) >= minScore)
-        : scored;
-    preFilter.sort(buildSearchResultComparator(query));
+    // The units path's magnitude-fused score (`fuseByEntry` in ranking.ts) is
+    // already the same [0, 1]-ish scale the ranking contributors and the
+    // belief-state ceiling are calibrated for (`stableFtsScore`'s floor/ceiling)
+    // — no separate minScore floor is applied. A demoting belief state already
+    // caps a hit's score and ranks it last via `buildSearchResultComparator`
+    // rather than dropping it.
+    scored.sort(buildSearchResultComparator(query));
     // Deduplicate by file path — keep only the highest-scored entry per file.
-    const deduped = deduplicateByPath(preFilter);
+    const deduped = deduplicateByPath(scored);
     // Source → scope → proposed-quality → derived-twin belief inheritance →
     // belief: the post-candidate filter chain shared with enumerateEntries (see
     // applyEntryFilters). Applied AFTER ranking so filtering narrows the result
@@ -475,6 +483,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             rankingMode,
             lexicalMatch: ranked.lexicalMatch,
             fragmentId: ranked.fragmentId,
+            matchedUnit: ranked.matchedUnit,
             indexedFragment: ranked.fragmentId ? (selectedFragmentByEntryId.get(ranked.id) ?? null) : undefined,
             defaultStashDir: stashDir,
             allSourceDirs,
@@ -489,24 +498,298 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
     }));
     return { embedMs, rankMs, hits, mode, semanticWarning };
 }
+// ── Units search (index-redesign-contract.md B3) ────────────────────────────
+//
+// Every write path (reconcile, and `indexWrittenAssets` for a just-written
+// asset) populates `unit_texts`/`units_fts`/`entry_units` atomically with the
+// `entries` row itself (B1's contract), so there is exactly one search path:
+// lexical `units_fts` fused with semantic `units_vec` by evidence magnitude
+// (`ranking.ts`'s `fuseByEntry` — see its doc for why magnitude, not rank
+// fusion). There is no longer a coverage check to branch on — B5a's generation bump
+// (index-schema.ts) discards `entries` outright on an incompatible schema, so
+// a readable `entries` row always has its `entry_units` sibling.
+/**
+ * `units_fts`/`units_vec` are keyed by UNIT, not by entry, and one entry can
+ * own several units (its structured-fields card plus one per Markdown
+ * fragment). Retrieving only `candidateLimit` units therefore yields fewer
+ * than `candidateLimit` distinct entries once grouped — this scales the
+ * requested `k` by the corpus's observed mean so entry-level recall stays
+ * comparable to the old per-entry candidate pool. 1 is the floor for a
+ * corpus with no `entry_units` rows yet (nothing to divide by).
+ */
+function meanUnitsPerEntry(db) {
+    const row = db
+        .prepare("SELECT AVG(cnt) AS mean FROM (SELECT COUNT(*) AS cnt FROM entry_units GROUP BY entry_id)")
+        .get();
+    const mean = row?.mean;
+    return typeof mean === "number" && Number.isFinite(mean) && mean > 0 ? mean : 1;
+}
+/**
+ * Build the `unit_hash IN (...)` clause that pushes a type predicate into the
+ * SQL BEFORE the candidate cap (item 3 — the confirmed defect: applying
+ * `typeFilter`/`excludeTypes` in JS after `fuseByEntry` filtered a pool that
+ * `LIMIT` already truncated could drop every eligible candidate). A unit is
+ * eligible if it has an owning entry (via `entry_units` → `entries`) that
+ * satisfies both predicates at once — the same entry, not independently
+ * matched rows — which is the correct reading for a unit hash shared by more
+ * than one entry (content-addressed reuse).
+ */
+function buildUnitTypeClause(typeOpts) {
+    if (!typeOpts?.typeFilter?.length && !typeOpts?.excludeTypes?.length)
+        return null;
+    const clauses = [];
+    const params = [];
+    if (typeOpts.typeFilter?.length) {
+        clauses.push(`e.type IN (${typeOpts.typeFilter.map(() => "?").join(",")})`);
+        params.push(...typeOpts.typeFilter);
+    }
+    if (typeOpts.excludeTypes?.length) {
+        clauses.push(`e.type NOT IN (${typeOpts.excludeTypes.map(() => "?").join(",")})`);
+        params.push(...typeOpts.excludeTypes);
+    }
+    return {
+        sql: `unit_hash IN (SELECT eu.unit_hash FROM entry_units eu JOIN entries e ON e.id = eu.entry_id WHERE ${clauses.join(" AND ")})`,
+        params,
+    };
+}
+function runUnitsFtsQuery(db, ftsQuery, lexicalMatch, k, kind, typeOpts) {
+    // `kind` filters via a subquery against `unit_texts` rather than joining
+    // (and aliasing) `units_fts` directly — FTS5's `bm25()` auxiliary function
+    // must name the exact identifier `units_fts` is referenced by in the FROM
+    // clause, so aliasing it would mean threading that alias through `bm25()`
+    // too. The subquery keeps `units_fts` unaliased and lets the MATCH still
+    // drive the query through FTS5's own index (`unit_texts_kind` then narrows
+    // it, see `files-repository.ts`). The type predicate (item 3) is pushed in
+    // the same way, and — critically — BEFORE the `LIMIT`, so an ineligible
+    // unit never occupies a slot a genuinely eligible one needed.
+    const conditions = ["units_fts MATCH ?"];
+    const params = [ftsQuery];
+    if (kind) {
+        conditions.push("unit_hash IN (SELECT unit_hash FROM unit_texts WHERE kind = ?)");
+        params.push(kind);
+    }
+    const typeClause = buildUnitTypeClause(typeOpts);
+    if (typeClause) {
+        conditions.push(typeClause.sql);
+        params.push(...typeClause.params);
+    }
+    params.push(k);
+    const rows = db
+        .prepare(`SELECT unit_hash AS unitHash, bm25(units_fts) AS score
+       FROM units_fts
+       WHERE ${conditions.join(" AND ")}
+       ORDER BY score ASC
+       LIMIT ?`)
+        .all(...params);
+    // Competition ranking (ties share a rank) rather than strict sequential
+    // position: SQLite gives no deterministic secondary order for an exact
+    // bm25 tie, and the final ranking comparator's content-based tie-break
+    // (`canonicalContentTieKey`) needs an exact score tie to survive to ever
+    // run — a strict `index + 1` would silently turn "these two units tied on
+    // relevance" into "this one wins".
+    let rank = 0;
+    let previousScore;
+    return rows.map((row, index) => {
+        if (previousScore === undefined || row.score !== previousScore)
+            rank = index + 1;
+        previousScore = row.score;
+        return { unitHash: row.unitHash, rank, bm25: row.score, lexicalMatch };
+    });
+}
+/**
+ * `units_fts` bm25 lexical search over unit text, ranked best-first,
+ * optionally scoped to one unit `kind`. Mirrors `searchFts`'s own exact →
+ * prefix → relaxed fallback (`index-fts-repository.ts`), but as a PRIORITY
+ * ORDER rather than an early exit (item 2): a unit is a card or one Markdown
+ * section, so a conjunctive query is rarely satisfied by any single unit —
+ * stopping at the first non-empty tier let one incidental hit (e.g. a pasted
+ * stack trace quoting every query token) suppress the far larger, more
+ * relevant relaxed pool. Instead, take the exact hits, then top up with
+ * prefix hits, then relaxed hits, until `k` is reached — each hit keeps the
+ * tier it came from in `lexicalMatch`. Magnitude fusion (`ranking.ts`'s
+ * `fuseByEntry`) is what makes topping up safe: a relaxed-tier junk match now
+ * scores at `stableFtsScore`'s 0.3 floor instead of near the top of a rank
+ * list.
+ *
+ * A genuine bm25 tie at the `k` boundary is never split across the cutoff:
+ * `addTier` finishes the whole tied group even if that pushes the result
+ * past `k`, because the final ranking comparator's content-based tie-break
+ * depends on that exact score tie surviving into `fuseByEntry`'s output.
+ * Ties are only tracked WITHIN one tier's own query — bm25 from different
+ * MATCH queries (exact vs. prefix vs. relaxed) is not comparable, so a
+ * later tier always starts its own fresh rank sequence, offset to continue
+ * numbering after the tiers already taken.
+ */
+function searchUnitsLexicalScoped(db, query, k, kind, typeOpts) {
+    if (k <= 0)
+        return [];
+    const plan = buildLexicalQueryPlan(query);
+    if (!plan.exact)
+        return [];
+    const hits = [];
+    const seen = new Set();
+    let rankOffset = 0;
+    const addTier = (tierHits) => {
+        let lastRank;
+        for (const hit of tierHits) {
+            if (seen.has(hit.unitHash))
+                continue;
+            if (hits.length >= k && hit.rank !== lastRank)
+                break;
+            seen.add(hit.unitHash);
+            hits.push({ ...hit, rank: hit.rank + rankOffset });
+            lastRank = hit.rank;
+        }
+        const tierMaxRank = tierHits[tierHits.length - 1]?.rank ?? 0;
+        rankOffset += tierMaxRank;
+    };
+    addTier(runUnitsFtsQuery(db, plan.exact, "exact", k, kind, typeOpts));
+    if (hits.length < k && plan.exactPrefix) {
+        addTier(runUnitsFtsQuery(db, plan.exactPrefix, "prefix", k, kind, typeOpts));
+    }
+    if (hits.length < k && plan.relaxed) {
+        addTier(runUnitsFtsQuery(db, plan.relaxed, "relaxed", k, kind, typeOpts));
+    }
+    return hits;
+}
+/**
+ * `units_fts` bm25 lexical search over EVERY unit, kind-agnostic — the
+ * original single-pool query, kept for callers that want one flat
+ * entry-level lexical ranking rather than the card/fragment split
+ * `collectSearchSignals` uses (below): `searchEntriesLexical`'s
+ * deterministic-only canary scoring for collapse-detector, which has no use
+ * for field emphasis.
+ */
+export function searchUnitsLexical(db, query, k) {
+    return searchUnitsLexicalScoped(db, query, k);
+}
+/**
+ * `units_fts` bm25 lexical search over BOTH kind-scoped pools (`"card"`,
+ * `"fragment"`) at once (index-redesign-contract.md B5f item 2). Structural
+ * field emphasis: `"card"` units hold name/description/tags/hints and are
+ * few (one per entry), so a name match ranks near the top of a SMALL pool
+ * instead of racing every fragment's body text in one shared BM25 ranking —
+ * the same effect the old per-column BM25 weights (name 10x, description
+ * 5x, ...) bought through tuning, gotten here from the units' own structure
+ * instead.
+ *
+ * Each pool runs its OWN exact → prefix → relaxed priority-order ladder
+ * (item 2 — `searchUnitsLexicalScoped`'s own doc), rather than sharing one
+ * tier decision as an earlier revision did: sharing let an incidental
+ * fragment-exact match (e.g. a pasted stack trace quoting every query token)
+ * lock the card pool out of ever escalating to its own relaxed recovery, so
+ * a well-named relevant entry disappeared behind an unrelated log dump.
+ * Magnitude fusion (`ranking.ts`'s `fuseByEntry`) is what makes independent
+ * ladders safe: a relaxed-tier junk match now scores at `stableFtsScore`'s
+ * 0.3 floor instead of competing on rank, so a stray fragment-side escalation
+ * can no longer crowd out a genuine card-side exact hit the way it would
+ * have under rank fusion.
+ */
+export function searchUnitsLexicalPair(db, query, k, typeOpts) {
+    return {
+        card: searchUnitsLexicalScoped(db, query, k, "card", typeOpts),
+        fragment: searchUnitsLexicalScoped(db, query, k, "fragment", typeOpts),
+    };
+}
+/** Count of `units` rows for the active identity. */
+function getUnitVectorCount(db, identity) {
+    try {
+        const row = db.prepare("SELECT COUNT(*) AS cnt FROM units WHERE identity = ?").get(identity);
+        return row?.cnt ?? 0;
+    }
+    catch {
+        // The design doc's migration story has units_fts populated (lexical
+        // ready) before the first embedding drain completes (`units` empty or
+        // absent) — an expected transient state, not a fault. Lexical-only
+        // results are the correct behavior until the drain catches up.
+        return 0;
+    }
+}
+async function tryUnitVecScores(db, query, k, config, typeOpts) {
+    if (config.semanticSearchMode === "off")
+        return { hits: null };
+    const identity = getMeta(db, "embeddingIdentity");
+    if (!identity || getUnitVectorCount(db, identity) === 0)
+        return { hits: null };
+    try {
+        const { embed } = await import("../../llm/embedder.js");
+        const queryEmbedding = await embed(query, config.embedding);
+        // item 5 — a query embedded under a different identity than the index
+        // must not be trusted, even when it happens to come back the same width
+        // (768/1024/1536 are all common across otherwise-unrelated models): a
+        // width match alone is not a vector-space match. `deriveObservedEmbeddingIdentity`
+        // is the same derivation `drain.ts` uses to learn/verify the identity it
+        // is embedding units under; there is no server-reported model for a
+        // single query `embed()` call (only `embedBatch`'s `onBatch` threads that
+        // through from the provider's response), so this derives from the
+        // CURRENT config the same way drain does whenever the provider's
+        // response echoes the configured model — the case `embedding.model`/
+        // `embedding.endpoint` being edited since the last index actually
+        // exercises. A genuine width mismatch is already safe (sqlite-vec throws
+        // below); this catches the same-width, different-model case that would
+        // otherwise silently compare incompatible vector spaces.
+        const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, undefined, queryEmbedding.length);
+        if (observedIdentity !== identity) {
+            return { hits: null, warning: buildIdentityMismatchWarning(config) };
+        }
+        return { hits: searchUnits(db, queryEmbedding, k, identity, typeOpts) };
+    }
+    catch (error) {
+        return { hits: null, warning: buildVectorFallbackWarning(config, error) };
+    }
+}
 async function collectSearchSignals(db, query, candidateLimit, typeFilter, excludeTypes, config) {
     const startedAt = Date.now();
-    const embeddingPromise = tryVecScores(db, query, candidateLimit, config);
-    const ftsResults = searchFts(db, query, candidateLimit, typeFilter, excludeTypes);
-    const embeddingResult = await embeddingPromise;
-    const mode = embeddingResult.warning
+    const unitK = Math.max(1, Math.round(candidateLimit * meanUnitsPerEntry(db)));
+    // item 3 — push the type predicate into the SQL of both the lexical and
+    // semantic candidate queries, not just into `fuseByEntry`'s post-grouping
+    // JS filter below: applying it only after each list was already cut to
+    // `unitK` by `LIMIT`/`k` let a whole type-excluded pool (e.g. 100 `session`
+    // cards exactly matching the query) crowd a genuinely-matching entry of a
+    // different type out of the candidate window entirely. `fuseByEntry`'s own
+    // filter stays as a cheap guard, not the mechanism.
+    const typeOpts = { typeFilter: typeFilter ? [typeFilter] : undefined, excludeTypes };
+    const semanticPromise = tryUnitVecScores(db, query, unitK, config, typeOpts);
+    // index-redesign-contract.md B5f item 2 — two kind-scoped lexical lists,
+    // not one mixed pool: a card (name/description/tags/hints) match ranks
+    // within its own small pool instead of competing against every fragment's
+    // body text on raw BM25, so field emphasis falls out of the units'
+    // structure rather than tuned per-column weights. Each pool runs its own
+    // priority-order ladder — see `searchUnitsLexicalPair`'s own doc.
+    const { card: cardLexicalHits, fragment: fragmentLexicalHits } = searchUnitsLexicalPair(db, query, unitK, typeOpts);
+    const semanticResult = await semanticPromise;
+    const mode = semanticResult.warning
         ? "fts-fallback"
-        : embeddingResult.scores !== null
+        : semanticResult.hits !== null
             ? "semantic"
             : "keyword";
+    const unitScored = fuseByEntry(db, cardLexicalHits, fragmentLexicalHits, semanticResult.hits ?? [], typeOpts);
     return {
-        ftsResults,
-        embeddingScores: embeddingResult.scores,
         embedMs: Date.now() - startedAt,
         mode,
-        semanticWarning: embeddingResult.warning,
+        semanticWarning: semanticResult.warning,
+        unitScored,
     };
 }
+/**
+ * Entry-level lexical-only search over units, best match first — for
+ * consumers that need ranked entries without semantic fusion (e.g.
+ * collapse-detector's canary scoring, which is deterministic-only by design:
+ * see `src/commands/improve/collapse-detector.ts`). The `units_fts` card unit
+ * carries name/description/tags/hints, so an entry-level lexical search is a
+ * units query grouped by entry — the same grouping `collectSearchSignals`
+ * uses, with an empty semantic list so `fuseByEntry`'s magnitude fusion
+ * degenerates to a pure lexical-bm25 ordering.
+ */
+export function searchEntriesLexical(db, query, k) {
+    const unitK = Math.max(1, Math.round(k * meanUnitsPerEntry(db)));
+    const lexicalHits = searchUnitsLexical(db, query, unitK);
+    // One flat kind-agnostic pool, not the card/fragment split
+    // `collectSearchSignals` uses — deliberately: this is the kind-agnostic
+    // single-list mode `fuseByEntry` still supports for a caller with no use
+    // for field emphasis (see `searchUnitsLexical`'s own doc).
+    return fuseByEntry(db, lexicalHits, [], []).sort((a, b) => b.score - a.score);
+}
 /**
  * The no-hits tip. A query in the retired `<type>:` / `<type>:<prefix>/` browse
  * grammar gets the conceptId spelling that replaces it: without this it comes
@@ -688,33 +971,6 @@ function matchBeliefFilter(beliefState, filter) {
         beliefState === "archived");
 }
 // ── Vector scorer ───────────────────────────────────────────────────────────
-async function tryVecScores(db, query, k, config) {
-    if (config.semanticSearchMode === "off")
-        return { scores: null };
-    // A real-time completeness fact, not a cached verdict: skip the network
-    // round trip only when the index has never embedded anything. A PARTIAL
-    // failure (some entries embedded, one write degraded) still attempts —
-    // and if the endpoint is genuinely down, the failure surfaces as a live
-    // `semanticWarning` below instead of silently skipping with no signal.
-    if (getEmbeddingCount(db) === 0)
-        return { scores: null };
-    try {
-        const { embed } = await import("../../llm/embedder.js");
-        const queryEmbedding = await embed(query, config.embedding);
-        const vecResults = searchVec(db, queryEmbedding, k);
-        const scores = new Map();
-        for (const { id, distance } of vecResults) {
-            // Convert L2 distance to cosine similarity (vectors are normalized).
-            // Guard against NaN/Infinity from sqlite-vec edge cases.
-            const raw = 1 - (distance * distance) / 2;
-            scores.set(id, Number.isFinite(raw) ? Math.max(0, raw) : 0);
-        }
-        return { scores };
-    }
-    catch (error) {
-        return { scores: null, warning: buildVectorFallbackWarning(config, error) };
-    }
-}
 function buildVectorFallbackWarning(config, error) {
     const endpoint = safeEmbeddingEndpoint(config);
     const reason = classifyVectorFailure(error);
@@ -726,6 +982,21 @@ function buildVectorFallbackWarning(config, error) {
     const unavailable = reason === "connection failed" ? `cannot reach ${target}` : `${target} is unavailable`;
     return `Vector search unavailable: ${unavailable} (${reason}) — falling back to keyword search.`;
 }
+/**
+ * item 5 — same shape as {@link buildVectorFallbackWarning}, for the case
+ * where embedding itself succeeded but the query was embedded under a
+ * different identity than the index (`embedding.model`/`embedding.endpoint`
+ * edited since the last index run).
+ */
+function buildIdentityMismatchWarning(config) {
+    const endpoint = safeEmbeddingEndpoint(config);
+    const target = endpoint
+        ? `embedding endpoint ${endpoint}`
+        : config.embedding?.endpoint
+            ? "configured embedding endpoint"
+            : "local embedding model";
+    return `Vector search unavailable: ${target} is embedding queries under a different identity than the index was built with (embedding.model/embedding.endpoint changed since the last index) — falling back to keyword search.`;
+}
 /**
  * Name the useful endpoint without ever carrying URL userinfo, query secrets,
  * or fragments into a warning. Invalid authored values fail closed.
@@ -792,24 +1063,36 @@ export async function buildDbHit(input) {
             ? (input.bundleId ?? undefined)
             : undefined);
     const parentRef = resolveSearchHitRef(input.entry, input, defaultBundleId);
-    // Fragments prove lexical relevance, but executable assets must retain the
-    // parent ref consumed by their advertised action (for example workflow run).
-    // The central type-presentation contract opts those types out explicitly.
-    const ref = input.fragmentId && allowsFragmentRef(input.entry.type) ? `${parentRef}#${input.fragmentId}` : parentRef;
+    // index-redesign-contract.md B5f item 1 — the hit's primary `ref` is ALWAYS
+    // the entry ref now, never `${parentRef}#${fragmentId}`. On the units path
+    // the best-matching unit for a hit is routinely a Markdown fragment (unit
+    // kind `fragment`), so a fragment-suffixed `ref` here would silently mismatch
+    // every consumer (a judgment, a stored `derivedFrom`, a copy-pasted CLI
+    // command) that names the bare entry. A consumer that genuinely wants the
+    // matched fragment's own ref reads `selectedRef` below instead — computed
+    // exactly the way `ref` itself used to be, so its availability (gated by
+    // `allowsFragmentRef`) is unchanged; only the PRIMARY ref stopped carrying it.
+    const ref = parentRef;
     const editable = isEditable(absolutePath, input.config, input.sources);
     const indexedFragment = input.indexedFragment === undefined
         ? input.fragmentId && input.db
             ? getIndexedMarkdownFragment(input.db, input.itemRef, input.fragmentId)
             : undefined
         : (input.indexedFragment ?? undefined);
-    const selectedRef = input.fragmentId && ref !== parentRef ? `${parentRef}#${input.fragmentId}` : undefined;
+    // Fragments prove lexical relevance, but executable assets must retain the
+    // parent ref consumed by their advertised action (for example workflow run).
+    // The central type-presentation contract opts those types out explicitly.
+    const selectedRef = input.fragmentId && allowsFragmentRef(input.entry.type) ? `${parentRef}#${input.fragmentId}` : undefined;
     const parentEstimatedTokens = typeof input.entry.fileSize === "number"
         ? Math.round(input.entry.fileSize / 4)
         : indexedFragment
             ? Math.round(indexedFragment.parentChars / 4)
             : undefined;
     const fragmentEstimatedTokens = indexedFragment ? Math.round(indexedFragment.fragmentChars / 4) : undefined;
-    const estimatedTokens = selectedRef === ref && fragmentEstimatedTokens !== undefined ? fragmentEstimatedTokens : parentEstimatedTokens;
+    // `ref` addresses the whole entry now (see above), so the size it stands for
+    // is always the parent's — a caller that wants the fragment's own size reads
+    // `fragmentEstimatedTokens` from the `selectedRef` block below.
+    const estimatedTokens = parentEstimatedTokens;
     const hit = {
         type: input.entry.type,
         name: input.entry.name,
@@ -861,6 +1144,7 @@ export async function buildDbHit(input) {
         // hit. Omitted when the hit has no FTS component (pure-semantic hybrid
         // contribution).
         ...(input.lexicalMatch ? { matchStage: input.lexicalMatch } : {}),
+        ...(input.matchedUnit ? { matchedUnit: input.matchedUnit } : {}),
     };
     attachDbHitAttribution(hit, input);
     if (input.entry.derivedFrom) {

package/dist/indexer/search/ranking-contributors.js

@@ -106,20 +106,25 @@ function beliefStateBoost(item) {
  * stash-conventions-code-spec.md — corrections demotion).
  *
  * Why the additive {@link beliefStateBoost} penalties alone are not enough:
- * keyword base scores have a bounded lexical floor (`normalizeFtsScores`),
- * while the boost sum then MULTIPLIES the base (`score *= 1 + boostSum`,
- * {@link applyScoreContributors}). A superseded incumbent can still earn
- * enough independent boosts to outrank its own correction, so additive
- * penalties alone cannot guarantee the corrections pattern's point ("so the
- * ranker demotes the stale version instead of letting it outrank your fix").
+ * keyword base scores have a bounded lexical floor (`stableFtsScore`, in
+ * `core/lexical-score.ts` — the calibrated per-row bm25 transform
+ * `ranking.ts`'s `fuseByEntry` fuses on), while the boost sum then
+ * MULTIPLIES the base (`score *= 1 + boostSum`, {@link applyScoreContributors}).
+ * A superseded incumbent can still earn enough independent boosts to
+ * outrank its own correction, so additive penalties alone cannot guarantee
+ * the corrections pattern's point ("so the ranker demotes the stale version
+ * instead of letting it outrank your fix").
  *
  * The ceilings guarantee the demotion while keeping flagged entries VISIBLE:
- * un-demoted keyword hits floor at a 0.3 base, so any un-demoted hit outranks
- * a ceilinged one; demoted entries still list (belief FILTERING stays a
- * separate opt-in axis, `--belief`), and scores already below a ceiling keep
- * their relative ordering. Ceiling order mirrors the additive-penalty
- * severity order pinned in tests/integration/belief-state-phase1a.test.ts:
- * deprecated (mildest) > superseded > contradicted > archived.
+ * un-demoted keyword hits floor at `stableFtsScore`'s 0.3 base, so any
+ * un-demoted matching hit outranks a ceilinged one (verified end-to-end by
+ * `search-units-fusion.test.ts`'s belief-ceiling-invariant case, not just the
+ * ceiling constants below); demoted entries still list (belief FILTERING
+ * stays a separate opt-in axis, `--belief`), and scores already below a
+ * ceiling keep their relative ordering. Ceiling order mirrors the
+ * additive-penalty severity order pinned in
+ * tests/integration/belief-state-phase1a.test.ts: deprecated (mildest) >
+ * superseded > contradicted > archived.
  */
 const BELIEF_STATE_SCORE_CEILINGS = {
     deprecated: 0.28,
@@ -134,10 +139,10 @@ const BELIEF_STATE_SCORE_CEILINGS = {
  * order and displayed scores stay consistent (single scoring pipeline).
  *
  * When the ceiling clamps, the pre-clamp score is recorded as
- * `preCeilingScore` so db-search's semantic-only `minScore` floor can judge
- * the hit by what it would have scored WITHOUT the demotion — a ceiling below
- * the floor (archived 0.15 < default minScore 0.2) must demote a hit to last
- * place, never silently drop it from the results.
+ * `preCeilingScore` so db-search's final ranking comparator can order
+ * demoted hits by what they would have scored WITHOUT the demotion — a
+ * ceilinged item still lists (ranked last among un-demoted peers), never
+ * silently dropped from the results.
  */
 export function applyBeliefStateScoreCeiling(item) {
     const state = item.entry.beliefState;