akm-cli 0.9.1 → 0.9.2-alpha.2

package/dist/indexer/search/db-search.js

@@ -1,14 +1,27 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Database-backed (SQLite + FTS5/vector) source search implementation.
+ *
+ * Extracted from source-search.ts to break the circular import:
+ *   source-search.ts → sources/providers/filesystem.ts → db-search.ts (no cycle)
+ *
+ * source-search.ts imports this module for the `searchLocal` export.
+ * sources/providers/filesystem.ts also imports `searchLocal` from here.
+ *
+ * Renamed from `local-search.ts` to signal that this is the DB-layer search
+ * implementation, not a "local vs. remote" distinction.
+ */
 import path from "node:path";
 import { buildActionFromContributors, defaultActionContributors } from "../../core/action-contributors.js";
 import { stashDirFor } from "../../core/asset/asset-placement.js";
 import { displayRef } from "../../core/asset/resolve-ref.js";
 import { classifyPathAccess } from "../../core/path-access.js";
 import { getDbPath } from "../../core/paths.js";
+import { systemErrorCode } from "../../core/system-error.js";
 import { defaultRendererRegistry } from "../../core/type-presentation.js";
-import { warn } from "../../core/warn.js";
+import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
 import { assertIndexPathReadable, closeDatabase, openExistingDatabase, } from "../../storage/repositories/index-connection.js";
 import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntryById, getEntryCount, getPositiveFeedbackCountsByIds, } from "../../storage/repositories/index-entries-repository.js";
 import { searchFts } from "../../storage/repositories/index-fts-repository.js";
@@ -19,8 +32,9 @@ import { ensureIndex } from "../ensure-index.js";
 import { collectGraphRelatedHit, loadGraphBoostContext } from "../graph/graph-boost.js";
 import { isProposedQuality } from "../passes/metadata.js";
 import { resolveProjectContext } from "../walk/project-context.js";
-import { parseRefPrefixQuery, parseRetiredTypePrefixQuery, sanitizeFtsQuery } from "./fts-query.js";
-import { applyRankingRules, combineSearchScores, normalizeFtsScores } from "./ranking.js";
+import { buildLexicalQueryPlan, parseRefPrefixQuery, parseRetiredTypePrefixQuery, } from "./fts-query.js";
+import { applyRankingRules, combineSearchScores, lexicalNameMatchTier, normalizeFtsScores } from "./ranking.js";
+import { typeBoostFor } from "./ranking-contributors.js";
 import { attachSearchHitAttribution, copySearchHitAttribution, getSearchHitAttribution } from "./search-attribution.js";
 import { enrichSearchHit } from "./search-hit-enrichers.js";
 import { buildEditHint, findSourceForPath, isEditable } from "./search-source.js";
@@ -188,7 +202,9 @@ export async function searchLocal(input) {
         const staleHint = buildStaleIndexHint(db);
         if (staleHint)
             warnings.push(staleHint);
-        const { hits, embedMs, rankMs, usedSemantic } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes, disableProjectContext, disableScopedUtility);
+        const { hits, embedMs, rankMs, mode, semanticWarning } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes, disableProjectContext, disableScopedUtility);
+        if (semanticWarning)
+            warnings.push(semanticWarning);
         return {
             hits,
             tip: hits.length === 0 ? emptyResultTip(query) : undefined,
@@ -197,7 +213,7 @@ export async function searchLocal(input) {
             rankMs,
             // Report the mode the search ACTUALLY used, carried explicitly from the
             // vector scorer — not inferred from elapsed embedding milliseconds.
-            mode: usedSemantic ? "semantic" : "keyword",
+            mode,
         };
     }
     finally {
@@ -205,8 +221,38 @@ export async function searchLocal(input) {
     }
 }
 // ── Database search ─────────────────────────────────────────────────────────
+/**
+ * Keep one deterministic ranking order before stable path deduplication. Exact
+ * names survive the public score ceiling, while raw contributor differences
+ * are quantized so utility-recency epsilon cannot reorder visible ties.
+ */
+function buildSearchResultComparator(query) {
+    const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
+    const displayScore = (score) => Math.round(Math.min(1, Math.max(0, score)) * 10000) / 10000;
+    const stableRankScore = (score) => Math.round(score * 10000) / 10000;
+    return (a, b) => {
+        const aNameTier = lexicalNameMatchTier(a.entry, queryTokens);
+        const bNameTier = lexicalNameMatchTier(b.entry, queryTokens);
+        if (aNameTier === 3 || bNameTier === 3) {
+            const nameDiff = bNameTier - aNameTier;
+            if (nameDiff !== 0)
+                return nameDiff;
+        }
+        const scoreDiff = displayScore(b.score) - displayScore(a.score);
+        if (scoreDiff !== 0)
+            return scoreDiff;
+        const rawScoreDiff = stableRankScore(b.score) - stableRankScore(a.score);
+        if (rawScoreDiff !== 0)
+            return rawScoreDiff;
+        const nameDiff = bNameTier - aNameTier;
+        if (nameDiff !== 0)
+            return nameDiff;
+        const typeDiff = typeBoostFor(b.entry.type) - typeBoostFor(a.entry.type);
+        return typeDiff || a.filePath.localeCompare(b.filePath);
+    };
+}
 async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false, disableProjectContext = false, disableScopedUtility = false) {
-    const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
+    const hasSearchableTokens = query.length > 0 && buildLexicalQueryPlan(query).tokens.length > 0;
     // #627 — resolve the default type-exclusion policy. It applies ONLY on the
     // untyped ('any') path and only when the caller did not opt back in via
     // `includeExcludedTypes`. When the config key is ABSENT a built-in default of
@@ -246,7 +292,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
                 conceptIdPrefix: refPrefix.conceptIdPrefix,
                 ...(refPrefix.bundle !== undefined ? { bundle: refPrefix.bundle } : {}),
             })),
-            usedSemantic: false,
+            mode: "keyword",
         };
     }
     // Empty queries — including ones that sanitize down to no searchable FTS
@@ -260,23 +306,13 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
                 typeFilter: searchType === "any" ? undefined : searchType,
                 excludeTypes: defaultExcludes,
             })),
-            usedSemantic: false,
+            mode: "keyword",
         };
     }
     // Start the async embedding request without awaiting, then run FTS
     // synchronously while the HTTP/local embedding request is in-flight.
     const typeFilter = searchType === "any" ? undefined : searchType;
-    const tEmbed0 = Date.now();
-    const embeddingPromise = tryVecScores(db, query, limit * 3, config);
-    const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
-    const embeddingScores = await embeddingPromise;
-    const embedMs = Date.now() - tEmbed0;
-    // The vector scorer returns a (possibly empty) Map when the embedding + vector
-    // search actually executed, or null when semantic was not runnable (disabled,
-    // no embeddings, or the embed call threw). This is the AUTHORITATIVE "semantic
-    // mode was used" signal — carried out to telemetry instead of guessing from
-    // elapsed milliseconds (which timed the concurrent FTS work too).
-    const usedSemantic = embeddingScores !== null;
+    const { ftsResults, embeddingScores, embedMs, mode, semanticWarning } = await collectSearchSignals(db, query, limit * 3, typeFilter, defaultExcludes, config);
     const tRank0 = Date.now();
     // ── Score normalization ──────────────────────────────────────────────
     // Normalized BM25 + cosine similarity with weighted addition
@@ -378,21 +414,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
     const preFilter = minScore > 0
         ? scored.filter((item) => item.rankingMode !== "semantic" || (item.preCeilingScore ?? item.score) >= minScore)
         : scored;
-    // Deterministic tiebreaker on equal scores.
-    //
-    // CRITICAL: sort on the SAME clamped+rounded value the user sees (see the
-    // `finalScore`/round-to-4dp logic below at buildDbHit), NOT the raw pre-clamp
-    // `item.score`. The boost loop can push scores above 1.0 (utility, graph,
-    // project boosts) and carries ~15 significant digits. Two entries that DISPLAY
-    // an identical score (e.g. both clamp to 1.0000) can still differ in their raw
-    // pre-clamp score by a timing-dependent epsilon — utility recency uses
-    // `Date.now()` and `last_used_at`, so the same query run twice in one process
-    // can yield raw scores that diverge at the 6th decimal. Sorting on the raw
-    // value lets that invisible epsilon decide the order, so the visible name
-    // tiebreaker never engages and the order flips run-to-run (Issue #14). Quantize
-    // to the display value first; only then does `localeCompare` break true ties.
-    const displayScore = (s) => Math.round(Math.min(1, Math.max(0, s)) * 10000) / 10000;
-    preFilter.sort((a, b) => displayScore(b.score) - displayScore(a.score) || a.entry.name.localeCompare(b.entry.name));
+    preFilter.sort(buildSearchResultComparator(query));
     // Deduplicate by file path — keep only the highest-scored entry per file.
     const deduped = deduplicateByPath(preFilter);
     // Source → scope → proposed-quality → derived-twin belief inheritance →
@@ -426,6 +448,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             score: Math.round(finalScore * 10000) / 10000,
             query,
             rankingMode,
+            lexicalMatch: ranked.lexicalMatch,
             defaultStashDir: stashDir,
             allSourceDirs,
             sources,
@@ -437,7 +460,25 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             db,
         });
     }));
-    return { embedMs, rankMs, hits, usedSemantic };
+    return { embedMs, rankMs, hits, mode, semanticWarning };
+}
+async function collectSearchSignals(db, query, candidateLimit, typeFilter, excludeTypes, config) {
+    const startedAt = Date.now();
+    const embeddingPromise = tryVecScores(db, query, candidateLimit, config);
+    const ftsResults = searchFts(db, query, candidateLimit, typeFilter, excludeTypes);
+    const embeddingResult = await embeddingPromise;
+    const mode = embeddingResult.warning
+        ? "fts-fallback"
+        : embeddingResult.scores !== null
+            ? "semantic"
+            : "keyword";
+    return {
+        ftsResults,
+        embeddingScores: embeddingResult.scores,
+        embedMs: Date.now() - startedAt,
+        mode,
+        semanticWarning: embeddingResult.warning,
+    };
 }
 /**
  * The no-hits tip. A query in the retired `<type>:` / `<type>:<prefix>/` browse
@@ -537,7 +578,7 @@ async function enumerateEntries(opts) {
  * What this does NOT unify — and deliberately leaves divergent — is CANDIDATE-
  * POOL construction, which is inherent search-vs-browse semantics: the scored
  * path's pool is `searchFts`/vector matches for the query's own tokens (FTS
- * indexes description/tags/searchHints/aliases, not raw body prose), while the
+ * includes structured fields and bounded adapter content), while the
  * enumerate path's pool is `getAllEntries` for the type, independent of query
  * text. A derived twin sharing no indexed token with the query is therefore an
  * enumerate-path candidate but never a scored-path candidate — see
@@ -623,10 +664,10 @@ function matchBeliefFilter(beliefState, filter) {
 async function tryVecScores(db, query, k, config) {
     const semanticStatus = getEffectiveSemanticStatus(config, readSemanticStatus());
     if (!isSemanticRuntimeReady(semanticStatus))
-        return null;
+        return { scores: null };
     const hasEmbeddings = getMeta(db, "hasEmbeddings");
     if (hasEmbeddings !== "1")
-        return null;
+        return { scores: null };
     try {
         const { embed } = await import("../../llm/embedder.js");
         const queryEmbedding = await embed(query, config.embedding);
@@ -638,12 +679,64 @@ async function tryVecScores(db, query, k, config) {
             const raw = 1 - (distance * distance) / 2;
             scores.set(id, Number.isFinite(raw) ? Math.max(0, raw) : 0);
         }
-        return scores;
+        return { scores };
     }
     catch (error) {
-        warn("Vector search failed, skipping:", error instanceof Error ? error.message : String(error));
-        return null;
+        return { scores: null, warning: buildVectorFallbackWarning(config, error) };
+    }
+}
+function buildVectorFallbackWarning(config, error) {
+    const endpoint = safeEmbeddingEndpoint(config);
+    const reason = classifyVectorFailure(error);
+    const target = endpoint
+        ? `embedding endpoint ${endpoint}`
+        : config.embedding?.endpoint
+            ? "configured embedding endpoint"
+            : "local embedding model";
+    const unavailable = reason === "connection failed" ? `cannot reach ${target}` : `${target} is unavailable`;
+    return `Vector search unavailable: ${unavailable} (${reason}) — falling back to keyword search.`;
+}
+/**
+ * Name the useful endpoint without ever carrying URL userinfo, query secrets,
+ * or fragments into a warning. Invalid authored values fail closed.
+ */
+function safeEmbeddingEndpoint(config) {
+    const endpoint = config.embedding?.endpoint;
+    if (!endpoint)
+        return undefined;
+    try {
+        const parsed = new URL(normalizeEmbeddingEndpoint(endpoint));
+        parsed.username = "";
+        parsed.password = "";
+        parsed.search = "";
+        parsed.hash = "";
+        return parsed.toString();
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Map untrusted runtime failures onto a small, non-secret diagnostic set. */
+function classifyVectorFailure(error) {
+    const code = systemErrorCode(error);
+    const message = error instanceof Error ? error.message : "";
+    if (code === "ECONNREFUSED" ||
+        code === "ECONNRESET" ||
+        code === "ENETUNREACH" ||
+        code === "EHOSTUNREACH" ||
+        code === "ENOTFOUND" ||
+        code === "EAI_AGAIN" ||
+        /typo in the url or port|connection (?:refused|failed|reset)|fetch failed/i.test(message)) {
+        return "connection failed";
     }
+    if (code === "ETIMEDOUT" || /timed? out|timeout/i.test(message))
+        return "request timed out";
+    const httpStatus = message.match(/Embedding (?:batch )?request failed \((\d{3})\)/i)?.[1];
+    if (httpStatus)
+        return `HTTP ${httpStatus}`;
+    if (/unexpected embedding response|missing data\[0\]\.embedding/i.test(message))
+        return "invalid embedding response";
+    return "request failed";
 }
 // ── Hit building ────────────────────────────────────────────────────────────
 export async function buildDbHit(input) {
@@ -661,7 +754,7 @@ export async function buildDbHit(input) {
     // Round to 4 decimal places, no boost multiplication
     const score = Math.round(input.score * 10000) / 10000;
     const graphBoost = getSearchHitAttribution(input.attributionSource ?? {})?.graphExtraction?.boost ?? 0;
-    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost);
+    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost, input.lexicalMatch);
     const graphHit = input.graphContext ? collectGraphRelatedHit(input.graphContext, absolutePath) : null;
     const source = findSourceForPath(absolutePath, input.sources);
     const defaultBundleId = input.config?.defaultBundle ??
@@ -693,8 +786,7 @@ export async function buildDbHit(input) {
         ...(input.entry.currentBeliefRefs ? { currentBeliefRefs: input.entry.currentBeliefRefs } : {}),
         ...(graphHit ? { graph: { entities: graphHit.entities, relations: graphHit.relations } } : {}),
     };
-    if (input.attributionSource)
-        copySearchHitAttribution(input.attributionSource, hit);
+    attachDbHitAttribution(hit, input);
     if (input.entry.derivedFrom) {
         attachSearchHitAttribution(hit, {
             memoryInference: { exposure: "direct" },
@@ -703,14 +795,27 @@ export async function buildDbHit(input) {
     await enrichSearchHit(hit, {
         type: input.entry.type,
         stashDir: entryStashDir,
+        bundleId: input.bundleId,
         rendererRegistry,
         db: input.db,
     });
     return hit;
 }
+function attachDbHitAttribution(hit, input) {
+    if (input.lexicalMatch) {
+        attachSearchHitAttribution(hit, {
+            lexical: {
+                execution: input.lexicalMatch,
+                nameMatchTier: lexicalNameMatchTier(input.entry, buildLexicalQueryPlan(input.query).tokens),
+            },
+        });
+    }
+    if (input.attributionSource)
+        copySearchHitAttribution(input.attributionSource, hit);
+}
 export function buildWhyMatched(entry, query, 
 // "hybrid" ranking mode
-rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost) {
+rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost, lexicalMatch) {
     const reasons = [
         rankingMode === "hybrid"
             ? "hybrid (fts + semantic)"
@@ -718,6 +823,8 @@ rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost) {
                 ? "semantic similarity"
                 : "fts bm25 relevance",
     ];
+    if (lexicalMatch === "relaxed")
+        reasons.push("lexical recovery after strict query returned no hits");
     const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
     const queryLower = query.toLowerCase().trim();
     const name = entry.name.toLowerCase();
@@ -782,14 +889,13 @@ export function deriveSize(bytes) {
     return "large";
 }
 /**
- * Deduplicate scored results by file path, keeping only the highest-scored
- * entry per unique path. Sorts by score descending internally to ensure the
- * precondition is always met regardless of caller.
+ * Deduplicate the already-ranked result stream by file path. The caller owns
+ * the one ranking order; re-sorting here would silently discard exact-name and
+ * relaxed-recovery ordering in favor of an internal pre-clamp score.
  */
 function deduplicateByPath(items) {
-    const sorted = [...items].sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.filePath.localeCompare(b.filePath));
     const seen = new Set();
-    return sorted.filter((item) => {
+    return items.filter((item) => {
         if (seen.has(item.filePath))
             return false;
         seen.add(item.filePath);

package/dist/indexer/search/fts-query.js

@@ -2,54 +2,54 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Pure FTS5 query-string helpers, extracted from indexer/db/db.ts.
+ * Pure FTS5 query planning and ref-query helpers.
  *
- * These transform a raw user query into an FTS5-safe MATCH expression. They
- * touch no database state, so they are unit-testable with zero DB setup.
+ * The lexical planner transforms a raw user query into bounded FTS5-safe
+ * MATCH expressions. It touches no database state, so it is unit-testable
+ * with zero DB setup.
  * `parseRefPrefixQuery` is the one non-FTS helper: it decides whether a raw
  * query should bypass FTS entirely (SPEC-4 ref-prefix enumeration).
  */
-/**
- * Sanitize a raw user query into an FTS5-safe implicit-AND expression.
- *
- * Allows only characters safe in FTS5 queries: letters, digits, underscores,
- * and whitespace. Everything else (hyphens, dots, quotes, parens, asterisks,
- * colons, carets, @, !, etc.) is replaced with a space so that compound
- * identifiers like "code-review" or "k8s.setup" become AND-joined tokens
- * ("code review", "k8s setup") rather than triggering FTS5 syntax errors.
- */
-export function sanitizeFtsQuery(query) {
-    let sanitized = query.replace(/[^a-zA-Z0-9_\s]/g, " ");
-    // Neutralize the NEAR operator (FTS5 proximity syntax)
-    sanitized = sanitized.replace(/\bNEAR\b/g, " ");
-    const tokens = sanitized.split(/\s+/).filter((t) => t.length >= 1);
-    if (tokens.length === 0)
-        return "";
-    // Use implicit AND (space-separated tokens) for precision. FTS5 treats
-    // space-separated tokens as an implicit AND, matching only rows that
-    // contain ALL terms.
-    return tokens.join(" ");
+/** Maximum number of distinct lexical terms one query may execute. */
+export const MAX_LEXICAL_QUERY_TOKENS = 16;
+const UNICODE_TOKEN = /[\p{L}\p{N}]+/gu;
+function quoteToken(token) {
+    return `"${token}"`;
+}
+function prefixToken(token) {
+    return [...token].length >= 3 ? `${quoteToken(token)}*` : quoteToken(token);
 }
 /**
- * Build a prefix query from an FTS5 query string by appending `*` to each
- * token that is 3+ characters long. Tokens shorter than 3 characters are
- * kept as-is (no prefix expansion) to avoid overly broad matches.
+ * Build the sole lexical retrieval plan from raw user input.
  *
- * Returns null if no tokens qualify for prefix expansion.
+ * Tokenization follows the useful portion of SQLite FTS5's `unicode61`
+ * tokenizer (Unicode letters and numbers). Quoting every term makes FTS
+ * operators ordinary searchable words. Tokens are normalized, deduplicated
+ * case-insensitively, and capped before any SQL executes.
  */
-export function buildPrefixQuery(ftsQuery) {
-    const tokens = ftsQuery.split(/\s+/).filter(Boolean);
-    let hasPrefix = false;
-    const prefixTokens = tokens.map((t) => {
-        if (t.length >= 3) {
-            hasPrefix = true;
-            return `${t}*`;
-        }
-        return t;
-    });
-    if (!hasPrefix)
-        return null;
-    return prefixTokens.join(" ");
+export function buildLexicalQueryPlan(query) {
+    const tokens = [];
+    const seen = new Set();
+    const normalized = query.normalize("NFKC");
+    for (const match of normalized.matchAll(UNICODE_TOKEN)) {
+        const token = match[0];
+        const key = token.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        tokens.push(token);
+        if (tokens.length === MAX_LEXICAL_QUERY_TOKENS)
+            break;
+    }
+    const exact = tokens.map(quoteToken).join(" ");
+    const prefixTokens = tokens.map(prefixToken);
+    const exactPrefix = prefixTokens.some((token) => token.endsWith("*")) ? prefixTokens.join(" ") : undefined;
+    // A slash-bearing, whitespace-free input is an identifier/ref lookup, not
+    // sentence prose. Keep it conjunctive so a mistyped/bare ref never fans out
+    // across every path token through OR recovery.
+    const isRefLikeIdentifier = !/\s/u.test(query.trim()) && query.includes("/");
+    const relaxed = tokens.length > 1 && !isRefLikeIdentifier ? prefixTokens.join(" OR ") : undefined;
+    return { tokens, exact, exactPrefix, relaxed };
 }
 /**
  * D4 — parse a conceptId-prefix browse query.

package/dist/indexer/search/ranking.js

@@ -2,6 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { getUtilityScoresByIds } from "../../storage/repositories/index-utility-repository.js";
+import { buildLexicalQueryPlan } from "./fts-query.js";
 import { applyBeliefStateScoreCeiling, applyContributorAblation, applyScoreContributors, applyUtilityContributors, defaultRankingContributors, defaultUtilityRankingContributors, } from "./ranking-contributors.js";
 export function normalizeFtsScores(results) {
     const ftsScoreMap = new Map();
@@ -33,6 +34,7 @@ export function combineSearchScores(options) {
             filePath: result.filePath,
             score: combinedScore,
             rankingMode: embedScore !== undefined ? "hybrid" : "fts",
+            lexicalMatch: result.lexicalMatch,
             itemRef: result.itemRef,
             bundleId: result.bundleId,
             conceptId: result.conceptId,
@@ -63,7 +65,7 @@ export function combineSearchScores(options) {
     return scored;
 }
 export function applyRankingRules(options) {
-    const queryTokens = options.query.toLowerCase().split(/\s+/).filter(Boolean);
+    const queryTokens = buildLexicalQueryPlan(options.query).tokens.map((token) => token.toLowerCase());
     const queryLower = options.query.toLowerCase().trim();
     const rankingContext = {
         db: options.db,
@@ -103,6 +105,7 @@ export function applyRankingRules(options) {
     };
     for (const item of options.items) {
         applyUtilityContributors(item, utilityContext, activeUtilityContributors);
+        applyRelaxedLexicalScoreCeiling(item, queryTokens);
         // SPEC-5: demoting belief states (superseded/contradicted/archived/
         // deprecated) cap the FINAL score. The additive belief penalty inside the
         // multiplicative boost sum cannot overcome the FTS min-max normalization
@@ -112,3 +115,35 @@ export function applyRankingRules(options) {
     }
     return options.items;
 }
+const RELAXED_NON_NAME_SCORE_CEILING = 0.65;
+/**
+ * Rank name evidence without relying on punctuation or ASCII-only splitting.
+ * The tiers are intentionally structural: an exact normalized name, all query
+ * tokens in a longer name, any query token in the name, or no name evidence.
+ */
+export function lexicalNameMatchTier(entry, queryTokens) {
+    if (queryTokens.length === 0)
+        return 0;
+    const nameBase = entry.name.toLowerCase().split("/").pop() ?? entry.name.toLowerCase();
+    const nameTokens = buildLexicalQueryPlan(nameBase).tokens.map((token) => token.toLowerCase());
+    const tokenMatches = (left, right) => left === right ||
+        (Math.min([...left].length, [...right].length) >= 3 && (left.startsWith(right) || right.startsWith(left)));
+    if (nameTokens.length === queryTokens.length &&
+        nameTokens.every((token, index) => tokenMatches(token, queryTokens[index]))) {
+        return 3;
+    }
+    const matched = queryTokens.filter((token) => nameTokens.some((nameToken) => tokenMatches(nameToken, token))).length;
+    if (matched === queryTokens.length)
+        return 2;
+    return matched > 0 ? 1 : 0;
+}
+/**
+ * A relaxed OR query admits intentionally weak candidates. Candidates with no
+ * query token in their name remain visible for body-only recall, but cannot
+ * saturate at the same displayed score as stronger name-bearing recoveries.
+ */
+function applyRelaxedLexicalScoreCeiling(item, queryTokens) {
+    if (item.lexicalMatch !== "relaxed" || lexicalNameMatchTier(item.entry, queryTokens) > 0)
+        return;
+    item.score = Math.min(item.score, RELAXED_NON_NAME_SCORE_CEILING);
+}

package/dist/indexer/search/search-attribution.js

@@ -17,11 +17,13 @@ export function copySearchHitAttribution(from, to, outputDescription) {
     const memorySurvives = memoryInference?.exposure !== "surface" ||
         (memoryInference.surfaceDescription !== undefined && memoryInference.surfaceDescription === outputDescription);
     const applicable = {
+        ...(attribution.lexical ? { lexical: attribution.lexical } : {}),
         ...(memorySurvives && memoryInference ? { memoryInference } : {}),
         ...(attribution.graphExtraction ? { graphExtraction: attribution.graphExtraction } : {}),
     };
-    if (applicable.memoryInference || applicable.graphExtraction)
+    if (applicable.lexical || applicable.memoryInference || applicable.graphExtraction) {
         attachSearchHitAttribution(to, applicable);
+    }
 }
 export function getSearchHitAttribution(target) {
     return target[ATTRIBUTION];

package/dist/indexer/search/search-fields.js

@@ -1,6 +1,8 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Structured metadata plus bounded body text supplied to embedding providers. */
+export const SEARCH_TEXT_MAX_CHARS = 8_192;
 /**
  * Return per-field search text for multi-column FTS5 indexing.
  *
@@ -9,7 +11,7 @@
  *  - description: entry description
  *  - tags: tags + aliases joined
  *  - hints: searchHints + examples + usage + intent fields
- *  - content: TOC headings + parameters + the config-gated body opening
+ *  - content: bounded native/adapter body projection + TOC headings + parameters
  *    (lowest-weight catch-all)
  */
 // NOTE (R5): the collapse detector's frozen canary queries are built from the
@@ -53,8 +55,6 @@ export function buildSearchFields(entry) {
         hintParts.push(entry.whenToUse);
     const hints = hintParts.join(" ").toLowerCase();
     const contentParts = [];
-    if (entry.content)
-        contentParts.push(entry.content);
     if (entry.toc) {
         contentParts.push(entry.toc.map((h) => h.text).join(" "));
     }
@@ -65,15 +65,8 @@ export function buildSearchFields(entry) {
                 contentParts.push(param.description);
         }
     }
-    // Stash-organization conventions (SPEC-8): the self-situating body opening
-    // (captured by the metadata pass only when `index.indexBodyOpening` is on)
-    // folds into the lowest-weight catch-all column — never name/description/
-    // tags/hints — so orientation prose is retrievable without outranking
-    // structured-field matches. The fold is unconditional on the entry field:
-    // `rebuildFts` rebuilds FTS rows from stored entry_json and must reproduce
-    // the same fields without re-reading config.
-    if (entry.bodyOpening)
-        contentParts.push(entry.bodyOpening);
+    if (entry.content)
+        contentParts.push(entry.content);
     const content = contentParts.join(" ").toLowerCase();
     return { name, description, tags, hints, content };
 }
@@ -84,7 +77,23 @@ export function buildSearchFields(entry) {
  */
 export function buildSearchText(entry) {
     const fields = buildSearchFields(entry);
-    return [fields.name, fields.description, fields.tags, fields.hints, fields.content]
-        .filter((s) => s.length > 0)
+    const structured = [fields.name, fields.description, fields.tags, fields.hints]
+        .filter((field) => field.length > 0)
         .join(" ");
+    if (structured.length >= SEARCH_TEXT_MAX_CHARS)
+        return truncateUnicodeSafe(structured, SEARCH_TEXT_MAX_CHARS);
+    if (!fields.content)
+        return structured;
+    const separator = structured ? " " : "";
+    const remaining = SEARCH_TEXT_MAX_CHARS - structured.length - separator.length;
+    return `${structured}${separator}${truncateUnicodeSafe(fields.content, remaining)}`;
+}
+function truncateUnicodeSafe(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    let cut = text.slice(0, maxChars);
+    const lastCode = cut.charCodeAt(cut.length - 1);
+    if (lastCode >= 0xd800 && lastCode <= 0xdbff)
+        cut = cut.slice(0, -1);
+    return cut.trimEnd();
 }

package/dist/indexer/search/search-hit-enrichers.js

@@ -53,7 +53,7 @@ export const derivedMemoryEnricher = {
         // column now stores this same conceptId grammar (Group-C item 2 flip — the
         // metadata producer + this consumer move together).
         const parentRef = `memories/${hit.name}`;
-        const derived = getDerivedForParent(ctx.db, parentRef, ctx.stashDir);
+        const derived = getDerivedForParent(ctx.db, parentRef, ctx.bundleId);
         if (!derived)
             return;
         // Swap description / searchHints / tags from the derived child.