akm-cli 0.9.2-alpha.1 → 0.9.2-alpha.2

package/dist/indexer/search/db-search.js

@@ -32,8 +32,9 @@ import { ensureIndex } from "../ensure-index.js";
 import { collectGraphRelatedHit, loadGraphBoostContext } from "../graph/graph-boost.js";
 import { isProposedQuality } from "../passes/metadata.js";
 import { resolveProjectContext } from "../walk/project-context.js";
-import { parseRefPrefixQuery, parseRetiredTypePrefixQuery, sanitizeFtsQuery } from "./fts-query.js";
-import { applyRankingRules, combineSearchScores, normalizeFtsScores } from "./ranking.js";
+import { buildLexicalQueryPlan, parseRefPrefixQuery, parseRetiredTypePrefixQuery, } from "./fts-query.js";
+import { applyRankingRules, combineSearchScores, lexicalNameMatchTier, normalizeFtsScores } from "./ranking.js";
+import { typeBoostFor } from "./ranking-contributors.js";
 import { attachSearchHitAttribution, copySearchHitAttribution, getSearchHitAttribution } from "./search-attribution.js";
 import { enrichSearchHit } from "./search-hit-enrichers.js";
 import { buildEditHint, findSourceForPath, isEditable } from "./search-source.js";
@@ -220,8 +221,38 @@ export async function searchLocal(input) {
     }
 }
 // ── Database search ─────────────────────────────────────────────────────────
+/**
+ * Keep one deterministic ranking order before stable path deduplication. Exact
+ * names survive the public score ceiling, while raw contributor differences
+ * are quantized so utility-recency epsilon cannot reorder visible ties.
+ */
+function buildSearchResultComparator(query) {
+    const queryTokens = buildLexicalQueryPlan(query).tokens.map((token) => token.toLowerCase());
+    const displayScore = (score) => Math.round(Math.min(1, Math.max(0, score)) * 10000) / 10000;
+    const stableRankScore = (score) => Math.round(score * 10000) / 10000;
+    return (a, b) => {
+        const aNameTier = lexicalNameMatchTier(a.entry, queryTokens);
+        const bNameTier = lexicalNameMatchTier(b.entry, queryTokens);
+        if (aNameTier === 3 || bNameTier === 3) {
+            const nameDiff = bNameTier - aNameTier;
+            if (nameDiff !== 0)
+                return nameDiff;
+        }
+        const scoreDiff = displayScore(b.score) - displayScore(a.score);
+        if (scoreDiff !== 0)
+            return scoreDiff;
+        const rawScoreDiff = stableRankScore(b.score) - stableRankScore(a.score);
+        if (rawScoreDiff !== 0)
+            return rawScoreDiff;
+        const nameDiff = bNameTier - aNameTier;
+        if (nameDiff !== 0)
+            return nameDiff;
+        const typeDiff = typeBoostFor(b.entry.type) - typeBoostFor(a.entry.type);
+        return typeDiff || a.filePath.localeCompare(b.filePath);
+    };
+}
 async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false, disableProjectContext = false, disableScopedUtility = false) {
-    const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
+    const hasSearchableTokens = query.length > 0 && buildLexicalQueryPlan(query).tokens.length > 0;
     // #627 — resolve the default type-exclusion policy. It applies ONLY on the
     // untyped ('any') path and only when the caller did not opt back in via
     // `includeExcludedTypes`. When the config key is ABSENT a built-in default of
@@ -383,21 +414,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
     const preFilter = minScore > 0
         ? scored.filter((item) => item.rankingMode !== "semantic" || (item.preCeilingScore ?? item.score) >= minScore)
         : scored;
-    // Deterministic tiebreaker on equal scores.
-    //
-    // CRITICAL: sort on the SAME clamped+rounded value the user sees (see the
-    // `finalScore`/round-to-4dp logic below at buildDbHit), NOT the raw pre-clamp
-    // `item.score`. The boost loop can push scores above 1.0 (utility, graph,
-    // project boosts) and carries ~15 significant digits. Two entries that DISPLAY
-    // an identical score (e.g. both clamp to 1.0000) can still differ in their raw
-    // pre-clamp score by a timing-dependent epsilon — utility recency uses
-    // `Date.now()` and `last_used_at`, so the same query run twice in one process
-    // can yield raw scores that diverge at the 6th decimal. Sorting on the raw
-    // value lets that invisible epsilon decide the order, so the visible name
-    // tiebreaker never engages and the order flips run-to-run (Issue #14). Quantize
-    // to the display value first; only then does `localeCompare` break true ties.
-    const displayScore = (s) => Math.round(Math.min(1, Math.max(0, s)) * 10000) / 10000;
-    preFilter.sort((a, b) => displayScore(b.score) - displayScore(a.score) || a.entry.name.localeCompare(b.entry.name));
+    preFilter.sort(buildSearchResultComparator(query));
     // Deduplicate by file path — keep only the highest-scored entry per file.
     const deduped = deduplicateByPath(preFilter);
     // Source → scope → proposed-quality → derived-twin belief inheritance →
@@ -431,6 +448,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             score: Math.round(finalScore * 10000) / 10000,
             query,
             rankingMode,
+            lexicalMatch: ranked.lexicalMatch,
             defaultStashDir: stashDir,
             allSourceDirs,
             sources,
@@ -560,7 +578,7 @@ async function enumerateEntries(opts) {
  * What this does NOT unify — and deliberately leaves divergent — is CANDIDATE-
  * POOL construction, which is inherent search-vs-browse semantics: the scored
  * path's pool is `searchFts`/vector matches for the query's own tokens (FTS
- * indexes description/tags/searchHints/aliases, not raw body prose), while the
+ * includes structured fields and bounded adapter content), while the
  * enumerate path's pool is `getAllEntries` for the type, independent of query
  * text. A derived twin sharing no indexed token with the query is therefore an
  * enumerate-path candidate but never a scored-path candidate — see
@@ -736,7 +754,7 @@ export async function buildDbHit(input) {
     // Round to 4 decimal places, no boost multiplication
     const score = Math.round(input.score * 10000) / 10000;
     const graphBoost = getSearchHitAttribution(input.attributionSource ?? {})?.graphExtraction?.boost ?? 0;
-    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost);
+    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost, input.lexicalMatch);
     const graphHit = input.graphContext ? collectGraphRelatedHit(input.graphContext, absolutePath) : null;
     const source = findSourceForPath(absolutePath, input.sources);
     const defaultBundleId = input.config?.defaultBundle ??
@@ -768,8 +786,7 @@ export async function buildDbHit(input) {
         ...(input.entry.currentBeliefRefs ? { currentBeliefRefs: input.entry.currentBeliefRefs } : {}),
         ...(graphHit ? { graph: { entities: graphHit.entities, relations: graphHit.relations } } : {}),
     };
-    if (input.attributionSource)
-        copySearchHitAttribution(input.attributionSource, hit);
+    attachDbHitAttribution(hit, input);
     if (input.entry.derivedFrom) {
         attachSearchHitAttribution(hit, {
             memoryInference: { exposure: "direct" },
@@ -784,9 +801,21 @@ export async function buildDbHit(input) {
     });
     return hit;
 }
+function attachDbHitAttribution(hit, input) {
+    if (input.lexicalMatch) {
+        attachSearchHitAttribution(hit, {
+            lexical: {
+                execution: input.lexicalMatch,
+                nameMatchTier: lexicalNameMatchTier(input.entry, buildLexicalQueryPlan(input.query).tokens),
+            },
+        });
+    }
+    if (input.attributionSource)
+        copySearchHitAttribution(input.attributionSource, hit);
+}
 export function buildWhyMatched(entry, query, 
 // "hybrid" ranking mode
-rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost) {
+rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost, lexicalMatch) {
     const reasons = [
         rankingMode === "hybrid"
             ? "hybrid (fts + semantic)"
@@ -794,6 +823,8 @@ rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost) {
                 ? "semantic similarity"
                 : "fts bm25 relevance",
     ];
+    if (lexicalMatch === "relaxed")
+        reasons.push("lexical recovery after strict query returned no hits");
     const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
     const queryLower = query.toLowerCase().trim();
     const name = entry.name.toLowerCase();
@@ -858,14 +889,13 @@ export function deriveSize(bytes) {
     return "large";
 }
 /**
- * Deduplicate scored results by file path, keeping only the highest-scored
- * entry per unique path. Sorts by score descending internally to ensure the
- * precondition is always met regardless of caller.
+ * Deduplicate the already-ranked result stream by file path. The caller owns
+ * the one ranking order; re-sorting here would silently discard exact-name and
+ * relaxed-recovery ordering in favor of an internal pre-clamp score.
  */
 function deduplicateByPath(items) {
-    const sorted = [...items].sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.filePath.localeCompare(b.filePath));
     const seen = new Set();
-    return sorted.filter((item) => {
+    return items.filter((item) => {
         if (seen.has(item.filePath))
             return false;
         seen.add(item.filePath);

package/dist/indexer/search/fts-query.js

@@ -2,54 +2,54 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Pure FTS5 query-string helpers, extracted from indexer/db/db.ts.
+ * Pure FTS5 query planning and ref-query helpers.
  *
- * These transform a raw user query into an FTS5-safe MATCH expression. They
- * touch no database state, so they are unit-testable with zero DB setup.
+ * The lexical planner transforms a raw user query into bounded FTS5-safe
+ * MATCH expressions. It touches no database state, so it is unit-testable
+ * with zero DB setup.
  * `parseRefPrefixQuery` is the one non-FTS helper: it decides whether a raw
  * query should bypass FTS entirely (SPEC-4 ref-prefix enumeration).
  */
-/**
- * Sanitize a raw user query into an FTS5-safe implicit-AND expression.
- *
- * Allows only characters safe in FTS5 queries: letters, digits, underscores,
- * and whitespace. Everything else (hyphens, dots, quotes, parens, asterisks,
- * colons, carets, @, !, etc.) is replaced with a space so that compound
- * identifiers like "code-review" or "k8s.setup" become AND-joined tokens
- * ("code review", "k8s setup") rather than triggering FTS5 syntax errors.
- */
-export function sanitizeFtsQuery(query) {
-    let sanitized = query.replace(/[^a-zA-Z0-9_\s]/g, " ");
-    // Neutralize the NEAR operator (FTS5 proximity syntax)
-    sanitized = sanitized.replace(/\bNEAR\b/g, " ");
-    const tokens = sanitized.split(/\s+/).filter((t) => t.length >= 1);
-    if (tokens.length === 0)
-        return "";
-    // Use implicit AND (space-separated tokens) for precision. FTS5 treats
-    // space-separated tokens as an implicit AND, matching only rows that
-    // contain ALL terms.
-    return tokens.join(" ");
+/** Maximum number of distinct lexical terms one query may execute. */
+export const MAX_LEXICAL_QUERY_TOKENS = 16;
+const UNICODE_TOKEN = /[\p{L}\p{N}]+/gu;
+function quoteToken(token) {
+    return `"${token}"`;
+}
+function prefixToken(token) {
+    return [...token].length >= 3 ? `${quoteToken(token)}*` : quoteToken(token);
 }
 /**
- * Build a prefix query from an FTS5 query string by appending `*` to each
- * token that is 3+ characters long. Tokens shorter than 3 characters are
- * kept as-is (no prefix expansion) to avoid overly broad matches.
+ * Build the sole lexical retrieval plan from raw user input.
  *
- * Returns null if no tokens qualify for prefix expansion.
+ * Tokenization follows the useful portion of SQLite FTS5's `unicode61`
+ * tokenizer (Unicode letters and numbers). Quoting every term makes FTS
+ * operators ordinary searchable words. Tokens are normalized, deduplicated
+ * case-insensitively, and capped before any SQL executes.
  */
-export function buildPrefixQuery(ftsQuery) {
-    const tokens = ftsQuery.split(/\s+/).filter(Boolean);
-    let hasPrefix = false;
-    const prefixTokens = tokens.map((t) => {
-        if (t.length >= 3) {
-            hasPrefix = true;
-            return `${t}*`;
-        }
-        return t;
-    });
-    if (!hasPrefix)
-        return null;
-    return prefixTokens.join(" ");
+export function buildLexicalQueryPlan(query) {
+    const tokens = [];
+    const seen = new Set();
+    const normalized = query.normalize("NFKC");
+    for (const match of normalized.matchAll(UNICODE_TOKEN)) {
+        const token = match[0];
+        const key = token.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        tokens.push(token);
+        if (tokens.length === MAX_LEXICAL_QUERY_TOKENS)
+            break;
+    }
+    const exact = tokens.map(quoteToken).join(" ");
+    const prefixTokens = tokens.map(prefixToken);
+    const exactPrefix = prefixTokens.some((token) => token.endsWith("*")) ? prefixTokens.join(" ") : undefined;
+    // A slash-bearing, whitespace-free input is an identifier/ref lookup, not
+    // sentence prose. Keep it conjunctive so a mistyped/bare ref never fans out
+    // across every path token through OR recovery.
+    const isRefLikeIdentifier = !/\s/u.test(query.trim()) && query.includes("/");
+    const relaxed = tokens.length > 1 && !isRefLikeIdentifier ? prefixTokens.join(" OR ") : undefined;
+    return { tokens, exact, exactPrefix, relaxed };
 }
 /**
  * D4 — parse a conceptId-prefix browse query.

package/dist/indexer/search/ranking.js

@@ -2,6 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { getUtilityScoresByIds } from "../../storage/repositories/index-utility-repository.js";
+import { buildLexicalQueryPlan } from "./fts-query.js";
 import { applyBeliefStateScoreCeiling, applyContributorAblation, applyScoreContributors, applyUtilityContributors, defaultRankingContributors, defaultUtilityRankingContributors, } from "./ranking-contributors.js";
 export function normalizeFtsScores(results) {
     const ftsScoreMap = new Map();
@@ -33,6 +34,7 @@ export function combineSearchScores(options) {
             filePath: result.filePath,
             score: combinedScore,
             rankingMode: embedScore !== undefined ? "hybrid" : "fts",
+            lexicalMatch: result.lexicalMatch,
             itemRef: result.itemRef,
             bundleId: result.bundleId,
             conceptId: result.conceptId,
@@ -63,7 +65,7 @@ export function combineSearchScores(options) {
     return scored;
 }
 export function applyRankingRules(options) {
-    const queryTokens = options.query.toLowerCase().split(/\s+/).filter(Boolean);
+    const queryTokens = buildLexicalQueryPlan(options.query).tokens.map((token) => token.toLowerCase());
     const queryLower = options.query.toLowerCase().trim();
     const rankingContext = {
         db: options.db,
@@ -103,6 +105,7 @@ export function applyRankingRules(options) {
     };
     for (const item of options.items) {
         applyUtilityContributors(item, utilityContext, activeUtilityContributors);
+        applyRelaxedLexicalScoreCeiling(item, queryTokens);
         // SPEC-5: demoting belief states (superseded/contradicted/archived/
         // deprecated) cap the FINAL score. The additive belief penalty inside the
         // multiplicative boost sum cannot overcome the FTS min-max normalization
@@ -112,3 +115,35 @@ export function applyRankingRules(options) {
     }
     return options.items;
 }
+const RELAXED_NON_NAME_SCORE_CEILING = 0.65;
+/**
+ * Rank name evidence without relying on punctuation or ASCII-only splitting.
+ * The tiers are intentionally structural: an exact normalized name, all query
+ * tokens in a longer name, any query token in the name, or no name evidence.
+ */
+export function lexicalNameMatchTier(entry, queryTokens) {
+    if (queryTokens.length === 0)
+        return 0;
+    const nameBase = entry.name.toLowerCase().split("/").pop() ?? entry.name.toLowerCase();
+    const nameTokens = buildLexicalQueryPlan(nameBase).tokens.map((token) => token.toLowerCase());
+    const tokenMatches = (left, right) => left === right ||
+        (Math.min([...left].length, [...right].length) >= 3 && (left.startsWith(right) || right.startsWith(left)));
+    if (nameTokens.length === queryTokens.length &&
+        nameTokens.every((token, index) => tokenMatches(token, queryTokens[index]))) {
+        return 3;
+    }
+    const matched = queryTokens.filter((token) => nameTokens.some((nameToken) => tokenMatches(nameToken, token))).length;
+    if (matched === queryTokens.length)
+        return 2;
+    return matched > 0 ? 1 : 0;
+}
+/**
+ * A relaxed OR query admits intentionally weak candidates. Candidates with no
+ * query token in their name remain visible for body-only recall, but cannot
+ * saturate at the same displayed score as stronger name-bearing recoveries.
+ */
+function applyRelaxedLexicalScoreCeiling(item, queryTokens) {
+    if (item.lexicalMatch !== "relaxed" || lexicalNameMatchTier(item.entry, queryTokens) > 0)
+        return;
+    item.score = Math.min(item.score, RELAXED_NON_NAME_SCORE_CEILING);
+}

package/dist/indexer/search/search-attribution.js

@@ -17,11 +17,13 @@ export function copySearchHitAttribution(from, to, outputDescription) {
     const memorySurvives = memoryInference?.exposure !== "surface" ||
         (memoryInference.surfaceDescription !== undefined && memoryInference.surfaceDescription === outputDescription);
     const applicable = {
+        ...(attribution.lexical ? { lexical: attribution.lexical } : {}),
         ...(memorySurvives && memoryInference ? { memoryInference } : {}),
         ...(attribution.graphExtraction ? { graphExtraction: attribution.graphExtraction } : {}),
     };
-    if (applicable.memoryInference || applicable.graphExtraction)
+    if (applicable.lexical || applicable.memoryInference || applicable.graphExtraction) {
         attachSearchHitAttribution(to, applicable);
+    }
 }
 export function getSearchHitAttribution(target) {
     return target[ATTRIBUTION];

package/dist/indexer/search/search-fields.js

@@ -1,6 +1,8 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Structured metadata plus bounded body text supplied to embedding providers. */
+export const SEARCH_TEXT_MAX_CHARS = 8_192;
 /**
  * Return per-field search text for multi-column FTS5 indexing.
  *
@@ -9,7 +11,7 @@
  *  - description: entry description
  *  - tags: tags + aliases joined
  *  - hints: searchHints + examples + usage + intent fields
- *  - content: TOC headings + parameters + the config-gated body opening
+ *  - content: bounded native/adapter body projection + TOC headings + parameters
  *    (lowest-weight catch-all)
  */
 // NOTE (R5): the collapse detector's frozen canary queries are built from the
@@ -53,8 +55,6 @@ export function buildSearchFields(entry) {
         hintParts.push(entry.whenToUse);
     const hints = hintParts.join(" ").toLowerCase();
     const contentParts = [];
-    if (entry.content)
-        contentParts.push(entry.content);
     if (entry.toc) {
         contentParts.push(entry.toc.map((h) => h.text).join(" "));
     }
@@ -65,15 +65,8 @@ export function buildSearchFields(entry) {
                 contentParts.push(param.description);
         }
     }
-    // Stash-organization conventions (SPEC-8): the self-situating body opening
-    // (captured by the metadata pass only when `index.indexBodyOpening` is on)
-    // folds into the lowest-weight catch-all column — never name/description/
-    // tags/hints — so orientation prose is retrievable without outranking
-    // structured-field matches. The fold is unconditional on the entry field:
-    // `rebuildFts` rebuilds FTS rows from stored document_json and must reproduce
-    // the same fields without re-reading config.
-    if (entry.bodyOpening)
-        contentParts.push(entry.bodyOpening);
+    if (entry.content)
+        contentParts.push(entry.content);
     const content = contentParts.join(" ").toLowerCase();
     return { name, description, tags, hints, content };
 }
@@ -84,7 +77,23 @@ export function buildSearchFields(entry) {
  */
 export function buildSearchText(entry) {
     const fields = buildSearchFields(entry);
-    return [fields.name, fields.description, fields.tags, fields.hints, fields.content]
-        .filter((s) => s.length > 0)
+    const structured = [fields.name, fields.description, fields.tags, fields.hints]
+        .filter((field) => field.length > 0)
         .join(" ");
+    if (structured.length >= SEARCH_TEXT_MAX_CHARS)
+        return truncateUnicodeSafe(structured, SEARCH_TEXT_MAX_CHARS);
+    if (!fields.content)
+        return structured;
+    const separator = structured ? " " : "";
+    const remaining = SEARCH_TEXT_MAX_CHARS - structured.length - separator.length;
+    return `${structured}${separator}${truncateUnicodeSafe(fields.content, remaining)}`;
+}
+function truncateUnicodeSafe(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    let cut = text.slice(0, maxChars);
+    const lastCode = cut.charCodeAt(cut.length - 1);
+    if (lastCode >= 0xd800 && lastCode <= 0xdbff)
+        cut = cut.slice(0, -1);
+    return cut.trimEnd();
 }

package/dist/output/text/command-format.js

@@ -582,7 +582,9 @@ export function formatUpdatePlain(r) {
         }
     }
     for (const item of plainSynced ?? []) {
-        lines.push(`update: ${item.id} synced (${item.kind})`);
+        lines.push(item.kind === "filesystem"
+            ? `update: ${item.id} reconciled (filesystem)`
+            : `update: ${item.id} synced (${item.kind})`);
     }
     for (const item of skipped ?? []) {
         lines.push(`update: ${item.id} skipped — ${item.reason}`);