akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/indexer/search/ranking.js

@@ -2,84 +2,162 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { stableFtsScore } from "../../core/lexical-score.js";
+import { getEntryById } from "../../storage/repositories/index-entries-repository.js";
 import { getUtilityScoresByIds } from "../../storage/repositories/index-utility-repository.js";
+import { groupUnitHitsByEntry } from "../../storage/repositories/units-repository.js";
 import { buildLexicalQueryPlan } from "./fts-query.js";
 import { lexicalNameTokens, structuralNameTokenMatch } from "./name-match.js";
 import { applyBeliefStateScoreCeiling, applyScoreContributors, applyUtilityContributors, defaultRankingContributors, defaultUtilityRankingContributors, } from "./ranking-contributors.js";
 /**
- * Lower bounds keep a lexical hit competitive with a vector-only neighbour;
- * the upper bound deliberately leaves room for the ranking contributors that
- * run after retrieval (notably the bounded graph boost).  This is a
- * calibration for the one search pipeline, not a claim that BM25 is
- * comparable across different queries or FTS tables.
+ * The pre-redesign lexical/semantic split (`combineSearchScores`, retired
+ * with the units path but restored here — measured against `curate-golden`,
+ * see docs/plans/index-redesign.md's Search section): with both present,
+ * lexical evidence dominates and semantic refines it; nothing here is a new
+ * tuned value.
  */
+const LEXICAL_WEIGHT = 0.7;
+const SEMANTIC_WEIGHT = 0.3;
 /**
- * Convert FTS5's negative BM25 value into the lexical contribution used by
- * this pipeline.  The transform is fixed and monotone: it depends only on a
- * row's own BM25 value, so appending weaker candidates cannot rewrite an
- * existing row's score. FTS5 commonly emits relevance near 1e-6 for broad
- * queries, so first put relevance on a log scale around that observed value.
- * The shape constant intentionally makes the curve approach its ceiling
- * slowly: rare-term scores retain separation instead of all reading as 0.8.
- *
- * FTS5 produces finite non-positive values in normal operation.  Keeping the
- * defensive cases here finite makes this boundary safe if a driver or fixture
- * hands us an invalid value: `-Infinity` is the strongest possible match,
- * while NaN, +Infinity, and positive scores contribute no lexical evidence.
+ * `unit_texts.kind` is redundant with `fragmentId` nullity by construction
+ * (A1's `deriveUnits`: ordinal 0 is always the one structured-fields "card"
+ * unit; every fragment-derived unit carries a non-null `fragmentId`), so
+ * `matchedUnit.kind` is derived here instead of a second table read.
  */
-export function normalizeFtsScores(results) {
-    const ftsScoreMap = new Map();
-    for (const result of results) {
-        ftsScoreMap.set(result.id, { score: result.lexicalScore ?? stableFtsScore(result.bm25Score), result });
-    }
-    return ftsScoreMap;
+function unitKindFromFragmentId(fragmentId) {
+    return fragmentId === null ? "card" : "fragment";
 }
-export function combineSearchScores(options) {
-    const FTS_WEIGHT = 0.7;
-    const VEC_WEIGHT = 0.3;
-    const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
-    const scored = [];
-    const seenIds = new Set();
-    for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
-        seenIds.add(id);
-        const embedScore = options.embedScoreMap.get(id);
-        const combinedScore = embedScore !== undefined ? ftsScore * FTS_WEIGHT + embedScore * VEC_WEIGHT : ftsScore;
-        scored.push({
-            id,
-            entry: result.entry,
-            filePath: result.filePath,
-            score: combinedScore,
-            rankingMode: embedScore !== undefined ? "hybrid" : "fts",
-            lexicalMatch: result.lexicalMatch,
-            itemRef: result.itemRef,
-            bundleId: result.bundleId,
-            conceptId: result.conceptId,
-            fragmentId: result.fragmentId,
-        });
+/**
+ * Group lexical unit hits to entries via `entry_units`, keeping the
+ * strongest (most negative bm25 — see `stableFtsScore`) unit per entry, not
+ * merely the best-RANKED one: magnitude, not rank, is what `fuseByEntry`
+ * scores on.
+ */
+function groupLexicalHitsByEntry(db, hits) {
+    const best = new Map();
+    if (hits.length === 0)
+        return best;
+    const hashes = [...new Set(hits.map((hit) => hit.unitHash))];
+    const placeholders = hashes.map(() => "?").join(",");
+    const rows = db
+        .prepare(`SELECT entry_id AS entryId, fragment_id AS fragmentId, unit_hash AS unitHash FROM entry_units WHERE unit_hash IN (${placeholders})`)
+        .all(...hashes);
+    const ownersByHash = new Map();
+    for (const row of rows) {
+        const owners = ownersByHash.get(row.unitHash) ?? [];
+        owners.push({ entryId: row.entryId, fragmentId: row.fragmentId });
+        ownersByHash.set(row.unitHash, owners);
     }
-    for (const [id, cosine] of options.embedScoreMap) {
-        if (seenIds.has(id))
-            continue;
-        const found = options.getEntryById(id);
+    for (const hit of hits) {
+        for (const owner of ownersByHash.get(hit.unitHash) ?? []) {
+            const existing = best.get(owner.entryId);
+            if (!existing || hit.bm25 < existing.bm25) {
+                best.set(owner.entryId, {
+                    bm25: hit.bm25,
+                    unitHash: hit.unitHash,
+                    fragmentId: owner.fragmentId,
+                    lexicalMatch: hit.lexicalMatch,
+                });
+            }
+        }
+    }
+    return best;
+}
+/**
+ * Which unit is reported as `matchedUnit`/`fragmentId`: a fixed priority
+ * order by evidence strength — card (name/description/tags/hints, the
+ * strongest, most structured signal), then fragment, then a vector
+ * neighbor — never a magnitude comparison across the three. At least one of
+ * the three is defined for every `entryId` this is called with.
+ */
+function pickFusionWinner(cardHit, fragmentHit, semanticHit) {
+    if (cardHit)
+        return { unitHash: cardHit.unitHash, fragmentId: cardHit.fragmentId };
+    if (fragmentHit)
+        return { unitHash: fragmentHit.unitHash, fragmentId: fragmentHit.fragmentId };
+    const winner = semanticHit;
+    return { unitHash: winner.hash, fragmentId: winner.fragmentId };
+}
+/**
+ * Cosine similarity from a `units_vec` L2 distance over normalized vectors —
+ * exactly the retired `tryVecScores`' own conversion, guarded the same way:
+ * `1 - distance²/2`, clamped at 0, non-finite results treated as no evidence.
+ */
+function semanticCosine(distance) {
+    const raw = 1 - (distance * distance) / 2;
+    return Number.isFinite(raw) ? Math.max(0, raw) : 0;
+}
+/**
+ * Fuse the card-lexical, fragment-lexical (both `units_fts`) and semantic
+ * (`units_vec`) unit-level result lists into one ranked entry list
+ * (index-redesign-contract.md B3, restructured for field emphasis by B5f
+ * item 2). Splitting lexical into two kind-scoped lists — rather than one
+ * pool mixing card and fragment units — is what replaces the old per-column
+ * BM25 weights (name 10x, description 5x, tags 3x, hints 2x, content 1x): a
+ * card (name/description/tags/hints) match ranks within its own small pool
+ * instead of racing every fragment's body text on raw BM25, so field
+ * emphasis falls out of the units' structure with no weight tuned.
+ *
+ * Each list is first grouped to entries via `entry_units`, keeping that
+ * list's own strongest unit per entry — `groupUnitHitsByEntry` from the
+ * stage-1 unit store does this (by distance) for the semantic side;
+ * `groupLexicalHitsByEntry` mirrors it (by bm25) for each lexical side.
+ *
+ * The fused score is MAGNITUDE, not reciprocal-rank fusion: rank-only fusion
+ * was measured against the `curate-golden` gate fixture and lost to this —
+ * see docs/plans/index-redesign.md's Search section for the table. Lexical
+ * evidence is `stableFtsScore(bm25, population)` (the calibrated, monotone,
+ * per-row transform — floor 0.3, ceiling 0.8), taken as the BEST of the
+ * entry's card ("parent" population) and fragment ("fragment" population)
+ * magnitudes; semantic evidence is cosine similarity. Combined the way the
+ * pre-redesign `combineSearchScores` combined FTS and vector scores: with
+ * both present, `lexical * 0.7 + semantic * 0.3`; lexical alone is itself;
+ * semantic alone is `semantic * 0.3` (never enough alone to outrank a real
+ * lexical hit's 0.3 floor). `matchedUnit` reports the unit by evidence
+ * priority, not magnitude (see `pickFusionWinner`).
+ */
+export function fuseByEntry(db, cardLexical, fragmentLexical, semantic, opts = {}) {
+    const cardByEntry = groupLexicalHitsByEntry(db, cardLexical);
+    const fragmentByEntry = groupLexicalHitsByEntry(db, fragmentLexical);
+    const semanticByEntry = groupUnitHitsByEntry(db, semantic);
+    const includeTypes = opts.typeFilter && opts.typeFilter.length > 0 ? new Set(opts.typeFilter) : null;
+    const excludeTypes = opts.excludeTypes && opts.excludeTypes.length > 0 ? new Set(opts.excludeTypes) : null;
+    const entryIds = new Set([...cardByEntry.keys(), ...fragmentByEntry.keys(), ...semanticByEntry.keys()]);
+    const results = [];
+    for (const entryId of entryIds) {
+        const cardHit = cardByEntry.get(entryId);
+        const fragmentHit = fragmentByEntry.get(entryId);
+        const semanticHit = semanticByEntry.get(entryId);
+        const lexicalHit = cardHit ?? fragmentHit;
+        const lexicalScore = Math.max(cardHit ? stableFtsScore(cardHit.bm25, "parent") : 0, fragmentHit ? stableFtsScore(fragmentHit.bm25, "fragment") : 0);
+        const semanticScore = semanticHit ? semanticCosine(semanticHit.distance) : undefined;
+        const score = lexicalHit
+            ? semanticScore === undefined
+                ? lexicalScore
+                : lexicalScore * LEXICAL_WEIGHT + semanticScore * SEMANTIC_WEIGHT
+            : (semanticScore ?? 0) * SEMANTIC_WEIGHT;
+        const { unitHash, fragmentId } = pickFusionWinner(cardHit, fragmentHit, semanticHit);
+        const found = getEntryById(db, entryId);
         if (!found)
             continue;
-        if (options.typeFilter && found.entry.type !== options.typeFilter)
+        if (includeTypes && !includeTypes.has(found.entry.type))
             continue;
-        // #627 — drop vector-only neighbors whose type is excluded on the default path.
-        if (excludeTypeSet?.has(found.entry.type))
+        if (excludeTypes?.has(found.entry.type))
             continue;
-        scored.push({
-            id,
+        results.push({
+            id: entryId,
             entry: found.entry,
             filePath: found.filePath,
-            score: cosine * VEC_WEIGHT,
-            rankingMode: "semantic",
+            score,
+            rankingMode: lexicalHit && semanticHit ? "hybrid" : lexicalHit ? "fts" : "semantic",
             itemRef: found.itemRef,
             bundleId: found.bundleId,
             conceptId: found.conceptId,
+            ...(lexicalHit ? { lexicalMatch: lexicalHit.lexicalMatch } : {}),
+            ...(fragmentId ? { fragmentId } : {}),
+            matchedUnit: { unitHash, fragmentId, kind: unitKindFromFragmentId(fragmentId) },
         });
     }
-    return scored;
+    return results;
 }
 export function applyRankingRules(options) {
     const queryTokens = buildLexicalQueryPlan(options.query).tokens.map((token) => token.toLowerCase());

package/dist/indexer/units/unit.js

@@ -0,0 +1,159 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { parseMarkdownToc } from "../../core/asset/markdown.js";
+import { splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
+import { hashEmbeddableText } from "../../core/hash.js";
+import { getMarkdownFragmentContent, hasMarkdownFragmentContent } from "../passes/metadata.js";
+import { buildSearchFields } from "../search/search-fields.js";
+/**
+ * Separator between the entry name and a fragment's section title in a unit
+ * header — the exact format the contract specifies, kept as one named
+ * constant so every header is built the same way.
+ */
+const UNIT_HEADER_SECTION_SEPARATOR = " › ";
+/** Unit 0's body: description, tags, hints, parameters — the non-empty ones, one per line, in that order. */
+function structuredFieldsText(source) {
+    const body = [source.description, source.tags, source.hints, source.parameters]
+        .filter((field) => field.length > 0)
+        .join("\n");
+    return `${source.name}\n${body}`;
+}
+/**
+ * One line per parameter — `name`, or `name: description` when the
+ * parameter has a description — lowercased to match `buildSearchFields`'s
+ * other structured fields (`units_fts` is case-insensitive either way; this
+ * keeps the card unit's casing uniform). `""` when `parameters` is absent or
+ * empty, so `structuredFieldsText`'s filter drops it cleanly.
+ */
+function parametersText(parameters) {
+    if (!parameters || parameters.length === 0)
+        return "";
+    return parameters
+        .map((param) => (param.description ? `${param.name}: ${param.description}` : param.name))
+        .join("\n")
+        .toLowerCase();
+}
+function fragmentHeaderText(name, sectionTitle) {
+    return sectionTitle ? `${name}${UNIT_HEADER_SECTION_SEPARATOR}${sectionTitle}` : name;
+}
+/**
+ * The section title standing over each fragment, in fragment order: a
+ * fragment's own first heading line when it starts with one, else the
+ * nearest heading at or before it (tracked while scanning fragments in
+ * document order), else `null`. Reuses `parseMarkdownToc`, the same heading
+ * list `splitMarkdownFragments` derives its section boundaries from, so a
+ * "#" inside a fenced code block is never mistaken for a real heading.
+ */
+function fragmentSectionTitles(safeMarkdown, fragments) {
+    const headings = parseMarkdownToc(safeMarkdown).headings;
+    let headingIndex = 0;
+    let current = null;
+    return fragments.map((fragment) => {
+        while (headingIndex < headings.length && headings[headingIndex].line <= fragment.startLine) {
+            current = headings[headingIndex].text;
+            headingIndex++;
+        }
+        return current;
+    });
+}
+/**
+ * Split `text` into pieces no longer than `maxChars`: cut at the last "\n"
+ * before the bound, else the last space, else hard-split a single unbroken
+ * run (e.g. a URL) so every piece still respects the bound and the loop
+ * always terminates. The cut character itself is dropped, not carried by
+ * either side.
+ */
+function splitOverflowingText(text, maxChars) {
+    const pieces = [];
+    let rest = text;
+    while (rest.length > maxChars) {
+        const window = rest.slice(0, maxChars);
+        const newlineCut = window.lastIndexOf("\n");
+        if (newlineCut > 0) {
+            pieces.push(rest.slice(0, newlineCut));
+            rest = rest.slice(newlineCut + 1);
+            continue;
+        }
+        const spaceCut = window.lastIndexOf(" ");
+        if (spaceCut > 0) {
+            pieces.push(rest.slice(0, spaceCut));
+            rest = rest.slice(spaceCut + 1);
+            continue;
+        }
+        pieces.push(rest.slice(0, maxChars));
+        rest = rest.slice(maxChars);
+    }
+    if (rest.length > 0)
+        pieces.push(rest);
+    return pieces;
+}
+/**
+ * maxChars bounds every unit's text; a unit over it is split at the last
+ * "\n" (else the last space) before the bound into sub-units that keep the
+ * source fragmentId and take the next ordinals.
+ */
+export function deriveUnits(source, maxChars) {
+    if (!Number.isFinite(maxChars) || maxChars <= 0) {
+        throw new RangeError("deriveUnits: maxChars must be a positive finite number");
+    }
+    const units = [];
+    let ordinal = 0;
+    const pushUnit = (fragmentId, text) => {
+        units.push({ entryId: source.entryId, ordinal: ordinal++, fragmentId, hash: hashEmbeddableText(text), text });
+    };
+    for (const text of splitOverflowingText(structuredFieldsText(source), maxChars))
+        pushUnit(null, text);
+    if (source.safeMarkdown != null) {
+        const fragments = splitMarkdownFragments(source.safeMarkdown);
+        const sectionTitles = fragmentSectionTitles(source.safeMarkdown, fragments);
+        fragments.forEach((fragment, index) => {
+            const header = fragmentHeaderText(source.name, sectionTitles[index] ?? null);
+            const text = `${header}\n${fragment.text}`;
+            for (const piece of splitOverflowingText(text, maxChars))
+                pushUnit(fragment.fragmentId, piece);
+        });
+    }
+    return units;
+}
+/**
+ * `UnitSource` from an already-parsed `IndexDocument` — `buildSearchFields`
+ * for the structured fields, the entry's own carried markdown for fragments.
+ * Shared by `reconcile.ts` (a freshly-parsed entry) and `enrich.ts` (the same
+ * entry merged with LLM-enriched description/tags/searchHints, index-redesign
+ * B5e) — a leaf in `units/`, not either caller, so importing it never creates
+ * a reconcile.ts ↔ enrich.ts cycle.
+ *
+ * `hasMarkdownFragmentContent`/`getMarkdownFragmentContent` is the `akm`
+ * adapter's own line-structure-preserving fragment projection
+ * (`applyPreContributorFields`, gated `.md`-only and excluding sensitive
+ * types), set during `recognize` and present ONLY for that adapter (or a
+ * caller that re-tags a derived copy via `setMarkdownFragmentContent`, as
+ * `enrich.ts` does). Every other adapter (`okf`, ...) never calls it, so
+ * `hasMarkdownFragmentContent` is always false for their entries — falling
+ * straight to `null` here would leave their body content in `entries_fts`'s
+ * single per-entry `content` column (`buildSearchFields`, unconditional) but
+ * in NO unit at all, an asymmetry that would silently blank a whole
+ * adapter's fragment search once unit coverage is complete
+ * (index-redesign-contract.md B3). `entry.content` — the same field already
+ * surfaced through search hits and `show`, so already that adapter's own
+ * public-safe projection — is the fallback fragment source for exactly this
+ * case.
+ */
+export function toUnitSource(entryId, entry) {
+    const fields = buildSearchFields(entry);
+    const safeMarkdown = hasMarkdownFragmentContent(entry)
+        ? (getMarkdownFragmentContent(entry) ?? null)
+        : typeof entry.content === "string" && entry.content.trim()
+            ? entry.content
+            : null;
+    return {
+        entryId,
+        name: fields.name,
+        description: fields.description,
+        tags: fields.tags,
+        hints: fields.hints,
+        parameters: parametersText(entry.parameters),
+        safeMarkdown,
+    };
+}

package/dist/llm/client.js

@@ -456,6 +456,15 @@ export async function probeLlmReachable(config) {
         return { reachable: false, error: err instanceof Error ? err.message : String(err) };
     }
 }
+/**
+ * Default bound for a best-effort capability/reachability probe (#914):
+ * generous enough for a cold local model server to answer a route-existence
+ * check, short enough that `akm health --probe` and the provider-limits
+ * probe (`src/llm/embedders/provider-limits.ts`, index-units) do not stall a
+ * run on a dead endpoint. Shared so both probes bound themselves to the same
+ * value instead of drifting apart.
+ */
+export const HEALTH_PROBE_TIMEOUT_MS = 3_000;
 /**
  * Reachability probe for `akm health` (#914): one GET against the
  * OpenAI-compatible `/models` route, bounded by `timeoutMs`. Any HTTP
@@ -463,7 +472,7 @@ export async function probeLlmReachable(config) {
  * answers, not whether the route exists or the credential is right — so a
  * cold local server is never asked to load a model just to be checked.
  */
-export async function probeLlmEndpoint(config, timeoutMs = 3_000) {
+export async function probeLlmEndpoint(config, timeoutMs = HEALTH_PROBE_TIMEOUT_MS) {
     try {
         await fetch(`${config.endpoint.replace(/\/+$/, "")}/models`, { signal: AbortSignal.timeout(timeoutMs) });
         return { reachable: true };

package/dist/llm/embedder.js

@@ -104,10 +104,17 @@ async function embedOnce(text, embeddingConfig, signal) {
  * `onBatch`, when given, fires once per provider/local batch as it completes
  * (#954) so a caller can commit each batch's rows durably as they land
  * rather than buffering the whole call — see `EmbeddingBatchCommit`.
+ *
+ * `packing`, when given, threads a remote request's window/exact-token-
+ * counter/Ollama `num_ctx` in from the provider's own probed limits
+ * (`probeProviderLimits`, `src/llm/embedders/provider-limits.ts`) instead of
+ * the retired `embedding.maxTokens`/`batchSize`/`contextLength` config keys
+ * (index redesign, B5) — see `EmbeddingRequestPacking`. Only the remote
+ * branch below consumes it; local/deterministic embedding ignores it.
  */
-export async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
+export async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing) {
     if (embedderOverrides?.embedBatch) {
-        return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch);
+        return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing);
     }
     if (texts.length === 0)
         return [];
@@ -121,7 +128,7 @@ export async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch
         return embeddings;
     }
     if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
-        return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch);
+        return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch, packing);
     }
     // Local transformer: use the batched path (chunks of 32 via LocalEmbedder).
     // When a localModel override is set we cannot share the singleton (which uses