akm-cli 0.8.0-rc1 → 0.8.0

package/dist/indexer/db-search.js

@@ -1,25 +1,18 @@
-/**
- * Database-backed (SQLite + FTS5/vector) source search implementation.
- *
- * Extracted from source-search.ts to break the circular import:
- *   source-search.ts → sources/providers/filesystem.ts → db-search.ts (no cycle)
- *
- * source-search.ts imports this module for the `searchLocal` export.
- * sources/providers/filesystem.ts also imports `searchLocal` from here.
- *
- * Renamed from `local-search.ts` to signal that this is the DB-layer search
- * implementation, not a "local vs. remote" distinction.
- */
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import { buildActionFromContributors, defaultActionContributors } from "../core/action-contributors";
 import { makeAssetRef } from "../core/asset-ref";
 import { defaultRendererRegistry } from "../core/asset-registry";
 import { getDbPath } from "../core/paths";
 import { warn } from "../core/warn";
-import { closeDatabase, getAllEntries, getEntryById, getEntryCount, getMeta, openExistingDatabase, sanitizeFtsQuery, searchFts, searchVec, } from "./db";
+import { getCurrentWorkflowScopeKey } from "../workflows/scope-key";
+import { closeDatabase, getAllEntries, getEntryById, getEntryCount, getMeta, getPositiveFeedbackCountsByIds, openExistingDatabase, sanitizeFtsQuery, searchFts, searchVec, } from "./db";
 import { ensureIndex } from "./ensure-index";
-import { collectGraphRelatedHit, loadGraphBoostContext } from "./graph-boost";
+import { collectGraphRelatedHit, computeGraphBoost, loadGraphBoostContext, } from "./graph-boost";
 import { isProposedQuality } from "./metadata";
+import { resolveProjectContext } from "./project-context";
 import { applyRankingRules, combineSearchScores, normalizeFtsScores } from "./ranking";
 import { enrichSearchHit } from "./search-hit-enrichers";
 import { buildEditHint, findSourceForPath, isEditable } from "./search-source";
@@ -36,12 +29,30 @@ function resolveSearchHitRef(entry, refName, source) {
 function resolveSearchHitOrigin(source) {
     return source?.wikiName ? null : (source?.registryId ?? null);
 }
+/**
+ * Phase 2A / Rec 5: gate for the per-search `getPositiveFeedbackCountsByIds`
+ * lookup. Returns `true` only when the user has explicitly opted into
+ * `improve.utilityDecay` AND configured a `feedbackStabilityBoost > 1.0`.
+ * Either condition being false makes the DB query pure overhead (the ranking
+ * contributor ignores `positiveFeedbackCounts` when `utilityDecayConfig` is
+ * absent, and `1.0^count == 1` collapses the boost into a no-op).
+ *
+ * Exported for unit testing — keeps the gate decision pinned so a future edit
+ * can't quietly broaden the hot path.
+ */
+export function shouldQueryPositiveFeedbackCounts(utilityDecayRaw) {
+    if (utilityDecayRaw === undefined)
+        return false;
+    const boost = utilityDecayRaw.feedbackStabilityBoost ?? 1.5;
+    return boost > 1.0;
+}
 // ── Main search entrypoint ───────────────────────────────────────────────────
 export async function searchLocal(input) {
     const { query, searchType, limit, stashDir, sources, config } = input;
     const filters = input.filters;
     const includeProposed = input.includeProposed === true;
     const beliefFilter = input.beliefFilter ?? "all";
+    const restrictToSources = input.restrictToSources === true;
     const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
     const allSourceDirs = sources.map((s) => s.path);
     const rawStatus = readSemanticStatus();
@@ -52,8 +63,18 @@ export async function searchLocal(input) {
         if (rawStatus && rawStatus.providerFingerprint !== currentFingerprint) {
             warnings.push("Embedding config changed. Run 'akm index --full' to rebuild the semantic index with the new provider.");
         }
+        else if (!config.embedding?.endpoint || !config.embedding?.model) {
+            // #480: when semantic mode is `auto` but no embedding provider is
+            // configured (e.g. `akm setup --yes` ran without picking one), telling
+            // the user to "run akm setup" is misleading — they just did. Surface
+            // the actual remediation: configure an embedding endpoint OR switch
+            // semanticSearchMode to `off` to silence the warning.
+            warnings.push("Semantic search is enabled (semanticSearchMode='auto') but no embedding provider is configured. " +
+                'Either: (a) `akm config set embedding \'{"endpoint":"...","model":"..."}\'`, or ' +
+                "(b) `akm config set semanticSearchMode off` to use keyword-only search.");
+        }
         else {
-            warnings.push("Semantic search is pending verification. Run 'akm setup' or 'akm index --full' to enable semantic search.");
+            warnings.push("Semantic search is pending verification. Run 'akm index --full' to build the semantic index now, or wait for the next background index pass.");
         }
     }
     if (config.semanticSearchMode === "auto" && semanticStatus === "blocked") {
@@ -81,7 +102,7 @@ export async function searchLocal(input) {
                 mode: "keyword",
             };
         }
-        const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter);
+        const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
         return {
             hits,
             tip: hits.length === 0
@@ -98,7 +119,7 @@ export async function searchLocal(input) {
     }
 }
 // ── Database search ─────────────────────────────────────────────────────────
-async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all") {
+async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
     const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
     // Empty queries — including ones that sanitize down to no searchable FTS
     // tokens such as "." — should enumerate matching entries instead of
@@ -114,12 +135,19 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             seenFilePaths.add(ie.filePath);
             return true;
         });
+        // Source filter: when the caller narrowed `sources` via `--source <name>`,
+        // drop entries whose filePath does not live under any of the requested
+        // sources. The FTS index spans every configured source, so without this
+        // filter a narrowed --source request would still leak results.
+        const sourceFiltered = restrictToSources
+            ? uniqueEntries.filter((ie) => findSourceForPath(ie.filePath, sources) !== undefined)
+            : uniqueEntries;
         // Scope filter: drop entries whose stored scope does not satisfy every
         // supplied scope key. Filtering happens BEFORE the limit slice so a
         // restrictive filter still returns up to `limit` results.
         const scopeFiltered = filters
-            ? uniqueEntries.filter((ie) => entryMatchesScope(ie.entry.scope, filters))
-            : uniqueEntries;
+            ? sourceFiltered.filter((ie) => entryMatchesScope(ie.entry.scope, filters))
+            : sourceFiltered;
         // Proposed-quality filter (v1 spec §4.2): exclude entries with
         // `quality: "proposed"` unless the caller explicitly opts in.
         const qualityFiltered = includeProposed
@@ -138,6 +166,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             sources,
             config,
             rendererRegistry,
+            db,
         })));
         return { hits };
     }
@@ -192,24 +221,84 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             return null;
         return loadGraphBoostContext(allSourceDirs, query, config, db);
     })();
-    applyRankingRules({ db, query, items: scored, graphContext });
+    // Resolve project-context tokens from the current working directory once
+    // per search invocation. Returns null when running from home dir / /tmp,
+    // or when the caller has set AKM_DISABLE_PROJECT_CONTEXT=1.
+    const projectContext = process.env.AKM_DISABLE_PROJECT_CONTEXT === "1" ? null : resolveProjectContext(process.cwd());
+    // Phase 2A / Rec 5: resolve forgetting-curve config and skip the feedback
+    // count query when the boost cannot make a difference (default ≤ 1.0 means
+    // boost^count == 1 — zero overhead for the common case).
+    const utilityDecayRaw = config.improve?.utilityDecay;
+    const halfLifeDays = utilityDecayRaw?.halfLifeDays ?? 30;
+    const feedbackStabilityBoost = utilityDecayRaw?.feedbackStabilityBoost ?? 1.5;
+    const utilityDecayConfig = utilityDecayRaw !== undefined ? { halfLifeDays, feedbackStabilityBoost } : undefined;
+    // Gate the feedback-count query on the user having explicitly opted into
+    // utilityDecay. Without an opt-in, `utilityDecayConfig` is undefined and the
+    // ranking contributor ignores `positiveFeedbackCounts` — so running the DB
+    // query here would be pure overhead. The boost > 1.0 sub-gate then skips the
+    // query when the configured boost is a no-op (1.5^count when boost==1 is 1).
+    const positiveFeedbackCounts = shouldQueryPositiveFeedbackCounts(utilityDecayRaw)
+        ? getPositiveFeedbackCountsByIds(db, scored.map((item) => item.id))
+        : undefined;
+    // Resolve per-project scope key for scoped utility scoring.
+    // AKM_DISABLE_SCOPED_UTILITY=1 opts out (e.g. for registry searches or tests).
+    let scopeKey;
+    try {
+        scopeKey = process.env.AKM_DISABLE_SCOPED_UTILITY === "1" ? undefined : getCurrentWorkflowScopeKey();
+    }
+    catch {
+        // Non-fatal — ranking proceeds without scoped utility on any error.
+    }
+    applyRankingRules({
+        db,
+        query,
+        items: scored,
+        graphContext,
+        projectContext,
+        utilityDecayConfig,
+        positiveFeedbackCounts,
+        scopeKey,
+    });
     // ── minScore floor ──────────────────────────────────────────────────────
     // Drop semantic-only hits (cosine-only, no FTS match) whose score falls
     // below the configured floor. FTS hits and hybrid hits are always kept.
     // Default floor: 0.2. Set search.minScore = 0 in config to disable.
     const minScore = config.search?.minScore ?? 0.2;
     const preFilter = minScore > 0 ? scored.filter((item) => item.rankingMode !== "semantic" || item.score >= minScore) : scored;
-    // Deterministic tiebreaker on equal scores
-    preFilter.sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
+    // Deterministic tiebreaker on equal scores.
+    //
+    // CRITICAL: sort on the SAME clamped+rounded value the user sees (see the
+    // `finalScore`/round-to-4dp logic below at buildDbHit), NOT the raw pre-clamp
+    // `item.score`. The boost loop can push scores above 1.0 (utility, graph,
+    // project boosts) and carries ~15 significant digits. Two entries that DISPLAY
+    // an identical score (e.g. both clamp to 1.0000) can still differ in their raw
+    // pre-clamp score by a timing-dependent epsilon — utility recency uses
+    // `Date.now()` and `last_used_at`, so the same query run twice in one process
+    // can yield raw scores that diverge at the 6th decimal. Sorting on the raw
+    // value lets that invisible epsilon decide the order, so the visible name
+    // tiebreaker never engages and the order flips run-to-run (Issue #14). Quantize
+    // to the display value first; only then does `localeCompare` break true ties.
+    const displayScore = (s) => Math.round(Math.min(1, Math.max(0, s)) * 10000) / 10000;
+    preFilter.sort((a, b) => displayScore(b.score) - displayScore(a.score) || a.entry.name.localeCompare(b.entry.name));
     // Deduplicate by file path — keep only the highest-scored entry per file.
     // Multiple .stash.json entries can map to the same file (e.g. entries without
     // a filename field all collapse to files[0]). Showing the same path/ref
     // multiple times clutters results.
     const deduped = deduplicateByPath(preFilter);
+    // Source filter: when the caller narrowed `sources` via `--source <name>`,
+    // drop hits whose filePath does not live under any of the requested
+    // sources. The FTS/vector index spans every configured source, so without
+    // this filter a narrowed --source request would still leak results from
+    // other sources that happened to match the query text.
+    const sourceFiltered = restrictToSources
+        ? deduped.filter((item) => findSourceForPath(item.filePath, sources) !== undefined)
+        : deduped;
     // Scope filter: drop hits whose stored scope does not satisfy every supplied
     // key. Applied AFTER ranking — filtering narrows the result set without
     // touching the single FTS5+boosts scoring pipeline.
-    const scopeFiltered = filters ? deduped.filter((item) => entryMatchesScope(item.entry.scope, filters)) : deduped;
+    const scopeFiltered = filters
+        ? sourceFiltered.filter((item) => entryMatchesScope(item.entry.scope, filters))
+        : sourceFiltered;
     // Proposed-quality filter (v1 spec §4.2): exclude entries with
     // `quality: "proposed"` unless the caller passed `--include-proposed`.
     // Applied AFTER ranking for the same reason as scope filtering.
@@ -239,6 +328,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
             utilityBoosted,
             graphContext,
             rendererRegistry,
+            db,
         });
     }));
     return { embedMs, rankMs, hits };
@@ -248,9 +338,16 @@ function matchBeliefFilter(type, beliefState, filter) {
         return true;
     if (type !== "memory")
         return true;
-    if (filter === "current")
-        return beliefState === undefined || beliefState === "active";
-    return beliefState === "contradicted" || beliefState === "superseded" || beliefState === "archived";
+    if (filter === "current") {
+        // Phase 1A: `asserted` is a "current" state (stronger authority than `active`);
+        // `deprecated` is excluded from current results.
+        return beliefState === undefined || beliefState === "active" || beliefState === "asserted";
+    }
+    // historical
+    return (beliefState === "contradicted" ||
+        beliefState === "superseded" ||
+        beliefState === "deprecated" ||
+        beliefState === "archived");
 }
 // ── Vector scorer ───────────────────────────────────────────────────────────
 async function tryVecScores(db, query, k, config) {
@@ -292,7 +389,8 @@ export async function buildDbHit(input) {
     const confidenceBoost = typeof input.entry.confidence === "number" ? Math.min(0.05, Math.max(0, input.entry.confidence) * 0.05) : 0;
     // Round to 4 decimal places, no boost multiplication
     const score = Math.round(input.score * 10000) / 10000;
-    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted);
+    const graphBoost = input.graphContext ? computeGraphBoost(input.graphContext, input.path) : 0;
+    const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost, input.utilityBoosted, graphBoost);
     const graphHit = input.graphContext ? collectGraphRelatedHit(input.graphContext, input.path) : null;
     const source = findSourceForPath(input.path, input.sources);
     const ref = resolveSearchHitRef(input.entry, input.entry.name, source);
@@ -326,12 +424,13 @@ export async function buildDbHit(input) {
         type: input.entry.type,
         stashDir: entryStashDir,
         rendererRegistry,
+        db: input.db,
     });
     return hit;
 }
 export function buildWhyMatched(entry, query, 
 // "hybrid" ranking mode
-rankingMode, qualityBoost, confidenceBoost, utilityBoosted) {
+rankingMode, qualityBoost, confidenceBoost, utilityBoosted, graphBoost) {
     const reasons = [
         rankingMode === "hybrid"
             ? "hybrid (fts + semantic)"
@@ -375,12 +474,21 @@ rankingMode, qualityBoost, confidenceBoost, utilityBoosted) {
         reasons.push("metadata confidence boost");
     if (entry.beliefState === "active")
         reasons.push("active belief state");
+    if (entry.beliefState === "asserted")
+        reasons.push("asserted belief state");
     if (entry.beliefState === "contradicted")
         reasons.push("contradicted belief state");
     if (entry.beliefState === "superseded")
         reasons.push("superseded belief state");
+    if (entry.beliefState === "deprecated")
+        reasons.push("deprecated belief state");
+    if (entry.beliefState === "archived")
+        reasons.push("archived belief state");
     if (utilityBoosted)
         reasons.push("usage history boost");
+    if (typeof graphBoost === "number" && graphBoost > 0) {
+        reasons.push(`graph boost +${graphBoost.toFixed(2)}`);
+    }
     return reasons;
 }
 // ── Utilities ────────────────────────────────────────────────────────────────