@rubytech/create-maxy-code 0.1.131 → 0.1.138

package/payload/platform/lib/graph-search/src/index.ts

@@ -41,9 +41,41 @@
 
 import { int, type Session } from "neo4j-driver";
 import { notTrashed } from "../../graph-trash/dist/index.js";
+import { fuseRrf } from "./rrf-fusion.js";
+import { expandQuery } from "./query-expansion.js";
+import { applyCompiledTruthBoost, applyBacklinkBoost } from "./boosts.js";
+import { dedup as dedupHits, DEFAULT_LAYERS, type DedupLayer } from "./dedup.js";
+import { pickIndexMix, type RetrievalClass } from "./route.js";
 
 const VECTOR_WEIGHT = 0.7;
 const BM25_WEIGHT = 0.3;
+
+/**
+ * Per-enhancement feature flags (Task 308). All read from env at the
+ * boundary of each `hybrid()` call so flips do not require a process
+ * restart. Per-call `HybridParams` fields override the env defaults so
+ * tests and A/B harnesses can opt in without touching the environment.
+ *
+ * Defaults are OFF — unflagged callers see exactly the baseline
+ * weighted-sum + nodeId-only-dedup baseline. Flip a flag on at the
+ * environment level after the soak window passes.
+ */
+function envFlag(name: string): boolean {
+  const v = process.env[name];
+  return v === "1" || v === "true" || v === "yes";
+}
+function readFlags() {
+  return {
+    rrf: envFlag("MAXY_GS_RRF"),
+    expansion: envFlag("MAXY_GS_EXPANSION"),
+    boosts: envFlag("MAXY_GS_BOOSTS"),
+    dedup: envFlag("MAXY_GS_DEDUP"),
+    route: envFlag("MAXY_GS_ROUTE"),
+  };
+}
+
+export type { DedupLayer, RetrievalClass };
+export { DEFAULT_LAYERS };
 /**
  * Name of the universal fulltext index. Mirrors the `CREATE FULLTEXT
  * INDEX` declaration in `platform/neo4j/schema.cypher`. The doctrine test
@@ -108,7 +140,17 @@ export interface ScoredNode {
   bm25Hit: boolean;
 }
 
-export type SearchMode = "hybrid" | "bm25";
+/**
+ * `"hybrid"` = weighted-sum fusion path (the baseline; the
+ * brief calls this `weighted-sum`).
+ * `"rrf"` = RRF fusion path (Task 308, flagged via `MAXY_GS_RRF`).
+ * `"bm25"` = degraded path after embed failure with `degradeOnEmbedFailure`.
+ *
+ * UI callers (e.g. `platform/ui/app/graph/page.tsx`) pattern-match on
+ * `"bm25"` to render the Ollama-down banner; non-`"bm25"` values collapse
+ * to the normal banner, so adding `"rrf"` is wire-compatible.
+ */
+export type SearchMode = "hybrid" | "rrf" | "bm25";
 
 export interface Bm25OnlyParams {
   query: string;
@@ -155,6 +197,33 @@ export interface HybridParams extends Bm25OnlyParams {
    * default; the lib itself takes no opinion on the value.
    */
   vectorThreshold?: number;
+
+  // --- Task 308 enhancements (all optional, all default off) ---
+
+  /**
+   * Routing hint from the inbound-gateway classifier (Task 304). When
+   * the `MAXY_GS_ROUTE` flag is on, the lib picks a label filter and
+   * fusion-weight plan per class. Omitted ⇒ `"general"` (unflagged
+   * callers see no change).
+   */
+  retrievalClass?: RetrievalClass;
+  /** Per-call override: enable Reciprocal Rank Fusion. */
+  enableRrf?: boolean;
+  /** Per-call override: enable Haiku multi-query expansion. */
+  enableExpansion?: boolean;
+  /** Per-call override: enable compiledTruth + backlink boosts. */
+  enableBoosts?: boolean;
+  /** Per-call override: enable 4-layer dedup (default layers below). */
+  enableDedup?: boolean;
+  /** Per-call override: enable retrieval-class routing. */
+  enableRoute?: boolean;
+  /**
+   * Layers passed to the 4-layer dedup when `enableDedup` is true.
+   * Defaults to `["nodeId","slug","canonicalName","contentHash"]`.
+   */
+  dedupLayers?: DedupLayer[];
+  /** RRF smoothing constant. Default 60 (gbrain default). */
+  rrfK?: number;
 }
 
 export interface HybridResponse {
@@ -190,6 +259,21 @@ export interface HybridResponse {
   suppressed: number;
   bm25Bypass: number;
   threshold: number | null;
+  /**
+   * Per-stage counts surfaced for the caller's `[graph-search:hybrid]`
+   * log line (Task 308). Always present; values reflect whichever
+   * enhancements ran on this call. `expansions` is 1 when expansion
+   * was off, `boosted - fused` is 0 when boosts were no-ops, etc.
+   */
+  stageCounts: {
+    expansions: number;
+    vector: number;
+    bm25: number;
+    fused: number;
+    boosted: number;
+    deduped: number;
+    final: number;
+  };
 }
 
 export type EmbedFn = (text: string) => Promise<number[]>;
@@ -331,20 +415,29 @@ export async function bm25Only(
  * Hybrid vector + BM25 search with graph expansion.
  *
  * Sequence:
- *  1. embed(query) — may throw if Ollama is unreachable. When
+ *  1. Resolve flags + route plan. Route may short-circuit (`retrievalClass=none`).
+ *  2. Multi-query expansion (Task 308 flag): Haiku produces 3-5 paraphrases.
+ *     Each paraphrase is embedded and run through vector+BM25 in parallel.
+ *  3. embed(query) — may throw if Ollama is unreachable. When
  *     `degradeOnEmbedFailure=true`, caller receives bm25Only() result
  *     with `mode: "bm25"`; otherwise the throw propagates.
- *  2. Vector search per label (one query per vector index discovered at
+ *  4. Vector search per label (one query per vector index discovered at
  *     boot). Nodes-by-label filter short-circuits when the requested
  *     labels have no index.
- *  3. BM25 search on the universal `entity_search` index —
+ *  5. BM25 search on the universal `entity_search` index —
  *     same filter semantics as vector half (scope/agent/keyword/trashed).
- *  4. Keyword subscriptions (when set): BM25 per keyword + property
+ *  6. Keyword subscriptions (when set): BM25 per keyword + property
  *     lookup against `node.keywords` array. Both bypass the agentSlug
  *     filter — subscriptions are scope-inclusive by design (matches
- *     memory-search.ts comment L319).
- *  5. Merge: 0.7*vector + 0.3*bm25_norm, dedup by nodeId, sort.
- *  6. Graph expand (1 hop by default, 0 to skip) — notTrashed filter +
+ *     memory-search.ts comment L319). Runs once, not per expansion.
+ *  7. Fusion: weighted-sum (`vw * vector + bw * bm25_norm`) by default,
+ *     RRF when `MAXY_GS_RRF` is on. Route plan supplies fusion weights.
+ *  8. Boosts (Task 308 flag): compiledTruth bump + backlink log-bump.
+ *  9. Dedup (Task 308 flag): 4-layer dedup over the union, default layers
+ *     `[nodeId, slug, canonicalName, contentHash]`.
+ * 10. Threshold + sort + slice. Counts (rawMerged / suppressed / bm25Bypass)
+ *     reflect the pre-threshold and post-threshold populations.
+ * 11. Graph expand (1 hop by default, 0 to skip) — notTrashed filter +
  *     scope + agent clauses mirrored.
  */
 export async function hybrid(
@@ -364,18 +457,62 @@ export async function hybrid(
     keywordSubscriptions,
     expandHops = 1,
     degradeOnEmbedFailure = false,
+    retrievalClass,
+    dedupLayers = DEFAULT_LAYERS,
+    rrfK = 60,
   } = params;
 
+  const envFlags = readFlags();
+  const useRoute = params.enableRoute ?? envFlags.route;
+  const useRrf = params.enableRrf ?? envFlags.rrf;
+  const useExpansion = params.enableExpansion ?? envFlags.expansion;
+  const useBoosts = params.enableBoosts ?? envFlags.boosts;
+  const useDedup = params.enableDedup ?? envFlags.dedup;
+
+  // --- Route plan (Task 308): pick label filter + fusion weights from class ---
+  const plan = useRoute
+    ? pickIndexMix(retrievalClass)
+    : { skip: false, vectorWeight: VECTOR_WEIGHT, bm25Weight: BM25_WEIGHT, labelFilter: undefined as string[] | undefined };
+
+  const emptyStageCounts = {
+    expansions: 1,
+    vector: 0,
+    bm25: 0,
+    fused: 0,
+    boosted: 0,
+    deduped: 0,
+    final: 0,
+  };
+
+  if (plan.skip) {
+    return {
+      mode: useRrf ? "rrf" : "hybrid",
+      results: [],
+      expandMs: 0,
+      rawMerged: 0,
+      suppressed: 0,
+      bm25Bypass: 0,
+      threshold: params.vectorThreshold ?? null,
+      stageCounts: emptyStageCounts,
+    };
+  }
+
+  // Per-call labels win over route-supplied label filter — operator-set
+  // filters are intentional; the route plan is a soft hint.
+  const effectiveLabels = (labels && labels.length > 0)
+    ? labels
+    : plan.labelFilter;
+
+  // --- Degraded path: embed fails, fall back to BM25-only ---
+  // Run a single embed up front so degradation is detected before any
+  // expansion / multi-query work spins up.
   let queryEmbedding: number[];
   try {
     queryEmbedding = await embed(query);
   } catch (err) {
     if (!degradeOnEmbedFailure) throw err;
     const msg = err instanceof Error ? err.message : String(err);
-    const bm25Hits = await bm25Only(session, params);
-    // even in the degraded path, surface granular components so
-    // the /data score-line renders consistently. Vector half is 0 (no embed
-    // happened); bm25 half is normalised across the local hit set.
+    const bm25Hits = await bm25Only(session, { ...params, labels: effectiveLabels });
     const normalised = normaliseBm25Scores(bm25Hits.map((h) => h.score));
     const results: SearchResult[] = bm25Hits.map((h, i) => ({
       ...h,
@@ -383,11 +520,6 @@ export async function hybrid(
       bm25Score: normalised[i] ?? 0,
       related: [],
     }));
-    // threshold has no meaning in the degraded path: no embed
-    // happened, so there is no vector cosine to compare against. Surface
-    // counts in the response (rawMerged = rendered, suppressed = 0,
-    // bm25Bypass = 0, threshold = null) so the caller's render path is
-    // uniform across hybrid + degraded modes.
     return {
       mode: "bm25",
       results,
@@ -397,32 +529,36 @@ export async function hybrid(
       suppressed: 0,
       bm25Bypass: 0,
       threshold: null,
+      stageCounts: {
+        expansions: 1,
+        vector: 0,
+        bm25: results.length,
+        fused: results.length,
+        boosted: results.length,
+        deduped: results.length,
+        final: results.length,
+      },
     };
   }
 
   const labelToIndex = await discoverIndexes(session);
 
-  const keywordFilter = buildKeywordFilter(keywords, keywordMatch);
-  const keywordClause = keywordFilter?.clause ?? "";
-  const keywordParams = keywordFilter?.params ?? {};
-
-  const scoreMap = new Map<string, ScoredNode>();
-
-  // --- Vector search (per label) ---
+  // --- Resolve vector indexes from the effective label set ---
   let indexesToQuery: string[];
-  if (labels && labels.length > 0) {
-    indexesToQuery = labels
+  if (effectiveLabels && effectiveLabels.length > 0) {
+    indexesToQuery = effectiveLabels
       .map((l) => labelToIndex.get(l))
       .filter((idx): idx is string => idx !== undefined);
     if (indexesToQuery.length === 0) {
       return {
-        mode: "hybrid",
+        mode: useRrf ? "rrf" : "hybrid",
         results: [],
         expandMs: 0,
         rawMerged: 0,
         suppressed: 0,
         bm25Bypass: 0,
         threshold: params.vectorThreshold ?? null,
+        stageCounts: emptyStageCounts,
       };
     }
   } else {
@@ -438,59 +574,198 @@ export async function hybrid(
     : "";
   const agentParams = agentSlug ? { agentSlug } : {};
 
-  for (const indexName of indexesToQuery) {
-    const vectorResult = await session.run(
-      `CALL db.index.vector.queryNodes($indexName, $limit, $embedding)
-       YIELD node, score
+  const keywordFilter = buildKeywordFilter(keywords, keywordMatch);
+  const keywordClause = keywordFilter?.clause ?? "";
+  const keywordParams = keywordFilter?.params ?? {};
+
+  // --- Multi-query expansion (Task 308) ---
+  // Original query is always first. On expansion failure expandQuery returns
+  // [query] only and logs `[graph-search:expansion-failed]`. Embeddings for
+  // additional paraphrases are computed in parallel; embed failures on a
+  // paraphrase fall back to skipping the vector half for that pass.
+  const expansions = useExpansion
+    ? await expandQuery({ query, retrievalClass, accountId })
+    : [query];
+
+  type Pass = {
+    queryText: string;
+    embedding: number[] | null;
+    vectorRanked: ScoredNode[];
+    bm25Ranked: ScoredNode[];
+  };
+
+  const passes: Pass[] = await Promise.all(
+    expansions.map(async (qText, idx) => {
+      let pEmbed: number[] | null;
+      if (idx === 0) {
+        pEmbed = queryEmbedding;
+      } else {
+        try {
+          pEmbed = await embed(qText);
+        } catch {
+          pEmbed = null;
+        }
+      }
+
+      const vectorRanked: ScoredNode[] = [];
+      const vectorSeen = new Map<string, number>();
+      if (pEmbed) {
+        for (const indexName of indexesToQuery) {
+          const vectorResult = await session.run(
+            `CALL db.index.vector.queryNodes($indexName, $limit, $embedding)
+             YIELD node, score
+             WHERE node.accountId = $accountId
+             ${scopeClause}
+             ${agentClause}
+             AND ${notTrashed("node")}
+             ${keywordClause}
+             RETURN node, score, labels(node) AS nodeLabels, elementId(node) AS nodeId
+             ORDER BY score DESC
+             LIMIT $limit`,
+            {
+              indexName,
+              embedding: pEmbed,
+              limit: int(limit),
+              accountId,
+              ...scopeParams,
+              ...agentParams,
+              ...keywordParams,
+            },
+          );
+          for (const record of vectorResult.records) {
+            const nodeId = record.get("nodeId") as string;
+            const scoreRaw = record.get("score");
+            const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw);
+            const existingIdx = vectorSeen.get(nodeId);
+            if (existingIdx !== undefined) {
+              const existing = vectorRanked[existingIdx];
+              if (score > existing.vectorScore) existing.vectorScore = score;
+              continue;
+            }
+            const node = record.get("node") as { properties: Record<string, unknown> };
+            vectorSeen.set(nodeId, vectorRanked.length);
+            vectorRanked.push({
+              nodeId,
+              labels: record.get("nodeLabels") as string[],
+              properties: plainProperties(node.properties),
+              vectorScore: score,
+              bm25Score: 0,
+              bm25Hit: false,
+            });
+          }
+        }
+        vectorRanked.sort((a, b) => b.vectorScore - a.vectorScore);
+      }
+
+      const bm25Hits = await bm25Only(session, {
+        ...params,
+        query: qText,
+        labels: effectiveLabels,
+      });
+      const bm25Norm = normaliseBm25Scores(bm25Hits.map((h) => h.score));
+      const bm25Ranked: ScoredNode[] = bm25Hits.map((h, i) => ({
+        nodeId: h.nodeId,
+        labels: h.labels,
+        properties: h.properties,
+        vectorScore: 0,
+        bm25Score: bm25Norm[i] ?? 0,
+        bm25Hit: true,
+      }));
+
+      return { queryText: qText, embedding: pEmbed, vectorRanked, bm25Ranked };
+    }),
+  );
+
+  // --- Fuse passes into the score map ---
+  // Weighted-sum: union per nodeId, taking the max vector score and max
+  // bm25-normalised score across passes, then combine with the route's
+  // weights. This preserves the baseline single-query behaviour exactly
+  // when expansions.length === 1.
+  // RRF: each pass contributes one vector-ranked list and one bm25-ranked
+  // list. Fusion is computed across the full union of ranked lists.
+  const scoreMap = new Map<string, ScoredNode>();
+  let totalVector = 0;
+  let totalBm25 = 0;
+
+  for (const pass of passes) {
+    totalVector += pass.vectorRanked.length;
+    totalBm25 += pass.bm25Ranked.length;
+    for (const v of pass.vectorRanked) {
+      const existing = scoreMap.get(v.nodeId);
+      if (existing) {
+        existing.vectorScore = Math.max(existing.vectorScore, v.vectorScore);
+      } else {
+        scoreMap.set(v.nodeId, { ...v });
+      }
+    }
+    for (const b of pass.bm25Ranked) {
+      mergeBm25Hit(scoreMap, {
+        nodeId: b.nodeId,
+        labels: b.labels,
+        properties: b.properties,
+        score: b.bm25Score,
+      }, b.bm25Score);
+    }
+  }
+
+  // --- Keyword subscriptions (scope-inclusive, agentSlug-bypassing) ---
+  // Runs once regardless of expansion count — subscriptions are keyed by
+  // operator config, not by the inbound query.
+  if (keywordSubscriptions && keywordSubscriptions.length > 0) {
+    for (const kw of keywordSubscriptions) {
+      const kwHits = await bm25Only(session, {
+        query: kw,
+        accountId,
+        limit,
+        allowedScopes,
+      });
+      if (kwHits.length === 0) continue;
+      const rawScores = kwHits.map((h) => h.score);
+      const normalised = normaliseBm25Scores(rawScores);
+      for (let i = 0; i < kwHits.length; i++) {
+        mergeBm25Hit(scoreMap, kwHits[i], normalised[i]);
+      }
+    }
+
+    const propScopeClause = allowedScopes
+      ? "AND (node.scope IS NULL OR node.scope IN $allowedScopes)"
+      : "";
+    const propResult = await session.run(
+      `MATCH (node)
        WHERE node.accountId = $accountId
-       ${scopeClause}
-       ${agentClause}
        AND ${notTrashed("node")}
-       ${keywordClause}
-       RETURN node, score, labels(node) AS nodeLabels, elementId(node) AS nodeId
-       ORDER BY score DESC
+       AND node.keywords IS NOT NULL
+       AND ANY(kw IN $kwSubs WHERE ANY(nk IN node.keywords WHERE toLower(nk) = kw))
+       ${propScopeClause}
+       RETURN node, labels(node) AS nodeLabels, elementId(node) AS nodeId
        LIMIT $limit`,
       {
-        indexName,
-        embedding: queryEmbedding,
-        limit: int(limit),
         accountId,
-        ...scopeParams,
-        ...agentParams,
-        ...keywordParams,
+        kwSubs: keywordSubscriptions,
+        limit: int(limit),
+        ...(allowedScopes ? { allowedScopes } : {}),
       },
     );
-    for (const record of vectorResult.records) {
+    for (const record of propResult.records) {
       const nodeId = record.get("nodeId") as string;
-      const scoreRaw = record.get("score");
-      const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw);
       const existing = scoreMap.get(nodeId);
       if (existing) {
-        existing.vectorScore = Math.max(existing.vectorScore, score);
+        existing.bm25Score = Math.max(existing.bm25Score, 1.0);
+        existing.bm25Hit = true;
       } else {
         const node = record.get("node") as { properties: Record<string, unknown> };
         scoreMap.set(nodeId, {
           nodeId,
           labels: record.get("nodeLabels") as string[],
           properties: plainProperties(node.properties),
-          vectorScore: score,
-          bm25Score: 0,
-          bm25Hit: false,
+          vectorScore: 0,
+          bm25Score: 1.0,
+          bm25Hit: true,
         });
       }
     }
   }
 
-  // --- BM25 half ---
-  const bm25Hits = await bm25Only(session, params);
-  if (bm25Hits.length > 0) {
-    const rawScores = bm25Hits.map((h) => h.score);
-    const normalised = normaliseBm25Scores(rawScores);
-    for (let i = 0; i < bm25Hits.length; i++) {
-      mergeBm25Hit(scoreMap, bm25Hits[i], normalised[i]);
-    }
-  }
-
   // --- Keyword subscriptions (scope-inclusive, agentSlug-bypassing) ---
   if (keywordSubscriptions && keywordSubscriptions.length > 0) {
     for (const kw of keywordSubscriptions) {
@@ -547,7 +822,69 @@ export async function hybrid(
     }
   }
 
-  // --- Merge, threshold, rank ---
+  // --- Fusion ---
+  //
+  // Weighted-sum (default): combined = vw * vector + bw * bm25_norm, with
+  // weights from the route plan (or the 0.7/0.3 lib defaults when routing
+  // is off). This preserves the baseline single-query single-fusion path
+  // when no flags are flipped on.
+  //
+  // RRF (flagged): contributions from each pass's vector and BM25 ranked
+  // lists are summed across all passes; combinedScore becomes the RRF
+  // score. Per-half components on `ScoredNode` (vectorScore / bm25Score)
+  // are retained so the /data UI's score-line renders consistently.
+  let allMerged: Array<ScoredNode & { combinedScore: number }>;
+  if (useRrf) {
+    const rankedLists: ScoredNode[][] = [];
+    for (const pass of passes) {
+      if (pass.vectorRanked.length > 0) rankedLists.push(pass.vectorRanked);
+      if (pass.bm25Ranked.length > 0) rankedLists.push(pass.bm25Ranked);
+    }
+    const fused = fuseRrf(rankedLists, rrfK);
+    allMerged = fused.map((f) => {
+      // Use the score-map representative so vectorScore / bm25Score reflect
+      // the cross-pass max, not just whichever list ranked first. The
+      // representative is guaranteed to exist (scoreMap aggregated the
+      // union); fall back to the fusion entry if not (defensive).
+      const rep = scoreMap.get(f.nodeId) ?? f;
+      return { ...rep, combinedScore: f.rrfScore };
+    });
+  } else {
+    allMerged = [...scoreMap.values()].map((node) => ({
+      ...node,
+      combinedScore: plan.vectorWeight * node.vectorScore + plan.bm25Weight * node.bm25Score,
+    }));
+  }
+  const fusedCount = allMerged.length;
+
+  // --- Boosts (Task 308) ---
+  // Both are no-ops on the current corpus until Task 305 (typed-edge
+  // backlinks) and Task 306 (compiledTruth property) land. Reading a
+  // missing property as zero is the correct behaviour — the boost is
+  // additive over hits that have the field.
+  if (useBoosts) {
+    allMerged = applyCompiledTruthBoost(
+      applyBacklinkBoost(allMerged.map((m) => ({
+        ...m,
+        score: m.combinedScore,
+      }))),
+    ).map((m) => ({ ...m, combinedScore: m.score }));
+  }
+  const boostedCount = allMerged.length;
+
+  // --- Dedup (Task 308) ---
+  // 4-layer dedup is a strict superset of the baseline nodeId-only dedup.
+  // The post-fusion scoreMap is already nodeId-deduped, so when no flag
+  // is set this step is a no-op.
+  if (useDedup) {
+    allMerged = dedupHits(
+      allMerged.map((m) => ({ ...m, score: m.combinedScore })),
+      dedupLayers,
+    ).map((m) => ({ ...m, combinedScore: m.score }));
+  }
+  const dedupedCount = allMerged.length;
+
+  // --- Threshold + sort + slice ---
   //
   // vectorThreshold filters BEFORE sort+slice. Order matters:
   // slicing first would consume the slot budget with rows the threshold
@@ -562,10 +899,6 @@ export async function hybrid(
   // normalisation pins the lowest BM25 row's score to 0.0, so checking
   // bm25Score > 0 would silently drop a literal-token match that just
   // happens to be the worst-scored BM25 hit in the set.
-  const allMerged = [...scoreMap.values()].map((node) => ({
-    ...node,
-    combinedScore: VECTOR_WEIGHT * node.vectorScore + BM25_WEIGHT * node.bm25Score,
-  }));
   const rawMerged = allMerged.length;
   const threshold = params.vectorThreshold;
   let kept = allMerged;
@@ -668,13 +1001,22 @@ export async function hybrid(
   }
 
   return {
-    mode: "hybrid",
+    mode: useRrf ? "rrf" : "hybrid",
     results,
     expandMs,
     rawMerged,
     suppressed,
     bm25Bypass,
     threshold: threshold ?? null,
+    stageCounts: {
+      expansions: expansions.length,
+      vector: totalVector,
+      bm25: totalBm25,
+      fused: fusedCount,
+      boosted: boostedCount,
+      deduped: dedupedCount,
+      final: results.length,
+    },
   };
 }