@gmickel/gno 1.9.0 → 1.10.1

package/src/pipeline/diagnose.ts

@@ -0,0 +1,302 @@
+/**
+ * Targeted query diagnostics.
+ *
+ * @module src/pipeline/diagnose
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type { DocumentRow, StoreResult } from "../store/types";
+import type { HybridSearchDeps } from "./hybrid";
+import type {
+  HybridSearchOptions,
+  QueryModeSummary,
+  QueryDiagnoseStageId,
+  QueryDiagnoseTraceCandidate,
+} from "./types";
+
+import { fingerprintContentTypeRules } from "../config";
+import { resolveDocRef } from "../core/ref-parser";
+import { err, ok } from "../store/types";
+import { evaluateQueryTargetFilters } from "./filters";
+import { searchHybrid } from "./hybrid";
+
+export type QueryDiagnoseTargetStatus =
+  | "not_found"
+  | "inactive"
+  | "no_indexed_content"
+  | "filtered_out"
+  | "diagnosed";
+
+export type QueryDiagnoseDropReason =
+  | "not_in_candidate_set"
+  | "below_cutoff"
+  | "skipped"
+  | null;
+
+export interface QueryDiagnoseStage {
+  id: QueryDiagnoseStageId;
+  status: "active" | "skipped";
+  sourceCount: number;
+  present: boolean;
+  rank: number | null;
+  score: number | null;
+  survived: boolean;
+  dropReason: QueryDiagnoseDropReason;
+  reason?: string;
+}
+
+export interface QueryDiagnoseResult {
+  schemaVersion: "1.0";
+  query: string;
+  target: {
+    ref: string;
+    status: QueryDiagnoseTargetStatus;
+    docid: string | null;
+    uri: string | null;
+    title: string | null;
+    contentType: string | null;
+    contentTypeSource: string | null;
+    categories: string[];
+    graphHints: string[];
+    contentTypeRulesFingerprint: string | null;
+    contentTypeFingerprintMatches: boolean | null;
+    mirrorHash: string | null;
+    chunkCount: number;
+    filterReasons: string[];
+  };
+  stages: QueryDiagnoseStage[];
+  chunk: {
+    seq: number | null;
+    startLine: number | null;
+    endLine: number | null;
+    language: string | null;
+  };
+  meta: {
+    mode: "bm25_only" | "hybrid";
+    vectorsUsed: boolean;
+    reranked: boolean;
+    totalResults: number;
+    queryModes?: QueryModeSummary;
+  };
+}
+
+export type QueryDiagnoseOptions = HybridSearchOptions & {
+  target: string;
+  contentTypeRules?: NormalizedContentTypeRule[];
+  contentTypeRulesFingerprint?: string;
+};
+
+function graphHintsForDoc(
+  doc: DocumentRow,
+  rules: NormalizedContentTypeRule[]
+): string[] {
+  if (!doc.contentType) {
+    return [];
+  }
+  return rules.find((rule) => rule.id === doc.contentType)?.graphHints ?? [];
+}
+
+function buildBaseResult(
+  query: string,
+  targetRef: string,
+  status: QueryDiagnoseTargetStatus,
+  doc: DocumentRow | null,
+  fields: {
+    graphHints?: string[];
+    chunkCount?: number;
+    filterReasons?: string[];
+    fingerprint?: string | null;
+    fingerprintMatches?: boolean | null;
+  } = {}
+): QueryDiagnoseResult {
+  return {
+    schemaVersion: "1.0",
+    query,
+    target: {
+      ref: targetRef,
+      status,
+      docid: doc?.docid ?? null,
+      uri: doc?.uri ?? null,
+      title: doc?.title ?? null,
+      contentType: doc?.contentType ?? null,
+      contentTypeSource: doc?.contentTypeSource ?? null,
+      categories: doc?.categories ?? [],
+      graphHints: fields.graphHints ?? [],
+      contentTypeRulesFingerprint: doc?.contentTypeRulesFingerprint ?? null,
+      contentTypeFingerprintMatches: fields.fingerprintMatches ?? null,
+      mirrorHash: doc?.mirrorHash ?? null,
+      chunkCount: fields.chunkCount ?? 0,
+      filterReasons: fields.filterReasons ?? [],
+    },
+    stages: [],
+    chunk: {
+      seq: null,
+      startLine: null,
+      endLine: null,
+      language: null,
+    },
+    meta: {
+      mode: "bm25_only",
+      vectorsUsed: false,
+      reranked: false,
+      totalResults: 0,
+    },
+  };
+}
+
+function findTargetCandidate(
+  candidates: QueryDiagnoseTraceCandidate[],
+  mirrorHash: string,
+  targetSeqs: Set<number>
+): QueryDiagnoseTraceCandidate | undefined {
+  return candidates.find(
+    (candidate) =>
+      candidate.mirrorHash === mirrorHash && targetSeqs.has(candidate.seq)
+  );
+}
+
+export async function diagnoseQueryTarget(
+  deps: HybridSearchDeps,
+  query: string,
+  options: QueryDiagnoseOptions
+): Promise<StoreResult<QueryDiagnoseResult>> {
+  const resolved = await resolveDocRef(deps.store, options.target);
+  if ("error" in resolved) {
+    return ok(buildBaseResult(query, options.target, "not_found", null));
+  }
+
+  const doc = resolved.doc;
+  const rules = options.contentTypeRules ?? [];
+  const expectedFingerprint =
+    options.contentTypeRulesFingerprint ?? fingerprintContentTypeRules(rules);
+  const fingerprintMatches = doc.contentTypeRulesFingerprint
+    ? doc.contentTypeRulesFingerprint === expectedFingerprint
+    : null;
+  const graphHints = graphHintsForDoc(doc, rules);
+
+  if (!doc.active) {
+    return ok(
+      buildBaseResult(query, options.target, "inactive", doc, {
+        graphHints,
+        fingerprintMatches,
+      })
+    );
+  }
+  if (!doc.mirrorHash) {
+    return ok(
+      buildBaseResult(query, options.target, "no_indexed_content", doc, {
+        graphHints,
+        fingerprintMatches,
+      })
+    );
+  }
+
+  const chunksResult = await deps.store.getChunks(doc.mirrorHash);
+  if (!chunksResult.ok) {
+    return err("QUERY_FAILED", chunksResult.error.message);
+  }
+  const chunks = chunksResult.value;
+  if (chunks.length === 0) {
+    return ok(
+      buildBaseResult(query, options.target, "no_indexed_content", doc, {
+        graphHints,
+        fingerprintMatches,
+      })
+    );
+  }
+
+  const filterEval = await evaluateQueryTargetFilters(
+    deps.store,
+    query,
+    doc,
+    chunks,
+    options
+  );
+  if (!filterEval.matches) {
+    return ok(
+      buildBaseResult(query, options.target, "filtered_out", doc, {
+        graphHints,
+        chunkCount: chunks.length,
+        filterReasons: filterEval.reasons,
+        fingerprintMatches,
+      })
+    );
+  }
+
+  const searchResult = await searchHybrid(deps, query, {
+    ...options,
+    diagnoseTrace: true,
+  });
+  if (!searchResult.ok) {
+    return err(searchResult.error.code, searchResult.error.message);
+  }
+
+  const trace = searchResult.value.meta.trace;
+  const targetSeqs = new Set(chunks.map((chunk) => chunk.seq));
+  let seenEarlier = false;
+  const stages: QueryDiagnoseStage[] =
+    trace?.stages.map((stage) => {
+      const candidate = findTargetCandidate(
+        stage.candidates,
+        doc.mirrorHash ?? "",
+        targetSeqs
+      );
+      const present = Boolean(candidate);
+      const dropReason: QueryDiagnoseDropReason =
+        stage.status === "skipped"
+          ? "skipped"
+          : present
+            ? null
+            : seenEarlier
+              ? "below_cutoff"
+              : "not_in_candidate_set";
+      if (present) {
+        seenEarlier = true;
+      }
+      return {
+        id: stage.id,
+        status: stage.status,
+        sourceCount: stage.sourceCount,
+        present,
+        rank: candidate?.rank ?? null,
+        score: candidate?.score ?? null,
+        survived: present,
+        dropReason,
+        reason: stage.reason,
+      };
+    }) ?? [];
+
+  const firstMatched = trace?.stages
+    .flatMap((stage) => stage.candidates)
+    .find(
+      (candidate) =>
+        candidate.mirrorHash === doc.mirrorHash && targetSeqs.has(candidate.seq)
+    );
+  const matchedChunk =
+    chunks.find((chunk) => chunk.seq === firstMatched?.seq) ??
+    chunks[0] ??
+    null;
+
+  return ok({
+    ...buildBaseResult(query, options.target, "diagnosed", doc, {
+      graphHints,
+      chunkCount: chunks.length,
+      fingerprintMatches,
+    }),
+    stages,
+    chunk: {
+      seq: matchedChunk?.seq ?? null,
+      startLine: matchedChunk?.startLine ?? null,
+      endLine: matchedChunk?.endLine ?? null,
+      language: matchedChunk?.language ?? null,
+    },
+    meta: {
+      mode:
+        searchResult.value.meta.mode === "bm25_only" ? "bm25_only" : "hybrid",
+      vectorsUsed: searchResult.value.meta.vectorsUsed ?? false,
+      reranked: searchResult.value.meta.reranked ?? false,
+      totalResults: searchResult.value.meta.totalResults,
+      queryModes: searchResult.value.meta.queryModes,
+    },
+  });
+}

package/src/pipeline/filters.ts

@@ -0,0 +1,119 @@
+/**
+ * Shared query filter evaluation for live query assembly and diagnostics.
+ *
+ * @module src/pipeline/filters
+ */
+
+import type { ChunkRow, DocumentRow, StorePort } from "../store/types";
+import type { HybridSearchOptions } from "./types";
+
+import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
+import { isWithinTemporalRange, resolveTemporalRange } from "./temporal";
+
+export interface QueryFilterEvaluation {
+  matches: boolean;
+  reasons: string[];
+}
+
+export function evaluateDocumentChunkFilters(
+  query: string,
+  doc: DocumentRow,
+  chunks: ChunkRow[],
+  options: HybridSearchOptions
+): QueryFilterEvaluation {
+  const reasons: string[] = [];
+  const temporalRange = resolveTemporalRange(
+    query,
+    options.since,
+    options.until
+  );
+
+  if (options.collection && doc.collection !== options.collection) {
+    reasons.push("collection");
+  }
+  if (!isWithinTemporalRange(doc.sourceMtime, temporalRange)) {
+    reasons.push("date");
+  }
+  if (
+    options.author &&
+    !doc.author?.toLowerCase().includes(options.author.toLowerCase())
+  ) {
+    reasons.push("author");
+  }
+  if (options.categories?.length) {
+    const allowed = new Set(options.categories.map((c) => c.toLowerCase()));
+    const contentTypeMatch = doc.contentType
+      ? allowed.has(doc.contentType.toLowerCase())
+      : false;
+    const categoryMatch = (doc.categories ?? []).some((c) =>
+      allowed.has(c.toLowerCase())
+    );
+    if (!contentTypeMatch && !categoryMatch) {
+      reasons.push("category");
+    }
+  }
+  if (
+    options.lang &&
+    !chunks.some((chunk) => chunk.language === options.lang)
+  ) {
+    reasons.push("lang");
+  }
+  if (
+    matchesExcludedText(
+      [
+        doc.title ?? "",
+        doc.relPath,
+        doc.author ?? "",
+        doc.contentType ?? "",
+        ...(doc.categories ?? []),
+      ],
+      options.exclude
+    ) ||
+    matchesExcludedChunks(chunks, options.exclude)
+  ) {
+    reasons.push("exclude");
+  }
+
+  return {
+    matches: reasons.length === 0,
+    reasons,
+  };
+}
+
+export async function evaluateQueryTargetFilters(
+  store: StorePort,
+  query: string,
+  doc: DocumentRow,
+  chunks: ChunkRow[],
+  options: HybridSearchOptions
+): Promise<QueryFilterEvaluation> {
+  const reasons = [
+    ...evaluateDocumentChunkFilters(query, doc, chunks, options).reasons,
+  ];
+
+  if (options.tagsAll?.length || options.tagsAny?.length) {
+    const tagsResult = await store.getTagsForDoc(doc.id);
+    if (!tagsResult.ok) {
+      reasons.push("tags");
+    } else {
+      const docTags = new Set(tagsResult.value.map((tag) => tag.tag));
+      if (
+        options.tagsAll?.length &&
+        !options.tagsAll.every((tag) => docTags.has(tag))
+      ) {
+        reasons.push("tagsAll");
+      }
+      if (
+        options.tagsAny?.length &&
+        !options.tagsAny.some((tag) => docTags.has(tag))
+      ) {
+        reasons.push("tagsAny");
+      }
+    }
+  }
+
+  return {
+    matches: reasons.length === 0,
+    reasons,
+  };
+}

package/src/pipeline/hybrid.ts

@@ -14,6 +14,8 @@ import type {
   ExplainLine,
   HybridSearchOptions,
   PipelineConfig,
+  QueryDiagnoseTrace,
+  QueryDiagnoseTraceCandidate,
   SearchResult,
   SearchResults,
 } from "./types";
@@ -22,7 +24,6 @@ import { embedTextsWithRecovery } from "../embed/batch";
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
 import { formatQueryForEmbedding } from "./contextual";
-import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
 import { expandQuery } from "./expansion";
 import {
   buildExplainResults,
@@ -36,6 +37,7 @@ import {
   explainTimings,
   explainVector,
 } from "./explain";
+import { evaluateDocumentChunkFilters } from "./filters";
 import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
 import { expandGraphCandidates } from "./graph-retrieval";
 import { selectBestChunkForSteering } from "./intent";
@@ -153,6 +155,7 @@ async function checkBm25Strength(
 interface ChunkId {
   mirrorHash: string;
   seq: number;
+  score?: number;
 }
 
 type FtsChunksResult =
@@ -196,6 +199,7 @@ async function searchFtsChunks(
     chunks: result.value.map((r) => ({
       mirrorHash: r.mirrorHash,
       seq: r.seq,
+      score: r.score,
     })),
   };
 }
@@ -236,6 +240,32 @@ async function searchVectorChunks(
   return searchResult.value.map((r) => ({
     mirrorHash: r.mirrorHash,
     seq: r.seq,
+    score: r.distance,
+  }));
+}
+
+function toTraceCandidates(chunks: ChunkId[]): QueryDiagnoseTraceCandidate[] {
+  return chunks.map((chunk, index) => ({
+    mirrorHash: chunk.mirrorHash,
+    seq: chunk.seq,
+    rank: index + 1,
+    score: chunk.score ?? index + 1,
+  }));
+}
+
+function candidatesToTrace(
+  candidates: Array<{
+    mirrorHash: string;
+    seq: number;
+    fusionScore?: number;
+    blendedScore?: number;
+  }>
+): QueryDiagnoseTraceCandidate[] {
+  return candidates.map((candidate, index) => ({
+    mirrorHash: candidate.mirrorHash,
+    seq: candidate.seq,
+    rank: index + 1,
+    score: candidate.blendedScore ?? candidate.fusionScore ?? index + 1,
   }));
 }
 
@@ -377,6 +407,9 @@ export async function searchHybrid(
   // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
   // ─────────────────────────────────────────────────────────────────────────
   const rankedInputs: RankedInput[] = [];
+  const diagnoseTrace: QueryDiagnoseTrace | undefined = options.diagnoseTrace
+    ? { stages: [] }
+    : undefined;
 
   const bm25StartedAt = performance.now();
 
@@ -401,6 +434,12 @@ export async function searchHybrid(
 
   const bm25Chunks = bm25Result.ok ? bm25Result.chunks : [];
   const bm25Count = bm25Chunks.length;
+  diagnoseTrace?.stages.push({
+    id: "bm25",
+    status: "active",
+    sourceCount: 1,
+    candidates: toTraceCandidates(bm25Chunks),
+  });
   if (bm25Count > 0) {
     rankedInputs.push(toRankedInput("bm25", bm25Chunks));
   }
@@ -446,6 +485,7 @@ export async function searchHybrid(
   }
 
   const vectorStartedAt = performance.now();
+  const vectorTraceChunks: ChunkId[] = [];
 
   if (vectorAvailable && vectorIndex && embedPort) {
     const vectorVariantQueries = [
@@ -469,6 +509,7 @@ export async function searchHybrid(
       );
 
       vecCount = vecChunks.length;
+      vectorTraceChunks.push(...vecChunks);
       if (vecCount > 0) {
         rankedInputs.push(toRankedInput("vector", vecChunks));
       }
@@ -520,6 +561,7 @@ export async function searchHybrid(
           if (variant.source === "vector") {
             vecCount = chunks.length;
           }
+          vectorTraceChunks.push(...chunks);
           if (chunks.length === 0) {
             continue;
           }
@@ -530,6 +572,14 @@ export async function searchHybrid(
   }
   timings.vectorMs = performance.now() - vectorStartedAt;
 
+  diagnoseTrace?.stages.push({
+    id: "vector",
+    status: vectorAvailable ? "active" : "skipped",
+    reason: vectorAvailable ? undefined : "vector_unavailable",
+    sourceCount: vectorAvailable ? 1 : 0,
+    candidates: toTraceCandidates(vectorTraceChunks),
+  });
+
   explainLines.push(
     explainVector(
       vecCount,
@@ -546,6 +596,12 @@ export async function searchHybrid(
   const candidateLimit =
     options.candidateLimit ?? pipelineConfig.rerankCandidates;
   let fusedCandidates = rrfFuse(rankedInputs, pipelineConfig.rrf);
+  diagnoseTrace?.stages.push({
+    id: "fusion",
+    status: "active",
+    sourceCount: rankedInputs.length,
+    candidates: candidatesToTrace(fusedCandidates),
+  });
 
   timings.fusionMs = performance.now() - fusionStartedAt;
   const graphStartedAt = performance.now();
@@ -570,6 +626,20 @@ export async function searchHybrid(
     fusedCandidates = rrfFuse(rankedInputs, pipelineConfig.rrf);
     timings.fusionMs += performance.now() - graphFusionStartedAt;
   }
+  diagnoseTrace?.stages.push({
+    id: "graph",
+    status: graphExpansion.meta.attempted ? "active" : "skipped",
+    reason: graphExpansion.meta.attempted
+      ? undefined
+      : graphExpansion.meta.fallbackReasons.join(", ") || "disabled",
+    sourceCount: graphExpansion.meta.attempted ? 1 : 0,
+    candidates: graphExpansion.candidates.map((candidate, index) => ({
+      mirrorHash: candidate.mirrorHash,
+      seq: candidate.seq,
+      rank: index + 1,
+      score: index + 1,
+    })),
+  });
   if (graphExpansion.meta.fallbackReasons.length > 0) {
     counters.fallbackEvents.push(...graphExpansion.meta.fallbackReasons);
   }
@@ -604,6 +674,16 @@ export async function searchHybrid(
   }
   timings.rerankMs = performance.now() - rerankStartedAt;
 
+  diagnoseTrace?.stages.push({
+    id: "rerank",
+    status: rerankResult.reranked ? "active" : "skipped",
+    reason: rerankResult.reranked
+      ? undefined
+      : (rerankResult.fallbackReason ?? "disabled"),
+    sourceCount: rerankResult.reranked ? 1 : 0,
+    candidates: candidatesToTrace(rerankResult.candidates),
+  });
+
   explainLines.push(
     explainRerank(!options.noRerank && rerankPort !== null, candidateLimit)
   );
@@ -754,22 +834,14 @@ export async function searchHybrid(
       continue;
     }
 
-    const excluded =
-      matchesExcludedText(
-        [
-          doc.title ?? "",
-          doc.relPath,
-          doc.author ?? "",
-          doc.contentType ?? "",
-          ...(doc.categories ?? []),
-        ],
-        options.exclude
-      ) ||
-      matchesExcludedChunks(
-        chunksMap.get(candidate.mirrorHash) ?? [],
-        options.exclude
-      );
-    if (excluded) {
+    const docChunks = chunksMap.get(candidate.mirrorHash) ?? [];
+    const filterEval = evaluateDocumentChunkFilters(
+      query,
+      doc,
+      docChunks,
+      options
+    );
+    if (!filterEval.matches) {
       continue;
     }
 
@@ -935,6 +1007,7 @@ export async function searchHybrid(
       queryLanguage,
       queryModes: queryModeSummary,
       explain: explainData,
+      trace: diagnoseTrace,
     },
   });
 }

package/src/pipeline/types.ts

@@ -104,6 +104,8 @@ export interface SearchMeta {
     lines: ExplainLine[];
     results: ExplainResult[];
   };
+  /** Internal diagnose trace, only populated when diagnoseTrace is enabled */
+  trace?: QueryDiagnoseTrace;
 }
 
 /** Complete search results wrapper */
@@ -182,6 +184,8 @@ export type HybridSearchOptions = SearchOptions & {
   noGraph?: boolean;
   /** Language hint for prompt selection (does NOT filter retrieval, only affects expansion prompts) */
   queryLanguageHint?: string;
+  /** Internal: capture per-stage candidates for query diagnose */
+  diagnoseTrace?: boolean;
 };
 
 /** Options for ask command */
@@ -258,6 +262,34 @@ export interface FusionCandidate {
   sources: FusionSource[];
 }
 
+export type QueryDiagnoseStageId =
+  | "bm25"
+  | "vector"
+  | "fusion"
+  | "graph"
+  | "rerank";
+
+export type QueryDiagnoseStageStatus = "active" | "skipped";
+
+export interface QueryDiagnoseTraceCandidate {
+  mirrorHash: string;
+  seq: number;
+  rank: number;
+  score: number;
+}
+
+export interface QueryDiagnoseTraceStage {
+  id: QueryDiagnoseStageId;
+  status: QueryDiagnoseStageStatus;
+  reason?: string;
+  sourceCount: number;
+  candidates: QueryDiagnoseTraceCandidate[];
+}
+
+export interface QueryDiagnoseTrace {
+  stages: QueryDiagnoseTraceStage[];
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Rerank & Blending Types
 // ─────────────────────────────────────────────────────────────────────────────

package/src/publish/export-service.ts

@@ -7,7 +7,7 @@
 import type { Collection } from "../config/types";
 import type { DocumentRow, StorePort, TagRow } from "../store/types";
 
-import { parseRef } from "../cli/commands/ref-parser";
+import { parseRef } from "../core/ref-parser";
 import { parseFrontmatter } from "../ingestion/frontmatter";
 import { getContentBatch } from "../store/content-batch";
 import {