@gmickel/gno 1.3.1 → 1.4.1

package/README.md

@@ -880,11 +880,13 @@ bun run lint && bun run typecheck
 Use retrieval benchmark commands to track quality and latency over time:
 
 ```bash
+gno bench docs/examples/bench-fixture.json
 bun run eval:hybrid
 bun run eval:hybrid:baseline
 bun run eval:hybrid:delta
 ```
 
+- Public fixture runner: `gno bench <fixture.json>` reports Precision@K, Recall@K, F1@K, MRR, nDCG@K, and latency across BM25/vector/hybrid modes.
 - Benchmark guide: [evals/README.md](./evals/README.md)
 - Latest baseline snapshot: [evals/fixtures/hybrid-baseline/latest.json](./evals/fixtures/hybrid-baseline/latest.json)
 

package/assets/skill/SKILL.md

@@ -131,6 +131,18 @@ gno query "auth" --json | jq -r '.results[0].uri' | xargs gno get
 gno search "error handling" --json | jq -r '.results[].uri' | xargs gno multi-get
 ```
 
+## MCP Retrieval Strategy
+
+When using GNO through MCP, prefer `gno_query` first for normal questions. It returns snippets plus `uri`, `docid`, and often `line`; follow with `gno_get` using `fromLine`/`lineCount` for a bounded read, or `gno_multi_get` to batch top result refs.
+
+Use narrower tools when the request tells you to:
+
+- `gno_search`: exact phrase, filename, identifier, stack trace, error text
+- `gno_vsearch`: conceptual similarity when exact wording differs
+- `gno_status`: stale results, missing embeddings, vector unavailable
+
+For ambiguous terms, pass `intent` instead of bloating the query text. For typed retrieval, use `queryModes`: `term` for lexical anchors, `intent` for disambiguation, one `hyde` for a hypothetical answer/document.
+
 ## Document Links & Similarity
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmickel/gno",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
   "keywords": [
     "embeddings",
@@ -69,6 +69,7 @@
     "eval:hybrid": "bun --bun evalite evals/hybrid.eval.ts",
     "eval:hybrid:baseline": "bun scripts/hybrid-benchmark.ts --write",
     "eval:hybrid:delta": "bun scripts/hybrid-benchmark.ts --delta",
+    "bench:ast-chunking": "bun scripts/ast-chunking-benchmark.ts",
     "bench:code-embeddings": "bun scripts/code-embedding-benchmark.ts",
     "bench:code-embeddings:write": "bun scripts/code-embedding-benchmark.ts --write",
     "bench:general-embeddings": "bun scripts/general-embedding-benchmark.ts",
@@ -121,7 +122,7 @@
     "website:og": "bun scripts/og-screenshots.ts",
     "website:sync-assets": "bun scripts/sync-assets.ts",
     "sync:agents": "scripts/sync-agents.sh",
-    "build:css": "bunx @tailwindcss/cli -i src/serve/public/globals.css -o src/serve/public/globals.built.css --minify",
+    "build:css": "tailwindcss -i src/serve/public/globals.css -o src/serve/public/globals.built.css --minify",
     "serve": "bun src/index.ts serve",
     "serve:dev": "NODE_ENV=development bun --hot src/index.ts serve",
     "version:patch": "npm version patch --no-git-tag-version",
@@ -133,73 +134,75 @@
     "prepare": "lefthook install"
   },
   "dependencies": {
-    "@codemirror/lang-markdown": "^6.5.0",
-    "@codemirror/theme-one-dark": "^6.1.3",
-    "@modelcontextprotocol/sdk": "^1.27.1",
-    "@radix-ui/react-collapsible": "^1.1.12",
-    "@radix-ui/react-dialog": "^1.1.15",
-    "@radix-ui/react-dropdown-menu": "^2.1.16",
-    "@radix-ui/react-hover-card": "^1.1.15",
-    "@radix-ui/react-progress": "^1.1.8",
-    "@radix-ui/react-scroll-area": "^1.2.10",
-    "@radix-ui/react-select": "^2.2.6",
-    "@radix-ui/react-separator": "^1.1.8",
-    "@radix-ui/react-slot": "^1.2.4",
-    "@radix-ui/react-tooltip": "^1.2.8",
-    "ai": "^6.0.5",
-    "bun-plugin-tailwind": "^0.1.2",
-    "class-variance-authority": "^0.7.1",
-    "clsx": "^2.1.1",
-    "cmdk": "^1.1.1",
-    "codemirror": "^6.0.2",
-    "commander": "^14.0.2",
-    "embla-carousel-react": "^8.6.0",
-    "franc": "^6.2.0",
-    "lucide-react": "^1.8.0",
-    "markitdown-ts": "^0.0.9",
-    "minimatch": "^10.1.1",
-    "nanoid": "^5.1.6",
-    "node-llama-cpp": "^3.17.1",
-    "officeparser": "^6.0.4",
-    "picocolors": "^1.1.1",
-    "react": "^19.2.3",
-    "react-dom": "^19.2.3",
-    "react-force-graph-2d": "^1.29.0",
-    "react-markdown": "^10.1.0",
-    "rehype-sanitize": "^6.0.0",
-    "remark-gfm": "^4.0.1",
-    "shiki": "^4.0.2",
-    "sqlite-vec": "^0.1.7-alpha.2",
-    "streamdown": "^2.0.1",
-    "tailwind-merge": "^3.4.0",
-    "tailwindcss": "^4.1.18",
-    "use-stick-to-bottom": "^1.1.1",
-    "zod": "^4.3.4"
+    "@codemirror/lang-markdown": "6.5.0",
+    "@codemirror/theme-one-dark": "6.1.3",
+    "@modelcontextprotocol/sdk": "1.27.1",
+    "@radix-ui/react-collapsible": "1.1.12",
+    "@radix-ui/react-dialog": "1.1.15",
+    "@radix-ui/react-dropdown-menu": "2.1.16",
+    "@radix-ui/react-hover-card": "1.1.15",
+    "@radix-ui/react-progress": "1.1.8",
+    "@radix-ui/react-scroll-area": "1.2.10",
+    "@radix-ui/react-select": "2.2.6",
+    "@radix-ui/react-separator": "1.1.8",
+    "@radix-ui/react-slot": "1.2.4",
+    "@radix-ui/react-tooltip": "1.2.8",
+    "ai": "6.0.68",
+    "bun-plugin-tailwind": "0.1.2",
+    "class-variance-authority": "0.7.1",
+    "clsx": "2.1.1",
+    "cmdk": "1.1.1",
+    "codemirror": "6.0.2",
+    "commander": "14.0.3",
+    "embla-carousel-react": "8.6.0",
+    "franc": "6.2.0",
+    "lucide-react": "1.8.0",
+    "markitdown-ts": "0.0.9",
+    "minimatch": "10.1.1",
+    "nanoid": "5.1.6",
+    "node-llama-cpp": "3.18.1",
+    "officeparser": "6.0.4",
+    "picocolors": "1.1.1",
+    "react": "19.2.4",
+    "react-dom": "19.2.4",
+    "react-force-graph-2d": "1.29.0",
+    "react-markdown": "10.1.0",
+    "rehype-sanitize": "6.0.0",
+    "remark-gfm": "4.0.1",
+    "shiki": "4.0.2",
+    "sqlite-vec": "0.1.9",
+    "streamdown": "2.1.0",
+    "tailwind-merge": "3.4.0",
+    "tailwindcss": "4.1.18",
+    "use-stick-to-bottom": "1.1.2",
+    "zod": "4.3.6"
   },
   "devDependencies": {
-    "@ai-sdk/openai": "^3.0.2",
+    "@ai-sdk/openai": "3.0.25",
     "@biomejs/biome": "2.3.14",
-    "@tailwindcss/cli": "^4.1.18",
-    "@testing-library/react": "^16.3.2",
-    "@testing-library/user-event": "^14.6.1",
-    "@types/bun": "latest",
-    "@types/react": "^19.2.13",
-    "@types/react-dom": "^19.2.3",
-    "ajv": "^8.17.1",
-    "ajv-formats": "^3.0.1",
-    "docx": "^9.5.1",
-    "evalite": "^1.0.0-beta.15",
-    "exceljs": "^4.4.0",
-    "happy-dom": "^20.8.9",
-    "lefthook": "^2.1.0",
-    "oxfmt": "^0.28.0",
-    "oxlint": "^1.42.0",
-    "oxlint-tsgolint": "^0.11.5",
-    "pdf-lib": "^1.17.1",
-    "playwright": "^1.58.2",
-    "pptxgenjs": "^4.0.1",
+    "@tailwindcss/cli": "4.1.18",
+    "@testing-library/react": "16.3.2",
+    "@testing-library/user-event": "14.6.1",
+    "@types/bun": "1.3.8",
+    "@types/react": "19.2.14",
+    "@types/react-dom": "19.2.3",
+    "@vscode/tree-sitter-wasm": "0.3.1",
+    "ajv": "8.17.1",
+    "ajv-formats": "3.0.1",
+    "docx": "9.5.1",
+    "evalite": "1.0.0-beta.16",
+    "exceljs": "4.4.0",
+    "happy-dom": "20.8.9",
+    "lefthook": "2.1.4",
+    "oxfmt": "0.28.0",
+    "oxlint": "1.43.0",
+    "oxlint-tsgolint": "0.11.5",
+    "pdf-lib": "1.17.1",
+    "playwright": "1.58.2",
+    "pptxgenjs": "4.0.1",
     "ultracite": "7.1.5",
-    "vitest": "^4.0.16"
+    "vitest": "4.0.18",
+    "web-tree-sitter": "0.26.8"
   },
   "peerDependencies": {
     "typescript": "^5"

package/src/bench/fixture.ts

@@ -0,0 +1,247 @@
+import { z } from "zod";
+
+import type { BenchFixture, BenchMode, BenchOptions } from "./types";
+
+const MODE_ALIASES = [
+  "bm25",
+  "vector",
+  "hybrid",
+  "fast",
+  "no-rerank",
+  "thorough",
+] as const;
+
+type BenchModeAlias = (typeof MODE_ALIASES)[number];
+
+const queryModeInputSchema = z.object({
+  mode: z.enum(["term", "intent", "hyde"]),
+  text: z.string().trim().min(1),
+});
+
+const modeObjectSchema = z.object({
+  name: z.string().trim().min(1).optional(),
+  type: z.enum(["bm25", "vector", "hybrid"]).optional(),
+  mode: z.enum(MODE_ALIASES).optional(),
+  noExpand: z.boolean().optional(),
+  noRerank: z.boolean().optional(),
+  candidateLimit: z.number().int().positive().optional(),
+  limit: z.number().int().positive().optional(),
+  queryModes: z.array(queryModeInputSchema).optional(),
+});
+
+const fixtureSchema = z.object({
+  version: z.literal(1),
+  metadata: z
+    .object({
+      name: z.string().optional(),
+      description: z.string().optional(),
+      tags: z.array(z.string()).optional(),
+    })
+    .optional(),
+  collection: z.string().trim().min(1).optional(),
+  topK: z.number().int().positive().optional(),
+  candidateLimit: z.number().int().positive().optional(),
+  modes: z.array(z.union([z.enum(MODE_ALIASES), modeObjectSchema])).optional(),
+  queries: z
+    .array(
+      z.object({
+        id: z.string().trim().min(1),
+        query: z.string().trim().min(1),
+        expected: z.array(z.string().trim().min(1)).optional(),
+        expectedDocuments: z.array(z.string().trim().min(1)).optional(),
+        expectedUris: z.array(z.string().trim().min(1)).optional(),
+        judgments: z
+          .array(
+            z.object({
+              docid: z.string().trim().min(1).optional(),
+              doc: z.string().trim().min(1).optional(),
+              uri: z.string().trim().min(1).optional(),
+              relevance: z.number().min(0),
+            })
+          )
+          .optional(),
+        collection: z.string().trim().min(1).optional(),
+        topK: z.number().int().positive().optional(),
+        queryModes: z.array(queryModeInputSchema).optional(),
+      })
+    )
+    .min(1),
+});
+
+type FixtureModeInput = NonNullable<
+  z.infer<typeof fixtureSchema>["modes"]
+>[number];
+
+export function normalizeBenchRef(value: string): string {
+  const trimmed = value.trim();
+  const queryIndex = trimmed.indexOf("?");
+  return queryIndex === -1 ? trimmed : trimmed.slice(0, queryIndex);
+}
+
+function normalizeMode(alias: BenchModeAlias): BenchMode {
+  switch (alias) {
+    case "bm25":
+      return { name: "bm25", type: "bm25" };
+    case "vector":
+      return { name: "vector", type: "vector" };
+    case "fast":
+      return {
+        name: "fast",
+        type: "hybrid",
+        noExpand: true,
+        noRerank: true,
+      };
+    case "no-rerank":
+      return { name: "no-rerank", type: "hybrid", noRerank: true };
+    case "thorough":
+      return { name: "thorough", type: "hybrid", depth: "thorough" };
+    case "hybrid":
+      return { name: "hybrid", type: "hybrid" };
+  }
+}
+
+function normalizeModeInput(input: FixtureModeInput): BenchMode {
+  if (typeof input === "string") {
+    return normalizeMode(input as BenchModeAlias);
+  }
+
+  const base = input.mode ? normalizeMode(input.mode) : undefined;
+  const type = input.type ?? base?.type ?? "hybrid";
+  const name = input.name ?? input.mode ?? type;
+  return {
+    ...base,
+    name,
+    type,
+    depth: base?.depth,
+    noExpand: input.noExpand ?? base?.noExpand,
+    noRerank: input.noRerank ?? base?.noRerank,
+    candidateLimit: input.candidateLimit,
+    limit: input.limit,
+    queryModes: input.queryModes,
+  };
+}
+
+function parseModeFlag(
+  mode: string
+): { ok: true; value: BenchMode } | { ok: false; error: string } {
+  const normalized = mode.trim() as BenchModeAlias;
+  if (!MODE_ALIASES.includes(normalized)) {
+    return {
+      ok: false,
+      error: `Unsupported bench mode: ${mode}. Supported: ${MODE_ALIASES.join(", ")}`,
+    };
+  }
+  return { ok: true, value: normalizeMode(normalized) };
+}
+
+function normalizeModes(
+  fixtureModes: z.infer<typeof fixtureSchema>["modes"],
+  optionModes?: string[]
+): BenchMode[] {
+  if (optionModes?.length) {
+    return optionModes.map((mode) => {
+      const parsed = parseModeFlag(mode);
+      if (!parsed.ok) {
+        throw new Error(parsed.error);
+      }
+      return parsed.value;
+    });
+  }
+
+  return (fixtureModes ?? ["bm25"]).map(normalizeModeInput);
+}
+
+function normalizeFixture(
+  parsed: z.infer<typeof fixtureSchema>,
+  options: BenchOptions
+): BenchFixture {
+  const modes = normalizeModes(parsed.modes, options.modes);
+  const topK = options.topK ?? parsed.topK ?? 10;
+  const candidateLimit = options.candidateLimit ?? parsed.candidateLimit;
+
+  return {
+    version: parsed.version,
+    metadata: parsed.metadata,
+    collection: options.collection ?? parsed.collection,
+    topK,
+    candidateLimit,
+    modes,
+    queries: parsed.queries.map((entry) => {
+      const explicitExpected = [
+        ...(entry.expected ?? []),
+        ...(entry.expectedDocuments ?? []),
+        ...(entry.expectedUris ?? []),
+      ].map(normalizeBenchRef);
+      const judgments =
+        entry.judgments?.flatMap((judgment) => {
+          const docid = judgment.docid ?? judgment.doc ?? judgment.uri;
+          return docid
+            ? [
+                {
+                  docid: normalizeBenchRef(docid),
+                  relevance: judgment.relevance,
+                },
+              ]
+            : [];
+        }) ?? [];
+      const expected =
+        explicitExpected.length > 0
+          ? explicitExpected
+          : judgments.map((judgment) => judgment.docid);
+
+      return {
+        id: entry.id,
+        query: entry.query,
+        expected,
+        judgments,
+        collection: options.collection ?? entry.collection ?? parsed.collection,
+        topK: entry.topK,
+        queryModes: entry.queryModes,
+      };
+    }),
+  };
+}
+
+export async function loadBenchFixture(
+  fixturePath: string,
+  options: BenchOptions
+): Promise<{ ok: true; fixture: BenchFixture } | { ok: false; error: string }> {
+  const file = Bun.file(fixturePath);
+  if (!(await file.exists())) {
+    return { ok: false, error: `Fixture not found: ${fixturePath}` };
+  }
+
+  let raw: unknown;
+  try {
+    raw = JSON.parse(await file.text());
+  } catch (error) {
+    return {
+      ok: false,
+      error: `Invalid JSON fixture: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+
+  const parsed = fixtureSchema.safeParse(raw);
+  if (!parsed.success) {
+    return { ok: false, error: z.prettifyError(parsed.error) };
+  }
+
+  try {
+    const fixture = normalizeFixture(parsed.data, options);
+    const missingExpected = fixture.queries.find(
+      (entry) => entry.expected.length === 0
+    );
+    if (missingExpected) {
+      return {
+        ok: false,
+        error: `Bench query "${missingExpected.id}" must define expected documents, expected URIs, or judgments`,
+      };
+    }
+    return { ok: true, fixture };
+  } catch (error) {
+    return {
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}

package/src/bench/metrics.ts

@@ -0,0 +1,137 @@
+/**
+ * Retrieval benchmark metric helpers.
+ *
+ * @module src/bench/metrics
+ */
+
+export interface RelevanceJudgment {
+  docid: string;
+  relevance: number;
+}
+
+export interface RetrievalMetrics {
+  precisionAtK: number;
+  recallAtK: number;
+  f1AtK: number;
+  mrr: number;
+  ndcgAtK: number;
+}
+
+function round(value: number, places = 4): number {
+  return Number(value.toFixed(places));
+}
+
+/**
+ * Compute Precision@K: fraction of retrieved top-K docs that are relevant.
+ */
+export function computePrecision(
+  output: string[],
+  expected: string[],
+  k: number
+): number {
+  if (k <= 0) {
+    return 0;
+  }
+  const expectedSet = new Set(expected);
+  const hits = output.slice(0, k).filter((docid) => expectedSet.has(docid));
+  return hits.length / k;
+}
+
+/**
+ * Compute Recall@K: fraction of relevant docs in top K results.
+ */
+export function computeRecall(
+  output: string[],
+  expected: string[],
+  k: number
+): number {
+  if (expected.length === 0) return 1;
+  const topK = output.slice(0, k);
+  const hits = expected.filter((docid) => topK.includes(docid)).length;
+  return hits / expected.length;
+}
+
+/**
+ * Compute F1@K from precision and recall.
+ */
+export function computeF1(precision: number, recall: number): number {
+  if (precision === 0 && recall === 0) {
+    return 0;
+  }
+  return (2 * precision * recall) / (precision + recall);
+}
+
+/**
+ * Compute nDCG@K: normalized discounted cumulative gain.
+ */
+export function computeNdcg(
+  output: string[],
+  judgments: RelevanceJudgment[],
+  k: number
+): number {
+  if (judgments.length === 0) return 1;
+  const relMap = new Map(judgments.map((j) => [j.docid, j.relevance]));
+  const dcg = output.slice(0, k).reduce((sum, docid, i) => {
+    const rel = relMap.get(docid) ?? 0;
+    return sum + (2 ** rel - 1) / Math.log2(i + 2);
+  }, 0);
+  const idcg = [...judgments]
+    .sort((a, b) => b.relevance - a.relevance)
+    .slice(0, k)
+    .reduce((sum, j, i) => sum + (2 ** j.relevance - 1) / Math.log2(i + 2), 0);
+  return idcg > 0 ? dcg / idcg : 1;
+}
+
+/**
+ * Compute Mean Reciprocal Rank (single-query form).
+ * Returns reciprocal rank of first relevant hit in output.
+ */
+export function computeMrr(output: string[], expected: string[]): number {
+  if (expected.length === 0) {
+    return 1;
+  }
+  const expectedSet = new Set(expected);
+  for (const [index, docid] of output.entries()) {
+    if (expectedSet.has(docid)) {
+      return 1 / (index + 1);
+    }
+  }
+  return 0;
+}
+
+export function computeRetrievalMetrics(input: {
+  output: string[];
+  expected: string[];
+  judgments: RelevanceJudgment[];
+  k: number;
+}): RetrievalMetrics {
+  const precision = computePrecision(input.output, input.expected, input.k);
+  const recall = computeRecall(input.output, input.expected, input.k);
+  const judgmentSource =
+    input.judgments.length > 0
+      ? input.judgments
+      : input.expected.map((docid) => ({ docid, relevance: 1 }));
+
+  return {
+    precisionAtK: round(precision),
+    recallAtK: round(recall),
+    f1AtK: round(computeF1(precision, recall)),
+    mrr: round(computeMrr(input.output, input.expected)),
+    ndcgAtK: round(computeNdcg(input.output, judgmentSource, input.k)),
+  };
+}
+
+export function averageMetrics(metrics: RetrievalMetrics[]): RetrievalMetrics {
+  const average = (values: number[]): number =>
+    values.length === 0
+      ? 0
+      : values.reduce((sum, value) => sum + value, 0) / values.length;
+
+  return {
+    precisionAtK: round(average(metrics.map((m) => m.precisionAtK))),
+    recallAtK: round(average(metrics.map((m) => m.recallAtK))),
+    f1AtK: round(average(metrics.map((m) => m.f1AtK))),
+    mrr: round(average(metrics.map((m) => m.mrr))),
+    ndcgAtK: round(average(metrics.map((m) => m.ndcgAtK))),
+  };
+}

package/src/bench/types.ts

@@ -0,0 +1,96 @@
+import type { QueryModeInput } from "../pipeline/types";
+import type { RelevanceJudgment, RetrievalMetrics } from "./metrics";
+
+export type BenchModeType = "bm25" | "vector" | "hybrid";
+
+export interface BenchMode {
+  name: string;
+  type: BenchModeType;
+  depth?: "thorough";
+  noExpand?: boolean;
+  noRerank?: boolean;
+  candidateLimit?: number;
+  limit?: number;
+  queryModes?: QueryModeInput[];
+}
+
+export interface BenchCase {
+  id: string;
+  query: string;
+  expected: string[];
+  judgments: RelevanceJudgment[];
+  collection?: string;
+  topK?: number;
+  queryModes?: QueryModeInput[];
+}
+
+export interface BenchFixture {
+  version: 1;
+  metadata?: {
+    name?: string;
+    description?: string;
+    tags?: string[];
+  };
+  collection?: string;
+  topK: number;
+  candidateLimit?: number;
+  modes: BenchMode[];
+  queries: BenchCase[];
+}
+
+export interface BenchOptions {
+  configPath?: string;
+  indexName?: string;
+  collection?: string;
+  topK?: number;
+  candidateLimit?: number;
+  modes?: string[];
+  json?: boolean;
+}
+
+export interface BenchCaseResult {
+  id: string;
+  query: string;
+  topK: number;
+  expected: string[];
+  hits: string[];
+  topDocs: string[];
+  metrics: RetrievalMetrics;
+  latencyMs: number;
+  error?: string;
+}
+
+export interface BenchModeResult {
+  name: string;
+  type: BenchModeType;
+  status: "ok" | "failed";
+  queryCount: number;
+  failures: number;
+  metrics: RetrievalMetrics;
+  latency: {
+    p50Ms: number;
+    p95Ms: number;
+    meanMs: number;
+  };
+  cases: BenchCaseResult[];
+}
+
+export interface BenchOutput {
+  fixture: {
+    path: string;
+    name?: string;
+    version: 1;
+    queryCount: number;
+    topK: number;
+  };
+  generatedAt: string;
+  modes: BenchModeResult[];
+  meta: {
+    indexName: string;
+    collection?: string;
+  };
+}
+
+export type BenchResult =
+  | { success: true; data: BenchOutput }
+  | { success: false; error: string; isValidation?: boolean };