@gmickel/gno 1.14.0 → 1.16.0

package/README.md

@@ -94,7 +94,7 @@ gno daemon --detach  # headless continuous indexing (background; --status / --st
 
 <!-- public-truth:current-version -->
 
-> Current release: **v1.14.0** — see [CHANGELOG.md](./CHANGELOG.md)
+> Current release: **v1.16.0** — see [CHANGELOG.md](./CHANGELOG.md)
 
 <!-- /public-truth -->
 
@@ -1036,9 +1036,30 @@ official production GGUF was validated for Nemotron.
 These small fixture results support keeping Qwen as the built-in default; they
 do not establish general language superiority. Query-language classification
 supports a broader set than the indexed-document detector (`en`, `de`, `fr`,
-`it`, `zh`, `ja`, `ko`), and the committed fixture covers only five languages.
-The BM25-only multilingual eval remains a legacy sanity lane; dedicated lexical
-CJK benchmarking is pending.
+`it`, `zh`, `ja`, `ko`), and the committed semantic fixture covers only five
+languages.
+
+<!-- public-truth:cjk-lexical-benchmark -->
+
+Lexical fallback has separate evidence. The immutable
+[July 22, 2026 CJK result](./evals/fixtures/cjk-lexical-benchmark/2026-07-22.md)
+uses 21 synthetic documents and 25 same-language queries across Chinese,
+Japanese, and Korean. Production BM25 lexical results and frozen floors:
+
+- Chinese: baseline Recall@10 `0.2222`, nDCG@10 `0.1481`, zero-result `0.7778`; promotion Recall@10 `0.4722`, nDCG@10 `0.3981`, maximum zero-result `0.5278`
+- Japanese: baseline Recall@10 `0.125`, nDCG@10 `0.125`, zero-result `0.875`; promotion Recall@10 `0.375`, nDCG@10 `0.375`, maximum zero-result `0.625`
+- Korean: baseline Recall@10 `0.5`, nDCG@10 `0.5`, zero-result `0.5`; promotion Recall@10 `0.75`, nDCG@10 `0.75`, maximum zero-result `0.25`
+
+The
+[promotion-gates.md](./evals/fixtures/cjk-lexical-benchmark/promotion-gates.md)
+also bind MRR, non-regression, and cost requirements. This lexical result does
+not reduce or replace the semantic evidence above. All positive qrels use
+relevance `3`, so
+nDCG measures placement but not distinctions among positive gain grades.
+Production tokenization is unchanged; improvements remain gated work for
+`fn-109`.
+
+<!-- /public-truth -->
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmickel/gno",
-  "version": "1.14.0",
+  "version": "1.16.0",
   "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
   "keywords": [
     "embeddings",
@@ -78,8 +78,10 @@
     "bench:general-embeddings:write": "bun scripts/general-embedding-benchmark.ts --write",
     "bench:cpu-embeddings": "bun scripts/cpu-embed-autoresearch.ts",
     "bench:cpu-embeddings:native-batch-probe": "bun scripts/native-embedding-batch-probe.ts",
+    "bench:cjk-lexical": "bun scripts/cjk-lexical-benchmark.ts",
     "eval:retrieval-candidates": "bun scripts/retrieval-candidate-benchmark.ts",
     "eval:retrieval-candidates:write": "bun scripts/retrieval-candidate-benchmark.ts --write",
+    "eval:agentic": "bun evals/agentic/cli.ts",
     "eval:watch": "bun --bun evalite watch",
     "research:finetune:bootstrap": "bun research/finetune/scripts/bootstrap-promotion-fixtures.ts",
     "research:finetune:autonomous:noop": "bun research/finetune/autonomous/scripts/noop-run.ts",

package/src/bench/cjk-fingerprint.ts

@@ -0,0 +1,42 @@
+const canonicalize = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([key, child]) => [key, canonicalize(child)])
+    );
+  }
+  return value;
+};
+
+export const stableJson = (value: unknown): string =>
+  JSON.stringify(canonicalize(value));
+
+export const fingerprint = (value: unknown): string => {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(stableJson(value));
+  return hasher.digest("hex");
+};
+
+const withoutVolatileFields = (value: unknown, parentKey?: string): unknown => {
+  if (Array.isArray(value)) {
+    return value.map((child) => withoutVolatileFields(child, parentKey));
+  }
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .filter(
+          ([key]) =>
+            key !== "generatedAt" &&
+            !key.endsWith("Ms") &&
+            !(parentKey === "fingerprints" && key === "result")
+        )
+        .map(([key, child]) => [key, withoutVolatileFields(child, key)])
+    );
+  }
+  return value;
+};
+
+export const fingerprintStableResult = (value: unknown): string =>
+  fingerprint(withoutVolatileFields(value));

package/src/bench/cjk-metrics.ts

@@ -0,0 +1,110 @@
+import type {
+  CjkBenchCaseResult,
+  CjkBenchFailure,
+  CjkBenchLanguage,
+  CjkBenchLanguageResult,
+  CjkBenchMetrics,
+} from "./types";
+
+import { computeMrr, computeNdcg, computeRecall } from "./metrics";
+
+const round = (value: number, places = 4): number =>
+  Number(value.toFixed(places));
+
+const average = (values: number[]): number =>
+  values.length === 0
+    ? 0
+    : values.reduce((sum, value) => sum + value, 0) / values.length;
+
+export const summarizeLatency = (
+  values: number[]
+): { p50Ms: number; p95Ms: number; meanMs: number } => {
+  if (values.length === 0) {
+    return { p50Ms: 0, p95Ms: 0, meanMs: 0 };
+  }
+  const sorted = [...values].sort((left, right) => left - right);
+  const percentile = (percent: number): number => {
+    const index = Math.ceil((percent / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, Math.min(sorted.length - 1, index))] ?? 0;
+  };
+  return {
+    p50Ms: round(percentile(50), 2),
+    p95Ms: round(percentile(95), 2),
+    meanMs: round(average(values), 2),
+  };
+};
+
+export const buildCjkCaseResult = (input: {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCaseResult["category"];
+  query: string;
+  expected: string[];
+  judgments: Array<{ docid: string; relevance: number }>;
+  topDocs: string[];
+  warmLatencyMs: number;
+  error?: string;
+}): CjkBenchCaseResult => ({
+  queryId: input.queryId,
+  language: input.language,
+  category: input.category,
+  query: input.query,
+  expected: input.expected,
+  topDocs: input.topDocs.slice(0, 10),
+  metrics: {
+    recallAt5: round(computeRecall(input.topDocs, input.expected, 5)),
+    recallAt10: round(computeRecall(input.topDocs, input.expected, 10)),
+    mrr: round(computeMrr(input.topDocs, input.expected)),
+    ndcgAt10: round(computeNdcg(input.topDocs, input.judgments, 10)),
+  },
+  zeroResult: input.topDocs.length === 0,
+  warmLatencyMs: round(input.warmLatencyMs, 2),
+  ...(input.error ? { error: input.error } : {}),
+});
+
+export const summarizeCjkMetrics = (
+  cases: CjkBenchCaseResult[]
+): CjkBenchMetrics => ({
+  recallAt5: round(average(cases.map((item) => item.metrics.recallAt5))),
+  recallAt10: round(average(cases.map((item) => item.metrics.recallAt10))),
+  mrr: round(average(cases.map((item) => item.metrics.mrr))),
+  ndcgAt10: round(average(cases.map((item) => item.metrics.ndcgAt10))),
+  zeroResultRate: round(
+    average(cases.map((item) => (item.zeroResult ? 1 : 0)))
+  ),
+});
+
+export const classifyCjkFailures = (
+  cases: CjkBenchCaseResult[]
+): CjkBenchFailure[] =>
+  cases.flatMap((item) => {
+    if (item.metrics.recallAt5 === 1) {
+      return [];
+    }
+    const reason = item.zeroResult
+      ? "zero-result"
+      : item.metrics.recallAt10 === 0
+        ? "not-in-top-10"
+        : "below-rank-5";
+    return [
+      {
+        queryId: item.queryId,
+        language: item.language,
+        category: item.category,
+        reason,
+        query: item.query,
+        expected: item.expected,
+        topDocs: item.topDocs.slice(0, 5),
+      },
+    ];
+  });
+
+export const summarizeCjkLanguage = (
+  language: CjkBenchLanguage,
+  cases: CjkBenchCaseResult[]
+): CjkBenchLanguageResult => ({
+  language,
+  queryCount: cases.length,
+  metrics: summarizeCjkMetrics(cases),
+  failures: classifyCjkFailures(cases),
+});

package/src/bench/types.ts

@@ -94,3 +94,122 @@ export interface BenchOutput {
 export type BenchResult =
   | { success: true; data: BenchOutput }
   | { success: false; error: string; isValidation?: boolean };
+
+export const CJK_BENCH_LANGUAGES = ["zh", "ja", "ko"] as const;
+export type CjkBenchLanguage = (typeof CJK_BENCH_LANGUAGES)[number];
+
+export type CjkBenchCategory =
+  | "exact-term"
+  | "filename"
+  | "identifier"
+  | "mixed-script"
+  | "normalization"
+  | "punctuation"
+  | "ranking"
+  | "token-boundary";
+
+export type CjkBenchLane =
+  | "bm25"
+  | "hybrid-no-models"
+  | "substring-raw"
+  | "substring-nfc";
+
+export interface CjkBenchMetrics {
+  recallAt5: number;
+  recallAt10: number;
+  mrr: number;
+  ndcgAt10: number;
+  zeroResultRate: number;
+}
+
+export interface CjkBenchLatency {
+  coldQueryMs: number;
+  warmQuery: {
+    p50Ms: number;
+    p95Ms: number;
+    meanMs: number;
+  };
+}
+
+export interface CjkBenchFailure {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCategory;
+  reason: "below-rank-5" | "not-in-top-10" | "zero-result";
+  query: string;
+  expected: string[];
+  topDocs: string[];
+}
+
+export interface CjkBenchCaseResult {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCategory;
+  query: string;
+  expected: string[];
+  normalization?: {
+    form: "NFC" | "NFKC";
+    source: string;
+    target: string;
+  };
+  topDocs: string[];
+  metrics: Omit<CjkBenchMetrics, "zeroResultRate">;
+  zeroResult: boolean;
+  warmLatencyMs: number;
+  error?: string;
+}
+
+export interface CjkBenchLanguageResult {
+  language: CjkBenchLanguage;
+  queryCount: number;
+  metrics: CjkBenchMetrics;
+  failures: CjkBenchFailure[];
+}
+
+export interface CjkBenchLaneResult {
+  id: CjkBenchLane;
+  description: string;
+  config: Record<string, string | number | boolean | null>;
+  queryCount: number;
+  metrics: CjkBenchMetrics;
+  latency: CjkBenchLatency;
+  languages: CjkBenchLanguageResult[];
+  cases: CjkBenchCaseResult[];
+}
+
+export interface CjkBenchOutput {
+  schemaVersion: 1;
+  generatedAt: string;
+  benchmark: "gno-cjk-lexical-degradation";
+  corpus: {
+    fixtureVersion: number;
+    documentCount: number;
+    queryCount: number;
+    languages: CjkBenchLanguage[];
+    provenance: string;
+    fingerprint: string;
+  };
+  runtime: {
+    bun: string;
+    platform: string;
+    arch: string;
+    sqlite: string;
+  };
+  index: {
+    tokenizer: string;
+    buildMs: number;
+    bytes: number;
+    pageCount: number;
+    pageSize: number;
+    vocabularyTerms: number;
+    vocabularyDocuments: number;
+    tokenOccurrences: number;
+  };
+  fingerprints: {
+    config: string;
+    runtime: string;
+    tokenizer: string;
+    result: string;
+  };
+  lanes: CjkBenchLaneResult[];
+}