@gmickel/gno 1.13.0 → 1.15.0

package/README.md

@@ -92,7 +92,12 @@ gno daemon --detach  # headless continuous indexing (background; --status / --st
 
 ## What's New
 
-> Latest release: see [CHANGELOG.md](./CHANGELOG.md)
+<!-- public-truth:current-version -->
+
+> Current release: **v1.15.0** — see [CHANGELOG.md](./CHANGELOG.md)
+
+<!-- /public-truth -->
+
 > Full release history: [CHANGELOG.md](./CHANGELOG.md)
 
 - **Retrieval-proven activation**: `gno status`, `gno doctor`, REST, and the
@@ -109,7 +114,9 @@ gno daemon --detach  # headless continuous indexing (background; --status / --st
 - **Publish to [gno.sh](https://gno.sh/publish)**: new `gno publish export` CLI and Web UI action produce a self-contained artifact you upload to the hosted reader — public, secret, invite-only, or locally encrypted before upload
 - **Retrieval Quality Upgrade**: stronger BM25 lexical handling, code-aware chunking, terminal result hyperlinks, and per-collection model overrides
 - **Code Embedding Benchmarks**: new benchmark workflow across canonical, real-GNO, and pinned OSS slices for comparing alternate embedding models
-- **Default Embed Model**: built-in presets now use `Qwen3-Embedding-0.6B-GGUF` after it beat `bge-m3` on both code and multilingual prose benchmark lanes
+<!-- public-truth:default-embed-model -->
+- **Default Embed Model**: all four built-in presets use `Qwen3-Embedding-0.6B-GGUF`; see the dated, fixture-scoped evidence below
+<!-- /public-truth -->
 - **Regression Fixes**: tightened phrase/negation/hyphen/underscore BM25 behavior, cleaned non-TTY hyperlink output, improved `gno doctor` chunking and embedding fingerprint visibility, and fixed the embedding autoresearch harness
 
 ### Upgrading Existing Collections
@@ -201,7 +208,11 @@ Manage the detached process with `gno daemon --status` and `gno daemon --stop`.
 
 ### Install GNO
 
-Requires [Bun](https://bun.sh/) >= 1.3.0.
+<!-- public-truth:runtime -->
+
+Requires [Bun](https://bun.sh/) >=1.3.0.
+
+<!-- /public-truth -->
 
 ```bash
 bun install -g @gmickel/gno
@@ -225,10 +236,15 @@ when its structured activation state is degraded. `gno doctor` exits 2 when any
 configured folder fails the lexical proof; semantic models may still be pending
 without blocking BM25 search.
 
-**Windows**: current validated target is `windows-x64`, with a packaged
+<!-- public-truth:supported-platforms -->
+
+GNO supports macOS, Linux, and Windows. The current validated Windows target is
+`windows-x64`, with a packaged
 desktop beta zip now published on GitHub Releases. See
 [docs/WINDOWS.md](./docs/WINDOWS.md) for support scope and validation notes.
 
+<!-- /public-truth -->
+
 Keep an index fresh continuously without opening the Web UI:
 
 ```bash
@@ -532,7 +548,9 @@ Connect GNO to Claude Desktop, Cursor, Raycast, and more:
 
 ![GNO MCP](./assets/screenshots/mcp.jpg)
 
-GNO exposes tools via [Model Context Protocol](https://modelcontextprotocol.io):
+GNO exposes 17 tools by default via [Model Context Protocol](https://modelcontextprotocol.io),
+including the core retrieval tools below. Starting MCP with `--enable-write`
+adds 11 opt-in mutation tools, for 28 total.
 
 | Tool            | Description                           |
 | :-------------- | :------------------------------------ |
@@ -547,7 +565,9 @@ GNO exposes tools via [Model Context Protocol](https://modelcontextprotocol.io):
 | `gno_graph`     | Get knowledge graph (nodes and edges) |
 | `gno_status`    | Index health check                    |
 
-**Design**: MCP tools are retrieval-only. Your AI assistant (Claude, GPT-4) synthesizes answers from retrieved context. Best retrieval (GNO) + best reasoning (your LLM).
+**Design**: Default MCP mode is read-only: retrieval, graph, status, and job
+inspection. Your AI assistant synthesizes answers from retrieved context. Write
+tools are available only through the explicit `--enable-write` opt-in.
 
 [MCP setup guide →](https://gno.sh/docs/MCP/)
 
@@ -644,7 +664,10 @@ Interactive visualization of document connections. Wiki links, markdown links, a
 
 Ask questions in natural language. GNO searches your documents and synthesizes answers with inline citations linking to sources.
 
-Everything runs locally. No cloud, no accounts, no data leaving your machine.
+The Web UI and local-model path run on your machine with no account or
+telemetry. Network access occurs when GNO downloads models, when you configure
+an HTTP model backend, or when you explicitly upload an exported artifact to
+gno.sh.
 
 > **Detailed docs**: [Web UI Guide](https://gno.sh/docs/WEB-UI/)
 
@@ -802,13 +825,13 @@ graph TD
 | **REST API**         | HTTP API for custom tools and integrations                                     |
 | **Multi-Format**     | Markdown, PDF, DOCX, XLSX, PPTX, plain text                                    |
 | **Local LLM**        | AI answers via llama.cpp, no API keys                                          |
-| **Remote Inference** | Offload to GPU servers via HTTP (llama-server, Ollama, LocalAI)                |
-| **Privacy First**    | 100% offline, zero telemetry, your data stays yours                            |
-| **MCP Server**       | Works with Claude Desktop, Cursor, Zed, + 8 more                               |
+| **Remote Inference** | Optional HTTP endpoints for embedding, reranking, expansion, and generation    |
+| **Privacy First**    | Local by default; no telemetry; network use is explicit or model provisioning  |
+| **MCP Server**       | 10 automatic client targets; 17 read-only tools, 28 with writes enabled        |
 | **Collections**      | Organize sources with patterns, excludes, contexts                             |
 | **Tag Filtering**    | Frontmatter tags with hierarchical paths, filter via `--tags-any`/`--tags-all` |
 | **Note Linking**     | Wiki links, backlinks, related notes, cross-collection navigation              |
-| **Multilingual**     | 30+ languages, auto-detection, cross-lingual search                            |
+| **Multilingual**     | Query classification, 7-language document detection, multilingual embeddings   |
 | **Incremental**      | SHA-256 tracking, only changed files re-indexed                                |
 | **Keyboard First**   | ⌘N capture, ⌘K search, ⌘/ shortcuts, ⌘S save                                   |
 
@@ -818,20 +841,20 @@ graph TD
 
 Models auto-download on first use to `~/.cache/gno/models/`. GNO validates cached GGUF files before loading and removes intercepted HTML/non-GGUF cache entries with a clear recovery error. For deterministic startup, set `GNO_NO_AUTO_DOWNLOAD=1` and use `gno models pull` explicitly. Alternatively, offload to a GPU server on your network using HTTP backends.
 
-| Model                  | Purpose                               | Size         |
-| :--------------------- | :------------------------------------ | :----------- |
-| Qwen3-Embedding-0.6B   | Embeddings (multilingual)             | ~640MB       |
-| Qwen3-Reranker-0.6B    | Cross-encoder reranking (32K context) | ~700MB       |
-| Qwen3 / Qwen2.5 family | Query expansion + AI answers          | ~600MB-2.5GB |
+| Model                  | Purpose                                          |
+| :--------------------- | :----------------------------------------------- |
+| Qwen3-Embedding-0.6B   | Embeddings                                       |
+| Qwen3-Reranker-0.6B    | Best-chunk-per-document cross-encoder reranking  |
+| Qwen3 / Qwen2.5 family | Query expansion and standalone answer generation |
 
 ### Model Presets
 
-| Preset       | Disk   | Best For                                                |
-| :----------- | :----- | :------------------------------------------------------ |
-| `slim-tuned` | ~1GB   | Current default, tuned retrieval in a compact footprint |
-| `slim`       | ~1GB   | Fast, good quality                                      |
-| `balanced`   | ~2GB   | Slightly larger model                                   |
-| `quality`    | ~2.5GB | Best answers                                            |
+| Preset       | Best For                                     |
+| :----------- | :------------------------------------------- |
+| `slim-tuned` | Current default; tuned query expansion       |
+| `slim`       | Untuned slim query expansion                 |
+| `balanced`   | Qwen2.5 3B expansion and answers             |
+| `quality`    | Qwen3 4B expansion and standalone AI answers |
 
 ```bash
 gno models use slim-tuned
@@ -849,7 +872,9 @@ GNO now has a published promoted retrieval model for the default slim path:
 
 Use it when you want the tuned retrieval expansion path immediately, without running local fine-tuning yourself.
 
-For private/internal products, use the same workflow but keep the final GGUF private and point `gen:` at a `file:` URI instead of publishing to Hugging Face.
+For private/internal products, use the same workflow but keep the final GGUF
+private and point `expand:` at a `file:` URI instead of publishing it to
+Hugging Face. The `gen:` role remains the standalone answer model.
 
 See:
 
@@ -873,7 +898,10 @@ models:
       gen: "http://192.168.1.100:8083/v1/chat/completions#qwen3-4b"
 ```
 
-Works with llama-server, Ollama, LocalAI, vLLM, or any OpenAI-compatible server.
+The HTTP adapter expects the OpenAI-compatible endpoint shapes documented in
+[Configuration](./docs/CONFIGURATION.md). Remote servers receive the query,
+chunk, or answer context sent to their configured model role; they are outside
+GNO's local trust boundary.
 
 > **Configuration**: [Model Setup](https://gno.sh/docs/CONFIGURATION/)
 
@@ -989,16 +1017,49 @@ bun run bench:general-embeddings --candidate bge-m3-incumbent --write
 bun run bench:general-embeddings --candidate qwen3-embedding-0.6b --write
 ```
 
-Current signal on the public multilingual FastAPI-docs fixture:
-
-- `bge-m3`: vector nDCG@10 `0.3508`, hybrid nDCG@10 `0.6756`
-- `Qwen3-Embedding-0.6B-GGUF`: vector nDCG@10 `0.9891`, hybrid nDCG@10 `0.9891`
-
-Interpretation:
-
-- Qwen is now the strongest general multilingual embedding model we have tested
-- built-in presets now use Qwen by default
-- existing users may need to run `gno embed` again after upgrading so current collections catch up
+<!-- public-truth:general-embedding-benchmark -->
+
+The immutable April 2026 FastAPI-docs run used 15 documents in five corpus
+languages (`en`, `de`, `fr`, `es`, `zh`) and 13 queries:
+
+- [bge-m3 incumbent](./evals/fixtures/general-embedding-benchmark/2026-04-06-bge-m3-incumbent.md): vector nDCG@10 `0.3503`, hybrid nDCG@10 `0.642`
+- [Qwen3 Embedding 0.6B](./evals/fixtures/general-embedding-benchmark/2026-04-06-qwen3-embedding-0-6b.md): vector nDCG@10 `0.8594`, hybrid nDCG@10 `0.947`
+<!-- /public-truth -->
+
+A separate [July 2026 Nemotron screen](./research/embeddings/2026-07-21-nemotron-3-embed-1b.md)
+reran the same 13-query multilingual lane after runtime/profile changes. It
+measured Qwen at `0.9891` vector / `0.9891` hybrid nDCG@10 and Nemotron 3 Embed
+1B at `0.9023` / `0.9461`. Nemotron used a temporary PyTorch HTTP adapter;
+Qwen used GNO's production GGUF path. Their timings are not comparable, and no
+official production GGUF was validated for Nemotron.
+
+These small fixture results support keeping Qwen as the built-in default; they
+do not establish general language superiority. Query-language classification
+supports a broader set than the indexed-document detector (`en`, `de`, `fr`,
+`it`, `zh`, `ja`, `ko`), and the committed semantic fixture covers only five
+languages.
+
+<!-- public-truth:cjk-lexical-benchmark -->
+
+Lexical fallback has separate evidence. The immutable
+[July 22, 2026 CJK result](./evals/fixtures/cjk-lexical-benchmark/2026-07-22.md)
+uses 21 synthetic documents and 25 same-language queries across Chinese,
+Japanese, and Korean. Production BM25 lexical results and frozen floors:
+
+- Chinese: baseline Recall@10 `0.2222`, nDCG@10 `0.1481`, zero-result `0.7778`; promotion Recall@10 `0.4722`, nDCG@10 `0.3981`, maximum zero-result `0.5278`
+- Japanese: baseline Recall@10 `0.125`, nDCG@10 `0.125`, zero-result `0.875`; promotion Recall@10 `0.375`, nDCG@10 `0.375`, maximum zero-result `0.625`
+- Korean: baseline Recall@10 `0.5`, nDCG@10 `0.5`, zero-result `0.5`; promotion Recall@10 `0.75`, nDCG@10 `0.75`, maximum zero-result `0.25`
+
+The
+[promotion-gates.md](./evals/fixtures/cjk-lexical-benchmark/promotion-gates.md)
+also bind MRR, non-regression, and cost requirements. This lexical result does
+not reduce or replace the semantic evidence above. All positive qrels use
+relevance `3`, so
+nDCG measures placement but not distinctions among positive gain grades.
+Production tokenization is unchanged; improvements remain gated work for
+`fn-109`.
+
+<!-- /public-truth -->
 
 ---
 

package/assets/skill/SKILL.md

@@ -6,7 +6,9 @@ allowed-tools: Bash(gno:*) Read
 
 # GNO - Local Knowledge Engine
 
-Fast local semantic search. Index once, search instantly. No cloud, no API keys.
+Fast local semantic search. Index once, search instantly. Local inference needs
+no API key; configured HTTP backends and explicit gno.sh publishing are separate
+network boundaries.
 
 ## When to Use This Skill
 

package/assets/skill/cli-reference.md

@@ -453,7 +453,8 @@ gno models list [--json|--md]
 gno models use <preset>
 ```
 
-Presets: `slim` (~1GB), `balanced` (~2GB), `quality` (~2.5GB)
+Built-ins: `slim-tuned` (default), `slim`, `balanced`, `quality`. Actual
+download/cache use depends on the selected artifacts and existing shared cache.
 
 ### gno models pull
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmickel/gno",
-  "version": "1.13.0",
+  "version": "1.15.0",
   "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
   "keywords": [
     "embeddings",
@@ -78,6 +78,7 @@
     "bench:general-embeddings:write": "bun scripts/general-embedding-benchmark.ts --write",
     "bench:cpu-embeddings": "bun scripts/cpu-embed-autoresearch.ts",
     "bench:cpu-embeddings:native-batch-probe": "bun scripts/native-embedding-batch-probe.ts",
+    "bench:cjk-lexical": "bun scripts/cjk-lexical-benchmark.ts",
     "eval:retrieval-candidates": "bun scripts/retrieval-candidate-benchmark.ts",
     "eval:retrieval-candidates:write": "bun scripts/retrieval-candidate-benchmark.ts --write",
     "eval:watch": "bun --bun evalite watch",
@@ -118,6 +119,7 @@
     "research:finetune:mlx:fuse": "bun research/finetune/scripts/fuse-mlx.ts",
     "research:finetune:mlx:smoke": "bun research/finetune/scripts/smoke-mlx.ts",
     "reset": "bun run src/index.ts reset --confirm",
+    "docs:truth": "bun run scripts/public-truth.ts",
     "docs:verify": "bun run scripts/docs-verify.ts",
     "website:install": "cd website && bundle install",
     "website:dev": "cd website && make serve",

package/src/bench/cjk-fingerprint.ts

@@ -0,0 +1,42 @@
+const canonicalize = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([key, child]) => [key, canonicalize(child)])
+    );
+  }
+  return value;
+};
+
+export const stableJson = (value: unknown): string =>
+  JSON.stringify(canonicalize(value));
+
+export const fingerprint = (value: unknown): string => {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(stableJson(value));
+  return hasher.digest("hex");
+};
+
+const withoutVolatileFields = (value: unknown, parentKey?: string): unknown => {
+  if (Array.isArray(value)) {
+    return value.map((child) => withoutVolatileFields(child, parentKey));
+  }
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .filter(
+          ([key]) =>
+            key !== "generatedAt" &&
+            !key.endsWith("Ms") &&
+            !(parentKey === "fingerprints" && key === "result")
+        )
+        .map(([key, child]) => [key, withoutVolatileFields(child, key)])
+    );
+  }
+  return value;
+};
+
+export const fingerprintStableResult = (value: unknown): string =>
+  fingerprint(withoutVolatileFields(value));

package/src/bench/cjk-metrics.ts

@@ -0,0 +1,110 @@
+import type {
+  CjkBenchCaseResult,
+  CjkBenchFailure,
+  CjkBenchLanguage,
+  CjkBenchLanguageResult,
+  CjkBenchMetrics,
+} from "./types";
+
+import { computeMrr, computeNdcg, computeRecall } from "./metrics";
+
+const round = (value: number, places = 4): number =>
+  Number(value.toFixed(places));
+
+const average = (values: number[]): number =>
+  values.length === 0
+    ? 0
+    : values.reduce((sum, value) => sum + value, 0) / values.length;
+
+export const summarizeLatency = (
+  values: number[]
+): { p50Ms: number; p95Ms: number; meanMs: number } => {
+  if (values.length === 0) {
+    return { p50Ms: 0, p95Ms: 0, meanMs: 0 };
+  }
+  const sorted = [...values].sort((left, right) => left - right);
+  const percentile = (percent: number): number => {
+    const index = Math.ceil((percent / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, Math.min(sorted.length - 1, index))] ?? 0;
+  };
+  return {
+    p50Ms: round(percentile(50), 2),
+    p95Ms: round(percentile(95), 2),
+    meanMs: round(average(values), 2),
+  };
+};
+
+export const buildCjkCaseResult = (input: {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCaseResult["category"];
+  query: string;
+  expected: string[];
+  judgments: Array<{ docid: string; relevance: number }>;
+  topDocs: string[];
+  warmLatencyMs: number;
+  error?: string;
+}): CjkBenchCaseResult => ({
+  queryId: input.queryId,
+  language: input.language,
+  category: input.category,
+  query: input.query,
+  expected: input.expected,
+  topDocs: input.topDocs.slice(0, 10),
+  metrics: {
+    recallAt5: round(computeRecall(input.topDocs, input.expected, 5)),
+    recallAt10: round(computeRecall(input.topDocs, input.expected, 10)),
+    mrr: round(computeMrr(input.topDocs, input.expected)),
+    ndcgAt10: round(computeNdcg(input.topDocs, input.judgments, 10)),
+  },
+  zeroResult: input.topDocs.length === 0,
+  warmLatencyMs: round(input.warmLatencyMs, 2),
+  ...(input.error ? { error: input.error } : {}),
+});
+
+export const summarizeCjkMetrics = (
+  cases: CjkBenchCaseResult[]
+): CjkBenchMetrics => ({
+  recallAt5: round(average(cases.map((item) => item.metrics.recallAt5))),
+  recallAt10: round(average(cases.map((item) => item.metrics.recallAt10))),
+  mrr: round(average(cases.map((item) => item.metrics.mrr))),
+  ndcgAt10: round(average(cases.map((item) => item.metrics.ndcgAt10))),
+  zeroResultRate: round(
+    average(cases.map((item) => (item.zeroResult ? 1 : 0)))
+  ),
+});
+
+export const classifyCjkFailures = (
+  cases: CjkBenchCaseResult[]
+): CjkBenchFailure[] =>
+  cases.flatMap((item) => {
+    if (item.metrics.recallAt5 === 1) {
+      return [];
+    }
+    const reason = item.zeroResult
+      ? "zero-result"
+      : item.metrics.recallAt10 === 0
+        ? "not-in-top-10"
+        : "below-rank-5";
+    return [
+      {
+        queryId: item.queryId,
+        language: item.language,
+        category: item.category,
+        reason,
+        query: item.query,
+        expected: item.expected,
+        topDocs: item.topDocs.slice(0, 5),
+      },
+    ];
+  });
+
+export const summarizeCjkLanguage = (
+  language: CjkBenchLanguage,
+  cases: CjkBenchCaseResult[]
+): CjkBenchLanguageResult => ({
+  language,
+  queryCount: cases.length,
+  metrics: summarizeCjkMetrics(cases),
+  failures: classifyCjkFailures(cases),
+});

package/src/bench/types.ts

@@ -94,3 +94,122 @@ export interface BenchOutput {
 export type BenchResult =
   | { success: true; data: BenchOutput }
   | { success: false; error: string; isValidation?: boolean };
+
+export const CJK_BENCH_LANGUAGES = ["zh", "ja", "ko"] as const;
+export type CjkBenchLanguage = (typeof CJK_BENCH_LANGUAGES)[number];
+
+export type CjkBenchCategory =
+  | "exact-term"
+  | "filename"
+  | "identifier"
+  | "mixed-script"
+  | "normalization"
+  | "punctuation"
+  | "ranking"
+  | "token-boundary";
+
+export type CjkBenchLane =
+  | "bm25"
+  | "hybrid-no-models"
+  | "substring-raw"
+  | "substring-nfc";
+
+export interface CjkBenchMetrics {
+  recallAt5: number;
+  recallAt10: number;
+  mrr: number;
+  ndcgAt10: number;
+  zeroResultRate: number;
+}
+
+export interface CjkBenchLatency {
+  coldQueryMs: number;
+  warmQuery: {
+    p50Ms: number;
+    p95Ms: number;
+    meanMs: number;
+  };
+}
+
+export interface CjkBenchFailure {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCategory;
+  reason: "below-rank-5" | "not-in-top-10" | "zero-result";
+  query: string;
+  expected: string[];
+  topDocs: string[];
+}
+
+export interface CjkBenchCaseResult {
+  queryId: string;
+  language: CjkBenchLanguage;
+  category: CjkBenchCategory;
+  query: string;
+  expected: string[];
+  normalization?: {
+    form: "NFC" | "NFKC";
+    source: string;
+    target: string;
+  };
+  topDocs: string[];
+  metrics: Omit<CjkBenchMetrics, "zeroResultRate">;
+  zeroResult: boolean;
+  warmLatencyMs: number;
+  error?: string;
+}
+
+export interface CjkBenchLanguageResult {
+  language: CjkBenchLanguage;
+  queryCount: number;
+  metrics: CjkBenchMetrics;
+  failures: CjkBenchFailure[];
+}
+
+export interface CjkBenchLaneResult {
+  id: CjkBenchLane;
+  description: string;
+  config: Record<string, string | number | boolean | null>;
+  queryCount: number;
+  metrics: CjkBenchMetrics;
+  latency: CjkBenchLatency;
+  languages: CjkBenchLanguageResult[];
+  cases: CjkBenchCaseResult[];
+}
+
+export interface CjkBenchOutput {
+  schemaVersion: 1;
+  generatedAt: string;
+  benchmark: "gno-cjk-lexical-degradation";
+  corpus: {
+    fixtureVersion: number;
+    documentCount: number;
+    queryCount: number;
+    languages: CjkBenchLanguage[];
+    provenance: string;
+    fingerprint: string;
+  };
+  runtime: {
+    bun: string;
+    platform: string;
+    arch: string;
+    sqlite: string;
+  };
+  index: {
+    tokenizer: string;
+    buildMs: number;
+    bytes: number;
+    pageCount: number;
+    pageSize: number;
+    vocabularyTerms: number;
+    vocabularyDocuments: number;
+    tokenOccurrences: number;
+  };
+  fingerprints: {
+    config: string;
+    runtime: string;
+    tokenizer: string;
+    result: string;
+  };
+  lanes: CjkBenchLaneResult[];
+}