@gmickel/gno 0.36.0 → 0.37.0

package/README.md

@@ -1,6 +1,6 @@
 # GNO
 
-**Your Local Second Brain**: Index, search, and synthesize your entire digital life.
+**Local search, retrieval, and synthesis for the files you actually work in.**
 
 [![npm](./assets/badges/npm.svg)](https://www.npmjs.com/package/@gmickel/gno)
 [![MIT License](./assets/badges/license.svg)](./LICENSE)
@@ -12,7 +12,57 @@
 
 ![GNO](./assets/og-image.png)
 
-GNO is a local knowledge engine that turns your documents into a searchable, connected knowledge graph. Index notes, code, PDFs, and Office docs. Get hybrid search, AI answers with citations, and wiki-style note linking—all 100% offline.
+GNO is a local knowledge engine for notes, code, PDFs, Office docs, meeting transcripts, and reference material. It gives you fast keyword search, semantic retrieval, grounded answers with citations, wiki-style linking, and a real workspace UI, while keeping the whole stack local by default.
+
+Use it when:
+
+- your notes live in more than one folder
+- your important knowledge is split across Markdown, code, PDFs, and Office files
+- you want one retrieval layer that works from the CLI, browser, MCP, and a Bun/TypeScript SDK
+- you want better local context for agents without shipping your docs to a cloud API
+
+### What GNO Gives You
+
+- **Fast local search**: BM25 for exact hits, vectors for concepts, hybrid for best quality
+- **Real retrieval surfaces**: CLI, Web UI, REST API, MCP, SDK
+- **Local-first answers**: grounded synthesis with citations when you want answers, raw retrieval when you do not
+- **Connected knowledge**: backlinks, related notes, graph view, cross-collection navigation
+- **Operational fit**: daemon mode, model presets, remote GPU backends, safe config/state on disk
+
+### One-Minute Tour
+
+```bash
+# Install
+bun install -g @gmickel/gno
+
+# Add a few collections
+gno init ~/notes --name notes
+gno collection add ~/work/docs --name work-docs --pattern "**/*.{md,pdf,docx}"
+gno collection add ~/work/gno/src --name gno-code --pattern "**/*.{ts,tsx,js,jsx}"
+
+# Add context so retrieval results come back with the right framing
+gno context add "notes:" "Personal notes, journal entries, and long-form ideas"
+gno context add "work-docs:" "Architecture docs, runbooks, RFCs, meeting notes"
+gno context add "gno-code:" "Source code for the GNO application"
+
+# Index + embed
+gno update --yes
+gno embed
+
+# Search in the way that fits the question
+gno search "DEC-0054"                            # exact keyword / identifier
+gno vsearch "retry failed jobs with backoff"     # natural-language semantic lookup
+gno query "JWT refresh token rotation" --explain # hybrid retrieval with score traces
+
+# Retrieve documents or export context for an agent
+gno get "gno://work-docs/architecture/auth.md"
+gno multi-get "gno-code/**/*.ts" --max-bytes 30000 --md
+gno query "deployment process" --all --files --min-score 0.35
+
+# Run the workspace
+gno serve
+gno daemon
+```
 
 ---
 
@@ -265,6 +315,14 @@ headless. In v0.30 it is foreground-only and does not expose built-in
 
 Embed GNO directly in another Bun or TypeScript app. No CLI subprocesses. No local server required.
 
+Install:
+
+```bash
+bun add @gmickel/gno
+```
+
+Minimal client:
+
 ```ts
 import { createDefaultConfig, createGnoClient } from "@gmickel/gno";
 
@@ -295,6 +353,43 @@ console.log(results.results[0]?.uri);
 await client.close();
 ```
 
+More SDK examples:
+
+```ts
+import { createGnoClient } from "@gmickel/gno";
+
+const client = await createGnoClient({
+  configPath: "/Users/me/.config/gno/index.yml",
+});
+
+// Fast exact search
+const bm25 = await client.search("DEC-0054", {
+  collection: "work-docs",
+});
+
+// Semantic code lookup
+const semantic = await client.vsearch("retry failed jobs with backoff", {
+  collection: "gno-code",
+});
+
+// Hybrid retrieval with explicit intent
+const hybrid = await client.query("token refresh", {
+  collection: "work-docs",
+  intent: "JWT refresh token rotation in our auth stack",
+  candidateLimit: 12,
+});
+
+// Fetch content directly
+const doc = await client.get("gno://work-docs/auth/refresh.md");
+const bundle = await client.multiGet(["gno-code/**/*.ts"], { maxBytes: 25000 });
+
+// Indexing / embedding
+await client.update({ collection: "work-docs" });
+await client.embed({ collection: "gno-code" });
+
+await client.close();
+```
+
 Core SDK surface:
 
 - `createGnoClient({ config | configPath, dbPath? })`
@@ -303,12 +398,6 @@ Core SDK surface:
 - `update`, `embed`, `index`
 - `close`
 
-Install in an app:
-
-```bash
-bun add @gmickel/gno
-```
-
 Full guide: [SDK docs](https://gno.sh/docs/SDK/)
 
 ---
@@ -338,6 +427,31 @@ gno ask "what did we decide" --answer # AI synthesis
 
 Output formats: `--json`, `--files`, `--csv`, `--md`, `--xml`
 
+### Common CLI Recipes
+
+```bash
+# Search one collection
+gno search "PostgreSQL connection pool" --collection work-docs
+
+# Export retrieval results for an agent
+gno query "authentication flow" --json -n 10
+gno query "deployment rollback" --all --files --min-score 0.4
+
+# Retrieve a document by URI or docid
+gno get "gno://work-docs/runbooks/deploy.md"
+gno get "#abc123"
+
+# Fetch many documents at once
+gno multi-get "work-docs/**/*.md" --max-bytes 20000 --md
+
+# Inspect how the hybrid rank was assembled
+gno query "refresh token rotation" --explain
+
+# Work with filters
+gno query "meeting notes" --since "last month" --category "meeting,notes"
+gno search "incident review" --tags-all "status/active,team/platform"
+```
+
 ### Retrieval V2 Controls
 
 Existing query calls still work. Retrieval v2 adds optional structured intent control and deeper explain output.
@@ -382,6 +496,20 @@ gno skill install --scope user
 
 Then ask your agent: _"Search my notes for the auth discussion"_
 
+Agent-friendly CLI examples:
+
+```bash
+# Structured retrieval output for an agent
+gno query "authentication" --json -n 10
+
+# File list for downstream retrieval
+gno query "error handling" --all --files --min-score 0.35
+
+# Full document content when the agent already knows the ref
+gno get "gno://work-docs/api-reference.md" --full
+gno multi-get "work-docs/**/*.md" --md --max-bytes 30000
+```
+
 [Skill setup guide →](https://gno.sh/docs/integrations/skills/)
 
 ### MCP Server
@@ -655,7 +783,7 @@ See:
 Offload inference to a GPU server on your network:
 
 ```yaml
-# ~/.config/gno/config.yaml
+# ~/.config/gno/index.yml
 models:
   activePreset: remote-gpu
   presets:
@@ -715,6 +843,61 @@ bun run eval:hybrid:delta
 - Benchmark guide: [evals/README.md](./evals/README.md)
 - Latest baseline snapshot: [evals/fixtures/hybrid-baseline/latest.json](./evals/fixtures/hybrid-baseline/latest.json)
 
+### Code Embedding Benchmark Harness
+
+GNO also has a dedicated harness for comparing alternate embedding models on code retrieval without touching product defaults:
+
+```bash
+# Establish the current incumbent baseline
+bun run bench:code-embeddings --candidate bge-m3-incumbent --write
+
+# Add candidate model URIs to the search space, then inspect them
+bun run research:embeddings:autonomous:list-search-candidates
+
+# Benchmark one candidate explicitly
+bun run research:embeddings:autonomous:run-candidate bge-m3-incumbent
+
+# Or let the bounded search harness walk the remaining candidates later
+bun run research:embeddings:autonomous:search --dry-run
+```
+
+See [research/embeddings/README.md](./research/embeddings/README.md).
+
+If a model turns out to be better specifically for code, the intended user story is:
+
+- keep the default global preset for mixed prose/docs collections
+- use per-collection `models.embed` overrides for code collections
+
+That lets GNO stay sane by default while still giving power users a clean path to code-specialist retrieval.
+
+Current code-focused recommendation:
+
+```yaml
+collections:
+  - name: gno-code
+    path: /Users/you/work/gno/src
+    pattern: "**/*.{ts,tsx,js,jsx,go,rs,py,swift,c}"
+    models:
+      embed: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf"
+```
+
+GNO treats that override like any other model URI:
+
+- auto-downloads on first use by default
+- manual-only if `GNO_NO_AUTO_DOWNLOAD=1`
+- offline-safe if the model is already cached
+
+Why this is the current recommendation:
+
+- matches `bge-m3` on the tiny canonical benchmark
+- significantly beats `bge-m3` on the real GNO `src/serve` code slice
+- also beats `bge-m3` on a pinned public-OSS code slice
+
+Trade-off:
+
+- Qwen is slower to embed than `bge-m3`
+- use it where code retrieval quality matters, not necessarily as the global default for every collection
+
 ---
 
 ## License

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmickel/gno",
-  "version": "0.36.0",
+  "version": "0.37.0",
   "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
   "keywords": [
     "embeddings",
@@ -69,6 +69,8 @@
     "eval:hybrid": "bun --bun evalite evals/hybrid.eval.ts",
     "eval:hybrid:baseline": "bun scripts/hybrid-benchmark.ts --write",
     "eval:hybrid:delta": "bun scripts/hybrid-benchmark.ts --delta",
+    "bench:code-embeddings": "bun scripts/code-embedding-benchmark.ts",
+    "bench:code-embeddings:write": "bun scripts/code-embedding-benchmark.ts --write",
     "eval:retrieval-candidates": "bun scripts/retrieval-candidate-benchmark.ts",
     "eval:retrieval-candidates:write": "bun scripts/retrieval-candidate-benchmark.ts --write",
     "eval:watch": "bun --bun evalite watch",
@@ -83,6 +85,11 @@
     "research:finetune:autonomous:confirm-winner": "bun research/finetune/autonomous/scripts/confirm-winner.ts",
     "research:finetune:autonomous:check-promotion-targets": "bun research/finetune/autonomous/scripts/check-promotion-targets.ts",
     "research:finetune:validate": "bun research/finetune/scripts/validate-sandbox.ts",
+    "research:embeddings:autonomous:list-search-candidates": "bun research/embeddings/autonomous/scripts/list-search-candidates.ts",
+    "research:embeddings:autonomous:run-candidate": "bun research/embeddings/autonomous/scripts/run-candidate.ts",
+    "research:embeddings:autonomous:leaderboard": "bun research/embeddings/autonomous/scripts/leaderboard.ts",
+    "research:embeddings:autonomous:confirm-winner": "bun research/embeddings/autonomous/scripts/confirm-winner.ts",
+    "research:embeddings:autonomous:search": "bun research/embeddings/autonomous/scripts/search.ts",
     "research:finetune:qmd-import:legacy": "bun research/finetune/scripts/import-qmd-training.ts",
     "research:finetune:mlx:build-dataset": "bun research/finetune/scripts/build-mlx-dataset.ts",
     "research:finetune:build-variant-dataset": "bun research/finetune/scripts/build-variant-dataset.ts",

package/src/cli/commands/ask.ts

@@ -14,7 +14,7 @@ import type { AskOptions, AskResult, Citation } from "../../pipeline/types";
 
 import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
 import { resolveDownloadPolicy } from "../../llm/policy";
-import { getActivePreset } from "../../llm/registry";
+import { resolveModelUri } from "../../llm/registry";
 import {
   generateGroundedAnswer,
   processAnswerResult,
@@ -90,7 +90,6 @@ export async function ask(
   let rerankPort: RerankPort | null = null;
 
   try {
-    const preset = getActivePreset(config);
     const llm = new LlmAdapter(config);
 
     // Resolve download policy from env/flags
@@ -106,7 +105,12 @@ export async function ask(
       : undefined;
 
     // Create embedding port
-    const embedUri = options.embedModel ?? preset.embed;
+    const embedUri = resolveModelUri(
+      config,
+      "embed",
+      options.embedModel,
+      options.collection
+    );
     const embedResult = await llm.createEmbeddingPort(embedUri, {
       policy,
       onProgress: downloadProgress
@@ -119,8 +123,12 @@ export async function ask(
 
     // Create expansion port when expansion is enabled.
     if (!options.noExpand && !options.queryModes?.length) {
-      const expandUri =
-        options.expandModel ?? options.genModel ?? preset.expand;
+      const expandUri = resolveModelUri(
+        config,
+        "expand",
+        options.expandModel ?? options.genModel,
+        options.collection
+      );
       const genResult = await llm.createExpansionPort(expandUri, {
         policy,
         onProgress: downloadProgress
@@ -134,7 +142,12 @@ export async function ask(
 
     // Create answer generation port when answers are requested.
     if (options.answer) {
-      const genUri = options.genModel ?? preset.gen;
+      const genUri = resolveModelUri(
+        config,
+        "gen",
+        options.genModel,
+        options.collection
+      );
       const genResult = await llm.createGenerationPort(genUri, {
         policy,
         onProgress: downloadProgress
@@ -148,7 +161,12 @@ export async function ask(
 
     // Create rerank port (unless --fast or --no-rerank)
     if (!options.noRerank) {
-      const rerankUri = options.rerankModel ?? preset.rerank;
+      const rerankUri = resolveModelUri(
+        config,
+        "rerank",
+        options.rerankModel,
+        options.collection
+      );
       const rerankResult = await llm.createRerankPort(rerankUri, {
         policy,
         onProgress: downloadProgress

package/src/cli/commands/doctor.ts

@@ -14,6 +14,7 @@ import type { Config } from "../../config/types";
 
 import { getIndexDbPath, getModelsCachePath } from "../../app/constants";
 import { getConfigPaths, isInitialized, loadConfig } from "../../config";
+import { getCodeChunkingStatus } from "../../ingestion/chunker";
 import { ModelCache } from "../../llm/cache";
 import { getActivePreset } from "../../llm/registry";
 import { loadFts5Snowball } from "../../store/sqlite/fts5-snowball";
@@ -122,6 +123,19 @@ async function checkModels(config: Config): Promise<DoctorCheck[]> {
   return checks;
 }
 
+function checkCodeChunking(): DoctorCheck {
+  const status = getCodeChunkingStatus();
+  return {
+    name: "code-chunking",
+    status: "ok",
+    message: `${status.mode} structural chunking for ${status.supportedExtensions.join(", ")}`,
+    details: [
+      "Unsupported extensions fall back to the default markdown chunker.",
+      "Chunking mode is automatic-only in the first pass.",
+    ],
+  };
+}
+
 async function checkNodeLlamaCpp(): Promise<DoctorCheck> {
   try {
     const { getLlama } = await import("node-llama-cpp");
@@ -319,6 +333,9 @@ export async function doctor(
   const sqliteChecks = await checkSqliteExtensions();
   checks.push(...sqliteChecks);
 
+  // Code chunking capability
+  checks.push(checkCodeChunking());
+
   // Determine overall health
   const hasErrors = checks.some((c) => c.status === "error");
 

package/src/cli/commands/embed.ts

@@ -19,7 +19,7 @@ import {
 } from "../../config";
 import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
 import { resolveDownloadPolicy } from "../../llm/policy";
-import { getActivePreset } from "../../llm/registry";
+import { resolveModelUri } from "../../llm/registry";
 import { formatDocForEmbedding } from "../../pipeline/contextual";
 import { SqliteAdapter } from "../../store/sqlite/adapter";
 import { err, ok } from "../../store/types";
@@ -271,8 +271,7 @@ async function initEmbedContext(
     return { ok: false, error: `Collection not found: ${collection}` };
   }
 
-  const preset = getActivePreset(config);
-  const modelUri = model ?? preset.embed;
+  const modelUri = resolveModelUri(config, "embed", model, collection);
 
   const store = new SqliteAdapter();
   const dbPath = getIndexDbPath();

package/src/cli/commands/query.ts

@@ -14,7 +14,7 @@ import type { HybridSearchOptions, SearchResults } from "../../pipeline/types";
 
 import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
 import { resolveDownloadPolicy } from "../../llm/policy";
-import { getActivePreset } from "../../llm/registry";
+import { resolveModelUri } from "../../llm/registry";
 import { type HybridSearchDeps, searchHybrid } from "../../pipeline/hybrid";
 import {
   createVectorIndexPort,
@@ -58,6 +58,7 @@ export interface QueryFormatOptions {
   format: "terminal" | "json" | "files" | "csv" | "md" | "xml";
   full?: boolean;
   lineNumbers?: boolean;
+  terminalLinks?: import("../format/search-results").FormatOptions["terminalLinks"];
 }
 
 export type QueryResult =
@@ -97,7 +98,6 @@ export async function query(
   let rerankPort: RerankPort | null = null;
 
   try {
-    const preset = getActivePreset(config);
     const llm = new LlmAdapter(config);
 
     // Resolve download policy from env/flags
@@ -113,7 +113,12 @@ export async function query(
       : undefined;
 
     // Create embedding port (for vector search)
-    const embedUri = options.embedModel ?? preset.embed;
+    const embedUri = resolveModelUri(
+      config,
+      "embed",
+      options.embedModel,
+      options.collection
+    );
     const embedResult = await llm.createEmbeddingPort(embedUri, {
       policy,
       onProgress: downloadProgress
@@ -127,8 +132,12 @@ export async function query(
     // Create expansion port - optional.
     // Skip when structured query modes are provided.
     if (!options.noExpand && !options.queryModes?.length) {
-      const expandUri =
-        options.expandModel ?? options.genModel ?? preset.expand;
+      const expandUri = resolveModelUri(
+        config,
+        "expand",
+        options.expandModel ?? options.genModel,
+        options.collection
+      );
       const genResult = await llm.createExpansionPort(expandUri, {
         policy,
         onProgress: downloadProgress
@@ -142,7 +151,12 @@ export async function query(
 
     // Create rerank port - optional
     if (!options.noRerank) {
-      const rerankUri = options.rerankModel ?? preset.rerank;
+      const rerankUri = resolveModelUri(
+        config,
+        "rerank",
+        options.rerankModel,
+        options.collection
+      );
       const rerankResult = await llm.createRerankPort(rerankUri, {
         policy,
         onProgress: downloadProgress
@@ -260,5 +274,6 @@ export function formatQuery(
     format: options.format,
     full: options.full,
     lineNumbers: options.lineNumbers,
+    terminalLinks: options.terminalLinks,
   });
 }

package/src/cli/commands/search.ts

@@ -31,6 +31,8 @@ export type SearchCommandOptions = SearchOptions & {
   xml?: boolean;
   /** Output files only */
   files?: boolean;
+  /** Terminal hyperlink policy */
+  terminalLinks?: FormatOptions["terminalLinks"];
 };
 
 export type SearchResult =
@@ -132,6 +134,7 @@ export function formatSearch(
     format: getFormatType(options),
     full: options.full,
     lineNumbers: options.lineNumbers,
+    terminalLinks: options.terminalLinks,
   };
 
   return formatSearchResults(result.data, formatOpts);

package/src/cli/commands/vsearch.ts

@@ -8,7 +8,7 @@
 import type { SearchOptions, SearchResults } from "../../pipeline/types";
 
 import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
-import { getActivePreset } from "../../llm/registry";
+import { resolveModelUri } from "../../llm/registry";
 import { formatQueryForEmbedding } from "../../pipeline/contextual";
 import {
   searchVectorWithEmbedding,
@@ -40,6 +40,8 @@ export type VsearchCommandOptions = SearchOptions & {
   xml?: boolean;
   /** Output files only */
   files?: boolean;
+  /** Terminal hyperlink policy */
+  terminalLinks?: FormatOptions["terminalLinks"];
 };
 
 export type VsearchResult =
@@ -76,8 +78,12 @@ export async function vsearch(
 
   try {
     // Get model URI from preset
-    const preset = getActivePreset(config);
-    const modelUri = options.model ?? preset.embed;
+    const modelUri = resolveModelUri(
+      config,
+      "embed",
+      options.model,
+      options.collection
+    );
 
     // Create LLM adapter for embeddings
     const llm = new LlmAdapter(config);
@@ -187,6 +193,7 @@ export function formatVsearch(
     format: getFormatType(options),
     full: options.full,
     lineNumbers: options.lineNumbers,
+    terminalLinks: options.terminalLinks,
   };
 
   return formatSearchResults(result.data, formatOpts);

package/src/cli/format/search-results.ts

@@ -5,6 +5,8 @@
  * @module src/cli/format/searchResults
  */
 
+import { pathToFileURL } from "node:url";
+
 import type { SearchResults } from "../../pipeline/types";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -15,6 +17,10 @@ export interface FormatOptions {
   full?: boolean;
   lineNumbers?: boolean;
   format: "terminal" | "json" | "files" | "csv" | "md" | "xml";
+  terminalLinks?: {
+    isTTY: boolean;
+    editorUriTemplate?: string | null;
+  };
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -76,7 +82,9 @@ function formatTerminal(data: SearchResults, options: FormatOptions): string {
 
   const lines: string[] = [];
   for (const r of data.results) {
-    lines.push(`[${r.docid}] ${r.uri} (score: ${r.score.toFixed(2)})`);
+    lines.push(
+      `[${r.docid}] ${formatTerminalUri(r, options)} (score: ${r.score.toFixed(2)})`
+    );
     if (r.title) {
       lines.push(`  ${r.title}`);
     }
@@ -100,6 +108,55 @@ function formatTerminal(data: SearchResults, options: FormatOptions): string {
   return lines.join("\n");
 }
 
+function formatTerminalUri(
+  result: SearchResults["results"][number],
+  options: FormatOptions
+): string {
+  const links = options.terminalLinks;
+  if (!links?.isTTY || !result.source.absPath) {
+    return result.uri;
+  }
+
+  const target = buildTerminalLinkTarget(
+    result.source.absPath,
+    result.snippetRange?.startLine,
+    links.editorUriTemplate ?? undefined
+  );
+
+  if (!target) {
+    return result.uri;
+  }
+
+  return wrapOsc8(target, result.uri);
+}
+
+function buildTerminalLinkTarget(
+  absPath: string,
+  line: number | undefined,
+  template?: string
+): string | null {
+  if (template) {
+    if (line === undefined && template.includes("{line}")) {
+      return null;
+    }
+
+    const withPath = template.replaceAll("{path}", absPath);
+    const withLine = withPath.replaceAll(
+      "{line}",
+      line !== undefined ? String(line) : ""
+    );
+    return withLine.replaceAll("{col}", line !== undefined ? "1" : "");
+  }
+
+  return pathToFileURL(absPath).toString();
+}
+
+function wrapOsc8(target: string, label: string): string {
+  const OSC = "\u001B]8;;";
+  const BEL = "\u0007";
+  return `${OSC}${target}${BEL}${label}${OSC}${BEL}`;
+}
+
 function formatMarkdown(data: SearchResults, options: FormatOptions): string {
   const modeLabel = data.meta.mode === "vector" ? "Vector " : "";
   if (data.results.length === 0) {