@gmickel/gno 0.16.0 → 0.18.0

package/README.md

@@ -32,7 +32,22 @@ GNO is a local knowledge engine that turns your documents into a searchable, con
 
 ---
 
-## What's New in v0.15
+## What's New in v0.18
+
+- **Intent Steering**: optional `intent` control for ambiguous queries across CLI, API, Web, and MCP query flows
+- **Rerank Controls**: `candidateLimit` lets you tune rerank cost vs. recall on slower or memory-constrained machines
+- **Stability**: query expansion now uses a bounded configurable context size (`models.expandContextSize`, default `2048`)
+- **Rerank Efficiency**: identical chunk texts are deduplicated before scoring and expanded back out deterministically
+
+### v0.17
+
+- **Structured Query Modes**: `term`, `intent`, and `hyde` controls across CLI, API, MCP, and Web
+- **Temporal Retrieval Upgrades**: `since`/`until`, date-range parsing, and recency sorting with frontmatter-date fallback
+- **Web Retrieval UX Polish**: richer advanced controls in Search and Ask (collection/date/category/author/tags + query modes)
+- **Metadata-Aware Retrieval**: ingestion now materializes document metadata/date fields for better filtering and ranking
+- **Migration Reliability**: SQLite-compatible migration path for existing indexes (including older SQLite engines)
+
+### v0.15
 
 - **HTTP Backends**: Offload embedding, reranking, and generation to remote GPU servers
 - Simple URI config: `http://host:port/path#modelname`
@@ -139,16 +154,41 @@ gno skill install --target all       # Both Claude + Codex
 
 **BM25** indexes full documents (not chunks) with Snowball stemming, so "running" matches "run".
 **Vector** embeds chunks with document titles for context awareness.
+All retrieval modes also support metadata filters: `--since`, `--until`, `--category`, `--author`, `--tags-all`, `--tags-any`.
 
 ```bash
 gno search "handleAuth"              # Find exact matches
 gno vsearch "error handling patterns" # Semantic similarity
 gno query "database optimization"    # Full pipeline
+gno query "meeting decisions" --since "last month" --category "meeting,notes" --author "gordon"
+gno query "performance" --intent "web performance and latency"
 gno ask "what did we decide" --answer # AI synthesis
 ```
 
 Output formats: `--json`, `--files`, `--csv`, `--md`, `--xml`
 
+### Retrieval V2 Controls
+
+Existing query calls still work. Retrieval v2 adds optional structured intent control and deeper explain output.
+
+```bash
+# Existing call (unchanged)
+gno query "auth flow" --thorough
+
+# Structured retrieval intent
+gno query "auth flow" \
+  --intent "web authentication and token lifecycle" \
+  --candidate-limit 12 \
+  --query-mode term:"jwt refresh token -oauth1" \
+  --query-mode intent:"how refresh token rotation works" \
+  --query-mode hyde:"Refresh tokens rotate on each use and previous tokens are revoked." \
+  --explain
+```
+
+- Modes: `term` (BM25-focused), `intent` (semantic-focused), `hyde` (single hypothetical passage)
+- Explain includes stage timings, fallback/cache counters, and per-result score components
+- `gno ask --json` includes `meta.answerContext` for adaptive source selection traces
+
 ---
 
 ## Agent Integration
@@ -372,7 +412,7 @@ graph TD
 
 ## Local Models
 
-Models auto-download on first use to `~/.cache/gno/models/`. Alternatively, offload to a GPU server on your network using HTTP backends.
+Models auto-download on first use to `~/.cache/gno/models/`. For deterministic startup, set `GNO_NO_AUTO_DOWNLOAD=1` and use `gno models pull` explicitly. Alternatively, offload to a GPU server on your network using HTTP backends.
 
 | Model               | Purpose                               | Size         |
 | :------------------ | :------------------------------------ | :----------- |
@@ -444,6 +484,19 @@ bun run lint && bun run typecheck
 
 > **Contributing**: [CONTRIBUTING.md](.github/CONTRIBUTING.md)
 
+### Evals and Benchmark Deltas
+
+Use retrieval benchmark commands to track quality and latency over time:
+
+```bash
+bun run eval:hybrid
+bun run eval:hybrid:baseline
+bun run eval:hybrid:delta
+```
+
+- Benchmark guide: [evals/README.md](./evals/README.md)
+- Latest baseline snapshot: [evals/fixtures/hybrid-baseline/latest.json](./evals/fixtures/hybrid-baseline/latest.json)
+
 ---
 
 ## License

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmickel/gno",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
   "keywords": [
     "embeddings",
@@ -51,6 +51,9 @@
     "test:fixtures": "bun scripts/generate-test-fixtures.ts",
     "evals": "bun scripts/update-eval-scores.ts",
     "eval": "bun --bun evalite",
+    "eval:hybrid": "bun --bun evalite evals/hybrid.eval.ts",
+    "eval:hybrid:baseline": "bun scripts/hybrid-benchmark.ts --write",
+    "eval:hybrid:delta": "bun scripts/hybrid-benchmark.ts --delta",
     "eval:watch": "bun --bun evalite watch",
     "reset": "bun run src/index.ts reset --confirm",
     "docs:verify": "bun run scripts/docs-verify.ts",

package/src/cli/commands/ask.ts

@@ -192,8 +192,16 @@ export async function ask(
       limit,
       collection: options.collection,
       lang: options.lang,
+      intent: options.intent,
+      since: options.since,
+      until: options.until,
+      categories: options.categories,
+      author: options.author,
+      tagsAll: options.tagsAll,
+      tagsAny: options.tagsAny,
       noExpand: options.noExpand,
       noRerank: options.noRerank,
+      candidateLimit: options.candidateLimit,
     });
 
     if (!searchResult.ok) {
@@ -205,6 +213,7 @@ export async function ask(
     // Generate grounded answer if requested
     let answer: string | undefined;
     let citations: Citation[] | undefined;
+    let answerContext: AskResult["meta"]["answerContext"] | undefined;
     let answerGenerated = false;
 
     // Only generate answer if:
@@ -236,6 +245,7 @@ export async function ask(
       const processed = processAnswerResult(rawResult);
       answer = processed.answer;
       citations = processed.citations;
+      answerContext = processed.answerContext;
       answerGenerated = true;
     }
 
@@ -250,8 +260,11 @@ export async function ask(
         expanded: searchResult.value.meta.expanded ?? false,
         reranked: searchResult.value.meta.reranked ?? false,
         vectorsUsed: searchResult.value.meta.vectorsUsed ?? false,
+        intent: searchResult.value.meta.intent,
+        candidateLimit: searchResult.value.meta.candidateLimit,
         answerGenerated,
         totalResults: results.length,
+        answerContext,
       },
     };
 

package/src/cli/commands/models/use.ts

@@ -58,6 +58,7 @@ export async function modelsUse(
       presets: config.models?.presets ?? [],
       loadTimeout: config.models?.loadTimeout ?? 60_000,
       inferenceTimeout: config.models?.inferenceTimeout ?? 30_000,
+      expandContextSize: config.models?.expandContextSize ?? 2_048,
       warmModelTtl: config.models?.warmModelTtl ?? 300_000,
     },
   };

package/src/cli/commands/query.ts

@@ -121,8 +121,9 @@ export async function query(
       embedPort = embedResult.value;
     }
 
-    // Create generation port (for expansion) - optional
-    if (!options.noExpand) {
+    // Create generation port (for expansion) - optional.
+    // Skip when structured query modes are provided.
+    if (!options.noExpand && !options.queryModes?.length) {
       const genUri = options.genModel ?? preset.gen;
       const genResult = await llm.createGenerationPort(genUri, {
         policy,

package/src/cli/pager.ts

@@ -167,7 +167,7 @@ export class Pager {
 
       // Write content to pager stdin
       if (proc.stdin) {
-        proc.stdin.write(content + "\n");
+        void proc.stdin.write(content + "\n");
         await proc.stdin.end();
       }
 

package/src/cli/program.ts

@@ -130,6 +130,17 @@ async function writeOutput(
   }
 }
 
+function parseCsvValues(raw: unknown): string[] | undefined {
+  if (typeof raw !== "string") {
+    return undefined;
+  }
+  const values = raw
+    .split(",")
+    .map((v) => v.trim().toLowerCase())
+    .filter((v) => v.length > 0);
+  return values.length > 0 ? values : undefined;
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Program Factory
 // ─────────────────────────────────────────────────────────────────────────────
@@ -203,6 +214,17 @@ function wireSearchCommands(program: Command): void {
     .option("--min-score <num>", "minimum score threshold")
     .option("-c, --collection <name>", "filter by collection")
     .option("--lang <code>", "language filter/hint (BCP-47)")
+    .option(
+      "--since <date>",
+      "modified-at lower bound (ISO date/time or token)"
+    )
+    .option(
+      "--until <date>",
+      "modified-at upper bound (ISO date/time or token)"
+    )
+    .option("--category <values>", "require category match (comma-separated)")
+    .option("--author <text>", "filter by author (case-insensitive contains)")
+    .option("--intent <text>", "disambiguating context for ambiguous queries")
     .option("--tags-all <tags>", "require ALL tags (comma-separated)")
     .option("--tags-any <tags>", "require ANY tag (comma-separated)")
     .option("--full", "include full content")
@@ -247,6 +269,7 @@ function wireSearchCommands(program: Command): void {
       const limit = cmdOpts.limit
         ? parsePositiveInt("limit", cmdOpts.limit)
         : getDefaultLimit(format);
+      const categories = parseCsvValues(cmdOpts.category);
 
       const { search, formatSearch } = await import("./commands/search");
       const result = await search(queryText, {
@@ -254,6 +277,11 @@ function wireSearchCommands(program: Command): void {
         minScore,
         collection: cmdOpts.collection as string | undefined,
         lang: cmdOpts.lang as string | undefined,
+        since: cmdOpts.since as string | undefined,
+        until: cmdOpts.until as string | undefined,
+        categories,
+        author: cmdOpts.author as string | undefined,
+        intent: cmdOpts.intent as string | undefined,
         tagsAll,
         tagsAny,
         full: Boolean(cmdOpts.full),
@@ -293,6 +321,17 @@ function wireSearchCommands(program: Command): void {
     .option("--min-score <num>", "minimum score threshold")
     .option("-c, --collection <name>", "filter by collection")
     .option("--lang <code>", "language filter/hint (BCP-47)")
+    .option(
+      "--since <date>",
+      "modified-at lower bound (ISO date/time or token)"
+    )
+    .option(
+      "--until <date>",
+      "modified-at upper bound (ISO date/time or token)"
+    )
+    .option("--category <values>", "require category match (comma-separated)")
+    .option("--author <text>", "filter by author (case-insensitive contains)")
+    .option("--intent <text>", "disambiguating context for ambiguous queries")
     .option("--tags-all <tags>", "require ALL tags (comma-separated)")
     .option("--tags-any <tags>", "require ANY tag (comma-separated)")
     .option("--full", "include full content")
@@ -337,6 +376,7 @@ function wireSearchCommands(program: Command): void {
       const limit = cmdOpts.limit
         ? parsePositiveInt("limit", cmdOpts.limit)
         : getDefaultLimit(format);
+      const categories = parseCsvValues(cmdOpts.category);
 
       const { vsearch, formatVsearch } = await import("./commands/vsearch");
       const result = await vsearch(queryText, {
@@ -344,6 +384,11 @@ function wireSearchCommands(program: Command): void {
         minScore,
         collection: cmdOpts.collection as string | undefined,
         lang: cmdOpts.lang as string | undefined,
+        since: cmdOpts.since as string | undefined,
+        until: cmdOpts.until as string | undefined,
+        categories,
+        author: cmdOpts.author as string | undefined,
+        intent: cmdOpts.intent as string | undefined,
         tagsAll,
         tagsAny,
         full: Boolean(cmdOpts.full),
@@ -378,6 +423,17 @@ function wireSearchCommands(program: Command): void {
     .option("--min-score <num>", "minimum score threshold")
     .option("-c, --collection <name>", "filter by collection")
     .option("--lang <code>", "language hint (BCP-47)")
+    .option(
+      "--since <date>",
+      "modified-at lower bound (ISO date/time or token)"
+    )
+    .option(
+      "--until <date>",
+      "modified-at upper bound (ISO date/time or token)"
+    )
+    .option("--category <values>", "require category match (comma-separated)")
+    .option("--author <text>", "filter by author (case-insensitive contains)")
+    .option("--intent <text>", "disambiguating context for ambiguous queries")
     .option("--tags-all <tags>", "require ALL tags (comma-separated)")
     .option("--tags-any <tags>", "require ANY tag (comma-separated)")
     .option("--full", "include full content")
@@ -386,6 +442,13 @@ function wireSearchCommands(program: Command): void {
     .option("--thorough", "enable query expansion (slower, ~5-8s)")
     .option("--no-expand", "disable query expansion")
     .option("--no-rerank", "disable reranking")
+    .option(
+      "--query-mode <mode:text>",
+      "structured mode entry (repeatable): term:<text>, intent:<text>, or hyde:<text>",
+      (value: string, previous: string[] = []) => [...previous, value],
+      []
+    )
+    .option("-C, --candidate-limit <num>", "max candidates passed to reranking")
     .option("--explain", "include scoring explanation")
     .option("--json", "JSON output")
     .option("--md", "Markdown output")
@@ -407,6 +470,17 @@ function wireSearchCommands(program: Command): void {
         throw new CliError("VALIDATION", "--min-score must be between 0 and 1");
       }
 
+      // Parse optional structured query modes
+      let queryModes: import("../pipeline/types").QueryModeInput[] | undefined;
+      if (Array.isArray(cmdOpts.queryMode) && cmdOpts.queryMode.length > 0) {
+        const { parseQueryModeSpecs } = await import("../pipeline/query-modes");
+        const parsed = parseQueryModeSpecs(cmdOpts.queryMode as string[]);
+        if (!parsed.ok) {
+          throw new CliError("VALIDATION", parsed.error.message);
+        }
+        queryModes = parsed.value;
+      }
+
       // Parse and validate tag filters
       let tagsAll: string[] | undefined;
       let tagsAny: string[] | undefined;
@@ -427,6 +501,10 @@ function wireSearchCommands(program: Command): void {
       const limit = cmdOpts.limit
         ? parsePositiveInt("limit", cmdOpts.limit)
         : getDefaultLimit(format);
+      const candidateLimit = cmdOpts.candidateLimit
+        ? parsePositiveInt("candidate-limit", cmdOpts.candidateLimit)
+        : undefined;
+      const categories = parseCsvValues(cmdOpts.category);
 
       // Determine expansion/rerank settings based on flags
       // Priority: --fast > --thorough > --no-expand/--no-rerank > default
@@ -458,12 +536,19 @@ function wireSearchCommands(program: Command): void {
         minScore,
         collection: cmdOpts.collection as string | undefined,
         lang: cmdOpts.lang as string | undefined,
+        since: cmdOpts.since as string | undefined,
+        until: cmdOpts.until as string | undefined,
+        categories,
+        author: cmdOpts.author as string | undefined,
+        intent: cmdOpts.intent as string | undefined,
         tagsAll,
         tagsAny,
         full: Boolean(cmdOpts.full),
         lineNumbers: Boolean(cmdOpts.lineNumbers),
         noExpand,
         noRerank,
+        candidateLimit,
+        queryModes,
         explain: Boolean(cmdOpts.explain),
         json: format === "json",
         md: format === "md",
@@ -490,8 +575,20 @@ function wireSearchCommands(program: Command): void {
     .option("-n, --limit <num>", "max source results")
     .option("-c, --collection <name>", "filter by collection")
     .option("--lang <code>", "language hint (BCP-47)")
+    .option(
+      "--since <date>",
+      "modified-at lower bound (ISO date/time or token)"
+    )
+    .option(
+      "--until <date>",
+      "modified-at upper bound (ISO date/time or token)"
+    )
+    .option("--category <values>", "require category match (comma-separated)")
+    .option("--author <text>", "filter by author (case-insensitive contains)")
+    .option("--intent <text>", "disambiguating context for ambiguous queries")
     .option("--fast", "skip expansion and reranking (fastest)")
     .option("--thorough", "enable query expansion (slower)")
+    .option("-C, --candidate-limit <num>", "max candidates passed to reranking")
     .option("--answer", "generate short grounded answer")
     .option("--no-answer", "force retrieval-only output")
     .option("--max-answer-tokens <num>", "max answer tokens")
@@ -510,11 +607,15 @@ function wireSearchCommands(program: Command): void {
       const limit = cmdOpts.limit
         ? parsePositiveInt("limit", cmdOpts.limit)
         : getDefaultLimit(format);
+      const candidateLimit = cmdOpts.candidateLimit
+        ? parsePositiveInt("candidate-limit", cmdOpts.candidateLimit)
+        : undefined;
 
       // Parse max-answer-tokens (optional, defaults to 512 in command impl)
       const maxAnswerTokens = cmdOpts.maxAnswerTokens
         ? parsePositiveInt("max-answer-tokens", cmdOpts.maxAnswerTokens)
         : undefined;
+      const categories = parseCsvValues(cmdOpts.category);
 
       // Determine expansion/rerank settings based on flags
       // Default: skip expansion (balanced mode)
@@ -535,8 +636,14 @@ function wireSearchCommands(program: Command): void {
         limit,
         collection: cmdOpts.collection as string | undefined,
         lang: cmdOpts.lang as string | undefined,
+        since: cmdOpts.since as string | undefined,
+        until: cmdOpts.until as string | undefined,
+        categories,
+        author: cmdOpts.author as string | undefined,
+        intent: cmdOpts.intent as string | undefined,
         noExpand,
         noRerank,
+        candidateLimit,
         // Per spec: --answer defaults to false, --no-answer forces retrieval-only
         // Commander creates separate cmdOpts.noAnswer for --no-answer flag
         answer: Boolean(cmdOpts.answer),

package/src/config/types.ts

@@ -209,6 +209,8 @@ export const ModelConfigSchema = z.object({
   loadTimeout: z.number().default(60_000),
   /** Inference timeout in ms */
   inferenceTimeout: z.number().default(30_000),
+  /** Context size used for query expansion generation */
+  expandContextSize: z.number().int().min(256).default(2_048),
   /** Keep warm model TTL in ms (5 min) */
   warmModelTtl: z.number().default(300_000),
 });

package/src/core/links.ts

@@ -80,6 +80,10 @@ const UNSAFE_PERCENT_CODES = new Set(["%2F", "%5C", "%00", "%2f", "%5c"]);
  */
 const WIKI_LINK_REGEX = /\[\[([^\]|]+(?:\|[^\]]+)?)\]\]/g;
 
+/** Logseq embed: {{embed [[Page]]}} or {{embed ((block-id))}} */
+const LOGSEQ_EMBED_REGEX =
+  /\{\{\s*embed\s+(\[\[[^\]]+\]\]|\(\([^)]+\)\))\s*\}\}/gi;
+
 /**
  * Markdown inline link: [text](url)
  * Captures: 1=text, 2=url (path and optional anchor)
@@ -273,6 +277,43 @@ export function parseLinks(
 ): ParsedLink[] {
   const links: ParsedLink[] = [];
 
+  const pushWikiLink = (
+    raw: string,
+    target: string,
+    startOffset: number,
+    endOffset: number,
+    displayText?: string
+  ): void => {
+    const trimmedTarget = target.trim();
+    const hasScheme =
+      /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmedTarget) ||
+      trimmedTarget.startsWith("mailto:");
+    if (hasScheme || trimmedTarget.startsWith("//")) {
+      return;
+    }
+
+    const parts = parseTargetParts(trimmedTarget);
+    if (!parts.ref) {
+      return;
+    }
+
+    const startPos = offsetToPosition(startOffset, lineOffsets);
+    const endPos = offsetToPosition(endOffset, lineOffsets);
+
+    links.push({
+      kind: "wiki",
+      raw,
+      targetRef: parts.ref,
+      targetAnchor: parts.anchor,
+      targetCollection: parts.collection,
+      displayText,
+      startLine: startPos.line,
+      startCol: startPos.col,
+      endLine: endPos.line,
+      endCol: endPos.col,
+    });
+  };
+
   // Parse wiki links
   WIKI_LINK_REGEX.lastIndex = 0;
   let match: RegExpExecArray | null;
@@ -281,6 +322,14 @@ export function parseLinks(
     const startOffset = match.index;
     const endOffset = startOffset + match[0].length;
 
+    // Skip [[target]] nested in Logseq alias syntax: [Display]([[target]])
+    if (
+      markdown.slice(Math.max(0, startOffset - 2), startOffset) === "](" &&
+      markdown.slice(endOffset, endOffset + 1) === ")"
+    ) {
+      continue;
+    }
+
     // Skip if inside excluded range
     if (rangeIntersectsExcluded(startOffset, endOffset, excludedRanges)) {
       continue;
@@ -307,31 +356,35 @@ export function parseLinks(
     }
 
     const trimmedTarget = targetPart.trim();
-    const hasScheme =
-      /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmedTarget) ||
-      trimmedTarget.startsWith("mailto:");
-    if (hasScheme || trimmedTarget.startsWith("//")) {
+    if (!trimmedTarget) {
       continue;
     }
+    pushWikiLink(match[0], trimmedTarget, startOffset, endOffset, displayText);
+  }
 
-    const parts = parseTargetParts(trimmedTarget);
-    if (!parts.ref) continue;
+  // Parse Logseq embeds as links
+  LOGSEQ_EMBED_REGEX.lastIndex = 0;
 
-    const startPos = offsetToPosition(startOffset, lineOffsets);
-    const endPos = offsetToPosition(endOffset, lineOffsets);
+  while ((match = LOGSEQ_EMBED_REGEX.exec(markdown)) !== null) {
+    const startOffset = match.index;
+    const endOffset = startOffset + match[0].length;
 
-    links.push({
-      kind: "wiki",
-      raw: match[0],
-      targetRef: parts.ref,
-      targetAnchor: parts.anchor,
-      targetCollection: parts.collection,
-      displayText,
-      startLine: startPos.line,
-      startCol: startPos.col,
-      endLine: endPos.line,
-      endCol: endPos.col,
-    });
+    if (rangeIntersectsExcluded(startOffset, endOffset, excludedRanges)) {
+      continue;
+    }
+
+    const embedTarget = match[1]?.trim();
+    if (!embedTarget) {
+      continue;
+    }
+
+    if (embedTarget.startsWith("((") && embedTarget.endsWith("))")) {
+      const blockId = embedTarget.slice(2, -2).trim();
+      if (blockId.length === 0) {
+        continue;
+      }
+      pushWikiLink(match[0], blockId, startOffset, endOffset);
+    }
   }
 
   // Parse markdown links
@@ -350,6 +403,25 @@ export function parseLinks(
     const url = match[2];
     if (!url) continue;
 
+    // Logseq alias syntax: [Display]([[Target]])
+    if (url.startsWith("[[") && url.endsWith("]]")) {
+      const innerTarget = url.slice(2, -2).trim();
+      if (innerTarget.length > 0) {
+        const displayText =
+          linkText && linkText !== innerTarget
+            ? truncateText(linkText, MAX_DISPLAY_TEXT_GRAPHEMES)
+            : undefined;
+        pushWikiLink(
+          match[0],
+          innerTarget,
+          startOffset,
+          endOffset,
+          displayText
+        );
+      }
+      continue;
+    }
+
     // Skip external URLs
     if (EXTERNAL_URL_REGEX.test(url)) {
       continue;