@gmickel/gno 0.3.5 → 0.5.0

package/src/pipeline/hybrid.ts

@@ -11,9 +11,11 @@ import type { StorePort } from '../store/types';
 import { err, ok } from '../store/types';
 import type { VectorIndexPort } from '../store/vector/types';
 import { createChunkLookup } from './chunk-lookup';
+import { formatQueryForEmbedding } from './contextual';
 import { expandQuery } from './expansion';
 import {
   buildExplainResults,
+  type ExpansionStatus,
   explainBm25,
   explainExpansion,
   explainFusion,
@@ -51,56 +53,64 @@ export interface HybridSearchDeps {
 // Score Normalization
 // ─────────────────────────────────────────────────────────────────────────────
 
-function _normalizeVectorScore(distance: number): number {
-  return Math.max(0, Math.min(1, 1 - distance / 2));
+// Removed: _normalizeVectorScore was dead code (vector distances normalized in vector index)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// BM25 Score Normalization
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Normalize raw BM25 score to 0-1 range using sigmoid.
+ * BM25 scores are negative in SQLite FTS5 (more negative = better match).
+ * Typical range: -15 (excellent) to -2 (weak match).
+ * Maps to 0-1 where higher is better.
+ */
+function normalizeBm25Score(rawScore: number): number {
+  const absScore = Math.abs(rawScore);
+  // Sigmoid with center=4.5, scale=2.8
+  // Maps: -15 → ~0.99, -5 → ~0.55, -2 → ~0.29
+  return 1 / (1 + Math.exp(-(absScore - 4.5) / 2.8));
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
 // BM25 Strength Check
 // ─────────────────────────────────────────────────────────────────────────────
 
+// Thresholds for strong signal detection (conservative - prefer expansion over speed)
+const STRONG_TOP_SCORE = 0.84; // ~84th percentile confidence
+const STRONG_GAP = 0.14; // Clear separation from #2
+
 /**
  * Check if BM25 results are strong enough to skip expansion.
- * Uses gap-based metric: how much better is #1 than #2?
- * Returns 0-1 where 1 = #1 is clearly dominant, 0 = results are similar.
- * Raw BM25: smaller (more negative) is better.
+ * Returns true if top result is both confident AND clearly separated.
+ * This prevents skipping on weak-but-separated results.
  */
 async function checkBm25Strength(
   store: StorePort,
   query: string,
   options?: { collection?: string; lang?: string }
-): Promise<number> {
+): Promise<boolean> {
   const result = await store.searchFts(query, {
     limit: 5,
     collection: options?.collection,
     language: options?.lang,
   });
+
   if (!result.ok || result.value.length === 0) {
-    return 0;
+    return false;
   }
 
-  // Only one result = strong signal
-  if (result.value.length === 1) {
-    return 1;
-  }
+  // Normalize scores (higher = better)
+  const scores = result.value
+    .map((r) => normalizeBm25Score(r.score))
+    .sort((a, b) => b - a); // Descending
 
-  // Get top 2 scores (smaller is better)
-  const scores = result.value.map((r) => r.score).sort((a, b) => a - b);
-  const best = scores[0] ?? 0;
-  const second = scores[1] ?? best;
-  const worst = scores.at(-1) ?? best;
-
-  // Compute gap-based strength
-  // If best and second are equal, gap = 0
-  // If second is much worse (larger), gap approaches 1
-  const range = worst - best;
-  if (range === 0) {
-    return 0; // All scores equal, no clear winner
-  }
+  const topScore = scores[0] ?? 0;
+  const secondScore = scores[1] ?? 0;
+  const gap = topScore - secondScore;
 
-  // Gap = how much worse is #2 relative to the range (clamped for safety)
-  const gap = (second - best) / range;
-  return Math.max(0, Math.min(1, gap));
+  // Strong signal requires BOTH: high confidence AND clear separation
+  return topScore >= STRONG_TOP_SCORE && gap >= STRONG_GAP;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -155,7 +165,8 @@ async function searchVectorChunks(
     return [];
   }
 
-  const embedResult = await embedPort.embed(query);
+  // Embed query with contextual formatting
+  const embedResult = await embedPort.embed(formatQueryForEmbedding(query));
   if (!embedResult.ok) {
     return [];
   }
@@ -225,17 +236,18 @@ export async function searchHybrid(
   // 1. Check if expansion needed
   // ─────────────────────────────────────────────────────────────────────────
   const shouldExpand = !options.noExpand && genPort !== null;
-  let skipExpansionDueToStrength = false;
+  let expansionStatus: ExpansionStatus = 'disabled';
 
   if (shouldExpand) {
-    const bm25Strength = await checkBm25Strength(store, query, {
+    const hasStrongSignal = await checkBm25Strength(store, query, {
       collection: options.collection,
       lang: options.lang,
     });
-    skipExpansionDueToStrength =
-      bm25Strength >= pipelineConfig.strongBm25Threshold;
 
-    if (!skipExpansionDueToStrength) {
+    if (hasStrongSignal) {
+      expansionStatus = 'skipped_strong';
+    } else {
+      expansionStatus = 'attempted';
       const expandResult = await expandQuery(genPort, query, {
         // Use queryLanguage for prompt selection, NOT options.lang (retrieval filter)
         lang: queryLanguage,
@@ -247,9 +259,7 @@ export async function searchHybrid(
     }
   }
 
-  explainLines.push(
-    explainExpansion(shouldExpand && !skipExpansionDueToStrength, expansion)
-  );
+  explainLines.push(explainExpansion(expansionStatus, expansion));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 2. Parallel retrieval using raw store/vector APIs for correct seq tracking
@@ -293,7 +303,8 @@ export async function searchHybrid(
 
   // Vector search
   let vecCount = 0;
-  const vectorAvailable = vectorIndex?.searchAvailable && embedPort !== null;
+  const vectorAvailable =
+    (vectorIndex?.searchAvailable && embedPort !== null) ?? false;
 
   if (vectorAvailable && vectorIndex && embedPort) {
     // Original query
@@ -335,7 +346,7 @@ export async function searchHybrid(
     }
   }
 
-  explainLines.push(explainVector(vecCount, vectorAvailable ?? false));
+  explainLines.push(explainVector(vecCount, vectorAvailable));
 
   // ─────────────────────────────────────────────────────────────────────────
   // 3. RRF Fusion
@@ -441,7 +452,13 @@ export async function searchHybrid(
     }
 
     // Get chunk via O(1) lookup
-    const chunk = getChunk(candidate.mirrorHash, candidate.seq);
+    // For doc-level FTS (seq=0), fall back to first available chunk if exact lookup fails
+    let chunk = getChunk(candidate.mirrorHash, candidate.seq);
+    if (!chunk && candidate.seq === 0) {
+      // Doc-level FTS uses seq=0 as placeholder - try first chunk
+      const docChunks = chunksMap.get(candidate.mirrorHash);
+      chunk = docChunks?.[0];
+    }
     if (!chunk) {
       continue;
     }
@@ -524,7 +541,7 @@ export async function searchHybrid(
       mode: vectorAvailable ? 'hybrid' : 'bm25_only',
       expanded: expansion !== null,
       reranked: rerankResult.reranked,
-      vectorsUsed: vectorAvailable ?? false,
+      vectorsUsed: vectorAvailable,
       totalResults: results.length,
       collection: options.collection,
       lang: options.lang,

package/src/pipeline/index.ts

@@ -4,11 +4,18 @@
  * @module src/pipeline
  */
 
+// Contextual embedding
+export {
+  extractTitle,
+  formatDocForEmbedding,
+  formatQueryForEmbedding,
+} from './contextual';
 // Expansion
 export { expandQuery, generateCacheKey } from './expansion';
 // Explain
 export {
   buildExplainResults,
+  type ExpansionStatus,
   explainBm25,
   explainExpansion,
   explainFusion,

package/src/pipeline/rerank.ts

@@ -7,7 +7,6 @@
 
 import type { RerankPort } from '../llm/types';
 import type { StorePort } from '../store/types';
-import { createChunkLookup } from './chunk-lookup';
 import type { BlendingTier, FusionCandidate, RerankedCandidate } from './types';
 import { DEFAULT_BLENDING_SCHEDULE } from './types';
 
@@ -121,32 +120,44 @@ export async function rerankCandidates(
   const toRerank = candidates.slice(0, maxCandidates);
   const remaining = candidates.slice(maxCandidates);
 
-  // Pre-fetch all chunks in one batch query (eliminates N+1)
+  // Dedupe by document - multiple chunks from same doc use single full-doc rerank
   const uniqueHashes = [...new Set(toRerank.map((c) => c.mirrorHash))];
-  const chunksMapResult = await store.getChunksBatch(uniqueHashes);
 
-  // If chunk fetch fails, degrade gracefully (fusion-only)
-  // Don't rerank on empty/missing texts - produces non-deterministic results
-  if (!chunksMapResult.ok) {
-    return {
-      candidates: candidates.map((c) => ({
-        ...c,
-        rerankScore: null,
-        blendedScore: normalizeFusionScore(c.fusionScore),
-      })),
-      reranked: false,
-    };
+  // Fetch full document content for each unique document (parallel)
+  // Max 128K chars per doc to fit in reranker context
+  const MAX_DOC_CHARS = 128_000;
+  const contentResults = await Promise.all(
+    uniqueHashes.map((hash) => store.getContent(hash))
+  );
+  const docContents = new Map<string, string>();
+  for (let i = 0; i < uniqueHashes.length; i++) {
+    const hash = uniqueHashes[i] as string;
+    const result = contentResults[i] as Awaited<
+      ReturnType<typeof store.getContent>
+    >;
+    if (result.ok && result.value) {
+      const content = result.value;
+      docContents.set(
+        hash,
+        content.length > MAX_DOC_CHARS
+          ? `${content.slice(0, MAX_DOC_CHARS)}...`
+          : content
+      );
+    } else {
+      // Fallback to empty string if content not available
+      docContents.set(hash, '');
+    }
   }
-  const chunksMap = chunksMapResult.value;
-  const getChunk = createChunkLookup(chunksMap);
 
-  // Build texts array for reranking (O(1) lookup per candidate)
-  const texts: string[] = toRerank.map((c) => {
-    const chunk = getChunk(c.mirrorHash, c.seq);
-    return chunk?.text ?? '';
-  });
+  // Build texts array for reranking (one per unique document)
+  const hashToIndex = new Map<string, number>();
+  const texts: string[] = [];
+  for (const hash of uniqueHashes) {
+    hashToIndex.set(hash, texts.length);
+    texts.push(docContents.get(hash) ?? '');
+  }
 
-  // Run reranking
+  // Run reranking on full documents
   const rerankResult = await rerankPort.rerank(query, texts);
 
   if (!rerankResult.ok) {
@@ -163,16 +174,33 @@ export async function rerankCandidates(
 
   // Map rerank scores to candidates
   // Note: We use normalizeFusionScore defined above (across ALL candidates)
-  // Build index->score map for O(1) lookup instead of O(n) find per candidate
-  const scoreByIndex = new Map(
+  // Build doc index->score map for O(1) lookup
+  // All chunks from same document share the same rerank score
+  const scoreByDocIndex = new Map(
     rerankResult.value.map((s) => [s.index, s.score])
   );
+
+  // Normalize rerank scores using min-max (models return varying scales)
+  const rerankScores = rerankResult.value.map((s) => s.score);
+  const minRerank = Math.min(...rerankScores);
+  const maxRerank = Math.max(...rerankScores);
+  const rerankRange = maxRerank - minRerank;
+
+  function normalizeRerankScore(score: number): number {
+    if (rerankRange < 1e-9) {
+      return 1; // All tied for best
+    }
+    return (score - minRerank) / rerankRange;
+  }
+
   const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
-    const rerankScore = scoreByIndex.get(i) ?? null;
+    // Get document-level rerank score (shared by all chunks from same doc)
+    const docIndex = hashToIndex.get(c.mirrorHash) ?? -1;
+    const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
 
-    // Normalize rerank score to 0-1 range (models may return different scales)
+    // Normalize rerank score to 0-1 range using min-max
     const normalizedRerankScore =
-      rerankScore !== null ? Math.max(0, Math.min(1, rerankScore)) : null;
+      rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
 
     // Calculate blended score using normalized fusion score
     const position = i + 1;

package/src/pipeline/types.ts

@@ -213,8 +213,6 @@ export type RerankedCandidate = FusionCandidate & {
 
 /** Search pipeline configuration */
 export interface PipelineConfig {
-  /** Strong BM25 threshold to skip expansion */
-  strongBm25Threshold: number;
   /** Expansion timeout in ms */
   expansionTimeout: number;
   /** Max candidates to rerank */
@@ -227,7 +225,6 @@ export interface PipelineConfig {
 
 /** Default pipeline configuration */
 export const DEFAULT_PIPELINE_CONFIG: PipelineConfig = {
-  strongBm25Threshold: 0.7,
   expansionTimeout: 5000,
   rerankCandidates: 20,
   rrf: DEFAULT_RRF_CONFIG,

package/src/pipeline/vsearch.ts

@@ -11,6 +11,7 @@ import type { StorePort } from '../store/types';
 import { err, ok } from '../store/types';
 import type { VectorIndexPort } from '../store/vector/types';
 import { createChunkLookup } from './chunk-lookup';
+import { formatQueryForEmbedding } from './contextual';
 import { detectQueryLanguage } from './query-language';
 import type { SearchOptions, SearchResult, SearchResults } from './types';
 
@@ -268,8 +269,8 @@ export async function searchVector(
     );
   }
 
-  // Embed query
-  const embedResult = await embedPort.embed(query);
+  // Embed query with contextual formatting
+  const embedResult = await embedPort.embed(formatQueryForEmbedding(query));
   if (!embedResult.ok) {
     return err(
       'QUERY_FAILED',

package/src/serve/CLAUDE.md

@@ -0,0 +1,91 @@
+# Web UI (gno serve)
+
+Local web server for GNO search and document browsing.
+
+## Architecture
+
+Uses same **"Ports without DI"** pattern as CLI/MCP (see root CLAUDE.md):
+- Adapters instantiated directly in `context.ts`
+- Pipeline code receives port interfaces
+- No dependency injection
+
+```
+src/serve/
+├── server.ts          # Bun.serve() entry point
+├── context.ts         # ServerContext with LLM ports
+├── routes/
+│   └── api.ts         # REST API handlers
+└── public/            # React frontend (Bun HTML imports)
+    ├── App.tsx        # Router
+    ├── pages/         # Page components
+    ├── components/    # UI components (ShadCN + AI Elements)
+    └── hooks/         # Custom hooks (useApi, etc.)
+```
+
+## Key Patterns
+
+### Ports (interfaces)
+- `EmbeddingPort` - vector embeddings
+- `GenerationPort` - LLM text generation
+- `RerankPort` - cross-encoder reranking
+- `VectorIndexPort` - vector search
+
+### ServerContext
+Created at startup, holds all LLM ports and capabilities:
+```typescript
+interface ServerContext {
+  store: SqliteAdapter;
+  config: Config;
+  vectorIndex: VectorIndexPort | null;
+  embedPort: EmbeddingPort | null;
+  genPort: GenerationPort | null;
+  rerankPort: RerankPort | null;
+  capabilities: { bm25, vector, hybrid, answer };
+}
+```
+
+### Shared Pipeline Code
+Answer generation uses shared module to stay in sync with CLI:
+- `src/pipeline/answer.ts` - generateGroundedAnswer, processAnswerResult
+
+## API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/api/health` | GET | Health check |
+| `/api/status` | GET | Index stats, collections |
+| `/api/capabilities` | GET | Available features |
+| `/api/collections` | GET | List collections |
+| `/api/docs` | GET | List documents |
+| `/api/doc` | GET | Get document content |
+| `/api/search` | POST | BM25 search |
+| `/api/query` | POST | Hybrid search |
+| `/api/ask` | POST | AI answer with citations |
+| `/api/presets` | GET | List model presets |
+| `/api/presets` | POST | Switch preset (hot-reload) |
+| `/api/models/status` | GET | Download progress |
+| `/api/models/pull` | POST | Start model download |
+
+## Frontend
+
+- **Framework**: React (via Bun HTML imports)
+- **Styling**: Tailwind CSS + ShadCN components
+- **AI Elements**: Conversation, Message, Sources, CodeBlock, Loader
+- **Routing**: Simple hash-free SPA routing in App.tsx
+
+## Development
+
+```bash
+# Start dev server with HMR
+bun run src/serve/index.ts
+
+# Or via CLI
+gno serve --port 3000
+```
+
+## Security
+
+- Binds to `127.0.0.1` only (no LAN exposure)
+- CSP headers on all responses
+- CORS protection on POST endpoints
+- No external font/script loading

package/src/serve/bunfig.toml

@@ -0,0 +1,2 @@
+[serve.static]
+plugins = ["bun-plugin-tailwind"]

package/src/serve/context.ts

@@ -0,0 +1,181 @@
+/**
+ * Server context for web UI.
+ * Manages LLM ports and vector index for hybrid search and AI answers.
+ *
+ * @module src/serve/context
+ */
+
+import type { Config } from '../config/types';
+import { LlmAdapter } from '../llm/nodeLlamaCpp/adapter';
+import { getActivePreset } from '../llm/registry';
+import type {
+  DownloadProgress,
+  EmbeddingPort,
+  GenerationPort,
+  ModelType,
+  RerankPort,
+} from '../llm/types';
+import type { SqliteAdapter } from '../store/sqlite/adapter';
+import { createVectorIndexPort, type VectorIndexPort } from '../store/vector';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Download State (in-memory, single user)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface DownloadState {
+  active: boolean;
+  currentType: ModelType | null;
+  progress: DownloadProgress | null;
+  completed: ModelType[];
+  failed: Array<{ type: ModelType; error: string }>;
+  startedAt: number | null;
+}
+
+/** Global download state for polling */
+export const downloadState: DownloadState = {
+  active: false,
+  currentType: null,
+  progress: null,
+  completed: [],
+  failed: [],
+  startedAt: null,
+};
+
+/** Reset download state */
+export function resetDownloadState(): void {
+  downloadState.active = false;
+  downloadState.currentType = null;
+  downloadState.progress = null;
+  downloadState.completed = [];
+  downloadState.failed = [];
+  downloadState.startedAt = null;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Server Context
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface ServerContext {
+  store: SqliteAdapter;
+  config: Config;
+  vectorIndex: VectorIndexPort | null;
+  embedPort: EmbeddingPort | null;
+  genPort: GenerationPort | null;
+  rerankPort: RerankPort | null;
+  capabilities: {
+    bm25: boolean;
+    vector: boolean;
+    hybrid: boolean;
+    answer: boolean;
+  };
+}
+
+/**
+ * Initialize server context with LLM ports.
+ * Attempts to load models; missing models are logged but don't fail.
+ */
+export async function createServerContext(
+  store: SqliteAdapter,
+  config: Config
+): Promise<ServerContext> {
+  let embedPort: EmbeddingPort | null = null;
+  let genPort: GenerationPort | null = null;
+  let rerankPort: RerankPort | null = null;
+  let vectorIndex: VectorIndexPort | null = null;
+
+  try {
+    const preset = getActivePreset(config);
+    const llm = new LlmAdapter(config);
+
+    // Try to create embedding port
+    const embedResult = await llm.createEmbeddingPort(preset.embed);
+    if (embedResult.ok) {
+      embedPort = embedResult.value;
+      const initResult = await embedPort.init();
+      if (initResult.ok) {
+        // Create vector index
+        const dimensions = embedPort.dimensions();
+        const db = store.getRawDb();
+        const vectorResult = await createVectorIndexPort(db, {
+          model: preset.embed,
+          dimensions,
+        });
+        if (vectorResult.ok) {
+          vectorIndex = vectorResult.value;
+          console.log('Vector search enabled');
+        }
+      }
+    }
+
+    // Try to create generation port
+    const genResult = await llm.createGenerationPort(preset.gen);
+    if (genResult.ok) {
+      genPort = genResult.value;
+      console.log('AI answer generation enabled');
+    }
+
+    // Try to create rerank port
+    const rerankResult = await llm.createRerankPort(preset.rerank);
+    if (rerankResult.ok) {
+      rerankPort = rerankResult.value;
+      console.log('Reranking enabled');
+    }
+  } catch (e) {
+    // Log but don't fail - models are optional
+    console.log(
+      'LLM initialization skipped:',
+      e instanceof Error ? e.message : String(e)
+    );
+  }
+
+  const capabilities = {
+    bm25: true, // Always available
+    vector: vectorIndex?.searchAvailable ?? false,
+    hybrid: (vectorIndex?.searchAvailable ?? false) && embedPort !== null,
+    answer: genPort !== null,
+  };
+
+  return {
+    store,
+    config,
+    vectorIndex,
+    embedPort,
+    genPort,
+    rerankPort,
+    capabilities,
+  };
+}
+
+/**
+ * Dispose server context resources.
+ * Each port is disposed independently to prevent one failure from blocking others.
+ */
+export async function disposeServerContext(ctx: ServerContext): Promise<void> {
+  const ports = [
+    { name: 'embed', port: ctx.embedPort },
+    { name: 'gen', port: ctx.genPort },
+    { name: 'rerank', port: ctx.rerankPort },
+  ];
+
+  for (const { name, port } of ports) {
+    if (port) {
+      try {
+        await port.dispose();
+      } catch (e) {
+        console.error(`Failed to dispose ${name} port:`, e);
+      }
+    }
+  }
+}
+
+/**
+ * Reload server context with potentially new config.
+ * Disposes existing ports and recreates them.
+ */
+export async function reloadServerContext(
+  ctx: ServerContext,
+  newConfig?: Config
+): Promise<ServerContext> {
+  await disposeServerContext(ctx);
+  return createServerContext(ctx.store, newConfig ?? ctx.config);
+}

package/src/serve/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Serve module exports.
+ *
+ * @module src/serve
+ */
+
+export { type ServeOptions, type ServeResult, startServer } from './server';