@gmickel/gno 0.3.5 → 0.5.0

package/src/serve/server.ts

@@ -0,0 +1,249 @@
+/**
+ * Bun.serve() web server for GNO web UI.
+ * Uses Bun's fullstack dev server with HTML imports.
+ * Opens DB once at startup, closes on shutdown.
+ *
+ * @module src/serve/server
+ */
+
+import { getIndexDbPath } from '../app/constants';
+import { getConfigPaths, isInitialized, loadConfig } from '../config';
+import { SqliteAdapter } from '../store/sqlite/adapter';
+import { createServerContext, disposeServerContext } from './context';
+// HTML import - Bun handles bundling TSX/CSS automatically via routes
+import homepage from './public/index.html';
+import {
+  handleAsk,
+  handleCapabilities,
+  handleCollections,
+  handleDoc,
+  handleDocs,
+  handleHealth,
+  handleModelPull,
+  handleModelStatus,
+  handlePresets,
+  handleQuery,
+  handleSearch,
+  handleSetPreset,
+  handleStatus,
+} from './routes/api';
+
+export interface ServeOptions {
+  /** Port to listen on (default: 3000) */
+  port?: number;
+  /** Config path override */
+  configPath?: string;
+  /** Index name (from --index flag) */
+  index?: string;
+}
+
+export interface ServeResult {
+  success: boolean;
+  error?: string;
+}
+
+// Hostname parsing helpers - preserved for future fetch handler use
+// function parseHostname(host: string): string { ... }
+// function isLoopback(hostname: string): boolean { ... }
+
+/**
+ * Get CSP based on environment.
+ * Dev mode allows WebSocket connections for HMR.
+ */
+function getCspHeader(isDev: boolean): string {
+  // Local fonts only - no Google Fonts for true offline-first
+  const base = [
+    "default-src 'self'",
+    "script-src 'self'",
+    "style-src 'self' 'unsafe-inline'",
+    "font-src 'self'",
+    "img-src 'self' data: blob:",
+    "frame-ancestors 'none'",
+    "base-uri 'none'", // Prevent base tag injection
+    "object-src 'none'", // Prevent plugin execution
+  ];
+
+  // Dev mode: allow WebSocket for HMR
+  if (isDev) {
+    base.push("connect-src 'self' ws:");
+  } else {
+    base.push("connect-src 'self'");
+  }
+
+  return base.join('; ');
+}
+
+/**
+ * Apply security headers to a Response.
+ */
+function withSecurityHeaders(response: Response, isDev: boolean): Response {
+  const headers = new Headers(response.headers);
+  headers.set('Content-Security-Policy', getCspHeader(isDev));
+  headers.set('X-Content-Type-Options', 'nosniff');
+  headers.set('X-Frame-Options', 'DENY');
+  headers.set('Referrer-Policy', 'no-referrer');
+  headers.set('Cross-Origin-Resource-Policy', 'same-origin');
+
+  return new Response(response.body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+}
+
+/**
+ * Start the web server.
+ * Opens DB once, closes on SIGINT/SIGTERM.
+ */
+export async function startServer(
+  options: ServeOptions = {}
+): Promise<ServeResult> {
+  const port = options.port ?? 3000;
+  const isDev = process.env.NODE_ENV !== 'production';
+
+  // Check initialization
+  const initialized = await isInitialized(options.configPath);
+  if (!initialized) {
+    return { success: false, error: 'GNO not initialized. Run: gno init' };
+  }
+
+  // Load config
+  const configResult = await loadConfig(options.configPath);
+  if (!configResult.ok) {
+    return { success: false, error: configResult.error.message };
+  }
+  const config = configResult.value;
+
+  // Open database once for server lifetime
+  const store = new SqliteAdapter();
+  const dbPath = getIndexDbPath(options.index);
+  // Use actual config path (from options or default) for consistency
+  const paths = getConfigPaths();
+  const actualConfigPath = options.configPath ?? paths.configFile;
+  store.setConfigPath(actualConfigPath);
+
+  const openResult = await store.open(dbPath, config.ftsTokenizer);
+  if (!openResult.ok) {
+    return { success: false, error: openResult.error.message };
+  }
+
+  // Create server context with LLM ports for hybrid search and AI answers
+  // Use holder pattern to allow hot-reloading presets
+  const ctxHolder = {
+    current: await createServerContext(store, config),
+    config, // Keep original config for reloading
+  };
+
+  // Shutdown controller for clean lifecycle
+  const shutdownController = new AbortController();
+
+  // Graceful shutdown handler
+  const shutdown = async () => {
+    console.log('\nShutting down...');
+    await disposeServerContext(ctxHolder.current);
+    await store.close();
+    shutdownController.abort();
+  };
+
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
+
+  // Start server with try/catch for port-in-use etc.
+  let server: ReturnType<typeof Bun.serve>;
+  try {
+    server = Bun.serve({
+      port,
+      hostname: '127.0.0.1', // Loopback only - no LAN exposure
+
+      // Enable development mode for HMR and console logging
+      development: isDev,
+
+      // Routes object - Bun handles HTML bundling and /_bun/* assets automatically
+      routes: {
+        // SPA routes - all serve the same React app
+        '/': homepage,
+        '/search': homepage,
+        '/browse': homepage,
+        '/doc': homepage,
+        '/ask': homepage,
+
+        // API routes
+        '/api/health': {
+          GET: () => withSecurityHeaders(handleHealth(), isDev),
+        },
+        '/api/status': {
+          GET: async () =>
+            withSecurityHeaders(await handleStatus(store), isDev),
+        },
+        '/api/collections': {
+          GET: async () =>
+            withSecurityHeaders(await handleCollections(store), isDev),
+        },
+        '/api/docs': {
+          GET: async (req: Request) => {
+            const url = new URL(req.url);
+            return withSecurityHeaders(await handleDocs(store, url), isDev);
+          },
+        },
+        '/api/doc': {
+          GET: async (req: Request) => {
+            const url = new URL(req.url);
+            return withSecurityHeaders(await handleDoc(store, url), isDev);
+          },
+        },
+        '/api/search': {
+          POST: async (req: Request) =>
+            withSecurityHeaders(await handleSearch(store, req), isDev),
+        },
+        '/api/query': {
+          POST: async (req: Request) =>
+            withSecurityHeaders(
+              await handleQuery(ctxHolder.current, req),
+              isDev
+            ),
+        },
+        '/api/ask': {
+          POST: async (req: Request) =>
+            withSecurityHeaders(await handleAsk(ctxHolder.current, req), isDev),
+        },
+        '/api/capabilities': {
+          GET: () =>
+            withSecurityHeaders(handleCapabilities(ctxHolder.current), isDev),
+        },
+        '/api/presets': {
+          GET: () =>
+            withSecurityHeaders(handlePresets(ctxHolder.current), isDev),
+          POST: async (req: Request) =>
+            withSecurityHeaders(await handleSetPreset(ctxHolder, req), isDev),
+        },
+        '/api/models/status': {
+          GET: () => withSecurityHeaders(handleModelStatus(), isDev),
+        },
+        '/api/models/pull': {
+          POST: () => withSecurityHeaders(handleModelPull(ctxHolder), isDev),
+        },
+      },
+
+      // No fetch fallback - let Bun handle /_bun/* assets and return 404 for others
+    });
+  } catch (e) {
+    await store.close();
+    return {
+      success: false,
+      error: e instanceof Error ? e.message : String(e),
+    };
+  }
+
+  console.log(`GNO server running at http://localhost:${server.port}`);
+  console.log('Press Ctrl+C to stop');
+
+  // Block until shutdown signal
+  await new Promise<void>((resolve) => {
+    shutdownController.signal.addEventListener('abort', () => resolve(), {
+      once: true,
+    });
+  });
+
+  server.stop(true);
+  return { success: true };
+}

package/src/store/migrations/002-documents-fts.ts

@@ -0,0 +1,40 @@
+/**
+ * Migration: Document-level FTS with Snowball stemmer.
+ *
+ * Replaces chunk-level content_fts with document-level documents_fts.
+ * Uses snowball tokenizer for multilingual stemming support.
+ *
+ * @module src/store/migrations/002-documents-fts
+ */
+
+import type { Database } from 'bun:sqlite';
+import type { FtsTokenizer } from '../../config/types';
+import type { Migration } from './runner';
+
+export const migration: Migration = {
+  version: 2,
+  name: 'documents_fts',
+
+  up(db: Database, ftsTokenizer: FtsTokenizer): void {
+    // Drop old chunk-level FTS (no backwards compat needed per epic)
+    db.exec('DROP TABLE IF EXISTS content_fts');
+
+    // Create document-level FTS with snowball stemmer
+    // Indexes: filepath (for path searches), title, body (full content)
+    // Note: NOT using content='' because contentless tables don't support DELETE
+    // The storage overhead is acceptable for simpler update semantics
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+        filepath,
+        title,
+        body,
+        tokenize='${ftsTokenizer}'
+      )
+    `);
+  },
+
+  down(db: Database): void {
+    db.exec('DROP TABLE IF EXISTS documents_fts');
+    // Note: Cannot restore content_fts - would need full reindex
+  },
+};

package/src/store/migrations/index.ts

@@ -15,6 +15,7 @@ export {
 
 // Import all migrations
 import { migration as m001 } from './001-initial';
+import { migration as m002 } from './002-documents-fts';
 
 /** All migrations in order */
-export const migrations = [m001];
+export const migrations = [m001, m002];

package/src/store/sqlite/adapter.ts

@@ -31,6 +31,7 @@ import type {
   StoreResult,
 } from '../types';
 import { err, ok } from '../types';
+import { loadFts5Snowball } from './fts5-snowball';
 import type { SqliteDbProvider } from './types';
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -103,6 +104,19 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         this.db.exec('PRAGMA journal_mode = WAL');
       }
 
+      // Load fts5-snowball extension if using snowball tokenizer
+      if (ftsTokenizer.startsWith('snowball')) {
+        const snowballResult = loadFts5Snowball(this.db);
+        if (!snowballResult.loaded) {
+          this.db.close();
+          this.db = null;
+          return err(
+            'EXTENSION_LOAD_FAILED',
+            `Failed to load fts5-snowball: ${snowballResult.error}`
+          );
+        }
+      }
+
       // Run migrations
       const result = runMigrations(this.db, migrations, ftsTokenizer);
       if (!result.ok) {
@@ -487,6 +501,53 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  async listDocumentsPaginated(options: {
+    collection?: string;
+    limit: number;
+    offset: number;
+  }): Promise<StoreResult<{ documents: DocumentRow[]; total: number }>> {
+    try {
+      const db = this.ensureOpen();
+      const { collection, limit, offset } = options;
+
+      // Get total count
+      const countRow = collection
+        ? db
+            .query<{ count: number }, [string]>(
+              'SELECT COUNT(*) as count FROM documents WHERE collection = ?'
+            )
+            .get(collection)
+        : db
+            .query<{ count: number }, []>(
+              'SELECT COUNT(*) as count FROM documents'
+            )
+            .get();
+
+      const total = countRow?.count ?? 0;
+
+      // Get paginated documents
+      const rows = collection
+        ? db
+            .query<DbDocumentRow, [string, number, number]>(
+              'SELECT * FROM documents WHERE collection = ? ORDER BY updated_at DESC LIMIT ? OFFSET ?'
+            )
+            .all(collection, limit, offset)
+        : db
+            .query<DbDocumentRow, [number, number]>(
+              'SELECT * FROM documents ORDER BY updated_at DESC LIMIT ? OFFSET ?'
+            )
+            .all(limit, offset);
+
+      return ok({ documents: rows.map(mapDocumentRow), total });
+    } catch (cause) {
+      return err(
+        'QUERY_FAILED',
+        cause instanceof Error ? cause.message : 'Failed to list documents',
+        cause
+      );
+    }
+  }
+
   async markInactive(
     collection: string,
     relPaths: string[]
@@ -697,16 +758,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       const db = this.ensureOpen();
       const limit = options.limit ?? 20;
 
-      // Join FTS results with chunks and documents
-      // Use bm25() function explicitly - fts.rank doesn't work with JOINs
-      // Note: Multiple docs can share mirror_hash (content-addressed storage)
-      // Deduplication by uri+seq is done in search.ts to avoid FTS function context issues
+      // Document-level FTS search using documents_fts
+      // Uses bm25() for relevance ranking (more negative = better match)
+      // Snippet from body column (index 2) with highlight markers
       const sql = `
         SELECT
-          c.mirror_hash,
-          c.seq,
-          bm25(content_fts) as score,
-          ${options.snippet ? "snippet(content_fts, 0, '<mark>', '</mark>', '...', 32) as snippet," : ''}
+          d.mirror_hash,
+          0 as seq,
+          bm25(documents_fts) as score,
+          ${options.snippet ? "snippet(documents_fts, 2, '<mark>', '</mark>', '...', 32) as snippet," : ''}
           d.docid,
           d.uri,
           d.title,
@@ -717,13 +777,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           d.source_mtime,
           d.source_size,
           d.source_hash
-        FROM content_fts fts
-        JOIN content_chunks c ON c.rowid = fts.rowid
-        JOIN documents d ON d.mirror_hash = c.mirror_hash AND d.active = 1
-        WHERE content_fts MATCH ?
+        FROM documents_fts fts
+        JOIN documents d ON d.id = fts.rowid AND d.active = 1
+        WHERE documents_fts MATCH ?
         ${options.collection ? 'AND d.collection = ?' : ''}
-        ${options.language ? 'AND c.language = ?' : ''}
-        ORDER BY bm25(content_fts)
+        ORDER BY bm25(documents_fts)
         LIMIT ?
       `;
 
@@ -731,9 +789,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       if (options.collection) {
         params.push(options.collection);
       }
-      if (options.language) {
-        params.push(options.language);
-      }
       params.push(limit);
 
       interface FtsRow {
@@ -788,29 +843,157 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
-  async rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>> {
+  /**
+   * Sync a document to documents_fts for full-text search.
+   * Must be called after document and content are both upserted.
+   * The FTS rowid matches documents.id for efficient JOINs.
+   */
+  async syncDocumentFts(
+    collection: string,
+    relPath: string
+  ): Promise<StoreResult<void>> {
     try {
       const db = this.ensureOpen();
 
       const transaction = db.transaction(() => {
-        // Get chunks for this hash
-        const chunks = db
-          .query<{ rowid: number; text: string }, [string]>(
-            'SELECT rowid, text FROM content_chunks WHERE mirror_hash = ?'
+        // Get document with its content
+        interface DocWithContent {
+          id: number;
+          rel_path: string;
+          title: string | null;
+          markdown: string | null;
+        }
+
+        const doc = db
+          .query<DocWithContent, [string, string]>(
+            `SELECT d.id, d.rel_path, d.title, c.markdown
+             FROM documents d
+             LEFT JOIN content c ON c.mirror_hash = d.mirror_hash
+             WHERE d.collection = ? AND d.rel_path = ? AND d.active = 1`
           )
-          .all(mirrorHash);
+          .get(collection, relPath);
 
-        // Delete old FTS entries for these rowids
-        for (const chunk of chunks) {
-          db.run('DELETE FROM content_fts WHERE rowid = ?', [chunk.rowid]);
+        if (!doc) {
+          return; // Document not found or inactive
+        }
+
+        // Delete existing FTS entry for this doc
+        db.run('DELETE FROM documents_fts WHERE rowid = ?', [doc.id]);
+
+        // Insert new FTS entry if we have content
+        if (doc.markdown) {
+          db.run(
+            'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)',
+            [doc.id, doc.rel_path, doc.title ?? '', doc.markdown]
+          );
         }
+      });
 
-        // Insert new FTS entries
+      transaction();
+      return ok(undefined);
+    } catch (cause) {
+      return err(
+        'QUERY_FAILED',
+        cause instanceof Error ? cause.message : 'Failed to sync document FTS',
+        cause
+      );
+    }
+  }
+
+  /**
+   * Rebuild entire documents_fts index from scratch.
+   * Use after migration or for recovery.
+   */
+  async rebuildAllDocumentsFts(): Promise<StoreResult<number>> {
+    try {
+      const db = this.ensureOpen();
+      let count = 0;
+
+      const transaction = db.transaction(() => {
+        // Clear FTS table
+        db.run('DELETE FROM documents_fts');
+
+        // Get all active documents with content
+        interface DocWithContent {
+          id: number;
+          rel_path: string;
+          title: string | null;
+          markdown: string;
+        }
+
+        const docs = db
+          .query<DocWithContent, []>(
+            `SELECT d.id, d.rel_path, d.title, c.markdown
+             FROM documents d
+             JOIN content c ON c.mirror_hash = d.mirror_hash
+             WHERE d.active = 1 AND d.mirror_hash IS NOT NULL`
+          )
+          .all();
+
+        // Insert FTS entries
         const stmt = db.prepare(
-          'INSERT INTO content_fts (rowid, text) VALUES (?, ?)'
+          'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)'
         );
-        for (const chunk of chunks) {
-          stmt.run(chunk.rowid, chunk.text);
+
+        for (const doc of docs) {
+          stmt.run(doc.id, doc.rel_path, doc.title ?? '', doc.markdown);
+          count++;
+        }
+      });
+
+      transaction();
+      return ok(count);
+    } catch (cause) {
+      return err(
+        'QUERY_FAILED',
+        cause instanceof Error
+          ? cause.message
+          : 'Failed to rebuild documents FTS',
+        cause
+      );
+    }
+  }
+
+  /**
+   * @deprecated Use syncDocumentFts for document-level FTS.
+   * Kept for backwards compat during migration.
+   */
+  async rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>> {
+    try {
+      const db = this.ensureOpen();
+
+      const transaction = db.transaction(() => {
+        // Get documents using this hash and sync their FTS
+        interface DocInfo {
+          id: number;
+          rel_path: string;
+          title: string | null;
+        }
+
+        const docs = db
+          .query<DocInfo, [string]>(
+            'SELECT id, rel_path, title FROM documents WHERE mirror_hash = ? AND active = 1'
+          )
+          .all(mirrorHash);
+
+        // Get content
+        const content = db
+          .query<{ markdown: string }, [string]>(
+            'SELECT markdown FROM content WHERE mirror_hash = ?'
+          )
+          .get(mirrorHash);
+
+        if (!content) {
+          return;
+        }
+
+        // Update FTS for each document using this hash
+        for (const doc of docs) {
+          db.run('DELETE FROM documents_fts WHERE rowid = ?', [doc.id]);
+          db.run(
+            'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)',
+            [doc.id, doc.rel_path, doc.title ?? '', content.markdown]
+          );
         }
       });
 
@@ -1069,10 +1252,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         `);
         expiredCache = cacheResult.changes;
 
-        // Rebuild FTS index (remove orphaned entries)
+        // Clean orphaned FTS entries (documents that no longer exist or are inactive)
         db.run(`
-          DELETE FROM content_fts WHERE rowid NOT IN (
-            SELECT rowid FROM content_chunks
+          DELETE FROM documents_fts WHERE rowid NOT IN (
+            SELECT id FROM documents WHERE active = 1
           )
         `);
       });