@gmickel/gno 0.4.0 → 0.5.0

package/src/store/sqlite/adapter.ts

@@ -31,6 +31,7 @@ import type {
   StoreResult,
 } from '../types';
 import { err, ok } from '../types';
+import { loadFts5Snowball } from './fts5-snowball';
 import type { SqliteDbProvider } from './types';
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -103,6 +104,19 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         this.db.exec('PRAGMA journal_mode = WAL');
       }
 
+      // Load fts5-snowball extension if using snowball tokenizer
+      if (ftsTokenizer.startsWith('snowball')) {
+        const snowballResult = loadFts5Snowball(this.db);
+        if (!snowballResult.loaded) {
+          this.db.close();
+          this.db = null;
+          return err(
+            'EXTENSION_LOAD_FAILED',
+            `Failed to load fts5-snowball: ${snowballResult.error}`
+          );
+        }
+      }
+
       // Run migrations
       const result = runMigrations(this.db, migrations, ftsTokenizer);
       if (!result.ok) {
@@ -744,16 +758,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       const db = this.ensureOpen();
       const limit = options.limit ?? 20;
 
-      // Join FTS results with chunks and documents
-      // Use bm25() function explicitly - fts.rank doesn't work with JOINs
-      // Note: Multiple docs can share mirror_hash (content-addressed storage)
-      // Deduplication by uri+seq is done in search.ts to avoid FTS function context issues
+      // Document-level FTS search using documents_fts
+      // Uses bm25() for relevance ranking (more negative = better match)
+      // Snippet from body column (index 2) with highlight markers
       const sql = `
         SELECT
-          c.mirror_hash,
-          c.seq,
-          bm25(content_fts) as score,
-          ${options.snippet ? "snippet(content_fts, 0, '<mark>', '</mark>', '...', 32) as snippet," : ''}
+          d.mirror_hash,
+          0 as seq,
+          bm25(documents_fts) as score,
+          ${options.snippet ? "snippet(documents_fts, 2, '<mark>', '</mark>', '...', 32) as snippet," : ''}
           d.docid,
           d.uri,
           d.title,
@@ -764,13 +777,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           d.source_mtime,
           d.source_size,
           d.source_hash
-        FROM content_fts fts
-        JOIN content_chunks c ON c.rowid = fts.rowid
-        JOIN documents d ON d.mirror_hash = c.mirror_hash AND d.active = 1
-        WHERE content_fts MATCH ?
+        FROM documents_fts fts
+        JOIN documents d ON d.id = fts.rowid AND d.active = 1
+        WHERE documents_fts MATCH ?
         ${options.collection ? 'AND d.collection = ?' : ''}
-        ${options.language ? 'AND c.language = ?' : ''}
-        ORDER BY bm25(content_fts)
+        ORDER BY bm25(documents_fts)
         LIMIT ?
       `;
 
@@ -778,9 +789,6 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       if (options.collection) {
         params.push(options.collection);
       }
-      if (options.language) {
-        params.push(options.language);
-      }
       params.push(limit);
 
       interface FtsRow {
@@ -835,29 +843,157 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
-  async rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>> {
+  /**
+   * Sync a document to documents_fts for full-text search.
+   * Must be called after document and content are both upserted.
+   * The FTS rowid matches documents.id for efficient JOINs.
+   */
+  async syncDocumentFts(
+    collection: string,
+    relPath: string
+  ): Promise<StoreResult<void>> {
     try {
       const db = this.ensureOpen();
 
       const transaction = db.transaction(() => {
-        // Get chunks for this hash
-        const chunks = db
-          .query<{ rowid: number; text: string }, [string]>(
-            'SELECT rowid, text FROM content_chunks WHERE mirror_hash = ?'
+        // Get document with its content
+        interface DocWithContent {
+          id: number;
+          rel_path: string;
+          title: string | null;
+          markdown: string | null;
+        }
+
+        const doc = db
+          .query<DocWithContent, [string, string]>(
+            `SELECT d.id, d.rel_path, d.title, c.markdown
+             FROM documents d
+             LEFT JOIN content c ON c.mirror_hash = d.mirror_hash
+             WHERE d.collection = ? AND d.rel_path = ? AND d.active = 1`
           )
-          .all(mirrorHash);
+          .get(collection, relPath);
 
-        // Delete old FTS entries for these rowids
-        for (const chunk of chunks) {
-          db.run('DELETE FROM content_fts WHERE rowid = ?', [chunk.rowid]);
+        if (!doc) {
+          return; // Document not found or inactive
         }
 
-        // Insert new FTS entries
+        // Delete existing FTS entry for this doc
+        db.run('DELETE FROM documents_fts WHERE rowid = ?', [doc.id]);
+
+        // Insert new FTS entry if we have content
+        if (doc.markdown) {
+          db.run(
+            'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)',
+            [doc.id, doc.rel_path, doc.title ?? '', doc.markdown]
+          );
+        }
+      });
+
+      transaction();
+      return ok(undefined);
+    } catch (cause) {
+      return err(
+        'QUERY_FAILED',
+        cause instanceof Error ? cause.message : 'Failed to sync document FTS',
+        cause
+      );
+    }
+  }
+
+  /**
+   * Rebuild entire documents_fts index from scratch.
+   * Use after migration or for recovery.
+   */
+  async rebuildAllDocumentsFts(): Promise<StoreResult<number>> {
+    try {
+      const db = this.ensureOpen();
+      let count = 0;
+
+      const transaction = db.transaction(() => {
+        // Clear FTS table
+        db.run('DELETE FROM documents_fts');
+
+        // Get all active documents with content
+        interface DocWithContent {
+          id: number;
+          rel_path: string;
+          title: string | null;
+          markdown: string;
+        }
+
+        const docs = db
+          .query<DocWithContent, []>(
+            `SELECT d.id, d.rel_path, d.title, c.markdown
+             FROM documents d
+             JOIN content c ON c.mirror_hash = d.mirror_hash
+             WHERE d.active = 1 AND d.mirror_hash IS NOT NULL`
+          )
+          .all();
+
+        // Insert FTS entries
         const stmt = db.prepare(
-          'INSERT INTO content_fts (rowid, text) VALUES (?, ?)'
+          'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)'
         );
-        for (const chunk of chunks) {
-          stmt.run(chunk.rowid, chunk.text);
+
+        for (const doc of docs) {
+          stmt.run(doc.id, doc.rel_path, doc.title ?? '', doc.markdown);
+          count++;
+        }
+      });
+
+      transaction();
+      return ok(count);
+    } catch (cause) {
+      return err(
+        'QUERY_FAILED',
+        cause instanceof Error
+          ? cause.message
+          : 'Failed to rebuild documents FTS',
+        cause
+      );
+    }
+  }
+
+  /**
+   * @deprecated Use syncDocumentFts for document-level FTS.
+   * Kept for backwards compat during migration.
+   */
+  async rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>> {
+    try {
+      const db = this.ensureOpen();
+
+      const transaction = db.transaction(() => {
+        // Get documents using this hash and sync their FTS
+        interface DocInfo {
+          id: number;
+          rel_path: string;
+          title: string | null;
+        }
+
+        const docs = db
+          .query<DocInfo, [string]>(
+            'SELECT id, rel_path, title FROM documents WHERE mirror_hash = ? AND active = 1'
+          )
+          .all(mirrorHash);
+
+        // Get content
+        const content = db
+          .query<{ markdown: string }, [string]>(
+            'SELECT markdown FROM content WHERE mirror_hash = ?'
+          )
+          .get(mirrorHash);
+
+        if (!content) {
+          return;
+        }
+
+        // Update FTS for each document using this hash
+        for (const doc of docs) {
+          db.run('DELETE FROM documents_fts WHERE rowid = ?', [doc.id]);
+          db.run(
+            'INSERT INTO documents_fts (rowid, filepath, title, body) VALUES (?, ?, ?, ?)',
+            [doc.id, doc.rel_path, doc.title ?? '', content.markdown]
+          );
         }
       });
 
@@ -1116,10 +1252,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         `);
         expiredCache = cacheResult.changes;
 
-        // Rebuild FTS index (remove orphaned entries)
+        // Clean orphaned FTS entries (documents that no longer exist or are inactive)
         db.run(`
-          DELETE FROM content_fts WHERE rowid NOT IN (
-            SELECT rowid FROM content_chunks
+          DELETE FROM documents_fts WHERE rowid NOT IN (
+            SELECT id FROM documents WHERE active = 1
           )
         `);
       });

package/src/store/sqlite/fts5-snowball.ts

@@ -0,0 +1,144 @@
+/**
+ * fts5-snowball extension loader.
+ *
+ * Loads vendored fts5-snowball extension for multilingual FTS5 stemming.
+ * Pattern mirrors sqlite-vec loader.
+ *
+ * @module src/store/sqlite/fts5-snowball
+ */
+
+import type { Database } from 'bun:sqlite';
+// node:fs: existsSync for sync file checks at load time
+import { existsSync } from 'node:fs';
+// node:path: join for cross-platform paths
+import { join } from 'node:path';
+// node:process: arch/platform detection (no Bun equivalent)
+import { arch, platform } from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Result of attempting to load fts5-snowball.
+ */
+export interface Fts5SnowballLoadResult {
+  loaded: boolean;
+  error?: string;
+  path?: string;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Platform Detection
+// ─────────────────────────────────────────────────────────────────────────────
+
+function getPlatformDir(): string | null {
+  const os = platform === 'win32' ? 'windows' : platform;
+  const archName = arch === 'arm64' ? 'arm64' : 'x64';
+
+  if (os === 'darwin') {
+    return `darwin-${archName}`;
+  }
+  if (os === 'linux' && archName === 'x64') {
+    return 'linux-x64';
+  }
+  if (os === 'windows' && archName === 'x64') {
+    return 'windows-x64';
+  }
+
+  return null;
+}
+
+function getExtensionSuffix(): string {
+  if (platform === 'win32') {
+    return 'dll';
+  }
+  if (platform === 'darwin') {
+    return 'dylib';
+  }
+  return 'so';
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Path Resolution
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Get path to vendored fts5-snowball extension.
+ * Returns null if not available for this platform.
+ */
+export function getExtensionPath(): string | null {
+  const platformDir = getPlatformDir();
+  if (!platformDir) {
+    return null;
+  }
+
+  const suffix = getExtensionSuffix();
+  const filename = `fts5stemmer.${suffix}`;
+
+  // Resolve relative to this module (ESM-safe)
+  const thisDir = fileURLToPath(new URL('.', import.meta.url));
+  const vendorPath = join(
+    thisDir,
+    '..',
+    '..',
+    '..',
+    'vendor',
+    'fts5-snowball',
+    platformDir,
+    filename
+  );
+
+  if (existsSync(vendorPath)) {
+    return vendorPath;
+  }
+
+  return null;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Extension Loading
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Load fts5-snowball extension into database.
+ *
+ * Must be called after Database.setCustomSQLite() on macOS.
+ * Safe to call multiple times - extension load is idempotent.
+ *
+ * @param db - Open database connection
+ * @returns Load result with success/error info
+ */
+export function loadFts5Snowball(db: Database): Fts5SnowballLoadResult {
+  const path = getExtensionPath();
+
+  if (!path) {
+    const platformDir = getPlatformDir();
+    return {
+      loaded: false,
+      error: platformDir
+        ? `fts5-snowball binary not found for ${platformDir}`
+        : `fts5-snowball not available for ${platform}-${arch}`,
+    };
+  }
+
+  try {
+    db.loadExtension(path);
+    return { loaded: true, path };
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    return {
+      loaded: false,
+      error: message,
+      path,
+    };
+  }
+}
+
+/**
+ * Check if fts5-snowball is available for this platform.
+ */
+export function isAvailable(): boolean {
+  return getExtensionPath() !== null;
+}

package/src/store/types.ts

@@ -18,6 +18,7 @@ export type StoreErrorCode =
   | 'CONSTRAINT_VIOLATION'
   | 'MIGRATION_FAILED'
   | 'CONNECTION_FAILED'
+  | 'EXTENSION_LOAD_FAILED'
   | 'QUERY_FAILED'
   | 'TRANSACTION_FAILED'
   | 'INVALID_INPUT'
@@ -195,7 +196,11 @@ export interface FtsSearchOptions {
   limit?: number;
   /** Filter by collection */
   collection?: string;
-  /** Filter by language */
+  /**
+   * Language hint (reserved for future use).
+   * Note: FTS5 snowball tokenizer is language-aware at index time,
+   * so runtime language filtering is not currently implemented.
+   */
   language?: string;
   /** Include snippet with highlights */
   snippet?: boolean;
@@ -469,7 +474,7 @@ export interface StorePort {
   // ─────────────────────────────────────────────────────────────────────────
 
   /**
-   * Search chunks using FTS5.
+   * Search documents using FTS5 (document-level).
    */
   searchFts(
     query: string,
@@ -477,8 +482,23 @@ export interface StorePort {
   ): Promise<StoreResult<FtsResult[]>>;
 
   /**
+   * Sync a document to documents_fts for full-text search.
+   * Must be called after document and content are both upserted.
+   */
+  syncDocumentFts(
+    collection: string,
+    relPath: string
+  ): Promise<StoreResult<void>>;
+
+  /**
+   * Rebuild entire documents_fts index from scratch.
+   * Use after migration or for recovery. Returns count of indexed docs.
+   */
+  rebuildAllDocumentsFts(): Promise<StoreResult<number>>;
+
+  /**
+   * @deprecated Use syncDocumentFts for document-level FTS.
    * Rebuild FTS index for a mirror hash.
-   * Called after upserting chunks.
    */
   rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>>;
 

package/src/store/vector/stats.ts

@@ -78,9 +78,11 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
 
         // Seek pagination: use cursor to avoid skipping items as backlog shrinks
         // Query structure changes based on whether we have a cursor
+        // Include document title for contextual embedding
         const sql = after
           ? `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v
@@ -108,6 +110,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
         `
           : `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v

package/src/store/vector/types.ts

@@ -38,6 +38,7 @@ export interface BacklogItem {
   mirrorHash: string;
   seq: number;
   text: string;
+  title: string | null;
   reason: 'new' | 'changed' | 'force';
 }