@gmickel/gno 0.3.5 → 0.5.0

package/src/store/sqlite/fts5-snowball.ts

@@ -0,0 +1,144 @@
+/**
+ * fts5-snowball extension loader.
+ *
+ * Loads vendored fts5-snowball extension for multilingual FTS5 stemming.
+ * Pattern mirrors sqlite-vec loader.
+ *
+ * @module src/store/sqlite/fts5-snowball
+ */
+
+import type { Database } from 'bun:sqlite';
+// node:fs: existsSync for sync file checks at load time
+import { existsSync } from 'node:fs';
+// node:path: join for cross-platform paths
+import { join } from 'node:path';
+// node:process: arch/platform detection (no Bun equivalent)
+import { arch, platform } from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Result of attempting to load fts5-snowball.
+ */
+export interface Fts5SnowballLoadResult {
+  loaded: boolean;
+  error?: string;
+  path?: string;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Platform Detection
+// ─────────────────────────────────────────────────────────────────────────────
+
+function getPlatformDir(): string | null {
+  const os = platform === 'win32' ? 'windows' : platform;
+  const archName = arch === 'arm64' ? 'arm64' : 'x64';
+
+  if (os === 'darwin') {
+    return `darwin-${archName}`;
+  }
+  if (os === 'linux' && archName === 'x64') {
+    return 'linux-x64';
+  }
+  if (os === 'windows' && archName === 'x64') {
+    return 'windows-x64';
+  }
+
+  return null;
+}
+
+function getExtensionSuffix(): string {
+  if (platform === 'win32') {
+    return 'dll';
+  }
+  if (platform === 'darwin') {
+    return 'dylib';
+  }
+  return 'so';
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Path Resolution
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Get path to vendored fts5-snowball extension.
+ * Returns null if not available for this platform.
+ */
+export function getExtensionPath(): string | null {
+  const platformDir = getPlatformDir();
+  if (!platformDir) {
+    return null;
+  }
+
+  const suffix = getExtensionSuffix();
+  const filename = `fts5stemmer.${suffix}`;
+
+  // Resolve relative to this module (ESM-safe)
+  const thisDir = fileURLToPath(new URL('.', import.meta.url));
+  const vendorPath = join(
+    thisDir,
+    '..',
+    '..',
+    '..',
+    'vendor',
+    'fts5-snowball',
+    platformDir,
+    filename
+  );
+
+  if (existsSync(vendorPath)) {
+    return vendorPath;
+  }
+
+  return null;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Extension Loading
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Load fts5-snowball extension into database.
+ *
+ * Must be called after Database.setCustomSQLite() on macOS.
+ * Safe to call multiple times - extension load is idempotent.
+ *
+ * @param db - Open database connection
+ * @returns Load result with success/error info
+ */
+export function loadFts5Snowball(db: Database): Fts5SnowballLoadResult {
+  const path = getExtensionPath();
+
+  if (!path) {
+    const platformDir = getPlatformDir();
+    return {
+      loaded: false,
+      error: platformDir
+        ? `fts5-snowball binary not found for ${platformDir}`
+        : `fts5-snowball not available for ${platform}-${arch}`,
+    };
+  }
+
+  try {
+    db.loadExtension(path);
+    return { loaded: true, path };
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    return {
+      loaded: false,
+      error: message,
+      path,
+    };
+  }
+}
+
+/**
+ * Check if fts5-snowball is available for this platform.
+ */
+export function isAvailable(): boolean {
+  return getExtensionPath() !== null;
+}

package/src/store/types.ts

@@ -18,6 +18,7 @@ export type StoreErrorCode =
   | 'CONSTRAINT_VIOLATION'
   | 'MIGRATION_FAILED'
   | 'CONNECTION_FAILED'
+  | 'EXTENSION_LOAD_FAILED'
   | 'QUERY_FAILED'
   | 'TRANSACTION_FAILED'
   | 'INVALID_INPUT'
@@ -195,7 +196,11 @@ export interface FtsSearchOptions {
   limit?: number;
   /** Filter by collection */
   collection?: string;
-  /** Filter by language */
+  /**
+   * Language hint (reserved for future use).
+   * Note: FTS5 snowball tokenizer is language-aware at index time,
+   * so runtime language filtering is not currently implemented.
+   */
   language?: string;
   /** Include snippet with highlights */
   snippet?: boolean;
@@ -399,6 +404,16 @@ export interface StorePort {
    */
   listDocuments(collection?: string): Promise<StoreResult<DocumentRow[]>>;
 
+  /**
+   * List documents with pagination support.
+   * Returns documents and total count for efficient browsing.
+   */
+  listDocumentsPaginated(options: {
+    collection?: string;
+    limit: number;
+    offset: number;
+  }): Promise<StoreResult<{ documents: DocumentRow[]; total: number }>>;
+
   /**
    * Mark documents as inactive (soft delete).
    * Returns count of affected documents.
@@ -459,7 +474,7 @@ export interface StorePort {
   // ─────────────────────────────────────────────────────────────────────────
 
   /**
-   * Search chunks using FTS5.
+   * Search documents using FTS5 (document-level).
    */
   searchFts(
     query: string,
@@ -467,8 +482,23 @@ export interface StorePort {
   ): Promise<StoreResult<FtsResult[]>>;
 
   /**
+   * Sync a document to documents_fts for full-text search.
+   * Must be called after document and content are both upserted.
+   */
+  syncDocumentFts(
+    collection: string,
+    relPath: string
+  ): Promise<StoreResult<void>>;
+
+  /**
+   * Rebuild entire documents_fts index from scratch.
+   * Use after migration or for recovery. Returns count of indexed docs.
+   */
+  rebuildAllDocumentsFts(): Promise<StoreResult<number>>;
+
+  /**
+   * @deprecated Use syncDocumentFts for document-level FTS.
    * Rebuild FTS index for a mirror hash.
-   * Called after upserting chunks.
    */
   rebuildFtsForHash(mirrorHash: string): Promise<StoreResult<void>>;
 

package/src/store/vector/stats.ts

@@ -78,9 +78,11 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
 
         // Seek pagination: use cursor to avoid skipping items as backlog shrinks
         // Query structure changes based on whether we have a cursor
+        // Include document title for contextual embedding
         const sql = after
           ? `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v
@@ -108,6 +110,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
         `
           : `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v

package/src/store/vector/types.ts

@@ -38,6 +38,7 @@ export interface BacklogItem {
   mirrorHash: string;
   seq: number;
   text: string;
+  title: string | null;
   reason: 'new' | 'changed' | 'force';
 }