@gmickel/gno 0.3.0

package/src/store/index.ts

@@ -0,0 +1,41 @@
+/**
+ * Store layer exports.
+ * Provides StorePort interface and SQLite implementation.
+ *
+ * @module src/store
+ */
+
+export type { Migration } from './migrations';
+// Migrations
+export {
+  getDbFtsTokenizer,
+  getSchemaVersion,
+  migrations,
+  needsFtsRebuild,
+  runMigrations,
+} from './migrations';
+
+// SQLite adapter
+export { SqliteAdapter } from './sqlite';
+// Types and interfaces
+export type {
+  ChunkInput,
+  ChunkRow,
+  CleanupStats,
+  CollectionRow,
+  CollectionStatus,
+  ContextRow,
+  DocumentInput,
+  DocumentRow,
+  FtsResult,
+  FtsSearchOptions,
+  IndexStatus,
+  IngestErrorInput,
+  IngestErrorRow,
+  MigrationResult,
+  StoreError,
+  StoreErrorCode,
+  StorePort,
+  StoreResult,
+} from './types';
+export { err, ok } from './types';

package/src/store/migrations/001-initial.ts

@@ -0,0 +1,196 @@
+/**
+ * Initial schema migration.
+ * Creates all core tables for GNO.
+ *
+ * @module src/store/migrations/001_initial
+ */
+
+import type { Database } from 'bun:sqlite';
+import type { FtsTokenizer } from '../../config/types';
+import type { Migration } from './runner';
+
+export const migration: Migration = {
+  version: 1,
+  name: 'initial_schema',
+
+  up(db: Database, ftsTokenizer: FtsTokenizer): void {
+    // Collections (synced from YAML config)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS collections (
+        name TEXT PRIMARY KEY,
+        path TEXT NOT NULL,
+        pattern TEXT NOT NULL DEFAULT '**/*',
+        include TEXT,
+        exclude TEXT,
+        update_cmd TEXT,
+        language_hint TEXT,
+        synced_at TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+
+    // Contexts (synced from YAML config)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS contexts (
+        scope_type TEXT NOT NULL CHECK (scope_type IN ('global', 'collection', 'prefix')),
+        scope_key TEXT NOT NULL,
+        text TEXT NOT NULL,
+        synced_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (scope_type, scope_key)
+      )
+    `);
+
+    // Documents (source file identity)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        collection TEXT NOT NULL,
+        rel_path TEXT NOT NULL,
+
+        source_hash TEXT NOT NULL,
+        source_mime TEXT NOT NULL,
+        source_ext TEXT NOT NULL,
+        source_size INTEGER NOT NULL,
+        source_mtime TEXT NOT NULL,
+
+        docid TEXT NOT NULL,
+        uri TEXT NOT NULL,
+
+        title TEXT,
+        mirror_hash TEXT,
+        converter_id TEXT,
+        converter_version TEXT,
+        language_hint TEXT,
+
+        active INTEGER NOT NULL DEFAULT 1,
+
+        last_error_code TEXT,
+        last_error_message TEXT,
+        last_error_at TEXT,
+
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+
+        UNIQUE (collection, rel_path),
+        FOREIGN KEY (collection) REFERENCES collections(name) ON DELETE CASCADE
+        -- Note: mirror_hash is NOT an FK - documents are tracked before content exists
+        -- Cleanup via cleanupOrphans() handles orphaned content
+      )
+    `);
+
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection)'
+    );
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_documents_active ON documents(active)'
+    );
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_documents_mirror_hash ON documents(mirror_hash)'
+    );
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_documents_docid ON documents(docid)'
+    );
+    db.exec('CREATE INDEX IF NOT EXISTS idx_documents_uri ON documents(uri)');
+
+    // Content (content-addressed markdown mirrors)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS content (
+        mirror_hash TEXT PRIMARY KEY,
+        markdown TEXT NOT NULL,
+        created_at TEXT NOT NULL DEFAULT (datetime('now'))
+      )
+    `);
+
+    // Content chunks (for FTS and vectors)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS content_chunks (
+        mirror_hash TEXT NOT NULL,
+        seq INTEGER NOT NULL,
+        pos INTEGER NOT NULL,
+        text TEXT NOT NULL,
+        start_line INTEGER NOT NULL,
+        end_line INTEGER NOT NULL,
+        language TEXT,
+        token_count INTEGER,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (mirror_hash, seq),
+        FOREIGN KEY (mirror_hash) REFERENCES content(mirror_hash) ON DELETE CASCADE
+      )
+    `);
+
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_chunks_mirror_hash ON content_chunks(mirror_hash)'
+    );
+
+    // FTS5 virtual table with configurable tokenizer
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS content_fts USING fts5(
+        text,
+        tokenize='${ftsTokenizer}'
+      )
+    `);
+
+    // Content vectors (EPIC 7)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS content_vectors (
+        mirror_hash TEXT NOT NULL,
+        seq INTEGER NOT NULL,
+        model TEXT NOT NULL,
+        embedding BLOB NOT NULL,
+        embedded_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (mirror_hash, seq, model),
+        FOREIGN KEY (mirror_hash, seq) REFERENCES content_chunks(mirror_hash, seq) ON DELETE CASCADE
+      )
+    `);
+
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_vectors_model ON content_vectors(model)'
+    );
+
+    // LLM cache (EPIC 6+)
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS llm_cache (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        expires_at TEXT
+      )
+    `);
+
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_llm_cache_expires ON llm_cache(expires_at)'
+    );
+
+    // Ingest errors
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS ingest_errors (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        collection TEXT NOT NULL,
+        rel_path TEXT NOT NULL,
+        occurred_at TEXT NOT NULL DEFAULT (datetime('now')),
+        code TEXT NOT NULL,
+        message TEXT NOT NULL,
+        details_json TEXT
+      )
+    `);
+
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_ingest_errors_occurred ON ingest_errors(occurred_at DESC)'
+    );
+    db.exec(
+      'CREATE INDEX IF NOT EXISTS idx_ingest_errors_collection ON ingest_errors(collection, rel_path)'
+    );
+  },
+
+  down(db: Database): void {
+    // Drop in reverse order of creation
+    db.exec('DROP TABLE IF EXISTS ingest_errors');
+    db.exec('DROP TABLE IF EXISTS llm_cache');
+    db.exec('DROP TABLE IF EXISTS content_vectors');
+    db.exec('DROP TABLE IF EXISTS content_fts');
+    db.exec('DROP TABLE IF EXISTS content_chunks');
+    db.exec('DROP TABLE IF EXISTS content');
+    db.exec('DROP TABLE IF EXISTS documents');
+    db.exec('DROP TABLE IF EXISTS contexts');
+    db.exec('DROP TABLE IF EXISTS collections');
+  },
+};

package/src/store/migrations/index.ts

@@ -0,0 +1,20 @@
+/**
+ * Migration registry.
+ * Exports all migrations in order.
+ *
+ * @module src/store/migrations
+ */
+
+export type { Migration } from './runner';
+export {
+  getDbFtsTokenizer,
+  getSchemaVersion,
+  needsFtsRebuild,
+  runMigrations,
+} from './runner';
+
+// Import all migrations
+import { migration as m001 } from './001-initial';
+
+/** All migrations in order */
+export const migrations = [m001];

package/src/store/migrations/runner.ts

@@ -0,0 +1,187 @@
+/**
+ * Database migration runner.
+ * Tracks applied migrations in schema_meta table.
+ *
+ * @module src/store/migrations/runner
+ */
+
+import type { Database } from 'bun:sqlite';
+import type { FtsTokenizer } from '../../config/types';
+import type { MigrationResult, StoreResult } from '../types';
+import { err, ok } from '../types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Migration definition */
+export interface Migration {
+  /** Version number (must be unique and sequential) */
+  version: number;
+  /** Human-readable name */
+  name: string;
+  /** Apply migration */
+  up(db: Database, ftsTokenizer: FtsTokenizer): void;
+  /** Rollback migration (optional) */
+  down?(db: Database): void;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Schema Meta Queries
+// ─────────────────────────────────────────────────────────────────────────────
+
+const BOOTSTRAP_META_TABLE = `
+  CREATE TABLE IF NOT EXISTS schema_meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL,
+    updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+  )
+`;
+
+const GET_META = 'SELECT value FROM schema_meta WHERE key = ?';
+
+const SET_META = `
+  INSERT INTO schema_meta (key, value, updated_at)
+  VALUES (?, ?, datetime('now'))
+  ON CONFLICT(key) DO UPDATE SET
+    value = excluded.value,
+    updated_at = datetime('now')
+`;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Migration Runner
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Get current schema version from database.
+ * Returns 0 if no version is set.
+ */
+export function getSchemaVersion(db: Database): number {
+  try {
+    const row = db.query<{ value: string }, [string]>(GET_META).get('version');
+    return row ? Number.parseInt(row.value, 10) : 0;
+  } catch {
+    // Table doesn't exist yet
+    return 0;
+  }
+}
+
+/**
+ * Get current FTS tokenizer from database.
+ * Returns null if not set.
+ */
+export function getDbFtsTokenizer(db: Database): FtsTokenizer | null {
+  try {
+    const row = db
+      .query<{ value: string }, [string]>(GET_META)
+      .get('fts_tokenizer');
+    return row ? (row.value as FtsTokenizer) : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Set metadata value.
+ */
+function setMeta(db: Database, key: string, value: string): void {
+  db.run(SET_META, [key, value]);
+}
+
+/**
+ * Run pending migrations.
+ *
+ * @param db - Open database connection
+ * @param migrations - Array of migrations to apply
+ * @param ftsTokenizer - FTS tokenizer from config
+ * @returns Migration result with applied versions
+ */
+export function runMigrations(
+  db: Database,
+  migrations: Migration[],
+  ftsTokenizer: FtsTokenizer
+): StoreResult<MigrationResult> {
+  try {
+    // Bootstrap schema_meta table
+    db.exec(BOOTSTRAP_META_TABLE);
+
+    // Get current version
+    const currentVersion = getSchemaVersion(db);
+
+    // Check FTS tokenizer consistency
+    const dbTokenizer = getDbFtsTokenizer(db);
+    if (dbTokenizer !== null && dbTokenizer !== ftsTokenizer) {
+      // Tokenizer mismatch - this requires special handling
+      // For now, we error out. EPIC 5 will handle FTS rebuild.
+      return err(
+        'MIGRATION_FAILED',
+        `FTS tokenizer mismatch: DB has '${dbTokenizer}', config has '${ftsTokenizer}'. ` +
+          'Run `gno index --rebuild-fts` to recreate FTS index with new tokenizer.'
+      );
+    }
+
+    // Sort migrations by version
+    const sorted = [...migrations].sort((a, b) => a.version - b.version);
+
+    // Validate sequential versions
+    for (let i = 0; i < sorted.length; i++) {
+      const migration = sorted[i];
+      if (migration && migration.version !== i + 1) {
+        return err(
+          'MIGRATION_FAILED',
+          `Migration versions must be sequential. Expected ${i + 1}, got ${migration.version}`
+        );
+      }
+    }
+
+    // Filter pending migrations
+    const pending = sorted.filter((m) => m.version > currentVersion);
+
+    if (pending.length === 0) {
+      return ok({
+        applied: [],
+        currentVersion,
+        ftsTokenizer: dbTokenizer ?? ftsTokenizer,
+      });
+    }
+
+    // Apply migrations in a transaction
+    const applied: number[] = [];
+    const transaction = db.transaction(() => {
+      for (const migration of pending) {
+        migration.up(db, ftsTokenizer);
+        setMeta(db, 'version', migration.version.toString());
+        applied.push(migration.version);
+      }
+
+      // Set FTS tokenizer if not already set
+      if (dbTokenizer === null) {
+        setMeta(db, 'fts_tokenizer', ftsTokenizer);
+        setMeta(db, 'created_at', new Date().toISOString());
+      }
+    });
+
+    transaction();
+
+    return ok({
+      applied,
+      currentVersion: pending.at(-1)?.version ?? currentVersion,
+      ftsTokenizer,
+    });
+  } catch (cause) {
+    const message =
+      cause instanceof Error ? cause.message : 'Unknown migration error';
+    return err('MIGRATION_FAILED', message, cause);
+  }
+}
+
+/**
+ * Check if FTS tokenizer change is needed.
+ */
+export function needsFtsRebuild(
+  db: Database,
+  configTokenizer: FtsTokenizer
+): boolean {
+  const dbTokenizer = getDbFtsTokenizer(db);
+  return dbTokenizer !== null && dbTokenizer !== configTokenizer;
+}