@velvetmonkey/vault-core 2.0.137 → 2.0.139

package/dist/migrations.d.ts

@@ -0,0 +1,21 @@
+/**
+ * SQLite Schema Migrations
+ *
+ * Database path resolution, schema initialization, migration logic,
+ * and database file management utilities.
+ */
+import Database from 'better-sqlite3';
+/**
+ * Get the database path for a vault
+ */
+export declare function getStateDbPath(vaultPath: string): string;
+/**
+ * Initialize schema and run migrations
+ */
+export declare function initSchema(db: Database.Database): void;
+export declare function deleteStateDbFiles(dbPath: string): void;
+/** Back up state.db before opening (skip if missing or 0 bytes). */
+export declare function backupStateDb(dbPath: string): void;
+/** Preserve a corrupted database for inspection before deleting. */
+export declare function preserveCorruptedDb(dbPath: string): void;
+//# sourceMappingURL=migrations.d.ts.map

package/dist/migrations.js

@@ -0,0 +1,320 @@
+/**
+ * SQLite Schema Migrations
+ *
+ * Database path resolution, schema initialization, migration logic,
+ * and database file management utilities.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { SCHEMA_VERSION, SCHEMA_SQL, STATE_DB_FILENAME, FLYWHEEL_DIR } from './schema.js';
+// =============================================================================
+// Database Path Resolution
+// =============================================================================
+/**
+ * Get the database path for a vault
+ */
+export function getStateDbPath(vaultPath) {
+    const flywheelDir = path.join(vaultPath, FLYWHEEL_DIR);
+    if (!fs.existsSync(flywheelDir)) {
+        fs.mkdirSync(flywheelDir, { recursive: true });
+    }
+    return path.join(flywheelDir, STATE_DB_FILENAME);
+}
+// =============================================================================
+// Schema Initialization & Migrations
+// =============================================================================
+/**
+ * Initialize schema and run migrations
+ */
+export function initSchema(db) {
+    // Enable WAL mode for better concurrent read performance
+    db.pragma('journal_mode = WAL');
+    // Enable foreign keys
+    db.pragma('foreign_keys = ON');
+    // Performance tuning
+    db.pragma('synchronous = NORMAL'); // Safe with WAL — fsync only on checkpoint, not every commit
+    db.pragma('cache_size = -64000'); // 64 MB page cache (default is ~2 MB)
+    db.pragma('temp_store = MEMORY'); // Temp tables/indices in RAM instead of disk
+    // Run schema creation
+    db.exec(SCHEMA_SQL);
+    // Guard: Verify critical tables were created
+    // This catches cases where schema execution silently failed (e.g., corrupted db)
+    const tables = db.prepare(`
+    SELECT name FROM sqlite_master
+    WHERE type='table' AND name IN ('entities', 'schema_version', 'metadata')
+  `).all();
+    if (tables.length < 3) {
+        const foundTables = tables.map(t => t.name).join(', ') || 'none';
+        throw new Error(`[vault-core] Schema validation failed: expected 3 critical tables, found ${tables.length} (${foundTables}). ` +
+            `Database may be corrupted. Delete ${db.name} and restart.`);
+    }
+    // Check and record schema version
+    const versionRow = db.prepare('SELECT MAX(version) as version FROM schema_version').get();
+    const currentVersion = versionRow?.version ?? 0;
+    if (currentVersion < SCHEMA_VERSION) {
+        // v2: Drop dead notes/links tables if they exist from v1
+        if (currentVersion < 2) {
+            db.exec('DROP TABLE IF EXISTS notes');
+            db.exec('DROP TABLE IF EXISTS links');
+        }
+        // v3: Rename crank_state → write_state
+        if (currentVersion < 3) {
+            const hasCrankState = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='crank_state'`).get();
+            const hasWriteState = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='write_state'`).get();
+            if (hasCrankState && !hasWriteState) {
+                db.exec('ALTER TABLE crank_state RENAME TO write_state');
+            }
+            else if (hasCrankState && hasWriteState) {
+                // Both exist (stale db) — drop the old one
+                db.exec('DROP TABLE crank_state');
+            }
+        }
+        // v4: vault_metrics, wikilink_feedback, wikilink_suppressions tables
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v5: wikilink_applications table (implicit feedback tracking)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v6: index_events table (index activity history)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v7: tool_invocations table (usage analytics)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v8: graph_snapshots table (structural evolution)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v9: note_embeddings table (semantic search)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v10: entity_embeddings table (semantic entity search)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v11: notes_fts gains frontmatter column (4-col: path, title, frontmatter, content)
+        // Virtual tables can't ALTER, so drop and recreate
+        if (currentVersion < 11) {
+            db.exec('DROP TABLE IF EXISTS notes_fts');
+            db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
+        path, title, frontmatter, content,
+        tokenize='porter'
+      )`);
+            // Clear FTS metadata to force rebuild with new schema
+            db.exec(`DELETE FROM fts_metadata WHERE key = 'last_built'`);
+        }
+        // v12: tasks cache table (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v13: merge_dismissals table (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v14: Add steps column to index_events (pipeline observability)
+        if (currentVersion < 14) {
+            const hasSteps = db.prepare(`SELECT COUNT(*) as cnt FROM pragma_table_info('index_events') WHERE name = 'steps'`).get();
+            if (hasSteps.cnt === 0) {
+                db.exec('ALTER TABLE index_events ADD COLUMN steps TEXT');
+            }
+        }
+        // v15: suggestion_events table (pipeline observability audit log)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v16: note_links table (forward-link persistence for diff-based feedback)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v17: entity_changes table (entity field change audit log)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v18: note_tags table (tag persistence for diff-based feedback)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v19: note_link_history table (wikilink survival tracking for positive feedback)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v20: note_moves table (records file renames/moves detected by the watcher)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v21: description TEXT column on entities table
+        if (currentVersion < 21) {
+            const hasDesc = db.prepare(`SELECT COUNT(*) as cnt FROM pragma_table_info('entities') WHERE name = 'description'`).get();
+            if (hasDesc.cnt === 0) {
+                db.exec('ALTER TABLE entities ADD COLUMN description TEXT');
+            }
+        }
+        // v22: Edge weight columns on note_links table
+        if (currentVersion < 22) {
+            const hasWeight = db.prepare(`SELECT COUNT(*) as cnt FROM pragma_table_info('note_links') WHERE name = 'weight'`).get();
+            if (hasWeight.cnt === 0) {
+                db.exec('ALTER TABLE note_links ADD COLUMN weight REAL NOT NULL DEFAULT 1.0');
+                db.exec('ALTER TABLE note_links ADD COLUMN weight_updated_at INTEGER');
+            }
+        }
+        // v23: Case-insensitive unique index on wikilink_applications
+        if (currentVersion < 23) {
+            db.exec('DROP INDEX IF EXISTS idx_wl_apps_unique');
+            db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_wl_apps_unique ON wikilink_applications(entity COLLATE NOCASE, note_path)');
+        }
+        // v24: corrections table (persistent correction records)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v25: confidence column on wikilink_feedback (signal quality weighting)
+        if (currentVersion < 25) {
+            const hasConfidence = db.prepare(`SELECT COUNT(*) as cnt FROM pragma_table_info('wikilink_feedback') WHERE name = 'confidence'`).get();
+            if (hasConfidence.cnt === 0) {
+                db.exec('ALTER TABLE wikilink_feedback ADD COLUMN confidence REAL NOT NULL DEFAULT 1.0');
+            }
+        }
+        // v26: memories table, memories_fts, session_summaries table
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v27: cooccurrence_cache table (persist co-occurrence index)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v28: content_hashes table (persist watcher content hashes across restarts)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v29: index on wikilink_feedback(note_path) for temporal analysis queries
+        // (created by SCHEMA_SQL above via CREATE INDEX IF NOT EXISTS)
+        // v31: proactive_queue table (deferred proactive linking)
+        // (created by SCHEMA_SQL above via CREATE TABLE IF NOT EXISTS)
+        // v30: token economics columns on tool_invocations
+        if (currentVersion < 30) {
+            const hasResponseTokens = db.prepare(`SELECT COUNT(*) as cnt FROM pragma_table_info('tool_invocations') WHERE name = 'response_tokens'`).get();
+            if (hasResponseTokens.cnt === 0) {
+                db.exec('ALTER TABLE tool_invocations ADD COLUMN response_tokens INTEGER');
+                db.exec('ALTER TABLE tool_invocations ADD COLUMN baseline_tokens INTEGER');
+            }
+        }
+        // v32: Drop composite PRIMARY KEY on entity_changes (was causing UNIQUE constraint
+        // crashes when two changes hit the same entity+field within one second).
+        // Recreate as rowid table — audit log doesn't need dedup.
+        if (currentVersion < 32) {
+            const hasTable = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='entity_changes'`).get();
+            if (hasTable) {
+                db.exec(`
+          CREATE TABLE IF NOT EXISTS entity_changes_new (
+            entity TEXT NOT NULL,
+            field TEXT NOT NULL,
+            old_value TEXT,
+            new_value TEXT,
+            changed_at TEXT NOT NULL DEFAULT (datetime('now'))
+          );
+          INSERT INTO entity_changes_new SELECT entity, field, old_value, new_value, changed_at FROM entity_changes;
+          DROP TABLE entity_changes;
+          ALTER TABLE entity_changes_new RENAME TO entity_changes;
+        `);
+            }
+        }
+        // v33: performance_benchmarks table (longitudinal tracking)
+        if (currentVersion < 33) {
+            const hasTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='performance_benchmarks'").get();
+            if (!hasTable) {
+                db.exec(`
+          CREATE TABLE performance_benchmarks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            timestamp INTEGER NOT NULL,
+            version TEXT NOT NULL,
+            benchmark TEXT NOT NULL,
+            mean_ms REAL NOT NULL,
+            p50_ms REAL,
+            p95_ms REAL,
+            iterations INTEGER NOT NULL DEFAULT 1
+          );
+          CREATE INDEX idx_perf_bench_ts ON performance_benchmarks(timestamp);
+          CREATE INDEX idx_perf_bench_name ON performance_benchmarks(benchmark, timestamp);
+        `);
+            }
+            db.prepare('INSERT OR REPLACE INTO schema_version (version) VALUES (?)').run(33);
+        }
+        // v34: Fix entities_fts — switch from content='entities' to contentless
+        // The old FTS5 declared content='entities' with column 'aliases', but the entities
+        // table has 'aliases_json'. This caused "no such column: T.aliases" on every query.
+        // Contentless FTS5 eliminates the column name dependency; triggers handle sync.
+        if (currentVersion < 34) {
+            // Drop old FTS5 and shadow tables
+            db.exec(`DROP TABLE IF EXISTS entities_fts`);
+            db.exec(`DROP TABLE IF EXISTS entities_fts_data`);
+            db.exec(`DROP TABLE IF EXISTS entities_fts_idx`);
+            db.exec(`DROP TABLE IF EXISTS entities_fts_docsize`);
+            db.exec(`DROP TABLE IF EXISTS entities_fts_config`);
+            // Recreate as contentless
+            db.exec(`
+        CREATE VIRTUAL TABLE entities_fts USING fts5(
+          name, aliases, category,
+          content='',
+          tokenize='porter unicode61'
+        )
+      `);
+            // Drop and recreate triggers (unchanged logic, now targeting contentless table)
+            db.exec(`DROP TRIGGER IF EXISTS entities_ai`);
+            db.exec(`DROP TRIGGER IF EXISTS entities_ad`);
+            db.exec(`DROP TRIGGER IF EXISTS entities_au`);
+            db.exec(`
+        CREATE TRIGGER entities_ai AFTER INSERT ON entities BEGIN
+          INSERT INTO entities_fts(rowid, name, aliases, category)
+          VALUES (
+            new.id,
+            new.name,
+            COALESCE((SELECT group_concat(value, ' ') FROM json_each(new.aliases_json)), ''),
+            new.category
+          );
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER entities_ad AFTER DELETE ON entities BEGIN
+          INSERT INTO entities_fts(entities_fts, rowid, name, aliases, category)
+          VALUES (
+            'delete',
+            old.id,
+            old.name,
+            COALESCE((SELECT group_concat(value, ' ') FROM json_each(old.aliases_json)), ''),
+            old.category
+          );
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER entities_au AFTER UPDATE ON entities BEGIN
+          INSERT INTO entities_fts(entities_fts, rowid, name, aliases, category)
+          VALUES (
+            'delete',
+            old.id,
+            old.name,
+            COALESCE((SELECT group_concat(value, ' ') FROM json_each(old.aliases_json)), ''),
+            old.category
+          );
+          INSERT INTO entities_fts(rowid, name, aliases, category)
+          VALUES (
+            new.id,
+            new.name,
+            COALESCE((SELECT group_concat(value, ' ') FROM json_each(new.aliases_json)), ''),
+            new.category
+          );
+        END
+      `);
+            // Populate FTS from existing entities
+            db.exec(`
+        INSERT INTO entities_fts(rowid, name, aliases, category)
+        SELECT id, name,
+          COALESCE((SELECT group_concat(value, ' ') FROM json_each(aliases_json)), ''),
+          category
+        FROM entities
+      `);
+            db.prepare('INSERT OR REPLACE INTO schema_version (version) VALUES (?)').run(34);
+        }
+        db.prepare('INSERT OR IGNORE INTO schema_version (version) VALUES (?)').run(SCHEMA_VERSION);
+    }
+}
+// =============================================================================
+// Database File Management
+// =============================================================================
+export function deleteStateDbFiles(dbPath) {
+    for (const suffix of ['', '-wal', '-shm']) {
+        const p = dbPath + suffix;
+        if (fs.existsSync(p))
+            fs.unlinkSync(p);
+    }
+}
+/** Back up state.db before opening (skip if missing or 0 bytes). */
+export function backupStateDb(dbPath) {
+    try {
+        if (!fs.existsSync(dbPath))
+            return;
+        const stat = fs.statSync(dbPath);
+        if (stat.size === 0)
+            return;
+        fs.copyFileSync(dbPath, dbPath + '.backup');
+    }
+    catch (err) {
+        console.error(`[vault-core] Failed to back up state.db: ${err instanceof Error ? err.message : err}`);
+    }
+}
+/** Preserve a corrupted database for inspection before deleting. */
+export function preserveCorruptedDb(dbPath) {
+    try {
+        if (fs.existsSync(dbPath)) {
+            fs.copyFileSync(dbPath, dbPath + '.corrupt');
+            console.error(`[vault-core] Corrupted database preserved at ${dbPath}.corrupt`);
+        }
+    }
+    catch {
+        // Best effort — don't block recovery
+    }
+}
+//# sourceMappingURL=migrations.js.map

package/dist/queries.d.ts

@@ -0,0 +1,209 @@
+/**
+ * SQLite Query Functions
+ *
+ * All database query operations: entity search, recency, write state,
+ * flywheel config, merge dismissals, metadata, vault index cache,
+ * and content hashes.
+ */
+import type { EntityIndex } from './types.js';
+import type { StateDb, EntitySearchResult, RecencyRow, StateDbMetadata } from './sqlite.js';
+/**
+ * Search entities using FTS5 with porter stemming
+ *
+ * @param stateDb - State database instance
+ * @param query - Search query (supports FTS5 syntax)
+ * @param limit - Maximum results to return
+ * @returns Array of matching entities with relevance scores
+ */
+export declare function searchEntities(stateDb: StateDb, query: string, limit?: number): EntitySearchResult[];
+/**
+ * Search entities by prefix for autocomplete
+ *
+ * @param stateDb - State database instance
+ * @param prefix - Prefix to search for
+ * @param limit - Maximum results to return
+ */
+export declare function searchEntitiesPrefix(stateDb: StateDb, prefix: string, limit?: number): EntitySearchResult[];
+/**
+ * Get entity by exact name (case-insensitive)
+ */
+export declare function getEntityByName(stateDb: StateDb, name: string): EntitySearchResult | null;
+/**
+ * Get all entities from the database
+ */
+export declare function getAllEntitiesFromDb(stateDb: StateDb): EntitySearchResult[];
+/**
+ * Convert database entities back to EntityIndex format
+ */
+export declare function getEntityIndexFromDb(stateDb: StateDb): EntityIndex;
+/**
+ * Get entities that have a given alias (case-insensitive)
+ *
+ * @param stateDb - State database instance
+ * @param alias - Alias to search for (case-insensitive)
+ * @returns Array of matching entities
+ */
+export declare function getEntitiesByAlias(stateDb: StateDb, alias: string): EntitySearchResult[];
+/**
+ * Record a mention of an entity
+ */
+export declare function recordEntityMention(stateDb: StateDb, entityName: string, mentionedAt?: Date): void;
+/**
+ * Get recency info for an entity
+ */
+export declare function getEntityRecency(stateDb: StateDb, entityName: string): RecencyRow | null;
+/**
+ * Get all recency data ordered by most recent
+ */
+export declare function getAllRecency(stateDb: StateDb): RecencyRow[];
+/**
+ * Set a write state value
+ */
+export declare function setWriteState(stateDb: StateDb, key: string, value: unknown): void;
+/**
+ * Get a write state value
+ */
+export declare function getWriteState<T>(stateDb: StateDb, key: string): T | null;
+/**
+ * Delete a write state key
+ */
+export declare function deleteWriteState(stateDb: StateDb, key: string): void;
+/** Flywheel config row from database */
+export interface FlywheelConfigRow {
+    key: string;
+    value: string;
+}
+/**
+ * Set a flywheel config value
+ */
+export declare function setFlywheelConfig(stateDb: StateDb, key: string, value: unknown): void;
+/**
+ * Get a flywheel config value
+ */
+export declare function getFlywheelConfig<T>(stateDb: StateDb, key: string): T | null;
+/**
+ * Get all flywheel config values as an object
+ */
+export declare function getAllFlywheelConfig(stateDb: StateDb): Record<string, unknown>;
+/**
+ * Delete a flywheel config key
+ */
+export declare function deleteFlywheelConfig(stateDb: StateDb, key: string): void;
+/**
+ * Save entire Flywheel config object to database
+ * Stores each top-level key as a separate row
+ */
+export declare function saveFlywheelConfigToDb(stateDb: StateDb, config: Record<string, unknown>): void;
+/**
+ * Load Flywheel config from database and reconstruct as typed object
+ */
+export declare function loadFlywheelConfigFromDb(stateDb: StateDb): Record<string, unknown> | null;
+/**
+ * Record a merge dismissal so the pair never reappears in suggestions.
+ */
+export declare function recordMergeDismissal(db: StateDb, sourcePath: string, targetPath: string, sourceName: string, targetName: string, reason: string): void;
+/**
+ * Get all dismissed merge pair keys for filtering.
+ */
+export declare function getDismissedMergePairs(db: StateDb): Set<string>;
+/**
+ * Get database metadata
+ */
+export declare function getStateDbMetadata(stateDb: StateDb): StateDbMetadata;
+/**
+ * Check if entity data is stale (older than threshold)
+ */
+export declare function isEntityDataStale(stateDb: StateDb, thresholdMs?: number): boolean;
+/**
+ * Escape special FTS5 characters in a query
+ */
+export declare function escapeFts5Query(query: string): string;
+/**
+ * Rebuild the entities_fts index from the entities table.
+ * Contentless FTS5 tables don't support the 'rebuild' command,
+ * so we manually delete all entries and re-insert from the entities table.
+ */
+export declare function rebuildEntitiesFts(stateDb: StateDb): void;
+/**
+ * Check if the state database exists for a vault
+ */
+export declare function stateDbExists(vaultPath: string): boolean;
+/**
+ * Delete the state database (for testing or reset)
+ */
+export declare function deleteStateDb(vaultPath: string): void;
+/** Serializable VaultIndex for caching */
+export interface VaultIndexCacheData {
+    notes: Array<{
+        path: string;
+        title: string;
+        aliases: string[];
+        frontmatter: Record<string, unknown>;
+        outlinks: Array<{
+            target: string;
+            alias?: string;
+            line: number;
+        }>;
+        tags: string[];
+        modified: number;
+        created?: number;
+    }>;
+    backlinks: Array<[string, Array<{
+        source: string;
+        line: number;
+        context?: string;
+    }>]>;
+    entities: Array<[string, string]>;
+    prospects?: Array<[string, {
+        displayName: string;
+        backlinkCount: number;
+    }]>;
+    tags: Array<[string, string[]]>;
+    builtAt: number;
+}
+/** Cache metadata */
+export interface VaultIndexCacheInfo {
+    builtAt: Date;
+    noteCount: number;
+    version: number;
+}
+/**
+ * Save VaultIndex to cache
+ *
+ * @param stateDb - State database instance
+ * @param indexData - Serialized VaultIndex data
+ */
+export declare function saveVaultIndexCache(stateDb: StateDb, indexData: VaultIndexCacheData): void;
+/**
+ * Load VaultIndex from cache
+ *
+ * @param stateDb - State database instance
+ * @returns Cached VaultIndex data or null if not found
+ */
+export declare function loadVaultIndexCache(stateDb: StateDb): VaultIndexCacheData | null;
+/**
+ * Get cache metadata without loading full data
+ */
+export declare function getVaultIndexCacheInfo(stateDb: StateDb): VaultIndexCacheInfo | null;
+/**
+ * Clear the vault index cache
+ */
+export declare function clearVaultIndexCache(stateDb: StateDb): void;
+/**
+ * Check if cache is valid (not too old and note count matches)
+ *
+ * @param stateDb - State database instance
+ * @param actualNoteCount - Current number of notes in vault
+ * @param maxAgeMs - Maximum cache age in milliseconds (default 24 hours)
+ */
+export declare function isVaultIndexCacheValid(stateDb: StateDb, actualNoteCount: number, maxAgeMs?: number): boolean;
+/** Load all persisted content hashes */
+export declare function loadContentHashes(stateDb: StateDb): Map<string, string>;
+/** Persist hash changes from a watcher batch (upserts + deletes in one transaction) */
+export declare function saveContentHashBatch(stateDb: StateDb, upserts: Array<{
+    path: string;
+    hash: string;
+}>, deletes: string[]): void;
+/** Rename a hash entry (for file renames) */
+export declare function renameContentHash(stateDb: StateDb, oldPath: string, newPath: string): void;
+//# sourceMappingURL=queries.d.ts.map