@cosmocoder/mcp-web-docs 2.0.9 → 2.0.11

package/build/storage/storage.js

@@ -2,12 +2,20 @@ import sqlite3 from 'sqlite3';
 import { open } from 'sqlite';
 import * as lancedb from '@lancedb/lancedb';
 import { PhraseQuery, MatchQuery, BooleanQuery, Occur } from '@lancedb/lancedb';
-import { Field, FixedSizeList, Float32, Schema, Utf8, Int32 } from 'apache-arrow';
+import { Bool, Field, FixedSizeList, Float32, Schema, Utf8, Int32 } from 'apache-arrow';
 import QuickLRU from 'quick-lru';
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
 import { mkdir } from 'fs/promises';
 import { dirname } from 'path';
 import { logger } from '../util/logger.js';
 import { escapeFilterValue } from '../util/security.js';
+const SEARCH_VISIBILITY_RETRY_WINDOW_MS = 2_000;
+const SEARCH_VISIBILITY_RETRY_MAX_BACKOFF_MS = 25;
+// Long enough for normal Lance writes; renewed while staging is in flight.
+const REPLACEMENT_LEASE_MS = 30_000;
+const REPLACEMENT_HEARTBEAT_MS = 10_000;
+const REPLACEMENT_LEASE_POLL_MS = 25;
 /**
  * Preprocesses a search query - keeps it generic for any documentation type.
  * Only extracts explicitly quoted phrases, otherwise passes through to LanceDB's
@@ -35,10 +43,14 @@ export class DocumentStore {
     vectorDbPath;
     embeddings;
     sqliteDb;
+    sqliteReadDb;
+    sqliteLeaseDb;
     lanceConn;
     lanceTable;
     searchCache;
     ftsIndexCreated = false;
+    // ponytail: one per-store FIFO; split only if mutation throughput becomes a measured bottleneck.
+    mutationTail = Promise.resolve();
     constructor(dbPath, vectorDbPath, embeddings, maxCacheSize = 1000) {
         this.dbPath = dbPath;
         this.vectorDbPath = vectorDbPath;
@@ -50,6 +62,28 @@ export class DocumentStore {
         });
         this.searchCache = new QuickLRU({ maxSize: maxCacheSize });
     }
+    runMutation(mutation, signal) {
+        if (signal?.aborted) {
+            return Promise.reject(signal.reason);
+        }
+        let onAbort;
+        const result = this.mutationTail.then(() => {
+            if (onAbort) {
+                signal?.removeEventListener('abort', onAbort);
+            }
+            signal?.throwIfAborted();
+            return mutation();
+        });
+        this.mutationTail = result.then(() => undefined, () => undefined);
+        if (!signal) {
+            return result;
+        }
+        return new Promise((resolve, reject) => {
+            onAbort = () => reject(signal.reason);
+            signal.addEventListener('abort', onAbort, { once: true });
+            result.then(resolve, reject);
+        });
+    }
     async initialize() {
         logger.debug(`[DocumentStore] Starting initialization with paths:`, {
             dbPath: this.dbPath,
@@ -95,6 +129,20 @@ export class DocumentStore {
       `);
             // Run database migrations
             await this.runMigrations();
+            // Reads use a separate connection so they never observe this instance's
+            // uncommitted publication transaction.
+            this.sqliteReadDb = await open({
+                filename: this.dbPath,
+                driver: sqlite3.Database,
+            });
+            await this.sqliteReadDb.exec('PRAGMA busy_timeout = 5000;');
+            // Lease renewal uses its own writer so slow Lance staging cannot block
+            // heartbeats behind this instance's publication connection.
+            this.sqliteLeaseDb = await open({
+                filename: this.dbPath,
+                driver: sqlite3.Database,
+            });
+            await this.sqliteLeaseDb.exec('PRAGMA busy_timeout = 5000;');
             // Initialize LanceDB with error handling
             try {
                 logger.debug(`[DocumentStore] Connecting to LanceDB at ${this.vectorDbPath}`);
@@ -108,6 +156,8 @@ export class DocumentStore {
                     // Define schema using Apache Arrow
                     const vectorType = new FixedSizeList(this.embeddings.dimensions, new Field('item', new Float32(), true));
                     const schema = new Schema([
+                        new Field('generation', new Utf8(), false),
+                        new Field('published', new Bool(), false),
                         new Field('url', new Utf8(), false),
                         new Field('title', new Utf8(), false),
                         new Field('content', new Utf8(), false),
@@ -127,15 +177,17 @@ export class DocumentStore {
                     // Create empty table with schema
                     this.lanceTable = await this.lanceConn.createEmptyTable('chunks', schema, { mode: 'create' });
                     logger.debug(`[DocumentStore] New chunks table created successfully`);
-                    // Create FTS index for better text search
-                    await this.createFTSIndex();
                 }
                 else {
                     logger.debug(`[DocumentStore] Using existing chunks table`);
                     this.lanceTable = await this.lanceConn.openTable('chunks');
-                    // Try to create FTS index if it doesn't exist
-                    await this.createFTSIndex();
+                    // Tables created before replacement journaling need a generation marker.
+                    await this.ensureLanceColumn('generation', "'legacy'");
+                    await this.ensureLanceColumn('published', 'true');
                 }
+                await this.recoverDocumentReplacements();
+                await this.reapOrphanedUnpublishedGenerations();
+                await this.createFTSIndex();
                 // Verify table is accessible
                 const rowCount = await this.lanceTable.countRows();
                 logger.debug(`[DocumentStore] Chunks table initialized, contains ${rowCount} rows`);
@@ -148,9 +200,58 @@ export class DocumentStore {
         }
         catch (error) {
             logger.error('[DocumentStore] Error initializing storage:', error);
+            try {
+                await this.close();
+            }
+            catch (cleanupError) {
+                logger.warn('[DocumentStore] Storage cleanup after initialization failure was incomplete:', cleanupError);
+            }
             throw error;
         }
     }
+    async close() {
+        const errors = [];
+        const closeResource = async (name, resource, clear) => {
+            if (!resource) {
+                return;
+            }
+            try {
+                await resource.close();
+                clear();
+            }
+            catch (error) {
+                errors.push(error);
+                logger.warn(`[DocumentStore] Failed to close ${name}:`, error);
+            }
+        };
+        await closeResource('LanceDB table', this.lanceTable, () => (this.lanceTable = undefined));
+        await closeResource('LanceDB connection', this.lanceConn, () => (this.lanceConn = undefined));
+        await closeResource('SQLite reader', this.sqliteReadDb, () => (this.sqliteReadDb = undefined));
+        await closeResource('SQLite lease writer', this.sqliteLeaseDb, () => (this.sqliteLeaseDb = undefined));
+        await closeResource('SQLite writer', this.sqliteDb, () => (this.sqliteDb = undefined));
+        if (errors.length > 0) {
+            throw new AggregateError(errors, 'Failed to close one or more storage resources');
+        }
+    }
+    async ensureLanceColumn(name, valueSql) {
+        if (!this.lanceTable) {
+            throw new Error('Storage not initialized');
+        }
+        const schema = await this.lanceTable.schema();
+        if (schema.fields.some((field) => field.name === name)) {
+            return;
+        }
+        try {
+            await this.lanceTable.addColumns([{ name, valueSql }]);
+        }
+        catch (error) {
+            await this.lanceTable.checkoutLatest();
+            const refreshedSchema = await this.lanceTable.schema();
+            if (!refreshedSchema.fields.some((field) => field.name === name)) {
+                throw error;
+            }
+        }
+    }
     /**
      * Database migrations list.
      * Each migration has a unique version number and SQL statements to execute.
@@ -213,6 +314,20 @@ export class DocumentStore {
         CREATE INDEX IF NOT EXISTS idx_collection_documents_url ON collection_documents(url);
       `,
         },
+        {
+            version: 5,
+            description: 'Add document replacement journal',
+            sql: `
+        CREATE TABLE IF NOT EXISTS document_replacements (
+          url TEXT PRIMARY KEY,
+          generation TEXT NOT NULL,
+          state TEXT NOT NULL CHECK (state IN ('prepared', 'published', 'deleting')),
+          owner_id TEXT NOT NULL,
+          lease_expires_at INTEGER NOT NULL,
+          cleanup_generations TEXT NOT NULL
+        );
+      `,
+        },
     ];
     /**
      * Run pending database migrations.
@@ -264,7 +379,7 @@ export class DocumentStore {
                     }
                 }
                 // Record successful migration
-                await this.sqliteDb.run('INSERT INTO schema_migrations (version, applied_at, description) VALUES (?, ?, ?)', [
+                await this.sqliteDb.run('INSERT OR IGNORE INTO schema_migrations (version, applied_at, description) VALUES (?, ?, ?)', [
                     migration.version,
                     new Date().toISOString(),
                     migration.description,
@@ -277,87 +392,380 @@ export class DocumentStore {
             }
         }
     }
-    async addDocument(doc) {
+    /**
+     * Stage a complete generation in LanceDB, then publish it through SQLite.
+     * Search only reads the published generation, so failed staging never leaks.
+     */
+    addDocument(doc, options = {}) {
+        return this.runMutation(() => this.addDocumentUnlocked(doc, options), options.signal);
+    }
+    async addDocumentUnlocked(doc, options) {
+        const { signal, tags } = options;
         logger.debug(`[DocumentStore] Starting addDocument for:`, {
             url: doc.metadata.url,
             title: doc.metadata.title,
             chunks: doc.chunks.length,
         });
-        // Add diagnostic logging for vector dimensions
         if (doc.chunks.length > 0) {
             logger.debug(`[DocumentStore] Sample vector dimensions: ${doc.chunks[0].vector.length}`);
             logger.debug(`[DocumentStore] Sample vector first 5 values: ${doc.chunks[0].vector.slice(0, 5)}`);
         }
-        // Validate storage initialization
-        if (!this.sqliteDb) {
-            logger.debug('[DocumentStore] SQLite not initialized during addDocument');
-            throw new Error('SQLite storage not initialized');
-        }
-        if (!this.lanceTable) {
-            logger.debug('[DocumentStore] LanceDB not initialized during addDocument');
-            throw new Error('LanceDB storage not initialized');
+        const sqliteDb = this.sqliteDb;
+        const sqliteReadDb = this.sqliteReadDb;
+        if (!sqliteDb || !sqliteReadDb || !this.sqliteLeaseDb || !this.lanceTable) {
+            throw new Error('Storage not initialized');
         }
+        const url = doc.metadata.url;
+        const generation = randomUUID();
+        const ownerId = randomUUID();
+        const newRows = this.toLanceRows(doc, generation);
+        const lease = await this.acquireReplacementLease(url, generation, ownerId, signal);
+        let stopHeartbeat = () => Promise.resolve();
+        let transactionStarted = false;
         try {
-            // Check if document already exists
-            const existing = await this.getDocument(doc.metadata.url);
-            if (existing) {
-                logger.debug(`[DocumentStore] Existing document found, will update:`, existing);
+            const preparedJournal = await this.captureCleanupGenerations(lease);
+            stopHeartbeat = this.startReplacementHeartbeat(preparedJournal);
+            // The committed journal advances reader data_version before staged rows
+            // can become queryable.
+            signal?.throwIfAborted();
+            if (newRows.length > 0) {
+                await this.lanceTable.add(newRows);
             }
-            logger.debug(`[DocumentStore] Starting SQLite transaction`);
-            await this.sqliteDb.run('BEGIN TRANSACTION');
-            // Add metadata to SQLite
-            await this.sqliteDb.run('INSERT OR REPLACE INTO documents (url, title, favicon, last_indexed, requires_auth, auth_domain, version) VALUES (?, ?, ?, ?, ?, ?, ?)', [
-                doc.metadata.url,
-                doc.metadata.title,
-                doc.metadata.favicon,
-                doc.metadata.lastIndexed.toISOString(),
-                doc.metadata.requiresAuth ? 1 : 0,
-                doc.metadata.authDomain || null,
-                doc.metadata.version || null,
-            ]);
-            logger.debug(`[DocumentStore] Added metadata to SQLite (requiresAuth: ${doc.metadata.requiresAuth}, authDomain: ${doc.metadata.authDomain}, version: ${doc.metadata.version})`);
-            // Delete existing chunks for this document (using escaped value to prevent injection)
-            await this.lanceTable.delete(`url = '${escapeFilterValue(doc.metadata.url)}'`);
-            logger.debug(`[DocumentStore] Deleted existing chunks`);
-            // Add new chunks to LanceDB
-            const rows = doc.chunks.map((chunk) => ({
-                url: doc.metadata.url,
-                title: doc.metadata.title,
-                content: chunk.content,
-                path: chunk.path,
-                startLine: chunk.startLine,
-                endLine: chunk.endLine,
-                vector: chunk.vector,
-                type: chunk.metadata.type,
-                lastUpdated: new Date().toISOString(),
-                version: '',
-                framework: '',
-                language: '',
-                // Serialize code blocks and props as JSON strings
-                codeBlocks: JSON.stringify(chunk.metadata.codeBlocks || []),
-                props: JSON.stringify(chunk.metadata.props || []),
-            }));
-            logger.debug(`[DocumentStore] Adding ${rows.length} chunks to LanceDB`);
-            await this.lanceTable.add(rows);
-            // Verify data was added
-            const rowCount = await this.lanceTable.countRows();
-            logger.debug(`[DocumentStore] Table now contains ${rowCount} rows`);
-            // Commit transaction
-            await this.sqliteDb.run('COMMIT');
-            logger.debug(`[DocumentStore] Committed transaction`);
-            // Clear search cache for this URL
-            this.clearCacheForUrl(doc.metadata.url);
+            await this.renewReplacementLease(preparedJournal);
+            await this.lanceTable.update({
+                where: `url = '${escapeFilterValue(url)}' AND generation = '${escapeFilterValue(generation)}'`,
+                values: { published: true },
+            });
+            await stopHeartbeat();
+            await this.renewReplacementLease(preparedJournal);
+            await sqliteDb.run('BEGIN TRANSACTION');
+            transactionStarted = true;
+            signal?.throwIfAborted();
+            await this.upsertMetadata(doc.metadata);
+            if (tags !== undefined) {
+                await this.replaceDocumentTags(url, tags);
+            }
+            signal?.throwIfAborted();
+            const publicationLeaseExpiresAt = Date.now() + REPLACEMENT_LEASE_MS;
+            const publication = await sqliteDb.run(`UPDATE document_replacements
+         SET state = 'published', lease_expires_at = ?
+         WHERE url = ? AND generation = ? AND owner_id = ? AND state = 'prepared'`, [publicationLeaseExpiresAt, url, generation, ownerId]);
+            if (publication.changes !== 1) {
+                throw new Error(`Replacement lease lost for ${url}`);
+            }
+            signal?.throwIfAborted();
+            await sqliteDb.run('COMMIT');
+            transactionStarted = false;
+            await this.finishDocumentReplacementBestEffort({
+                ...preparedJournal,
+                state: 'published',
+                lease_expires_at: publicationLeaseExpiresAt,
+            });
         }
         catch (error) {
-            // Rollback on error
-            if (this.sqliteDb) {
-                await this.sqliteDb.run('ROLLBACK');
+            await stopHeartbeat().catch((heartbeatError) => {
+                logger.warn(`[DocumentStore] Replacement heartbeat stopped with an error for ${url}:`, heartbeatError);
+            });
+            if (transactionStarted) {
+                try {
+                    await sqliteDb.run('ROLLBACK');
+                }
+                catch {
+                    // The transaction may already have been rolled back by SQLite.
+                }
+            }
+            let durableJournal;
+            try {
+                durableJournal = await sqliteReadDb.get('SELECT * FROM document_replacements WHERE url = ? AND generation = ? AND owner_id = ?', [url, generation, ownerId]);
+            }
+            catch (journalError) {
+                logger.error(`[DocumentStore] Could not determine durable replacement state for ${url}:`, journalError);
+                throw error;
+            }
+            if (durableJournal) {
+                await this.finishDocumentReplacementBestEffort(durableJournal);
+            }
+            else {
+                await this.deleteUnpublishedGenerationBestEffort(url, generation);
             }
             logger.error('[DocumentStore] Error adding document:', error);
             throw error;
         }
     }
+    async acquireReplacementLease(url, generation, ownerId, signal) {
+        if (!this.sqliteLeaseDb) {
+            throw new Error('Storage not initialized');
+        }
+        while (true) {
+            signal?.throwIfAborted();
+            const now = Date.now();
+            const leaseExpiresAt = now + REPLACEMENT_LEASE_MS;
+            const insertion = await this.sqliteLeaseDb.run(`INSERT OR IGNORE INTO document_replacements
+           (url, generation, state, owner_id, lease_expires_at, cleanup_generations)
+         VALUES (?, ?, 'prepared', ?, ?, '[]')`, [url, generation, ownerId, leaseExpiresAt]);
+            if (insertion.changes === 1) {
+                return { url, generation, state: 'prepared', owner_id: ownerId, lease_expires_at: leaseExpiresAt, cleanup_generations: '[]' };
+            }
+            const existing = await this.sqliteLeaseDb.get('SELECT * FROM document_replacements WHERE url = ?', [url]);
+            if (!existing) {
+                continue;
+            }
+            if (existing.lease_expires_at <= now) {
+                const claim = await this.sqliteLeaseDb.run(`UPDATE document_replacements
+           SET owner_id = ?, lease_expires_at = ?
+           WHERE url = ? AND generation = ? AND owner_id = ? AND state = ? AND lease_expires_at <= ?`, [ownerId, leaseExpiresAt, url, existing.generation, existing.owner_id, existing.state, now]);
+                if (claim.changes === 1) {
+                    await this.finishDocumentReplacement({ ...existing, owner_id: ownerId, lease_expires_at: leaseExpiresAt });
+                }
+                continue;
+            }
+            await delay(Math.max(1, Math.min(REPLACEMENT_LEASE_POLL_MS, existing.lease_expires_at - now)), undefined, { signal });
+        }
+    }
+    async captureCleanupGenerations(journal) {
+        if (!this.lanceTable || !this.sqliteLeaseDb) {
+            throw new Error('Storage not initialized');
+        }
+        await this.lanceTable.checkoutLatest();
+        const rows = await this.lanceTable
+            .query()
+            .where(`url = '${escapeFilterValue(journal.url)}'`)
+            .select(['generation'])
+            .toArray();
+        const cleanupGenerations = JSON.stringify([...new Set(rows.map((row) => String(row.generation)))].sort());
+        const leaseExpiresAt = Date.now() + REPLACEMENT_LEASE_MS;
+        const update = await this.sqliteLeaseDb.run(`UPDATE document_replacements
+       SET cleanup_generations = ?, lease_expires_at = ?
+       WHERE url = ? AND generation = ? AND owner_id = ? AND state = 'prepared'`, [cleanupGenerations, leaseExpiresAt, journal.url, journal.generation, journal.owner_id]);
+        if (update.changes !== 1) {
+            throw new Error(`Replacement lease lost for ${journal.url}`);
+        }
+        return { ...journal, cleanup_generations: cleanupGenerations, lease_expires_at: leaseExpiresAt };
+    }
+    startReplacementHeartbeat(journal) {
+        let stopped = false;
+        let timer;
+        let renewal = Promise.resolve();
+        let renewalError;
+        const schedule = () => {
+            timer = setTimeout(() => {
+                renewal = this.renewReplacementLease(journal)
+                    .catch((error) => {
+                    renewalError = error;
+                })
+                    .finally(() => {
+                    if (!stopped && !renewalError) {
+                        schedule();
+                    }
+                });
+            }, REPLACEMENT_HEARTBEAT_MS);
+            timer.unref();
+        };
+        schedule();
+        return async () => {
+            stopped = true;
+            if (timer) {
+                clearTimeout(timer);
+            }
+            await renewal;
+            if (renewalError) {
+                throw renewalError;
+            }
+        };
+    }
+    async renewReplacementLease(journal) {
+        if (!this.sqliteLeaseDb) {
+            throw new Error('Storage not initialized');
+        }
+        const renewal = await this.sqliteLeaseDb.run(`UPDATE document_replacements
+       SET lease_expires_at = ?
+       WHERE url = ? AND generation = ? AND owner_id = ? AND state = ?`, [Date.now() + REPLACEMENT_LEASE_MS, journal.url, journal.generation, journal.owner_id, journal.state]);
+        if (renewal.changes !== 1) {
+            throw new Error(`Replacement lease lost for ${journal.url}`);
+        }
+    }
+    toLanceRows(doc, generation) {
+        const lastUpdated = new Date().toISOString();
+        return doc.chunks.map((chunk) => ({
+            generation,
+            published: false,
+            url: doc.metadata.url,
+            title: doc.metadata.title,
+            content: chunk.content,
+            path: chunk.path,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            vector: chunk.vector,
+            type: chunk.metadata.type,
+            lastUpdated,
+            version: chunk.metadata.version ?? '',
+            framework: chunk.metadata.framework ?? '',
+            language: chunk.metadata.language ?? '',
+            codeBlocks: JSON.stringify(chunk.metadata.codeBlocks || []),
+            props: JSON.stringify(chunk.metadata.props || []),
+        }));
+    }
+    async upsertMetadata(metadata) {
+        if (!this.sqliteDb) {
+            throw new Error('Storage not initialized');
+        }
+        await this.sqliteDb.run(`INSERT INTO documents (url, title, favicon, last_indexed, requires_auth, auth_domain, version)
+       VALUES (?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(url) DO UPDATE SET
+         title = excluded.title,
+         favicon = excluded.favicon,
+         last_indexed = excluded.last_indexed,
+         requires_auth = excluded.requires_auth,
+         auth_domain = excluded.auth_domain,
+         version = excluded.version`, [
+            metadata.url,
+            metadata.title,
+            metadata.favicon ?? null,
+            metadata.lastIndexed.toISOString(),
+            metadata.requiresAuth ? 1 : 0,
+            metadata.authDomain ?? null,
+            metadata.version ?? null,
+        ]);
+    }
+    async recoverDocumentReplacements() {
+        if (!this.sqliteReadDb || !this.sqliteLeaseDb || !this.lanceTable) {
+            return;
+        }
+        const now = Date.now();
+        const journals = await this.sqliteReadDb.all('SELECT * FROM document_replacements WHERE lease_expires_at <= ?', [now]);
+        for (const journal of journals) {
+            const ownerId = randomUUID();
+            const leaseExpiresAt = Date.now() + REPLACEMENT_LEASE_MS;
+            const claim = await this.sqliteLeaseDb.run(`UPDATE document_replacements
+         SET owner_id = ?, lease_expires_at = ?
+         WHERE url = ? AND generation = ? AND owner_id = ? AND state = ? AND lease_expires_at <= ?`, [ownerId, leaseExpiresAt, journal.url, journal.generation, journal.owner_id, journal.state, Date.now()]);
+            if (claim.changes === 1) {
+                await this.finishDocumentReplacement({ ...journal, owner_id: ownerId, lease_expires_at: leaseExpiresAt });
+                logger.info(`[DocumentStore] Recovered ${journal.state} replacement for ${journal.url}`);
+            }
+        }
+    }
+    async reapOrphanedUnpublishedGenerations() {
+        if (!this.sqliteReadDb || !this.lanceTable) {
+            return;
+        }
+        try {
+            await this.lanceTable.checkoutLatest();
+            const rows = await this.lanceTable.query().where('published = false').select(['generation']).toArray();
+            const orphaned = new Set(rows.map((row) => String(row.generation)));
+            if (orphaned.size === 0) {
+                return;
+            }
+            const journals = await this.sqliteReadDb.all('SELECT generation FROM document_replacements');
+            for (const { generation } of journals) {
+                orphaned.delete(generation);
+            }
+            if (orphaned.size > 0) {
+                const generations = [...orphaned].map((generation) => `'${escapeFilterValue(generation)}'`).join(', ');
+                await this.lanceTable.delete(`published = false AND generation IN (${generations})`);
+            }
+        }
+        catch (error) {
+            logger.warn('[DocumentStore] Orphaned unpublished generation cleanup deferred:', error);
+        }
+    }
+    async finishDocumentReplacement(journal) {
+        if (!this.sqliteLeaseDb || !this.lanceTable) {
+            throw new Error('Storage not initialized');
+        }
+        await this.renewReplacementLease(journal);
+        const url = escapeFilterValue(journal.url);
+        const cleanupGenerations = journal.state === 'prepared' ? [journal.generation] : this.parseCleanupGenerations(journal.cleanup_generations);
+        for (const generation of cleanupGenerations) {
+            await this.lanceTable.delete(`url = '${url}' AND generation = '${escapeFilterValue(generation)}'`);
+        }
+        const deletion = await this.sqliteLeaseDb.run('DELETE FROM document_replacements WHERE url = ? AND generation = ? AND owner_id = ? AND state = ?', [journal.url, journal.generation, journal.owner_id, journal.state]);
+        if (deletion.changes !== 1) {
+            throw new Error(`Replacement lease lost for ${journal.url}`);
+        }
+    }
+    parseCleanupGenerations(value) {
+        const parsed = JSON.parse(value);
+        if (!Array.isArray(parsed) || !parsed.every((generation) => typeof generation === 'string')) {
+            throw new Error('Invalid replacement cleanup generations');
+        }
+        return [...new Set(parsed)];
+    }
+    async deleteUnpublishedGenerationBestEffort(url, generation) {
+        try {
+            await this.lanceTable?.delete(`url = '${escapeFilterValue(url)}' AND generation = '${escapeFilterValue(generation)}' AND published = false`);
+        }
+        catch (error) {
+            logger.warn(`[DocumentStore] Stale replacement generation cleanup deferred for ${url}:`, error);
+        }
+    }
+    async finishDocumentReplacementBestEffort(journal) {
+        try {
+            await this.finishDocumentReplacement(journal);
+        }
+        catch (error) {
+            logger.warn(`[DocumentStore] ${journal.state} replacement cleanup deferred for ${journal.url}:`, error);
+        }
+    }
+    async getJournalVisibilityFilter() {
+        if (!this.sqliteReadDb) {
+            throw new Error('Storage not initialized');
+        }
+        const journals = await this.sqliteReadDb.all('SELECT * FROM document_replacements');
+        const journalFilters = journals
+            .map(({ url, generation, state }) => {
+            const escapedUrl = escapeFilterValue(url);
+            const escapedGeneration = escapeFilterValue(generation);
+            if (state === 'prepared') {
+                return `(url != '${escapedUrl}' OR generation != '${escapedGeneration}')`;
+            }
+            if (state === 'published') {
+                return `(url != '${escapedUrl}' OR generation = '${escapedGeneration}')`;
+            }
+            return `url != '${escapedUrl}'`;
+        })
+            .join(' AND ');
+        return journalFilters ? `published = true AND ${journalFilters}` : 'published = true';
+    }
+    async withStableSearch(operation) {
+        let deadline;
+        let backoffMs = 1;
+        while (true) {
+            const dataVersion = await this.getDataVersion();
+            if (!this.lanceTable) {
+                throw new Error('Storage not initialized');
+            }
+            // Other DocumentStore instances hold their own Lance table handle; refresh
+            // it to the latest committed table version before evaluating this snapshot.
+            await this.lanceTable.checkoutLatest();
+            const result = await operation(dataVersion);
+            if (dataVersion === (await this.getDataVersion())) {
+                return { result, dataVersion };
+            }
+            deadline ??= Date.now() + SEARCH_VISIBILITY_RETRY_WINDOW_MS;
+            if (Date.now() >= deadline) {
+                throw new Error('Search visibility kept changing; please retry');
+            }
+            await delay(backoffMs);
+            backoffMs = Math.min(backoffMs * 2, SEARCH_VISIBILITY_RETRY_MAX_BACKOFF_MS);
+        }
+    }
+    async getDataVersion() {
+        if (!this.sqliteReadDb) {
+            throw new Error('Storage not initialized');
+        }
+        // PRAGMA data_version is connection-relative: comparing successive reads
+        // on this reader detects commits from this instance's writer and other processes.
+        const row = await this.sqliteReadDb.get('PRAGMA data_version');
+        if (!row) {
+            throw new Error('Could not read SQLite data version');
+        }
+        return row.data_version;
+    }
     async searchDocuments(queryVector, options = {}) {
+        return (await this.withStableSearch(() => this.searchDocumentsOnce(queryVector, options))).result;
+    }
+    async searchDocumentsOnce(queryVector, options) {
         if (!this.lanceTable) {
             throw new Error('Storage not initialized');
         }
@@ -418,10 +826,9 @@ export class DocumentStore {
                 }
                 logger.debug(`[DocumentStore] Tag filter matched ${tagFilteredUrls.length} documents for vector search`);
             }
-            // Create search query
-            let query = this.lanceTable.search(queryVector).limit(limit);
             // Build WHERE conditions
-            const conditions = [];
+            const visibilityFilter = await this.getJournalVisibilityFilter();
+            const conditions = [`(${visibilityFilter})`];
             if (filterByType) {
                 conditions.push(`type = '${escapeFilterValue(filterByType)}'`);
             }
@@ -429,9 +836,7 @@ export class DocumentStore {
                 const urlConditions = tagFilteredUrls.map((u) => `url = '${escapeFilterValue(u)}'`).join(' OR ');
                 conditions.push(`(${urlConditions})`);
             }
-            if (conditions.length > 0) {
-                query = query.where(conditions.join(' AND '));
-            }
+            const query = this.lanceTable.search(queryVector).where(conditions.join(' AND ')).limit(limit);
             const results = await query.toArray();
             logger.debug(`[DocumentStore] Found ${results.length} results`);
             // Log the first result for debugging if available
@@ -524,14 +929,23 @@ export class DocumentStore {
         }
     }
     async searchByText(query, options = {}) {
+        const { result, dataVersion } = await this.withStableSearch((version) => this.searchByTextOnce(query, options, version));
+        this.searchCache.set(this.textSearchCacheKey(query, options, dataVersion), result);
+        return result;
+    }
+    textSearchCacheKey(query, options, dataVersion) {
+        return `text:${dataVersion}:${query}:${JSON.stringify(options)}`;
+    }
+    async searchByTextOnce(query, options, dataVersion) {
         logger.debug(`[DocumentStore] Searching documents by text:`, { query, options });
-        const cacheKey = `text:${query}:${JSON.stringify(options)}`;
+        const cacheKey = this.textSearchCacheKey(query, options, dataVersion);
         const cached = this.searchCache.get(cacheKey);
         if (cached) {
             logger.debug(`[DocumentStore] Returning cached results`);
             return cached;
         }
         const { limit = 10, filterByType, filterUrl, filterByTags } = options;
+        const visibilityFilter = await this.getJournalVisibilityFilter();
         // If filtering by tags, get the list of URLs that have all those tags
         let tagFilteredUrls;
         if (filterByTags && filterByTags.length > 0) {
@@ -540,14 +954,13 @@ export class DocumentStore {
                 // No documents match the tag filter, cache and return empty results
                 logger.debug(`[DocumentStore] No documents match tag filter:`, filterByTags);
                 const emptyResults = [];
-                this.searchCache.set(cacheKey, emptyResults);
                 return emptyResults;
             }
             logger.debug(`[DocumentStore] Tag filter matched ${tagFilteredUrls.length} documents`);
         }
         // Build WHERE clause for filtering (using escaped values to prevent injection)
         const buildWhereClause = () => {
-            const conditions = [];
+            const conditions = [`(${visibilityFilter})`];
             if (filterByType) {
                 conditions.push(`type = '${escapeFilterValue(filterByType)}'`);
             }
@@ -562,7 +975,7 @@ export class DocumentStore {
                 const urlConditions = tagFilteredUrls.map((u) => `url = '${escapeFilterValue(u)}'`).join(' OR ');
                 conditions.push(`(${urlConditions})`);
             }
-            return conditions.length > 0 ? conditions.join(' AND ') : undefined;
+            return conditions.join(' AND ');
         };
         const whereClause = buildWhereClause();
         try {
@@ -589,25 +1002,22 @@ export class DocumentStore {
                         queries.push([Occur.Should, new MatchQuery(processedQuery.cleanedQuery, 'content', { fuzziness: 1 })]);
                     }
                     const boolQuery = new BooleanQuery(queries);
-                    let ftsQuery = this.lanceTable
+                    const ftsQuery = this.lanceTable
                         .query()
                         .fullTextSearch(boolQuery)
+                        .where(whereClause)
                         .limit(limit * 2);
-                    if (whereClause) {
-                        ftsQuery = ftsQuery.where(whereClause);
-                    }
                     const ftsResults = await ftsQuery.toArray();
                     logger.debug(`[DocumentStore] Phrase-based FTS returned ${ftsResults.length} results`);
                     if (ftsResults.length > 0) {
                         // Combine with vector search for semantic relevance
-                        let vectorQuery = this.lanceTable.search(queryVector).limit(limit * 2);
-                        if (whereClause) {
-                            vectorQuery = vectorQuery.where(whereClause);
-                        }
+                        const vectorQuery = this.lanceTable
+                            .search(queryVector)
+                            .where(whereClause)
+                            .limit(limit * 2);
                         const vectorResults = await vectorQuery.toArray();
                         const mergedResults = this.mergeAndRankResults(ftsResults, vectorResults, limit);
                         const searchResults = this.formatSearchResults(mergedResults);
-                        this.searchCache.set(cacheKey, searchResults);
                         return searchResults;
                     }
                 }
@@ -622,27 +1032,24 @@ export class DocumentStore {
                     // LanceDB's FTS already handles stop words and stemming
                     // Add fuzziness for typo tolerance
                     const matchQuery = new MatchQuery(processedQuery.cleanedQuery, 'content', { fuzziness: 1 });
-                    let ftsQuery = this.lanceTable
+                    const ftsQuery = this.lanceTable
                         .query()
                         .fullTextSearch(matchQuery)
+                        .where(whereClause)
                         .limit(limit * 2);
-                    if (whereClause) {
-                        ftsQuery = ftsQuery.where(whereClause);
-                    }
                     const ftsResults = await ftsQuery.toArray();
                     logger.debug(`[DocumentStore] FTS returned ${ftsResults.length} results`);
                     // Always combine with vector search for best results
-                    let vectorQuery = this.lanceTable.search(queryVector).limit(limit * 2);
-                    if (whereClause) {
-                        vectorQuery = vectorQuery.where(whereClause);
-                    }
+                    const vectorQuery = this.lanceTable
+                        .search(queryVector)
+                        .where(whereClause)
+                        .limit(limit * 2);
                     const vectorResults = await vectorQuery.toArray();
                     logger.debug(`[DocumentStore] Vector search returned ${vectorResults.length} results`);
                     // Merge using RRF even if one is empty - ensures we get results
                     const mergedResults = this.mergeAndRankResults(ftsResults, vectorResults, limit);
                     if (mergedResults.length > 0) {
                         const searchResults = this.formatSearchResults(mergedResults);
-                        this.searchCache.set(cacheKey, searchResults);
                         return searchResults;
                     }
                 }
@@ -653,9 +1060,7 @@ export class DocumentStore {
             }
             // Strategy 3: Fallback to pure vector search (semantic similarity)
             logger.debug('[DocumentStore] Falling back to pure vector search');
-            const results = await this.searchDocuments(queryVector, options);
-            this.searchCache.set(cacheKey, results);
-            return results;
+            return this.searchDocumentsOnce(queryVector, options);
         }
         catch (error) {
             logger.error('[DocumentStore] Error searching documents by text:', error);
@@ -731,15 +1136,16 @@ export class DocumentStore {
         });
     }
     async listDocuments() {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         logger.debug(`[DocumentStore] Listing documents`);
         try {
-            const rows = await this.sqliteDb.all('SELECT url, title, favicon, last_indexed, requires_auth, auth_domain, version FROM documents ORDER BY last_indexed DESC');
+            const rows = await this.sqliteReadDb.all(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
+                COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
+         FROM documents d
+         ORDER BY d.last_indexed DESC`);
             logger.debug(`[DocumentStore] Found ${rows.length} documents`);
-            // Fetch tags for all documents
-            const tagsMap = await this.getAllDocumentTags();
             return rows.map((row) => ({
                 url: row.url,
                 title: row.title,
@@ -747,7 +1153,7 @@ export class DocumentStore {
                 lastIndexed: new Date(row.last_indexed),
                 requiresAuth: row.requires_auth === 1,
                 authDomain: row.auth_domain ?? undefined,
-                tags: tagsMap.get(row.url) || [],
+                tags: this.parseDocumentTags(row.tags_json),
                 version: row.version ?? undefined,
             }));
         }
@@ -756,60 +1162,78 @@ export class DocumentStore {
             throw error;
         }
     }
-    /**
-     * Get all tags for all documents as a Map
-     */
-    async getAllDocumentTags() {
-        if (!this.sqliteDb) {
-            return new Map();
-        }
-        const rows = await this.sqliteDb.all('SELECT url, tag FROM document_tags ORDER BY url, tag');
-        const tagsMap = new Map();
-        for (const row of rows) {
-            const existing = tagsMap.get(row.url) || [];
-            existing.push(row.tag);
-            tagsMap.set(row.url, existing);
-        }
-        return tagsMap;
+    deleteDocument(url) {
+        return this.runMutation(() => this.deleteDocumentUnlocked(url));
     }
-    async deleteDocument(url) {
-        if (!this.sqliteDb || !this.lanceTable) {
+    async deleteDocumentUnlocked(url) {
+        const sqliteDb = this.sqliteDb;
+        const sqliteReadDb = this.sqliteReadDb;
+        if (!sqliteDb || !sqliteReadDb || !this.sqliteLeaseDb || !this.lanceTable) {
             throw new Error('Storage not initialized');
         }
         logger.debug(`[DocumentStore] Deleting document: ${url}`);
+        const generation = `delete:${randomUUID()}`;
+        const ownerId = randomUUID();
+        const lease = await this.acquireReplacementLease(url, generation, ownerId);
+        let transactionStarted = false;
         try {
-            await this.sqliteDb.run('BEGIN TRANSACTION');
+            const preparedJournal = await this.captureCleanupGenerations(lease);
+            await this.renewReplacementLease(preparedJournal);
+            await sqliteDb.run('BEGIN TRANSACTION');
+            transactionStarted = true;
             // Delete tags first (in case foreign key cascade isn't enabled)
-            await this.sqliteDb.run('DELETE FROM document_tags WHERE url = ?', [url]);
-            await this.sqliteDb.run('DELETE FROM documents WHERE url = ?', [url]);
-            await this.lanceTable.delete(`url = '${escapeFilterValue(url)}'`);
-            await this.sqliteDb.run('COMMIT');
-            // Clear cache for this URL
+            await sqliteDb.run('DELETE FROM document_tags WHERE url = ?', [url]);
+            await sqliteDb.run('DELETE FROM documents WHERE url = ?', [url]);
+            const leaseExpiresAt = Date.now() + REPLACEMENT_LEASE_MS;
+            const transition = await sqliteDb.run(`UPDATE document_replacements
+         SET state = 'deleting', lease_expires_at = ?
+         WHERE url = ? AND generation = ? AND owner_id = ? AND state = 'prepared'`, [leaseExpiresAt, url, generation, ownerId]);
+            if (transition.changes !== 1) {
+                throw new Error(`Replacement lease lost for ${url}`);
+            }
+            await sqliteDb.run('COMMIT');
+            transactionStarted = false;
+            await this.finishDocumentReplacementBestEffort({
+                ...preparedJournal,
+                state: 'deleting',
+                lease_expires_at: leaseExpiresAt,
+            });
             this.clearCacheForUrl(url);
             logger.debug(`[DocumentStore] Document deleted successfully`);
         }
         catch (error) {
-            if (this.sqliteDb) {
-                await this.sqliteDb.run('ROLLBACK');
+            if (transactionStarted) {
+                try {
+                    await sqliteDb.run('ROLLBACK');
+                }
+                catch {
+                    // The transaction may already have been rolled back by SQLite.
+                }
+            }
+            const durableJournal = await sqliteReadDb.get('SELECT * FROM document_replacements WHERE url = ? AND generation = ? AND owner_id = ?', [url, generation, ownerId]);
+            if (durableJournal) {
+                await this.finishDocumentReplacementBestEffort(durableJournal);
+                if (durableJournal.state === 'deleting') {
+                    this.clearCacheForUrl(url);
+                    return;
+                }
             }
             logger.error('[DocumentStore] Error deleting document:', error);
             throw error;
         }
     }
     async getDocument(url) {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         logger.debug(`[DocumentStore] Getting document: ${url}`);
         try {
-            // Check if SQLite is properly initialized
-            if (!this.sqliteDb) {
-                logger.debug('[DocumentStore] SQLite not initialized during getDocument');
-                throw new Error('Storage not initialized');
-            }
             // Log the query being executed
             logger.debug(`[DocumentStore] Executing SQLite query for URL: ${url}`);
-            const row = await this.sqliteDb.get('SELECT url, title, favicon, last_indexed, requires_auth, auth_domain, version FROM documents WHERE url = ?', [url]);
+            const row = await this.sqliteReadDb.get(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
+                COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
+         FROM documents d
+         WHERE d.url = ?`, [url]);
             if (!row) {
                 logger.debug(`[DocumentStore] Document not found in SQLite: ${url}`);
                 return null;
@@ -819,8 +1243,6 @@ export class DocumentStore {
                 const chunks = await this.lanceTable.countRows(`url = '${escapeFilterValue(url)}'`);
                 logger.debug(`[DocumentStore] Found ${chunks} chunks in LanceDB for ${url}`);
             }
-            // Fetch tags for this document
-            const tags = await this.getDocumentTags(url);
             logger.debug(`[DocumentStore] Document found in SQLite:`, row);
             return {
                 url: row.url,
@@ -829,7 +1251,7 @@ export class DocumentStore {
                 lastIndexed: new Date(row.last_indexed),
                 requiresAuth: row.requires_auth === 1,
                 authDomain: row.auth_domain ?? undefined,
-                tags,
+                tags: this.parseDocumentTags(row.tags_json),
                 version: row.version ?? undefined,
             };
         }
@@ -838,22 +1260,34 @@ export class DocumentStore {
             throw error;
         }
     }
-    /**
-     * Get tags for a specific document
-     */
-    async getDocumentTags(url) {
-        if (!this.sqliteDb) {
+    parseDocumentTags(value) {
+        try {
+            const parsed = JSON.parse(value);
+            return Array.isArray(parsed) && parsed.every((tag) => typeof tag === 'string') ? parsed.sort() : [];
+        }
+        catch {
             return [];
         }
-        const rows = await this.sqliteDb.all('SELECT tag FROM document_tags WHERE url = ? ORDER BY tag', [url]);
-        return rows.map((row) => row.tag);
+    }
+    async replaceDocumentTags(url, tags) {
+        if (!this.sqliteDb) {
+            throw new Error('Storage not initialized');
+        }
+        await this.sqliteDb.run('DELETE FROM document_tags WHERE url = ?', [url]);
+        const normalizedTags = [...new Set(tags.map((tag) => tag.trim().toLowerCase()).filter(Boolean))];
+        for (const tag of normalizedTags) {
+            await this.sqliteDb.run('INSERT INTO document_tags (url, tag) VALUES (?, ?)', [url, tag]);
+        }
     }
     /**
      * Set tags for a documentation site. Replaces any existing tags.
      * @param url - The URL of the documentation site
      * @param tags - Array of tags to assign (empty array removes all tags)
      */
-    async setTags(url, tags) {
+    setTags(url, tags) {
+        return this.runMutation(() => this.setTagsUnlocked(url, tags));
+    }
+    async setTagsUnlocked(url, tags) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -866,13 +1300,7 @@ export class DocumentStore {
                 await this.sqliteDb.run('ROLLBACK');
                 throw new Error('Documentation not found');
             }
-            // Delete existing tags
-            await this.sqliteDb.run('DELETE FROM document_tags WHERE url = ?', [url]);
-            // Insert new tags (deduplicated and normalized)
-            const uniqueTags = [...new Set(tags.map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0))];
-            for (const tag of uniqueTags) {
-                await this.sqliteDb.run('INSERT INTO document_tags (url, tag) VALUES (?, ?)', [url, tag]);
-            }
+            await this.replaceDocumentTags(url, tags);
             await this.sqliteDb.run('COMMIT');
             // Clear cached search results that may be affected by tag changes
             this.clearCacheForUrl(url);
@@ -896,12 +1324,12 @@ export class DocumentStore {
      * @returns Array of tags with counts, sorted by count descending
      */
     async listAllTags() {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         logger.debug(`[DocumentStore] Listing all tags`);
         try {
-            const rows = await this.sqliteDb.all('SELECT tag, COUNT(*) as count FROM document_tags GROUP BY tag ORDER BY count DESC, tag ASC');
+            const rows = await this.sqliteReadDb.all('SELECT tag, COUNT(*) as count FROM document_tags GROUP BY tag ORDER BY count DESC, tag ASC');
             logger.debug(`[DocumentStore] Found ${rows.length} unique tags`);
             return rows;
         }
@@ -916,7 +1344,7 @@ export class DocumentStore {
      * @returns Array of matching document URLs
      */
     async getUrlsByTags(tags) {
-        if (!this.sqliteDb || tags.length === 0) {
+        if (!this.sqliteReadDb || tags.length === 0) {
             return [];
         }
         // Normalize tags
@@ -935,7 +1363,7 @@ export class DocumentStore {
         GROUP BY url
         HAVING COUNT(DISTINCT tag) = ?
       `;
-            const rows = await this.sqliteDb.all(query, [...normalizedTags, normalizedTags.length]);
+            const rows = await this.sqliteReadDb.all(query, [...normalizedTags, normalizedTags.length]);
             logger.debug(`[DocumentStore] Found ${rows.length} URLs matching all tags`);
             return rows.map((row) => row.url);
         }
@@ -951,7 +1379,10 @@ export class DocumentStore {
      * @param description - Optional description
      * @throws Error if collection already exists
      */
-    async createCollection(name, description) {
+    createCollection(name, description) {
+        return this.runMutation(() => this.createCollectionUnlocked(name, description));
+    }
+    async createCollectionUnlocked(name, description) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -981,7 +1412,10 @@ export class DocumentStore {
      * @param name - Name of the collection to delete
      * @throws Error if collection doesn't exist
      */
-    async deleteCollection(name) {
+    deleteCollection(name) {
+        return this.runMutation(() => this.deleteCollectionUnlocked(name));
+    }
+    async deleteCollectionUnlocked(name) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -1000,7 +1434,10 @@ export class DocumentStore {
      * @param updates - Fields to update
      * @throws Error if collection doesn't exist
      */
-    async updateCollection(name, updates) {
+    updateCollection(name, updates) {
+        return this.runMutation(() => this.updateCollectionUnlocked(name, updates));
+    }
+    async updateCollectionUnlocked(name, updates) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -1077,11 +1514,11 @@ export class DocumentStore {
      * @returns Array of collections sorted by name
      */
     async listCollections() {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         logger.debug(`[DocumentStore] Listing collections`);
-        const rows = await this.sqliteDb.all(`
+        const rows = await this.sqliteReadDb.all(`
       SELECT c.name, c.description, c.created_at, c.updated_at,
              COUNT(cd.url) as document_count
       FROM collections c
@@ -1104,27 +1541,26 @@ export class DocumentStore {
      * @returns Collection with documents, or null if not found
      */
     async getCollection(name) {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         const normalizedName = name.trim();
         logger.debug(`[DocumentStore] Getting collection: ${normalizedName}`);
         // Get collection metadata
-        const row = await this.sqliteDb.get('SELECT name, description, created_at, updated_at FROM collections WHERE name = ?', [normalizedName]);
+        const row = await this.sqliteReadDb.get('SELECT name, description, created_at, updated_at FROM collections WHERE name = ?', [normalizedName]);
         if (!row) {
             logger.debug(`[DocumentStore] Collection not found: ${normalizedName}`);
             return null;
         }
         // Get documents in the collection
-        const docRows = await this.sqliteDb.all(`
-      SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version
+        const docRows = await this.sqliteReadDb.all(`
+      SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
+             COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
       FROM documents d
       INNER JOIN collection_documents cd ON d.url = cd.url
       WHERE cd.collection_name = ?
       ORDER BY d.title ASC
     `, [normalizedName]);
-        // Fetch tags for all documents
-        const tagsMap = await this.getAllDocumentTags();
         const documents = docRows.map((doc) => ({
             url: doc.url,
             title: doc.title,
@@ -1132,7 +1568,7 @@ export class DocumentStore {
             lastIndexed: new Date(doc.last_indexed),
             requiresAuth: doc.requires_auth === 1,
             authDomain: doc.auth_domain ?? undefined,
-            tags: tagsMap.get(doc.url) || [],
+            tags: this.parseDocumentTags(doc.tags_json),
             version: doc.version ?? undefined,
         }));
         logger.debug(`[DocumentStore] Collection "${normalizedName}" has ${documents.length} documents`);
@@ -1151,7 +1587,10 @@ export class DocumentStore {
      * @param urls - URLs of documents to add
      * @throws Error if collection doesn't exist
      */
-    async addToCollection(name, urls) {
+    addToCollection(name, urls) {
+        return this.runMutation(() => this.addToCollectionUnlocked(name, urls));
+    }
+    async addToCollectionUnlocked(name, urls) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -1219,7 +1658,10 @@ export class DocumentStore {
      * @param urls - URLs of documents to remove
      * @throws Error if collection doesn't exist
      */
-    async removeFromCollection(name, urls) {
+    removeFromCollection(name, urls) {
+        return this.runMutation(() => this.removeFromCollectionUnlocked(name, urls));
+    }
+    async removeFromCollectionUnlocked(name, urls) {
         if (!this.sqliteDb) {
             throw new Error('Storage not initialized');
         }
@@ -1270,12 +1712,12 @@ export class DocumentStore {
      * @returns Array of document URLs
      */
     async getCollectionUrls(name) {
-        if (!this.sqliteDb) {
+        if (!this.sqliteReadDb) {
             throw new Error('Storage not initialized');
         }
         const normalizedName = name.trim();
         logger.debug(`[DocumentStore] Getting URLs for collection: ${normalizedName}`);
-        const rows = await this.sqliteDb.all('SELECT url FROM collection_documents WHERE collection_name = ?', [
+        const rows = await this.sqliteReadDb.all('SELECT url FROM collection_documents WHERE collection_name = ?', [
             normalizedName,
         ]);
         return rows.map((row) => row.url);
@@ -1299,15 +1741,17 @@ export class DocumentStore {
             throw new Error('Storage not initialized');
         }
         try {
+            const visibilityFilter = await this.getJournalVisibilityFilter();
             // Get total row count
-            const rowCount = await this.lanceTable.countRows();
+            const rowCount = await this.lanceTable.countRows(visibilityFilter);
             logger.debug(`[DocumentStore] Vector validation: Table contains ${rowCount} rows`);
             if (rowCount === 0) {
                 logger.debug('[DocumentStore] Vector validation: No rows found in vector table');
                 return false;
             }
             // Get a sample row using a query
-            const sample = await this.lanceTable.query().limit(1).toArray();
+            const sampleQuery = this.lanceTable.query().where(visibilityFilter).limit(1);
+            const sample = await sampleQuery.toArray();
             if (sample.length === 0) {
                 logger.debug('[DocumentStore] Vector validation: No rows returned from query');
                 return false;
@@ -1323,7 +1767,8 @@ export class DocumentStore {
             // Try a simple vector search with a random vector
             const testVector = new Array(this.embeddings.dimensions).fill(0).map(() => Math.random());
             logger.debug(`[DocumentStore] Testing vector search with random vector of length ${testVector.length}`);
-            const searchResults = await this.lanceTable.search(testVector).limit(1).toArray();
+            const searchQuery = this.lanceTable.search(testVector).where(visibilityFilter).limit(1);
+            const searchResults = await searchQuery.toArray();
             logger.debug(`[DocumentStore] Vector search test returned ${searchResults.length} results`);
             if (searchResults.length > 0) {
                 logger.debug('[DocumentStore] Vector search test result:', {
@@ -1352,7 +1797,10 @@ export class DocumentStore {
      *
      * @returns Promise with optimization statistics
      */
-    async optimize() {
+    optimize() {
+        return this.runMutation(() => this.optimizeUnlocked());
+    }
+    async optimizeUnlocked() {
         if (!this.lanceTable) {
             logger.debug('[DocumentStore] Cannot optimize: Storage not initialized');
             return { compacted: false, cleanedUp: false, error: 'Storage not initialized' };