@docstack/pouchdb-adapter-googledrive 0.0.6 → 0.0.9

package/lib/drive.js

@@ -17,6 +17,9 @@ const DEFAULT_CACHE_SIZE = 1000; // Number of docs
  *   └── changes-*.ndjson     # Append logs
  */
 class DriveHandler {
+    log(...args) {
+        console.log(`[googledrive-drive] [${this.meta.dbName}]`, ...args);
+    }
     constructor(options, dbName) {
         this.folderId = null;
         this.meta = {
@@ -34,7 +37,35 @@ class DriveHandler {
         this.currentLogSizeEstimate = 0;
         this.listeners = [];
         this.pollingInterval = null;
-        this.client = new client_1.GoogleDriveClient(options);
+        this.loadingPromise = null;
+        this.isPollingActive = false;
+        this.processedLogIds = new Set();
+        this.currentSnapshotIndexId = null;
+        this.debug = false;
+        this.isCompacting = false;
+        const clientOptions = { ...options };
+        if (options.testMode) {
+            const serverUrl = options.testServerUrl || 'http://localhost:3000';
+            // @ts-ignore - baseUrl/uploadUrl might not be in the strict type if we didn't update types.ts definition for DriveClientOptions in client.ts yet? 
+            // We did update DriveClientOptions in client.ts.
+            // But GoogleDriveAdapterOptions extends DriveClientOptions?
+            // types.ts: export interface GoogleDriveAdapterOptions extends DriveClientOptions
+            // client.ts: export interface DriveClientOptions { accessToken: ...; baseUrl?: string; uploadUrl?: string; }
+            // So typescript should be happy.
+            // Using /drive/v3/files as base for the test server if simplified?
+            // The TestServer mounts at /drive/v3/files.
+            // But the client appends /drive/v3/files to BASE_URL? 
+            // In client.ts default is `https://www.googleapis.com/drive/v3/files`.
+            // Our TestServer mounts `/drive/v3/files`.
+            // So testUrl should be `http://localhost:3000/drive/v3/files`.
+            const testBase = `${serverUrl}/drive/v3/files`;
+            const testUpload = `${serverUrl}/upload/drive/v3/files`;
+            // @ts-ignore
+            clientOptions.baseUrl = testBase;
+            // @ts-ignore
+            clientOptions.uploadUrl = testUpload;
+        }
+        this.client = new client_1.GoogleDriveClient(clientOptions);
         this.options = options;
         this.folderId = options.folderId || null;
         this.folderName = options.folderName || dbName;
@@ -42,7 +73,9 @@ class DriveHandler {
         this.compactionThreshold = options.compactionThreshold || DEFAULT_COMPACTION_THRESHOLD;
         this.compactionSizeThreshold = options.compactionSizeThreshold || DEFAULT_SIZE_THRESHOLD;
         this.meta.dbName = dbName;
+        this.debug = !!options.debug;
         this.docCache = new cache_1.LRUCache(options.cacheSize || DEFAULT_CACHE_SIZE);
+        this.fileCache = new cache_1.LRUCache(100); // Cache for last 100 files
         // Polling will be started in load() after folderId is resolved
     }
     // Public getter for Sequence (used by adapter)
@@ -51,69 +84,104 @@ class DriveHandler {
     }
     /** Load the database (Index Only) */
     async load() {
-        if (!this.folderId) {
-            this.folderId = await this.findOrCreateFolder();
-        }
-        const metaFile = await this.findFile('_meta.json');
-        if (metaFile) {
-            this.meta = await this.downloadJson(metaFile.id);
-            this.metaEtag = metaFile.etag || null;
-            this.metaModifiedTime = metaFile.modifiedTime || null;
-        }
-        else {
-            await this.saveMeta(this.meta);
-        }
-        // Initialize Index
-        this.index = {};
-        // 1. Load Snapshot Index
-        if (this.meta.snapshotIndexId) {
+        if (this.loadingPromise)
+            return this.loadingPromise;
+        this.loadingPromise = (async () => {
             try {
-                // Try strictly as new format first
-                const snapshotIdx = await this.downloadJson(this.meta.snapshotIndexId);
-                // Check if it's actually a legacy snapshot (has 'docs' with bodies)
-                if (snapshotIdx.docs) {
-                    // Migration Path: Handle legacy snapshot
-                    this.filesFromLegacySnapshot(snapshotIdx);
+                this.log('Loading database, options', { options: this.options });
+                if (!this.folderId) {
+                    this.folderId = await this.findOrCreateFolder();
+                    this.log('Retrieved folder', { folderId: this.folderId });
+                }
+                const metaFile = await this.findFile('_meta.json');
+                if (metaFile) {
+                    this.log('Retrieved meta file', { fileId: metaFile.id });
+                    this.meta = await this.downloadJson(metaFile.id, true); // No cache for meta
+                    this.metaEtag = metaFile.etag || null;
+                    this.metaModifiedTime = metaFile.modifiedTime || null;
                 }
                 else {
-                    this.index = snapshotIdx.entries || {};
-                    // We assume seq is synced with meta usually, but use snapshot's seq as base
+                    this.log('Meta file not found, creating new');
+                    await this.saveMeta(this.meta);
+                }
+                if (this.meta.snapshotIndexId !== this.currentSnapshotIndexId) {
+                    this.log('Snapshot index changed, loading index', {
+                        snapshotIndexId: this.meta.snapshotIndexId,
+                        currentSnapshotIndexId: this.currentSnapshotIndexId
+                    });
+                    // Compaction occurred or initial load
+                    this.index = {};
+                    this.processedLogIds.clear();
+                    this.currentSnapshotIndexId = this.meta.snapshotIndexId;
+                    if (this.meta.snapshotIndexId) {
+                        try {
+                            const snapshotIdx = await this.downloadJson(this.meta.snapshotIndexId);
+                            if (snapshotIdx.docs) {
+                                this.filesFromLegacySnapshot(snapshotIdx);
+                            }
+                            else {
+                                this.index = snapshotIdx.entries || {};
+                            }
+                        }
+                        catch (e) {
+                            console.warn('Failed to load snapshot index', e);
+                        }
+                    }
+                    else if (this.meta.snapshotId) {
+                        this.log('Legacy snapshot found, loading index', {
+                            snapshotId: this.meta.snapshotId
+                        });
+                        try {
+                            const legacySnapshot = await this.downloadJson(this.meta.snapshotId);
+                            this.filesFromLegacySnapshot(legacySnapshot);
+                        }
+                        catch (e) {
+                            console.warn('Failed to load legacy snapshot', e);
+                        }
+                    }
+                }
+                // 2. Replay NEW Change Logs (Metadata only updates)
+                this.log('Replaying change logs');
+                for (const logId of this.meta.changeLogIds) {
+                    if (this.processedLogIds.has(logId))
+                        continue;
+                    this.log('Replaying change log', logId);
+                    let changes = await this.downloadNdjson(logId);
+                    if (!Array.isArray(changes)) {
+                        this.log("Unexpected changes", { changes, logId });
+                        this.log('Downloaded changes not an array, wrapping/ignoring', typeof changes);
+                        changes = changes ? [changes] : []; // Fallback
+                    }
+                    this.currentLogSizeEstimate += 100 * changes.length;
+                    this.log('Replayed change log', logId);
+                    for (const change of changes) {
+                        this.log('Processing change, sequence', change.seq);
+                        this.updateIndex(change, logId);
+                        if (this.docCache.get(change.id)) {
+                            this.docCache.remove(change.id);
+                        }
+                    }
+                    this.processedLogIds.add(logId);
+                    this.log('Processed log', logId);
+                }
+                // 3. Start Polling (if enabled)
+                if (this.options.pollingIntervalMs) {
+                    this.log('Starting polling with interval', this.options.pollingIntervalMs);
+                    this.startPolling(Number(this.options.pollingIntervalMs));
+                }
+                else {
+                    this.log('Polling disabled (no interval provided)');
                 }
             }
             catch (e) {
-                console.warn('Failed to load snapshot index', e);
-                this.index = {};
-            }
-        }
-        else if (this.meta.snapshotId) {
-            // Legacy support: field was renamed
-            try {
-                const legacySnapshot = await this.downloadJson(this.meta.snapshotId);
-                this.filesFromLegacySnapshot(legacySnapshot);
-            }
-            catch (e) {
-                console.warn('Failed to load legacy snapshot', e);
+                console.error('Failed to load database', e);
+                throw e;
             }
-        }
-        // 2. Replay Change Logs (Metadata only updates)
-        this.pendingChanges = [];
-        this.currentLogSizeEstimate = 0;
-        for (const logId of this.meta.changeLogIds) {
-            const changes = await this.downloadNdjson(logId);
-            this.currentLogSizeEstimate += 100 * changes.length;
-            for (const change of changes) {
-                this.updateIndex(change, logId);
-                // We do NOT load body into cache automatically
-                // But we must invalidate cache if we had old data
-                if (this.docCache.get(change.id)) {
-                    this.docCache.remove(change.id);
-                }
+            finally {
+                this.loadingPromise = null;
             }
-        }
-        // 3. Start Polling (if enabled)
-        if (this.options.pollingIntervalMs) {
-            this.startPolling(this.options.pollingIntervalMs);
-        }
+        })();
+        return this.loadingPromise;
     }
     // Migration helper
     filesFromLegacySnapshot(snapshot) {
@@ -141,17 +209,12 @@ class DriveHandler {
             return null;
         if (entry.deleted)
             return null;
-        // 1. Check Cache
-        const cached = this.docCache.get(id);
-        if (cached)
-            return cached;
-        // 2. Fetch from Drive
-        // If it's a legacy entry currently in memory (should have been cached), returns null if evicted?
+        // 1. Check Doc Cache
+        const cachedDoc = this.docCache.get(id);
+        if (cachedDoc)
+            return cachedDoc;
+        // 2. Fetch from Drive (via File Cache)
         if (entry.location.fileId === 'LEGACY_MEMORY') {
-            // If evicted, we are in trouble unless we re-download the legacy snapshot.
-            // For robustness, let's say we reload the legacy snapshot if needed.
-            // OR simpler: we assume compaction will fix this soon.
-            // Let's implement fetch for safety.
             if (this.meta.snapshotId) {
                 const legacy = await this.downloadJson(this.meta.snapshotId);
                 if (legacy.docs[id]) {
@@ -159,36 +222,83 @@ class DriveHandler {
                     return legacy.docs[id];
                 }
             }
-            return null; // Should not happen
+            return null;
         }
         const fileId = entry.location.fileId;
-        // Is it a change file (NDJSON) or snapshot file (JSON)?
-        // We can infer or we could have stored type. 
-        // Usually, we just download the file.
-        // Optimization: If we have many docs in one file, we might want to cache that file's contents?
-        // For now, naive fetch: download file, find doc.
-        const content = await this.downloadFileAny(fileId);
+        const content = await this.fetchFile(fileId);
         let doc = null;
         if (Array.isArray(content)) {
-            // It's a change log (array of entries)
-            // Find the *last* entry for this ID in this file
-            const match = content.reverse().find((c) => c.id === id);
+            // It's a change log (NDJSON parsed as array)
+            const match = [...content].reverse().find((c) => c.id === id);
             doc = match ? match.doc : null;
         }
-        else if (content.docs) {
+        else if (content && content.docs) {
             // It's a snapshot-data chunk
             doc = content.docs[id];
         }
-        else {
-            // Single doc file? (Not used yet)
+        else if (content && content.id === id && content.doc) {
+            // It's a single ChangeEntry object (parsed from single-line NDJSON)
+            doc = content.doc;
+        }
+        else if (content && (content._id === id || content.id === id)) {
+            // Single doc file or raw doc body
             doc = content;
         }
         if (doc) {
             this.docCache.put(id, doc);
-            doc._rev = entry.rev; // Ensure consistent rev
+            doc._rev = entry.rev;
         }
         return doc;
     }
+    /** Generic Download with Caching and Parsing */
+    async fetchFile(fileId, skipCache = false) {
+        if (!skipCache) {
+            const cached = this.fileCache.get(fileId);
+            if (cached) {
+                this.log('fetchFile cache hit', fileId);
+                return cached;
+            }
+        }
+        this.log('fetchFile downloading', fileId);
+        const data = await this.client.getFile(fileId);
+        let parsed;
+        if (typeof data === 'string') {
+            const trimmed = data.trim();
+            if (trimmed.startsWith('{')) {
+                // Could be JSON or NDJSON
+                if (trimmed.includes('\n')) {
+                    // Definitely NDJSON (multiple lines)
+                    try {
+                        const lines = trimmed.split('\n').filter(l => l);
+                        parsed = lines.map(line => JSON.parse(line));
+                    }
+                    catch (e) {
+                        parsed = data;
+                    }
+                }
+                else {
+                    // Single line. Try regular JSON first.
+                    try {
+                        parsed = JSON.parse(trimmed);
+                        // Optional: if we know it was supposed to be NDJSON? 
+                        // Our change files are always NDJSON. 
+                        // But _meta.json and snapshot-*.json are regular JSON.
+                    }
+                    catch (e) {
+                        parsed = data;
+                    }
+                }
+            }
+            else {
+                parsed = data;
+            }
+        }
+        else {
+            parsed = data;
+        }
+        this.fileCache.put(fileId, parsed);
+        return parsed;
+    }
     /** Get multiple docs (Atomic-ish) used for _allDocs */
     async getMulti(ids) {
         // Naive parallel fetch
@@ -222,16 +332,20 @@ class DriveHandler {
         // Fetch files
         for (const [fileId, docIds] of Object.entries(byFile)) {
             try {
-                const content = await this.downloadFileAny(fileId);
+                const content = await this.fetchFile(fileId);
                 for (const docId of docIds) {
                     let doc = null;
                     if (Array.isArray(content)) {
-                        const match = content.reverse().find((c) => c.id === docId);
+                        const match = [...content].reverse().find((c) => c.id === docId);
                         doc = match ? match.doc : null;
                     }
-                    else if (content.docs) {
+                    else if (content && content.docs) {
                         doc = content.docs[docId];
                     }
+                    else if (content && content.id === docId && content.doc) {
+                        // Single ChangeEntry object
+                        doc = content.doc;
+                    }
                     if (doc) {
                         // Add entry.rev to doc just in case
                         if (this.index[docId])
@@ -253,7 +367,9 @@ class DriveHandler {
         return ids.map(id => results[id]);
     }
     /** Return all keys in Index */
-    getIndexKeys() {
+    async getIndexKeys() {
+        if (this.loadingPromise)
+            await this.loadingPromise;
         return Object.keys(this.index);
     }
     /** Get metadata for a specific ID from Index */
@@ -296,29 +412,37 @@ class DriveHandler {
     async tryAppendChanges(changes) {
         // 1. Write Log File (Upload Data)
         const fileId = await this.writeChangeFile(changes);
-        // 2. Prepare speculative meta update
-        const nextMeta = { ...this.meta };
-        nextMeta.changeLogIds = [...nextMeta.changeLogIds, fileId];
-        nextMeta.seq = changes[changes.length - 1].seq;
-        // 3. Commit Lock
-        await this.saveMeta(nextMeta, this.metaEtag);
-        // 4. Update Local State
-        this.meta = nextMeta;
-        for (const change of changes) {
-            this.updateIndex(change, fileId);
-            if (change.doc) {
-                this.docCache.put(change.id, change.doc);
+        try {
+            // 2. Prepare speculative meta update
+            const nextMeta = { ...this.meta };
+            nextMeta.changeLogIds = [...nextMeta.changeLogIds, fileId];
+            nextMeta.seq = changes[changes.length - 1].seq;
+            // 3. Commit Lock
+            await this.saveMeta(nextMeta, this.metaEtag);
+            // 4. Update Local State
+            this.meta = nextMeta;
+            for (const change of changes) {
+                this.updateIndex(change, fileId);
+                if (change.doc) {
+                    this.docCache.put(change.id, change.doc);
+                }
+                else if (change.deleted) {
+                    this.docCache.remove(change.id);
+                }
             }
-            else if (change.deleted) {
-                this.docCache.remove(change.id);
+            // Notify local changes feed listeners about our own write
+            this.notifyListeners();
+            // 5. Compaction Check
+            const totalChanges = await this.countTotalChanges();
+            if (totalChanges >= this.compactionThreshold ||
+                this.currentLogSizeEstimate >= this.compactionSizeThreshold) {
+                this.compact().catch(e => console.error('Compaction failed', e));
             }
         }
-        // 5. Compaction Check
-        // Count changes since last compaction *pointer*, not just list length
-        const totalChanges = await this.countTotalChanges();
-        if (totalChanges >= this.compactionThreshold ||
-            this.currentLogSizeEstimate >= this.compactionSizeThreshold) {
-            this.compact().catch(e => console.error('Compaction failed', e));
+        catch (err) {
+            // Cleanup orphaned log file on metadata update failure
+            this.client.deleteFile(fileId).catch(e => this.log('Failed to cleanup orphaned log', fileId, e));
+            throw err;
         }
     }
     /** Update Index with a new change */
@@ -348,55 +472,67 @@ class DriveHandler {
     }
     /** Compact: Create SnapshotIndex + SnapshotData */
     async compact() {
-        const snapshotSeq = this.meta.seq;
-        const oldLogIds = [...this.meta.changeLogIds];
-        const oldIndexId = this.meta.snapshotIndexId;
-        // 1. Fetch ALL active documents
-        // We need them to build the new large snapshot-data file
-        // This is the one time we download everything if not cached. 
-        // Optimization: We could reuse existing `snapshot-data` chunks and only append new data 
-        // to a new chunk, but for simplicity: Merge All.
-        const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted);
-        const allDocs = await this.getMulti(allIds);
-        const snapshotData = { docs: {} };
-        allIds.forEach((id, i) => {
-            if (allDocs[i])
-                snapshotData.docs[id] = allDocs[i];
-        });
-        // 2. Upload Data File
-        const dataContent = JSON.stringify(snapshotData);
-        const dataRes = await this.client.createFile(`snapshot-data-${Date.now()}.json`, [this.folderId], 'application/json', dataContent);
-        const dataFileId = dataRes.id;
-        // 3. Create Index pointing to this Data File
-        const newIndexEntries = {};
-        for (const id of Object.keys(snapshotData.docs)) {
-            newIndexEntries[id] = {
-                rev: this.index[id].rev,
-                seq: this.index[id].seq,
-                location: { fileId: dataFileId }
+        if (this.isCompacting)
+            return;
+        this.isCompacting = true;
+        try {
+            this.log('Starting compaction');
+            const snapshotSeq = this.meta.seq;
+            const oldLogIds = [...this.meta.changeLogIds];
+            const oldIndexId = this.meta.snapshotIndexId;
+            // 1. Fetch ALL active documents
+            // We need them to build the new large snapshot-data file
+            // This is the one time we download everything if not cached. 
+            // Optimization: We could reuse existing `snapshot-data` chunks and only append new data 
+            // to a new chunk, but for simplicity: Merge All.
+            const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted);
+            const allDocs = await this.getMulti(allIds);
+            const snapshotData = { docs: {} };
+            allIds.forEach((id, i) => {
+                if (allDocs[i])
+                    snapshotData.docs[id] = allDocs[i];
+            });
+            // 2. Upload Data File
+            const dataContent = JSON.stringify(snapshotData);
+            const dataRes = await this.client.createFile(`snapshot-data-${Date.now()}.json`, [this.folderId], 'application/json', dataContent);
+            const dataFileId = dataRes.id;
+            // 3. Create Index pointing to this Data File
+            const newIndexEntries = {};
+            for (const id of Object.keys(snapshotData.docs)) {
+                newIndexEntries[id] = {
+                    rev: this.index[id].rev,
+                    seq: this.index[id].seq,
+                    location: { fileId: dataFileId }
+                };
+            }
+            const snapshotIndex = {
+                entries: newIndexEntries,
+                seq: snapshotSeq,
+                createdAt: Date.now()
             };
+            const indexContent = JSON.stringify(snapshotIndex);
+            const indexRes = await this.client.createFile(`snapshot-index-${Date.now()}.json`, [this.folderId], 'application/json', indexContent);
+            const newIndexId = indexRes.id;
+            // 4. Update Meta
+            let filesToDelete = [];
+            await this.atomicUpdateMeta((latest) => {
+                const remainingLogs = latest.changeLogIds.filter(id => !oldLogIds.includes(id));
+                // Only delete files that were in oldLogIds but not in remainingLogs
+                filesToDelete = oldLogIds.filter(id => !remainingLogs.includes(id));
+                return {
+                    ...latest,
+                    snapshotIndexId: newIndexId,
+                    changeLogIds: remainingLogs,
+                    lastCompaction: Date.now()
+                };
+            });
+            // 5. Cleanup - Only delete files that were confirmed removed from metadata
+            await this.cleanupOldFiles(oldIndexId, filesToDelete);
+            this.currentLogSizeEstimate = 0;
+        }
+        finally {
+            this.isCompacting = false;
         }
-        const snapshotIndex = {
-            entries: newIndexEntries,
-            seq: snapshotSeq,
-            createdAt: Date.now()
-        };
-        const indexContent = JSON.stringify(snapshotIndex);
-        const indexRes = await this.client.createFile(`snapshot-index-${Date.now()}.json`, [this.folderId], 'application/json', indexContent);
-        const newIndexId = indexRes.id;
-        // 4. Update Meta
-        await this.atomicUpdateMeta((latest) => {
-            const remainingLogs = latest.changeLogIds.filter(id => !oldLogIds.includes(id));
-            return {
-                ...latest,
-                snapshotIndexId: newIndexId,
-                changeLogIds: remainingLogs,
-                lastCompaction: Date.now()
-            };
-        });
-        // 5. Cleanup
-        this.cleanupOldFiles(oldIndexId, oldLogIds); // And potentially old data files if we tracked them
-        this.currentLogSizeEstimate = 0;
     }
     // ... Helpers (atomicUpdateMeta, saveMeta, writeChangeFile same as before) ...
     async atomicUpdateMeta(modifier) {
@@ -407,7 +543,7 @@ class DriveHandler {
                 const metaFile = await this.findFile('_meta.json');
                 if (!metaFile)
                     throw new Error('Meta missing');
-                const validMeta = await this.downloadJson(metaFile.id);
+                const validMeta = await this.downloadJson(metaFile.id, true); // No cache
                 const newMeta = modifier(validMeta);
                 await this.saveMeta(newMeta, metaFile.etag);
                 this.meta = newMeta;
@@ -439,30 +575,33 @@ class DriveHandler {
         const safeName = this.escapeQuery(name);
         const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
         const files = await this.client.listFiles(q);
-        if (files.length > 0)
+        if (files.length > 0) {
+            let file = files[0];
+            if (!file.etag) {
+                // Robustness: Fetch metadata for the file if etag is missing from list
+                try {
+                    file = await this.client.getFileMetadata(file.id);
+                }
+                catch (e) {
+                    this.log('Failed to fetch file metadata for etag', file.id, e);
+                }
+            }
             return {
-                id: files[0].id,
-                etag: files[0].etag || '',
-                modifiedTime: files[0].modifiedTime || ''
+                id: file.id,
+                etag: file.etag,
+                modifiedTime: file.modifiedTime
             };
+        }
         return null;
     }
-    async downloadJson(fileId) {
-        return await this.client.getFile(fileId);
+    async downloadJson(fileId, skipCache = false) {
+        return await this.fetchFile(fileId, skipCache);
     }
     async downloadFileAny(fileId) {
-        return await this.client.getFile(fileId);
+        return await this.fetchFile(fileId);
     }
     async downloadNdjson(fileId) {
-        const data = await this.client.getFile(fileId);
-        // data will likely be a string if NDJSON is returned and getFile sees weird content-type
-        // Or if getFile auto-parsed standard "application/json" but NDJSON is just text.
-        // Google Drive might return application/json for everything if we aren't careful?
-        // Actually .ndjson is separate.
-        // Safest: Handle string or object.
-        const content = typeof data === 'string' ? data : JSON.stringify(data);
-        const lines = content.trim().split('\n').filter((l) => l);
-        return lines.map((line) => JSON.parse(line));
+        return await this.fetchFile(fileId);
     }
     async writeChangeFile(changes) {
         const lines = changes.map(c => JSON.stringify(c)).join('\n') + '\n';
@@ -479,6 +618,7 @@ class DriveHandler {
             const res = await this.client.updateFile(metaFile.id, content, expectedEtag || undefined);
             this.metaEtag = res.etag;
             this.metaModifiedTime = res.modifiedTime;
+            this.fileCache.remove(metaFile.id); // Invalidate cache
         }
         else {
             const res = await this.client.createFile('_meta.json', [this.folderId], 'application/json', content);
@@ -487,41 +627,75 @@ class DriveHandler {
         }
     }
     async countTotalChanges() {
-        // Calculate diff between meta.seq and snapshot seq
-        // But we don't store snapshot seq in meta directly?
-        // We can approximate by pending changes count + known gaps?
-        // Actually we used to check snapshot.seq. 
-        // We can assume snapshot is somewhat recent.
-        return this.pendingChanges.length + 10; // dummy for now, rely on log size
+        // If no snapshot exists yet, total changes = meta.seq (all changes)
+        if (!this.meta.snapshotIndexId) {
+            return this.meta.seq;
+        }
+        // Each log file ID in changeLogIds represents some number of changes.
+        // For simplicity and to trigger compaction based on file count (which is what matters for Drive),
+        // we can return the number of log files. 
+        // But since compactionThreshold is usually in ENTRIES, let's keep a rough estimate
+        // or just return the log file count if that's what the user expects.
+        // The previous "* 5" was too aggressive. 
+        // Let's assume on average 1 change per log file in tests (worst case).
+        return this.meta.changeLogIds.length + this.pendingChanges.length;
     }
     async cleanupOldFiles(oldIndexId, oldLogIds) {
-        if (oldIndexId)
+        const deleteFile = async (fileId) => {
             try {
-                await this.client.deleteFile(oldIndexId);
+                await this.client.deleteFile(fileId);
+                this.log('Deleted file', fileId);
             }
-            catch { }
-        for (const id of oldLogIds)
-            try {
-                await this.client.deleteFile(id);
+            catch (err) {
+                // 404 is ok - file already deleted or doesn't exist
+                if (err.status === 404 || err.code === 404) {
+                    this.log('File already deleted or not found', fileId);
+                    return;
+                }
+                // Log other errors but don't fail
+                this.log('Failed to delete file', fileId, err);
             }
-            catch { }
+        };
+        if (oldIndexId) {
+            await deleteFile(oldIndexId);
+        }
+        for (const id of oldLogIds) {
+            await deleteFile(id);
+        }
     }
     startPolling(intervalMs) {
+        this.log('Starting polling with interval', { intervalMs });
+        if (isNaN(intervalMs) || intervalMs <= 0)
+            return;
         if (this.pollingInterval)
             clearInterval(this.pollingInterval);
         this.pollingInterval = setInterval(async () => {
+            this.log('Polling tick...');
+            if (this.isPollingActive) {
+                this.log('Polling already in progress, skipping tick');
+                return;
+            }
+            this.isPollingActive = true;
             try {
                 const metaFile = await this.findFile('_meta.json');
-                if (!metaFile)
+                if (!metaFile) {
+                    this.log('Polling: _meta.json not found');
                     return;
-                // Use modifiedTime for polling as it's readable in projections
-                if (metaFile.modifiedTime !== this.metaModifiedTime) {
+                }
+                // Compare etags, falling back to modifiedTime
+                this.log('Polling: comparing etag', metaFile.etag, 'with', this.metaEtag);
+                if ((metaFile.etag && this.metaEtag && metaFile.etag !== this.metaEtag) ||
+                    (!metaFile.etag && metaFile.modifiedTime !== this.metaModifiedTime)) {
+                    this.log('Polling detected change!', metaFile.etag || metaFile.modifiedTime);
                     await this.load();
                     this.notifyListeners();
                 }
             }
             catch (err) {
-                console.error('Polling error', err);
+                this.log('Polling error', err);
+            }
+            finally {
+                this.isPollingActive = false;
             }
         }, intervalMs);
     }
@@ -534,13 +708,23 @@ class DriveHandler {
         // Adapter needs to handle this.
         const changes = {};
         for (const [id, entry] of Object.entries(this.index)) {
-            changes[id] = { _id: id, _rev: entry.rev, _deleted: entry.deleted };
+            changes[id] = {
+                _id: id,
+                _rev: entry.rev,
+                _deleted: !!entry.deleted,
+                seq: entry.seq // IMPORTANT: Missing previously, preventing filtered changes from working
+            };
         }
-        for (const l of this.listeners)
-            l(changes);
+        for (const cb of this.listeners)
+            cb(changes);
     }
     // For tests/debug
-    onChange(cb) { this.listeners.push(cb); }
+    onChange(cb) {
+        this.listeners.push(cb);
+        return () => {
+            this.listeners = this.listeners.filter(l => l !== cb);
+        };
+    }
     stopPolling() { if (this.pollingInterval)
         clearInterval(this.pollingInterval); }
     escapeQuery(value) {