@docstack/pouchdb-adapter-googledrive 0.1.3 → 0.1.5

package/lib/adapter.js

@@ -193,74 +193,72 @@ function GoogleDriveAdapter(PouchDB) {
                 opts = {};
             }
             const promise = (async () => {
-                const keys = await db.getIndexKeys();
-                const total = keys.length; // Total keys (including deleted?)
-                let startIndex = opts.skip || 0;
-                let limit = typeof opts.limit === 'number' ? opts.limit : keys.length;
-                let filteredKeys = keys;
-                if (opts.startkey)
-                    filteredKeys = filteredKeys.filter(k => k >= opts.startkey);
-                if (opts.endkey)
-                    filteredKeys = filteredKeys.filter(k => k <= opts.endkey);
-                if (opts.key)
-                    filteredKeys = filteredKeys.filter(k => k === opts.key);
-                if (opts.keys)
-                    filteredKeys = opts.keys;
-                filteredKeys.sort();
-                if (opts.descending)
-                    filteredKeys.reverse();
-                const sliced = filteredKeys.slice(startIndex, startIndex + limit);
-                // Fetch actual docs if needed
-                if (opts.include_docs) {
-                    const docs = await db.getMulti(sliced);
-                    const rows = sliced.map((id, i) => {
-                        const doc = docs[i];
-                        const entry = db.getIndexEntry(id);
-                        if (!doc && (!entry || entry.deleted))
-                            return { key: id, error: 'not_found' };
-                        if (!doc && entry) {
-                            // This implies fetch failed but exists in index? Or null result.
-                            return { key: id, error: 'not_found' };
-                        }
-                        const row = {
-                            id,
-                            key: id,
-                            value: { rev: entry?.rev || doc._rev }
-                        };
-                        row.doc = doc;
-                        return row;
-                    });
-                    const result = {
-                        total_rows: total,
-                        offset: startIndex,
-                        rows: rows.filter(r => !r.error || !opts.keys) // Filter errored unless specifically asked via keys?
-                        // CouchDB usually returns error row if distinct keys requested.
-                    };
-                    if (opts.update_seq)
-                        result.update_seq = db.seq;
-                    return result;
+                // _local/* docs live in a separate namespace and never appear in allDocs,
+                // same as CouchDB - they aren't part of the B-tree of regular documents.
+                const indexKeys = (await db.getIndexKeys()).filter(k => !k.startsWith('_local/'));
+                // total_rows reflects live (non-deleted) documents, matching db.info().doc_count -
+                // deleted entries stay in the index forever but were never meant to count here.
+                let total = 0;
+                for (const k of indexKeys) {
+                    const entry = db.getIndexEntry(k);
+                    if (entry && !entry.deleted)
+                        total++;
+                }
+                const startIndex = opts.skip || 0;
+                let candidateKeys;
+                if (opts.keys) {
+                    // Explicit keys: return exactly these, in the given order. Deleted docs are
+                    // included here (flagged via value.deleted), not treated as errors - only
+                    // keys absent from the index entirely are 'not_found'.
+                    candidateKeys = opts.keys;
                 }
                 else {
-                    // Index only (Fast!)
-                    const rows = sliced.map(id => {
-                        const entry = db.getIndexEntry(id);
-                        if (!entry || entry.deleted)
-                            return { key: id, error: 'not_found' };
-                        return {
-                            id,
-                            key: id,
-                            value: { rev: entry.rev }
-                        };
+                    // Default listing: deleted docs never appear, same as a real CouchDB/PouchDB
+                    // allDocs() with no keys specified.
+                    candidateKeys = indexKeys.filter(k => {
+                        const entry = db.getIndexEntry(k);
+                        return entry && !entry.deleted;
                     });
-                    const result = {
-                        total_rows: total,
-                        offset: startIndex,
-                        rows
-                    };
-                    if (opts.update_seq)
-                        result.update_seq = db.seq;
-                    return result;
+                    if (opts.startkey)
+                        candidateKeys = candidateKeys.filter(k => k >= opts.startkey);
+                    if (opts.endkey)
+                        candidateKeys = candidateKeys.filter(k => k <= opts.endkey);
+                    if (opts.key)
+                        candidateKeys = candidateKeys.filter(k => k === opts.key);
+                    candidateKeys.sort();
+                    if (opts.descending)
+                        candidateKeys.reverse();
                 }
+                const limit = typeof opts.limit === 'number' ? opts.limit : candidateKeys.length;
+                const sliced = opts.keys ? candidateKeys : candidateKeys.slice(startIndex, startIndex + limit);
+                const docs = opts.include_docs ? await db.getMulti(sliced) : null;
+                const rows = sliced.map((id, i) => {
+                    const entry = db.getIndexEntry(id);
+                    if (!entry)
+                        return { key: id, error: 'not_found' };
+                    if (entry.deleted) {
+                        const row = { id, key: id, value: { rev: entry.rev, deleted: true } };
+                        if (opts.include_docs)
+                            row.doc = null;
+                        return row;
+                    }
+                    const row = { id, key: id, value: { rev: entry.rev } };
+                    if (opts.include_docs) {
+                        const doc = docs[i];
+                        if (!doc)
+                            return { key: id, error: 'not_found' };
+                        row.doc = doc;
+                    }
+                    return row;
+                });
+                const result = {
+                    total_rows: total,
+                    offset: startIndex,
+                    rows
+                };
+                if (opts.update_seq)
+                    result.update_seq = db.seq;
+                return result;
             })();
             if (callback) {
                 promise.then(res => callback(null, res)).catch(err => callback(err));
@@ -314,7 +312,10 @@ function GoogleDriveAdapter(PouchDB) {
         api._bulkDocs = function (req, opts, callback) {
             const docs = req.docs;
             const results = [];
-            const newEdits = opts.new_edits !== false;
+            // new_edits lives on `req` (the CouchDB _bulk_docs request body shape), not `opts`.
+            // api.bulkDocs = api._bulkDocs below means callers reach this directly, bypassing
+            // AbstractPouchDB.prototype.bulkDocs's req->opts normalization - so read it from req.
+            const newEdits = req.new_edits !== false;
             const changes = [];
             // We need to validate revisions against Index
             // This does NOT require fetching bodies usually
@@ -498,7 +499,11 @@ function GoogleDriveAdapter(PouchDB) {
             };
         };
         // Manual compaction trigger
-        api._compact = function (callback) {
+        api._compact = function (opts, callback) {
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
             const promise = db.compact().then(() => {
                 const result = { ok: true };
                 if (callback)

package/lib/drive.js

@@ -287,6 +287,12 @@ class DriveHandler {
             doc = content;
         }
         if (doc) {
+            // HEAL: Ensure PouchDB core validation doesn't fail due to history corruption
+            // This prevents the infinite replicator retry loop when start < ids.length
+            if (doc._revisions && doc._revisions.start < doc._revisions.ids.length) {
+                const start = doc._revisions.start || 1;
+                doc._revisions.ids = doc._revisions.ids.slice(0, start);
+            }
             this.docCache.put(id, doc);
             doc._rev = entry.rev;
         }
@@ -622,6 +628,27 @@ class DriveHandler {
             const snapshotSeq = this.meta.seq;
             const oldLogIds = [...this.meta.changeLogIds];
             const oldIndexId = this.meta.snapshotIndexId;
+            // Resolve which snapshot-data file(s) the OLD index points to, so we can
+            // delete them too once the new snapshot is safely in place. Without this,
+            // every compaction leaves its predecessor's data file behind as orphaned
+            // clutter (the index gets cleaned up, but the (often much larger) data
+            // file it pointed to never does).
+            let oldDataFileIds = [];
+            if (oldIndexId) {
+                try {
+                    const oldIndex = await this.downloadJson(oldIndexId, true);
+                    const ids = new Set();
+                    for (const entry of Object.values(oldIndex.entries || {})) {
+                        const fid = entry.location?.fileId;
+                        if (fid && fid !== 'LEGACY_MEMORY')
+                            ids.add(fid);
+                    }
+                    oldDataFileIds = [...ids];
+                }
+                catch (e) {
+                    this.log('Failed to load old snapshot index for data-file cleanup', oldIndexId, e);
+                }
+            }
             // 1. Fetch ALL active documents
             // We need them to build the new large snapshot-data file
             // This is the one time we download everything if not cached. 
@@ -677,8 +704,11 @@ class DriveHandler {
                     lastCompaction: Date.now()
                 };
             });
-            // 5. Cleanup - Only delete files that were confirmed removed from metadata
-            await this.cleanupOldFiles(oldIndexId, filesToDelete);
+            // 5. Cleanup - Only delete files that were confirmed removed from metadata.
+            // Also delete the old snapshot-data file(s) the old index pointed to (excluding
+            // the freshly-created one, which can't appear here since it's brand new).
+            const staleDataFileIds = oldDataFileIds.filter(id => id !== dataFileId);
+            await this.cleanupOldFiles(oldIndexId, [...filesToDelete, ...staleDataFileIds]);
             this.currentLogSizeEstimate = 0;
         }
         finally {

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "@docstack/pouchdb-adapter-googledrive",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "PouchDB adapter for Google Drive",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "tsc",
     "test": "jest",
-    "test:prod": "TEST_ENV=production jest"
+    "test:prod": "TEST_ENV=production jest",
+    "test:prod:explore": "TEST_ENV=production jest tests/production.explore.test.ts",
+    "test:prod:replication": "TEST_ENV=production jest tests/production.replication.test.ts"
   },
   "keywords": [
     "pouchdb",