@docstack/pouchdb-adapter-googledrive 0.1.0 → 0.1.2

package/lib/adapter.js

@@ -350,15 +350,27 @@ function GoogleDriveAdapter(PouchDB) {
                 }
                 else {
                     let newRev;
+                    let savedDoc;
                     if (newEdits) {
                         const oldRev = entry?.rev || '0-0';
                         const revNum = parseInt(oldRev.split('-')[0], 10) + 1;
-                        newRev = revNum + '-' + generateRevId();
+                        const revHash = generateRevId();
+                        newRev = revNum + '-' + revHash;
+                        savedDoc = Object.assign({}, doc, { _rev: newRev });
+                        if (doc._revisions) {
+                            savedDoc._revisions = {
+                                start: revNum,
+                                ids: [revHash, ...(doc._revisions.ids || [])]
+                            };
+                            if (savedDoc._revisions.ids.length > 500) {
+                                savedDoc._revisions.ids = savedDoc._revisions.ids.slice(0, 500);
+                            }
+                        }
                     }
                     else {
                         newRev = doc._rev;
+                        savedDoc = Object.assign({}, doc, { _rev: newRev });
                     }
-                    const savedDoc = Object.assign({}, doc, { _rev: newRev });
                     changes.push({
                         seq,
                         id,
@@ -558,8 +570,16 @@ function GoogleDriveAdapter(PouchDB) {
                 opts = {};
             }
             const id = doc._id;
-            const rev = '0-1';
+            const revNum = 1;
+            const revHash = generateRevId();
+            const rev = revNum + '-' + revHash;
             const savedDoc = Object.assign({}, doc, { _rev: rev });
+            if (doc._revisions) {
+                savedDoc._revisions = {
+                    start: revNum,
+                    ids: [revHash, ...(doc._revisions.ids || [])]
+                };
+            }
             const change = {
                 seq: db.getNextSeq(),
                 id,

package/lib/client.js

@@ -70,7 +70,7 @@ class GoogleDriveClient {
     async listFiles(q) {
         const params = new URLSearchParams({
             q,
-            fields: 'files(id,name,mimeType,parents,modifiedTime)'
+            fields: 'files(id,name,mimeType,parents,modifiedTime,md5Checksum)'
         });
         // FIX: URLSearchParams uses '+', but Drive API is safer with '%20'
         const queryString = params.toString().replace(/\+/g, '%20');
@@ -103,7 +103,7 @@ class GoogleDriveClient {
     }
     // Single metadata get (for etag check)
     async getFileMetadata(fileId) {
-        const params = new URLSearchParams({ fields: 'id,name,mimeType,parents,modifiedTime' });
+        const params = new URLSearchParams({ fields: 'id,name,mimeType,parents,modifiedTime,md5Checksum' });
         const queryString = params.toString().replace(/\+/g, '%20');
         const res = await this.fetch(`${this.baseUrl}/${fileId}?${queryString}`, { method: 'GET' });
         const data = await res.json();

package/lib/drive.d.ts

@@ -19,7 +19,9 @@ export declare class DriveHandler {
     private compactionSizeThreshold;
     private meta;
     private metaEtag;
+    private metaMd5;
     private metaModifiedTime;
+    private localDocsEtag;
     private index;
     private docCache;
     private pendingChanges;
@@ -33,6 +35,8 @@ export declare class DriveHandler {
     private currentSnapshotIndexId;
     private debug;
     private isCompacting;
+    private pendingDownloads;
+    private pendingFinds;
     private log;
     constructor(options: GoogleDriveAdapterOptions, dbName: string);
     get seq(): number;
@@ -56,6 +60,7 @@ export declare class DriveHandler {
     appendChange(change: ChangeEntry): Promise<void>;
     /** Append changes with OCC */
     appendChanges(changes: ChangeEntry[]): Promise<void>;
+    private appendLocalDocs;
     private tryAppendChanges;
     /** Update Index with a new change */
     private updateIndex;

package/lib/drive.js

@@ -30,7 +30,9 @@ class DriveHandler {
             dbName: ''
         };
         this.metaEtag = null;
+        this.metaMd5 = null;
         this.metaModifiedTime = null;
+        this.localDocsEtag = null;
         // In-Memory Index: ID -> Metadata/Pointer
         this.index = {};
         this.pendingChanges = [];
@@ -43,6 +45,8 @@ class DriveHandler {
         this.currentSnapshotIndexId = null;
         this.debug = false;
         this.isCompacting = false;
+        this.pendingDownloads = new Map();
+        this.pendingFinds = new Map();
         const clientOptions = { ...options };
         if (options.testMode) {
             const serverUrl = options.testServerUrl || 'http://localhost:3000';
@@ -95,9 +99,10 @@ class DriveHandler {
                 }
                 const metaFile = await this.findFile('_meta.json');
                 if (metaFile) {
-                    this.log('Retrieved meta file', { fileId: metaFile.id });
-                    this.meta = await this.downloadJson(metaFile.id, true); // No cache for meta
+                    this.log('Retrieved meta file', { fileId: metaFile.fileId });
+                    this.meta = await this.downloadJson(metaFile.fileId, true); // No cache for meta
                     this.metaEtag = metaFile.etag || null;
+                    this.metaMd5 = metaFile.md5Checksum || null;
                     this.metaModifiedTime = metaFile.modifiedTime || null;
                 }
                 else {
@@ -142,36 +147,64 @@ class DriveHandler {
                 }
                 // 2. Replay NEW Change Logs (Metadata only updates)
                 this.log('Replaying change logs');
-                for (const logId of this.meta.changeLogIds) {
-                    if (this.processedLogIds.has(logId))
-                        continue;
-                    this.log('Replaying change log', logId);
-                    let changes = await this.downloadNdjson(logId);
-                    if (!Array.isArray(changes)) {
-                        this.log("Unexpected changes", { changes, logId });
-                        this.log('Downloaded changes not an array, wrapping/ignoring', typeof changes);
-                        changes = changes ? [changes] : []; // Fallback
-                    }
-                    this.currentLogSizeEstimate += 100 * changes.length;
-                    this.log('Replayed change log', logId);
-                    for (const change of changes) {
-                        this.log('Processing change, sequence', change.seq);
-                        this.updateIndex(change, logId);
-                        if (this.docCache.get(change.id)) {
-                            this.docCache.remove(change.id);
+                const pendingLogs = this.meta.changeLogIds.filter(id => !this.processedLogIds.has(id));
+                if (pendingLogs.length > 0) {
+                    this.log(`Downloading ${pendingLogs.length} change logs in parallel`);
+                    const logResults = await Promise.all(pendingLogs.map(async (id) => {
+                        try {
+                            const changes = await this.downloadNdjson(id);
+                            return { id, changes };
+                        }
+                        catch (e) {
+                            this.log(`Failed to download change log ${id}`, e);
+                            return { id, changes: null };
+                        }
+                    }));
+                    for (const { id, changes } of logResults) {
+                        if (!changes) {
+                            this.log(`Skipping failed log ${id}`);
+                            continue;
+                        }
+                        let changesArray = Array.isArray(changes) ? changes : [changes];
+                        this.currentLogSizeEstimate += 100 * changesArray.length;
+                        for (const change of changesArray) {
+                            this.log('Processing change, sequence', change.seq);
+                            this.updateIndex(change, id);
+                            if (this.docCache.get(change.id)) {
+                                this.docCache.remove(change.id);
+                            }
                         }
+                        this.processedLogIds.add(id);
+                        this.log('Processed log', id);
                     }
-                    this.processedLogIds.add(logId);
-                    this.log('Processed log', logId);
-                }
-                // 3. Start Polling (if enabled)
-                if (this.options.pollingIntervalMs) {
-                    this.log('Starting polling with interval', this.options.pollingIntervalMs);
-                    this.startPolling(Number(this.options.pollingIntervalMs));
                 }
-                else {
-                    this.log('Polling disabled (no interval provided)');
+                // 2. Replay NEW Change Logs (Metadata only updates)
+                // ... (previous logic for change logs)
+                // (Already updated in previous turn, keep it)
+                // 2b. Load Local Documents Store (Pinned in meta)
+                if (this.meta.localDocsId) {
+                    this.log('Loading local docs store', this.meta.localDocsId);
+                    try {
+                        const localStore = await this.client.getFileMetadata(this.meta.localDocsId);
+                        this.localDocsEtag = localStore.etag || null;
+                        const localDocsChunk = await this.downloadJson(this.meta.localDocsId, true);
+                        if (localDocsChunk && localDocsChunk.docs) {
+                            for (const [id, doc] of Object.entries(localDocsChunk.docs)) {
+                                this.log('Merging local doc', id);
+                                this.index[id] = {
+                                    rev: doc._rev,
+                                    seq: 0, // Local docs don't participate in shared sequences
+                                    location: { fileId: this.meta.localDocsId }
+                                };
+                                this.docCache.put(id, doc);
+                            }
+                        }
+                    }
+                    catch (e) {
+                        this.log('Failed to load local docs store', e);
+                    }
                 }
+                // 3. Start Polling ...
             }
             catch (e) {
                 console.error('Failed to load database', e);
@@ -259,45 +292,61 @@ class DriveHandler {
                 return cached;
             }
         }
-        this.log('fetchFile downloading', fileId);
-        const data = await this.client.getFile(fileId);
-        let parsed;
-        if (typeof data === 'string') {
-            const trimmed = data.trim();
-            if (trimmed.startsWith('{')) {
-                // Could be JSON or NDJSON
-                if (trimmed.includes('\n')) {
-                    // Definitely NDJSON (multiple lines)
-                    try {
-                        const lines = trimmed.split('\n').filter(l => l);
-                        parsed = lines.map(line => JSON.parse(line));
+        // Always check pending downloads. A download in progress is as fresh as it 
+        // can be right now, so we can reuse it even if skipCache is true.
+        const pending = this.pendingDownloads.get(fileId);
+        if (pending) {
+            this.log('fetchFile reuse pending download', fileId);
+            return await pending;
+        }
+        const downloadPromise = (async () => {
+            try {
+                this.log('fetchFile downloading', fileId);
+                const data = await this.client.getFile(fileId);
+                let parsed;
+                if (typeof data === 'string') {
+                    const trimmed = data.trim();
+                    if (trimmed.startsWith('{')) {
+                        // Could be JSON or NDJSON
+                        if (trimmed.includes('\n')) {
+                            // Definitely NDJSON (multiple lines)
+                            try {
+                                const lines = trimmed.split('\n').filter(l => l);
+                                parsed = lines.map(line => JSON.parse(line));
+                            }
+                            catch (e) {
+                                parsed = data;
+                            }
+                        }
+                        else {
+                            // Single line. Try regular JSON first.
+                            try {
+                                parsed = JSON.parse(trimmed);
+                            }
+                            catch (e) {
+                                parsed = data;
+                            }
+                        }
                     }
-                    catch (e) {
+                    else {
                         parsed = data;
                     }
                 }
                 else {
-                    // Single line. Try regular JSON first.
-                    try {
-                        parsed = JSON.parse(trimmed);
-                        // Optional: if we know it was supposed to be NDJSON? 
-                        // Our change files are always NDJSON. 
-                        // But _meta.json and snapshot-*.json are regular JSON.
-                    }
-                    catch (e) {
-                        parsed = data;
-                    }
+                    parsed = data;
                 }
+                if (!skipCache)
+                    this.fileCache.put(fileId, parsed);
+                return parsed;
             }
-            else {
-                parsed = data;
+            finally {
+                if (!skipCache)
+                    this.pendingDownloads.delete(fileId);
             }
-        }
-        else {
-            parsed = data;
-        }
-        this.fileCache.put(fileId, parsed);
-        return parsed;
+        })();
+        if (!skipCache)
+            this.pendingDownloads.set(fileId, downloadPromise);
+        return await downloadPromise;
     }
     /** Get multiple docs (Atomic-ish) used for _allDocs */
     async getMulti(ids) {
@@ -383,24 +432,33 @@ class DriveHandler {
     /** Append changes with OCC */
     async appendChanges(changes) {
         const MAX_RETRIES = 5;
-        let attempt = 0;
-        while (attempt < MAX_RETRIES) {
+        let attemptNum = 0;
+        const local = changes.filter(c => c.id.startsWith('_local/'));
+        const remote = changes.filter(c => !c.id.startsWith('_local/'));
+        // Handle Local Docs (Pinned Store)
+        if (local.length > 0) {
+            await this.appendLocalDocs(local);
+        }
+        // Handle Remote Docs (App Log)
+        if (remote.length === 0)
+            return;
+        while (attemptNum < MAX_RETRIES) {
             try {
-                return await this.tryAppendChanges(changes);
+                return await this.tryAppendChanges(remote);
             }
             catch (err) {
                 if (err.status === 412 || err.status === 409) {
                     // Reload and RETRY
                     await this.load();
                     // Check conflicts against Index (Metadata sufficient)
-                    this.checkConflicts(changes);
+                    this.checkConflicts(remote);
                     // Reseq
                     let currentSeq = this.meta.seq;
-                    for (const change of changes) {
+                    for (const change of remote) {
                         currentSeq++;
                         change.seq = currentSeq;
                     }
-                    attempt++;
+                    attemptNum++;
                     await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
                     continue;
                 }
@@ -409,6 +467,66 @@ class DriveHandler {
         }
         throw new Error('Failed to append changes');
     }
+    async appendLocalDocs(changes) {
+        const MAX_RETRIES = 5;
+        let attempt = 0;
+        while (attempt < MAX_RETRIES) {
+            try {
+                // 1. Download current local docs (no cache)
+                let store = { docs: {} };
+                let currentEtag = null;
+                if (this.meta.localDocsId) {
+                    try {
+                        const fileMeta = await this.client.getFileMetadata(this.meta.localDocsId);
+                        currentEtag = fileMeta.etag || null;
+                        store = await this.downloadJson(this.meta.localDocsId, true);
+                    }
+                    catch (e) {
+                        if (e.status !== 404)
+                            throw e;
+                    }
+                }
+                // 2. Merge changes
+                for (const change of changes) {
+                    if (change.deleted) {
+                        delete store.docs[change.id];
+                    }
+                    else if (change.doc) {
+                        store.docs[change.id] = change.doc;
+                    }
+                }
+                // 3. Save back
+                const content = JSON.stringify(store);
+                let res;
+                if (this.meta.localDocsId) {
+                    res = await this.client.updateFile(this.meta.localDocsId, content, currentEtag || undefined);
+                }
+                else {
+                    res = await this.client.createFile('_local_docs.json', [this.folderId], 'application/json', content);
+                    // Update Meta with new File ID
+                    await this.atomicUpdateMeta((latest) => ({ ...latest, localDocsId: res.id }));
+                }
+                this.localDocsEtag = res.etag;
+                // Update Index
+                for (const change of changes) {
+                    this.updateIndex(change, res.id);
+                    if (change.doc)
+                        this.docCache.put(change.id, change.doc);
+                    else
+                        this.docCache.remove(change.id);
+                }
+                return;
+            }
+            catch (err) {
+                if (err.status === 412 || err.status === 409) {
+                    attempt++;
+                    await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
+                    continue;
+                }
+                throw err;
+            }
+        }
+    }
     async tryAppendChanges(changes) {
         // 1. Write Log File (Upload Data)
         const fileId = await this.writeChangeFile(changes);
@@ -485,13 +603,22 @@ class DriveHandler {
             // This is the one time we download everything if not cached. 
             // Optimization: We could reuse existing `snapshot-data` chunks and only append new data 
             // to a new chunk, but for simplicity: Merge All.
-            const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted);
+            const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted && !id.startsWith('_local/'));
             const allDocs = await this.getMulti(allIds);
             const snapshotData = { docs: {} };
+            const missingDocs = [];
             allIds.forEach((id, i) => {
-                if (allDocs[i])
+                if (allDocs[i]) {
                     snapshotData.docs[id] = allDocs[i];
+                }
+                else {
+                    missingDocs.push(id);
+                }
             });
+            if (missingDocs.length > 0) {
+                this.log('Compaction ABORTED: Failed to fetch documents', missingDocs);
+                throw new Error(`Compaction failed: missing ${missingDocs.length} documents. Aborting to prevent data loss.`);
+            }
             // 2. Upload Data File
             const dataContent = JSON.stringify(snapshotData);
             const dataRes = await this.client.createFile(`snapshot-data-${Date.now()}.json`, [this.folderId], 'application/json', dataContent);
@@ -543,7 +670,7 @@ class DriveHandler {
                 const metaFile = await this.findFile('_meta.json');
                 if (!metaFile)
                     throw new Error('Meta missing');
-                const validMeta = await this.downloadJson(metaFile.id, true); // No cache
+                const validMeta = await this.downloadJson(metaFile.fileId, true); // No cache
                 const newMeta = modifier(validMeta);
                 await this.saveMeta(newMeta, metaFile.etag);
                 this.meta = newMeta;
@@ -570,29 +697,42 @@ class DriveHandler {
         return createRes.id;
     }
     async findFile(name) {
-        if (!this.folderId)
-            return null;
-        const safeName = this.escapeQuery(name);
-        const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
-        const files = await this.client.listFiles(q);
-        if (files.length > 0) {
-            let file = files[0];
-            if (!file.etag) {
-                // Robustness: Fetch metadata for the file if etag is missing from list
-                try {
-                    file = await this.client.getFileMetadata(file.id);
-                }
-                catch (e) {
-                    this.log('Failed to fetch file metadata for etag', file.id, e);
+        const pending = this.pendingFinds.get(name);
+        if (pending) {
+            this.log('findFile reuse pending search', name);
+            return await pending;
+        }
+        const findPromise = (async () => {
+            const safeName = this.escapeQuery(name);
+            const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
+            try {
+                const files = await this.client.listFiles(q);
+                if (files.length > 0) {
+                    let file = files[0];
+                    if (!file.etag) {
+                        // Robustness: Fetch metadata for the file if etag is missing from list
+                        try {
+                            file = await this.client.getFileMetadata(file.id);
+                        }
+                        catch (e) {
+                            this.log('Failed to fetch file metadata for etag', file.id, e);
+                        }
+                    }
+                    return {
+                        fileId: file.id,
+                        etag: file.etag,
+                        md5Checksum: file.md5Checksum,
+                        modifiedTime: file.modifiedTime
+                    };
                 }
+                return null;
             }
-            return {
-                id: file.id,
-                etag: file.etag,
-                modifiedTime: file.modifiedTime
-            };
-        }
-        return null;
+            finally {
+                this.pendingFinds.delete(name);
+            }
+        })();
+        this.pendingFinds.set(name, findPromise);
+        return await findPromise;
     }
     async downloadJson(fileId, skipCache = false) {
         return await this.fetchFile(fileId, skipCache);
@@ -615,14 +755,16 @@ class DriveHandler {
         const content = JSON.stringify(meta);
         const metaFile = await this.findFile('_meta.json');
         if (metaFile) {
-            const res = await this.client.updateFile(metaFile.id, content, expectedEtag || undefined);
+            const res = await this.client.updateFile(metaFile.fileId, content, expectedEtag || undefined);
             this.metaEtag = res.etag;
+            this.metaMd5 = res.md5Checksum || null;
             this.metaModifiedTime = res.modifiedTime;
-            this.fileCache.remove(metaFile.id); // Invalidate cache
+            this.fileCache.remove(metaFile.fileId); // Invalidate cache
         }
         else {
             const res = await this.client.createFile('_meta.json', [this.folderId], 'application/json', content);
             this.metaEtag = res.etag;
+            this.metaMd5 = res.md5Checksum || null;
             this.metaModifiedTime = res.modifiedTime;
         }
     }
@@ -682,11 +824,25 @@ class DriveHandler {
                     this.log('Polling: _meta.json not found');
                     return;
                 }
-                // Compare etags, falling back to modifiedTime
-                this.log('Polling: comparing etag', metaFile.etag, 'with', this.metaEtag);
-                if ((metaFile.etag && this.metaEtag && metaFile.etag !== this.metaEtag) ||
-                    (!metaFile.etag && metaFile.modifiedTime !== this.metaModifiedTime)) {
-                    this.log('Polling detected change!', metaFile.etag || metaFile.modifiedTime);
+                // Compare etags, falling back to md5Checksum or modifiedTime
+                const remoteEtag = metaFile.etag;
+                const remoteMd5 = metaFile.md5Checksum;
+                const remoteModified = metaFile.modifiedTime;
+                this.log('Polling: comparing etag', remoteEtag, 'with', this.metaEtag, 'md5', remoteMd5, 'with', this.metaMd5);
+                let changed = false;
+                if (remoteEtag && this.metaEtag) {
+                    if (remoteEtag !== this.metaEtag)
+                        changed = true;
+                }
+                else if (remoteMd5 && this.metaMd5) {
+                    if (remoteMd5 !== this.metaMd5)
+                        changed = true;
+                }
+                else if (remoteModified !== this.metaModifiedTime) {
+                    changed = true;
+                }
+                if (changed) {
+                    this.log('Polling detected change!', remoteEtag || remoteMd5 || remoteModified);
                     await this.load();
                     this.notifyListeners();
                 }

package/lib/types.d.ts

@@ -40,6 +40,9 @@ export interface ChangeEntry {
 /** Location pointer for lazy loading */
 export interface FilePointer {
     fileId: string;
+    etag?: string;
+    md5Checksum?: string;
+    modifiedTime?: string;
     /** Optional offset/length for future optimization (packed files) */
     offset?: number;
     length?: number;
@@ -87,6 +90,8 @@ export interface MetaData {
     lastCompaction: number | null;
     /** Database name */
     dbName: string;
+    /** File ID for _local documents store (optimizes by avoiding app-log writes) */
+    localDocsId?: string | null;
     /** Schema Version (for migration) */
     version?: number;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docstack/pouchdb-adapter-googledrive",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "PouchDB adapter for Google Drive",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/jest.config.js

@@ -1,8 +0,0 @@
-module.exports = {
-    preset: 'ts-jest',
-    testEnvironment: 'node',
-    testMatch: ['**/tests/**/*.test.ts'],
-    transform: {
-        '^.+\\.ts$': 'ts-jest',
-    },
-};