@docstack/pouchdb-adapter-googledrive 0.0.4 → 0.0.6

package/DOCUMENTATION.md

@@ -0,0 +1,54 @@
+# Architecture & Design Documentation
+
+## 1. Core Principles
+The `pouchdb-adapter-googledrive` implementation is built on three core pillars to ensure data integrity and performance on a file-based remote storage system.
+
+### A. Append-Only Log (Storage)
+Instead of modifying a single database file (which is prone to conflicts), we use an **Append-Only** strategy.
+- **Changes**: Every write operation (or batch of writes) creates a **new, immutable file** (e.g., `changes-{seq}-{uuid}.ndjson`).
+- **Snapshots**: Periodically, the log is compacted into a `snapshot` file.
+- **Benefit**: Historical data is preserved until compaction, and file-write conflicts are minimized.
+
+### B. Optimistic Concurrency Control (OCC)
+To prevent race conditions (two clients writing simultaneously), we use **ETag-based locking** on a single entry point: `_meta.json`.
+- **The Lock**: `_meta.json` holds the current Sequence Number and the list of active log files.
+- **The Protocol**:
+    1. Reader fetches `_meta.json` and its `ETag`.
+    2. Writer prepares a new change file and uploads it (orphaned initially).
+    3. Writer attempts to update `_meta.json` with the new file reference, sending `If-Match: <Old-ETag>`.
+    4. **Success**: The change is now officially part of the DB.
+    5. **Failure (412/409)**: Another client updated the DB. The writer deletes its orphaned file, pulls the new state, and retries the logical operation.
+
+### C. Remote-First "Lazy" Loading (Memory Optimization)
+To support large databases without exhausting client memory, we separate **Metadata** from **Content**.
+
+#### Storage Structure
+- `_meta.json`: Root pointer. Small.
+- `snapshot-index.json`: A map of `{ docId: { rev, filePointer } }`. Medium size (~100 bytes/doc). Loaded at startup.
+- `snapshot-data.json`: The actual document bodies. Large. **Never fully loaded.**
+- `changes-*.ndjson`: Recent updates.
+
+#### Client Startup Sequence
+1.  **Fetch Meta**: Download `_meta.json` and get the `snapshotIndexId`.
+2.  **Fetch Index**: Download `snapshot-index.json`. This builds the "Revision Tree" in memory.
+3.  **Replay Logs**: Download and parse only the small `changes-*.ndjson` files created since the snapshot to update the in-memory Index.
+4.  **Ready**: The client is now ready to query keys. No document content has been downloaded yet.
+
+#### On-Demand Usage
+- **`db.get(id)`**: 
+    1. Look up `id` in the **Memory Index** to find the `filePointer`.
+    2. Check **LRU Cache**.
+    3. If missing, fetch the specific file/range from Google Drive.
+- **`db.allDocs({ keys: [...] })`**: Efficiently looks up pointers and fetches only requested docs.
+
+## 2. Technical Patterns
+
+### Atomic Compaction
+Compaction is a critical maintenance task that merges the `snapshot-data` with recent `changes` to create a new baseline.
+- **Safe**: Limits memory usage by streaming/batching.
+- **Atomic**: Uploads the new snapshot as a new file. Swaps the pointer in `_meta.json` using OCC.
+- **Zero-Downtime**: Clients can continue reading/writing to the old logs while compaction runs. Writes that happen *during* compaction are detected via the ETag check, causing the compaction to abort/retry safeley.
+
+### Conflict Handling
+- **PouchDB Level**: Standard CouchDB revision conflicts (409) are preserved. A "winner" is chosen deterministically, but conflicting revisions are kept in the tree (requires `snapshot-index` to store the full revision tree, not just the winner).
+- **Adapter Level**: Drive API 409s handling (retry logic) ensures the transport layer is reliable.

package/lib/client.d.ts

@@ -4,6 +4,7 @@ export interface DriveFile {
     mimeType: string;
     parents?: string[];
     etag?: string;
+    modifiedTime?: string;
 }
 export interface DriveClientOptions {
     accessToken: string | (() => Promise<string>);
@@ -19,10 +20,12 @@ export declare class GoogleDriveClient {
     createFile(name: string, parents: string[] | undefined, mimeType: string, content: string): Promise<{
         id: string;
         etag: string;
+        modifiedTime: string;
     }>;
     updateFile(fileId: string, content: string, expectedEtag?: string): Promise<{
         id: string;
         etag: string;
+        modifiedTime: string;
     }>;
     deleteFile(fileId: string): Promise<void>;
     private buildMultipart;

package/lib/client.js

@@ -14,23 +14,44 @@ class GoogleDriveClient {
         return this.options.accessToken;
     }
     async fetch(url, init) {
+        const method = init.method || 'GET';
         const token = await this.getToken();
         const headers = new Headers(init.headers);
         headers.set('Authorization', `Bearer ${token}`);
-        const res = await fetch(url, { ...init, headers });
+        let res;
+        try {
+            res = await fetch(url, { ...init, headers });
+        }
+        catch (networkErr) {
+            const err = new Error(`Network Error: ${networkErr.message} (${method} ${url})`);
+            err.code = 'network_error';
+            err.url = url;
+            err.method = method;
+            throw err;
+        }
         if (!res.ok) {
-            // Basic error handling
             const text = await res.text();
-            let errorMsg = `Drive API Error: ${res.status} ${res.statusText}`;
+            let errorMsg = `Drive API Error: ${res.status} ${res.statusText} (${method} ${url})`;
+            let reason = res.statusText;
             try {
                 const json = JSON.parse(text);
-                if (json.error && json.error.message) {
-                    errorMsg += ` - ${json.error.message}`;
+                const gError = json.error;
+                if (gError) {
+                    errorMsg += ` - ${gError.message || 'Unknown Error'}`;
+                    if (Array.isArray(gError.errors) && gError.errors.length > 0) {
+                        reason = gError.errors[0].reason || reason;
+                        if (gError.errors[0].message && gError.errors[0].message !== gError.message) {
+                            errorMsg += ` (${gError.errors[0].message})`;
+                        }
+                    }
                 }
             }
             catch { }
             const err = new Error(errorMsg);
             err.status = res.status;
+            err.code = reason;
+            err.url = url;
+            err.method = method;
             throw err;
         }
         return res;
@@ -38,11 +59,11 @@ class GoogleDriveClient {
     async listFiles(q) {
         const params = new URLSearchParams({
             q,
-            fields: 'files(id, name, mimeType, parents, etag)',
-            spaces: 'drive',
-            pageSize: '1000' // Ensure we get enough
+            fields: 'files(id,name,mimeType,parents,modifiedTime)'
         });
-        const res = await this.fetch(`${BASE_URL}?${params.toString()}`, { method: 'GET' });
+        // FIX: URLSearchParams uses '+', but Drive API is safer with '%20'
+        const queryString = params.toString().replace(/\+/g, '%20');
+        const res = await this.fetch(`${BASE_URL}?${queryString}`, { method: 'GET' });
         const data = await res.json();
         return data.files || [];
     }
@@ -50,7 +71,8 @@ class GoogleDriveClient {
         // Try getting media
         try {
             const params = new URLSearchParams({ alt: 'media' });
-            const res = await this.fetch(`${BASE_URL}/${fileId}?${params.toString()}`, { method: 'GET' });
+            const queryString = params.toString().replace(/\+/g, '%20');
+            const res = await this.fetch(`${BASE_URL}/${fileId}?${queryString}`, { method: 'GET' });
             // Standard fetch handles JSON/Text transparency? 
             // We expect JSON mostly, but sometimes we might want text.
             // PouchDB adapter flow: downloadJson, downloadNdjson
@@ -69,8 +91,9 @@ class GoogleDriveClient {
     }
     // Single metadata get (for etag check)
     async getFileMetadata(fileId) {
-        const params = new URLSearchParams({ fields: 'id, name, mimeType, parents, etag' });
-        const res = await this.fetch(`${BASE_URL}/${fileId}?${params.toString()}`, { method: 'GET' });
+        const params = new URLSearchParams({ fields: 'id,name,mimeType,parents,modifiedTime' });
+        const queryString = params.toString().replace(/\+/g, '%20');
+        const res = await this.fetch(`${BASE_URL}/${fileId}?${queryString}`, { method: 'GET' });
         return await res.json();
     }
     async createFile(name, parents, mimeType, content) {
@@ -79,25 +102,49 @@ class GoogleDriveClient {
             mimeType,
             parents
         };
+        // Folders or empty content can use simple metadata-only POST
+        if (!content && mimeType === 'application/vnd.google-apps.folder') {
+            const res = await this.fetch(`${BASE_URL}?fields=id,modifiedTime`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(metadata)
+            });
+            const data = await res.json();
+            return {
+                id: data.id,
+                etag: data.etag || '',
+                modifiedTime: data.modifiedTime || ''
+            };
+        }
         const multipartBody = this.buildMultipart(metadata, content, mimeType);
-        const res = await this.fetch(`${UPLOAD_URL}?uploadType=multipart&fields=id,etag`, {
+        const res = await this.fetch(`${UPLOAD_URL}?uploadType=multipart&fields=id,modifiedTime`, {
             method: 'POST',
             headers: {
                 'Content-Type': `multipart/related; boundary=${multipartBody.boundary}`
             },
             body: multipartBody.body
         });
-        return await res.json();
+        const data = await res.json();
+        return {
+            id: data.id,
+            etag: data.etag || '',
+            modifiedTime: data.modifiedTime || ''
+        };
     }
     async updateFile(fileId, content, expectedEtag) {
         // Update content (media) usually, but sometimes meta?
         // In our usage (saveMeta), we update body.
-        const res = await this.fetch(`${UPLOAD_URL}/${fileId}?uploadType=media&fields=id,etag`, {
+        const res = await this.fetch(`${UPLOAD_URL}/${fileId}?uploadType=media&fields=id,modifiedTime`, {
             method: 'PATCH',
             headers: expectedEtag ? { 'If-Match': expectedEtag, 'Content-Type': 'application/json' } : { 'Content-Type': 'application/json' },
             body: content
         });
-        return await res.json();
+        const data = await res.json();
+        return {
+            id: data.id,
+            etag: data.etag || '',
+            modifiedTime: data.modifiedTime || ''
+        };
     }
     async deleteFile(fileId) {
         await this.fetch(`${BASE_URL}/${fileId}`, { method: 'DELETE' });

package/lib/drive.d.ts

@@ -11,6 +11,7 @@ import { GoogleDriveAdapterOptions, ChangeEntry, IndexEntry } from './types';
  */
 export declare class DriveHandler {
     private client;
+    private options;
     private folderId;
     private folderName;
     private parents;
@@ -18,6 +19,7 @@ export declare class DriveHandler {
     private compactionSizeThreshold;
     private meta;
     private metaEtag;
+    private metaModifiedTime;
     private index;
     private docCache;
     private pendingChanges;
@@ -64,6 +66,7 @@ export declare class DriveHandler {
     private notifyListeners;
     onChange(cb: any): void;
     stopPolling(): void;
+    private escapeQuery;
     deleteFolder(): Promise<void>;
     getNextSeq(): number;
 }

package/lib/drive.js

@@ -27,6 +27,7 @@ class DriveHandler {
             dbName: ''
         };
         this.metaEtag = null;
+        this.metaModifiedTime = null;
         // In-Memory Index: ID -> Metadata/Pointer
         this.index = {};
         this.pendingChanges = [];
@@ -34,6 +35,7 @@ class DriveHandler {
         this.listeners = [];
         this.pollingInterval = null;
         this.client = new client_1.GoogleDriveClient(options);
+        this.options = options;
         this.folderId = options.folderId || null;
         this.folderName = options.folderName || dbName;
         this.parents = options.parents || [];
@@ -41,9 +43,7 @@ class DriveHandler {
         this.compactionSizeThreshold = options.compactionSizeThreshold || DEFAULT_SIZE_THRESHOLD;
         this.meta.dbName = dbName;
         this.docCache = new cache_1.LRUCache(options.cacheSize || DEFAULT_CACHE_SIZE);
-        if (options.pollingIntervalMs) {
-            this.startPolling(options.pollingIntervalMs);
-        }
+        // Polling will be started in load() after folderId is resolved
     }
     // Public getter for Sequence (used by adapter)
     get seq() {
@@ -58,6 +58,7 @@ class DriveHandler {
         if (metaFile) {
             this.meta = await this.downloadJson(metaFile.id);
             this.metaEtag = metaFile.etag || null;
+            this.metaModifiedTime = metaFile.modifiedTime || null;
         }
         else {
             await this.saveMeta(this.meta);
@@ -109,6 +110,10 @@ class DriveHandler {
                 }
             }
         }
+        // 3. Start Polling (if enabled)
+        if (this.options.pollingIntervalMs) {
+            this.startPolling(this.options.pollingIntervalMs);
+        }
     }
     // Migration helper
     filesFromLegacySnapshot(snapshot) {
@@ -420,7 +425,8 @@ class DriveHandler {
     }
     // Reused helpers
     async findOrCreateFolder() {
-        const q = `name = '${this.folderName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
+        const safeName = this.escapeQuery(this.folderName);
+        const q = `name = '${safeName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
         const files = await this.client.listFiles(q);
         if (files.length > 0)
             return files[0].id;
@@ -428,10 +434,17 @@ class DriveHandler {
         return createRes.id;
     }
     async findFile(name) {
-        const q = `name = '${name}' and '${this.folderId}' in parents and trashed = false`;
+        if (!this.folderId)
+            return null;
+        const safeName = this.escapeQuery(name);
+        const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
         const files = await this.client.listFiles(q);
         if (files.length > 0)
-            return { id: files[0].id, etag: files[0].etag || '' };
+            return {
+                id: files[0].id,
+                etag: files[0].etag || '',
+                modifiedTime: files[0].modifiedTime || ''
+            };
         return null;
     }
     async downloadJson(fileId) {
@@ -465,10 +478,12 @@ class DriveHandler {
         if (metaFile) {
             const res = await this.client.updateFile(metaFile.id, content, expectedEtag || undefined);
             this.metaEtag = res.etag;
+            this.metaModifiedTime = res.modifiedTime;
         }
         else {
             const res = await this.client.createFile('_meta.json', [this.folderId], 'application/json', content);
             this.metaEtag = res.etag;
+            this.metaModifiedTime = res.modifiedTime;
         }
     }
     async countTotalChanges() {
@@ -499,8 +514,8 @@ class DriveHandler {
                 const metaFile = await this.findFile('_meta.json');
                 if (!metaFile)
                     return;
-                // Etag check
-                if (metaFile.etag !== this.metaEtag) {
+                // Use modifiedTime for polling as it's readable in projections
+                if (metaFile.modifiedTime !== this.metaModifiedTime) {
                     await this.load();
                     this.notifyListeners();
                 }
@@ -528,6 +543,9 @@ class DriveHandler {
     onChange(cb) { this.listeners.push(cb); }
     stopPolling() { if (this.pollingInterval)
         clearInterval(this.pollingInterval); }
+    escapeQuery(value) {
+        return value.replace(/'/g, "\\'");
+    }
     async deleteFolder() { if (this.folderId)
         await this.client.deleteFile(this.folderId); }
     getNextSeq() { return this.meta.seq + 1; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docstack/pouchdb-adapter-googledrive",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "PouchDB adapter for Google Drive",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",