@docstack/pouchdb-adapter-googledrive 0.0.4 → 0.0.5

package/DOCUMENTATION.md

@@ -0,0 +1,54 @@
+# Architecture & Design Documentation
+
+## 1. Core Principles
+The `pouchdb-adapter-googledrive` implementation is built on three core pillars to ensure data integrity and performance on a file-based remote storage system.
+
+### A. Append-Only Log (Storage)
+Instead of modifying a single database file (which is prone to conflicts), we use an **Append-Only** strategy.
+- **Changes**: Every write operation (or batch of writes) creates a **new, immutable file** (e.g., `changes-{seq}-{uuid}.ndjson`).
+- **Snapshots**: Periodically, the log is compacted into a `snapshot` file.
+- **Benefit**: Historical data is preserved until compaction, and file-write conflicts are minimized.
+
+### B. Optimistic Concurrency Control (OCC)
+To prevent race conditions (two clients writing simultaneously), we use **ETag-based locking** on a single entry point: `_meta.json`.
+- **The Lock**: `_meta.json` holds the current Sequence Number and the list of active log files.
+- **The Protocol**:
+    1. Reader fetches `_meta.json` and its `ETag`.
+    2. Writer prepares a new change file and uploads it (orphaned initially).
+    3. Writer attempts to update `_meta.json` with the new file reference, sending `If-Match: <Old-ETag>`.
+    4. **Success**: The change is now officially part of the DB.
+    5. **Failure (412/409)**: Another client updated the DB. The writer deletes its orphaned file, pulls the new state, and retries the logical operation.
+
+### C. Remote-First "Lazy" Loading (Memory Optimization)
+To support large databases without exhausting client memory, we separate **Metadata** from **Content**.
+
+#### Storage Structure
+- `_meta.json`: Root pointer. Small.
+- `snapshot-index.json`: A map of `{ docId: { rev, filePointer } }`. Medium size (~100 bytes/doc). Loaded at startup.
+- `snapshot-data.json`: The actual document bodies. Large. **Never fully loaded.**
+- `changes-*.ndjson`: Recent updates.
+
+#### Client Startup Sequence
+1.  **Fetch Meta**: Download `_meta.json` and get the `snapshotIndexId`.
+2.  **Fetch Index**: Download `snapshot-index.json`. This builds the "Revision Tree" in memory.
+3.  **Replay Logs**: Download and parse only the small `changes-*.ndjson` files created since the snapshot to update the in-memory Index.
+4.  **Ready**: The client is now ready to query keys. No document content has been downloaded yet.
+
+#### On-Demand Usage
+- **`db.get(id)`**: 
+    1. Look up `id` in the **Memory Index** to find the `filePointer`.
+    2. Check **LRU Cache**.
+    3. If missing, fetch the specific file/range from Google Drive.
+- **`db.allDocs({ keys: [...] })`**: Efficiently looks up pointers and fetches only requested docs.
+
+## 2. Technical Patterns
+
+### Atomic Compaction
+Compaction is a critical maintenance task that merges the `snapshot-data` with recent `changes` to create a new baseline.
+- **Safe**: Limits memory usage by streaming/batching.
+- **Atomic**: Uploads the new snapshot as a new file. Swaps the pointer in `_meta.json` using OCC.
+- **Zero-Downtime**: Clients can continue reading/writing to the old logs while compaction runs. Writes that happen *during* compaction are detected via the ETag check, causing the compaction to abort/retry safeley.
+
+### Conflict Handling
+- **PouchDB Level**: Standard CouchDB revision conflicts (409) are preserved. A "winner" is chosen deterministically, but conflicting revisions are kept in the tree (requires `snapshot-index` to store the full revision tree, not just the winner).
+- **Adapter Level**: Drive API 409s handling (retry logic) ensures the transport layer is reliable.

package/lib/client.js

@@ -18,10 +18,11 @@ class GoogleDriveClient {
         const headers = new Headers(init.headers);
         headers.set('Authorization', `Bearer ${token}`);
         const res = await fetch(url, { ...init, headers });
+        const method = init.method || 'GET';
         if (!res.ok) {
             // Basic error handling
             const text = await res.text();
-            let errorMsg = `Drive API Error: ${res.status} ${res.statusText}`;
+            let errorMsg = `Drive API Error: ${res.status} ${res.statusText} (${method} ${url})`;
             try {
                 const json = JSON.parse(text);
                 if (json.error && json.error.message) {
@@ -79,6 +80,15 @@ class GoogleDriveClient {
             mimeType,
             parents
         };
+        // Folders or empty content can use simple metadata-only POST
+        if (!content && mimeType === 'application/vnd.google-apps.folder') {
+            const res = await this.fetch(`${BASE_URL}?fields=id,etag`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(metadata)
+            });
+            return await res.json();
+        }
         const multipartBody = this.buildMultipart(metadata, content, mimeType);
         const res = await this.fetch(`${UPLOAD_URL}?uploadType=multipart&fields=id,etag`, {
             method: 'POST',

package/lib/drive.d.ts

@@ -11,6 +11,7 @@ import { GoogleDriveAdapterOptions, ChangeEntry, IndexEntry } from './types';
  */
 export declare class DriveHandler {
     private client;
+    private options;
     private folderId;
     private folderName;
     private parents;
@@ -64,6 +65,7 @@ export declare class DriveHandler {
     private notifyListeners;
     onChange(cb: any): void;
     stopPolling(): void;
+    private escapeQuery;
     deleteFolder(): Promise<void>;
     getNextSeq(): number;
 }

package/lib/drive.js

@@ -34,6 +34,7 @@ class DriveHandler {
         this.listeners = [];
         this.pollingInterval = null;
         this.client = new client_1.GoogleDriveClient(options);
+        this.options = options;
         this.folderId = options.folderId || null;
         this.folderName = options.folderName || dbName;
         this.parents = options.parents || [];
@@ -41,9 +42,7 @@ class DriveHandler {
         this.compactionSizeThreshold = options.compactionSizeThreshold || DEFAULT_SIZE_THRESHOLD;
         this.meta.dbName = dbName;
         this.docCache = new cache_1.LRUCache(options.cacheSize || DEFAULT_CACHE_SIZE);
-        if (options.pollingIntervalMs) {
-            this.startPolling(options.pollingIntervalMs);
-        }
+        // Polling will be started in load() after folderId is resolved
     }
     // Public getter for Sequence (used by adapter)
     get seq() {
@@ -109,6 +108,10 @@ class DriveHandler {
                 }
             }
         }
+        // 3. Start Polling (if enabled)
+        if (this.options.pollingIntervalMs) {
+            this.startPolling(this.options.pollingIntervalMs);
+        }
     }
     // Migration helper
     filesFromLegacySnapshot(snapshot) {
@@ -420,7 +423,8 @@ class DriveHandler {
     }
     // Reused helpers
     async findOrCreateFolder() {
-        const q = `name = '${this.folderName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
+        const safeName = this.escapeQuery(this.folderName);
+        const q = `name = '${safeName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
         const files = await this.client.listFiles(q);
         if (files.length > 0)
             return files[0].id;
@@ -428,7 +432,10 @@ class DriveHandler {
         return createRes.id;
     }
     async findFile(name) {
-        const q = `name = '${name}' and '${this.folderId}' in parents and trashed = false`;
+        if (!this.folderId)
+            return null;
+        const safeName = this.escapeQuery(name);
+        const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
         const files = await this.client.listFiles(q);
         if (files.length > 0)
             return { id: files[0].id, etag: files[0].etag || '' };
@@ -528,6 +535,9 @@ class DriveHandler {
     onChange(cb) { this.listeners.push(cb); }
     stopPolling() { if (this.pollingInterval)
         clearInterval(this.pollingInterval); }
+    escapeQuery(value) {
+        return value.replace(/'/g, "\\'");
+    }
     async deleteFolder() { if (this.folderId)
         await this.client.deleteFile(this.folderId); }
     getNextSeq() { return this.meta.seq + 1; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docstack/pouchdb-adapter-googledrive",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "PouchDB adapter for Google Drive",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",