@docstack/pouchdb-adapter-googledrive 0.0.1

package/.env.example

@@ -0,0 +1,6 @@
+# Google Drive API Credentials
+GOOGLE_CLIENT_ID=your_client_id
+GOOGLE_CLIENT_SECRET=your_client_secret
+GOOGLE_REDIRECT_URL=your_redirect_url
+GOOGLE_ACCESS_TOKEN=your_access_token
+GOOGLE_REFRESH_TOKEN=your_refresh_token

package/DOCUMENTATION.md

@@ -0,0 +1,54 @@
+# Architecture & Design Documentation
+
+## 1. Core Principles
+The `pouchdb-adapter-googledrive` implementation is built on three core pillars to ensure data integrity and performance on a file-based remote storage system.
+
+### A. Append-Only Log (Storage)
+Instead of modifying a single database file (which is prone to conflicts), we use an **Append-Only** strategy.
+- **Changes**: Every write operation (or batch of writes) creates a **new, immutable file** (e.g., `changes-{seq}-{uuid}.ndjson`).
+- **Snapshots**: Periodically, the log is compacted into a `snapshot` file.
+- **Benefit**: Historical data is preserved until compaction, and file-write conflicts are minimized.
+
+### B. Optimistic Concurrency Control (OCC)
+To prevent race conditions (two clients writing simultaneously), we use **ETag-based locking** on a single entry point: `_meta.json`.
+- **The Lock**: `_meta.json` holds the current Sequence Number and the list of active log files.
+- **The Protocol**:
+    1. Reader fetches `_meta.json` and its `ETag`.
+    2. Writer prepares a new change file and uploads it (orphaned initially).
+    3. Writer attempts to update `_meta.json` with the new file reference, sending `If-Match: <Old-ETag>`.
+    4. **Success**: The change is now officially part of the DB.
+    5. **Failure (412/409)**: Another client updated the DB. The writer deletes its orphaned file, pulls the new state, and retries the logical operation.
+
+### C. Remote-First "Lazy" Loading (Memory Optimization)
+To support large databases without exhausting client memory, we separate **Metadata** from **Content**.
+
+#### Storage Structure
+- `_meta.json`: Root pointer. Small.
+- `snapshot-index.json`: A map of `{ docId: { rev, filePointer } }`. Medium size (~100 bytes/doc). Loaded at startup.
+- `snapshot-data.json`: The actual document bodies. Large. **Never fully loaded.**
+- `changes-*.ndjson`: Recent updates.
+
+#### Client Startup Sequence
+1.  **Fetch Meta**: Download `_meta.json` and get the `snapshotIndexId`.
+2.  **Fetch Index**: Download `snapshot-index.json`. This builds the "Revision Tree" in memory.
+3.  **Replay Logs**: Download and parse only the small `changes-*.ndjson` files created since the snapshot to update the in-memory Index.
+4.  **Ready**: The client is now ready to query keys. No document content has been downloaded yet.
+
+#### On-Demand Usage
+- **`db.get(id)`**: 
+    1. Look up `id` in the **Memory Index** to find the `filePointer`.
+    2. Check **LRU Cache**.
+    3. If missing, fetch the specific file/range from Google Drive.
+- **`db.allDocs({ keys: [...] })`**: Efficiently looks up pointers and fetches only requested docs.
+
+## 2. Technical Patterns
+
+### Atomic Compaction
+Compaction is a critical maintenance task that merges the `snapshot-data` with recent `changes` to create a new baseline.
+- **Safe**: Limits memory usage by streaming/batching.
+- **Atomic**: Uploads the new snapshot as a new file. Swaps the pointer in `_meta.json` using OCC.
+- **Zero-Downtime**: Clients can continue reading/writing to the old logs while compaction runs. Writes that happen *during* compaction are detected via the ETag check, causing the compaction to abort/retry safeley.
+
+### Conflict Handling
+- **PouchDB Level**: Standard CouchDB revision conflicts (409) are preserved. A "winner" is chosen deterministically, but conflicting revisions are kept in the tree (requires `snapshot-index` to store the full revision tree, not just the winner).
+- **Adapter Level**: Drive API 409s handling (retry logic) ensures the transport layer is reliable.

package/README.md

@@ -0,0 +1,78 @@
+# PouchDB Adapter for Google Drive
+
+A PouchDB adapter that uses Google Drive as a backend storage.
+
+## Features
+
+- **Append-Only Log**: Uses an efficient append-only log pattern for fast writes.
+- **Auto-Compaction**: Automatically merges logs into a snapshot when thresholds are met.
+- **Offline/Sync**: Supports PouchDB's replication and sync capabilities.
+- **TypeScript**: Written in TypeScript with full type definitions.
+
+## Installation
+
+```bash
+npm install @docstack/pouchdb-adapter-googledrive
+```
+
+## Usage
+
+```typescript
+import PouchDB from 'pouchdb-core';
+import GoogleDriveAdapter from '@docstack/pouchdb-adapter-googledrive';
+import { google } from 'googleapis';
+
+// Register the adapter
+PouchDB.plugin(GoogleDriveAdapter);
+
+// Setup Google Drive Client (Authenticated)
+const oauth2Client = new google.auth.OAuth2(
+  YOUR_CLIENT_ID,
+  YOUR_CLIENT_SECRET,
+  YOUR_REDIRECT_URL
+);
+oauth2Client.setCredentials({ access_token: '...' });
+
+const drive = google.drive({ version: 'v3', auth: oauth2Client });
+
+// Create Database
+const db = new PouchDB('my-drive-db', {
+  adapter: 'googledrive',
+  drive: drive,              // valid googleapis Drive instance
+  folderId: '...',           // Optional: Folder ID to store database files
+  folderName: 'my-db',       // Optional: Folder name (created if not exists)
+  pollingIntervalMs: 2000,   // Optional: Check for remote changes
+  compactionThreshold: 50    // Optional: Number of changes before auto-compaction
+});
+```
+
+## How it works
+
+The adapter implements an **append-only log** pattern for efficiency and reliability:
+
+1. **Folder Structure**: Each database is a folder in Google Drive.
+2. **`_meta.json`**: Tracks the current sequence number and active log files.
+3. **`snapshot.json`**: Contains the full database state at a specific sequence point.
+4. **`changes-{timestamp}.ndjson`**: New changes are appended to these newline-delimited JSON files.
+
+### Compaction
+
+To prevent the change logs from growing indefinitely, the adapter performs auto-compaction:
+- When the number of pending changes exceeds `compactionThreshold` (default: 100).
+- Or when the log file size exceeds `compactionSizeThreshold` (default: 1MB).
+
+Compaction merges the snapshot and all change logs into a new `snapshot.json` and deletes old log files.
+
+## Testing
+
+To run the tests, you need to provide Google Drive API credentials.
+
+1. Copy `.env.example` to `.env`:
+   ```bash
+   cp .env.example .env
+   ```
+2. Fill in your Google Cloud credentials in `.env`.
+3. Run the tests:
+   ```bash
+   npm test
+   ```

package/error.log

@@ -0,0 +1,21 @@
+FAIL tests/adapter.test.ts
+  ΓùÅ Test suite failed to run
+
+    tests/adapter.test.ts:51:13 - error TS2353: Object literal may only specify known properties, and 'drive' does not exist in type 'DatabaseConfiguration'.
+
+    51             drive: drive,
+                   ~~~~~
+    tests/adapter.test.ts:57:21 - error TS2339: Property 'backend_adapter' does not exist on type 'DatabaseInfo'.
+
+    57         expect(info.backend_adapter).toBe('googledrive');
+                           ~~~~~~~~~~~~~~~
+    tests/adapter.test.ts:65:24 - error TS2339: Property 'title' does not exist on type 'IdMeta & GetMeta'.
+
+    65         expect(fetched.title).toBe('Start Wars');
+                              ~~~~~
+
+Test Suites: 1 failed, 1 total
+Tests:       0 total
+Snapshots:   0 total
+Time:        14.401 s
+Ran all test suites matching tests/adapter.test.ts.

package/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    testMatch: ['**/tests/**/*.test.ts'],
+    transform: {
+        '^.+\\.ts$': 'ts-jest',
+    },
+};

package/lib/adapter.d.ts

@@ -0,0 +1,17 @@
+import { GoogleDriveAdapterOptions } from './types';
+/** Combined options type for PouchDB adapter */
+interface AdapterOptions extends GoogleDriveAdapterOptions {
+    name: string;
+}
+/** Callback type for adapter initialization */
+type AdapterCallback = (err: Error | null, api?: any) => void;
+/**
+ * GoogleDriveAdapter - PouchDB adapter for Google Drive storage.
+ * Updated for Lazy Loading (Async Access).
+ */
+export declare function GoogleDriveAdapter(PouchDB: any): {
+    (this: any, opts: AdapterOptions, callback: AdapterCallback): void;
+    valid(): boolean;
+    use_prefix: boolean;
+};
+export default GoogleDriveAdapter;

package/lib/adapter.js

@@ -0,0 +1,440 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GoogleDriveAdapter = GoogleDriveAdapter;
+const drive_1 = require("./drive");
+/**
+ * Schedule a function to run asynchronously.
+ */
+function nextTick(fn) {
+    queueMicrotask(fn);
+}
+/**
+ * GoogleDriveAdapter - PouchDB adapter for Google Drive storage.
+ * Updated for Lazy Loading (Async Access).
+ */
+function GoogleDriveAdapter(PouchDB) {
+    function GoogleDrivePouch(opts, callback) {
+        const api = this;
+        const name = opts.name;
+        // Clone options to avoid mutation
+        const adapterOpts = Object.assign({}, opts);
+        // Internal state
+        let instanceId;
+        let db;
+        // Initialize DriveHandler
+        db = new drive_1.DriveHandler(adapterOpts, name);
+        // After database is initialized
+        function afterDBCreated() {
+            instanceId = 'gdrive-' + name + '-' + Date.now().toString(36);
+            nextTick(function () {
+                callback(null, api);
+            });
+        }
+        // Load data from Drive and initialize
+        db.load().then(() => {
+            afterDBCreated();
+        }).catch((err) => {
+            callback(err);
+        });
+        // ============ PouchDB Adapter API Methods ============
+        api._remote = false;
+        api.type = function () {
+            return 'googledrive';
+        };
+        api._id = function (callback) {
+            callback(null, instanceId);
+        };
+        // Info now must be async-ish (calculated from Index)
+        api._info = function (callback) {
+            const keys = db.getIndexKeys();
+            const docCount = keys.length; // Approximate (doesn't account for deleted unless filtered)
+            // Filter deleted for accurate count
+            let alive = 0;
+            for (const k of keys) {
+                const entry = db.getIndexEntry(k);
+                if (entry && !entry.deleted)
+                    alive++;
+            }
+            const res = {
+                doc_count: alive,
+                update_seq: db.seq,
+                backend_adapter: 'googledrive'
+            };
+            nextTick(function () {
+                callback(null, res);
+            });
+        };
+        // Get a single document by ID (Async fetch)
+        api._get = function (id, opts, callback) {
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
+            db.get(id).then(doc => {
+                if (!doc) {
+                    return callback({
+                        status: 404,
+                        error: true,
+                        name: 'not_found',
+                        message: 'missing',
+                        reason: 'missing'
+                    });
+                }
+                const metadata = {
+                    id: doc._id,
+                    rev: doc._rev,
+                    winningRev: doc._rev,
+                    deleted: !!doc._deleted
+                };
+                callback(null, { doc, metadata });
+            }).catch(err => {
+                callback(err);
+            });
+        };
+        // Get all documents (Lazy stream or fetch)
+        api._allDocs = function (opts, callback) {
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
+            const keys = db.getIndexKeys();
+            const total = keys.length; // Total keys (including deleted?)
+            let startIndex = opts.skip || 0;
+            let limit = typeof opts.limit === 'number' ? opts.limit : keys.length;
+            let filteredKeys = keys;
+            if (opts.startkey)
+                filteredKeys = filteredKeys.filter(k => k >= opts.startkey);
+            if (opts.endkey)
+                filteredKeys = filteredKeys.filter(k => k <= opts.endkey);
+            if (opts.key)
+                filteredKeys = filteredKeys.filter(k => k === opts.key);
+            if (opts.keys)
+                filteredKeys = opts.keys;
+            filteredKeys.sort();
+            if (opts.descending)
+                filteredKeys.reverse();
+            const sliced = filteredKeys.slice(startIndex, startIndex + limit);
+            // Fetch actual docs if needed
+            if (opts.include_docs) {
+                db.getMulti(sliced).then(docs => {
+                    const rows = sliced.map((id, i) => {
+                        const doc = docs[i];
+                        const entry = db.getIndexEntry(id);
+                        if (!doc && (!entry || entry.deleted))
+                            return { key: id, error: 'not_found' };
+                        if (!doc && entry) {
+                            // This implies fetch failed but exists in index? Or null result.
+                            return { key: id, error: 'not_found' };
+                        }
+                        const row = {
+                            id,
+                            key: id,
+                            value: { rev: entry?.rev || doc._rev }
+                        };
+                        row.doc = doc;
+                        return row;
+                    });
+                    const result = {
+                        total_rows: total,
+                        offset: startIndex,
+                        rows: rows.filter(r => !r.error || !opts.keys) // Filter errored unless specifically asked via keys?
+                        // CouchDB usually returns error row if distinct keys requested.
+                    };
+                    if (opts.update_seq)
+                        result.update_seq = db.seq;
+                    callback(null, result);
+                }).catch(err => callback(err));
+            }
+            else {
+                // Index only (Fast!)
+                const rows = sliced.map(id => {
+                    const entry = db.getIndexEntry(id);
+                    if (!entry || entry.deleted)
+                        return { key: id, error: 'not_found' };
+                    return {
+                        id,
+                        key: id,
+                        value: { rev: entry.rev }
+                    };
+                });
+                const result = {
+                    total_rows: total,
+                    offset: startIndex,
+                    rows
+                };
+                if (opts.update_seq)
+                    result.update_seq = db.seq;
+                nextTick(() => callback(null, result));
+            }
+        };
+        // Bulk document operations
+        api._bulkDocs = function (req, opts, callback) {
+            const docs = req.docs;
+            const results = [];
+            const newEdits = opts.new_edits !== false;
+            const changes = [];
+            // We need to validate revisions against Index
+            // This does NOT require fetching bodies usually
+            for (const doc of docs) {
+                const id = doc._id;
+                const seq = db.getNextSeq() + changes.length;
+                const entry = db.getIndexEntry(id);
+                if (doc._deleted) {
+                    if (!entry || entry.deleted) {
+                        results.push({
+                            ok: false,
+                            id,
+                            error: 'not_found',
+                            reason: 'missing'
+                        });
+                        continue;
+                    }
+                    // Check rev
+                    const oldRev = entry.rev || '0-0'; // Index has latest
+                    // If mismatch? PouchDB handles conflict logic before calling us sometimes?
+                    // But we should verify. 
+                    // If doc._rev matches entry.rev, we are good.
+                    const revNum = parseInt(oldRev.split('-')[0], 10) + 1;
+                    const newRev = revNum + '-' + generateRevId();
+                    changes.push({
+                        seq,
+                        id,
+                        rev: newRev,
+                        deleted: true,
+                        timestamp: Date.now()
+                    });
+                    results.push({ ok: true, id, rev: newRev });
+                }
+                else {
+                    let newRev;
+                    if (newEdits) {
+                        const oldRev = entry?.rev || '0-0';
+                        const revNum = parseInt(oldRev.split('-')[0], 10) + 1;
+                        newRev = revNum + '-' + generateRevId();
+                    }
+                    else {
+                        newRev = doc._rev;
+                    }
+                    const savedDoc = Object.assign({}, doc, { _rev: newRev });
+                    changes.push({
+                        seq,
+                        id,
+                        rev: newRev,
+                        doc: savedDoc,
+                        timestamp: Date.now()
+                    });
+                    results.push({ ok: true, id, rev: newRev });
+                }
+            }
+            // Append changes to log
+            db.appendChanges(changes).then(() => {
+                nextTick(() => callback(null, results));
+            }).catch((err) => {
+                callback(err);
+            });
+        };
+        // Changes feed
+        api._changes = function (opts) {
+            opts = Object.assign({}, opts);
+            const since = opts.since || 0;
+            const limit = typeof opts.limit === 'number' ? opts.limit : Infinity;
+            const returnDocs = opts.return_docs !== false;
+            const results = [];
+            let lastSeq = since;
+            let complete = false;
+            // Should we iterate Index or Logs?
+            // "Index" only has LATEST state. _changes usually wants history if `since` is old.
+            // But this adapter is an "Index + Log" adapter.
+            // If `since` is 0, we can walk the Index.
+            // If `since` is recent, we can maybe walk pending changes?
+            // Correct implementation of `_changes` with Append-Only Log requires reading the log files essentially.
+            // BUT, standard PouchDB `_changes` often just iterates all docs if it can't stream.
+            // For now, let's iterate the INDEX (Winning Revisions) which implies "since=0" behavior effectively (State of the World).
+            function processChanges() {
+                const keys = db.getIndexKeys(); // IDs
+                let processed = 0;
+                // Index-based iteration
+                for (const id of keys) {
+                    if (complete || processed >= limit)
+                        break;
+                    const entry = db.getIndexEntry(id);
+                    if (!entry)
+                        continue;
+                    // Filter by seq?
+                    if (entry.seq <= since)
+                        continue; // Already seen
+                    const change = {
+                        id: id,
+                        seq: entry.seq,
+                        changes: [{ rev: entry.rev }],
+                    };
+                    if (opts.include_docs) {
+                        // We need to fetch it!
+                        // This makes _changes with include_docs SLOW.
+                        // We can't do this synchronously here easily because `processChanges` is sync in original code?
+                        // Wait, original was `nextTick(processChanges)`.
+                        // We need to be async here.
+                    }
+                    // Supporting async processChanges is cleaner.
+                }
+                // ... This requires rewrite for async iteration ...
+            }
+            // Simplified Async Version
+            async function processChangesAsync() {
+                const keys = db.getIndexKeys();
+                let processed = 0;
+                for (const id of keys) {
+                    if (complete || processed >= limit)
+                        break;
+                    const entry = db.getIndexEntry(id);
+                    if (!entry || entry.seq <= since)
+                        continue;
+                    const change = {
+                        id: id,
+                        seq: entry.seq,
+                        changes: [{ rev: entry.rev }]
+                    };
+                    if (opts.include_docs) {
+                        change.doc = await db.get(id);
+                    }
+                    if (opts.onChange)
+                        opts.onChange(change);
+                    if (returnDocs)
+                        results.push(change);
+                    processed++;
+                    lastSeq = Math.max(lastSeq, entry.seq);
+                }
+                if (opts.complete && !complete) {
+                    opts.complete(null, { results, last_seq: lastSeq });
+                }
+            }
+            if (opts.live) {
+                db.onChange((changes) => {
+                    // Update feed
+                    // ...
+                });
+            }
+            nextTick(() => {
+                processChangesAsync().catch(err => {
+                    console.error('Changes feed error', err);
+                    if (opts.complete)
+                        opts.complete(err);
+                });
+            });
+            return {
+                cancel() {
+                    complete = true;
+                }
+            };
+        };
+        // Manual compaction trigger
+        api._compact = function (callback) {
+            db.compact().then(() => {
+                callback(null, { ok: true });
+            }).catch((err) => {
+                callback(err);
+            });
+        };
+        api._getRevisionTree = function (docId, callback) {
+            db.get(docId).then(doc => {
+                if (!doc) {
+                    return callback({ status: 404, error: true, name: 'not_found', message: 'missing' });
+                }
+                const revTree = [{
+                        pos: 1,
+                        ids: [doc._rev.split('-')[1], { status: 'available' }, []]
+                    }];
+                callback(null, revTree);
+            }).catch(callback);
+        };
+        api._close = function (callback) {
+            db.stopPolling();
+            nextTick(callback);
+        };
+        api._destroy = function (opts, callback) {
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
+            db.stopPolling();
+            if (opts.deleteFolder) {
+                db.deleteFolder().then(() => {
+                    callback(null, { ok: true });
+                }).catch((err) => {
+                    callback(err);
+                });
+            }
+            else {
+                nextTick(() => callback(null, { ok: true }));
+            }
+        };
+        api._putLocal = function (doc, opts, callback) {
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
+            const id = doc._id;
+            const rev = '0-1';
+            const savedDoc = Object.assign({}, doc, { _rev: rev });
+            const change = {
+                seq: db.getNextSeq(),
+                id,
+                rev,
+                doc: savedDoc,
+                timestamp: Date.now()
+            };
+            db.appendChanges([change]).then(() => {
+                callback(null, { ok: true, id, rev });
+            }).catch((err) => {
+                callback(err);
+            });
+        };
+        api._getLocal = function (id, callback) {
+            db.get(id).then(doc => {
+                if (!doc)
+                    return callback({ status: 404, error: true, name: 'not_found' });
+                callback(null, doc);
+            }).catch(callback);
+        };
+        api._removeLocal = function (doc, opts, callback) {
+            // ... Similar async update ...
+            if (typeof opts === 'function') {
+                callback = opts;
+                opts = {};
+            }
+            const id = doc._id;
+            // Check existence async if we want to be strict, but index check is ok
+            if (!db.getIndexEntry(id)) {
+                return callback({ status: 404, error: true, name: 'not_found' });
+            }
+            // ...
+            // Simplified removeLocal
+            const change = {
+                seq: db.getNextSeq(),
+                id,
+                rev: '0-0',
+                deleted: true,
+                timestamp: Date.now()
+            };
+            db.appendChanges([change]).then(() => {
+                callback(null, { ok: true, id, rev: '0-0' });
+            }).catch((err) => {
+                callback(err);
+            });
+        };
+    }
+    // Static properties
+    GoogleDrivePouch.valid = function () {
+        return true;
+    };
+    GoogleDrivePouch.use_prefix = false;
+    return GoogleDrivePouch;
+}
+/**
+ * Generate a random revision ID
+ */
+function generateRevId() {
+    return Math.random().toString(36).substring(2, 11) +
+        Math.random().toString(36).substring(2, 11);
+}
+exports.default = GoogleDriveAdapter;

package/lib/cache.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Simple Least Recently Used (LRU) Cache
+ */
+export declare class LRUCache<K, V> {
+    private capacity;
+    private map;
+    constructor(capacity: number);
+    get(key: K): V | undefined;
+    put(key: K, value: V): void;
+    remove(key: K): void;
+    clear(): void;
+}

package/lib/cache.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LRUCache = void 0;
+/**
+ * Simple Least Recently Used (LRU) Cache
+ */
+class LRUCache {
+    constructor(capacity) {
+        this.capacity = capacity;
+        this.map = new Map();
+    }
+    get(key) {
+        const value = this.map.get(key);
+        if (value !== undefined) {
+            // Refresh: delete and re-add to end (most contiguous)
+            this.map.delete(key);
+            this.map.set(key, value);
+        }
+        return value;
+    }
+    put(key, value) {
+        if (this.map.has(key)) {
+            // Update existing
+            this.map.delete(key);
+        }
+        else if (this.map.size >= this.capacity) {
+            // Evict least recently used (first item)
+            const firstKey = this.map.keys().next().value;
+            if (firstKey !== undefined) {
+                this.map.delete(firstKey);
+            }
+        }
+        this.map.set(key, value);
+    }
+    remove(key) {
+        this.map.delete(key);
+    }
+    clear() {
+        this.map.clear();
+    }
+}
+exports.LRUCache = LRUCache;