@docstack/pouchdb-adapter-googledrive 0.0.1

package/lib/drive.d.ts

@@ -0,0 +1,69 @@
+import { GoogleDriveAdapterOptions, ChangeEntry, IndexEntry } from './types';
+/**
+ * DriveHandler - Lazy Loading Implementation
+ *
+ * Storage structure:
+ * /db-folder/
+ *   ├── _meta.json
+ *   ├── snapshot-index.json  # Map<DocId, IndexEntry>
+ *   ├── snapshot-data.json   # Map<DocId, DocBody>
+ *   └── changes-*.ndjson     # Append logs
+ */
+export declare class DriveHandler {
+    private drive;
+    private folderId;
+    private folderName;
+    private parents;
+    private compactionThreshold;
+    private compactionSizeThreshold;
+    private meta;
+    private metaEtag;
+    private index;
+    private docCache;
+    private pendingChanges;
+    private currentLogSizeEstimate;
+    private listeners;
+    private pollingInterval;
+    constructor(options: GoogleDriveAdapterOptions, dbName: string);
+    get seq(): number;
+    /** Load the database (Index Only) */
+    load(): Promise<void>;
+    private filesFromLegacySnapshot;
+    /**
+     * Get a document body.
+     * Index -> Cache -> Fetch
+     */
+    get(id: string): Promise<any | null>;
+    /** Get multiple docs (Atomic-ish) used for _allDocs */
+    getMulti(ids: string[]): Promise<any[]>;
+    /** Return all keys in Index */
+    getIndexKeys(): string[];
+    /** Get metadata for a specific ID from Index */
+    getIndexEntry(id: string): IndexEntry | undefined;
+    /** Single change wrapper */
+    appendChange(change: ChangeEntry): Promise<void>;
+    /** Append changes with OCC */
+    appendChanges(changes: ChangeEntry[]): Promise<void>;
+    private tryAppendChanges;
+    /** Update Index with a new change */
+    private updateIndex;
+    private checkConflicts;
+    /** Compact: Create SnapshotIndex + SnapshotData */
+    compact(): Promise<void>;
+    private atomicUpdateMeta;
+    private findOrCreateFolder;
+    private findFile;
+    private downloadJson;
+    private downloadFileAny;
+    private downloadNdjson;
+    private writeChangeFile;
+    private saveMeta;
+    private countTotalChanges;
+    private cleanupOldFiles;
+    private startPolling;
+    private notifyListeners;
+    onChange(cb: any): void;
+    stopPolling(): void;
+    deleteFolder(): Promise<void>;
+    getNextSeq(): number;
+}

package/lib/drive.js

@@ -0,0 +1,573 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DriveHandler = void 0;
+const cache_1 = require("./cache");
+const DEFAULT_COMPACTION_THRESHOLD = 100; // entries
+const DEFAULT_SIZE_THRESHOLD = 1024 * 1024; // 1MB
+const DEFAULT_CACHE_SIZE = 1000; // Number of docs
+/**
+ * DriveHandler - Lazy Loading Implementation
+ *
+ * Storage structure:
+ * /db-folder/
+ *   ├── _meta.json
+ *   ├── snapshot-index.json  # Map<DocId, IndexEntry>
+ *   ├── snapshot-data.json   # Map<DocId, DocBody>
+ *   └── changes-*.ndjson     # Append logs
+ */
+class DriveHandler {
+    constructor(options, dbName) {
+        this.folderId = null;
+        this.meta = {
+            seq: 0,
+            changeLogIds: [],
+            snapshotIndexId: null,
+            lastCompaction: null,
+            dbName: ''
+        };
+        this.metaEtag = null;
+        // In-Memory Index: ID -> Metadata/Pointer
+        this.index = {};
+        this.pendingChanges = [];
+        this.currentLogSizeEstimate = 0;
+        this.listeners = [];
+        this.pollingInterval = null;
+        this.drive = options.drive;
+        this.folderId = options.folderId || null;
+        this.folderName = options.folderName || dbName;
+        this.parents = options.parents || [];
+        this.compactionThreshold = options.compactionThreshold || DEFAULT_COMPACTION_THRESHOLD;
+        this.compactionSizeThreshold = options.compactionSizeThreshold || DEFAULT_SIZE_THRESHOLD;
+        this.meta.dbName = dbName;
+        this.docCache = new cache_1.LRUCache(options.cacheSize || DEFAULT_CACHE_SIZE);
+        if (options.pollingIntervalMs) {
+            this.startPolling(options.pollingIntervalMs);
+        }
+    }
+    // Public getter for Sequence (used by adapter)
+    get seq() {
+        return this.meta.seq;
+    }
+    /** Load the database (Index Only) */
+    async load() {
+        if (!this.folderId) {
+            this.folderId = await this.findOrCreateFolder();
+        }
+        const metaFile = await this.findFile('_meta.json');
+        if (metaFile) {
+            this.meta = await this.downloadJson(metaFile.id);
+            this.metaEtag = metaFile.etag || null;
+        }
+        else {
+            await this.saveMeta(this.meta);
+        }
+        // Initialize Index
+        this.index = {};
+        // 1. Load Snapshot Index
+        if (this.meta.snapshotIndexId) {
+            try {
+                // Try strictly as new format first
+                const snapshotIdx = await this.downloadJson(this.meta.snapshotIndexId);
+                // Check if it's actually a legacy snapshot (has 'docs' with bodies)
+                if (snapshotIdx.docs) {
+                    // Migration Path: Handle legacy snapshot
+                    this.filesFromLegacySnapshot(snapshotIdx);
+                }
+                else {
+                    this.index = snapshotIdx.entries || {};
+                    // We assume seq is synced with meta usually, but use snapshot's seq as base
+                }
+            }
+            catch (e) {
+                console.warn('Failed to load snapshot index', e);
+                this.index = {};
+            }
+        }
+        else if (this.meta.snapshotId) {
+            // Legacy support: field was renamed
+            try {
+                const legacySnapshot = await this.downloadJson(this.meta.snapshotId);
+                this.filesFromLegacySnapshot(legacySnapshot);
+            }
+            catch (e) {
+                console.warn('Failed to load legacy snapshot', e);
+            }
+        }
+        // 2. Replay Change Logs (Metadata only updates)
+        this.pendingChanges = [];
+        this.currentLogSizeEstimate = 0;
+        for (const logId of this.meta.changeLogIds) {
+            const changes = await this.downloadNdjson(logId);
+            this.currentLogSizeEstimate += 100 * changes.length;
+            for (const change of changes) {
+                this.updateIndex(change, logId);
+                // We do NOT load body into cache automatically
+                // But we must invalidate cache if we had old data
+                if (this.docCache.get(change.id)) {
+                    this.docCache.remove(change.id);
+                }
+            }
+        }
+    }
+    // Migration helper
+    filesFromLegacySnapshot(snapshot) {
+        // Convert Legacy Snapshot (Docs in memory) to Index
+        // Since we don't have a separate file pointer for each doc in legacy snapshot,
+        // we say they are in the snapshot file itself.
+        // BUT, lazy loading requires being able to fetch them. 
+        // We will cache them ALL now (since we downloaded them) and index them.
+        for (const [id, doc] of Object.entries(snapshot.docs)) {
+            this.index[id] = {
+                rev: doc._rev,
+                seq: snapshot.seq, // Approximate
+                location: { fileId: 'LEGACY_MEMORY' } // Special validity marker
+            };
+            this.docCache.put(id, doc);
+        }
+    }
+    /**
+     * Get a document body.
+     * Index -> Cache -> Fetch
+     */
+    async get(id) {
+        const entry = this.index[id];
+        if (!entry)
+            return null;
+        if (entry.deleted)
+            return null;
+        // 1. Check Cache
+        const cached = this.docCache.get(id);
+        if (cached)
+            return cached;
+        // 2. Fetch from Drive
+        // If it's a legacy entry currently in memory (should have been cached), returns null if evicted?
+        if (entry.location.fileId === 'LEGACY_MEMORY') {
+            // If evicted, we are in trouble unless we re-download the legacy snapshot.
+            // For robustness, let's say we reload the legacy snapshot if needed.
+            // OR simpler: we assume compaction will fix this soon.
+            // Let's implement fetch for safety.
+            if (this.meta.snapshotId) {
+                const legacy = await this.downloadJson(this.meta.snapshotId);
+                if (legacy.docs[id]) {
+                    this.docCache.put(id, legacy.docs[id]);
+                    return legacy.docs[id];
+                }
+            }
+            return null; // Should not happen
+        }
+        const fileId = entry.location.fileId;
+        // Is it a change file (NDJSON) or snapshot file (JSON)?
+        // We can infer or we could have stored type. 
+        // Usually, we just download the file.
+        // Optimization: If we have many docs in one file, we might want to cache that file's contents?
+        // For now, naive fetch: download file, find doc.
+        const content = await this.downloadFileAny(fileId);
+        let doc = null;
+        if (Array.isArray(content)) {
+            // It's a change log (array of entries)
+            // Find the *last* entry for this ID in this file
+            const match = content.reverse().find((c) => c.id === id);
+            doc = match ? match.doc : null;
+        }
+        else if (content.docs) {
+            // It's a snapshot-data chunk
+            doc = content.docs[id];
+        }
+        else {
+            // Single doc file? (Not used yet)
+            doc = content;
+        }
+        if (doc) {
+            this.docCache.put(id, doc);
+            doc._rev = entry.rev; // Ensure consistent rev
+        }
+        return doc;
+    }
+    /** Get multiple docs (Atomic-ish) used for _allDocs */
+    async getMulti(ids) {
+        // Naive parallel fetch
+        // Optimization: Group by fileID to reduce requests
+        const byFile = {};
+        const results = {};
+        for (const id of ids) {
+            const entry = this.index[id];
+            if (!entry || entry.deleted) {
+                results[id] = null;
+                continue;
+            }
+            // Check cache
+            const cached = this.docCache.get(id);
+            if (cached) {
+                results[id] = cached;
+                continue;
+            }
+            // Group by file
+            if (entry.location.fileId === 'LEGACY_MEMORY') {
+                // Handle legacy separately
+                const doc = await this.get(id); // fallback
+                results[id] = doc;
+            }
+            else {
+                if (!byFile[entry.location.fileId])
+                    byFile[entry.location.fileId] = [];
+                byFile[entry.location.fileId].push(id);
+            }
+        }
+        // Fetch files
+        for (const [fileId, docIds] of Object.entries(byFile)) {
+            try {
+                const content = await this.downloadFileAny(fileId);
+                for (const docId of docIds) {
+                    let doc = null;
+                    if (Array.isArray(content)) {
+                        const match = content.reverse().find((c) => c.id === docId);
+                        doc = match ? match.doc : null;
+                    }
+                    else if (content.docs) {
+                        doc = content.docs[docId];
+                    }
+                    if (doc) {
+                        // Add entry.rev to doc just in case
+                        if (this.index[docId])
+                            doc._rev = this.index[docId].rev;
+                        this.docCache.put(docId, doc);
+                        results[docId] = doc;
+                    }
+                    else {
+                        results[docId] = null;
+                    }
+                }
+            }
+            catch (e) {
+                console.error(`Failed to fetch file ${fileId} for docs ${docIds}`, e);
+                // Return nulls
+                docIds.forEach(id => results[id] = null);
+            }
+        }
+        return ids.map(id => results[id]);
+    }
+    /** Return all keys in Index */
+    getIndexKeys() {
+        return Object.keys(this.index);
+    }
+    /** Get metadata for a specific ID from Index */
+    getIndexEntry(id) {
+        return this.index[id];
+    }
+    /** Single change wrapper */
+    async appendChange(change) {
+        return this.appendChanges([change]);
+    }
+    /** Append changes with OCC */
+    async appendChanges(changes) {
+        const MAX_RETRIES = 5;
+        let attempt = 0;
+        while (attempt < MAX_RETRIES) {
+            try {
+                return await this.tryAppendChanges(changes);
+            }
+            catch (err) {
+                if (err.code === 412 || err.code === 409) {
+                    // Reload and RETRY
+                    await this.load();
+                    // Check conflicts against Index (Metadata sufficient)
+                    this.checkConflicts(changes);
+                    // Reseq
+                    let currentSeq = this.meta.seq;
+                    for (const change of changes) {
+                        currentSeq++;
+                        change.seq = currentSeq;
+                    }
+                    attempt++;
+                    await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
+                    continue;
+                }
+                throw err;
+            }
+        }
+        throw new Error('Failed to append changes');
+    }
+    async tryAppendChanges(changes) {
+        // 1. Write Log File (Upload Data)
+        const fileId = await this.writeChangeFile(changes);
+        // 2. Prepare speculative meta update
+        const nextMeta = { ...this.meta };
+        nextMeta.changeLogIds = [...nextMeta.changeLogIds, fileId];
+        nextMeta.seq = changes[changes.length - 1].seq;
+        // 3. Commit Lock
+        await this.saveMeta(nextMeta, this.metaEtag);
+        // 4. Update Local State
+        this.meta = nextMeta;
+        for (const change of changes) {
+            this.updateIndex(change, fileId);
+            if (change.doc) {
+                this.docCache.put(change.id, change.doc);
+            }
+            else if (change.deleted) {
+                this.docCache.remove(change.id);
+            }
+        }
+        // 5. Compaction Check
+        // Count changes since last compaction *pointer*, not just list length
+        const totalChanges = await this.countTotalChanges();
+        if (totalChanges >= this.compactionThreshold ||
+            this.currentLogSizeEstimate >= this.compactionSizeThreshold) {
+            this.compact().catch(e => console.error('Compaction failed', e));
+        }
+    }
+    /** Update Index with a new change */
+    updateIndex(change, fileId) {
+        this.index[change.id] = {
+            rev: change.rev,
+            seq: change.seq,
+            deleted: !!change.deleted,
+            location: { fileId }
+        };
+    }
+    checkConflicts(changes) {
+        for (const change of changes) {
+            const docId = change.id;
+            const newRevNum = parseInt(change.rev.split('-')[0], 10);
+            const existing = this.index[docId];
+            if (existing) {
+                const currentRevNum = parseInt(existing.rev.split('-')[0], 10);
+                if (currentRevNum >= newRevNum) {
+                    const err = new Error('Document update conflict');
+                    err.status = 409;
+                    err.name = 'conflict'; // PouchDB expectation
+                    throw err;
+                }
+            }
+        }
+    }
+    /** Compact: Create SnapshotIndex + SnapshotData */
+    async compact() {
+        const snapshotSeq = this.meta.seq;
+        const oldLogIds = [...this.meta.changeLogIds];
+        const oldIndexId = this.meta.snapshotIndexId;
+        // 1. Fetch ALL active documents
+        // We need them to build the new large snapshot-data file
+        // This is the one time we download everything if not cached. 
+        // Optimization: We could reuse existing `snapshot-data` chunks and only append new data 
+        // to a new chunk, but for simplicity: Merge All.
+        const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted);
+        const allDocs = await this.getMulti(allIds);
+        const snapshotData = { docs: {} };
+        allIds.forEach((id, i) => {
+            if (allDocs[i])
+                snapshotData.docs[id] = allDocs[i];
+        });
+        // 2. Upload Data File
+        const dataContent = JSON.stringify(snapshotData);
+        const dataRes = await this.drive.files.create({
+            requestBody: {
+                name: `snapshot-data-${Date.now()}.json`,
+                parents: [this.folderId],
+                mimeType: 'application/json'
+            },
+            media: { mimeType: 'application/json', body: dataContent },
+            fields: 'id'
+        });
+        const dataFileId = dataRes.data.id;
+        // 3. Create Index pointing to this Data File
+        const newIndexEntries = {};
+        for (const id of Object.keys(snapshotData.docs)) {
+            newIndexEntries[id] = {
+                rev: this.index[id].rev,
+                seq: this.index[id].seq,
+                location: { fileId: dataFileId }
+            };
+        }
+        const snapshotIndex = {
+            entries: newIndexEntries,
+            seq: snapshotSeq,
+            createdAt: Date.now()
+        };
+        const indexContent = JSON.stringify(snapshotIndex);
+        const indexRes = await this.drive.files.create({
+            requestBody: {
+                name: `snapshot-index-${Date.now()}.json`,
+                parents: [this.folderId],
+                mimeType: 'application/json'
+            },
+            media: { mimeType: 'application/json', body: indexContent },
+            fields: 'id'
+        });
+        const newIndexId = indexRes.data.id;
+        // 4. Update Meta
+        await this.atomicUpdateMeta((latest) => {
+            const remainingLogs = latest.changeLogIds.filter(id => !oldLogIds.includes(id));
+            return {
+                ...latest,
+                snapshotIndexId: newIndexId,
+                changeLogIds: remainingLogs,
+                lastCompaction: Date.now()
+            };
+        });
+        // 5. Cleanup
+        this.cleanupOldFiles(oldIndexId, oldLogIds); // And potentially old data files if we tracked them
+        this.currentLogSizeEstimate = 0;
+    }
+    // ... Helpers (atomicUpdateMeta, saveMeta, writeChangeFile same as before) ...
+    async atomicUpdateMeta(modifier) {
+        const MAX_RETRIES = 5;
+        let attempt = 0;
+        while (attempt < MAX_RETRIES) {
+            try {
+                const metaFile = await this.findFile('_meta.json');
+                if (!metaFile)
+                    throw new Error('Meta missing');
+                const validMeta = await this.downloadJson(metaFile.id);
+                const newMeta = modifier(validMeta);
+                await this.saveMeta(newMeta, metaFile.etag);
+                this.meta = newMeta;
+                return;
+            }
+            catch (err) {
+                if (err.code === 412 || err.code === 409) {
+                    attempt++;
+                    await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
+                    continue;
+                }
+                throw err;
+            }
+        }
+    }
+    // Reused helpers
+    async findOrCreateFolder() {
+        const q = `name = '${this.folderName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
+        const res = await this.drive.files.list({ q, spaces: 'drive', fields: 'files(id)' });
+        if (res.data.files && res.data.files.length > 0)
+            return res.data.files[0].id;
+        const createRes = await this.drive.files.create({
+            requestBody: { name: this.folderName, mimeType: 'application/vnd.google-apps.folder', parents: this.parents.length ? this.parents : undefined },
+            fields: 'id'
+        });
+        return createRes.data.id;
+    }
+    async findFile(name) {
+        const q = `name = '${name}' and '${this.folderId}' in parents and trashed = false`;
+        const res = await this.drive.files.list({ q, spaces: 'drive', fields: 'files(id, etag)' });
+        if (res.data.files && res.data.files.length > 0)
+            return { id: res.data.files[0].id, etag: res.data.files[0].etag };
+        return null;
+    }
+    async downloadJson(fileId) {
+        const res = await this.drive.files.get({ fileId, alt: 'media' });
+        return res.data;
+    }
+    async downloadFileAny(fileId) {
+        const res = await this.drive.files.get({ fileId, alt: 'media' });
+        if (typeof res.data === 'string') {
+            // NDJSON or JSON string
+            try {
+                return JSON.parse(res.data);
+            }
+            catch {
+                // NDJSON?
+                const lines = res.data.trim().split('\n').filter((l) => l);
+                return lines.map((line) => JSON.parse(line));
+            }
+        }
+        return res.data;
+    }
+    async downloadNdjson(fileId) {
+        const res = await this.drive.files.get({ fileId, alt: 'media' });
+        const content = typeof res.data === 'string' ? res.data : JSON.stringify(res.data);
+        const lines = content.trim().split('\n').filter((l) => l);
+        return lines.map((line) => JSON.parse(line));
+    }
+    async writeChangeFile(changes) {
+        const lines = changes.map(c => JSON.stringify(c)).join('\n') + '\n';
+        const startSeq = changes[0].seq;
+        const name = `changes-${startSeq}-${Math.random().toString(36).substring(7)}.ndjson`;
+        const res = await this.drive.files.create({
+            requestBody: { name, parents: [this.folderId], mimeType: 'application/x-ndjson' },
+            media: { mimeType: 'application/x-ndjson', body: lines },
+            fields: 'id'
+        });
+        this.currentLogSizeEstimate += new Blob([lines]).size;
+        return res.data.id;
+    }
+    async saveMeta(meta, expectedEtag = null) {
+        const content = JSON.stringify(meta);
+        const metaFile = await this.findFile('_meta.json');
+        if (metaFile) {
+            const res = await this.drive.files.update({
+                fileId: metaFile.id,
+                headers: expectedEtag ? { 'If-Match': expectedEtag } : undefined,
+                media: { mimeType: 'application/json', body: content },
+                fields: 'id, etag'
+            });
+            this.metaEtag = res.data.etag;
+        }
+        else {
+            const res = await this.drive.files.create({
+                requestBody: { name: '_meta.json', parents: [this.folderId], mimeType: 'application/json' },
+                media: { mimeType: 'application/json', body: content },
+                fields: 'id, etag'
+            });
+            this.metaEtag = res.data.etag;
+        }
+    }
+    async countTotalChanges() {
+        // Calculate diff between meta.seq and snapshot seq
+        // But we don't store snapshot seq in meta directly?
+        // We can approximate by pending changes count + known gaps?
+        // Actually we used to check snapshot.seq. 
+        // We can assume snapshot is somewhat recent.
+        return this.pendingChanges.length + 10; // dummy for now, rely on log size
+    }
+    async cleanupOldFiles(oldIndexId, oldLogIds) {
+        if (oldIndexId)
+            try {
+                await this.drive.files.delete({ fileId: oldIndexId });
+            }
+            catch { }
+        for (const id of oldLogIds)
+            try {
+                await this.drive.files.delete({ fileId: id });
+            }
+            catch { }
+    }
+    startPolling(intervalMs) {
+        if (this.pollingInterval)
+            clearInterval(this.pollingInterval);
+        this.pollingInterval = setInterval(async () => {
+            try {
+                const metaFile = await this.findFile('_meta.json');
+                if (!metaFile)
+                    return;
+                if (metaFile.etag !== this.metaEtag) {
+                    await this.load();
+                    this.notifyListeners();
+                }
+            }
+            catch (err) {
+                console.error('Polling error', err);
+            }
+        }, intervalMs);
+    }
+    notifyListeners() {
+        // Observers expecting 'docs' object might be broken if they expect FULL body.
+        // We can pass empty object or partials?
+        // Real PouchDB changes feed calls `db.changes()`. 
+        // Our `adapter.js` uses `db.onChange` effectively.
+        // We should pass a map of { ID: { _rev, ... } } (Index entries)
+        // Adapter needs to handle this.
+        const changes = {};
+        for (const [id, entry] of Object.entries(this.index)) {
+            changes[id] = { _id: id, _rev: entry.rev, _deleted: entry.deleted };
+        }
+        for (const l of this.listeners)
+            l(changes);
+    }
+    // For tests/debug
+    onChange(cb) { this.listeners.push(cb); }
+    stopPolling() { if (this.pollingInterval)
+        clearInterval(this.pollingInterval); }
+    async deleteFolder() { if (this.folderId)
+        await this.drive.files.delete({ fileId: this.folderId }); }
+    getNextSeq() { return this.meta.seq + 1; }
+}
+exports.DriveHandler = DriveHandler;

package/lib/index.d.ts

@@ -0,0 +1 @@
+export default function (PouchDB: any): void;

package/lib/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = default_1;
+const adapter_1 = require("./adapter");
+function default_1(PouchDB) {
+    PouchDB.adapter('googledrive', (0, adapter_1.GoogleDriveAdapter)(PouchDB), true);
+}