mbkbucket 4.0.1 → 4.2.0

package/lib/src/core/storage/gdrive-provider.js

@@ -6,11 +6,13 @@
 
 import { Readable } from "node:stream";
 import { StorageProvider } from "./storage-provider.js";
+import { StorageCapabilities } from "./capabilities.js";
 import { StorageFile, StorageFolder, StorageListResult } from "./models.js";
 import { StorageNotFoundError, StorageConflictError, StorageAccessDeniedError, StorageValidationError, StorageConfigError, StorageError } from "./errors.js";
 import { refreshGoogleAccessToken, getServiceAccountAccessToken } from "./gdrive-auth.js";
 import { createLogger } from "#logger";
-import { nowIso, trimSlashes, trimLeadingSlashes, getBaseName, getFolderPath } from "#helpers";
+import { nowIso, getFolderPath } from "#helpers";
+import { trimSlashes, trimLeadingSlashes, getBaseName } from "mbkutils";
 
 const debugGDrive = createLogger('gdrive-provider');
 
@@ -19,19 +21,49 @@ function escapeGdriveQuery(str = '') {
 }
 
 export class GoogleDriveStorageProvider extends StorageProvider {
-  /**
-   * @param {string} name - Connection identifier
-   * @param {object} config - Configuration object
-   */
   constructor(name, config = {}) {
     super(name, config);
     this.rootFolderId = config.folder_id || config.root_folder_id || 'root';
     this._accessToken = config.access_token || null;
     this._tokenExpiry = config.expiry_date || 0;
     this._folderCache = new Map();
-    this._folderCache.set('', this.rootFolderId);
-    this._folderCache.set('.', this.rootFolderId);
-    this._folderCache.set('/', this.rootFolderId);
+    this._setFolderCache('', this.rootFolderId);
+    this._setFolderCache('.', this.rootFolderId);
+    this._setFolderCache('/', this.rootFolderId);
+  }
+
+  _setFolderCache(path, id) {
+    if (this._folderCache.size >= 500) {
+      const oldestKey = this._folderCache.keys().next().value;
+      this._folderCache.delete(oldestKey);
+    }
+    this._folderCache.set(path, { id, expires: Date.now() + 300000 });
+  }
+
+  _getFolderCache(path) {
+    const entry = this._folderCache.get(path);
+    if (!entry) return null;
+    if (typeof entry === 'object' && entry.expires && entry.expires < Date.now()) {
+      this._folderCache.delete(path);
+      return null;
+    }
+    return typeof entry === 'object' && entry.id ? entry.id : entry;
+  }
+
+  _invalidateFolderCache(prefix = '') {
+    if (!prefix) {
+      this._folderCache.clear();
+      this._setFolderCache('', this.rootFolderId);
+      this._setFolderCache('.', this.rootFolderId);
+      this._setFolderCache('/', this.rootFolderId);
+      return;
+    }
+    this._folderCache.delete(prefix);
+    for (const k of this._folderCache.keys()) {
+      if (k.startsWith(`${prefix}/`)) {
+        this._folderCache.delete(k);
+      }
+    }
   }
 
   get type() {
@@ -39,16 +71,19 @@ export class GoogleDriveStorageProvider extends StorageProvider {
   }
 
   get capabilities() {
-    return {
+    return new StorageCapabilities({
       multipart: false,
-      presignedUrls: false,
+      resumable: false,
       nativeFolders: true,
       ranges: true,
       copy: true,
       move: true,
       search: true,
-      directDownloadUrl: true
-    };
+      directDownloadUrl: true,
+      versioning: false,
+      publicUrls: true,
+      metadata: true
+    });
   }
 
   async getAccessToken() {
@@ -102,29 +137,45 @@ export class GoogleDriveStorageProvider extends StorageProvider {
   }
 
   async _fetch(url, options = {}) {
-    const token = await this.getAccessToken();
-    const headers = new Headers(options.headers || {});
-    headers.set('Authorization', `Bearer ${token}`);
-
-    let response = await fetch(url, { ...options, headers });
-
-    // Handle 401 token expiration retry once
-    if (response.status === 401 && (this.config.refresh_token || this.config.client_email)) {
-      debugGDrive('Received 401 from Google Drive API, retrying with refreshed token...');
-      this._accessToken = null;
-      this._tokenExpiry = 0;
-      const newToken = await this.getAccessToken();
-      headers.set('Authorization', `Bearer ${newToken}`);
-      response = await fetch(url, { ...options, headers });
-    }
+    const maxRetries = options.maxRetries ?? 3;
+    let attempt = 0;
+
+    while (attempt <= maxRetries) {
+      const token = await this.getAccessToken();
+      const headers = new Headers(options.headers || {});
+      headers.set('Authorization', `Bearer ${token}`);
+
+      let response = await fetch(url, { ...options, headers });
+
+      // Handle 401 token expiration retry once
+      if (response.status === 401 && attempt === 0 && (this.config.refresh_token || this.config.client_email)) {
+        debugGDrive('Received 401 from Google Drive API, retrying with refreshed token...');
+        this._accessToken = null;
+        this._tokenExpiry = 0;
+        const newToken = await this.getAccessToken();
+        headers.set('Authorization', `Bearer ${newToken}`);
+        response = await fetch(url, { ...options, headers });
+      }
+
+      // Handle 429 Too Many Requests or 503/500 temporary service errors with exponential backoff
+      if ((response.status === 429 || response.status === 503 || response.status === 500) && attempt < maxRetries) {
+        attempt++;
+        const retryAfter = Number(response.headers.get('retry-after')) || 0;
+        const delayMs = retryAfter > 0 ? retryAfter * 1000 : Math.min(300 * (2 ** (attempt - 1)) + Math.random() * 150, 4000);
+        debugGDrive('Google Drive API rate limit/error (%s). Retrying attempt %s in %sms...', response.status, attempt, Math.round(delayMs));
+        await new Promise(r => setTimeout(r, delayMs));
+        continue;
+      }
 
-    return response;
+      return response;
+    }
   }
 
   async _resolvePathToFolderId(path = '') {
     const cleanPath = trimSlashes(path);
     if (!cleanPath) return this.rootFolderId;
-    if (this._folderCache.has(cleanPath)) return this._folderCache.get(cleanPath);
+    const cached = this._getFolderCache(cleanPath);
+    if (cached) return cached;
 
     const segments = cleanPath.split('/').filter(Boolean);
     let currentParentId = this.rootFolderId;
@@ -132,8 +183,9 @@ export class GoogleDriveStorageProvider extends StorageProvider {
 
     for (const segment of segments) {
       currentPath = currentPath ? `${currentPath}/${segment}` : segment;
-      if (this._folderCache.has(currentPath)) {
-        currentParentId = this._folderCache.get(currentPath);
+      const segCached = this._getFolderCache(currentPath);
+      if (segCached) {
+        currentParentId = segCached;
         continue;
       }
 
@@ -151,7 +203,7 @@ export class GoogleDriveStorageProvider extends StorageProvider {
       }
 
       currentParentId = data.files[0].id;
-      this._folderCache.set(currentPath, currentParentId);
+      this._setFolderCache(currentPath, currentParentId);
     }
 
     return currentParentId;
@@ -160,7 +212,8 @@ export class GoogleDriveStorageProvider extends StorageProvider {
   async _ensureFolderPath(path = '') {
     const cleanPath = trimSlashes(path);
     if (!cleanPath) return this.rootFolderId;
-    if (this._folderCache.has(cleanPath)) return this._folderCache.get(cleanPath);
+    const cached = this._getFolderCache(cleanPath);
+    if (cached) return cached;
 
     const segments = cleanPath.split('/').filter(Boolean);
     let currentParentId = this.rootFolderId;
@@ -168,8 +221,9 @@ export class GoogleDriveStorageProvider extends StorageProvider {
 
     for (const segment of segments) {
       currentPath = currentPath ? `${currentPath}/${segment}` : segment;
-      if (this._folderCache.has(currentPath)) {
-        currentParentId = this._folderCache.get(currentPath);
+      const segCached = this._getFolderCache(currentPath);
+      if (segCached) {
+        currentParentId = segCached;
         continue;
       }
 
@@ -202,7 +256,7 @@ export class GoogleDriveStorageProvider extends StorageProvider {
         currentParentId = createData.id;
       }
 
-      this._folderCache.set(currentPath, currentParentId);
+      this._setFolderCache(currentPath, currentParentId);
     }
 
     return currentParentId;
@@ -304,7 +358,11 @@ export class GoogleDriveStorageProvider extends StorageProvider {
       for (const item of (data.files || [])) {
         if (item.mimeType === 'application/vnd.google-apps.folder') {
           const folderPrefix = `${prefixPath}${item.name}/`;
-          commonPrefixes.push(new StorageFolder(folderPrefix, item.id));
+          commonPrefixes.push(new StorageFolder({
+            prefix: folderPrefix,
+            id: item.id,
+            provider: 'gdrive'
+          }));
           this._folderCache.set(trimSlashes(folderPrefix), item.id);
         } else {
           const itemKey = `${prefixPath}${item.name}`;
@@ -312,12 +370,16 @@ export class GoogleDriveStorageProvider extends StorageProvider {
             key: itemKey,
             size: parseInt(item.size || 0, 10),
             lastModified: item.modifiedTime || item.createdTime || Date.now(),
+            createdAt: item.createdTime,
             etag: item.md5Checksum ? `"${item.md5Checksum}"` : `"${item.id}"`,
             storageClass: 'GOOGLE_DRIVE',
+            provider: 'gdrive',
             id: item.id,
+            nativeId: item.id,
             mimeType: item.mimeType,
             webViewLink: item.webViewLink,
-            webContentLink: item.webContentLink
+            webContentLink: item.webContentLink,
+            metadata: item.properties || {}
           }));
         }
       }
@@ -528,12 +590,7 @@ export class GoogleDriveStorageProvider extends StorageProvider {
         throw new StorageError(`Failed to delete Google Drive folder: ${data.error?.message || res.statusText}`);
       }
 
-      this._folderCache.delete(cleanPrefix);
-      for (const [k] of this._folderCache.entries()) {
-        if (k.startsWith(`${cleanPrefix}/`)) {
-          this._folderCache.delete(k);
-        }
-      }
+      this._invalidateFolderCache(cleanPrefix);
 
       return { deletedCount: 1, prefix: cleanPrefix, deletedAt: nowIso() };
     } catch (error) {
@@ -587,6 +644,7 @@ export class GoogleDriveStorageProvider extends StorageProvider {
   }
 
   async copyFile(sourceKey, destKey, _options = {}) {
+    this.assertCapability('copy', 'copyFile');
     try {
       const srcClean = trimLeadingSlashes(sourceKey);
       const dstClean = trimLeadingSlashes(destKey);
@@ -622,6 +680,7 @@ export class GoogleDriveStorageProvider extends StorageProvider {
   }
 
   async moveFile(sourceKey, destKey, _options = {}) {
+    this.assertCapability('move', 'moveFile');
     try {
       const srcClean = trimLeadingSlashes(sourceKey);
       const dstClean = trimLeadingSlashes(destKey);

package/lib/src/core/storage/index.js

@@ -5,10 +5,13 @@
  */
 
 export { StorageProvider } from "./storage-provider.js";
+export { StorageCapabilities } from "./capabilities.js";
 export { S3StorageProvider } from "./s3-provider.js";
 export { GoogleDriveStorageProvider } from "./gdrive-provider.js";
-export { StorageFile, StorageFolder, StorageListResult } from "./models.js";
+export { LocalStorageProvider } from "./local-provider.js";
+export { StorageItem, StorageFile, StorageFolder, StorageListResult } from "./models.js";
 export { StorageError, StorageNotFoundError, StorageAccessDeniedError, StorageConflictError, StorageValidationError, StorageConfigError, StorageQuotaExceededError, StorageUnsupportedOperationError } from "./errors.js";
 export { PathStrategy } from "./path-strategy.js";
 export { getGoogleAuthUrl, exchangeCodeForTokens, refreshGoogleAccessToken, getServiceAccountAccessToken, GOOGLE_DRIVE_SCOPES } from "./gdrive-auth.js";
 export { StorageManager, storageManager } from "./storage-manager.js";
+