@sinoia/hubdoc-tools 1.3.1 → 1.3.3

package/plugins/dropbox/index.ts

@@ -0,0 +1,451 @@
+import axios, { AxiosInstance } from 'axios';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface DropboxConfig extends PluginConfig {
+  accessToken: string;
+  appKey?: string;
+  appSecret?: string;
+  rootPath?: string; // e.g., '/Documents' or ''
+}
+
+interface DropboxFileMetadata {
+  '.tag': 'file' | 'folder';
+  name: string;
+  path_lower: string;
+  path_display: string;
+  id: string;
+  size?: number;
+  server_modified?: string;
+  client_modified?: string;
+  content_hash?: string;
+}
+
+export default class DropboxPlugin implements DocumentSourcePlugin {
+  readonly name = 'dropbox';
+  readonly version = '1.0.0';
+  readonly description = 'Dropbox document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: DropboxConfig;
+  private apiClient?: AxiosInstance;
+  private readonly apiUrl = 'https://api.dropboxapi.com/2';
+  private readonly contentUrl = 'https://content.dropboxapi.com/2';
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const client = this.createApiClient(config as DropboxConfig);
+      const response = await client.post('/users/get_current_account');
+      return response.status === 200;
+    } catch (error: any) {
+      console.error(`Dropbox connection test failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as DropboxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const rootPath = this.config.rootPath || '';
+      const items = await this.scanFolder(rootPath, options);
+
+      for (const item of items) {
+        if (item['.tag'] === 'file') {
+          const source: DocumentSource = {
+            id: item.path_display,
+            name: item.name,
+            path: this.getRelativePath(item.path_display, rootPath),
+            size: item.size || 0,
+            mimeType: this.getMimeType(item.name),
+            lastModified: new Date(item.server_modified || item.client_modified || Date.now()),
+            metadata: {
+              dropboxId: item.id,
+              pathLower: item.path_lower,
+              contentHash: item.content_hash
+            }
+          };
+
+          // Apply filters
+          if (this.shouldIncludeSource(source, options)) {
+            sources.push(source);
+            totalSize += source.size;
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`Dropbox scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  private async scanFolder(folderPath: string, options?: PluginImportOptions): Promise<DropboxFileMetadata[]> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const allItems: DropboxFileMetadata[] = [];
+    let cursor: string | undefined;
+
+    try {
+      // Initial request
+      const response = await this.apiClient.post('/files/list_folder', {
+        path: folderPath || '',
+        recursive: true,
+        limit: 2000,
+        include_media_info: false,
+        include_deleted: false,
+        include_has_explicit_shared_members: false
+      });
+
+      allItems.push(...response.data.entries);
+      
+      // Handle pagination
+      while (response.data.has_more) {
+        cursor = response.data.cursor;
+        const continueResponse = await this.apiClient.post('/files/list_folder/continue', {
+          cursor
+        });
+
+        allItems.push(...continueResponse.data.entries);
+        
+        if (!continueResponse.data.has_more) break;
+        response.data = continueResponse.data;
+      }
+
+      return allItems;
+    } catch (error: any) {
+      if (error.response?.data?.error?.['.tag'] === 'path_not_found') {
+        console.warn(`Warning: Folder not found: ${folderPath}`);
+        return [];
+      }
+      throw error;
+    }
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as DropboxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 10;
+
+    // Process in batches
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(100);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.apiClient) throw new Error('API client not initialized');
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+
+      // Download file from Dropbox
+      const downloadClient = axios.create({
+        baseURL: this.contentUrl,
+        headers: {
+          'Authorization': `Bearer ${this.config!.accessToken}`,
+          'Dropbox-API-Arg': JSON.stringify({
+            path: source.id
+          })
+        },
+        responseType: 'stream'
+      });
+
+      const response = await downloadClient.post('/files/download');
+      
+      const writer = fs.createWriteStream(targetPath);
+      response.data.pipe(writer);
+
+      return new Promise((resolve) => {
+        writer.on('finish', () => {
+          resolve({
+            success: true,
+            source,
+            localPath: targetPath,
+            bytesTransferred: source.size
+          });
+        });
+
+        writer.on('error', (error) => {
+          resolve({
+            success: false,
+            source,
+            error: error.message
+          });
+        });
+      });
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as DropboxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ExportResult[] = [];
+
+    for (const source of localSources) {
+      try {
+        const targetPath = options?.preserveStructure 
+          ? `${this.config.rootPath || ''}/${options.targetPath}/${source.path}`
+          : `${this.config.rootPath || ''}/${options?.targetPath || ''}/${source.name}`;
+
+        // Read file content
+        const fileContent = await fs.readFile(source.id);
+        const fileSize = fileContent.length;
+
+        // Upload file to Dropbox
+        const uploadClient = axios.create({
+          baseURL: this.contentUrl,
+          headers: {
+            'Authorization': `Bearer ${this.config.accessToken}`,
+            'Content-Type': 'application/octet-stream',
+            'Dropbox-API-Arg': JSON.stringify({
+              path: targetPath,
+              mode: options?.overwrite ? 'overwrite' : 'add',
+              autorename: !options?.overwrite,
+              mute: false
+            })
+          }
+        });
+
+        // Use upload sessions for large files (>150MB)
+        if (fileSize > 150 * 1024 * 1024) {
+          await this.uploadLargeFile(uploadClient, fileContent, targetPath);
+        } else {
+          await uploadClient.post('/files/upload', fileContent);
+        }
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: fileSize
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async uploadLargeFile(uploadClient: AxiosInstance, content: Buffer, targetPath: string): Promise<void> {
+    const chunkSize = 4 * 1024 * 1024; // 4MB chunks
+    let offset = 0;
+    let sessionId: string | undefined;
+
+    while (offset < content.length) {
+      const chunk = content.slice(offset, offset + chunkSize);
+      const isLastChunk = offset + chunkSize >= content.length;
+
+      if (!sessionId) {
+        // Start upload session
+        const startResponse = await uploadClient.post('/files/upload_session/start', chunk, {
+          headers: {
+            'Dropbox-API-Arg': JSON.stringify({ close: false })
+          }
+        });
+        sessionId = startResponse.data.session_id;
+      } else if (!isLastChunk) {
+        // Continue upload session
+        await uploadClient.post('/files/upload_session/append_v2', chunk, {
+          headers: {
+            'Dropbox-API-Arg': JSON.stringify({
+              cursor: { session_id: sessionId, offset },
+              close: false
+            })
+          }
+        });
+      } else {
+        // Finish upload session
+        await uploadClient.post('/files/upload_session/finish', chunk, {
+          headers: {
+            'Dropbox-API-Arg': JSON.stringify({
+              cursor: { session_id: sessionId, offset },
+              commit: {
+                path: targetPath,
+                mode: 'add',
+                autorename: true
+              }
+            })
+          }
+        });
+      }
+
+      offset += chunkSize;
+    }
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        accessToken: {
+          type: 'string',
+          description: 'Dropbox Access Token',
+          required: true
+        },
+        appKey: {
+          type: 'string',
+          description: 'Dropbox App Key (optional)',
+          required: false
+        },
+        appSecret: {
+          type: 'string',
+          description: 'Dropbox App Secret (optional)',
+          required: false
+        },
+        rootPath: {
+          type: 'string',
+          description: 'Root path to scan (e.g., /Documents)',
+          default: ''
+        }
+      },
+      required: ['accessToken']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as DropboxConfig;
+    
+    if (!this.config.accessToken) {
+      throw new Error('Dropbox access token is required');
+    }
+
+    this.apiClient = this.createApiClient(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.apiClient = undefined;
+  }
+
+  private createApiClient(config: DropboxConfig): AxiosInstance {
+    return axios.create({
+      baseURL: this.apiUrl,
+      headers: {
+        'Authorization': `Bearer ${config.accessToken}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+  }
+
+  private getRelativePath(fullPath: string, rootPath: string): string {
+    if (!rootPath) return fullPath.startsWith('/') ? fullPath.substring(1) : fullPath;
+    
+    const relativePath = fullPath.startsWith(rootPath) 
+      ? fullPath.substring(rootPath.length) 
+      : fullPath;
+    
+    return relativePath.startsWith('/') ? relativePath.substring(1) : relativePath;
+  }
+
+  private getMimeType(fileName: string): string {
+    const ext = path.extname(fileName).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.pdf': 'application/pdf',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.xls': 'application/vnd.ms-excel',
+      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      '.ppt': 'application/vnd.ms-powerpoint',
+      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+      '.txt': 'text/plain',
+      '.csv': 'text/csv',
+      '.json': 'application/json',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.png': 'image/png',
+      '.gif': 'image/gif',
+      '.zip': 'application/zip'
+    };
+
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/dropbox/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "dropbox",
+  "version": "1.0.0",
+  "description": "Dropbox document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.js",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "axios": "^1.5.0",
+    "fs-extra": "^11.1.0"
+  }
+}