@sinoia/hubdoc-tools 1.3.2 → 1.3.4

package/plugins/googledrive/index.ts

@@ -0,0 +1,463 @@
+import { google, drive_v3 } from 'googleapis';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface GoogleDriveConfig extends PluginConfig {
+  clientId: string;
+  clientSecret: string;
+  redirectUri: string;
+  accessToken?: string;
+  refreshToken?: string;
+  rootFolderId?: string; // Specific folder ID to scan, defaults to 'root'
+}
+
+export default class GoogleDrivePlugin implements DocumentSourcePlugin {
+  readonly name = 'googledrive';
+  readonly version = '1.0.0';
+  readonly description = 'Google Drive document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: GoogleDriveConfig;
+  private drive?: drive_v3.Drive;
+  private auth?: any;
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const driveConfig = config as GoogleDriveConfig;
+      const auth = await this.createAuth(driveConfig);
+      const drive = google.drive({ version: 'v3', auth });
+      
+      const response = await drive.about.get({ fields: 'user' });
+      return !!response.data.user;
+    } catch (error: any) {
+      console.error(`Google Drive connection test failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as GoogleDriveConfig;
+    this.auth = await this.createAuth(this.config);
+    this.drive = google.drive({ version: 'v3', auth: this.auth });
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const rootFolderId = this.config.rootFolderId || 'root';
+      const files = await this.scanFolder(rootFolderId, '', options);
+
+      for (const file of files) {
+        if (file.mimeType && !file.mimeType.includes('folder')) {
+          const source: DocumentSource = {
+            id: file.id!,
+            name: file.name!,
+            path: (file as any).path!,
+            size: parseInt(file.size || '0'),
+            mimeType: file.mimeType,
+            lastModified: new Date(file.modifiedTime!),
+            metadata: {
+              googleDriveId: file.id,
+              parents: file.parents,
+              webViewLink: file.webViewLink
+            }
+          };
+
+          // Apply filters
+          if (this.shouldIncludeSource(source, options)) {
+            sources.push(source);
+            totalSize += source.size;
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`Google Drive scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  private async scanFolder(folderId: string, parentPath: string, options?: PluginImportOptions): Promise<any[]> {
+    if (!this.drive) throw new Error('Drive client not initialized');
+
+    const allFiles: any[] = [];
+    let pageToken: string | undefined;
+
+    do {
+      try {
+        const response = await this.drive.files.list({
+          q: `'${folderId}' in parents and trashed=false`,
+          fields: 'nextPageToken, files(id, name, mimeType, size, modifiedTime, parents, webViewLink)',
+          pageSize: 1000,
+          pageToken
+        });
+
+        const files = response.data.files || [];
+
+        for (const file of files) {
+          const filePath = parentPath ? `${parentPath}/${file.name}` : file.name!;
+          (file as any).path = filePath;
+
+          if (file.mimeType === 'application/vnd.google-apps.folder') {
+            // Recursively scan subfolders
+            const subFiles = await this.scanFolder(file.id!, filePath, options);
+            allFiles.push(...subFiles);
+          } else {
+            allFiles.push(file);
+          }
+        }
+
+        pageToken = response.data.nextPageToken || undefined;
+      } catch (error: any) {
+        console.warn(`Warning: Failed to scan folder ${folderId}: ${error.message}`);
+        break;
+      }
+    } while (pageToken);
+
+    return allFiles;
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as GoogleDriveConfig;
+    this.auth = await this.createAuth(this.config);
+    this.drive = google.drive({ version: 'v3', auth: this.auth });
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 5; // Google Drive has rate limits
+
+    // Process in batches
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(200);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.drive) throw new Error('Drive client not initialized');
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+
+      // Handle Google Workspace documents (export to common formats)
+      if (this.isGoogleWorkspaceDoc(source.mimeType)) {
+        const exportMimeType = this.getExportMimeType(source.mimeType);
+        const exportExt = this.getExportExtension(exportMimeType);
+        const exportPath = targetPath.replace(path.extname(targetPath), exportExt);
+
+        const response = await this.drive.files.export({
+          fileId: source.id,
+          mimeType: exportMimeType
+        }, { responseType: 'stream' });
+
+        const writer = fs.createWriteStream(exportPath);
+        response.data.pipe(writer);
+
+        return new Promise((resolve) => {
+          writer.on('finish', async () => {
+            const stats = await fs.stat(exportPath);
+            resolve({
+              success: true,
+              source,
+              localPath: exportPath,
+              bytesTransferred: stats.size
+            });
+          });
+
+          writer.on('error', (error) => {
+            resolve({
+              success: false,
+              source,
+              error: error.message
+            });
+          });
+        });
+      } else {
+        // Regular file download
+        const response = await this.drive.files.get({
+          fileId: source.id,
+          alt: 'media'
+        }, { responseType: 'stream' });
+
+        const writer = fs.createWriteStream(targetPath);
+        response.data.pipe(writer);
+
+        return new Promise((resolve) => {
+          writer.on('finish', () => {
+            resolve({
+              success: true,
+              source,
+              localPath: targetPath,
+              bytesTransferred: source.size
+            });
+          });
+
+          writer.on('error', (error) => {
+            resolve({
+              success: false,
+              source,
+              error: error.message
+            });
+          });
+        });
+      }
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as GoogleDriveConfig;
+    this.auth = await this.createAuth(this.config);
+    this.drive = google.drive({ version: 'v3', auth: this.auth });
+
+    const results: ExportResult[] = [];
+    const rootFolderId = this.config.rootFolderId || 'root';
+
+    for (const source of localSources) {
+      try {
+        // Determine target folder
+        let targetFolderId = rootFolderId;
+        
+        if (options?.preserveStructure && source.path.includes('/')) {
+          const folderPath = path.dirname(source.path);
+          targetFolderId = await this.createFolderStructure(folderPath, rootFolderId);
+        }
+
+        // Upload file
+        const fileContent = await fs.readFile(source.id);
+        
+        const response = await this.drive!.files.create({
+          requestBody: {
+            name: source.name,
+            parents: [targetFolderId]
+          },
+          media: {
+            mimeType: source.mimeType,
+            body: fileContent
+          }
+        });
+
+        const targetPath = options?.preserveStructure ? source.path : source.name;
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: source.size
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async createFolderStructure(folderPath: string, parentId: string): Promise<string> {
+    if (!this.drive) throw new Error('Drive client not initialized');
+
+    const parts = folderPath.split('/').filter(part => part.length > 0);
+    let currentParentId = parentId;
+
+    for (const folderName of parts) {
+      // Check if folder already exists
+      const existingFolder = await this.drive.files.list({
+        q: `name='${folderName}' and '${currentParentId}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false`,
+        fields: 'files(id)'
+      });
+
+      if (existingFolder.data.files && existingFolder.data.files.length > 0) {
+        currentParentId = existingFolder.data.files[0].id!;
+      } else {
+        // Create new folder
+        const newFolder = await this.drive.files.create({
+          requestBody: {
+            name: folderName,
+            mimeType: 'application/vnd.google-apps.folder',
+            parents: [currentParentId]
+          }
+        });
+        currentParentId = newFolder.data.id!;
+      }
+    }
+
+    return currentParentId;
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        clientId: {
+          type: 'string',
+          description: 'Google OAuth2 Client ID',
+          required: true
+        },
+        clientSecret: {
+          type: 'string',
+          description: 'Google OAuth2 Client Secret',
+          required: true
+        },
+        redirectUri: {
+          type: 'string',
+          description: 'OAuth2 Redirect URI',
+          required: true
+        },
+        accessToken: {
+          type: 'string',
+          description: 'OAuth2 Access Token (will be obtained automatically)',
+          required: false
+        },
+        refreshToken: {
+          type: 'string',
+          description: 'OAuth2 Refresh Token (will be obtained automatically)',
+          required: false
+        },
+        rootFolderId: {
+          type: 'string',
+          description: 'Root folder ID to scan (defaults to root)',
+          default: 'root'
+        }
+      },
+      required: ['clientId', 'clientSecret', 'redirectUri']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as GoogleDriveConfig;
+    
+    if (!this.config.accessToken) {
+      throw new Error('Google Drive authentication required. Please run authentication flow first.');
+    }
+
+    this.auth = await this.createAuth(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.drive = undefined;
+    this.auth = undefined;
+  }
+
+  private async createAuth(config: GoogleDriveConfig) {
+    const auth = new google.auth.OAuth2(
+      config.clientId,
+      config.clientSecret,
+      config.redirectUri
+    );
+
+    if (config.accessToken) {
+      auth.setCredentials({
+        access_token: config.accessToken,
+        refresh_token: config.refreshToken
+      });
+    }
+
+    return auth;
+  }
+
+  private isGoogleWorkspaceDoc(mimeType: string): boolean {
+    return [
+      'application/vnd.google-apps.document',
+      'application/vnd.google-apps.spreadsheet',
+      'application/vnd.google-apps.presentation'
+    ].includes(mimeType);
+  }
+
+  private getExportMimeType(googleMimeType: string): string {
+    const exportMap: Record<string, string> = {
+      'application/vnd.google-apps.document': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      'application/vnd.google-apps.presentation': 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+    };
+    
+    return exportMap[googleMimeType] || 'application/pdf';
+  }
+
+  private getExportExtension(mimeType: string): string {
+    const extensionMap: Record<string, string> = {
+      'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+      'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+      'application/pdf': '.pdf'
+    };
+    
+    return extensionMap[mimeType] || '.pdf';
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/googledrive/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "googledrive",
+  "version": "1.0.0",
+  "description": "Google Drive document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.js",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "googleapis": "^128.0.0",
+    "fs-extra": "^11.1.0"
+  }
+}