@sinoia/hubdoc-tools 1.3.1 → 1.3.3

package/plugins/aws-s3/index.ts

@@ -0,0 +1,471 @@
+import {
+  S3Client,
+  ListObjectsV2Command,
+  GetObjectCommand,
+  PutObjectCommand,
+  CreateBucketCommand,
+  HeadBucketCommand
+} from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface S3Config extends PluginConfig {
+  accessKeyId: string;
+  secretAccessKey: string;
+  region: string;
+  bucketName: string;
+  prefix?: string;
+  endpoint?: string; // For S3-compatible services
+  limit?: number;
+}
+
+interface S3Object {
+  key: string;
+  size: number;
+  lastModified: Date;
+  etag: string;
+  storageClass?: string;
+  metadata?: Record<string, string>;
+}
+
+export default class S3Plugin implements DocumentSourcePlugin {
+  readonly name = 's3';
+  readonly version = '1.0.0';
+  readonly description = 'AWS S3 document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: S3Config;
+  private s3Client?: S3Client;
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const s3Config = config as S3Config;
+      const client = this.createS3Client(s3Config);
+      
+      await client.send(new HeadBucketCommand({ 
+        Bucket: s3Config.bucketName 
+      }));
+      
+      return true;
+    } catch (error: any) {
+      console.error(`S3 connection test failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as S3Config;
+    this.s3Client = this.createS3Client(this.config);
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const limit = (this.config as any).limit || (options as any)?.limit;
+      console.log(`🔍 Scanning S3 bucket: ${this.config.bucketName}${limit ? ` (limit: ${limit})` : ''}...`);
+
+      const objects = await this.listAllObjects(this.config.prefix);
+
+      let processedCount = 0;
+      for (const obj of objects) {
+        const source: DocumentSource = {
+          id: obj.key,
+          // Normalize filename to NFC to handle accented characters consistently across platforms
+          name: path.basename(obj.key).normalize('NFC'),
+          path: obj.key,
+          size: obj.size,
+          mimeType: this.getMimeType(obj.key),
+          lastModified: obj.lastModified,
+          metadata: {
+            s3Key: obj.key,
+            etag: obj.etag,
+            storageClass: obj.storageClass,
+            s3Metadata: obj.metadata
+          }
+        };
+
+        // Apply filters
+        if (this.shouldIncludeSource(source, options)) {
+          sources.push(source);
+          totalSize += source.size;
+          processedCount++;
+
+          // Check limit
+          if (limit && processedCount >= limit) {
+            console.log(`📏 Reached limit of ${limit} files`);
+            break;
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`S3 scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as S3Config;
+    this.s3Client = this.createS3Client(this.config);
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 5;
+
+    // Process in batches to manage memory and connections
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(100);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.s3Client || !this.config) throw new Error('S3 client not initialized');
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+
+      // Download file from S3
+      const command = new GetObjectCommand({
+        Bucket: this.config.bucketName,
+        Key: source.id
+      });
+
+      const response = await this.s3Client.send(command);
+      
+      if (!response.Body) {
+        throw new Error('Empty response body from S3');
+      }
+
+      // Stream to file
+      const writer = fs.createWriteStream(targetPath);
+      
+      // Handle different body types
+      if (response.Body instanceof ReadableStream) {
+        const reader = response.Body.getReader();
+        const pump = async (): Promise<void> => {
+          const { done, value } = await reader.read();
+          if (done) {
+            writer.end();
+            return;
+          }
+          writer.write(Buffer.from(value));
+          return pump();
+        };
+        await pump();
+      } else if (typeof response.Body.pipe === 'function') {
+        // Node.js stream
+        (response.Body as any).pipe(writer);
+      } else {
+        // Buffer or Uint8Array
+        writer.write(response.Body as Buffer);
+        writer.end();
+      }
+
+      return new Promise((resolve) => {
+        writer.on('finish', () => {
+          resolve({
+            success: true,
+            source,
+            localPath: targetPath,
+            bytesTransferred: source.size
+          });
+        });
+
+        writer.on('error', (error) => {
+          resolve({
+            success: false,
+            source,
+            error: error.message
+          });
+        });
+      });
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as S3Config;
+    this.s3Client = this.createS3Client(this.config);
+
+    const results: ExportResult[] = [];
+
+    for (const source of localSources) {
+      try {
+        if (!this.s3Client || !this.config) throw new Error('S3 client not initialized');
+
+        // Determine S3 key
+        let s3Key: string;
+        if (options?.preserveStructure) {
+          s3Key = this.config.prefix ? `${this.config.prefix}/${source.path}` : source.path;
+        } else {
+          s3Key = this.config.prefix ? `${this.config.prefix}/${source.name}` : source.name;
+        }
+
+        // Read local file
+        const fileContent = await fs.readFile(source.id);
+        
+        // Upload to S3 using multipart upload for large files
+        const upload = new Upload({
+          client: this.s3Client,
+          params: {
+            Bucket: this.config.bucketName,
+            Key: s3Key,
+            Body: fileContent,
+            ContentType: source.mimeType,
+            Metadata: {
+              originalPath: source.path,
+              originalName: source.name,
+              uploadedBy: 'hubdoc-tools'
+            }
+          }
+        });
+
+        await upload.done();
+
+        const targetPath = options?.preserveStructure ? source.path : source.name;
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: source.size
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async listAllObjects(prefix?: string): Promise<S3Object[]> {
+    if (!this.s3Client || !this.config) throw new Error('S3 client not initialized');
+
+    const allObjects: S3Object[] = [];
+    let continuationToken: string | undefined;
+
+    try {
+      do {
+        const command = new ListObjectsV2Command({
+          Bucket: this.config.bucketName,
+          Prefix: prefix,
+          ContinuationToken: continuationToken,
+          MaxKeys: 1000
+        });
+
+        const response = await this.s3Client.send(command);
+        const objects = response.Contents || [];
+
+        for (const obj of objects) {
+          if (obj.Key && obj.Size !== undefined && obj.LastModified) {
+            // Skip folders (keys ending with /)
+            if (!obj.Key.endsWith('/')) {
+              allObjects.push({
+                key: obj.Key,
+                size: obj.Size,
+                lastModified: obj.LastModified,
+                etag: obj.ETag || '',
+                storageClass: obj.StorageClass
+              });
+            }
+          }
+        }
+
+        continuationToken = response.NextContinuationToken;
+      } while (continuationToken);
+
+      return allObjects;
+    } catch (error: any) {
+      throw new Error(`Failed to list S3 objects: ${error.message}`);
+    }
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        accessKeyId: {
+          type: 'string',
+          description: 'AWS Access Key ID',
+          required: true
+        },
+        secretAccessKey: {
+          type: 'string',
+          description: 'AWS Secret Access Key',
+          required: true
+        },
+        region: {
+          type: 'string',
+          description: 'AWS Region (e.g., us-east-1)',
+          required: true
+        },
+        bucketName: {
+          type: 'string',
+          description: 'S3 Bucket name',
+          required: true
+        },
+        prefix: {
+          type: 'string',
+          description: 'S3 key prefix to filter objects (optional)',
+          required: false
+        },
+        endpoint: {
+          type: 'string',
+          description: 'Custom S3 endpoint for S3-compatible services (optional)',
+          required: false
+        },
+        limit: {
+          type: 'number',
+          description: 'Maximum number of documents to scan (useful for testing)',
+          required: false
+        }
+      },
+      required: ['accessKeyId', 'secretAccessKey', 'region', 'bucketName']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as S3Config;
+    
+    if (!this.config.accessKeyId || !this.config.secretAccessKey) {
+      throw new Error('AWS credentials are required');
+    }
+    
+    if (!this.config.region) {
+      throw new Error('AWS region is required');
+    }
+    
+    if (!this.config.bucketName) {
+      throw new Error('S3 bucket name is required');
+    }
+
+    this.s3Client = this.createS3Client(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.s3Client = undefined;
+  }
+
+  private createS3Client(config: S3Config): S3Client {
+    const clientConfig: any = {
+      region: config.region,
+      credentials: {
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey
+      }
+    };
+
+    // Support for S3-compatible services
+    if (config.endpoint) {
+      clientConfig.endpoint = config.endpoint;
+      clientConfig.forcePathStyle = true; // Required for most S3-compatible services
+    }
+
+    return new S3Client(clientConfig);
+  }
+
+  private getMimeType(key: string): string {
+    const ext = path.extname(key).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.pdf': 'application/pdf',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.xls': 'application/vnd.ms-excel',
+      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      '.ppt': 'application/vnd.ms-powerpoint',
+      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+      '.txt': 'text/plain',
+      '.csv': 'text/csv',
+      '.json': 'application/json',
+      '.xml': 'application/xml',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.png': 'image/png',
+      '.gif': 'image/gif',
+      '.zip': 'application/zip',
+      '.tar': 'application/x-tar',
+      '.gz': 'application/gzip'
+    };
+
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/aws-s3/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "aws-s3",
+  "version": "1.0.0",
+  "description": "AWS S3 document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.ts",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "aws-sdk": "^2.1000.0",
+    "fs-extra": "^11.1.0"
+  }
+}