@sinoia/hubdoc-tools 1.3.5 → 1.3.7

package/plugins/azure-blob/index.ts

@@ -1,420 +0,0 @@
-import { BlobServiceClient, StorageSharedKeyCredential, BlobSASPermissions, generateBlobSASQueryParameters } from '@azure/storage-blob';
-import fs from 'fs-extra';
-import path from 'path';
-import {
-  DocumentSourcePlugin,
-  DocumentSource,
-  PluginConfig,
-  ScanResult,
-  PluginImportOptions,
-  PluginExportOptions,
-  ImportResult,
-  ExportResult
-} from '../../src/types/plugins';
-
-interface AzureBlobConfig extends PluginConfig {
-  accountName: string;
-  accountKey?: string;
-  sasToken?: string;
-  connectionString?: string;
-  containerName: string;
-  prefix?: string;
-  limit?: number;
-}
-
-interface BlobItem {
-  name: string;
-  size: number;
-  lastModified: Date;
-  etag: string;
-  contentType?: string;
-  metadata?: Record<string, string>;
-  tags?: Record<string, string>;
-}
-
-export default class AzureBlobPlugin implements DocumentSourcePlugin {
-  readonly name = 'azure-blob';
-  readonly version = '1.0.0';
-  readonly description = 'Azure Blob Storage document source';
-  readonly supportedOperations = ['import', 'export', 'both'] as const;
-
-  private config?: AzureBlobConfig;
-  private blobServiceClient?: BlobServiceClient;
-
-  async testConnection(config: PluginConfig): Promise<boolean> {
-    try {
-      const blobConfig = config as AzureBlobConfig;
-      const client = this.createBlobServiceClient(blobConfig);
-      
-      const containerClient = client.getContainerClient(blobConfig.containerName);
-      await containerClient.getProperties();
-      
-      return true;
-    } catch (error: any) {
-      console.error(`Azure Blob connection test failed: ${error.message}`);
-      return false;
-    }
-  }
-
-  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
-    this.config = config as AzureBlobConfig;
-    this.blobServiceClient = this.createBlobServiceClient(this.config);
-
-    const sources: DocumentSource[] = [];
-    const errors: string[] = [];
-    let totalSize = 0;
-
-    try {
-      const limit = (this.config as any).limit || (options as any)?.limit;
-      console.log(`🔍 Scanning Azure Blob container: ${this.config.containerName}${limit ? ` (limit: ${limit})` : ''}...`);
-
-      const containerClient = this.blobServiceClient.getContainerClient(this.config.containerName);
-      const blobs = await this.listAllBlobs(containerClient, this.config.prefix);
-
-      let processedCount = 0;
-      for (const blob of blobs) {
-        const source: DocumentSource = {
-          id: blob.name,
-          // Normalize filename to NFC to handle accented characters consistently across platforms
-          name: path.basename(blob.name).normalize('NFC'),
-          path: blob.name,
-          size: blob.size,
-          mimeType: blob.contentType || this.getMimeType(blob.name),
-          lastModified: blob.lastModified,
-          metadata: {
-            blobName: blob.name,
-            etag: blob.etag,
-            azureMetadata: blob.metadata,
-            tags: blob.tags
-          }
-        };
-
-        // Apply filters
-        if (this.shouldIncludeSource(source, options)) {
-          sources.push(source);
-          totalSize += source.size;
-          processedCount++;
-
-          // Check limit
-          if (limit && processedCount >= limit) {
-            console.log(`📏 Reached limit of ${limit} files`);
-            break;
-          }
-        }
-      }
-
-      return {
-        sources,
-        totalCount: sources.length,
-        totalSize,
-        errors
-      };
-    } catch (error: any) {
-      return {
-        sources: [],
-        totalCount: 0,
-        totalSize: 0,
-        errors: [`Azure Blob scan failed: ${error.message}`]
-      };
-    }
-  }
-
-  async import(
-    config: PluginConfig,
-    sources: DocumentSource[],
-    targetDir: string,
-    options?: PluginImportOptions
-  ): Promise<ImportResult[]> {
-    this.config = config as AzureBlobConfig;
-    this.blobServiceClient = this.createBlobServiceClient(this.config);
-
-    const results: ImportResult[] = [];
-    const batchSize = options?.batchSize || 5;
-
-    // Process in batches to manage connections
-    for (let i = 0; i < sources.length; i += batchSize) {
-      const batch = sources.slice(i, i + batchSize);
-      
-      for (const source of batch) {
-        const result = await this.importSingle(source, targetDir);
-        results.push(result);
-        
-        // Small delay to respect rate limits
-        await this.sleep(100);
-      }
-    }
-
-    return results;
-  }
-
-  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
-    try {
-      if (!this.blobServiceClient || !this.config) throw new Error('Blob service client not initialized');
-
-      const targetPath = path.join(targetDir, source.path);
-      const targetDirectory = path.dirname(targetPath);
-      
-      await fs.ensureDir(targetDirectory);
-
-      const containerClient = this.blobServiceClient.getContainerClient(this.config.containerName);
-      const blobClient = containerClient.getBlobClient(source.id);
-
-      // Download blob to file
-      await blobClient.downloadToFile(targetPath);
-
-      return {
-        success: true,
-        source,
-        localPath: targetPath,
-        bytesTransferred: source.size
-      };
-    } catch (error: any) {
-      return {
-        success: false,
-        source,
-        error: error.message
-      };
-    }
-  }
-
-  async export?(
-    config: PluginConfig,
-    localSources: DocumentSource[],
-    options?: PluginExportOptions
-  ): Promise<ExportResult[]> {
-    this.config = config as AzureBlobConfig;
-    this.blobServiceClient = this.createBlobServiceClient(this.config);
-
-    const results: ExportResult[] = [];
-
-    for (const source of localSources) {
-      try {
-        if (!this.blobServiceClient || !this.config) throw new Error('Blob service client not initialized');
-
-        // Determine blob name
-        let blobName: string;
-        if (options?.preserveStructure) {
-          blobName = this.config.prefix ? `${this.config.prefix}/${source.path}` : source.path;
-        } else {
-          blobName = this.config.prefix ? `${this.config.prefix}/${source.name}` : source.name;
-        }
-
-        const containerClient = this.blobServiceClient.getContainerClient(this.config.containerName);
-        const blockBlobClient = containerClient.getBlockBlobClient(blobName);
-
-        // Upload file to Azure Blob Storage
-        const fileContent = await fs.readFile(source.id);
-        
-        await blockBlobClient.upload(fileContent, fileContent.length, {
-          blobHTTPHeaders: {
-            blobContentType: source.mimeType
-          },
-          metadata: {
-            originalPath: source.path,
-            originalName: source.name,
-            uploadedBy: 'hubdoc-tools',
-            uploadDate: new Date().toISOString()
-          }
-        });
-
-        const targetPath = options?.preserveStructure ? source.path : source.name;
-
-        results.push({
-          success: true,
-          targetPath,
-          source,
-          bytesTransferred: source.size
-        });
-      } catch (error: any) {
-        results.push({
-          success: false,
-          targetPath: options?.targetPath || '',
-          source,
-          error: error.message
-        });
-      }
-    }
-
-    return results;
-  }
-
-  private async listAllBlobs(containerClient: any, prefix?: string): Promise<BlobItem[]> {
-    const allBlobs: BlobItem[] = [];
-
-    try {
-      const listBlobsOptions: any = {
-        includeMetadata: true,
-        includeTags: true
-      };
-
-      if (prefix) {
-        listBlobsOptions.prefix = prefix;
-      }
-
-      for await (const blob of containerClient.listBlobsFlat(listBlobsOptions)) {
-        // Skip directories (virtual folders)
-        if (!blob.name.endsWith('/')) {
-          allBlobs.push({
-            name: blob.name,
-            size: blob.properties.contentLength || 0,
-            lastModified: blob.properties.lastModified || new Date(),
-            etag: blob.properties.etag || '',
-            contentType: blob.properties.contentType,
-            metadata: blob.metadata,
-            tags: blob.tags
-          });
-        }
-      }
-
-      return allBlobs;
-    } catch (error: any) {
-      throw new Error(`Failed to list Azure blobs: ${error.message}`);
-    }
-  }
-
-  getConfigSchema(): Record<string, any> {
-    return {
-      type: 'object',
-      properties: {
-        accountName: {
-          type: 'string',
-          description: 'Azure Storage Account name',
-          required: true
-        },
-        accountKey: {
-          type: 'string',
-          description: 'Azure Storage Account key (if using key authentication)',
-          required: false
-        },
-        sasToken: {
-          type: 'string',
-          description: 'Azure Storage SAS token (alternative to account key)',
-          required: false
-        },
-        connectionString: {
-          type: 'string',
-          description: 'Azure Storage connection string (alternative to account name/key)',
-          required: false
-        },
-        containerName: {
-          type: 'string',
-          description: 'Azure Blob Storage container name',
-          required: true
-        },
-        prefix: {
-          type: 'string',
-          description: 'Blob name prefix to filter objects (optional)',
-          required: false
-        },
-        limit: {
-          type: 'number',
-          description: 'Maximum number of documents to scan (useful for testing)',
-          required: false
-        }
-      },
-      required: ['accountName', 'containerName'],
-      oneOf: [
-        { required: ['accountName', 'accountKey', 'containerName'] },
-        { required: ['accountName', 'sasToken', 'containerName'] },
-        { required: ['connectionString', 'containerName'] }
-      ]
-    };
-  }
-
-  async initialize(config: PluginConfig): Promise<void> {
-    this.config = config as AzureBlobConfig;
-    
-    if (!this.config.containerName) {
-      throw new Error('Azure Blob container name is required');
-    }
-
-    // Validate authentication method
-    if (!this.config.connectionString && !this.config.accountName) {
-      throw new Error('Either connection string or account name is required');
-    }
-
-    if (this.config.accountName && !this.config.accountKey && !this.config.sasToken) {
-      throw new Error('When using account name, either account key or SAS token is required');
-    }
-
-    this.blobServiceClient = this.createBlobServiceClient(this.config);
-  }
-
-  async destroy(): Promise<void> {
-    this.config = undefined;
-    this.blobServiceClient = undefined;
-  }
-
-  private createBlobServiceClient(config: AzureBlobConfig): BlobServiceClient {
-    // Option 1: Connection string (simplest)
-    if (config.connectionString) {
-      return BlobServiceClient.fromConnectionString(config.connectionString);
-    }
-
-    // Option 2: Account name + SAS token
-    if (config.accountName && config.sasToken) {
-      const blobServiceUri = `https://${config.accountName}.blob.core.windows.net`;
-      return new BlobServiceClient(`${blobServiceUri}?${config.sasToken}`);
-    }
-
-    // Option 3: Account name + Account key
-    if (config.accountName && config.accountKey) {
-      const sharedKeyCredential = new StorageSharedKeyCredential(config.accountName, config.accountKey);
-      const blobServiceUri = `https://${config.accountName}.blob.core.windows.net`;
-      return new BlobServiceClient(blobServiceUri, sharedKeyCredential);
-    }
-
-    throw new Error('Invalid Azure Blob Storage configuration');
-  }
-
-  private getMimeType(blobName: string): string {
-    const ext = path.extname(blobName).toLowerCase();
-    const mimeTypes: Record<string, string> = {
-      '.pdf': 'application/pdf',
-      '.doc': 'application/msword',
-      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-      '.xls': 'application/vnd.ms-excel',
-      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-      '.ppt': 'application/vnd.ms-powerpoint',
-      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-      '.txt': 'text/plain',
-      '.csv': 'text/csv',
-      '.json': 'application/json',
-      '.xml': 'application/xml',
-      '.jpg': 'image/jpeg',
-      '.jpeg': 'image/jpeg',
-      '.png': 'image/png',
-      '.gif': 'image/gif',
-      '.zip': 'application/zip',
-      '.tar': 'application/x-tar',
-      '.gz': 'application/gzip'
-    };
-
-    return mimeTypes[ext] || 'application/octet-stream';
-  }
-
-  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
-    // Apply size filter
-    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
-      return false;
-    }
-
-    // Apply date range filter
-    if (options?.filters?.dateRange) {
-      const { from, to } = options.filters.dateRange;
-      if (from && source.lastModified < from) return false;
-      if (to && source.lastModified > to) return false;
-    }
-
-    // Apply MIME type filter
-    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
-      return false;
-    }
-
-    return true;
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-}

package/plugins/azure-blob/plugin.json

@@ -1,12 +0,0 @@
-{
-  "name": "azure-blob",
-  "version": "1.0.0",
-  "description": "Azure Blob Storage document source plugin",
-  "author": "HubDoc Tools",
-  "main": "index.ts",
-  "hubdocToolVersion": "^1.0.0",
-  "dependencies": {
-    "@azure/storage-blob": "^12.0.0",
-    "fs-extra": "^11.1.0"
-  }
-}