@sinoia/hubdoc-tools 1.3.4 → 1.3.5

package/dist/plugins/aws-s3/index.js

@@ -0,0 +1,383 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_s3_1 = require("@aws-sdk/client-s3");
+const lib_storage_1 = require("@aws-sdk/lib-storage");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+class S3Plugin {
+    constructor() {
+        this.name = 's3';
+        this.version = '1.0.0';
+        this.description = 'AWS S3 document source';
+        this.supportedOperations = ['import', 'export', 'both'];
+    }
+    async testConnection(config) {
+        try {
+            const s3Config = config;
+            const client = this.createS3Client(s3Config);
+            await client.send(new client_s3_1.HeadBucketCommand({
+                Bucket: s3Config.bucketName
+            }));
+            return true;
+        }
+        catch (error) {
+            console.error(`S3 connection test failed: ${error.message}`);
+            return false;
+        }
+    }
+    async scan(config, options) {
+        this.config = config;
+        this.s3Client = this.createS3Client(this.config);
+        const sources = [];
+        const errors = [];
+        let totalSize = 0;
+        try {
+            const limit = this.config.limit || options?.limit;
+            console.log(`🔍 Scanning S3 bucket: ${this.config.bucketName}${limit ? ` (limit: ${limit})` : ''}...`);
+            const objects = await this.listAllObjects(this.config.prefix);
+            let processedCount = 0;
+            for (const obj of objects) {
+                const source = {
+                    id: obj.key,
+                    // Normalize filename to NFC to handle accented characters consistently across platforms
+                    name: path_1.default.basename(obj.key).normalize('NFC'),
+                    path: obj.key,
+                    size: obj.size,
+                    mimeType: this.getMimeType(obj.key),
+                    lastModified: obj.lastModified,
+                    metadata: {
+                        s3Key: obj.key,
+                        etag: obj.etag,
+                        storageClass: obj.storageClass,
+                        s3Metadata: obj.metadata
+                    }
+                };
+                // Apply filters
+                if (this.shouldIncludeSource(source, options)) {
+                    sources.push(source);
+                    totalSize += source.size;
+                    processedCount++;
+                    // Check limit
+                    if (limit && processedCount >= limit) {
+                        console.log(`📏 Reached limit of ${limit} files`);
+                        break;
+                    }
+                }
+            }
+            return {
+                sources,
+                totalCount: sources.length,
+                totalSize,
+                errors
+            };
+        }
+        catch (error) {
+            return {
+                sources: [],
+                totalCount: 0,
+                totalSize: 0,
+                errors: [`S3 scan failed: ${error.message}`]
+            };
+        }
+    }
+    async import(config, sources, targetDir, options) {
+        this.config = config;
+        this.s3Client = this.createS3Client(this.config);
+        const results = [];
+        const batchSize = options?.batchSize || 5;
+        // Process in batches to manage memory and connections
+        for (let i = 0; i < sources.length; i += batchSize) {
+            const batch = sources.slice(i, i + batchSize);
+            for (const source of batch) {
+                const result = await this.importSingle(source, targetDir);
+                results.push(result);
+                // Small delay to respect rate limits
+                await this.sleep(100);
+            }
+        }
+        return results;
+    }
+    async importSingle(source, targetDir) {
+        try {
+            if (!this.s3Client || !this.config)
+                throw new Error('S3 client not initialized');
+            const targetPath = path_1.default.join(targetDir, source.path);
+            const targetDirectory = path_1.default.dirname(targetPath);
+            await fs_extra_1.default.ensureDir(targetDirectory);
+            // Download file from S3
+            const command = new client_s3_1.GetObjectCommand({
+                Bucket: this.config.bucketName,
+                Key: source.id
+            });
+            const response = await this.s3Client.send(command);
+            if (!response.Body) {
+                throw new Error('Empty response body from S3');
+            }
+            // Stream to file
+            const writer = fs_extra_1.default.createWriteStream(targetPath);
+            // Handle different body types
+            if (response.Body instanceof ReadableStream) {
+                const reader = response.Body.getReader();
+                const pump = async () => {
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        writer.end();
+                        return;
+                    }
+                    writer.write(Buffer.from(value));
+                    return pump();
+                };
+                await pump();
+            }
+            else if (typeof response.Body.pipe === 'function') {
+                // Node.js stream
+                response.Body.pipe(writer);
+            }
+            else {
+                // Buffer or Uint8Array
+                writer.write(response.Body);
+                writer.end();
+            }
+            return new Promise((resolve) => {
+                writer.on('finish', () => {
+                    resolve({
+                        success: true,
+                        source,
+                        localPath: targetPath,
+                        bytesTransferred: source.size
+                    });
+                });
+                writer.on('error', (error) => {
+                    resolve({
+                        success: false,
+                        source,
+                        error: error.message
+                    });
+                });
+            });
+        }
+        catch (error) {
+            return {
+                success: false,
+                source,
+                error: error.message
+            };
+        }
+    }
+    async export(config, localSources, options) {
+        this.config = config;
+        this.s3Client = this.createS3Client(this.config);
+        const results = [];
+        for (const source of localSources) {
+            try {
+                if (!this.s3Client || !this.config)
+                    throw new Error('S3 client not initialized');
+                // Determine S3 key
+                let s3Key;
+                if (options?.preserveStructure) {
+                    s3Key = this.config.prefix ? `${this.config.prefix}/${source.path}` : source.path;
+                }
+                else {
+                    s3Key = this.config.prefix ? `${this.config.prefix}/${source.name}` : source.name;
+                }
+                // Read local file
+                const fileContent = await fs_extra_1.default.readFile(source.id);
+                // Upload to S3 using multipart upload for large files
+                const upload = new lib_storage_1.Upload({
+                    client: this.s3Client,
+                    params: {
+                        Bucket: this.config.bucketName,
+                        Key: s3Key,
+                        Body: fileContent,
+                        ContentType: source.mimeType,
+                        Metadata: {
+                            originalPath: source.path,
+                            originalName: source.name,
+                            uploadedBy: 'hubdoc-tools'
+                        }
+                    }
+                });
+                await upload.done();
+                const targetPath = options?.preserveStructure ? source.path : source.name;
+                results.push({
+                    success: true,
+                    targetPath,
+                    source,
+                    bytesTransferred: source.size
+                });
+            }
+            catch (error) {
+                results.push({
+                    success: false,
+                    targetPath: options?.targetPath || '',
+                    source,
+                    error: error.message
+                });
+            }
+        }
+        return results;
+    }
+    async listAllObjects(prefix) {
+        if (!this.s3Client || !this.config)
+            throw new Error('S3 client not initialized');
+        const allObjects = [];
+        let continuationToken;
+        try {
+            do {
+                const command = new client_s3_1.ListObjectsV2Command({
+                    Bucket: this.config.bucketName,
+                    Prefix: prefix,
+                    ContinuationToken: continuationToken,
+                    MaxKeys: 1000
+                });
+                const response = await this.s3Client.send(command);
+                const objects = response.Contents || [];
+                for (const obj of objects) {
+                    if (obj.Key && obj.Size !== undefined && obj.LastModified) {
+                        // Skip folders (keys ending with /)
+                        if (!obj.Key.endsWith('/')) {
+                            allObjects.push({
+                                key: obj.Key,
+                                size: obj.Size,
+                                lastModified: obj.LastModified,
+                                etag: obj.ETag || '',
+                                storageClass: obj.StorageClass
+                            });
+                        }
+                    }
+                }
+                continuationToken = response.NextContinuationToken;
+            } while (continuationToken);
+            return allObjects;
+        }
+        catch (error) {
+            throw new Error(`Failed to list S3 objects: ${error.message}`);
+        }
+    }
+    getConfigSchema() {
+        return {
+            type: 'object',
+            properties: {
+                accessKeyId: {
+                    type: 'string',
+                    description: 'AWS Access Key ID',
+                    required: true
+                },
+                secretAccessKey: {
+                    type: 'string',
+                    description: 'AWS Secret Access Key',
+                    required: true
+                },
+                region: {
+                    type: 'string',
+                    description: 'AWS Region (e.g., us-east-1)',
+                    required: true
+                },
+                bucketName: {
+                    type: 'string',
+                    description: 'S3 Bucket name',
+                    required: true
+                },
+                prefix: {
+                    type: 'string',
+                    description: 'S3 key prefix to filter objects (optional)',
+                    required: false
+                },
+                endpoint: {
+                    type: 'string',
+                    description: 'Custom S3 endpoint for S3-compatible services (optional)',
+                    required: false
+                },
+                limit: {
+                    type: 'number',
+                    description: 'Maximum number of documents to scan (useful for testing)',
+                    required: false
+                }
+            },
+            required: ['accessKeyId', 'secretAccessKey', 'region', 'bucketName']
+        };
+    }
+    async initialize(config) {
+        this.config = config;
+        if (!this.config.accessKeyId || !this.config.secretAccessKey) {
+            throw new Error('AWS credentials are required');
+        }
+        if (!this.config.region) {
+            throw new Error('AWS region is required');
+        }
+        if (!this.config.bucketName) {
+            throw new Error('S3 bucket name is required');
+        }
+        this.s3Client = this.createS3Client(this.config);
+    }
+    async destroy() {
+        this.config = undefined;
+        this.s3Client = undefined;
+    }
+    createS3Client(config) {
+        const clientConfig = {
+            region: config.region,
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey
+            }
+        };
+        // Support for S3-compatible services
+        if (config.endpoint) {
+            clientConfig.endpoint = config.endpoint;
+            clientConfig.forcePathStyle = true; // Required for most S3-compatible services
+        }
+        return new client_s3_1.S3Client(clientConfig);
+    }
+    getMimeType(key) {
+        const ext = path_1.default.extname(key).toLowerCase();
+        const mimeTypes = {
+            '.pdf': 'application/pdf',
+            '.doc': 'application/msword',
+            '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            '.xls': 'application/vnd.ms-excel',
+            '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            '.ppt': 'application/vnd.ms-powerpoint',
+            '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+            '.txt': 'text/plain',
+            '.csv': 'text/csv',
+            '.json': 'application/json',
+            '.xml': 'application/xml',
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.zip': 'application/zip',
+            '.tar': 'application/x-tar',
+            '.gz': 'application/gzip'
+        };
+        return mimeTypes[ext] || 'application/octet-stream';
+    }
+    shouldIncludeSource(source, options) {
+        // Apply size filter
+        if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+            return false;
+        }
+        // Apply date range filter
+        if (options?.filters?.dateRange) {
+            const { from, to } = options.filters.dateRange;
+            if (from && source.lastModified < from)
+                return false;
+            if (to && source.lastModified > to)
+                return false;
+        }
+        // Apply MIME type filter
+        if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+            return false;
+        }
+        return true;
+    }
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+exports.default = S3Plugin;
+//# sourceMappingURL=index.js.map

package/dist/plugins/aws-s3/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../plugins/aws-s3/index.ts"],"names":[],"mappings":";;;;;AAAA,kDAO4B;AAC5B,sDAA8C;AAC9C,wDAA0B;AAC1B,gDAAwB;AA+BxB,MAAqB,QAAQ;IAA7B;QACW,SAAI,GAAG,IAAI,CAAC;QACZ,YAAO,GAAG,OAAO,CAAC;QAClB,gBAAW,GAAG,wBAAwB,CAAC;QACvC,wBAAmB,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;IAyavE,CAAC;IApaC,KAAK,CAAC,cAAc,CAAC,MAAoB;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAkB,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAE7C,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,6BAAiB,CAAC;gBACtC,MAAM,EAAE,QAAQ,CAAC,UAAU;aAC5B,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAoB,EAAE,OAA6B;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAkB,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,MAAM,KAAK,GAAI,IAAI,CAAC,MAAc,CAAC,KAAK,IAAK,OAAe,EAAE,KAAK,CAAC;YACpE,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAEvG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE9D,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAmB;oBAC7B,EAAE,EAAE,GAAG,CAAC,GAAG;oBACX,wFAAwF;oBACxF,IAAI,EAAE,cAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;oBAC7C,IAAI,EAAE,GAAG,CAAC,GAAG;oBACb,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;oBACnC,YAAY,EAAE,GAAG,CAAC,YAAY;oBAC9B,QAAQ,EAAE;wBACR,KAAK,EAAE,GAAG,CAAC,GAAG;wBACd,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,YAAY,EAAE,GAAG,CAAC,YAAY;wBAC9B,UAAU,EAAE,GAAG,CAAC,QAAQ;qBACzB;iBACF,CAAC;gBAEF,gBAAgB;gBAChB,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC9C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACrB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;oBACzB,cAAc,EAAE,CAAC;oBAEjB,cAAc;oBACd,IAAI,KAAK,IAAI,cAAc,IAAI,KAAK,EAAE,CAAC;wBACrC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,QAAQ,CAAC,CAAC;wBAClD,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO;gBACP,UAAU,EAAE,OAAO,CAAC,MAAM;gBAC1B,SAAS;gBACT,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,mBAAmB,KAAK,CAAC,OAAO,EAAE,CAAC;aAC7C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAoB,EACpB,OAAyB,EACzB,SAAiB,EACjB,OAA6B;QAE7B,IAAI,CAAC,MAAM,GAAG,MAAkB,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;QAE1C,sDAAsD;QACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAE9C,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC1D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAErB,qCAAqC;gBACrC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAsB,EAAE,SAAiB;QAClE,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAEjF,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,eAAe,GAAG,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEjD,MAAM,kBAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAEpC,wBAAwB;YACxB,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;gBAC9B,GAAG,EAAE,MAAM,CAAC,EAAE;aACf,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEnD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;YACjD,CAAC;YAED,iBAAiB;YACjB,MAAM,MAAM,GAAG,kBAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAEhD,8BAA8B;YAC9B,IAAI,QAAQ,CAAC,IAAI,YAAY,cAAc,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACzC,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;oBACrC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,CAAC,GAAG,EAAE,CAAC;wBACb,OAAO;oBACT,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBACjC,OAAO,IAAI,EAAE,CAAC;gBAChB,CAAC,CAAC;gBACF,MAAM,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,IAAI,OAAQ,QAAQ,CAAC,IAAY,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC7D,iBAAiB;gBAChB,QAAQ,CAAC,IAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,uBAAuB;gBACvB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC;gBACnC,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,CAAC;YAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC7B,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;oBACvB,OAAO,CAAC;wBACN,OAAO,EAAE,IAAI;wBACb,MAAM;wBACN,SAAS,EAAE,UAAU;wBACrB,gBAAgB,EAAE,MAAM,CAAC,IAAI;qBAC9B,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;oBAC3B,OAAO,CAAC;wBACN,OAAO,EAAE,KAAK;wBACd,MAAM;wBACN,KAAK,EAAE,KAAK,CAAC,OAAO;qBACrB,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM;gBACN,KAAK,EAAE,KAAK,CAAC,OAAO;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAoB,EACpB,YAA8B,EAC9B,OAA6B;QAE7B,IAAI,CAAC,MAAM,GAAG,MAAkB,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBAEjF,mBAAmB;gBACnB,IAAI,KAAa,CAAC;gBAClB,IAAI,OAAO,EAAE,iBAAiB,EAAE,CAAC;oBAC/B,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;gBACpF,CAAC;qBAAM,CAAC;oBACN,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;gBACpF,CAAC;gBAED,kBAAkB;gBAClB,MAAM,WAAW,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAEjD,sDAAsD;gBACtD,MAAM,MAAM,GAAG,IAAI,oBAAM,CAAC;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ;oBACrB,MAAM,EAAE;wBACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;wBAC9B,GAAG,EAAE,KAAK;wBACV,IAAI,EAAE,WAAW;wBACjB,WAAW,EAAE,MAAM,CAAC,QAAQ;wBAC5B,QAAQ,EAAE;4BACR,YAAY,EAAE,MAAM,CAAC,IAAI;4BACzB,YAAY,EAAE,MAAM,CAAC,IAAI;4BACzB,UAAU,EAAE,cAAc;yBAC3B;qBACF;iBACF,CAAC,CAAC;gBAEH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEpB,MAAM,UAAU,GAAG,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;gBAE1E,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,IAAI;oBACb,UAAU;oBACV,MAAM;oBACN,gBAAgB,EAAE,MAAM,CAAC,IAAI;iBAC9B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,EAAE;oBACrC,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,OAAO;iBACrB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAe;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAEjF,MAAM,UAAU,GAAe,EAAE,CAAC;QAClC,IAAI,iBAAqC,CAAC;QAE1C,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,MAAM,OAAO,GAAG,IAAI,gCAAoB,CAAC;oBACvC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;oBAC9B,MAAM,EAAE,MAAM;oBACd,iBAAiB,EAAE,iBAAiB;oBACpC,OAAO,EAAE,IAAI;iBACd,CAAC,CAAC;gBAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAExC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC1B,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;wBAC1D,oCAAoC;wBACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC3B,UAAU,CAAC,IAAI,CAAC;gCACd,GAAG,EAAE,GAAG,CAAC,GAAG;gCACZ,IAAI,EAAE,GAAG,CAAC,IAAI;gCACd,YAAY,EAAE,GAAG,CAAC,YAAY;gCAC9B,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gCACpB,YAAY,EAAE,GAAG,CAAC,YAAY;6BAC/B,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,iBAAiB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;YACrD,CAAC,QAAQ,iBAAiB,EAAE;YAE5B,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,eAAe;QACb,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,mBAAmB;oBAChC,QAAQ,EAAE,IAAI;iBACf;gBACD,eAAe,EAAE;oBACf,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uBAAuB;oBACpC,QAAQ,EAAE,IAAI;iBACf;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,8BAA8B;oBAC3C,QAAQ,EAAE,IAAI;iBACf;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,gBAAgB;oBAC7B,QAAQ,EAAE,IAAI;iBACf;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,4CAA4C;oBACzD,QAAQ,EAAE,KAAK;iBAChB;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,0DAA0D;oBACvE,QAAQ,EAAE,KAAK;iBAChB;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,0DAA0D;oBACvE,QAAQ,EAAE,KAAK;iBAChB;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,EAAE,iBAAiB,EAAE,QAAQ,EAAE,YAAY,CAAC;SACrE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAoB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAkB,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEO,cAAc,CAAC,MAAgB;QACrC,MAAM,YAAY,GAAQ;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE;gBACX,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;aACxC;SACF,CAAC;QAEF,qCAAqC;QACrC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,YAAY,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACxC,YAAY,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,2CAA2C;QACjF,CAAC;QAED,OAAO,IAAI,oBAAQ,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,MAAM,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAA2B;YACxC,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,oBAAoB;YAC5B,OAAO,EAAE,yEAAyE;YAClF,MAAM,EAAE,0BAA0B;YAClC,OAAO,EAAE,mEAAmE;YAC5E,MAAM,EAAE,+BAA+B;YACvC,OAAO,EAAE,2EAA2E;YACpF,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,kBAAkB;YAC3B,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,YAAY;YACpB,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,kBAAkB;SAC1B,CAAC;QAEF,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,0BAA0B,CAAC;IACtD,CAAC;IAEO,mBAAmB,CAAC,MAAsB,EAAE,OAA6B;QAC/E,oBAAoB;QACpB,IAAI,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YAChC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;YAC/C,IAAI,IAAI,IAAI,MAAM,CAAC,YAAY,GAAG,IAAI;gBAAE,OAAO,KAAK,CAAC;YACrD,IAAI,EAAE,IAAI,MAAM,CAAC,YAAY,GAAG,EAAE;gBAAE,OAAO,KAAK,CAAC;QACnD,CAAC;QAED,yBAAyB;QACzB,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF;AA7aD,2BA6aC"}

package/dist/plugins/azure-blob/index.d.ts

@@ -0,0 +1,23 @@
+import { DocumentSourcePlugin, DocumentSource, PluginConfig, ScanResult, PluginImportOptions, PluginExportOptions, ImportResult, ExportResult } from '../../src/types/plugins';
+export default class AzureBlobPlugin implements DocumentSourcePlugin {
+    readonly name = "azure-blob";
+    readonly version = "1.0.0";
+    readonly description = "Azure Blob Storage document source";
+    readonly supportedOperations: readonly ["import", "export", "both"];
+    private config?;
+    private blobServiceClient?;
+    testConnection(config: PluginConfig): Promise<boolean>;
+    scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult>;
+    import(config: PluginConfig, sources: DocumentSource[], targetDir: string, options?: PluginImportOptions): Promise<ImportResult[]>;
+    private importSingle;
+    export?(config: PluginConfig, localSources: DocumentSource[], options?: PluginExportOptions): Promise<ExportResult[]>;
+    private listAllBlobs;
+    getConfigSchema(): Record<string, any>;
+    initialize(config: PluginConfig): Promise<void>;
+    destroy(): Promise<void>;
+    private createBlobServiceClient;
+    private getMimeType;
+    private shouldIncludeSource;
+    private sleep;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/plugins/azure-blob/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../plugins/azure-blob/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,mBAAmB,EACnB,YAAY,EACZ,YAAY,EACb,MAAM,yBAAyB,CAAC;AAsBjC,MAAM,CAAC,OAAO,OAAO,eAAgB,YAAW,oBAAoB;IAClE,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,OAAO,WAAW;IAC3B,QAAQ,CAAC,WAAW,wCAAwC;IAC5D,QAAQ,CAAC,mBAAmB,wCAAyC;IAErE,OAAO,CAAC,MAAM,CAAC,CAAkB;IACjC,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAExC,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;IAetD,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC;IA+D9E,MAAM,CACV,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,YAAY,EAAE,CAAC;YAuBZ,YAAY;IA8BpB,MAAM,CAAC,CACX,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,cAAc,EAAE,EAC9B,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,YAAY,EAAE,CAAC;YAyDZ,YAAY;IAkC1B,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAiDhC,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB/C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9B,OAAO,CAAC,uBAAuB;IAsB/B,OAAO,CAAC,WAAW;IA0BnB,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,KAAK;CAGd"}