@sinoia/hubdoc-tools 1.3.2 → 1.3.3

package/plugins/filesystem/index.ts

@@ -0,0 +1,360 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { glob } from 'glob';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+import { XmlMetadataParser } from '../../src/utils/xml-metadata';
+
+interface FileSystemConfig extends PluginConfig {
+  basePath: string;
+  includeHidden?: boolean;
+  followSymlinks?: boolean;
+  useXmlMetadata?: boolean;  // Enable XML metadata parsing
+  xmlMetadataPattern?: string;  // Pattern for metadata files
+}
+
+export default class FileSystemPlugin implements DocumentSourcePlugin {
+  readonly name = 'filesystem';
+  readonly version = '1.0.0';
+  readonly description = 'Local filesystem document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: FileSystemConfig;
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    const fsConfig = config as FileSystemConfig;
+    try {
+      const stats = await fs.stat(fsConfig.basePath);
+      return stats.isDirectory();
+    } catch {
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    const fsConfig = config as FileSystemConfig;
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+
+    try {
+      let pattern = '**/*';
+      if (options?.filters?.path) {
+        pattern = path.join(options.filters.path, '**/*');
+      }
+
+      const globOptions = {
+        cwd: fsConfig.basePath,
+        absolute: true,
+        dot: fsConfig.includeHidden || false,
+        followSymbolicLinks: fsConfig.followSymlinks || false
+      };
+
+      const files = await glob(pattern, globOptions);
+      let totalSize = 0;
+      let processedCount = 0;
+
+      console.log(`📁 Found ${files.length} files${options?.limit ? ` (limit: ${options.limit})` : ''}`);
+
+      for (const filePath of files) {
+        // Check limit
+        if (options?.limit && processedCount >= options.limit) {
+          console.log(`📏 Reached limit of ${options.limit} files`);
+          break;
+        }
+        try {
+          const stats = await fs.stat(filePath);
+          
+          if (!stats.isFile()) continue;
+
+          // Apply filters
+          if (options?.filters?.maxSize && stats.size > options.filters.maxSize) {
+            continue;
+          }
+
+          if (options?.filters?.dateRange) {
+            const { from, to } = options.filters.dateRange;
+            if (from && stats.mtime < from) continue;
+            if (to && stats.mtime > to) continue;
+          }
+
+          const relativePath = path.relative(fsConfig.basePath, filePath);
+          const mimeType = this.getMimeType(filePath);
+
+          if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(mimeType)) {
+            continue;
+          }
+
+          // Skip metadata files from being processed as documents
+          if (this.isMetadataFile(filePath)) {
+            continue;
+          }
+
+          // Build base metadata
+          const baseMetadata = {
+            fullPath: filePath,
+            directory: path.dirname(relativePath),
+            extension: path.extname(filePath)
+          };
+
+          // Try to find and parse XML metadata if enabled
+          let xmlMetadata = {};
+          if (fsConfig.useXmlMetadata !== false) { // Default to true unless explicitly disabled
+            try {
+              const metadataFile = XmlMetadataParser.findMetadataFile(filePath);
+              if (metadataFile) {
+                const parsedXmlMetadata = await XmlMetadataParser.parseMetadataFile(metadataFile);
+                xmlMetadata = XmlMetadataParser.getHubDocMetadata(parsedXmlMetadata);
+                console.log(`📋 Found metadata for ${path.basename(filePath)}`);
+              }
+            } catch (error) {
+              console.warn(`⚠️  Failed to parse metadata for ${filePath}: ${error}`);
+            }
+          }
+
+          sources.push({
+            id: filePath,
+            // Normalize filename to NFC to handle accented characters consistently across platforms
+            name: path.basename(filePath).normalize('NFC'),
+            path: relativePath,
+            size: stats.size,
+            mimeType,
+            lastModified: stats.mtime,
+            metadata: {
+              ...baseMetadata,
+              ...xmlMetadata
+            }
+          });
+
+          totalSize += stats.size;
+          processedCount++;
+        } catch (error: any) {
+          errors.push(`Error reading ${filePath}: ${error.message}`);
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`Scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 10;
+
+    // Process in batches
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      const batchPromises = batch.map(source => this.importSingle(source, targetDir));
+      
+      const batchResults = await Promise.allSettled(batchPromises);
+      
+      for (const result of batchResults) {
+        if (result.status === 'fulfilled') {
+          results.push(result.value);
+        } else {
+          results.push({
+            success: false,
+            source: batch[results.length % batch.length],
+            error: result.reason?.message || 'Unknown error'
+          });
+        }
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+      await fs.copy(source.id, targetPath);
+      
+      const stats = await fs.stat(targetPath);
+      
+      return {
+        success: true,
+        source,
+        localPath: targetPath,
+        bytesTransferred: stats.size
+      };
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    const fsConfig = config as FileSystemConfig;
+    const results: ExportResult[] = [];
+
+    for (const source of localSources) {
+      try {
+        let targetPath: string;
+        
+        if (options?.preserveStructure) {
+          targetPath = path.join(fsConfig.basePath, options.targetPath, source.path);
+        } else {
+          targetPath = path.join(fsConfig.basePath, options?.targetPath || '', source.name);
+        }
+
+        const targetDir = path.dirname(targetPath);
+        await fs.ensureDir(targetDir);
+
+        if (!options?.overwrite && await fs.pathExists(targetPath)) {
+          results.push({
+            success: false,
+            targetPath,
+            source,
+            error: 'File already exists and overwrite is disabled'
+          });
+          continue;
+        }
+
+        await fs.copy(source.id, targetPath);
+        const stats = await fs.stat(targetPath);
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: stats.size
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        basePath: {
+          type: 'string',
+          description: 'Base directory path to scan',
+          required: true
+        },
+        includeHidden: {
+          type: 'boolean',
+          description: 'Include hidden files and directories',
+          default: false
+        },
+        followSymlinks: {
+          type: 'boolean',
+          description: 'Follow symbolic links',
+          default: false
+        },
+        useXmlMetadata: {
+          type: 'boolean',
+          description: 'Parse XML metadata files (e.g., *_metadata.xml)',
+          default: true
+        },
+        xmlMetadataPattern: {
+          type: 'string',
+          description: 'Pattern for metadata files',
+          default: '*_metadata.xml'
+        }
+      },
+      required: ['basePath']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as FileSystemConfig;
+    
+    // Validate base path exists
+    if (!await fs.pathExists(this.config.basePath)) {
+      throw new Error(`Base path does not exist: ${this.config.basePath}`);
+    }
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+  }
+
+  /**
+   * Check if a file is a metadata file and should be skipped from document processing
+   */
+  private isMetadataFile(filePath: string): boolean {
+    const fileName = path.basename(filePath).toLowerCase();
+    
+    // Common metadata file patterns
+    const metadataPatterns = [
+      /_metadata\.xml$/,
+      /\.metadata\.xml$/,
+      /_meta\.xml$/,
+      /^metadata_.*\.xml$/
+    ];
+
+    return metadataPatterns.some(pattern => pattern.test(fileName));
+  }
+
+  private getMimeType(filePath: string): string {
+    const ext = path.extname(filePath).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.pdf': 'application/pdf',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.xls': 'application/vnd.ms-excel',
+      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      '.ppt': 'application/vnd.ms-powerpoint',
+      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+      '.txt': 'text/plain',
+      '.csv': 'text/csv',
+      '.json': 'application/json',
+      '.xml': 'application/xml',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.png': 'image/png',
+      '.gif': 'image/gif',
+      '.bmp': 'image/bmp',
+      '.tiff': 'image/tiff',
+      '.zip': 'application/zip',
+      '.rar': 'application/vnd.rar',
+      '.7z': 'application/x-7z-compressed'
+    };
+
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+}

package/plugins/filesystem/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "filesystem",
+  "version": "1.0.0",
+  "description": "Local filesystem document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.js",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "fs-extra": "^11.1.0",
+    "glob": "^10.3.0"
+  }
+}