@sinoia/hubdoc-tools 1.3.2 → 1.3.4

package/plugins/box/index.ts

@@ -0,0 +1,495 @@
+import axios, { AxiosInstance } from 'axios';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface BoxConfig extends PluginConfig {
+  accessToken: string;
+  clientId?: string;
+  clientSecret?: string;
+  rootFolderId?: string; // Default to '0' (root)
+  limit?: number;
+}
+
+interface BoxItem {
+  type: 'file' | 'folder';
+  id: string;
+  name: string;
+  size?: number;
+  modified_at: string;
+  created_at: string;
+  description?: string;
+  path_collection?: {
+    entries: Array<{
+      type: string;
+      id: string;
+      name: string;
+    }>;
+  };
+  created_by?: {
+    type: string;
+    id: string;
+    name: string;
+    login: string;
+  };
+  modified_by?: {
+    type: string;
+    id: string;
+    name: string;
+    login: string;
+  };
+  shared_link?: {
+    url: string;
+    download_url: string;
+  };
+  tags?: string[];
+}
+
+export default class BoxPlugin implements DocumentSourcePlugin {
+  readonly name = 'box';
+  readonly version = '1.0.0';
+  readonly description = 'Box cloud storage document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: BoxConfig;
+  private apiClient?: AxiosInstance;
+  private readonly baseUrl = 'https://api.box.com/2.0';
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const client = this.createApiClient(config as BoxConfig);
+      const response = await client.get('/users/me');
+      return response.status === 200;
+    } catch (error: any) {
+      console.error(`Box connection test failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as BoxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const limit = (this.config as any).limit || (options as any)?.limit;
+      console.log(`🔍 Scanning Box documents${limit ? ` (limit: ${limit})` : ''}...`);
+      
+      const rootFolderId = this.config.rootFolderId || '0';
+      const items = await this.scanFolder(rootFolderId, '', options);
+
+      let processedCount = 0;
+      for (const item of items) {
+        if (item.type === 'file') {
+          const source: DocumentSource = {
+            id: item.id,
+            name: item.name,
+            path: this.getItemPath(item),
+            size: item.size || 0,
+            mimeType: this.getMimeType(item.name),
+            lastModified: new Date(item.modified_at),
+            metadata: {
+              boxId: item.id,
+              description: item.description,
+              createdAt: item.created_at,
+              createdBy: item.created_by?.name,
+              modifiedBy: item.modified_by?.name,
+              tags: item.tags,
+              sharedLink: item.shared_link?.url,
+              pathCollection: item.path_collection
+            }
+          };
+
+          // Apply filters
+          if (this.shouldIncludeSource(source, options)) {
+            sources.push(source);
+            totalSize += source.size;
+            processedCount++;
+
+            // Check limit
+            if (limit && processedCount >= limit) {
+              console.log(`📏 Reached limit of ${limit} files`);
+              break;
+            }
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`Box scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  private async scanFolder(folderId: string, parentPath: string, options?: PluginImportOptions): Promise<BoxItem[]> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const allItems: BoxItem[] = [];
+    let offset = 0;
+    const limit = 1000; // Box API max
+
+    try {
+      while (true) {
+        const response = await this.apiClient.get(`/folders/${folderId}/items`, {
+          params: {
+            fields: 'id,name,type,size,modified_at,created_at,description,path_collection,created_by,modified_by,shared_link,tags',
+            limit,
+            offset
+          }
+        });
+
+        const items: BoxItem[] = response.data.entries || [];
+        
+        for (const item of items) {
+          const itemPath = parentPath ? `${parentPath}/${item.name}` : item.name;
+          
+          if (item.type === 'file') {
+            // Add path information to the item
+            (item as any).fullPath = itemPath;
+            allItems.push(item);
+          } else if (item.type === 'folder') {
+            // Recursively scan subfolders
+            const subItems = await this.scanFolder(item.id, itemPath, options);
+            allItems.push(...subItems);
+          }
+        }
+
+        // Check if there are more items
+        if (items.length < limit) {
+          break;
+        }
+        offset += limit;
+      }
+
+      return allItems;
+    } catch (error: any) {
+      if (error.response?.status === 404) {
+        console.warn(`Warning: Folder not found: ${folderId}`);
+        return [];
+      }
+      throw error;
+    }
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as BoxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 5;
+
+    // Process in batches to respect API limits
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(200);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.apiClient) throw new Error('API client not initialized');
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+
+      // Download file from Box
+      const response = await this.apiClient.get(`/files/${source.id}/content`, {
+        responseType: 'stream',
+        maxRedirects: 5 // Box returns redirects to download URLs
+      });
+      
+      const writer = fs.createWriteStream(targetPath);
+      response.data.pipe(writer);
+
+      return new Promise((resolve) => {
+        writer.on('finish', () => {
+          resolve({
+            success: true,
+            source,
+            localPath: targetPath,
+            bytesTransferred: source.size
+          });
+        });
+
+        writer.on('error', (error) => {
+          resolve({
+            success: false,
+            source,
+            error: error.message
+          });
+        });
+      });
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as BoxConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ExportResult[] = [];
+    const rootFolderId = this.config.rootFolderId || '0';
+
+    for (const source of localSources) {
+      try {
+        // Determine target folder
+        let targetFolderId = rootFolderId;
+        
+        if (options?.preserveStructure && source.path.includes('/')) {
+          const folderPath = path.dirname(source.path);
+          targetFolderId = await this.createFolderStructure(folderPath, rootFolderId);
+        }
+
+        // Upload file to Box
+        const fileContent = await fs.readFile(source.id);
+        // Normalize filename to NFC to handle accented characters consistently across platforms
+        const fileName = (options?.preserveStructure ? path.basename(source.path) : source.name).normalize('NFC');
+        
+        // Box requires multipart form data for uploads
+        const FormData = require('form-data');
+        const form = new FormData();
+        
+        form.append('attributes', JSON.stringify({
+          name: fileName,
+          parent: { id: targetFolderId }
+        }));
+        form.append('file', fileContent, fileName);
+
+        const response = await this.apiClient!.post('/files/content', form, {
+          headers: {
+            ...form.getHeaders()
+          }
+        });
+
+        const targetPath = options?.preserveStructure ? source.path : source.name;
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: source.size
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async createFolderStructure(folderPath: string, parentId: string): Promise<string> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const parts = folderPath.split('/').filter(part => part.length > 0);
+    let currentParentId = parentId;
+
+    for (const folderName of parts) {
+      // Check if folder already exists
+      try {
+        const response = await this.apiClient.get(`/folders/${currentParentId}/items`, {
+          params: {
+            fields: 'id,name,type',
+            limit: 1000
+          }
+        });
+
+        const existingFolder = response.data.entries.find(
+          (item: any) => item.type === 'folder' && item.name === folderName
+        );
+
+        if (existingFolder) {
+          currentParentId = existingFolder.id;
+        } else {
+          // Create new folder
+          const createResponse = await this.apiClient.post('/folders', {
+            name: folderName,
+            parent: { id: currentParentId }
+          });
+          currentParentId = createResponse.data.id;
+        }
+      } catch (error: any) {
+        throw new Error(`Failed to create folder structure: ${error.message}`);
+      }
+    }
+
+    return currentParentId;
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        accessToken: {
+          type: 'string',
+          description: 'Box Access Token (get from Box Developer Console)',
+          required: true
+        },
+        clientId: {
+          type: 'string',
+          description: 'Box Client ID (optional, for OAuth flow)',
+          required: false
+        },
+        clientSecret: {
+          type: 'string',
+          description: 'Box Client Secret (optional, for OAuth flow)',
+          required: false
+        },
+        rootFolderId: {
+          type: 'string',
+          description: 'Root folder ID to scan (default: 0 for root)',
+          default: '0'
+        },
+        limit: {
+          type: 'number',
+          description: 'Maximum number of documents to scan (useful for testing)',
+          required: false
+        }
+      },
+      required: ['accessToken']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as BoxConfig;
+    
+    if (!this.config.accessToken) {
+      throw new Error('Box access token is required');
+    }
+
+    this.apiClient = this.createApiClient(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.apiClient = undefined;
+  }
+
+  private createApiClient(config: BoxConfig): AxiosInstance {
+    return axios.create({
+      baseURL: this.baseUrl,
+      headers: {
+        'Authorization': `Bearer ${config.accessToken}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+  }
+
+  private getItemPath(item: BoxItem): string {
+    // Use the full path if available, otherwise construct from path_collection
+    if ((item as any).fullPath) {
+      return (item as any).fullPath;
+    }
+
+    if (item.path_collection && item.path_collection.entries) {
+      const pathParts = item.path_collection.entries
+        .filter(entry => entry.type === 'folder' && entry.id !== '0')
+        .map(entry => entry.name);
+      pathParts.push(item.name);
+      return pathParts.join('/');
+    }
+
+    return item.name;
+  }
+
+  private getMimeType(fileName: string): string {
+    const ext = path.extname(fileName).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.pdf': 'application/pdf',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.xls': 'application/vnd.ms-excel',
+      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      '.ppt': 'application/vnd.ms-powerpoint',
+      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+      '.txt': 'text/plain',
+      '.csv': 'text/csv',
+      '.json': 'application/json',
+      '.xml': 'application/xml',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.png': 'image/png',
+      '.gif': 'image/gif',
+      '.zip': 'application/zip'
+    };
+
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/box/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "box",
+  "version": "1.0.0",
+  "description": "Box document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.ts",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "box-node-sdk": "^2.0.0",
+    "fs-extra": "^11.1.0"
+  }
+}

package/plugins/core/README.md

@@ -0,0 +1,122 @@
+# Core Plugin
+
+Plugin pour importer des documents depuis le système Core vers HubDoc.
+
+## Description
+
+Ce plugin permet d'importer des documents stockés dans un système Core (Document Management System) vers HubDoc. Il utilise l'API REST de Core pour lister et télécharger les documents.
+
+## Fonctionnalités
+
+- **Scan** : Liste tous les documents disponibles dans Core avec pagination
+- **Import** : Télécharge les documents vers un buffer local puis les transfère vers HubDoc
+- **Authentification** : Support de l'authentification Bearer Token avec Core
+- **Filtrage** : Support des filtres par taille, date, et type MIME
+- **Métadonnées** : Préservation des métadonnées Core (tags, type, sous-type, etc.)
+
+## Configuration
+
+Le plugin nécessite les paramètres suivants :
+
+### Paramètres obligatoires
+
+- **baseUrl** : URL de base de l'API Core (ex: `https://your-core-instance.com`)
+- **username** : Nom d'utilisateur pour l'authentification Core
+- **password** : Mot de passe pour l'authentification Core
+
+### Paramètres optionnels
+
+- **bearerToken** : Token Bearer pré-obtenu (optionnel, sera récupéré automatiquement si non fourni)
+- **tempDir** : Répertoire temporaire pour les fichiers buffer (par défaut : `{targetDir}/.temp`)
+
+## Exemple de configuration
+
+```json
+{
+  "baseUrl": "https://my-core-instance.com",
+  "username": "myuser",
+  "password": "mypassword",
+  "tempDir": "/tmp/core-import"
+}
+```
+
+## Stratégie d'import
+
+Le plugin utilise une stratégie de buffer local pour l'import :
+
+1. **Scan** : Liste les documents via `/api/d2/docs` avec pagination
+2. **Fetch** : Récupère le détail de chaque document via `/api/d2/docs/{id}`
+3. **Buffer** : Crée un fichier temporaire local avec le contenu du document
+4. **Transfer** : Déplace le fichier vers le répertoire cible avec la structure appropriée
+5. **Cleanup** : Supprime les fichiers temporaires
+
+## Structure des documents
+
+Les documents Core sont organisés selon cette structure :
+```
+{type}/{subtype}/{document_title}.txt
+```
+
+Exemples :
+- `document/general/My_Document.txt`
+- `html/report/Monthly_Report.txt`
+- `unknown/general/Document_123.txt`
+
+## Métadonnées préservées
+
+Le plugin préserve les métadonnées suivantes de Core :
+
+- **coreId** : ID unique du document dans Core
+- **coreType** : Type de document Core
+- **coreSubtype** : Sous-type de document Core
+- **corePath** : Chemin original dans Core
+- **tags** : Tags associés au document
+- **hasAttachments** : Présence de pièces jointes
+- **signed** : Statut de signature
+- **percent** : Pourcentage de completion
+- **note** : Note attribuée
+- **body** : Extrait du contenu (500 premiers caractères)
+
+## Limitations
+
+- **Export non supporté** : L'export vers Core n'est pas implémenté car les documents doivent être gérés dans Core même
+- **Contenu textuel uniquement** : Les documents sont extraits en format texte
+- **Pièces jointes** : Les attachments Core ne sont pas téléchargés directement
+
+## API Core utilisée
+
+Le plugin utilise les endpoints suivants de l'API Core :
+
+- `POST /api/token` : Authentification et récupération du token
+- `GET /api/d2/docs` : Liste des documents avec pagination
+- `GET /api/d2/docs/{id}` : Détail d'un document spécifique
+
+## Gestion des erreurs
+
+Le plugin gère les erreurs suivantes :
+
+- **Erreurs d'authentification** : Token invalide ou expiré
+- **Erreurs de pagination** : Pages manquantes ou inaccessibles
+- **Erreurs de téléchargement** : Documents inaccessibles
+- **Erreurs de filesystem** : Problèmes de buffer local
+
+## Utilisation
+
+```bash
+# Configuration du plugin
+hubdoc-tool connect core
+
+# Scan des documents Core
+hubdoc-tool plugin-scan core-connection
+
+# Import vers HubDoc
+hubdoc-tool plugin-import core-mapping.csv
+```
+
+## Exemple de mapping CSV généré
+
+```csv
+File Path,Target Folder,Workspace,Metadata (JSON),Permissions
+document/general/My_Document.txt,Core Documents,Core Project,"{\"coreId\":\"123\",\"coreType\":\"document\"}",
+html/report/Monthly_Report.txt,Core Reports,Core Project,"{\"coreId\":\"456\",\"coreType\":\"html\"}",
+```