@sinoia/hubdoc-tools 1.3.2 → 1.3.4

package/plugins/onedrive/TESTING.md

@@ -0,0 +1,197 @@
+# Guide de test du plugin OneDrive
+
+Ce guide vous permet de tester le plugin OneDrive avec Microsoft Graph API.
+
+## Prérequis
+
+1. **Azure CLI installé** : `brew install azure-cli` (macOS) ou voir [docs Azure](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)
+2. **Compte Microsoft/Office 365** avec accès OneDrive
+3. **HubDoc Tools configuré**
+
+> ⚠️ **Note** : Pour un test rapide, seul Azure CLI est nécessaire. L'enregistrement d'application Azure AD n'est requis que pour une intégration production.
+
+## Configuration de test
+
+### 1. Enregistrer une application Azure AD
+
+1. Aller sur [Azure Portal](https://portal.azure.com)
+2. Naviguer vers **Azure Active Directory** → **App registrations**
+3. Cliquer **New registration**
+4. Configurer les permissions Microsoft Graph :
+   - `Files.Read.All` (pour lire tous les fichiers)
+   - `Sites.Read.All` (pour SharePoint)
+   - `User.Read` (pour les informations utilisateur)
+
+### 2. Obtenir un token d'accès (Test rapide avec Azure CLI)
+
+```bash
+# Installer Azure CLI si nécessaire
+brew install azure-cli
+
+# Se connecter à votre compte Microsoft/Office 365
+az login
+
+# Obtenir un token pour Microsoft Graph
+az account get-access-token --resource https://graph.microsoft.com/
+```
+
+La commande retournera un JSON avec le token dans le champ `accessToken` :
+```json
+{
+  "accessToken": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIs...",
+  "expiresOn": "2025-01-10 15:30:00.000000",
+  ...
+}
+```
+
+### 3. Configuration pour test rapide
+
+Pour un test avec Azure CLI (pas besoin d'enregistrer d'app) :
+
+```json
+{
+  "clientId": "dummy",
+  "clientSecret": "dummy", 
+  "tenantId": "dummy",
+  "accessToken": "YOUR_TOKEN_FROM_AZ_CLI_COMMAND",
+  "rootPath": "/",
+  "limit": 10
+}
+```
+
+> 💡 Les champs `clientId`, `clientSecret`, et `tenantId` peuvent être des valeurs factices quand vous utilisez un token direct d'Azure CLI.
+
+### 4. Tester la connexion
+
+```bash
+# Lister les plugins disponibles
+npm run dev -- plugins
+
+# Vérifier que OneDrive est bien listé
+```
+
+### 5. Scanner les documents (test rapide)
+
+Utiliser le script de test :
+
+```bash
+# Modifier le fichier test-onedrive.js avec vos vraies valeurs
+node test-onedrive.js
+```
+
+## Résultats attendus
+
+### Scan réussi
+- ✅ Connection: SUCCESS
+- ✅ Documents trouvés avec métadonnées OneDrive
+- ✅ Limitation respectée (paramètre `limit`)
+- ✅ Types de documents variés (Office, PDF, images, etc.)
+
+### Structure des documents
+- **Chemin** : Structure de dossiers OneDrive/SharePoint
+- **Métadonnées** : OneDrive ID, download URL, parent path
+- **Types MIME** : Détection automatique basée sur l'extension
+
+## Test d'intégration complète
+
+### 1. Configuration d'une connexion
+```bash
+echo '{
+  "clientId": "YOUR_CLIENT_ID",
+  "clientSecret": "YOUR_CLIENT_SECRET", 
+  "tenantId": "YOUR_TENANT_ID",
+  "accessToken": "YOUR_TOKEN",
+  "rootPath": "/Documents",
+  "limit": 10
+}' > onedrive-config.json
+
+npm run dev -- connect onedrive --config onedrive-config.json --name test-onedrive
+```
+
+### 2. Scan des documents
+```bash
+npm run dev -- plugin-scan test-onedrive --output onedrive-mapping.csv
+```
+
+### 3. Vérification du mapping
+```bash
+head -10 onedrive-mapping.csv
+```
+
+Le fichier doit contenir :
+- Colonnes : File Path, Target Folder, Workspace, Metadata (JSON), Permissions
+- Métadonnées JSON avec OneDrive ID, download URL
+- Chemins organisés selon la structure OneDrive
+
+## Dépannage
+
+### Token expiré
+```
+Error: Request failed with status code 401
+```
+→ Renouveler le token avec `az account get-access-token --resource https://graph.microsoft.com/`
+
+### Permissions insuffisantes
+```
+Error: Insufficient privileges to complete the operation
+```
+→ Vérifier les permissions Microsoft Graph de l'application Azure
+
+### Dossier introuvable
+```
+Error: The resource could not be found
+```
+→ Vérifier le `rootPath` ou utiliser `/` pour la racine
+
+### Limite de débit
+```
+Error: Too many requests
+```
+→ Le plugin a déjà des délais intégrés, augmenter si nécessaire
+
+## Performance
+
+Avec limit=10 :
+- Durée : ~2-3 secondes
+- Documents : 10 maximum  
+- API calls : 1-2 requêtes
+
+Avec limit=100 :
+- Durée : ~10-15 secondes
+- Documents : 100 maximum
+- API calls : Pagination automatique
+
+## Paramètres utiles
+
+- **limit** : Nombre max de documents (utile pour tests)
+- **rootPath** : Dossier spécifique à scanner (ex: `/Documents`)
+- **accessToken** : Token direct (évite l'OAuth flow complexe)
+- **clientId/clientSecret** : Pour l'authentification OAuth2
+
+## Types de documents OneDrive observés
+
+- **Office** : .docx, .xlsx, .pptx (exportés automatiquement)
+- **PDF** : Documents PDF
+- **Images** : .jpg, .png, .gif
+- **Archives** : .zip, .rar
+- **Texte** : .txt, .csv, .json
+
+## Différences avec SharePoint
+
+Le plugin fonctionne pour :
+- **OneDrive personnel** : `/me/drive/`
+- **SharePoint** : Modifier l'URL de base si nécessaire
+- **OneDrive Entreprise** : Même API que personnel
+
+## Commandes utiles
+
+```bash
+# Test rapide de connexion
+node test-onedrive.js
+
+# Scan avec limite
+npm run dev -- plugin-scan test-onedrive --limit 5
+
+# Import vers dossier local
+npm run dev -- plugin-import test-onedrive ./downloads --limit 3
+```

package/plugins/onedrive/index.ts

@@ -0,0 +1,447 @@
+import axios, { AxiosInstance } from 'axios';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface OneDriveConfig extends PluginConfig {
+  clientId: string;
+  clientSecret: string;
+  tenantId: string;
+  accessToken?: string;
+  refreshToken?: string;
+  rootPath?: string; // e.g., '/Documents' or '/Shared Documents'
+  limit?: number;
+}
+
+interface OneDriveItem {
+  id: string;
+  name: string;
+  size?: number;
+  file?: {
+    mimeType: string;
+  };
+  folder?: {
+    childCount: number;
+  };
+  lastModifiedDateTime: string;
+  parentReference?: {
+    path: string;
+  };
+  '@microsoft.graph.downloadUrl'?: string;
+}
+
+export default class OneDrivePlugin implements DocumentSourcePlugin {
+  readonly name = 'onedrive';
+  readonly version = '1.0.0';
+  readonly description = 'Microsoft OneDrive/SharePoint document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: OneDriveConfig;
+  private apiClient?: AxiosInstance;
+  private baseUrl = 'https://graph.microsoft.com/v1.0';
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const client = this.createApiClient(config as OneDriveConfig);
+      // Try simpler endpoint first
+      const response = await client.get('/me');
+      return response.status === 200;
+    } catch (error: any) {
+      console.error(`OneDrive connection test failed: ${error.message}`);
+      if (error.response?.data) {
+        console.error('API Response:', error.response.data);
+      }
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as OneDriveConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const limit = (this.config as any).limit || (options as any)?.limit;
+      console.log(`🔍 Scanning OneDrive documents${limit ? ` (limit: ${limit})` : ''}...`);
+      
+      const rootPath = this.config.rootPath || '/';
+      const items = await this.scanFolder(rootPath, options);
+
+      let processedCount = 0;
+      for (const item of items) {
+        if (item.file) { // Only process files, not folders
+          const source: DocumentSource = {
+            id: item.id,
+            name: item.name,
+            path: this.getRelativePath(item),
+            size: item.size || 0,
+            mimeType: item.file.mimeType,
+            lastModified: new Date(item.lastModifiedDateTime),
+            metadata: {
+              oneDriveId: item.id,
+              downloadUrl: item['@microsoft.graph.downloadUrl'],
+              parentPath: item.parentReference?.path
+            }
+          };
+
+          // Apply filters
+          if (this.shouldIncludeSource(source, options)) {
+            sources.push(source);
+            totalSize += source.size;
+            processedCount++;
+
+            // Check limit
+            if (limit && processedCount >= limit) {
+              console.log(`📏 Reached limit of ${limit} files`);
+              break;
+            }
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`OneDrive scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  private async scanFolder(folderPath: string, options?: PluginImportOptions): Promise<OneDriveItem[]> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const items: OneDriveItem[] = [];
+    let nextLink: string | undefined;
+
+    do {
+      try {
+        // Handle root path correctly
+        let url: string;
+        if (nextLink) {
+          url = nextLink;
+        } else if (!folderPath || folderPath === '/') {
+          url = '/me/drive/root/children';
+        } else {
+          url = `/me/drive/root:${folderPath}:/children`;
+        }
+        
+        const response = await this.apiClient.get(url);
+        
+        const folderItems: OneDriveItem[] = response.data.value;
+        items.push(...folderItems);
+
+        // Recursively scan subfolders
+        for (const item of folderItems) {
+          if (item.folder && item.folder.childCount > 0) {
+            const subPath = `${folderPath}/${item.name}`;
+            const subItems = await this.scanFolder(subPath, options);
+            items.push(...subItems);
+          }
+        }
+
+        nextLink = response.data['@odata.nextLink'];
+      } catch (error: any) {
+        console.warn(`Warning: Failed to scan folder ${folderPath}: ${error.message}`);
+        break;
+      }
+    } while (nextLink);
+
+    return items;
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as OneDriveConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 5; // OneDrive has rate limits
+
+    // Process in batches to respect API limits
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(100);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.apiClient) throw new Error('API client not initialized');
+
+      // Get download URL if not in metadata
+      let downloadUrl = source.metadata?.downloadUrl;
+      
+      if (!downloadUrl) {
+        const response = await this.apiClient.get(`/me/drive/items/${source.id}`);
+        downloadUrl = response.data['@microsoft.graph.downloadUrl'];
+      }
+
+      if (!downloadUrl) {
+        throw new Error('No download URL available');
+      }
+
+      // Download file
+      const response = await axios.get(downloadUrl, {
+        responseType: 'stream'
+      });
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+      
+      const writer = fs.createWriteStream(targetPath);
+      response.data.pipe(writer);
+
+      return new Promise((resolve) => {
+        writer.on('finish', () => {
+          resolve({
+            success: true,
+            source,
+            localPath: targetPath,
+            bytesTransferred: source.size
+          });
+        });
+
+        writer.on('error', (error) => {
+          resolve({
+            success: false,
+            source,
+            error: error.message
+          });
+        });
+      });
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as OneDriveConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ExportResult[] = [];
+
+    for (const source of localSources) {
+      try {
+        const targetPath = options?.preserveStructure 
+          ? `${options.targetPath}/${source.path}`
+          : `${options?.targetPath || ''}/${source.name}`;
+
+        // Create folder structure if needed
+        const folderPath = path.dirname(targetPath);
+        if (folderPath && folderPath !== '.') {
+          await this.createFolderStructure(folderPath);
+        }
+
+        // Upload file
+        const fileContent = await fs.readFile(source.id);
+        const uploadUrl = `/me/drive/root:${targetPath}:/content`;
+        
+        await this.apiClient!.put(uploadUrl, fileContent, {
+          headers: {
+            'Content-Type': source.mimeType
+          }
+        });
+
+        results.push({
+          success: true,
+          targetPath,
+          source,
+          bytesTransferred: source.size
+        });
+      } catch (error: any) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: error.message
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async createFolderStructure(folderPath: string): Promise<void> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const parts = folderPath.split('/').filter(part => part.length > 0);
+    let currentPath = '';
+
+    for (const part of parts) {
+      currentPath += `/${part}`;
+      
+      try {
+        // Try to get the folder
+        await this.apiClient.get(`/me/drive/root:${currentPath}`);
+      } catch (error: any) {
+        if (error.response?.status === 404) {
+          // Folder doesn't exist, create it
+          const parentPath = path.dirname(currentPath) || '/';
+          await this.apiClient.post(`/me/drive/root:${parentPath}:/children`, {
+            name: part,
+            folder: {}
+          });
+        }
+      }
+    }
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        clientId: {
+          type: 'string',
+          description: 'Azure App Client ID (can be dummy when using accessToken)',
+          required: false
+        },
+        clientSecret: {
+          type: 'string',
+          description: 'Azure App Client Secret (can be dummy when using accessToken)',
+          required: false
+        },
+        tenantId: {
+          type: 'string',
+          description: 'Azure Tenant ID (can be dummy when using accessToken)',
+          required: false
+        },
+        accessToken: {
+          type: 'string',
+          description: 'OAuth2 Access Token (required - get with az account get-access-token)',
+          required: true
+        },
+        refreshToken: {
+          type: 'string',
+          description: 'OAuth2 Refresh Token (will be obtained automatically)',
+          required: false
+        },
+        rootPath: {
+          type: 'string',
+          description: 'Root path to scan (e.g., /Documents)',
+          default: '/'
+        },
+        limit: {
+          type: 'number',
+          description: 'Maximum number of documents to scan (useful for testing)',
+          required: false
+        }
+      },
+      required: ['accessToken']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as OneDriveConfig;
+    
+    // If no access token, we need to authenticate
+    if (!this.config.accessToken) {
+      console.log('ℹ️  OneDrive authentication required. Please obtain an access token first.');
+      console.log('💡 You can use Azure CLI: az account get-access-token --resource https://graph.microsoft.com/');
+      throw new Error('OneDrive authentication required. Please provide an access token.');
+    }
+
+    // Set default values for required fields if not provided
+    if (!this.config.clientId) this.config.clientId = 'dummy';
+    if (!this.config.clientSecret) this.config.clientSecret = 'dummy';
+    if (!this.config.tenantId) this.config.tenantId = 'dummy';
+
+    this.apiClient = this.createApiClient(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.apiClient = undefined;
+  }
+
+  private createApiClient(config: OneDriveConfig): AxiosInstance {
+    return axios.create({
+      baseURL: this.baseUrl,
+      headers: {
+        'Authorization': `Bearer ${config.accessToken}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+  }
+
+  private getRelativePath(item: OneDriveItem): string {
+    if (!item.parentReference?.path) {
+      return item.name;
+    }
+
+    // Remove the root drive path
+    const parentPath = item.parentReference.path
+      .replace(/^\/drive\/root:?/, '')
+      .replace(/^\//, '');
+    
+    return parentPath ? `${parentPath}/${item.name}` : item.name;
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/onedrive/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "onedrive",
+  "version": "1.0.0", 
+  "description": "Microsoft OneDrive document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.js",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "axios": "^1.5.0",
+    "fs-extra": "^11.1.0"
+  }
+}