@sinoia/hubdoc-tools 1.3.5 → 1.3.7

package/plugins/core/TESTING.md

@@ -1,155 +0,0 @@
-# Guide de test du plugin Core
-
-Ce guide vous permet de tester le plugin Core avec l'instance https://rec.plugandwork.fr.
-
-## Prérequis
-
-1. Token d'authentification Core valide
-2. HubDoc Tools configuré
-
-## Configuration de test
-
-### 1. Créer un fichier de configuration
-
-```json
-{
-  "baseUrl": "https://rec.plugandwork.fr",
-  "bearerToken": "YOUR_BEARER_TOKEN_HERE",
-  "limit": 10
-}
-```
-
-### 2. Tester la connexion
-
-```bash
-# Lister les plugins disponibles
-npm run dev -- plugins
-
-# Vérifier que Core est bien listé
-```
-
-### 3. Scanner les documents (test rapide)
-
-Créer un script de test :
-
-```javascript
-// test-core.js
-const fs = require('fs-extra');
-const { register } = require('ts-node');
-register({ transpileOnly: true, compilerOptions: { esModuleInterop: true } });
-
-const CorePlugin = require('./plugins/core/index.ts').default;
-
-async function test() {
-  const config = {
-    baseUrl: "https://rec.plugandwork.fr",
-    bearerToken: "YOUR_TOKEN",
-    limit: 5
-  };
-  
-  const plugin = new CorePlugin();
-  
-  console.log('Testing connection...');
-  const connected = await plugin.testConnection(config);
-  console.log('Connected:', connected);
-  
-  if (connected) {
-    console.log('Scanning documents...');
-    const result = await plugin.scan(config);
-    console.log(`Found ${result.totalCount} documents`);
-    result.sources.slice(0, 3).forEach(doc => {
-      console.log(`- ${doc.name} (${doc.path})`);
-    });
-  }
-}
-
-test().catch(console.error);
-```
-
-Puis exécuter :
-```bash
-node test-core.js
-```
-
-## Résultats attendus
-
-### Scan réussi
-- ✅ Connection: SUCCESS
-- ✅ Documents trouvés avec métadonnées Core
-- ✅ Limitation respectée (paramètre `limit`)
-- ✅ Types de documents variés (file, contact, note, email, etc.)
-
-### Structure des documents
-- **Chemin** : `{type}/{subtype}/{title}.txt`
-- **Métadonnées** : Core ID, type, tags, etc.
-- **Types MIME** : text/plain par défaut
-
-## Test d'intégration complète
-
-### 1. Configuration d'une connexion
-```bash
-echo '{"baseUrl":"https://rec.plugandwork.fr","bearerToken":"YOUR_TOKEN","limit":10}' > core-config.json
-npm run dev -- connect core --config core-config.json --name test-core
-```
-
-### 2. Scan des documents
-```bash
-npm run dev -- plugin-scan test-core --output core-mapping.csv
-```
-
-### 3. Vérification du mapping
-```bash
-head -10 core-mapping.csv
-```
-
-Le fichier doit contenir :
-- Colonnes : File Path, Target Folder, Workspace, Metadata (JSON), Permissions
-- Métadonnées JSON avec Core ID, type, tags
-- Chemins organisés par type Core
-
-## Dépannage
-
-### Token expiré
-```
-Error: Request failed with status code 401
-```
-→ Vérifier/renouveler le token Bearer
-
-### URL incorrecte
-```
-Error: ENOTFOUND rec.plugandwork.fr
-```
-→ Vérifier l'URL de base
-
-### Limite dépassée
-```
-Error: Too many requests
-```
-→ Augmenter les délais ou réduire la limit
-
-## Performance
-
-Avec limit=10 :
-- Durée : ~1-2 secondes
-- Documents : 10 maximum
-- Pagination : 1 page
-
-Avec limit=100 :
-- Durée : ~5-10 secondes  
-- Documents : 100 maximum
-- Pagination : Multiple pages
-
-## Paramètres utiles
-
-- **limit** : Nombre max de documents (utile pour tests)
-- **tempDir** : Répertoire temporaire pour les buffers
-- **bearerToken** : Token d'auth direct (évite username/password)
-
-## Types de documents Core observés
-
-- **file** : Fichiers généraux
-- **contact** : Contacts/fiches client  
-- **note** : Notes internes
-- **email** : Emails
-- **task** : Tâches
-- **doc** : Documents structurés

package/plugins/core/index.ts

@@ -1,510 +0,0 @@
-import axios, { AxiosInstance } from 'axios';
-import fs from 'fs-extra';
-import path from 'path';
-import { ConcurrentProcessor } from '../../src/utils/concurrent-processor';
-import {
-  DocumentSourcePlugin,
-  DocumentSource,
-  PluginConfig,
-  ScanResult,
-  PluginImportOptions,
-  PluginExportOptions,
-  ImportResult,
-  ExportResult
-} from '../../src/types/plugins';
-
-interface CoreConfig extends PluginConfig {
-  baseUrl: string;
-  username: string;
-  password: string;
-  bearerToken?: string;
-  tempDir?: string; // Directory for local buffer files
-}
-
-interface CoreDocResponse {
-  data: CoreDoc[];
-  meta?: {
-    pagination?: {
-      page?: number;
-      pages?: number;
-      count?: number;
-      total?: number;
-    };
-  };
-}
-
-interface CoreDoc {
-  id: string;
-  type: string;
-  attributes: {
-    title: string;
-    body?: string;
-    type?: string;
-    subtype?: string;
-    path?: string;
-    created_at?: string;
-    updated_at?: string;
-    tags?: string[];
-    percent?: number;
-    note?: number;
-    has_attachments?: boolean;
-    signed?: boolean;
-  };
-  relationships?: any;
-}
-
-export default class CorePlugin implements DocumentSourcePlugin {
-  readonly name = 'core';
-  readonly version = '1.0.0';
-  readonly description = 'Core document management system source';
-  readonly supportedOperations = ['import', 'both'] as const;
-
-  private config?: CoreConfig;
-  private apiClient?: AxiosInstance;
-
-  async testConnection(config: PluginConfig): Promise<boolean> {
-    try {
-      const coreConfig = config as CoreConfig;
-      const client = await this.createApiClient(coreConfig);
-      
-      // Test connection by trying to fetch docs (first page only)
-      const response = await client.get('/api/d2/docs', {
-        params: {
-          '[page][number]': 1,
-          '[page][size]': 1
-        }
-      });
-      
-      return response.status === 200;
-    } catch (error: any) {
-      console.error(`Core connection test failed: ${error.message}`);
-      return false;
-    }
-  }
-
-  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
-    this.config = config as CoreConfig;
-    this.apiClient = await this.createApiClient(this.config);
-
-    const sources: DocumentSource[] = [];
-    const errors: string[] = [];
-    let totalSize = 0;
-
-    try {
-      const limit = (this.config as any).limit || (options as any)?.limit;
-      console.log(`🔍 Scanning Core documents${limit ? ` (limit: ${limit})` : ''}...`);
-      
-      // Scan all pages of documents
-      let currentPage = 1;
-      let hasMorePages = true;
-      const pageSize = limit && limit < 100 ? limit : 100; // Use limit if smaller than default batch
-
-      while (hasMorePages && (!limit || sources.length < limit)) {
-        try {
-          const response = await this.apiClient.get<CoreDocResponse>('/api/d2/docs', {
-            params: {
-              '[page][number]': currentPage,
-              '[page][size]': pageSize,
-              // Add filters if provided in options
-              ...(options?.filters?.path && { path: options.filters.path }),
-              ...(options?.filters?.mimeTypes && { type: options.filters.mimeTypes.join(',') })
-            }
-          });
-
-          const docs = response.data.data || [];
-          console.log(`📄 Processing page ${currentPage}: ${docs.length} documents`);
-
-          for (const doc of docs) {
-            if (doc.attributes && doc.attributes.title) {
-              const source: DocumentSource = {
-                id: doc.id,
-                name: doc.attributes.title,
-                path: this.buildDocumentPath(doc),
-                size: this.estimateDocumentSize(doc),
-                mimeType: this.determineMimeType(doc),
-                lastModified: new Date(doc.attributes.updated_at || doc.attributes.created_at || Date.now()),
-                metadata: {
-                  coreId: doc.id,
-                  coreType: doc.attributes.type,
-                  coreSubtype: doc.attributes.subtype,
-                  corePath: doc.attributes.path,
-                  tags: doc.attributes.tags,
-                  hasAttachments: doc.attributes.has_attachments,
-                  signed: doc.attributes.signed,
-                  percent: doc.attributes.percent,
-                  note: doc.attributes.note,
-                  body: doc.attributes.body?.substring(0, 500) + '...' // Truncate body for metadata
-                }
-              };
-
-              // Apply filters
-              if (this.shouldIncludeSource(source, options)) {
-                sources.push(source);
-                totalSize += source.size;
-                
-                // Stop if we've reached the limit
-                if (limit && sources.length >= limit) {
-                  console.log(`📏 Reached limit of ${limit} documents`);
-                  break;
-                }
-              }
-            }
-          }
-
-          // Check for more pages
-          const pagination = response.data.meta?.pagination;
-          if (pagination && pagination.page && pagination.pages) {
-            hasMorePages = pagination.page < pagination.pages;
-            currentPage++;
-          } else {
-            // Fallback: if no docs returned, assume we've reached the end
-            hasMorePages = docs.length === pageSize;
-            if (hasMorePages) currentPage++;
-          }
-
-          // Add small delay to respect API limits
-          await this.sleep(100);
-
-        } catch (pageError: any) {
-          console.warn(`⚠️  Warning: Failed to fetch page ${currentPage}: ${pageError.message}`);
-          errors.push(`Failed to fetch page ${currentPage}: ${pageError.message}`);
-          break;
-        }
-      }
-
-      console.log(`✅ Scan completed: ${sources.length} documents found`);
-
-      return {
-        sources,
-        totalCount: sources.length,
-        totalSize,
-        errors
-      };
-    } catch (error: any) {
-      return {
-        sources: [],
-        totalCount: 0,
-        totalSize: 0,
-        errors: [`Core scan failed: ${error.message}`]
-      };
-    }
-  }
-
-  async import(
-    config: PluginConfig,
-    sources: DocumentSource[],
-    targetDir: string,
-    options?: PluginImportOptions
-  ): Promise<ImportResult[]> {
-    this.config = config as CoreConfig;
-    this.apiClient = await this.createApiClient(this.config);
-
-    const concurrency = options?.concurrent || 1;
-    const tempDir = this.config.tempDir || path.join(targetDir, '.temp');
-
-    // Ensure temp directory exists
-    await fs.ensureDir(tempDir);
-
-    console.log(`📥 Starting concurrent import of ${sources.length} documents from Core (${concurrency} jobs)`);
-
-    // Use concurrent processor
-    const processingResult = await ConcurrentProcessor.processWithProgress(
-      sources,
-      async (source: DocumentSource, index: number) => {
-        const result = await this.importSingle(source, targetDir, tempDir);
-        
-        // Rate limiting delay (only for concurrent > 1)
-        if (concurrency > 1) {
-          await this.sleep(100); // Reduced delay for concurrent operations
-        }
-        
-        return result;
-      },
-      {
-        concurrency,
-        operation: 'Import from Core',
-        itemName: 'documents'
-      }
-    );
-
-    // Cleanup temp directory
-    try {
-      await fs.remove(tempDir);
-      console.log('🧹 Temporary files cleaned up');
-    } catch (cleanupError) {
-      console.warn('⚠️  Warning: Failed to cleanup temporary directory');
-    }
-
-    // Filter valid results and handle errors
-    const validResults = processingResult.results.filter(r => r !== undefined);
-    const errorResults: ImportResult[] = processingResult.errors.map(({ item, error }) => ({
-      success: false,
-      source: item,
-      error: error.message
-    }));
-
-    return [...validResults, ...errorResults];
-  }
-
-  private async importSingle(source: DocumentSource, targetDir: string, tempDir: string): Promise<ImportResult> {
-    try {
-      if (!this.apiClient) throw new Error('API client not initialized');
-
-      console.log(`📄 Importing: ${source.name}`);
-
-      // Step 1: Fetch document details and content from Core
-      const docResponse = await this.apiClient.get(`/api/d2/docs/${source.id}`);
-      const docData = docResponse.data.data;
-
-      if (!docData) {
-        throw new Error('Document not found in Core');
-      }
-
-      // Step 2: Create local buffer file
-      const tempFileName = `${source.id}_${source.name.replace(/[^a-zA-Z0-9.-]/g, '_')}`;
-      const tempFilePath = path.join(tempDir, tempFileName);
-
-      // Step 3: Generate document content (Core stores content in body field)
-      let content = '';
-      
-      if (docData.attributes.body) {
-        content = docData.attributes.body;
-      } else {
-        // Fallback: create a minimal document with metadata
-        content = `Document: ${source.name}\n\n`;
-        content += `Type: ${source.metadata?.coreType || 'unknown'}\n`;
-        content += `Subtype: ${source.metadata?.coreSubtype || 'unknown'}\n`;
-        content += `Created: ${source.lastModified.toISOString()}\n`;
-        if (source.metadata?.tags && Array.isArray(source.metadata.tags)) {
-          content += `Tags: ${source.metadata.tags.join(', ')}\n`;
-        }
-        content += `\nImported from Core (ID: ${source.id})`;
-      }
-
-      // Step 4: Write content to temporary file
-      await fs.writeFile(tempFilePath, content, 'utf8');
-
-      // Step 5: Move to target directory with proper structure
-      const targetPath = path.join(targetDir, source.path);
-      const targetDirectory = path.dirname(targetPath);
-      
-      await fs.ensureDir(targetDirectory);
-      await fs.move(tempFilePath, targetPath, { overwrite: true });
-
-      const stats = await fs.stat(targetPath);
-
-      console.log(`✅ Imported: ${source.name} (${stats.size} bytes)`);
-
-      return {
-        success: true,
-        source,
-        localPath: targetPath,
-        bytesTransferred: stats.size
-      };
-    } catch (error: any) {
-      console.error(`❌ Failed to import ${source.name}: ${error.message}`);
-      return {
-        success: false,
-        source,
-        error: error.message
-      };
-    }
-  }
-
-  // Export is not supported for Core (documents are managed within Core)
-  async export?(
-    config: PluginConfig,
-    localSources: DocumentSource[],
-    options?: PluginExportOptions
-  ): Promise<ExportResult[]> {
-    throw new Error('Export to Core is not supported. Core documents should be managed within the Core system.');
-  }
-
-  getConfigSchema(): Record<string, any> {
-    return {
-      type: 'object',
-      properties: {
-        baseUrl: {
-          type: 'string',
-          description: 'Core API base URL (e.g., https://your-core-instance.com)',
-          required: true
-        },
-        username: {
-          type: 'string',
-          description: 'Core username for authentication',
-          required: true
-        },
-        password: {
-          type: 'string',
-          description: 'Core password for authentication',
-          required: true
-        },
-        bearerToken: {
-          type: 'string',
-          description: 'Pre-obtained bearer token (optional, will be fetched if not provided)',
-          required: false
-        },
-        tempDir: {
-          type: 'string',
-          description: 'Temporary directory for local buffer files (optional)',
-          required: false
-        },
-        limit: {
-          type: 'integer',
-          description: 'Maximum number of documents to scan (useful for testing)',
-          required: false
-        }
-      },
-      required: ['baseUrl', 'username', 'password']
-    };
-  }
-
-  async initialize(config: PluginConfig): Promise<void> {
-    this.config = config as CoreConfig;
-    
-    if (!this.config.baseUrl) {
-      throw new Error('Core plugin requires baseUrl');
-    }
-    
-    if (!this.config.bearerToken && (!this.config.username || !this.config.password)) {
-      throw new Error('Core plugin requires either bearerToken or username/password');
-    }
-
-    // Test authentication
-    try {
-      this.apiClient = await this.createApiClient(this.config);
-      console.log('✅ Core plugin initialized successfully');
-    } catch (error: any) {
-      throw new Error(`Failed to initialize Core plugin: ${error.message}`);
-    }
-  }
-
-  async destroy(): Promise<void> {
-    this.config = undefined;
-    this.apiClient = undefined;
-  }
-
-  private async createApiClient(config: CoreConfig): Promise<AxiosInstance> {
-    let bearerToken = config.bearerToken;
-
-    // If no bearer token provided, authenticate to get one
-    if (!bearerToken && config.username && config.password) {
-      console.log('🔐 No bearer token provided, authenticating...');
-      bearerToken = await this.authenticate(config);
-    } else if (!bearerToken) {
-      throw new Error('Either bearerToken or username/password must be provided');
-    } else {
-      console.log('🔑 Using provided bearer token');
-    }
-
-    return axios.create({
-      baseURL: config.baseUrl,
-      headers: {
-        'Authorization': `Bearer ${bearerToken}`,
-        'Content-Type': 'application/json'
-      },
-      timeout: 30000
-    });
-  }
-
-  private async authenticate(config: CoreConfig): Promise<string> {
-    try {
-      console.log('🔐 Authenticating with Core...');
-      
-      const authClient = axios.create({
-        baseURL: config.baseUrl,
-        timeout: 30000
-      });
-
-      // Use the token endpoint from the swagger
-      const response = await authClient.post('/api/token', null, {
-        params: {
-          'auth[username]': config.username,
-          'auth[password]': config.password,
-          'grant_type': 'password'
-        }
-      });
-
-      if (response.data && response.data.access_token) {
-        console.log('✅ Core authentication successful');
-        return response.data.access_token;
-      } else {
-        throw new Error('No access token received from Core API');
-      }
-    } catch (error: any) {
-      throw new Error(`Core authentication failed: ${error.response?.data?.message || error.message}`);
-    }
-  }
-
-  private buildDocumentPath(doc: CoreDoc): string {
-    // Build a meaningful path structure
-    const type = doc.attributes.type || 'unknown';
-    const subtype = doc.attributes.subtype || 'general';
-    const title = doc.attributes.title || doc.id;
-    
-    // Clean title for filesystem
-    const cleanTitle = title.replace(/[^a-zA-Z0-9.-]/g, '_');
-    
-    return `${type}/${subtype}/${cleanTitle}.txt`;
-  }
-
-  private estimateDocumentSize(doc: CoreDoc): number {
-    // Estimate size based on content
-    let size = 0;
-    
-    if (doc.attributes.body) {
-      size += doc.attributes.body.length;
-    }
-    
-    if (doc.attributes.title) {
-      size += doc.attributes.title.length;
-    }
-    
-    // Add base size for metadata
-    size += 500;
-    
-    return Math.max(size, 100); // Minimum 100 bytes
-  }
-
-  private determineMimeType(doc: CoreDoc): string {
-    const type = doc.attributes.type?.toLowerCase();
-    const subtype = doc.attributes.subtype?.toLowerCase();
-    
-    // Map Core types to MIME types
-    if (type === 'document' || subtype === 'document') {
-      return 'text/plain';
-    } else if (type === 'html' || subtype === 'html') {
-      return 'text/html';
-    } else if (doc.attributes.has_attachments) {
-      return 'application/octet-stream';
-    }
-    
-    // Default to plain text
-    return 'text/plain';
-  }
-
-  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
-    // Apply size filter
-    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
-      return false;
-    }
-
-    // Apply date range filter
-    if (options?.filters?.dateRange) {
-      const { from, to } = options.filters.dateRange;
-      if (from && source.lastModified < from) return false;
-      if (to && source.lastModified > to) return false;
-    }
-
-    // Apply MIME type filter
-    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
-      return false;
-    }
-
-    return true;
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-}

package/plugins/core/plugin.json

@@ -1,26 +0,0 @@
-{
-  "name": "core",
-  "version": "1.0.0",
-  "description": "Core document management system source plugin",
-  "author": "HubDoc Tools",
-  "main": "index.ts",
-  "dependencies": {
-    "axios": "^1.5.0",
-    "fs-extra": "^11.1.0"
-  },
-  "hubdocToolVersion": "^1.0.0",
-  "capabilities": {
-    "scan": true,
-    "import": true,
-    "export": false
-  },
-  "apiEndpoints": {
-    "authentication": "/api/token",
-    "documents": "/api/d2/docs",
-    "documentDetail": "/api/d2/docs/{id}"
-  },
-  "configuration": {
-    "required": ["baseUrl", "username", "password"],
-    "optional": ["bearerToken", "tempDir"]
-  }
-}