@sinoia/hubdoc-tools 1.3.2 → 1.3.4

package/plugins/sharepoint/index.ts

@@ -0,0 +1,509 @@
+import axios, { AxiosInstance } from 'axios';
+import fs from 'fs-extra';
+import path from 'path';
+import {
+  DocumentSourcePlugin,
+  DocumentSource,
+  PluginConfig,
+  ScanResult,
+  PluginImportOptions,
+  PluginExportOptions,
+  ImportResult,
+  ExportResult
+} from '../../src/types/plugins';
+
+interface SharePointConfig extends PluginConfig {
+  siteUrl: string;
+  accessToken: string;
+  driveId?: string; // Specific drive ID, otherwise uses default
+  folderId?: string; // Starting folder ID, defaults to root
+  limit?: number;
+}
+
+interface SharePointDriveItem {
+  id: string;
+  name: string;
+  size?: number;
+  lastModifiedDateTime: string;
+  createdDateTime: string;
+  file?: {
+    mimeType: string;
+    hashes: {
+      quickXorHash?: string;
+      sha1Hash?: string;
+    };
+  };
+  folder?: {
+    childCount: number;
+  };
+  createdBy: {
+    user: {
+      displayName: string;
+      email?: string;
+    };
+  };
+  lastModifiedBy: {
+    user: {
+      displayName: string;
+      email?: string;
+    };
+  };
+  parentReference?: {
+    driveId: string;
+    id: string;
+    path: string;
+  };
+  webUrl: string;
+}
+
+export default class SharePointPlugin implements DocumentSourcePlugin {
+  readonly name = 'sharepoint';
+  readonly version = '1.0.0';
+  readonly description = 'SharePoint Online document source';
+  readonly supportedOperations = ['import', 'export', 'both'] as const;
+
+  private config?: SharePointConfig;
+  private apiClient?: AxiosInstance;
+  private readonly baseUrl = 'https://graph.microsoft.com/v1.0';
+
+  async testConnection(config: PluginConfig): Promise<boolean> {
+    try {
+      const client = this.createApiClient(config as SharePointConfig);
+      const siteInfo = await this.getSiteInfo(client, (config as SharePointConfig).siteUrl);
+      return !!siteInfo.id;
+    } catch (error: any) {
+      console.error(`SharePoint connection test failed: ${error.message}`);
+      return false;
+    }
+  }
+
+  async scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult> {
+    this.config = config as SharePointConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const sources: DocumentSource[] = [];
+    const errors: string[] = [];
+    let totalSize = 0;
+
+    try {
+      const limit = (this.config as any).limit || (options as any)?.limit;
+      console.log(`🔍 Scanning SharePoint site${limit ? ` (limit: ${limit})` : ''}...`);
+
+      // Get site information
+      const siteInfo = await this.getSiteInfo(this.apiClient, this.config.siteUrl);
+      
+      // Get drive ID (default drive or specified drive)
+      const driveId = this.config.driveId || await this.getDefaultDriveId(siteInfo.id);
+      
+      // Scan items starting from root or specified folder
+      const startingFolderId = this.config.folderId || 'root';
+      const items = await this.scanDriveFolder(driveId, startingFolderId, '');
+
+      let processedCount = 0;
+      for (const item of items) {
+        if (item.file) {
+          const source: DocumentSource = {
+            id: item.id,
+            name: item.name,
+            path: this.getItemPath(item),
+            size: item.size || 0,
+            mimeType: item.file.mimeType,
+            lastModified: new Date(item.lastModifiedDateTime),
+            metadata: {
+              sharepointId: item.id,
+              driveId: driveId,
+              createdAt: item.createdDateTime,
+              createdBy: item.createdBy.user.displayName,
+              modifiedBy: item.lastModifiedBy.user.displayName,
+              webUrl: item.webUrl,
+              quickXorHash: item.file.hashes?.quickXorHash,
+              sha1Hash: item.file.hashes?.sha1Hash
+            }
+          };
+
+          // Apply filters
+          if (this.shouldIncludeSource(source, options)) {
+            sources.push(source);
+            totalSize += source.size;
+            processedCount++;
+
+            // Check limit
+            if (limit && processedCount >= limit) {
+              console.log(`📏 Reached limit of ${limit} files`);
+              break;
+            }
+          }
+        }
+      }
+
+      return {
+        sources,
+        totalCount: sources.length,
+        totalSize,
+        errors
+      };
+    } catch (error: any) {
+      return {
+        sources: [],
+        totalCount: 0,
+        totalSize: 0,
+        errors: [`SharePoint scan failed: ${error.message}`]
+      };
+    }
+  }
+
+  private async scanDriveFolder(driveId: string, folderId: string, parentPath: string): Promise<SharePointDriveItem[]> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const allItems: SharePointDriveItem[] = [];
+    let nextLink: string | null = null;
+
+    try {
+      do {
+        const url = nextLink || `/drives/${driveId}/items/${folderId}/children`;
+        const response = await this.apiClient.get(url, {
+          params: nextLink ? {} : {
+            $expand: 'thumbnails',
+            $select: 'id,name,size,lastModifiedDateTime,createdDateTime,file,folder,createdBy,lastModifiedBy,parentReference,webUrl'
+          }
+        });
+
+        const items: SharePointDriveItem[] = response.data.value || [];
+
+        for (const item of items) {
+          const itemPath = parentPath ? `${parentPath}/${item.name}` : item.name;
+          
+          if (item.file) {
+            // Add path information to the item
+            (item as any).fullPath = itemPath;
+            allItems.push(item);
+          } else if (item.folder) {
+            // Recursively scan subfolders
+            const subItems = await this.scanDriveFolder(driveId, item.id, itemPath);
+            allItems.push(...subItems);
+          }
+        }
+
+        nextLink = response.data['@odata.nextLink'] || null;
+      } while (nextLink);
+
+      return allItems;
+    } catch (error: any) {
+      if (error.response?.status === 404) {
+        console.warn(`Warning: Folder not found: ${folderId}`);
+        return [];
+      }
+      throw error;
+    }
+  }
+
+  async import(
+    config: PluginConfig,
+    sources: DocumentSource[],
+    targetDir: string,
+    options?: PluginImportOptions
+  ): Promise<ImportResult[]> {
+    this.config = config as SharePointConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ImportResult[] = [];
+    const batchSize = options?.batchSize || 5;
+
+    // Process in batches to respect API limits
+    for (let i = 0; i < sources.length; i += batchSize) {
+      const batch = sources.slice(i, i + batchSize);
+      
+      for (const source of batch) {
+        const result = await this.importSingle(source, targetDir);
+        results.push(result);
+        
+        // Small delay to respect rate limits
+        await this.sleep(200);
+      }
+    }
+
+    return results;
+  }
+
+  private async importSingle(source: DocumentSource, targetDir: string): Promise<ImportResult> {
+    try {
+      if (!this.apiClient) throw new Error('API client not initialized');
+
+      const targetPath = path.join(targetDir, source.path);
+      const targetDirectory = path.dirname(targetPath);
+      
+      await fs.ensureDir(targetDirectory);
+
+      // Get the drive ID from metadata
+      const driveId = source.metadata?.driveId;
+      if (!driveId) {
+        throw new Error('Drive ID not found in source metadata');
+      }
+
+      // Download file content from SharePoint
+      const response = await this.apiClient.get(`/drives/${driveId}/items/${source.id}/content`, {
+        responseType: 'stream'
+      });
+      
+      const writer = fs.createWriteStream(targetPath);
+      response.data.pipe(writer);
+
+      return new Promise((resolve) => {
+        writer.on('finish', () => {
+          resolve({
+            success: true,
+            source,
+            localPath: targetPath,
+            bytesTransferred: source.size
+          });
+        });
+
+        writer.on('error', (error) => {
+          resolve({
+            success: false,
+            source,
+            error: error.message
+          });
+        });
+      });
+    } catch (error: any) {
+      return {
+        success: false,
+        source,
+        error: error.message
+      };
+    }
+  }
+
+  async export?(
+    config: PluginConfig,
+    localSources: DocumentSource[],
+    options?: PluginExportOptions
+  ): Promise<ExportResult[]> {
+    this.config = config as SharePointConfig;
+    this.apiClient = this.createApiClient(this.config);
+
+    const results: ExportResult[] = [];
+
+    try {
+      // Get site and drive information
+      const siteInfo = await this.getSiteInfo(this.apiClient, this.config.siteUrl);
+      const driveId = this.config.driveId || await this.getDefaultDriveId(siteInfo.id);
+      const rootFolderId = this.config.folderId || 'root';
+
+      for (const source of localSources) {
+        try {
+          // Determine target folder
+          let targetFolderId = rootFolderId;
+          
+          if (options?.preserveStructure && source.path.includes('/')) {
+            const folderPath = path.dirname(source.path);
+            targetFolderId = await this.createFolderStructure(driveId, folderPath, rootFolderId);
+          }
+
+          // Read local file
+          const fileContent = await fs.readFile(source.id);
+          // Normalize filename to NFC to handle accented characters consistently across platforms
+          const fileName = (options?.preserveStructure ? path.basename(source.path) : source.name).normalize('NFC');
+
+          // Upload file to SharePoint using simple upload (for files < 4MB)
+          // For larger files, we should use resumable upload sessions
+          const uploadUrl = `/drives/${driveId}/items/${targetFolderId}:/${fileName}:/content`;
+          
+          await this.apiClient.put(uploadUrl, fileContent, {
+            headers: {
+              'Content-Type': source.mimeType || 'application/octet-stream'
+            }
+          });
+
+          const targetPath = options?.preserveStructure ? source.path : source.name;
+
+          results.push({
+            success: true,
+            targetPath,
+            source,
+            bytesTransferred: source.size
+          });
+        } catch (error: any) {
+          results.push({
+            success: false,
+            targetPath: options?.targetPath || '',
+            source,
+            error: error.message
+          });
+        }
+      }
+    } catch (error: any) {
+      // If we can't get site/drive info, fail all exports
+      for (const source of localSources) {
+        results.push({
+          success: false,
+          targetPath: options?.targetPath || '',
+          source,
+          error: `Failed to initialize SharePoint connection: ${error.message}`
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async getSiteInfo(client: AxiosInstance, siteUrl: string): Promise<any> {
+    // Extract hostname and site path from URL
+    const url = new URL(siteUrl);
+    const hostname = url.hostname;
+    const sitePath = url.pathname;
+
+    const response = await client.get(`/sites/${hostname}:${sitePath}`);
+    return response.data;
+  }
+
+  private async getDefaultDriveId(siteId: string): Promise<string> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const response = await this.apiClient.get(`/sites/${siteId}/drive`);
+    return response.data.id;
+  }
+
+  private async createFolderStructure(driveId: string, folderPath: string, parentId: string): Promise<string> {
+    if (!this.apiClient) throw new Error('API client not initialized');
+
+    const parts = folderPath.split('/').filter(part => part.length > 0);
+    let currentParentId = parentId;
+
+    for (const folderName of parts) {
+      try {
+        // Check if folder already exists
+        const searchUrl = `/drives/${driveId}/items/${currentParentId}/children`;
+        const response = await this.apiClient.get(searchUrl, {
+          params: {
+            $filter: `name eq '${folderName}' and folder ne null`
+          }
+        });
+
+        const existingFolder = response.data.value[0];
+        
+        if (existingFolder) {
+          currentParentId = existingFolder.id;
+        } else {
+          // Create new folder
+          const createResponse = await this.apiClient.post(`/drives/${driveId}/items/${currentParentId}/children`, {
+            name: folderName,
+            folder: {},
+            '@microsoft.graph.conflictBehavior': 'rename'
+          });
+          currentParentId = createResponse.data.id;
+        }
+      } catch (error: any) {
+        throw new Error(`Failed to create folder structure: ${error.message}`);
+      }
+    }
+
+    return currentParentId;
+  }
+
+  getConfigSchema(): Record<string, any> {
+    return {
+      type: 'object',
+      properties: {
+        siteUrl: {
+          type: 'string',
+          description: 'SharePoint site URL (e.g., https://contoso.sharepoint.com/sites/mysite)',
+          required: true
+        },
+        accessToken: {
+          type: 'string',
+          description: 'Microsoft Graph API access token with SharePoint permissions',
+          required: true
+        },
+        driveId: {
+          type: 'string',
+          description: 'Specific drive ID to scan (optional, uses default site drive)',
+          required: false
+        },
+        folderId: {
+          type: 'string',
+          description: 'Starting folder ID (optional, defaults to drive root)',
+          required: false
+        },
+        limit: {
+          type: 'number',
+          description: 'Maximum number of documents to scan (useful for testing)',
+          required: false
+        }
+      },
+      required: ['siteUrl', 'accessToken']
+    };
+  }
+
+  async initialize(config: PluginConfig): Promise<void> {
+    this.config = config as SharePointConfig;
+    
+    if (!this.config.siteUrl || !this.config.accessToken) {
+      throw new Error('SharePoint site URL and access token are required');
+    }
+
+    this.apiClient = this.createApiClient(this.config);
+  }
+
+  async destroy(): Promise<void> {
+    this.config = undefined;
+    this.apiClient = undefined;
+  }
+
+  private createApiClient(config: SharePointConfig): AxiosInstance {
+    return axios.create({
+      baseURL: this.baseUrl,
+      headers: {
+        'Authorization': `Bearer ${config.accessToken}`,
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+  }
+
+  private getItemPath(item: SharePointDriveItem): string {
+    // Use the full path if available from scanning
+    if ((item as any).fullPath) {
+      return (item as any).fullPath;
+    }
+
+    // Construct path from parent reference
+    if (item.parentReference?.path) {
+      const pathParts = item.parentReference.path
+        .split('/root:/')
+        .pop()
+        ?.split('/') || [];
+      
+      pathParts.push(item.name);
+      return pathParts.filter(part => part.length > 0).join('/');
+    }
+
+    return item.name;
+  }
+
+  private shouldIncludeSource(source: DocumentSource, options?: PluginImportOptions): boolean {
+    // Apply size filter
+    if (options?.filters?.maxSize && source.size > options.filters.maxSize) {
+      return false;
+    }
+
+    // Apply date range filter
+    if (options?.filters?.dateRange) {
+      const { from, to } = options.filters.dateRange;
+      if (from && source.lastModified < from) return false;
+      if (to && source.lastModified > to) return false;
+    }
+
+    // Apply MIME type filter
+    if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(source.mimeType)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

package/plugins/sharepoint/plugin.json

@@ -0,0 +1,12 @@
+{
+  "name": "sharepoint",
+  "version": "1.0.0",
+  "description": "SharePoint document source plugin",
+  "author": "HubDoc Tools",
+  "main": "index.ts",
+  "hubdocToolVersion": "^1.0.0",
+  "dependencies": {
+    "@pnp/sp": "^3.0.0",
+    "fs-extra": "^11.1.0"
+  }
+}