@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/services/upload/ApiUploadService.js

@@ -0,0 +1,676 @@
+import { Blob } from 'buffer';
+import { FormData } from 'formdata-node';
+import fs from 'fs';
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
+import fetch from 'node-fetch';
+import path from 'path';
+
+import appConfig from '../../config/config.js';
+import logger from '../LoggingService.js';
+import { BaseUploadService } from './BaseUploadService.js';
+
+/**
+ * API Upload Service
+ * Handles uploads to the Arela API with automatic processing
+ * Supports multiple API targets (agencia, cliente, default)
+ */
+export class ApiUploadService extends BaseUploadService {
+  constructor() {
+    super();
+    // Flag to indicate if config is externally set (cross-tenant mode)
+    this._isExternalConfig = false;
+    // Get initial API config (can be overridden at runtime via setApiTarget)
+    this.#updateApiConfig();
+
+    // Get API connection settings from config/environment
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000;
+
+    // Initialize HTTP agents optimized for multiple API replicas
+    this.httpAgent = new Agent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    this.httpsAgent = new HttpsAgent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections, // Match your API replica count
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5, // Buffer for peak usage
+      timeout: connectionTimeout,
+      scheduling: 'fifo', // First-in-first-out scheduling
+    });
+
+    console.log(
+      `🔗 HTTP Agent configured for ${maxApiConnections} concurrent API connections`,
+    );
+  }
+
+  /**
+   * Update API configuration from appConfig
+   * Called on initialization and when API target changes
+   * Skip if externally configured (cross-tenant mode)
+   * @private
+   */
+  #updateApiConfig() {
+    // Skip update if config is externally set
+    if (this._isExternalConfig) {
+      return;
+    }
+    const apiConfig = appConfig.getApiConfig();
+    this.baseUrl = apiConfig.baseUrl;
+    this.token = apiConfig.token;
+  }
+
+  /**
+   * Set external configuration (for cross-tenant mode)
+   * Prevents automatic config refresh from overwriting these values
+   * @param {string} baseUrl - API base URL
+   * @param {string} token - API token
+   */
+  setExternalConfig(baseUrl, token) {
+    this._isExternalConfig = true;
+    this.baseUrl = baseUrl;
+    this.token = token;
+  }
+
+  /**
+   * Get current API configuration (refreshes from appConfig)
+   * This ensures we always use the latest target configuration
+   * @returns {Object} Current API config with baseUrl and token
+   */
+  getApiConfig() {
+    this.#updateApiConfig();
+    return {
+      baseUrl: this.baseUrl,
+      token: this.token,
+    };
+  }
+
+  /**
+   * Upload files to Arela API with automatic detection and organization
+   * @param {Array} files - Array of file objects
+   * @param {Object} options - Upload options
+   * @returns {Promise<Object>} API response
+   */
+  async upload(files, options) {
+    // Refresh config to get current API target
+    this.#updateApiConfig();
+    // Validate files parameter
+    if (!files || !Array.isArray(files)) {
+      logger.warn(`Invalid files parameter: ${typeof files}`);
+      throw new Error('Files must be an array');
+    }
+
+    const formData = new FormData();
+
+    // Filter out system files (macOS, Windows, etc.)
+    const systemFilePattern =
+      /^\.|__pycache__|\.pyc|\.swp|\.swo|Thumbs\.db|desktop\.ini|DS_Store|\$RECYCLE\.BIN|System Volume Information|~\$|\.tmp/i;
+    const filteredFiles = files.filter((file) => {
+      const fileName = file.name || path.basename(file.path);
+      if (systemFilePattern.test(fileName)) {
+        logger.warn(`Skipping system file from upload: ${fileName}`);
+        return false;
+      }
+      return true;
+    });
+
+    if (filteredFiles.length === 0) {
+      throw new Error('No valid files to upload after filtering system files');
+    }
+
+    // Add files to form data asynchronously
+    for (const file of filteredFiles) {
+      try {
+        // Check file size for streaming vs buffer approach
+        let size = file.size;
+        if (size === undefined || size === null) {
+          const stats = await fs.promises.stat(file.path);
+          size = stats.size;
+        }
+        const fileSizeThreshold = 10 * 1024 * 1024; // 10MB threshold
+
+        if (size > fileSizeThreshold) {
+          // Use streaming for large files
+          const fileStream = fs.createReadStream(file.path);
+          formData.append('files', fileStream, {
+            filename: file.name,
+            contentType: file.contentType,
+            knownLength: size,
+          });
+        } else {
+          // Use buffer for smaller files
+          const fileBuffer = await fs.promises.readFile(file.path);
+          const blob = new Blob([fileBuffer], { type: file.contentType });
+          formData.append('files', blob, file.name);
+        }
+      } catch (error) {
+        throw new Error(`Failed to read file ${file.path}: ${error.message}`);
+      }
+    }
+
+    // Add configuration parameters
+    if (appConfig.supabase.bucket) {
+      formData.append('bucket', appConfig.supabase.bucket);
+    }
+
+    if (options.prefix) {
+      formData.append('prefix', options.prefix);
+    }
+
+    // Handle folder structure configuration
+    const folderStructure = this.#buildFolderStructure(files, options);
+    if (folderStructure) {
+      formData.append('folderStructure', folderStructure);
+    }
+
+    // Add client path if specified
+    if (options.clientPath) {
+      formData.append('clientPath', options.clientPath);
+    }
+
+    // Add RFC for multi-database routing (required for cross-tenant uploads)
+    if (options.rfc) {
+      formData.append('rfc', options.rfc);
+    }
+
+    // Add processing options
+    formData.append('autoDetect', String(options.autoDetect ?? true));
+    formData.append('autoOrganize', String(options.autoOrganize ?? false));
+    formData.append('batchSize', String(options.batchSize || 10));
+    formData.append('clientVersion', appConfig.packageVersion);
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const response = await fetch(
+        `${this.baseUrl}/api/storage/batch-upload-and-process`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': this.token,
+          },
+          body: formData,
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `API request failed: ${response.status} ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      const result = await response.json();
+
+      // Determine if the operation was successful
+      // Success means at least one file was uploaded successfully, even if some failed
+      const hasUploads = result.uploaded && result.uploaded.length > 0;
+      const hasErrors = result.errors && result.errors.length > 0;
+      // Consider it a success if there are uploads, or if there are no errors at all
+      const success =
+        hasUploads || (!hasErrors && result.stats?.totalFiles > 0);
+
+      // Normalize response format to match DatabaseService expectations
+      return { success, data: result };
+    } catch (fetchError) {
+      // Return normalized error format
+      return { success: false, error: fetchError.message };
+    }
+  }
+
+  /**
+   * Check if API service is available
+   * @returns {Promise<boolean>} True if available
+   */
+  async isAvailable() {
+    // Refresh config to get current API target
+    this.#updateApiConfig();
+
+    if (!this.baseUrl || !this.token) {
+      return false;
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const response = await fetch(`${this.baseUrl}/api/health`, {
+        headers: {
+          'x-api-key': this.token,
+        },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
+      });
+
+      return response.ok;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  /**
+   * Batch upsert file stats to uploader table
+   * @param {Array} records - Array of record objects to upsert
+   * @returns {Promise<Object>} Upsert results { inserted, updated, total }
+   */
+  async batchUpsertStats(records) {
+    if (!records || !Array.isArray(records) || records.length === 0) {
+      return { inserted: 0, updated: 0, total: 0 };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const response = await fetch(
+        `${this.baseUrl}/api/uploader/batch-upsert`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': this.token,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(records),
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `Batch upsert failed: ${response.status} ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      const result = await response.json();
+      return result;
+    } catch (error) {
+      logger.error(`Batch upsert API error: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Fetch PDF records for pedimento detection
+   * @param {Object} options - Query options
+   * @param {number} options.offset - Pagination offset
+   * @param {number} options.limit - Number of records to fetch
+   * @returns {Promise<Object>} { data: Array, error: Error|null }
+   */
+  async fetchPdfRecordsForDetection(options = {}) {
+    const { offset = 0, limit = 100 } = options;
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const url = new URL(`${this.baseUrl}/api/uploader/pdf-records`);
+      url.searchParams.append('offset', offset);
+      url.searchParams.append('limit', limit);
+      url.searchParams.append('status', 'fs-stats');
+      url.searchParams.append('file_extension', 'pdf');
+      url.searchParams.append('is_like_simplificado', 'true');
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          'x-api-key': this.token,
+          'Content-Type': 'application/json',
+        },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          data: null,
+          error: new Error(
+            `Failed to fetch PDF records: ${response.status} ${response.statusText} - ${errorText}`,
+          ),
+        };
+      }
+
+      const data = await response.json();
+      return { data, error: null };
+    } catch (error) {
+      logger.error(`API fetch PDF records error: ${error.message}`);
+      return { data: null, error };
+    }
+  }
+
+  /**
+   * Batch update detection results
+   * @param {Array} updates - Array of update objects with { id, ...updateData }
+   * @returns {Promise<Object>} Update result { success: boolean, updated: number, errors: Array }
+   */
+  async batchUpdateDetectionResults(updates) {
+    if (!updates || !Array.isArray(updates) || updates.length === 0) {
+      return { success: true, updated: 0, errors: [] };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const response = await fetch(
+        `${this.baseUrl}/api/uploader/batch-update-detection`,
+        {
+          method: 'PATCH',
+          headers: {
+            'x-api-key': this.token,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ updates }),
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `Batch update failed: ${response.status} ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      const result = await response.json();
+      return result;
+    } catch (error) {
+      logger.error(`Batch update API error: ${error.message}`);
+      return {
+        success: false,
+        updated: 0,
+        errors: [{ message: error.message }],
+      };
+    }
+  }
+
+  /**
+   * Execute arela_path propagation on the backend
+   * This triggers a server-side process that propagates arela_path from pedimentos to related files
+   * @param {Object} options - Propagation options
+   * @param {Array} options.years - Optional year filter
+   * @returns {Promise<Object>} Propagation result { success: boolean, processedCount, updatedCount, errorCount }
+   */
+  async propagateArelaPath(options = {}) {
+    const { years = [] } = options;
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+
+      // Longer timeout for propagation (can take several minutes for large datasets)
+      const propagationTimeout = 5 * 60 * 1000; // 5 minutes
+      const controller = new AbortController();
+      const timeoutId = setTimeout(
+        () => controller.abort(),
+        propagationTimeout,
+      );
+
+      const response = await fetch(
+        `${this.baseUrl}/api/uploader/propagate-arela-path`,
+        {
+          method: 'POST',
+          headers: {
+            'x-api-key': this.token,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ years }),
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
+          signal: controller.signal,
+        },
+      );
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `Arela path propagation failed: ${response.status} ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      const result = await response.json();
+      return result;
+    } catch (error) {
+      logger.error(`Propagate arela_path API error: ${error.message}`);
+      return {
+        success: false,
+        processedCount: 0,
+        updatedCount: 0,
+        errorCount: 1,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Fetch RFC file count
+   * @param {Object} options - Query options
+   * @param {Array} options.rfcs - Array of RFC values to filter
+   * @returns {Promise<Object>} { count: number, error: Error|null }
+   */
+  async fetchRfcFileCount(options = {}) {
+    const { rfcs = [] } = options;
+
+    if (!rfcs || rfcs.length === 0) {
+      return { count: 0, error: null };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const url = new URL(`${this.baseUrl}/api/uploader/rfc-file-count`);
+      url.searchParams.append('rfcs', rfcs.join(','));
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          'x-api-key': this.token,
+          'Content-Type': 'application/json',
+        },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          count: 0,
+          error: new Error(
+            `Failed to fetch RFC file count: ${response.status} ${response.statusText} - ${errorText}`,
+          ),
+        };
+      }
+
+      const data = await response.json();
+      return { count: data.count || 0, error: null };
+    } catch (error) {
+      logger.error(`API fetch RFC file count error: ${error.message}`);
+      return { count: 0, error };
+    }
+  }
+
+  /**
+   * Fetch pedimento records by RFC
+   * @param {Object} options - Query options
+   * @param {Array} options.rfcs - Array of RFC values to filter
+   * @param {Array} options.years - Optional year filter
+   * @param {number} options.offset - Pagination offset
+   * @param {number} options.limit - Number of records to fetch
+   * @returns {Promise<Object>} { data: Array, error: Error|null }
+   */
+  async fetchPedimentosByRfc(options = {}) {
+    const { rfcs = [], years = [], offset = 0, limit = 500 } = options;
+
+    if (!rfcs || rfcs.length === 0) {
+      return { data: [], error: null };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const url = new URL(`${this.baseUrl}/api/uploader/pedimentos-by-rfc`);
+      url.searchParams.append('rfcs', rfcs.join(','));
+      if (years && years.length > 0) {
+        url.searchParams.append('years', years.join(','));
+      }
+      url.searchParams.append('offset', offset);
+      url.searchParams.append('limit', limit);
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          'x-api-key': this.token,
+          'Content-Type': 'application/json',
+        },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          data: null,
+          error: new Error(
+            `Failed to fetch pedimentos by RFC: ${response.status} ${response.statusText} - ${errorText}`,
+          ),
+        };
+      }
+
+      const data = await response.json();
+      return { data, error: null };
+    } catch (error) {
+      logger.error(`API fetch pedimentos by RFC error: ${error.message}`);
+      return { data: null, error };
+    }
+  }
+
+  /**
+   * Fetch files for upload by arela_path
+   * @param {Object} options - Query options
+   * @param {Array} options.arelaPaths - Array of arela_path values to filter
+   * @param {number} options.offset - Pagination offset
+   * @param {number} options.limit - Number of records to fetch
+   * @returns {Promise<Object>} { data: Array, error: Error|null }
+   */
+  async fetchFilesForUpload(options = {}) {
+    const { arelaPaths = [], offset = 0, limit = 1000 } = options;
+
+    if (!arelaPaths || arelaPaths.length === 0) {
+      return { data: [], error: null };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const url = new URL(`${this.baseUrl}/api/uploader/files-for-upload`);
+      url.searchParams.append('arela_paths', arelaPaths.join('|'));
+      url.searchParams.append('offset', offset);
+      url.searchParams.append('limit', limit);
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          'x-api-key': this.token,
+          'Content-Type': 'application/json',
+        },
+        agent: isHttps ? this.httpsAgent : this.httpAgent,
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          data: null,
+          error: new Error(
+            `Failed to fetch files for upload: ${response.status} ${response.statusText} - ${errorText}`,
+          ),
+        };
+      }
+
+      const data = await response.json();
+      return { data, error: null };
+    } catch (error) {
+      logger.error(`API fetch files for upload error: ${error.message}`);
+      return { data: null, error };
+    }
+  }
+
+  /**
+   * Update file status after upload
+   * @param {Array} updates - Array of update objects with { id, status, message, processing_status }
+   * @returns {Promise<Object>} Update result { success: boolean, updated: number, errors: Array }
+   */
+  async updateFileStatus(updates) {
+    if (!updates || !Array.isArray(updates) || updates.length === 0) {
+      return { success: true, updated: 0, errors: [] };
+    }
+
+    try {
+      const isHttps = this.baseUrl.startsWith('https');
+      const response = await fetch(
+        `${this.baseUrl}/api/uploader/batch-update-status`,
+        {
+          method: 'PATCH',
+          headers: {
+            'x-api-key': this.token,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ updates }),
+          agent: isHttps ? this.httpsAgent : this.httpAgent,
+        },
+      );
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(
+          `Batch status update failed: ${response.status} ${response.statusText} - ${errorText}`,
+        );
+      }
+
+      const result = await response.json();
+      return result;
+    } catch (error) {
+      logger.error(`Batch status update API error: ${error.message}`);
+      return {
+        success: false,
+        updated: 0,
+        errors: [{ message: error.message }],
+      };
+    }
+  }
+
+  /**
+   * Get service name
+   * @returns {string} Service name
+   */
+  getServiceName() {
+    return 'Arela API';
+  }
+
+  /**
+   * Build folder structure from options and auto-detection
+   * @private
+   * @param {Array} files - Files being uploaded
+   * @param {Object} options - Upload options
+   * @returns {string|null} Folder structure or null
+   */
+  #buildFolderStructure(files, options) {
+    let combinedStructure = null;
+
+    if (
+      options.folderStructure &&
+      options.autoDetectStructure &&
+      files.length > 0
+    ) {
+      // This would require the path detection utility - we'll implement this after creating that module
+      console.log(`📁 Custom folder structure: ${options.folderStructure}`);
+      return options.folderStructure;
+    } else if (options.folderStructure) {
+      console.log(
+        `📁 Using custom folder structure: ${options.folderStructure}`,
+      );
+      return options.folderStructure;
+    } else if (options.autoDetectStructure && files.length > 0) {
+      // Auto-detection logic would go here
+      console.log('📁 Auto-detection requested but not yet implemented');
+      return null;
+    }
+
+    return null;
+  }
+}

package/.claude/worktrees/agent-multi-profile/src/services/upload/BaseUploadService.js

@@ -0,0 +1,36 @@
+import FormData from 'form-data';
+import fs from 'fs';
+import fetch from 'node-fetch';
+
+/**
+ * Base upload service interface
+ * Defines the contract for upload implementations
+ */
+export class BaseUploadService {
+  /**
+   * Upload files using the specific implementation
+   * @param {Array} files - Array of file objects with path, name, and contentType
+   * @param {Object} options - Upload options
+   * @returns {Promise<Object>} Upload result
+   * @throws {Error} If upload fails
+   */
+  async upload(files, options) {
+    throw new Error('upload() method must be implemented by subclass');
+  }
+
+  /**
+   * Check if the service is available and properly configured
+   * @returns {Promise<boolean>} True if service is available
+   */
+  async isAvailable() {
+    throw new Error('isAvailable() method must be implemented by subclass');
+  }
+
+  /**
+   * Get the service name for logging purposes
+   * @returns {string} Service name
+   */
+  getServiceName() {
+    throw new Error('getServiceName() method must be implemented by subclass');
+  }
+}