@arela/uploader 1.0.1 → 1.0.3

package/src/services/ScanApiService.js

@@ -0,0 +1,646 @@
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
+import fetch from 'node-fetch';
+
+import appConfig from '../config/config.js';
+import logger from './LoggingService.js';
+
+/**
+ * Scan API Service
+ * Handles API communication for the arela scan command
+ */
+export class ScanApiService {
+  constructor() {
+    const apiConfig = appConfig.getApiConfig();
+    this.baseUrl = apiConfig.baseUrl;
+    this.token = apiConfig.token;
+
+    // Get API connection settings
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 60000;
+
+    // Get retry configuration
+    this.maxRetries = parseInt(process.env.API_MAX_RETRIES) || 3;
+    this.useExponentialBackoff =
+      process.env.API_RETRY_EXPONENTIAL_BACKOFF !== 'false'; // Default true
+    this.fixedRetryDelay = parseInt(process.env.API_RETRY_DELAY) || 1000;
+
+    // Initialize HTTP agents for connection pooling
+    this.httpAgent = new Agent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5,
+      timeout: connectionTimeout,
+      scheduling: 'fifo',
+    });
+
+    this.httpsAgent = new HttpsAgent({
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5,
+      timeout: connectionTimeout,
+      scheduling: 'fifo',
+    });
+
+    logger.debug(
+      `🔗 Scan API Service configured with ${maxApiConnections} concurrent connections`,
+    );
+  }
+
+  /**
+   * Get the appropriate HTTP agent based on URL protocol
+   * @private
+   */
+  #getAgent(url) {
+    return url.startsWith('https://') ? this.httpsAgent : this.httpAgent;
+  }
+
+  /**
+   * Check if error is retryable
+   * @private
+   * @param {Error} error - Error to check
+   * @param {Response} response - HTTP response (if available)
+   * @returns {boolean} True if error is retryable
+   */
+  #isRetryableError(error, response = null) {
+    // Network errors are retryable
+    if (
+      error.code === 'ECONNRESET' ||
+      error.code === 'ETIMEDOUT' ||
+      error.code === 'ECONNREFUSED' ||
+      error.code === 'ENOTFOUND' ||
+      error.code === 'EAI_AGAIN'
+    ) {
+      return true;
+    }
+
+    // HTTP status codes that are retryable
+    if (response) {
+      const status = response.status;
+      // 429 Too Many Requests - should retry with backoff
+      // 5xx Server errors - temporary issues
+      if (status === 429 || (status >= 500 && status < 600)) {
+        return true;
+      }
+    }
+
+    // Timeout errors
+    if (error.message && error.message.includes('timeout')) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Calculate backoff delay
+   * @private
+   * @param {number} attempt - Current attempt number (1-based)
+   * @returns {number} Delay in milliseconds
+   */
+  #calculateBackoff(attempt) {
+    if (!this.useExponentialBackoff) {
+      // Fixed delay with jitter
+      const jitter = this.fixedRetryDelay * 0.2 * (Math.random() * 2 - 1);
+      return Math.floor(this.fixedRetryDelay + jitter);
+    }
+
+    // Exponential backoff: 1s, 2s, 4s, 8s, 16s
+    const baseDelay = 1000;
+    const maxDelay = 16000;
+    const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+
+    // Add jitter (±20%) to prevent thundering herd
+    const jitter = delay * 0.2 * (Math.random() * 2 - 1);
+    return Math.floor(delay + jitter);
+  }
+
+  /**
+   * Sleep for specified milliseconds
+   * @private
+   * @param {number} ms - Milliseconds to sleep
+   * @returns {Promise<void>}
+   */
+  async #sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Make API request with retry logic and exponential backoff
+   * @private
+   * @param {string} endpoint - API endpoint
+   * @param {string} method - HTTP method
+   * @param {Object} body - Request body
+   * @param {Object} headers - Additional headers
+   * @param {number} maxRetries - Maximum retry attempts (defaults to configured value)
+   * @returns {Promise<Object>} Response data
+   */
+  async #request(
+    endpoint,
+    method = 'GET',
+    body = null,
+    headers = {},
+    maxRetries = null,
+  ) {
+    // Use configured maxRetries if not specified
+    const retries = maxRetries !== null ? maxRetries : this.maxRetries;
+
+    const url = `${this.baseUrl}${endpoint}`;
+
+    const options = {
+      method,
+      headers: {
+        'x-api-key': this.token,
+        'Content-Type': 'application/json',
+        ...headers,
+      },
+      agent: this.#getAgent(url),
+    };
+
+    if (body) {
+      options.body = JSON.stringify(body);
+    }
+
+    let lastError;
+    let lastResponse = null;
+
+    for (let attempt = 1; attempt <= retries + 1; attempt++) {
+      try {
+        const response = await fetch(url, options);
+        lastResponse = response;
+
+        if (!response.ok) {
+          const errorText = await response.text();
+          let errorMessage = `API request failed: ${response.status} ${response.statusText}`;
+
+          try {
+            const errorJson = JSON.parse(errorText);
+            errorMessage = errorJson.message || errorMessage;
+          } catch {
+            errorMessage = errorText || errorMessage;
+          }
+
+          const error = new Error(errorMessage);
+          error.status = response.status;
+
+          // Check if error is retryable
+          if (this.#isRetryableError(error, response)) {
+            if (attempt <= retries) {
+              const backoffDelay = this.#calculateBackoff(attempt);
+              logger.warn(
+                `API request failed (attempt ${attempt}/${retries + 1}): ${errorMessage}. Retrying in ${backoffDelay}ms...`,
+              );
+              await this.#sleep(backoffDelay);
+              continue;
+            }
+          }
+
+          throw error;
+        }
+
+        // Success - log retry success if this wasn't the first attempt
+        if (attempt > 1) {
+          logger.info(
+            `API request succeeded on attempt ${attempt}/${retries + 1}`,
+          );
+        }
+
+        return await response.json();
+      } catch (error) {
+        lastError = error;
+
+        // Check if this is a retryable error
+        if (this.#isRetryableError(error, lastResponse)) {
+          if (attempt <= retries) {
+            const backoffDelay = this.#calculateBackoff(attempt);
+            logger.warn(
+              `API request failed (attempt ${attempt}/${retries + 1}): ${error.message}. Retrying in ${backoffDelay}ms...`,
+            );
+            await this.#sleep(backoffDelay);
+            continue;
+          }
+        }
+
+        // Non-retryable error or max retries reached
+        logger.error(
+          `API request failed after ${attempt} attempt(s): ${error.message}`,
+        );
+        throw error;
+      }
+    }
+
+    // Should not reach here, but just in case
+    throw lastError;
+  }
+
+  /**
+   * Register a scan instance with the API
+   * @param {Object} config - Instance configuration
+   * @returns {Promise<Object>} Registration result
+   */
+  async registerInstance(config) {
+    logger.debug('Registering scan instance...');
+
+    const result = await this.#request('/api/uploader/scan/register', 'POST', {
+      companySlug: config.companySlug,
+      serverId: config.serverId,
+      basePathLabel: config.basePathLabel,
+      basePathFull: config.basePathFull,
+    });
+
+    logger.debug(`Instance registered: ${result.tableName}`);
+    return result;
+  }
+
+  /**
+   * Bulk insert file stats
+   * @param {string} tableName - Target table name
+   * @param {Array} records - File stat records
+   * @returns {Promise<Object>} Insert result
+   */
+  async batchInsertStats(tableName, records) {
+    if (!records || records.length === 0) {
+      return { inserted: 0 };
+    }
+
+    logger.debug(`Uploading batch of ${records.length} records...`);
+
+    const result = await this.#request(
+      '/api/uploader/scan/batch-insert',
+      'POST',
+      records,
+      {
+        'x-table-name': tableName,
+      },
+    );
+
+    logger.debug(`Batch uploaded: ${result.inserted} inserted`);
+    return result;
+  }
+
+  /**
+   * Complete a scan and update statistics
+   * @param {Object} data - Completion data
+   * @returns {Promise<Object>} Completion result
+   */
+  async completeScan(data) {
+    logger.debug('Completing scan...');
+
+    const result = await this.#request('/api/uploader/scan/complete', 'PATCH', {
+      tableName: data.tableName,
+      totalFiles: data.totalFiles,
+      totalSizeBytes: data.totalSizeBytes,
+    });
+
+    logger.debug('Scan completed');
+    return result;
+  }
+
+  /**
+   * Get all scan instances
+   * @returns {Promise<Array>} List of scan instances
+   */
+  async getAllInstances() {
+    logger.debug('Fetching scan instances...');
+    return await this.#request('/api/uploader/scan/instances', 'GET');
+  }
+
+  /**
+   * Get stale scan instances
+   * @param {number} days - Days threshold
+   * @returns {Promise<Array>} List of stale instances
+   */
+  async getStaleInstances(days = 90) {
+    logger.debug(`Fetching stale instances (${days} days)...`);
+    return await this.#request(
+      `/api/uploader/scan/stale-instances?days=${days}`,
+      'GET',
+    );
+  }
+
+  /**
+   * Get all tables for a specific instance
+   * @param {string} companySlug - Company slug
+   * @param {string} serverId - Server ID
+   * @param {string} basePathLabel - Base path label
+   * @returns {Promise<Array>} List of tables for the instance
+   */
+  async getInstanceTables(companySlug, serverId, basePathLabel) {
+    logger.debug(
+      `Fetching instance tables for ${companySlug}/${serverId}/${basePathLabel}...`,
+    );
+    return await this.#request(
+      `/api/uploader/scan/instance-tables?companySlug=${encodeURIComponent(companySlug)}&serverId=${encodeURIComponent(serverId)}&basePathLabel=${encodeURIComponent(basePathLabel)}`,
+      'GET',
+    );
+  }
+
+  /**
+   * Deactivate a scan instance
+   * @param {string} tableName - Table name to deactivate
+   * @returns {Promise<Object>} Deactivation result
+   */
+  async deactivateInstance(tableName) {
+    logger.debug(`Deactivating instance: ${tableName}`);
+
+    const result = await this.#request(
+      '/api/uploader/scan/deactivate',
+      'PATCH',
+      {
+        tableName,
+      },
+    );
+
+    logger.debug('Instance deactivated');
+    return result;
+  }
+
+  // ============================================================================
+  // DETECTION OPERATIONS (for arela identify command)
+  // ============================================================================
+
+  /**
+   * Fetch PDF files for detection
+   * @param {string} tableName - Target table name
+   * @param {number} offset - Pagination offset
+   * @param {number} limit - Number of records to fetch
+   * @returns {Promise<Object>} { data: Array, hasMore: boolean }
+   */
+  async fetchPdfsForDetection(tableName, offset = 0, limit = 100) {
+    logger.debug(
+      `Fetching PDFs for detection (offset: ${offset}, limit: ${limit})...`,
+    );
+
+    const result = await this.#request(
+      `/api/uploader/scan/pdfs-for-detection?tableName=${encodeURIComponent(tableName)}&offset=${offset}&limit=${limit}`,
+      'GET',
+    );
+
+    logger.debug(
+      `Fetched ${result.data.length} PDFs, hasMore: ${result.hasMore}`,
+    );
+    return result;
+  }
+
+  /**
+   * Batch update detection results
+   * @param {string} tableName - Target table name
+   * @param {Array} updates - Detection results
+   * @returns {Promise<Object>} { updated: number, errors: number }
+   */
+  async batchUpdateDetection(tableName, updates) {
+    if (!updates || updates.length === 0) {
+      return { updated: 0, errors: 0 };
+    }
+
+    logger.debug(`Updating detection results for ${updates.length} files...`);
+
+    const result = await this.#request(
+      `/api/uploader/scan/batch-update-detection?tableName=${encodeURIComponent(tableName)}`,
+      'PATCH',
+      updates,
+    );
+
+    logger.debug(
+      `Detection updated: ${result.updated} successful, ${result.errors} errors`,
+    );
+    return result;
+  }
+
+  /**
+   * Get detection statistics
+   * @param {string} tableName - Target table name
+   * @returns {Promise<Object>} { totalPdfs, detected, pending, errors }
+   */
+  async getDetectionStats(tableName) {
+    logger.debug('Fetching detection statistics...');
+
+    const result = await this.#request(
+      `/api/uploader/scan/detection-stats?tableName=${encodeURIComponent(tableName)}`,
+      'GET',
+    );
+
+    logger.debug(
+      `Detection stats: ${result.detected}/${result.totalPdfs} detected, ${result.pending} pending`,
+    );
+    return result;
+  }
+
+  // ============================================================================
+  // PROPAGATION API METHODS (for arela propagate command)
+  // ============================================================================
+
+  /**
+   * Mark files needing propagation
+   * @param {string} tableName - Target table name
+   * @returns {Promise<Object>} { markedCount: number }
+   */
+  async markFilesNeedingPropagation(tableName) {
+    logger.debug('Marking files needing propagation...');
+
+    const result = await this.#request(
+      `/api/uploader/scan/mark-propagation?tableName=${encodeURIComponent(tableName)}`,
+      'POST',
+    );
+
+    logger.debug(`Marked ${result.markedCount} files for propagation`);
+    return result;
+  }
+
+  /**
+   * Fetch pedimento sources for propagation
+   * @param {string} tableName - Target table name
+   * @param {number} offset - Pagination offset
+   * @param {number} limit - Number of records to fetch
+   * @returns {Promise<Array>} Array of pedimento sources
+   */
+  async fetchPedimentoSources(tableName, offset = 0, limit = 100) {
+    logger.debug(
+      `Fetching pedimento sources (offset: ${offset}, limit: ${limit})...`,
+    );
+
+    const result = await this.#request(
+      `/api/uploader/scan/pedimento-sources?tableName=${encodeURIComponent(tableName)}&offset=${offset}&limit=${limit}`,
+      'GET',
+    );
+
+    // Validate response is an array
+    if (!Array.isArray(result)) {
+      logger.error(
+        'fetchPedimentoSources: Expected array, got:',
+        typeof result,
+      );
+      logger.error('Response data:', JSON.stringify(result).substring(0, 200));
+      return [];
+    }
+
+    logger.debug(`Fetched ${result.length} pedimento sources`);
+    return result;
+  }
+
+  /**
+   * Fetch files needing propagation by directory
+   * @param {string} tableName - Target table name
+   * @param {string} directoryPath - Directory path to query
+   * @returns {Promise<Array>} Array of files needing propagation
+   */
+  async fetchFilesNeedingPropagationByDirectory(tableName, directoryPath) {
+    const result = await this.#request(
+      `/api/uploader/scan/files-by-directory?tableName=${encodeURIComponent(tableName)}&directoryPath=${encodeURIComponent(directoryPath)}`,
+      'GET',
+    );
+
+    // Validate response is an array
+    if (!Array.isArray(result)) {
+      logger.error(
+        'fetchFilesNeedingPropagationByDirectory: Expected array, got:',
+        typeof result,
+      );
+      return [];
+    }
+
+    return result;
+  }
+
+  /**
+   * Batch update propagation results
+   * @param {string} tableName - Target table name
+   * @param {Array} updates - Propagation results
+   * @returns {Promise<Object>} { updated: number, errors: number }
+   */
+  async batchUpdatePropagation(tableName, updates) {
+    if (!updates || updates.length === 0) {
+      return { updated: 0, errors: 0 };
+    }
+
+    logger.debug(`Updating propagation results for ${updates.length} files...`);
+
+    const result = await this.#request(
+      `/api/uploader/scan/batch-update-propagation?tableName=${encodeURIComponent(tableName)}`,
+      'PATCH',
+      { updates },
+    );
+
+    logger.debug(
+      `Propagation updated: ${result.updated} successful, ${result.errors} errors`,
+    );
+    return result;
+  }
+
+  /**
+   * Get propagation statistics
+   * @param {string} tableName - Target table name
+   * @returns {Promise<Object>} { totalFiles, withArelaPath, needsPropagation, pending, errors, maxAttemptsReached, pedimentoSources }
+   */
+  async getPropagationStats(tableName) {
+    logger.debug('Fetching propagation statistics...');
+
+    const result = await this.#request(
+      `/api/uploader/scan/propagation-stats?tableName=${encodeURIComponent(tableName)}`,
+      'GET',
+    );
+
+    logger.debug(
+      `Propagation stats: ${result.withArelaPath}/${result.totalFiles} with arela_path, ${result.pending} pending`,
+    );
+    return result;
+  }
+
+  // ============================================================================
+  // PUSH OPERATIONS
+  // ============================================================================
+
+  /**
+   * Fetch files ready for upload (push command)
+   * @param {string} tableName - Target table name
+   * @param {Object} options - Query options
+   * @param {string[]} options.rfcs - RFCs to filter by
+   * @param {number[]} options.years - Years to filter by
+   * @param {number} options.offset - Pagination offset
+   * @param {number} options.limit - Pagination limit
+   * @returns {Promise<Array>} Array of files ready for upload
+   */
+  async fetchFilesForPush(tableName, options = {}) {
+    const { rfcs, years, offset = 0, limit = 100 } = options;
+
+    // Build query string
+    const params = new URLSearchParams({
+      tableName,
+      offset: offset.toString(),
+      limit: limit.toString(),
+    });
+
+    if (rfcs && rfcs.length > 0) {
+      params.append('rfcs', rfcs.join(','));
+    }
+
+    if (years && years.length > 0) {
+      params.append('years', years.join(','));
+    }
+
+    const result = await this.#request(
+      `/api/uploader/scan/files-for-push?${params.toString()}`,
+      'GET',
+    );
+
+    // Validate response is an array
+    if (!Array.isArray(result)) {
+      logger.error('fetchFilesForPush: Expected array, got:', typeof result);
+      return [];
+    }
+
+    logger.debug(`Fetched ${result.length} files for push`);
+    return result;
+  }
+
+  /**
+   * Batch update upload results
+   * @param {string} tableName - Target table name
+   * @param {Array} updates - Upload results
+   * @returns {Promise<Object>} { updated: number, errors: number }
+   */
+  async batchUpdateUpload(tableName, updates) {
+    if (!updates || updates.length === 0) {
+      return { updated: 0, errors: 0 };
+    }
+
+    logger.debug(`Updating upload results for ${updates.length} files...`);
+
+    const result = await this.#request(
+      `/api/uploader/scan/batch-update-upload?tableName=${encodeURIComponent(tableName)}`,
+      'PATCH',
+      { updates },
+    );
+
+    logger.debug(
+      `Upload updated: ${result.updated} successful, ${result.errors} errors`,
+    );
+    return result;
+  }
+
+  /**
+   * Get push statistics
+   * @param {string} tableName - Target table name
+   * @returns {Promise<Object>} { totalWithArelaPath, uploaded, pending, errors, maxAttemptsReached, byRfc }
+   */
+  async getPushStats(tableName) {
+    logger.debug('Fetching push statistics...');
+
+    const result = await this.#request(
+      `/api/uploader/scan/push-stats?tableName=${encodeURIComponent(tableName)}`,
+      'GET',
+    );
+
+    logger.debug(
+      `Push stats: ${result.uploaded}/${result.totalWithArelaPath} uploaded, ${result.pending} pending`,
+    );
+    return result;
+  }
+}
+
+export default ScanApiService;

package/src/services/upload/ApiUploadService.js

@@ -397,6 +397,15 @@ export class ApiUploadService extends BaseUploadService {
 
     try {
       const isHttps = this.baseUrl.startsWith('https');
+
+      // Longer timeout for propagation (can take several minutes for large datasets)
+      const propagationTimeout = 5 * 60 * 1000; // 5 minutes
+      const controller = new AbortController();
+      const timeoutId = setTimeout(
+        () => controller.abort(),
+        propagationTimeout,
+      );
+
       const response = await fetch(
         `${this.baseUrl}/api/uploader/propagate-arela-path`,
         {
@@ -407,9 +416,12 @@ export class ApiUploadService extends BaseUploadService {
           },
           body: JSON.stringify({ years }),
           agent: isHttps ? this.httpsAgent : this.httpAgent,
+          signal: controller.signal,
         },
       );
 
+      clearTimeout(timeoutId);
+
       if (!response.ok) {
         const errorText = await response.text();
         throw new Error(