@arela/uploader 1.1.4 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/services/DatastageApiService.js

@@ -0,0 +1,240 @@
+import FormData from 'form-data';
+import fs from 'fs';
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
+import fetch from 'node-fetch';
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import logger from './LoggingService.js';
+
+/**
+ * Datastage API Service
+ * Handles API communication for the arela datastage command:
+ *  - tracking endpoints under /api/uploader/datastage/*
+ *  - zip upload endpoint POST /api/datastage (multipart, field: zipFile)
+ */
+export class DatastageApiService {
+  /**
+   * @param {string|null} apiTarget - 'default'|'agencia'|'cliente'
+   */
+  constructor(apiTarget = null) {
+    this.apiTarget = apiTarget;
+    const apiConfig = appConfig.getApiConfig(apiTarget);
+    this.baseUrl = apiConfig.baseUrl;
+    this.token = apiConfig.token;
+
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 300000;
+
+    this.maxRetries = parseInt(process.env.API_MAX_RETRIES) || 3;
+    this.useExponentialBackoff =
+      process.env.API_RETRY_EXPONENTIAL_BACKOFF !== 'false';
+    this.fixedRetryDelay = parseInt(process.env.API_RETRY_DELAY) || 1000;
+
+    const agentOpts = {
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5,
+      timeout: connectionTimeout,
+      scheduling: 'fifo',
+    };
+    this.httpAgent = new Agent(agentOpts);
+    this.httpsAgent = new HttpsAgent(agentOpts);
+
+    logger.debug(
+      `🔗 Datastage API Service configured (target=${apiTarget || 'default'})`,
+    );
+  }
+
+  #getAgent(url) {
+    return url.startsWith('https://') ? this.httpsAgent : this.httpAgent;
+  }
+
+  #isRetryableError(error, response = null) {
+    if (
+      error?.code === 'ECONNRESET' ||
+      error?.code === 'ETIMEDOUT' ||
+      error?.code === 'ECONNREFUSED' ||
+      error?.code === 'ENOTFOUND' ||
+      error?.code === 'EAI_AGAIN'
+    ) {
+      return true;
+    }
+    if (response) {
+      const s = response.status;
+      if (s === 429 || (s >= 500 && s < 600)) return true;
+    }
+    if (error?.message && error.message.includes('timeout')) return true;
+    return false;
+  }
+
+  #calculateBackoff(attempt) {
+    if (!this.useExponentialBackoff) {
+      const jitter = this.fixedRetryDelay * 0.2 * (Math.random() * 2 - 1);
+      return Math.floor(this.fixedRetryDelay + jitter);
+    }
+    const baseDelay = 1000;
+    const maxDelay = 16000;
+    const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+    const jitter = delay * 0.2 * (Math.random() * 2 - 1);
+    return Math.floor(delay + jitter);
+  }
+
+  #sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+  }
+
+  async #requestJson(endpoint, method = 'GET', body = null, headers = {}) {
+    const url = `${this.baseUrl}${endpoint}`;
+    const options = {
+      method,
+      headers: {
+        'x-api-key': this.token,
+        'Content-Type': 'application/json',
+        ...headers,
+      },
+      agent: this.#getAgent(url),
+    };
+    if (body) options.body = JSON.stringify(body);
+
+    let lastError;
+    let lastResponse = null;
+    const retries = this.maxRetries;
+
+    for (let attempt = 1; attempt <= retries + 1; attempt++) {
+      try {
+        const response = await fetch(url, options);
+        lastResponse = response;
+        if (!response.ok) {
+          const errorText = await response.text();
+          let errorMessage = `API ${method} ${endpoint} failed: ${response.status} ${response.statusText}`;
+          try {
+            const j = JSON.parse(errorText);
+            errorMessage = j.message || errorMessage;
+          } catch {
+            errorMessage = errorText || errorMessage;
+          }
+          const err = new Error(errorMessage);
+          err.status = response.status;
+          if (this.#isRetryableError(err, response) && attempt <= retries) {
+            const d = this.#calculateBackoff(attempt);
+            logger.warn(
+              `Retrying ${method} ${endpoint} (attempt ${attempt}/${retries + 1}) in ${d}ms: ${errorMessage}`,
+            );
+            await this.#sleep(d);
+            continue;
+          }
+          throw err;
+        }
+        return await response.json();
+      } catch (error) {
+        lastError = error;
+        if (this.#isRetryableError(error, lastResponse) && attempt <= retries) {
+          const d = this.#calculateBackoff(attempt);
+          logger.warn(
+            `Retrying ${method} ${endpoint} (attempt ${attempt}/${retries + 1}) in ${d}ms: ${error.message}`,
+          );
+          await this.#sleep(d);
+          continue;
+        }
+        throw error;
+      }
+    }
+    throw lastError;
+  }
+
+  // --- Tracking endpoints ---
+
+  async registerUpload({
+    absolutePath,
+    fileName,
+    sizeBytes,
+    fileModifiedAt,
+    sourceDirectory,
+  }) {
+    return this.#requestJson('/api/uploader/datastage/register', 'POST', {
+      absolutePath,
+      fileName,
+      sizeBytes,
+      fileModifiedAt,
+      sourceDirectory,
+    });
+  }
+
+  async getPending(sourceDirectory = null) {
+    const qs = sourceDirectory
+      ? `?sourceDirectory=${encodeURIComponent(sourceDirectory)}`
+      : '';
+    return this.#requestJson(`/api/uploader/datastage/pending${qs}`, 'GET');
+  }
+
+  async getStats(sourceDirectory = null) {
+    const qs = sourceDirectory
+      ? `?sourceDirectory=${encodeURIComponent(sourceDirectory)}`
+      : '';
+    return this.#requestJson(`/api/uploader/datastage/stats${qs}`, 'GET');
+  }
+
+  async markUploaded(id, { datastageId, folio }) {
+    return this.#requestJson(
+      `/api/uploader/datastage/${id}/mark-uploaded`,
+      'PATCH',
+      { datastageId, folio },
+    );
+  }
+
+  async markFailed(id, error) {
+    return this.#requestJson(
+      `/api/uploader/datastage/${id}/mark-failed`,
+      'PATCH',
+      { error: String(error || 'unknown') },
+    );
+  }
+
+  // --- Zip upload ---
+
+  /**
+   * Upload a single zip file to POST /api/datastage (multipart, field name 'zipFile').
+   * Returns the created Datastage row { id, folio, ... }.
+   */
+  async uploadZip(localPath) {
+    const url = `${this.baseUrl}/api/datastage`;
+    const form = new FormData();
+    const fileName = path.basename(localPath);
+    form.append('zipFile', fs.createReadStream(localPath), {
+      filename: fileName,
+      contentType: 'application/zip',
+    });
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'x-api-key': this.token,
+        ...form.getHeaders(),
+      },
+      body: form,
+      agent: this.#getAgent(url),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      let msg = `Datastage upload failed: ${response.status} ${response.statusText}`;
+      try {
+        const j = JSON.parse(text);
+        msg = j.message || msg;
+      } catch {
+        msg = text || msg;
+      }
+      const err = new Error(msg);
+      err.status = response.status;
+      throw err;
+    }
+    return await response.json();
+  }
+}
+
+export default DatastageApiService;

package/.claude/worktrees/agent-multi-profile/src/services/ErrorMonitor.js

@@ -0,0 +1,275 @@
+/**
+ * Error Monitor Service
+ * Tracks, logs, and monitors errors during signal handling and cleanup operations
+ * Provides detailed error reporting and failure recovery mechanisms
+ */
+export class ErrorMonitor {
+  constructor(logger, cleanupManager, signalHandler) {
+    this.logger = logger;
+    this.cleanupManager = cleanupManager;
+    this.signalHandler = signalHandler;
+
+    /**
+     * Array of recorded errors
+     * @type {Array<Object>}
+     */
+    this.errors = [];
+
+    /**
+     * Error categories
+     * @type {Object}
+     */
+    this.errorCategories = {
+      SIGNAL: 'signal_error',
+      CLEANUP: 'cleanup_error',
+      DATABASE: 'database_error',
+      WATCH_SERVICE: 'watch_service_error',
+      LOGGING: 'logging_error',
+      GENERAL: 'general_error',
+    };
+
+    /**
+     * Error statistics
+     * @type {Object}
+     */
+    this.stats = {
+      totalErrors: 0,
+      errorsByCategory: {},
+      fatalErrors: 0,
+      recoveredErrors: 0,
+      lastErrorTime: null,
+    };
+
+    // Initialize error category stats
+    for (const category of Object.values(this.errorCategories)) {
+      this.stats.errorsByCategory[category] = 0;
+    }
+  }
+
+  /**
+   * Record an error occurrence
+   * @param {string} category - Error category
+   * @param {string} message - Error message
+   * @param {Error} error - Error object
+   * @param {Object} context - Additional context
+   * @param {boolean} isFatal - Whether error is fatal
+   * @returns {Object} Error record
+   */
+  recordError(category, message, error, context = {}, isFatal = false) {
+    const errorRecord = {
+      id: `err_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+      timestamp: new Date().toISOString(),
+      category,
+      message,
+      isFatal,
+      stack: error?.stack || null,
+      context,
+      signal: this.signalHandler?.getSignalReceived() || null,
+      resolved: false,
+    };
+
+    this.errors.push(errorRecord);
+    this.stats.totalErrors++;
+    this.stats.errorsByCategory[category]++;
+    this.stats.lastErrorTime = errorRecord.timestamp;
+
+    if (isFatal) {
+      this.stats.fatalErrors++;
+      this.logger.error(
+        `🚨 FATAL ERROR [${category}]: ${message} - ${error?.message}`,
+      );
+    } else {
+      this.logger.warn(
+        `⚠️  ERROR [${category}]: ${message} - ${error?.message}`,
+      );
+    }
+
+    return errorRecord;
+  }
+
+  /**
+   * Mark an error as resolved
+   * @param {string} errorId - Error ID to resolve
+   * @returns {boolean} True if resolved
+   */
+  resolveError(errorId) {
+    const errorRecord = this.errors.find((e) => e.id === errorId);
+    if (errorRecord) {
+      errorRecord.resolved = true;
+      this.stats.recoveredErrors++;
+      this.logger.info(`✅ Error resolved: ${errorId}`);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Get all unresolved errors
+   * @returns {Array<Object>} Unresolved error records
+   */
+  getUnresolvedErrors() {
+    return this.errors.filter((e) => !e.resolved);
+  }
+
+  /**
+   * Get errors by category
+   * @param {string} category - Error category
+   * @returns {Array<Object>} Errors in category
+   */
+  getErrorsByCategory(category) {
+    return this.errors.filter((e) => e.category === category);
+  }
+
+  /**
+   * Check if critical errors occurred
+   * @returns {boolean} True if there are fatal errors
+   */
+  hasCriticalErrors() {
+    return this.stats.fatalErrors > 0;
+  }
+
+  /**
+   * Get error statistics
+   * @returns {Object} Error statistics
+   */
+  getStatistics() {
+    return {
+      totalErrors: this.stats.totalErrors,
+      fatalErrors: this.stats.fatalErrors,
+      recoveredErrors: this.stats.recoveredErrors,
+      unresolvedErrors: this.getUnresolvedErrors().length,
+      errorsByCategory: { ...this.stats.errorsByCategory },
+      lastErrorTime: this.stats.lastErrorTime,
+      successRate:
+        this.stats.totalErrors > 0
+          ? (
+              (this.stats.recoveredErrors / this.stats.totalErrors) *
+              100
+            ).toFixed(2)
+          : 100,
+    };
+  }
+
+  /**
+   * Generate detailed error report
+   * @returns {string} Formatted error report
+   */
+  generateErrorReport() {
+    const stats = this.getStatistics();
+    let report =
+      '\n═══════════════════════════════════════════════════════════\n';
+    report += 'ERROR MONITORING REPORT\n';
+    report += '═══════════════════════════════════════════════════════════\n\n';
+
+    report += 'STATISTICS\n';
+    report += `├─ Total Errors:        ${stats.totalErrors}\n`;
+    report += `├─ Fatal Errors:        ${stats.fatalErrors}\n`;
+    report += `├─ Recovered Errors:    ${stats.recoveredErrors}\n`;
+    report += `├─ Unresolved Errors:   ${stats.unresolvedErrors}\n`;
+    report += `├─ Recovery Rate:       ${stats.successRate}%\n`;
+    report += `└─ Last Error:          ${stats.lastErrorTime || 'None'}\n\n`;
+
+    if (this.errors.length > 0) {
+      report += 'ERROR DETAILS (Last 10)\n';
+      const recentErrors = this.errors.slice(-10);
+      recentErrors.forEach((err, idx) => {
+        const status = err.resolved ? '✅' : '❌';
+        const severity = err.isFatal ? '🚨' : '⚠️ ';
+        report += `${idx + 1}. ${status} ${severity} [${err.category}]\n`;
+        report += `   Message: ${err.message}\n`;
+        report += `   Time: ${err.timestamp}\n`;
+        if (err.signal) {
+          report += `   Signal: ${err.signal}\n`;
+        }
+        report += '\n';
+      });
+    }
+
+    report += '═══════════════════════════════════════════════════════════\n';
+    return report;
+  }
+
+  /**
+   * Clear error history (useful after processing)
+   * @returns {void}
+   */
+  clearErrors() {
+    const count = this.errors.length;
+    this.errors = [];
+    this.logger.debug(`ErrorMonitor: Cleared ${count} error records`);
+  }
+
+  /**
+   * Get error monitor status
+   * @returns {Object} Status information
+   */
+  getStatus() {
+    return {
+      isHealthy: this.stats.fatalErrors === 0,
+      hasErrors: this.stats.totalErrors > 0,
+      unresolvedCount: this.getUnresolvedErrors().length,
+      stats: this.getStatistics(),
+    };
+  }
+
+  /**
+   * Validate cleanup operation for errors
+   * @param {Object} cleanupResult - Result from cleanup operation
+   * @returns {Object} Validation result
+   */
+  validateCleanupResult(cleanupResult) {
+    const validation = {
+      isValid: cleanupResult.failureCount === 0,
+      failureCount: cleanupResult.failureCount,
+      successCount: cleanupResult.successCount,
+      issues: [],
+    };
+
+    if (cleanupResult.results) {
+      for (const result of cleanupResult.results) {
+        if (!result.success) {
+          validation.issues.push({
+            resource: result.name,
+            error: result.error,
+            duration: result.duration,
+          });
+
+          this.recordError(
+            this.errorCategories.CLEANUP,
+            `Cleanup failed for ${result.name}`,
+            new Error(result.error),
+            { resource: result.name, duration: result.duration },
+            false,
+          );
+        }
+      }
+    }
+
+    return validation;
+  }
+
+  /**
+   * Reset monitor state (for testing)
+   * @returns {void}
+   */
+  reset() {
+    this.errors = [];
+    this.stats.totalErrors = 0;
+    this.stats.fatalErrors = 0;
+    this.stats.recoveredErrors = 0;
+    this.stats.lastErrorTime = null;
+
+    for (const category of Object.keys(this.stats.errorsByCategory)) {
+      this.stats.errorsByCategory[category] = 0;
+    }
+
+    this.logger.debug('ErrorMonitor: Reset to initial state');
+  }
+}
+
+// Export singleton
+export function createErrorMonitor(logger, cleanupManager, signalHandler) {
+  return new ErrorMonitor(logger, cleanupManager, signalHandler);
+}
+
+export default ErrorMonitor;