@arela/uploader 1.0.20 → 1.0.22

package/src/services/DatastageApiService.js

@@ -0,0 +1,240 @@
+import FormData from 'form-data';
+import fs from 'fs';
+import { Agent } from 'http';
+import { Agent as HttpsAgent } from 'https';
+import fetch from 'node-fetch';
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import logger from './LoggingService.js';
+
+/**
+ * Datastage API Service
+ * Handles API communication for the arela datastage command:
+ *  - tracking endpoints under /api/uploader/datastage/*
+ *  - zip upload endpoint POST /api/datastage (multipart, field: zipFile)
+ */
+export class DatastageApiService {
+  /**
+   * @param {string|null} apiTarget - 'default'|'agencia'|'cliente'
+   */
+  constructor(apiTarget = null) {
+    this.apiTarget = apiTarget;
+    const apiConfig = appConfig.getApiConfig(apiTarget);
+    this.baseUrl = apiConfig.baseUrl;
+    this.token = apiConfig.token;
+
+    const maxApiConnections = parseInt(process.env.MAX_API_CONNECTIONS) || 10;
+    const connectionTimeout =
+      parseInt(process.env.API_CONNECTION_TIMEOUT) || 300000;
+
+    this.maxRetries = parseInt(process.env.API_MAX_RETRIES) || 3;
+    this.useExponentialBackoff =
+      process.env.API_RETRY_EXPONENTIAL_BACKOFF !== 'false';
+    this.fixedRetryDelay = parseInt(process.env.API_RETRY_DELAY) || 1000;
+
+    const agentOpts = {
+      keepAlive: true,
+      keepAliveMsecs: 30000,
+      maxSockets: maxApiConnections,
+      maxFreeSockets: Math.ceil(maxApiConnections / 2),
+      maxTotalSockets: maxApiConnections + 5,
+      timeout: connectionTimeout,
+      scheduling: 'fifo',
+    };
+    this.httpAgent = new Agent(agentOpts);
+    this.httpsAgent = new HttpsAgent(agentOpts);
+
+    logger.debug(
+      `🔗 Datastage API Service configured (target=${apiTarget || 'default'})`,
+    );
+  }
+
+  #getAgent(url) {
+    return url.startsWith('https://') ? this.httpsAgent : this.httpAgent;
+  }
+
+  #isRetryableError(error, response = null) {
+    if (
+      error?.code === 'ECONNRESET' ||
+      error?.code === 'ETIMEDOUT' ||
+      error?.code === 'ECONNREFUSED' ||
+      error?.code === 'ENOTFOUND' ||
+      error?.code === 'EAI_AGAIN'
+    ) {
+      return true;
+    }
+    if (response) {
+      const s = response.status;
+      if (s === 429 || (s >= 500 && s < 600)) return true;
+    }
+    if (error?.message && error.message.includes('timeout')) return true;
+    return false;
+  }
+
+  #calculateBackoff(attempt) {
+    if (!this.useExponentialBackoff) {
+      const jitter = this.fixedRetryDelay * 0.2 * (Math.random() * 2 - 1);
+      return Math.floor(this.fixedRetryDelay + jitter);
+    }
+    const baseDelay = 1000;
+    const maxDelay = 16000;
+    const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+    const jitter = delay * 0.2 * (Math.random() * 2 - 1);
+    return Math.floor(delay + jitter);
+  }
+
+  #sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+  }
+
+  async #requestJson(endpoint, method = 'GET', body = null, headers = {}) {
+    const url = `${this.baseUrl}${endpoint}`;
+    const options = {
+      method,
+      headers: {
+        'x-api-key': this.token,
+        'Content-Type': 'application/json',
+        ...headers,
+      },
+      agent: this.#getAgent(url),
+    };
+    if (body) options.body = JSON.stringify(body);
+
+    let lastError;
+    let lastResponse = null;
+    const retries = this.maxRetries;
+
+    for (let attempt = 1; attempt <= retries + 1; attempt++) {
+      try {
+        const response = await fetch(url, options);
+        lastResponse = response;
+        if (!response.ok) {
+          const errorText = await response.text();
+          let errorMessage = `API ${method} ${endpoint} failed: ${response.status} ${response.statusText}`;
+          try {
+            const j = JSON.parse(errorText);
+            errorMessage = j.message || errorMessage;
+          } catch {
+            errorMessage = errorText || errorMessage;
+          }
+          const err = new Error(errorMessage);
+          err.status = response.status;
+          if (this.#isRetryableError(err, response) && attempt <= retries) {
+            const d = this.#calculateBackoff(attempt);
+            logger.warn(
+              `Retrying ${method} ${endpoint} (attempt ${attempt}/${retries + 1}) in ${d}ms: ${errorMessage}`,
+            );
+            await this.#sleep(d);
+            continue;
+          }
+          throw err;
+        }
+        return await response.json();
+      } catch (error) {
+        lastError = error;
+        if (this.#isRetryableError(error, lastResponse) && attempt <= retries) {
+          const d = this.#calculateBackoff(attempt);
+          logger.warn(
+            `Retrying ${method} ${endpoint} (attempt ${attempt}/${retries + 1}) in ${d}ms: ${error.message}`,
+          );
+          await this.#sleep(d);
+          continue;
+        }
+        throw error;
+      }
+    }
+    throw lastError;
+  }
+
+  // --- Tracking endpoints ---
+
+  async registerUpload({
+    absolutePath,
+    fileName,
+    sizeBytes,
+    fileModifiedAt,
+    sourceDirectory,
+  }) {
+    return this.#requestJson('/api/uploader/datastage/register', 'POST', {
+      absolutePath,
+      fileName,
+      sizeBytes,
+      fileModifiedAt,
+      sourceDirectory,
+    });
+  }
+
+  async getPending(sourceDirectory = null) {
+    const qs = sourceDirectory
+      ? `?sourceDirectory=${encodeURIComponent(sourceDirectory)}`
+      : '';
+    return this.#requestJson(`/api/uploader/datastage/pending${qs}`, 'GET');
+  }
+
+  async getStats(sourceDirectory = null) {
+    const qs = sourceDirectory
+      ? `?sourceDirectory=${encodeURIComponent(sourceDirectory)}`
+      : '';
+    return this.#requestJson(`/api/uploader/datastage/stats${qs}`, 'GET');
+  }
+
+  async markUploaded(id, { datastageId, folio }) {
+    return this.#requestJson(
+      `/api/uploader/datastage/${id}/mark-uploaded`,
+      'PATCH',
+      { datastageId, folio },
+    );
+  }
+
+  async markFailed(id, error) {
+    return this.#requestJson(
+      `/api/uploader/datastage/${id}/mark-failed`,
+      'PATCH',
+      { error: String(error || 'unknown') },
+    );
+  }
+
+  // --- Zip upload ---
+
+  /**
+   * Upload a single zip file to POST /api/datastage (multipart, field name 'zipFile').
+   * Returns the created Datastage row { id, folio, ... }.
+   */
+  async uploadZip(localPath) {
+    const url = `${this.baseUrl}/api/datastage`;
+    const form = new FormData();
+    const fileName = path.basename(localPath);
+    form.append('zipFile', fs.createReadStream(localPath), {
+      filename: fileName,
+      contentType: 'application/zip',
+    });
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'x-api-key': this.token,
+        ...form.getHeaders(),
+      },
+      body: form,
+      agent: this.#getAgent(url),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      let msg = `Datastage upload failed: ${response.status} ${response.statusText}`;
+      try {
+        const j = JSON.parse(text);
+        msg = j.message || msg;
+      } catch {
+        msg = text || msg;
+      }
+      const err = new Error(msg);
+      err.status = response.status;
+      throw err;
+    }
+    return await response.json();
+  }
+}
+
+export default DatastageApiService;

package/src/services/GoogleDriveService.js

@@ -0,0 +1,217 @@
+import fs from 'fs';
+import { google } from 'googleapis';
+
+import appConfig from '../config/config.js';
+import PathNormalizer from '../utils/PathNormalizer.js';
+import logger from './LoggingService.js';
+
+const NATIVE_DOC_PREFIX = 'application/vnd.google-apps.';
+const FOLDER_MIME = 'application/vnd.google-apps.folder';
+const SHORTCUT_MIME = 'application/vnd.google-apps.shortcut';
+
+const DEFAULT_FILE_FIELDS =
+  'id,name,mimeType,modifiedTime,size,md5Checksum,parents,trashed,shortcutDetails';
+
+/**
+ * Google Drive Service
+ * Thin wrapper around googleapis Drive v3 with Service Account auth.
+ * Read-only by design (scope: drive.readonly).
+ */
+export class GoogleDriveService {
+  constructor() {
+    this.config = appConfig.getGDriveConfig();
+    this.drive = null;
+    this.commonParams = {
+      supportsAllDrives: true,
+      includeItemsFromAllDrives: true,
+    };
+  }
+
+  /**
+   * Lazy-init Drive v3 client.
+   * @returns {import('googleapis').drive_v3.Drive}
+   */
+  async getClient() {
+    if (this.drive) return this.drive;
+
+    let credentials = null;
+    let keyFile = null;
+
+    if (this.config.serviceAccountJson) {
+      try {
+        credentials = JSON.parse(this.config.serviceAccountJson);
+      } catch (err) {
+        throw new Error(
+          `GDRIVE_SERVICE_ACCOUNT_JSON is not valid JSON: ${err.message}`,
+        );
+      }
+    } else if (this.config.serviceAccountFile) {
+      keyFile = PathNormalizer.toAbsolutePath(this.config.serviceAccountFile);
+      if (!fs.existsSync(keyFile)) {
+        throw new Error(`Service account file not found: ${keyFile}`);
+      }
+    } else {
+      throw new Error(
+        'No Google Drive credentials configured (GDRIVE_SERVICE_ACCOUNT_FILE or GDRIVE_SERVICE_ACCOUNT_JSON)',
+      );
+    }
+
+    const auth = new google.auth.GoogleAuth({
+      credentials,
+      keyFile,
+      scopes: ['https://www.googleapis.com/auth/drive.readonly'],
+    });
+
+    this.drive = google.drive({ version: 'v3', auth });
+    return this.drive;
+  }
+
+  /**
+   * Get metadata for a single file/folder.
+   */
+  async getFile(fileId, fields = DEFAULT_FILE_FIELDS) {
+    const drive = await this.getClient();
+    const res = await drive.files.get({
+      fileId,
+      fields,
+      ...this.commonParams,
+    });
+    return res.data;
+  }
+
+  /**
+   * List direct children of a folder, paginated.
+   * Yields each child file metadata one at a time.
+   * @param {string} folderId
+   * @param {Object} [options]
+   * @param {string} [options.fields]
+   * @param {number} [options.pageSize]
+   * @returns {AsyncGenerator<Object>}
+   */
+  async *listChildren(folderId, options = {}) {
+    const drive = await this.getClient();
+    const fields = options.fields || DEFAULT_FILE_FIELDS;
+    const pageSize = options.pageSize || this.config.pageSize || 1000;
+
+    let pageToken = undefined;
+
+    do {
+      const res = await this.#withRetry(() =>
+        drive.files.list({
+          q: `'${folderId}' in parents and trashed = false`,
+          fields: `nextPageToken, files(${fields})`,
+          pageSize,
+          pageToken,
+          orderBy: 'folder,name',
+          ...this.commonParams,
+        }),
+      );
+
+      const files = res.data.files || [];
+      for (const file of files) {
+        yield file;
+      }
+      pageToken = res.data.nextPageToken;
+    } while (pageToken);
+  }
+
+  /**
+   * Download a file's binary content to a writable stream.
+   * Returns total bytes written.
+   * @param {string} fileId
+   * @param {import('stream').Writable} destStream
+   * @returns {Promise<number>}
+   */
+  async downloadFile(fileId, destStream) {
+    const drive = await this.getClient();
+
+    const res = await this.#withRetry(() =>
+      drive.files.get(
+        {
+          fileId,
+          alt: 'media',
+          ...this.commonParams,
+        },
+        { responseType: 'stream' },
+      ),
+    );
+
+    return await new Promise((resolve, reject) => {
+      let bytes = 0;
+      res.data.on('data', (chunk) => {
+        bytes += chunk.length;
+      });
+      res.data.on('error', reject);
+      destStream.on('error', reject);
+      destStream.on('finish', () => resolve(bytes));
+      res.data.pipe(destStream);
+    });
+  }
+
+  /**
+   * Static helpers
+   */
+  static isFolder(file) {
+    return file?.mimeType === FOLDER_MIME;
+  }
+
+  static isShortcut(file) {
+    return file?.mimeType === SHORTCUT_MIME;
+  }
+
+  static isNativeGoogleDoc(file) {
+    return (
+      typeof file?.mimeType === 'string' &&
+      file.mimeType.startsWith(NATIVE_DOC_PREFIX) &&
+      !GoogleDriveService.isFolder(file) &&
+      !GoogleDriveService.isShortcut(file)
+    );
+  }
+
+  /**
+   * Retry wrapper with exponential backoff for transient errors.
+   * @private
+   */
+  async #withRetry(fn, maxAttempts = 5) {
+    let attempt = 0;
+    let lastErr;
+
+    while (attempt < maxAttempts) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastErr = err;
+        const status = err?.code || err?.response?.status;
+        const reason =
+          err?.errors?.[0]?.reason ||
+          err?.response?.data?.error?.errors?.[0]?.reason;
+
+        const retryable =
+          status === 429 ||
+          status === 500 ||
+          status === 502 ||
+          status === 503 ||
+          status === 504 ||
+          reason === 'rateLimitExceeded' ||
+          reason === 'userRateLimitExceeded' ||
+          reason === 'backendError';
+
+        if (!retryable || attempt === maxAttempts - 1) {
+          throw err;
+        }
+
+        const delayMs =
+          Math.min(60_000, 2 ** attempt * 1000) + Math.random() * 500;
+        logger.warn(
+          `⚠️  Drive API ${status || ''} ${reason || ''} — retry ${attempt + 1}/${maxAttempts} in ${Math.round(delayMs)}ms`,
+        );
+        await new Promise((r) => setTimeout(r, delayMs));
+        attempt += 1;
+      }
+    }
+
+    throw lastErr;
+  }
+}
+
+export default GoogleDriveService;

package/src/services/ScanApiService.js

@@ -375,6 +375,20 @@ export class ScanApiService {
    * @param {boolean} allTypes - When true, fetch all supported file types instead of just likely-simplificado PDFs
    * @returns {Promise<Object>} { data: Array, hasMore: boolean }
    */
+  /**
+   * Get a single file record by ID (for single-file identify mode).
+   * @param {string} tableName - Scan table name (with or without cli. prefix)
+   * @param {string} fileId - UUID of the file record
+   * @returns {Promise<{ id: string, file_name: string, file_extension: string, absolute_path: string }>}
+   */
+  async getFileRecord(tableName, fileId) {
+    const cleanTable = tableName.replace(/^cli\./, '');
+    const url = `/api/uploader/scan/file-record?tableName=${encodeURIComponent(cleanTable)}&fileId=${encodeURIComponent(fileId)}`;
+    const result = await this.#request(url, 'GET');
+    logger.debug(`Fetched file record ${fileId} from ${cleanTable}`);
+    return result;
+  }
+
   async fetchPdfsForDetection(
     tableName,
     offset = 0,
@@ -398,6 +412,22 @@ export class ScanApiService {
     return result;
   }
 
+  /**
+   * Reset detection_attempts to 0 for undetected files so they can be re-processed.
+   * @param {string} tableName - Target scan table name
+   * @param {string|null} absolutePath - If provided, reset only this specific file
+   * @returns {Promise<{ reset: number }>}
+   */
+  async resetDetectionAttempts(tableName, absolutePath = null) {
+    let url = `/api/uploader/scan/reset-detection-attempts?tableName=${encodeURIComponent(tableName)}`;
+    if (absolutePath) {
+      url += `&absolutePath=${encodeURIComponent(absolutePath)}`;
+    }
+    const result = await this.#request(url, 'PATCH');
+    logger.debug(`Reset ${result.reset} detection attempt(s) in ${tableName}`);
+    return result;
+  }
+
   /**
    * Batch update detection results
    * @param {string} tableName - Target table name