@elisra-devops/docgen-data-provider 1.18.0 → 1.19.0

package/src/helpers/tfs.ts

@@ -1,16 +1,34 @@
-import axios from 'axios';
+import axios, { AxiosInstance, AxiosRequestConfig } from 'axios';
 import logger from '../utils/logger';
 
 export class TFSServices {
+  // Connection pooling
+  private static connectionPool = {
+    httpAgent: new (require('http').Agent)({
+      keepAlive: true,
+      maxSockets: 50, // Allow more connections
+      keepAliveMsecs: 30000, // Keep connections alive longer
+    }),
+    httpsAgent: new (require('https').Agent)({
+      keepAlive: true,
+      maxSockets: 50,
+      keepAliveMsecs: 30000,
+    }),
+  };
+
+  // Axios instance with connection reuse
+  private static axiosInstance: AxiosInstance = axios.create({
+    httpAgent: this.connectionPool.httpAgent,
+    httpsAgent: this.connectionPool.httpsAgent,
+  });
+
   public static async downloadZipFile(url: string, pat: string): Promise<any> {
     try {
-      let res = await axios.request({
+      const res = await this.axiosInstance.request({
         url: url,
         headers: { 'Content-Type': 'application/zip' },
-        auth: {
-          username: '',
-          password: pat,
-        },
+        auth: { username: '', password: pat },
+        timeout: 15000, // Increased timeout for large files
       });
       return res;
     } catch (e) {
@@ -27,52 +45,22 @@ export class TFSServices {
     customHeaders: any = {},
     printError: boolean = true
   ): Promise<any> {
-    let config: any = {
+    const config: AxiosRequestConfig = {
       headers: customHeaders,
       method: requestMethod,
-      auth: {
-        username: '',
-        password: pat,
-      },
+      auth: { username: '', password: pat },
       data: data,
       responseType: 'arraybuffer', // Important for binary data
-      timeout: 5000, // Set timeout to 5 seconds
+      timeout: 8000, // Increased timeout for images
     };
-    let json;
-    let attempts = 0;
-    const maxAttempts = 3;
-
-    logger.silly(`making request:
-    url: ${url}
-    config: ${JSON.stringify(config)}`);
 
-    while (attempts < maxAttempts) {
-      try {
-        const response = await axios(url, config);
-
-        // Convert binary data to Base64
-        const base64String = Buffer.from(response.data, 'binary').toString('base64');
-        const contentType = response.headers['content-type']; // e.g., "image/png; api-version=7.1"
-        const mimeType = contentType.split(';')[0].trim(); // Extracts "image/png"
-        return `data:${mimeType};base64,${base64String}`;
-      } catch (e: any) {
-        attempts++;
-        if (e.message.includes('ETIMEDOUT') && attempts < maxAttempts) {
-          logger.warn(`Request timed out. Retrying attempt ${attempts} of ${maxAttempts}...`);
-          continue;
-        }
-        if (printError) {
-          if (e.response) {
-            logger.error(`Error fetching image from Azure DevOps at ${url}: ${e.message}`);
-            logger.error(`Status: ${e.response.status}`);
-            logger.error(`Response Data: ${JSON.stringify(e.response.data)}`);
-          } else {
-            logger.error(`Error fetching image from Azure DevOps at ${url}: ${e.message}`);
-          }
-        }
-        throw e;
-      }
-    }
+    return this.executeWithRetry(url, config, printError, (response) => {
+      // Convert binary data to Base64
+      const base64String = Buffer.from(response.data, 'binary').toString('base64');
+      const contentType = response.headers['content-type'] || 'application/octet-stream';
+      const mimeType = contentType.split(';')[0].trim();
+      return `data:${mimeType};base64,${base64String}`;
+    });
   }
 
   public static async getItemContent(
@@ -83,87 +71,37 @@ export class TFSServices {
     customHeaders: any = {},
     printError: boolean = true
   ): Promise<any> {
-    let config: any = {
+    // Clean URL
+    const cleanUrl = url.replace(/ /g, '%20');
+
+    const config: AxiosRequestConfig = {
       headers: customHeaders,
       method: requestMethod,
-      auth: {
-        username: '',
-        password: pat,
-      },
+      auth: { username: '', password: pat },
       data: data,
-      timeout: 3000, // Set timeout to 3 seconds
+      timeout: 10000, // More reasonable timeout
     };
-    let json;
-    let attempts = 0;
-    const maxAttempts = 3;
-    url = url.replace(/ /g, '%20');
-    logger.silly(`making request:
-    url: ${url}
-    config: ${JSON.stringify(config)}`);
-
-    while (attempts < maxAttempts) {
-      try {
-        let result = await axios(url, config);
-        json = JSON.parse(JSON.stringify(result.data));
-        return json;
-      } catch (e: any) {
-        logger.warn(`error fetching item content from azure devops at ${url}`);
-        logger.warn(`error: ${JSON.stringify(e.response?.data?.message)}`);
-        if (e.response?.data?.message.includes('could not be found')) {
-          logger.info(`File does not exist, or you do not have permissions to read it.`);
-          return undefined;
-        }
 
-        attempts++;
-        if (
-          e.message.includes('ETIMEDOUT') ||
-          e.message.includes('timeout') ||
-          (e.response?.data?.message?.includes('timeout') && attempts < maxAttempts)
-        ) {
-          logger.warn(`Request timed out. Retrying attempt ${attempts} of ${maxAttempts}...`);
-          continue;
-        }
-        if (printError) {
-          if (e.response) {
-            // Log detailed error information including the URL
-            logger.error(`Error making request to Azure DevOps at ${url}: ${e.message}`);
-            logger.error(`Status: ${e.response.status}`);
-            logger.error(`Response Data: ${JSON.stringify(e.response.data?.message)}`);
-          } else {
-            // Handle other errors (network, etc.)
-            logger.error(`Error making request to Azure DevOps at ${url}: ${e.message}`);
-          }
-        }
-        throw e;
-      }
-    }
+    return this.executeWithRetry(cleanUrl, config, printError, (response) => {
+      // Direct return of data without extra JSON parsing
+      return response.data;
+    });
   }
 
   public static async getJfrogRequest(url: string, header?: any) {
-    let config: any = {
+    const config: AxiosRequestConfig = {
       method: 'GET',
+      headers: header,
+      timeout: 8000, // Reasonable timeout
     };
-    if (header) {
-      config['headers'] = header;
-    }
 
-    let json;
     try {
-      let result = await axios(url, config);
-      json = JSON.parse(JSON.stringify(result.data));
+      const result = await this.axiosInstance.request(config);
+      return result.data;
     } catch (e: any) {
-      if (e.response) {
-        // Log detailed error information including the URL
-        logger.error(`Error making request Jfrog at ${url}: ${e.message}`);
-        logger.error(`Status: ${e.response.status}`);
-        logger.error(`Response Data: ${JSON.stringify(e.response.data)}`);
-      } else {
-        // Handle other errors (network, etc.)
-        logger.error(`Error making request to Jfrog at ${url}: ${e.message}`);
-      }
+      this.logDetailedError(e, url);
       throw e;
     }
-    return json;
   }
 
   public static async postRequest(
@@ -173,32 +111,129 @@ export class TFSServices {
     data: any,
     customHeaders: any = { headers: { 'Content-Type': 'application/json' } }
   ): Promise<any> {
-    let config: any = {
+    const config: AxiosRequestConfig = {
       headers: customHeaders,
       method: requestMethod,
-      auth: {
-        username: '',
-        password: pat,
-      },
+      auth: { username: '', password: pat },
       data: data,
+      timeout: 10000, // More reasonable timeout
     };
-    let result;
-    logger.silly(`making request:
-    url: ${url}
-    config: ${JSON.stringify(config)}`);
+
+    // Use shorter log format for better performance
+    logger.silly(`Request: ${url} [${requestMethod}]`);
+
     try {
-      result = await axios(url, config);
+      const result = await this.axiosInstance.request(config);
+      return result;
     } catch (e: any) {
-      if (e.response) {
-        // Log detailed error information including the URL
-        logger.error(`Error making request to Azure DevOps at ${url}: ${e.message}`);
-        logger.error(`Status: ${e.response.status}`);
-        logger.error(`Response Data: ${JSON.stringify(e.response.data?.message)}`);
-      } else {
-        // Handle other errors (network, etc.)
-        logger.error(`Error making request to Azure DevOps at ${url}: ${e.message}`);
+      this.logDetailedError(e, url);
+      throw e;
+    }
+  }
+
+  /**
+   * Execute a request with intelligent retry logic
+   */
+  private static async executeWithRetry(
+    url: string,
+    config: AxiosRequestConfig,
+    printError: boolean,
+    responseProcessor: (response: any) => any
+  ): Promise<any> {
+    let attempts = 0;
+    const maxAttempts = 3;
+    const baseDelay = 500; // Start with 500ms delay
+
+    while (true) {
+      try {
+        const result = await this.axiosInstance.request({ ...config, url });
+        return responseProcessor(result);
+      } catch (e: any) {
+        attempts++;
+        const errorMessage = this.getErrorMessage(e);
+
+        // Handle not found errors
+        if (errorMessage.includes('could not be found')) {
+          logger.info(`File does not exist, or you do not have permissions to read it.`);
+          return undefined;
+        }
+
+        // Check if we should retry
+        if (attempts < maxAttempts && this.isRetryableError(e)) {
+          // Calculate exponential backoff with jitter
+          const jitter = Math.random() * 0.3 + 0.85; // Between 0.85 and 1.15
+          const delay = Math.min(baseDelay * Math.pow(2, attempts - 1) * jitter, 5000);
+
+          logger.warn(`Request failed. Retrying in ${Math.round(delay)}ms (${attempts}/${maxAttempts})`);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+          continue;
+        }
+
+        // Log error if needed
+        if (printError) {
+          this.logDetailedError(e, url);
+        }
+
+        throw e;
       }
     }
-    return result;
+  }
+
+  /**
+   * Check if an error is retryable
+   */
+  private static isRetryableError(error: any): boolean {
+    // Network errors
+    if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT' || error.message.includes('timeout')) {
+      return true;
+    }
+
+    // Server errors (5xx)
+    if (error.response?.status >= 500) {
+      return true;
+    }
+
+    // Rate limiting (429)
+    if (error.response?.status === 429) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Log detailed error information
+   */
+  private static logDetailedError(error: any, url: string): void {
+    if (error.response) {
+      logger.error(`Error for ${url}: ${error.message}`);
+      logger.error(`Status: ${error.response.status}`);
+
+      if (error.response.data) {
+        if (typeof error.response.data === 'string') {
+          logger.error(`Response: ${error.response.data.substring(0, 200)}`);
+        } else {
+          const dataMessage =
+            error.response.data.message || JSON.stringify(error.response.data).substring(0, 200);
+          logger.error(`Response: ${dataMessage}`);
+        }
+      }
+    } else {
+      logger.error(`Error for ${url}: ${error.message}`);
+    }
+  }
+
+  private static getErrorMessage(error: any): string {
+    if (error.response?.data?.message) {
+      return JSON.stringify(error.response.data.message);
+    } else if (error.response?.data) {
+      return JSON.stringify(error.response.data);
+    } else if (error.response) {
+      return `HTTP ${error.response.status}`;
+    } else if (error.message) {
+      return error.message;
+    } else {
+      return 'Unknown error occurred';
+    }
   }
 }