@jackchuka/gql-ingest 1.2.0 → 1.3.0

package/src/cli.ts

@@ -2,7 +2,7 @@ import { Command } from "commander";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { DataMapper } from "./mapper";
 import { MetricsCollector } from "./metrics";
-import { loadConfig, getEntityConfig } from "./config";
+import { loadConfig, getEntityConfig, getRetryConfig } from "./config";
 import { DependencyResolver } from "./dependency-resolver";
 import { basename } from "path";
 
@@ -114,7 +114,8 @@ async function processEntitiesSequentially(
     try {
       const entityName = basename(configPath, ".json");
       const entityConfig = getEntityConfig(entityName, config);
-      await mapper.processEntity(configPath, entityConfig);
+      const retryConfig = getRetryConfig(entityName, config);
+      await mapper.processEntity(configPath, entityConfig, retryConfig);
     } catch (error) {
       console.warn(`Warning: Could not process ${configPath}:`, error);
     }
@@ -142,14 +143,15 @@ async function processEntitiesInWaves(
     // Process entities in controlled batches based on entityConcurrency
     const entityConcurrency = config.parallelProcessing.entityConcurrency;
     const chunks = chunkArray(wave.entities, entityConcurrency);
-    
+
     for (const chunk of chunks) {
       const entityPromises = chunk.map(async (entityName) => {
         const configPath = pathMap.get(entityName);
         if (configPath) {
           try {
             const entityConfig = getEntityConfig(entityName, config);
-            await mapper.processEntity(configPath, entityConfig);
+            const retryConfig = getRetryConfig(entityName, config);
+            await mapper.processEntity(configPath, entityConfig, retryConfig);
           } catch (error) {
             console.warn(`Warning: Could not process ${configPath}:`, error);
           }

package/src/config.test.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
+import { loadConfig, getEntityConfig, getRetryConfig, DEFAULT_CONFIG } from "./config";
 
 jest.mock("fs");
 const mockFs = fs as jest.Mocked<typeof fs>;
@@ -112,6 +112,13 @@ entityConfig:
 
   describe("getEntityConfig", () => {
     const globalConfig = {
+      retry: {
+        maxAttempts: 3,
+        baseDelay: 1000,
+        maxDelay: 30000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+      },
       parallelProcessing: {
         concurrency: 10,
         entityConcurrency: 3,
@@ -201,4 +208,65 @@ entityConfig:
       consoleSpy.mockRestore();
     });
   });
+
+  describe("getRetryConfig", () => {
+    const globalConfig = {
+      retry: {
+        maxAttempts: 3,
+        baseDelay: 1000,
+        maxDelay: 30000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+      },
+      parallelProcessing: {
+        concurrency: 10,
+        entityConcurrency: 3,
+        preserveRowOrder: false,
+      },
+      entityConfig: {
+        important: {
+          retry: {
+            maxAttempts: 5,
+            baseDelay: 500,
+          },
+        },
+        fast: {
+          retry: {
+            maxAttempts: 1,
+          },
+        },
+      },
+      entityDependencies: {},
+    };
+
+    it("should return global retry config for entity without overrides", () => {
+      const retryConfig = getRetryConfig("regular", globalConfig);
+
+      expect(retryConfig).toEqual(globalConfig.retry);
+    });
+
+    it("should merge entity retry overrides with global config", () => {
+      const retryConfig = getRetryConfig("important", globalConfig);
+
+      expect(retryConfig.maxAttempts).toBe(5); // overridden
+      expect(retryConfig.baseDelay).toBe(500); // overridden
+      expect(retryConfig.maxDelay).toBe(30000); // from global
+      expect(retryConfig.exponentialBackoff).toBe(true); // from global
+      expect(retryConfig.retryableStatusCodes).toEqual([408, 429, 500, 502, 503, 504]); // from global
+    });
+
+    it("should handle partial retry overrides", () => {
+      const retryConfig = getRetryConfig("fast", globalConfig);
+
+      expect(retryConfig.maxAttempts).toBe(1); // overridden
+      expect(retryConfig.baseDelay).toBe(1000); // from global
+      expect(retryConfig.maxDelay).toBe(30000); // from global
+    });
+
+    it("should handle entity with no retry config", () => {
+      const retryConfig = getRetryConfig("undefined-entity", globalConfig);
+
+      expect(retryConfig).toEqual(globalConfig.retry);
+    });
+  });
 });

package/src/config.ts

@@ -8,12 +8,22 @@ export interface ParallelProcessingConfig {
   preserveRowOrder: boolean;
 }
 
+export interface RetryConfig {
+  maxAttempts: number;
+  baseDelay: number;
+  maxDelay: number;
+  exponentialBackoff: boolean;
+  retryableStatusCodes: number[];
+}
+
 export interface EntityConfig {
   concurrency?: number;
   preserveRowOrder?: boolean;
+  retry?: Partial<RetryConfig>;
 }
 
 export interface ProcessingConfig {
+  retry: RetryConfig;
   parallelProcessing: ParallelProcessingConfig;
   entityConfig: Record<string, EntityConfig>;
   entityDependencies: Record<string, string[]>;
@@ -23,6 +33,14 @@ export interface FullConfig extends ProcessingConfig {
   // Future: additional config sections can be added here
 }
 
+export const DEFAULT_RETRY_CONFIG: RetryConfig = {
+  maxAttempts: 3,
+  baseDelay: 1000,
+  maxDelay: 30000,
+  exponentialBackoff: true,
+  retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+};
+
 export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
   concurrency: 1,
   entityConcurrency: 1,
@@ -30,6 +48,7 @@ export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
 };
 
 export const DEFAULT_CONFIG: ProcessingConfig = {
+  retry: DEFAULT_RETRY_CONFIG,
   parallelProcessing: DEFAULT_PARALLEL_CONFIG,
   entityConfig: {},
   entityDependencies: {},
@@ -58,6 +77,10 @@ export function loadConfig(configDir: string): ProcessingConfig {
 
 function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
   return {
+    retry: {
+      ...DEFAULT_RETRY_CONFIG,
+      ...(yamlConfig.retry || {}),
+    },
     parallelProcessing: {
       ...DEFAULT_PARALLEL_CONFIG,
       ...(yamlConfig.parallelProcessing || {}),
@@ -88,3 +111,15 @@ export function getEntityConfig(
 
   return finalConfig;
 }
+
+export function getRetryConfig(
+  entityName: string,
+  globalConfig: ProcessingConfig
+): RetryConfig {
+  const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
+
+  return {
+    ...globalConfig.retry,
+    ...entityOverrides,
+  };
+}

package/src/graphql-client.test.ts

@@ -63,7 +63,7 @@ describe("GraphQLClientWrapper", () => {
       clientWrapper.executeMutation(mutation, variables)
     ).rejects.toThrow("GraphQL error");
 
-    expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed:", error);
+    expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed after 3 attempts:", error);
 
     consoleSpy.mockRestore();
   });
@@ -75,4 +75,130 @@ describe("GraphQLClientWrapper", () => {
 
     expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
   });
+
+  describe("retry functionality", () => {
+    it("should retry on retryable status codes", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest
+        .mockRejectedValueOnce(serverError)
+        .mockRejectedValueOnce(serverError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+      expect(result).toEqual({ data: { result: "success" } });
+    });
+
+    it("should not retry on non-retryable status codes", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const clientError = new Error("Bad Request");
+      (clientError as any).response = { status: 400 };
+
+      mockRequest.mockRejectedValueOnce(clientError);
+
+      await expect(
+        clientWrapper.executeMutation(mutation, variables, retryConfig)
+      ).rejects.toThrow("Bad Request");
+
+      expect(mockRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it("should retry on network errors (no response)", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 2,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const networkError = new Error("Network Error");
+      // No response property = network error
+
+      mockRequest
+        .mockRejectedValueOnce(networkError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+
+      expect(mockRequest).toHaveBeenCalledTimes(2);
+      expect(result).toEqual({ data: { result: "success" } });
+    });
+
+    it("should fail after max attempts are exhausted", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 2,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest.mockRejectedValue(serverError);
+
+      await expect(
+        clientWrapper.executeMutation(mutation, variables, retryConfig)
+      ).rejects.toThrow("Server Error");
+
+      expect(mockRequest).toHaveBeenCalledTimes(2);
+    });
+
+    it("should use exponential backoff when configured", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest
+        .mockRejectedValueOnce(serverError)
+        .mockRejectedValueOnce(serverError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const startTime = Date.now();
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+      const totalTime = Date.now() - startTime;
+
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+      expect(result).toEqual({ data: { result: "success" } });
+      // Should have some delay for retries (base + exponential)
+      expect(totalTime).toBeGreaterThan(200); // At least 100ms base + 200ms exponential
+    });
+  });
 });

package/src/graphql-client.ts

@@ -1,51 +1,151 @@
-import { GraphQLClient } from 'graphql-request';
-import { MetricsCollector } from './metrics';
+import { GraphQLClient } from "graphql-request";
+import { MetricsCollector } from "./metrics";
+import { DEFAULT_RETRY_CONFIG, RetryConfig } from "./config";
 
 export class GraphQLClientWrapper {
   private client: GraphQLClient;
   private metrics?: MetricsCollector;
   private verbose: boolean;
 
-  constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose: boolean = false) {
+  constructor(
+    endpoint: string,
+    headers?: Record<string, string>,
+    metrics?: MetricsCollector,
+    verbose: boolean = false
+  ) {
     this.client = new GraphQLClient(endpoint, {
-      headers: headers || {}
+      headers: headers || {},
     });
     this.metrics = metrics;
     this.verbose = verbose;
   }
 
-  async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
-    const startTime = Date.now();
-    
-    try {
-      const result = await this.client.request(mutation, variables);
-      
-      if (this.metrics) {
-        const duration = Date.now() - startTime;
-        this.metrics.recordRequestDuration(duration);
-      }
-      
-      if (this.verbose) {
-        console.log(`✓ GraphQL request completed in ${Date.now() - startTime}ms:`, result);
-      }
-      
-      return result;
-    } catch (error) {
-      if (this.metrics) {
-        const duration = Date.now() - startTime;
-        this.metrics.recordRequestDuration(duration);
-      }
-      
-      if (this.verbose) {
-        console.error(`✗ GraphQL request failed in ${Date.now() - startTime}ms:`, error);
-      } else {
-        console.error('GraphQL mutation failed:', error);
+  async executeMutation(
+    mutation: string,
+    variables: Record<string, any>,
+    retryConfig?: RetryConfig
+  ): Promise<any> {
+    const config = retryConfig || DEFAULT_RETRY_CONFIG;
+
+    let lastError: any;
+    const totalStartTime = Date.now();
+
+    for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
+      const attemptStartTime = Date.now();
+
+      try {
+        const result = await this.client.request(mutation, variables);
+
+        if (this.metrics) {
+          const duration = Date.now() - attemptStartTime;
+          this.metrics.recordRequestDuration(duration);
+          if (attempt > 0) {
+            this.metrics.recordRetrySuccess(attempt);
+          }
+        }
+
+        if (this.verbose) {
+          const totalDuration = Date.now() - totalStartTime;
+          const retryInfo =
+            attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : "";
+          console.log(
+            `✓ GraphQL request completed in ${totalDuration}ms${retryInfo}:`,
+            result
+          );
+        }
+
+        return result;
+      } catch (error: any) {
+        lastError = error;
+        const duration = Date.now() - attemptStartTime;
+
+        if (this.metrics) {
+          this.metrics.recordRequestDuration(duration);
+        }
+
+        // Check if this is the last attempt
+        if (attempt === config.maxAttempts - 1) {
+          if (this.metrics && attempt > 0) {
+            this.metrics.recordRetryFailure(attempt);
+          }
+          break;
+        }
+
+        // Check if error is retryable
+        if (!this.isRetryableError(error, config)) {
+          if (this.verbose) {
+            console.error(
+              `✗ GraphQL request failed with non-retryable error in ${duration}ms:`,
+              error
+            );
+          } else {
+            console.error("GraphQL mutation failed (non-retryable):", error);
+          }
+          throw error;
+        }
+
+        // Calculate delay
+        const delay = this.calculateDelay(attempt, config);
+
+        if (this.verbose) {
+          console.log(
+            `⏳ GraphQL request failed (attempt ${attempt + 1}/${
+              config.maxAttempts
+            }), retrying in ${delay}ms...`
+          );
+        }
+
+        // Wait before retry
+        await this.sleep(delay);
       }
-      throw error;
     }
+
+    // All retries exhausted
+    if (this.verbose) {
+      const totalDuration = Date.now() - totalStartTime;
+      console.error(
+        `✗ GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms:`,
+        lastError
+      );
+    } else {
+      console.error(
+        `GraphQL mutation failed after ${config.maxAttempts} attempts:`,
+        lastError
+      );
+    }
+
+    throw lastError;
+  }
+
+  private isRetryableError(error: any, config: RetryConfig): boolean {
+    // Network errors (no response)
+    if (!error.response) {
+      return true;
+    }
+
+    // Check HTTP status codes
+    const status = error.response.status;
+    return config.retryableStatusCodes.includes(status);
+  }
+
+  private calculateDelay(attempt: number, config: RetryConfig): number {
+    if (!config.exponentialBackoff) {
+      return config.baseDelay;
+    }
+
+    const exponentialDelay = config.baseDelay * Math.pow(2, attempt);
+    const cappedDelay = Math.min(exponentialDelay, config.maxDelay);
+
+    // Add jitter (±20% randomization)
+    const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);
+    return Math.max(0, cappedDelay + jitter);
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
   }
 
   setHeaders(headers: Record<string, string>) {
     this.client.setHeaders(headers);
   }
-}
+}

package/src/mapper.test.ts

@@ -141,11 +141,11 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "Jane",
         email: "jane@example.com",
-      });
+      }, undefined);
 
       consoleSpy.mockRestore();
     });
@@ -223,7 +223,7 @@ describe("DataMapper", () => {
         name: "Widget",
         price: "19.99",
         sku: "W001",
-      });
+      }, undefined);
     });
 
     it("should handle missing CSV columns gracefully", async () => {
@@ -260,7 +260,7 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
     });
 
     it("should call metrics methods during successful processing", async () => {

package/src/mapper.ts

@@ -3,7 +3,7 @@ import path from "path";
 import { readCsvFile, CsvRow } from "./csv-reader";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
-import { ParallelProcessingConfig } from "./config";
+import { ParallelProcessingConfig, RetryConfig } from "./config";
 
 export interface MappingConfig {
   csvFile: string;
@@ -48,7 +48,8 @@ export class DataMapper {
 
   async processEntity(
     configPath: string,
-    parallelConfig?: ParallelProcessingConfig
+    parallelConfig?: ParallelProcessingConfig,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const entityName = path.basename(configPath, ".json");
     console.log(`Processing entity: ${configPath}`);
@@ -79,14 +80,16 @@ export class DataMapper {
         mutation,
         config.mapping,
         entityName,
-        parallelConfig
+        parallelConfig,
+        retryConfig
       );
     } else {
       await this.processRowsSequentially(
         csvData,
         mutation,
         config.mapping,
-        entityName
+        entityName,
+        retryConfig
       );
     }
 
@@ -97,7 +100,8 @@ export class DataMapper {
     csvData: CsvRow[],
     mutation: string,
     mapping: Record<string, string>,
-    entityName: string
+    entityName: string,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const totalRows = csvData.length;
     
@@ -106,7 +110,7 @@ export class DataMapper {
       const variables = this.mapCsvRowToVariables(row, mapping);
 
       try {
-        await this.client.executeMutation(mutation, variables);
+        await this.client.executeMutation(mutation, variables, retryConfig);
         this.metrics.recordSuccess(entityName);
         
         // Show progress every 10% or at the end (only in non-verbose mode)
@@ -128,7 +132,8 @@ export class DataMapper {
     mutation: string,
     mapping: Record<string, string>,
     entityName: string,
-    parallelConfig: ParallelProcessingConfig
+    parallelConfig: ParallelProcessingConfig,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const concurrency = parallelConfig.concurrency;
     console.log(
@@ -146,7 +151,7 @@ export class DataMapper {
         const variables = this.mapCsvRowToVariables(row, mapping);
 
         try {
-          const result = await this.client.executeMutation(mutation, variables);
+          const result = await this.client.executeMutation(mutation, variables, retryConfig);
           this.metrics.recordSuccess(entityName);
           return { success: true, result, row };
         } catch (error) {

package/src/metrics.ts

@@ -12,6 +12,9 @@ export interface ProcessingMetrics {
   totalFailures: number;
   entityMetrics: Map<string, EntityMetrics>;
   requestDurations: number[];
+  retryAttempts: number;
+  retrySuccesses: number;
+  retryFailures: number;
   startTime: number;
   endTime?: number;
 }
@@ -26,6 +29,9 @@ export class MetricsCollector {
       totalFailures: 0,
       entityMetrics: new Map(),
       requestDurations: [],
+      retryAttempts: 0,
+      retrySuccesses: 0,
+      retryFailures: 0,
       startTime: Date.now(),
     };
   }
@@ -88,6 +94,16 @@ export class MetricsCollector {
     this.metrics.requestDurations.push(duration);
   }
 
+  recordRetrySuccess(attempts: number): void {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retrySuccesses++;
+  }
+
+  recordRetryFailure(attempts: number): void {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retryFailures++;
+  }
+
   getAverageRequestDuration(): number {
     if (this.metrics.requestDurations.length === 0) return 0;
     const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);
@@ -114,6 +130,12 @@ export class MetricsCollector {
     if (this.metrics.requestDurations.length > 0) {
       summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
     }
+    
+    if (this.metrics.retryAttempts > 0) {
+      summary += `   Retry Attempts: ${this.metrics.retryAttempts}\n`;
+      summary += `   Retry Successes: ${this.metrics.retrySuccesses}\n`;
+      summary += `   Retry Failures: ${this.metrics.retryFailures}\n`;
+    }
 
     if (this.metrics.entityMetrics.size > 1) {
       summary += `\n📋 Per-Entity Breakdown:\n`;