@jackchuka/gql-ingest 1.1.0 → 1.3.0

package/src/cli.ts

@@ -2,10 +2,20 @@ import { Command } from "commander";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { DataMapper } from "./mapper";
 import { MetricsCollector } from "./metrics";
-import { loadConfig, getEntityConfig } from "./config";
+import { loadConfig, getEntityConfig, getRetryConfig } from "./config";
 import { DependencyResolver } from "./dependency-resolver";
 import { basename } from "path";
 
+// Utility function to chunk array into smaller arrays
+function chunkArray<T>(array: T[], chunkSize: number): T[][] {
+  if (chunkSize <= 0) return [array];
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
+
 const program = new Command();
 
 program
@@ -81,10 +91,10 @@ program
       }
 
       // Process entities in dependency-aware waves
-      if (config.parallelProcessing.enableEntityParallelization) {
-        await processEntitiesInWaves(mappingPaths, resolver, mapper, config);
-      } else {
+      if (config.parallelProcessing.entityConcurrency === 1) {
         await processEntitiesSequentially(mappingPaths, mapper, config);
+      } else {
+        await processEntitiesInWaves(mappingPaths, resolver, mapper, config);
       }
 
       metrics.finishProcessing();
@@ -104,7 +114,8 @@ async function processEntitiesSequentially(
     try {
       const entityName = basename(configPath, ".json");
       const entityConfig = getEntityConfig(entityName, config);
-      await mapper.processEntity(configPath, entityConfig);
+      const retryConfig = getRetryConfig(entityName, config);
+      await mapper.processEntity(configPath, entityConfig, retryConfig);
     } catch (error) {
       console.warn(`Warning: Could not process ${configPath}:`, error);
     }
@@ -129,27 +140,18 @@ async function processEntitiesInWaves(
       `Wave ${wave.wave + 1}: Processing entities [${wave.entities.join(", ")}]`
     );
 
-    if (config.parallelProcessing.preserveEntityOrder) {
-      // Process entities in this wave sequentially
-      for (const entityName of wave.entities) {
-        const configPath = pathMap.get(entityName);
-        if (configPath) {
-          try {
-            const entityConfig = getEntityConfig(entityName, config);
-            await mapper.processEntity(configPath, entityConfig);
-          } catch (error) {
-            console.warn(`Warning: Could not process ${configPath}:`, error);
-          }
-        }
-      }
-    } else {
-      // Process entities in this wave concurrently
-      const entityPromises = wave.entities.map(async (entityName) => {
+    // Process entities in controlled batches based on entityConcurrency
+    const entityConcurrency = config.parallelProcessing.entityConcurrency;
+    const chunks = chunkArray(wave.entities, entityConcurrency);
+
+    for (const chunk of chunks) {
+      const entityPromises = chunk.map(async (entityName) => {
         const configPath = pathMap.get(entityName);
         if (configPath) {
           try {
             const entityConfig = getEntityConfig(entityName, config);
-            await mapper.processEntity(configPath, entityConfig);
+            const retryConfig = getRetryConfig(entityName, config);
+            await mapper.processEntity(configPath, entityConfig, retryConfig);
           } catch (error) {
             console.warn(`Warning: Could not process ${configPath}:`, error);
           }

package/src/config.test.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
+import { loadConfig, getEntityConfig, getRetryConfig, DEFAULT_CONFIG } from "./config";
 
 jest.mock("fs");
 const mockFs = fs as jest.Mocked<typeof fs>;
@@ -32,7 +32,7 @@ describe("Configuration", () => {
       const yamlContent = `
 parallelProcessing:
   concurrency: 5
-  enableEntityParallelization: true
+  entityConcurrency: 3
 
 entityConfig:
   users:
@@ -49,7 +49,7 @@ entityDependencies:
       const config = loadConfig(testConfigDir);
 
       expect(config.parallelProcessing.concurrency).toBe(5);
-      expect(config.parallelProcessing.enableEntityParallelization).toBe(true);
+      expect(config.parallelProcessing.entityConcurrency).toBe(3);
       expect(config.entityConfig.users.concurrency).toBe(2);
       expect(config.entityConfig.users.preserveRowOrder).toBe(true);
       expect(config.entityDependencies.products).toEqual(["users"]);
@@ -71,7 +71,7 @@ entityConfig:
 
       // Should merge with defaults
       expect(config.parallelProcessing.concurrency).toBe(10);
-      expect(config.parallelProcessing.enableEntityParallelization).toBe(false); // default
+      expect(config.parallelProcessing.entityConcurrency).toBe(1); // default
       expect(config.parallelProcessing.preserveRowOrder).toBe(true); // default
     });
 
@@ -112,11 +112,17 @@ entityConfig:
 
   describe("getEntityConfig", () => {
     const globalConfig = {
+      retry: {
+        maxAttempts: 3,
+        baseDelay: 1000,
+        maxDelay: 30000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+      },
       parallelProcessing: {
         concurrency: 10,
-        enableEntityParallelization: true,
+        entityConcurrency: 3,
         preserveRowOrder: false,
-        preserveEntityOrder: false,
       },
       entityConfig: {
         users: {
@@ -140,7 +146,7 @@ entityConfig:
       const entityConfig = getEntityConfig("products", globalConfig);
 
       expect(entityConfig.concurrency).toBe(20); // overridden
-      expect(entityConfig.enableEntityParallelization).toBe(true); // from global
+      expect(entityConfig.entityConcurrency).toBe(3); // from global
       expect(entityConfig.preserveRowOrder).toBe(false); // from global
     });
 
@@ -202,4 +208,65 @@ entityConfig:
       consoleSpy.mockRestore();
     });
   });
+
+  describe("getRetryConfig", () => {
+    const globalConfig = {
+      retry: {
+        maxAttempts: 3,
+        baseDelay: 1000,
+        maxDelay: 30000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+      },
+      parallelProcessing: {
+        concurrency: 10,
+        entityConcurrency: 3,
+        preserveRowOrder: false,
+      },
+      entityConfig: {
+        important: {
+          retry: {
+            maxAttempts: 5,
+            baseDelay: 500,
+          },
+        },
+        fast: {
+          retry: {
+            maxAttempts: 1,
+          },
+        },
+      },
+      entityDependencies: {},
+    };
+
+    it("should return global retry config for entity without overrides", () => {
+      const retryConfig = getRetryConfig("regular", globalConfig);
+
+      expect(retryConfig).toEqual(globalConfig.retry);
+    });
+
+    it("should merge entity retry overrides with global config", () => {
+      const retryConfig = getRetryConfig("important", globalConfig);
+
+      expect(retryConfig.maxAttempts).toBe(5); // overridden
+      expect(retryConfig.baseDelay).toBe(500); // overridden
+      expect(retryConfig.maxDelay).toBe(30000); // from global
+      expect(retryConfig.exponentialBackoff).toBe(true); // from global
+      expect(retryConfig.retryableStatusCodes).toEqual([408, 429, 500, 502, 503, 504]); // from global
+    });
+
+    it("should handle partial retry overrides", () => {
+      const retryConfig = getRetryConfig("fast", globalConfig);
+
+      expect(retryConfig.maxAttempts).toBe(1); // overridden
+      expect(retryConfig.baseDelay).toBe(1000); // from global
+      expect(retryConfig.maxDelay).toBe(30000); // from global
+    });
+
+    it("should handle entity with no retry config", () => {
+      const retryConfig = getRetryConfig("undefined-entity", globalConfig);
+
+      expect(retryConfig).toEqual(globalConfig.retry);
+    });
+  });
 });

package/src/config.ts

@@ -4,17 +4,26 @@ import * as yaml from "js-yaml";
 
 export interface ParallelProcessingConfig {
   concurrency: number;
-  enableEntityParallelization: boolean;
+  entityConcurrency: number;
   preserveRowOrder: boolean;
-  preserveEntityOrder: boolean;
+}
+
+export interface RetryConfig {
+  maxAttempts: number;
+  baseDelay: number;
+  maxDelay: number;
+  exponentialBackoff: boolean;
+  retryableStatusCodes: number[];
 }
 
 export interface EntityConfig {
   concurrency?: number;
   preserveRowOrder?: boolean;
+  retry?: Partial<RetryConfig>;
 }
 
 export interface ProcessingConfig {
+  retry: RetryConfig;
   parallelProcessing: ParallelProcessingConfig;
   entityConfig: Record<string, EntityConfig>;
   entityDependencies: Record<string, string[]>;
@@ -24,14 +33,22 @@ export interface FullConfig extends ProcessingConfig {
   // Future: additional config sections can be added here
 }
 
+export const DEFAULT_RETRY_CONFIG: RetryConfig = {
+  maxAttempts: 3,
+  baseDelay: 1000,
+  maxDelay: 30000,
+  exponentialBackoff: true,
+  retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+};
+
 export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
   concurrency: 1,
-  enableEntityParallelization: false,
+  entityConcurrency: 1,
   preserveRowOrder: true,
-  preserveEntityOrder: true,
 };
 
 export const DEFAULT_CONFIG: ProcessingConfig = {
+  retry: DEFAULT_RETRY_CONFIG,
   parallelProcessing: DEFAULT_PARALLEL_CONFIG,
   entityConfig: {},
   entityDependencies: {},
@@ -60,6 +77,10 @@ export function loadConfig(configDir: string): ProcessingConfig {
 
 function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
   return {
+    retry: {
+      ...DEFAULT_RETRY_CONFIG,
+      ...(yamlConfig.retry || {}),
+    },
     parallelProcessing: {
       ...DEFAULT_PARALLEL_CONFIG,
       ...(yamlConfig.parallelProcessing || {}),
@@ -90,3 +111,15 @@ export function getEntityConfig(
 
   return finalConfig;
 }
+
+export function getRetryConfig(
+  entityName: string,
+  globalConfig: ProcessingConfig
+): RetryConfig {
+  const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
+
+  return {
+    ...globalConfig.retry,
+    ...entityOverrides,
+  };
+}

package/src/graphql-client.test.ts

@@ -63,7 +63,7 @@ describe("GraphQLClientWrapper", () => {
       clientWrapper.executeMutation(mutation, variables)
     ).rejects.toThrow("GraphQL error");
 
-    expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed:", error);
+    expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed after 3 attempts:", error);
 
     consoleSpy.mockRestore();
   });
@@ -75,4 +75,130 @@ describe("GraphQLClientWrapper", () => {
 
     expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
   });
+
+  describe("retry functionality", () => {
+    it("should retry on retryable status codes", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest
+        .mockRejectedValueOnce(serverError)
+        .mockRejectedValueOnce(serverError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+      expect(result).toEqual({ data: { result: "success" } });
+    });
+
+    it("should not retry on non-retryable status codes", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const clientError = new Error("Bad Request");
+      (clientError as any).response = { status: 400 };
+
+      mockRequest.mockRejectedValueOnce(clientError);
+
+      await expect(
+        clientWrapper.executeMutation(mutation, variables, retryConfig)
+      ).rejects.toThrow("Bad Request");
+
+      expect(mockRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it("should retry on network errors (no response)", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 2,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const networkError = new Error("Network Error");
+      // No response property = network error
+
+      mockRequest
+        .mockRejectedValueOnce(networkError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+
+      expect(mockRequest).toHaveBeenCalledTimes(2);
+      expect(result).toEqual({ data: { result: "success" } });
+    });
+
+    it("should fail after max attempts are exhausted", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 2,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: false,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest.mockRejectedValue(serverError);
+
+      await expect(
+        clientWrapper.executeMutation(mutation, variables, retryConfig)
+      ).rejects.toThrow("Server Error");
+
+      expect(mockRequest).toHaveBeenCalledTimes(2);
+    });
+
+    it("should use exponential backoff when configured", async () => {
+      const mutation = "mutation { createUser(name: $name) { id } }";
+      const variables = { name: "John" };
+      const retryConfig = {
+        maxAttempts: 3,
+        baseDelay: 100,
+        maxDelay: 1000,
+        exponentialBackoff: true,
+        retryableStatusCodes: [500],
+      };
+
+      const serverError = new Error("Server Error");
+      (serverError as any).response = { status: 500 };
+
+      mockRequest
+        .mockRejectedValueOnce(serverError)
+        .mockRejectedValueOnce(serverError)
+        .mockResolvedValueOnce({ data: { result: "success" } });
+
+      const startTime = Date.now();
+      const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
+      const totalTime = Date.now() - startTime;
+
+      expect(mockRequest).toHaveBeenCalledTimes(3);
+      expect(result).toEqual({ data: { result: "success" } });
+      // Should have some delay for retries (base + exponential)
+      expect(totalTime).toBeGreaterThan(200); // At least 100ms base + 200ms exponential
+    });
+  });
 });

package/src/graphql-client.ts

@@ -1,51 +1,151 @@
-import { GraphQLClient } from 'graphql-request';
-import { MetricsCollector } from './metrics';
+import { GraphQLClient } from "graphql-request";
+import { MetricsCollector } from "./metrics";
+import { DEFAULT_RETRY_CONFIG, RetryConfig } from "./config";
 
 export class GraphQLClientWrapper {
   private client: GraphQLClient;
   private metrics?: MetricsCollector;
   private verbose: boolean;
 
-  constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose: boolean = false) {
+  constructor(
+    endpoint: string,
+    headers?: Record<string, string>,
+    metrics?: MetricsCollector,
+    verbose: boolean = false
+  ) {
     this.client = new GraphQLClient(endpoint, {
-      headers: headers || {}
+      headers: headers || {},
     });
     this.metrics = metrics;
     this.verbose = verbose;
   }
 
-  async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
-    const startTime = Date.now();
-    
-    try {
-      const result = await this.client.request(mutation, variables);
-      
-      if (this.metrics) {
-        const duration = Date.now() - startTime;
-        this.metrics.recordRequestDuration(duration);
-      }
-      
-      if (this.verbose) {
-        console.log(`✓ GraphQL request completed in ${Date.now() - startTime}ms:`, result);
-      }
-      
-      return result;
-    } catch (error) {
-      if (this.metrics) {
-        const duration = Date.now() - startTime;
-        this.metrics.recordRequestDuration(duration);
-      }
-      
-      if (this.verbose) {
-        console.error(`✗ GraphQL request failed in ${Date.now() - startTime}ms:`, error);
-      } else {
-        console.error('GraphQL mutation failed:', error);
+  async executeMutation(
+    mutation: string,
+    variables: Record<string, any>,
+    retryConfig?: RetryConfig
+  ): Promise<any> {
+    const config = retryConfig || DEFAULT_RETRY_CONFIG;
+
+    let lastError: any;
+    const totalStartTime = Date.now();
+
+    for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
+      const attemptStartTime = Date.now();
+
+      try {
+        const result = await this.client.request(mutation, variables);
+
+        if (this.metrics) {
+          const duration = Date.now() - attemptStartTime;
+          this.metrics.recordRequestDuration(duration);
+          if (attempt > 0) {
+            this.metrics.recordRetrySuccess(attempt);
+          }
+        }
+
+        if (this.verbose) {
+          const totalDuration = Date.now() - totalStartTime;
+          const retryInfo =
+            attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : "";
+          console.log(
+            `✓ GraphQL request completed in ${totalDuration}ms${retryInfo}:`,
+            result
+          );
+        }
+
+        return result;
+      } catch (error: any) {
+        lastError = error;
+        const duration = Date.now() - attemptStartTime;
+
+        if (this.metrics) {
+          this.metrics.recordRequestDuration(duration);
+        }
+
+        // Check if this is the last attempt
+        if (attempt === config.maxAttempts - 1) {
+          if (this.metrics && attempt > 0) {
+            this.metrics.recordRetryFailure(attempt);
+          }
+          break;
+        }
+
+        // Check if error is retryable
+        if (!this.isRetryableError(error, config)) {
+          if (this.verbose) {
+            console.error(
+              `✗ GraphQL request failed with non-retryable error in ${duration}ms:`,
+              error
+            );
+          } else {
+            console.error("GraphQL mutation failed (non-retryable):", error);
+          }
+          throw error;
+        }
+
+        // Calculate delay
+        const delay = this.calculateDelay(attempt, config);
+
+        if (this.verbose) {
+          console.log(
+            `⏳ GraphQL request failed (attempt ${attempt + 1}/${
+              config.maxAttempts
+            }), retrying in ${delay}ms...`
+          );
+        }
+
+        // Wait before retry
+        await this.sleep(delay);
       }
-      throw error;
     }
+
+    // All retries exhausted
+    if (this.verbose) {
+      const totalDuration = Date.now() - totalStartTime;
+      console.error(
+        `✗ GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms:`,
+        lastError
+      );
+    } else {
+      console.error(
+        `GraphQL mutation failed after ${config.maxAttempts} attempts:`,
+        lastError
+      );
+    }
+
+    throw lastError;
+  }
+
+  private isRetryableError(error: any, config: RetryConfig): boolean {
+    // Network errors (no response)
+    if (!error.response) {
+      return true;
+    }
+
+    // Check HTTP status codes
+    const status = error.response.status;
+    return config.retryableStatusCodes.includes(status);
+  }
+
+  private calculateDelay(attempt: number, config: RetryConfig): number {
+    if (!config.exponentialBackoff) {
+      return config.baseDelay;
+    }
+
+    const exponentialDelay = config.baseDelay * Math.pow(2, attempt);
+    const cappedDelay = Math.min(exponentialDelay, config.maxDelay);
+
+    // Add jitter (±20% randomization)
+    const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);
+    return Math.max(0, cappedDelay + jitter);
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
   }
 
   setHeaders(headers: Record<string, string>) {
     this.client.setHeaders(headers);
   }
-}
+}

package/src/mapper.test.ts

@@ -141,11 +141,11 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "Jane",
         email: "jane@example.com",
-      });
+      }, undefined);
 
       consoleSpy.mockRestore();
     });
@@ -223,7 +223,7 @@ describe("DataMapper", () => {
         name: "Widget",
         price: "19.99",
         sku: "W001",
-      });
+      }, undefined);
     });
 
     it("should handle missing CSV columns gracefully", async () => {
@@ -260,7 +260,7 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
     });
 
     it("should call metrics methods during successful processing", async () => {