@jackchuka/gql-ingest 1.2.0 → 1.4.0

package/src/mapper.test.ts

@@ -141,11 +141,11 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "Jane",
         email: "jane@example.com",
-      });
+      }, undefined);
 
       consoleSpy.mockRestore();
     });
@@ -223,7 +223,7 @@ describe("DataMapper", () => {
         name: "Widget",
         price: "19.99",
         sku: "W001",
-      });
+      }, undefined);
     });
 
     it("should handle missing CSV columns gracefully", async () => {
@@ -260,7 +260,7 @@ describe("DataMapper", () => {
       expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
         name: "John",
         email: "john@example.com",
-      });
+      }, undefined);
     });
 
     it("should call metrics methods during successful processing", async () => {
@@ -331,5 +331,226 @@ describe("DataMapper", () => {
       const metrics = dataMapper.getMetrics();
       expect(metrics).toBe(mockMetrics);
     });
+
+    it("should convert numeric types from CSV strings to proper GraphQL types", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          price: "product_price",
+          quantity: "product_quantity",
+          active: "product_active",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          product_price: "19.99",
+          product_quantity: "10",
+          product_active: "true",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $price: Float!, $quantity: Int!, $active: Boolean!) {
+          createProduct(input: { name: $name, price: $price, quantity: $quantity, active: $active }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        price: 19.99,
+        quantity: 10,
+        active: true,
+      }, undefined);
+    });
+
+    it("should handle invalid numeric conversions gracefully", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          price: "product_price",
+          quantity: "product_quantity",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          product_price: "invalid_price",
+          product_quantity: "invalid_quantity",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $price: Float!, $quantity: Int!) {
+          createProduct(input: { name: $name, price: $price, quantity: $quantity }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        price: "invalid_price",
+        quantity: "invalid_quantity",
+      }, undefined);
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Warning: Cannot convert "invalid_price" to Float for variable $price. Expected a valid number. Using original value.'
+      );
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Warning: Cannot convert "invalid_quantity" to Int for variable $quantity. Expected a valid integer. Using original value.'
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should handle edge cases in numeric conversion safely", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          int_field: "int_value",
+          float_field: "float_value",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          int_value: "1.5", // Float in Int field - should remain string
+          float_value: "Infinity", // Invalid float - should remain string
+        },
+        {
+          int_value: "not_a_number", // Invalid int - should remain string
+          float_value: "1.2.3", // Invalid number format - should remain string
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($int_field: Int!, $float_field: Float!) {
+          createProduct(input: { int_field: $int_field, float_field: $float_field }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      // Should keep invalid values as strings
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        int_field: "1.5",
+        float_field: "Infinity",
+      }, undefined);
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        int_field: "not_a_number",
+        float_field: "1.2.3",
+      }, undefined);
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should keep unknown scalar types as strings", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          custom_field: "custom_value",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          custom_value: "123",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $custom_field: CustomScalar!) {
+          createProduct(input: { name: $name, custom_field: $custom_field }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      // Create verbose mapper to test the logging
+      const verboseMapper = new DataMapper(mockClient, testBasePath, mockMetrics, true);
+      const consoleSpy = jest.spyOn(console, "log").mockImplementation();
+
+      await verboseMapper.processEntity("configs/test/mappings/products.json");
+
+      // Should keep custom scalar as string
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        custom_field: "123",
+      }, undefined);
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Unknown GraphQL type "CustomScalar" for variable $custom_field. Keeping value as string.'
+      );
+
+      consoleSpy.mockRestore();
+    });
   });
 });

package/src/mapper.ts

@@ -1,9 +1,10 @@
 import fs from "fs";
 import path from "path";
+import { parse, DocumentNode, VariableDefinitionNode } from "graphql";
 import { readCsvFile, CsvRow } from "./csv-reader";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
-import { ParallelProcessingConfig } from "./config";
+import { ParallelProcessingConfig, RetryConfig } from "./config";
 
 export interface MappingConfig {
   csvFile: string;
@@ -48,7 +49,8 @@ export class DataMapper {
 
   async processEntity(
     configPath: string,
-    parallelConfig?: ParallelProcessingConfig
+    parallelConfig?: ParallelProcessingConfig,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const entityName = path.basename(configPath, ".json");
     console.log(`Processing entity: ${configPath}`);
@@ -79,14 +81,16 @@ export class DataMapper {
         mutation,
         config.mapping,
         entityName,
-        parallelConfig
+        parallelConfig,
+        retryConfig
       );
     } else {
       await this.processRowsSequentially(
         csvData,
         mutation,
         config.mapping,
-        entityName
+        entityName,
+        retryConfig
       );
     }
 
@@ -97,27 +101,37 @@ export class DataMapper {
     csvData: CsvRow[],
     mutation: string,
     mapping: Record<string, string>,
-    entityName: string
+    entityName: string,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const totalRows = csvData.length;
-    
+    const variableTypes = this.extractVariableTypes(mutation);
+
     for (let i = 0; i < csvData.length; i++) {
       const row = csvData[i];
-      const variables = this.mapCsvRowToVariables(row, mapping);
+      const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
 
       try {
-        await this.client.executeMutation(mutation, variables);
+        await this.client.executeMutation(mutation, variables, retryConfig);
         this.metrics.recordSuccess(entityName);
-        
+
         // Show progress every 10% or at the end (only in non-verbose mode)
-        if (!this.verbose && ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1)) {
+        if (
+          !this.verbose &&
+          ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 ||
+            i === totalRows - 1)
+        ) {
           const progress = (((i + 1) / totalRows) * 100).toFixed(1);
           console.log(`📊 Progress: ${i + 1}/${totalRows} (${progress}%) ✓`);
         }
       } catch (error) {
         this.metrics.recordFailure(entityName);
         if (!this.verbose) {
-          console.error(`✗ Failed to create entity for row ${i + 1}:`, row, error);
+          console.error(
+            `✗ Failed to create entity for row ${i + 1}:`,
+            row,
+            error
+          );
         }
       }
     }
@@ -128,13 +142,17 @@ export class DataMapper {
     mutation: string,
     mapping: Record<string, string>,
     entityName: string,
-    parallelConfig: ParallelProcessingConfig
+    parallelConfig: ParallelProcessingConfig,
+    retryConfig?: RetryConfig
   ): Promise<void> {
     const concurrency = parallelConfig.concurrency;
     console.log(
       `Processing ${csvData.length} rows with concurrency: ${concurrency}`
     );
 
+    // Extract variable types once for all rows
+    const variableTypes = this.extractVariableTypes(mutation);
+
     // Split data into chunks for concurrent processing
     const chunks = this.chunkArray(csvData, concurrency);
     let processedCount = 0;
@@ -143,10 +161,14 @@ export class DataMapper {
     for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
       const chunk = chunks[chunkIndex];
       const promises = chunk.map(async (row) => {
-        const variables = this.mapCsvRowToVariables(row, mapping);
+        const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
 
         try {
-          const result = await this.client.executeMutation(mutation, variables);
+          const result = await this.client.executeMutation(
+            mutation,
+            variables,
+            retryConfig
+          );
           this.metrics.recordSuccess(entityName);
           return { success: true, result, row };
         } catch (error) {
@@ -161,7 +183,7 @@ export class DataMapper {
       // Count successes and failures in this chunk
       let chunkSuccesses = 0;
       let chunkFailures = 0;
-      
+
       results.forEach((result) => {
         if (result.status === "fulfilled") {
           const { success, error, row } = result.value;
@@ -184,7 +206,11 @@ export class DataMapper {
       // Show progress update (only in non-verbose mode)
       if (!this.verbose) {
         const progress = ((processedCount / totalRows) * 100).toFixed(1);
-        console.log(`📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${chunkIndex + 1}: ${chunkSuccesses} ✓, ${chunkFailures} ✗`);
+        console.log(
+          `📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${
+            chunkIndex + 1
+          }: ${chunkSuccesses} ✓, ${chunkFailures} ✗`
+        );
       }
     }
   }
@@ -199,19 +225,120 @@ export class DataMapper {
 
   private mapCsvRowToVariables(
     row: CsvRow,
-    mapping: Record<string, string>
+    mapping: Record<string, string>,
+    variableTypes: Record<string, string>
   ): Record<string, any> {
     const variables: Record<string, any> = {};
 
     for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
       if (row[csvColumn] !== undefined) {
-        variables[graphqlVar] = row[csvColumn];
+        const rawValue = row[csvColumn];
+        const type = variableTypes[graphqlVar];
+        variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
       }
     }
 
     return variables;
   }
 
+  private extractVariableTypes(mutation: string): Record<string, string> {
+    const types: Record<string, string> = {};
+
+    try {
+      const document: DocumentNode = parse(mutation);
+
+      // Find the operation (mutation/query) and extract variable definitions
+      for (const definition of document.definitions) {
+        if (
+          definition.kind === "OperationDefinition" &&
+          definition.variableDefinitions
+        ) {
+          for (const variableDef of definition.variableDefinitions) {
+            const varName = variableDef.variable.name.value;
+            const typeName = this.extractTypeName(variableDef);
+            if (typeName) {
+              types[varName] = typeName;
+            }
+          }
+        }
+      }
+    } catch (error) {
+      console.error("Error parsing GraphQL mutation:", error);
+    }
+
+    return types;
+  }
+
+  private extractTypeName(variableDef: VariableDefinitionNode): string | null {
+    const type = variableDef.type;
+
+    if (type.kind === "NonNullType") {
+      // Handle non-null types like String!
+      if (type.type.kind === "NamedType") {
+        return type.type.name.value;
+      }
+    } else if (type.kind === "NamedType") {
+      // Handle nullable types like String
+      return type.name.value;
+    }
+
+    return null;
+  }
+
+  private convertValue(value: string, type: string | undefined, varName: string): any {
+    if (!type) {
+      // No type information available, keep as string
+      return value;
+    }
+
+    const trimmedValue = value.trim();
+
+    switch (type) {
+      case "Int":
+        const intValue = Number(trimmedValue);
+        // Validate that it's a valid integer (no decimals, NaN, or Infinity)
+        if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {
+          console.warn(
+            `Warning: Cannot convert "${value}" to Int for variable $${varName}. Expected a valid integer. Using original value.`
+          );
+          return value;
+        }
+        return intValue;
+
+      case "Float":
+        const floatValue = Number(trimmedValue);
+        // Number() is more strict than parseFloat() - it requires the entire string to be valid
+        if (isNaN(floatValue) || !isFinite(floatValue)) {
+          console.warn(
+            `Warning: Cannot convert "${value}" to Float for variable $${varName}. Expected a valid number. Using original value.`
+          );
+          return value;
+        }
+        return floatValue;
+
+      case "Boolean":
+        const lowerValue = trimmedValue.toLowerCase();
+        if (lowerValue === "true" || lowerValue === "1") return true;
+        if (lowerValue === "false" || lowerValue === "0") return false;
+        console.warn(
+          `Warning: Cannot convert "${value}" to Boolean for variable $${varName}. Expected "true", "false", "1", or "0". Using original value.`
+        );
+        return value;
+
+      case "String":
+        return value;
+
+      default:
+        // Unknown scalar type - keep as string for safety
+        if (this.verbose) {
+          console.log(
+            `Unknown GraphQL type "${type}" for variable $${varName}. Keeping value as string.`
+          );
+        }
+        return value;
+    }
+  }
+
   getMetrics(): MetricsCollector {
     return this.metrics;
   }

package/src/metrics.ts

@@ -12,6 +12,9 @@ export interface ProcessingMetrics {
   totalFailures: number;
   entityMetrics: Map<string, EntityMetrics>;
   requestDurations: number[];
+  retryAttempts: number;
+  retrySuccesses: number;
+  retryFailures: number;
   startTime: number;
   endTime?: number;
 }
@@ -26,6 +29,9 @@ export class MetricsCollector {
       totalFailures: 0,
       entityMetrics: new Map(),
       requestDurations: [],
+      retryAttempts: 0,
+      retrySuccesses: 0,
+      retryFailures: 0,
       startTime: Date.now(),
     };
   }
@@ -88,6 +94,16 @@ export class MetricsCollector {
     this.metrics.requestDurations.push(duration);
   }
 
+  recordRetrySuccess(attempts: number): void {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retrySuccesses++;
+  }
+
+  recordRetryFailure(attempts: number): void {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retryFailures++;
+  }
+
   getAverageRequestDuration(): number {
     if (this.metrics.requestDurations.length === 0) return 0;
     const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);
@@ -114,6 +130,12 @@ export class MetricsCollector {
     if (this.metrics.requestDurations.length > 0) {
       summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
     }
+    
+    if (this.metrics.retryAttempts > 0) {
+      summary += `   Retry Attempts: ${this.metrics.retryAttempts}\n`;
+      summary += `   Retry Successes: ${this.metrics.retrySuccesses}\n`;
+      summary += `   Retry Failures: ${this.metrics.retryFailures}\n`;
+    }
 
     if (this.metrics.entityMetrics.size > 1) {
       summary += `\n📋 Per-Entity Breakdown:\n`;