@jackchuka/gql-ingest 1.3.0 → 1.4.0

package/dist/mapper.d.ts

@@ -18,6 +18,9 @@ export declare class DataMapper {
     private processRowsConcurrently;
     private chunkArray;
     private mapCsvRowToVariables;
+    private extractVariableTypes;
+    private extractTypeName;
+    private convertValue;
     getMetrics(): MetricsCollector;
 }
 //# sourceMappingURL=mapper.d.ts.map

package/dist/mapper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YA+BvB,uBAAuB;IAmErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAe5B,UAAU,IAAI,gBAAgB;CAG/B"}
+{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAsDpB,UAAU,IAAI,gBAAgB;CAG/B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jackchuka/gql-ingest",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "A CLI tool for ingesting data from CSV files into a GraphQL API",
   "type": "module",
   "main": "dist/cli.js",

package/src/mapper.test.ts

@@ -331,5 +331,226 @@ describe("DataMapper", () => {
       const metrics = dataMapper.getMetrics();
       expect(metrics).toBe(mockMetrics);
     });
+
+    it("should convert numeric types from CSV strings to proper GraphQL types", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          price: "product_price",
+          quantity: "product_quantity",
+          active: "product_active",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          product_price: "19.99",
+          product_quantity: "10",
+          product_active: "true",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $price: Float!, $quantity: Int!, $active: Boolean!) {
+          createProduct(input: { name: $name, price: $price, quantity: $quantity, active: $active }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        price: 19.99,
+        quantity: 10,
+        active: true,
+      }, undefined);
+    });
+
+    it("should handle invalid numeric conversions gracefully", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          price: "product_price",
+          quantity: "product_quantity",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          product_price: "invalid_price",
+          product_quantity: "invalid_quantity",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $price: Float!, $quantity: Int!) {
+          createProduct(input: { name: $name, price: $price, quantity: $quantity }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        price: "invalid_price",
+        quantity: "invalid_quantity",
+      }, undefined);
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Warning: Cannot convert "invalid_price" to Float for variable $price. Expected a valid number. Using original value.'
+      );
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Warning: Cannot convert "invalid_quantity" to Int for variable $quantity. Expected a valid integer. Using original value.'
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should handle edge cases in numeric conversion safely", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          int_field: "int_value",
+          float_field: "float_value",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          int_value: "1.5", // Float in Int field - should remain string
+          float_value: "Infinity", // Invalid float - should remain string
+        },
+        {
+          int_value: "not_a_number", // Invalid int - should remain string
+          float_value: "1.2.3", // Invalid number format - should remain string
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($int_field: Int!, $float_field: Float!) {
+          createProduct(input: { int_field: $int_field, float_field: $float_field }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      await dataMapper.processEntity("configs/test/mappings/products.json");
+
+      // Should keep invalid values as strings
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        int_field: "1.5",
+        float_field: "Infinity",
+      }, undefined);
+
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        int_field: "not_a_number",
+        float_field: "1.2.3",
+      }, undefined);
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should keep unknown scalar types as strings", async () => {
+      const mockConfig = {
+        csvFile: "data/products.csv",
+        graphqlFile: "graphql/products.graphql",
+        mapping: {
+          name: "product_name",
+          custom_field: "custom_value",
+        },
+      };
+
+      const mockCsvData = [
+        {
+          product_name: "Widget",
+          custom_value: "123",
+        },
+      ];
+
+      const mockMutation = `
+        mutation CreateProduct($name: String!, $custom_field: CustomScalar!) {
+          createProduct(input: { name: $name, custom_field: $custom_field }) {
+            id
+          }
+        }
+      `;
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createProduct: { id: "123" },
+      });
+
+      // Create verbose mapper to test the logging
+      const verboseMapper = new DataMapper(mockClient, testBasePath, mockMetrics, true);
+      const consoleSpy = jest.spyOn(console, "log").mockImplementation();
+
+      await verboseMapper.processEntity("configs/test/mappings/products.json");
+
+      // Should keep custom scalar as string
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
+        name: "Widget",
+        custom_field: "123",
+      }, undefined);
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        'Unknown GraphQL type "CustomScalar" for variable $custom_field. Keeping value as string.'
+      );
+
+      consoleSpy.mockRestore();
+    });
   });
 });

package/src/mapper.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
 import path from "path";
+import { parse, DocumentNode, VariableDefinitionNode } from "graphql";
 import { readCsvFile, CsvRow } from "./csv-reader";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
@@ -104,24 +105,33 @@ export class DataMapper {
     retryConfig?: RetryConfig
   ): Promise<void> {
     const totalRows = csvData.length;
-    
+    const variableTypes = this.extractVariableTypes(mutation);
+
     for (let i = 0; i < csvData.length; i++) {
       const row = csvData[i];
-      const variables = this.mapCsvRowToVariables(row, mapping);
+      const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
 
       try {
         await this.client.executeMutation(mutation, variables, retryConfig);
         this.metrics.recordSuccess(entityName);
-        
+
         // Show progress every 10% or at the end (only in non-verbose mode)
-        if (!this.verbose && ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1)) {
+        if (
+          !this.verbose &&
+          ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 ||
+            i === totalRows - 1)
+        ) {
           const progress = (((i + 1) / totalRows) * 100).toFixed(1);
           console.log(`📊 Progress: ${i + 1}/${totalRows} (${progress}%) ✓`);
         }
       } catch (error) {
         this.metrics.recordFailure(entityName);
         if (!this.verbose) {
-          console.error(`✗ Failed to create entity for row ${i + 1}:`, row, error);
+          console.error(
+            `✗ Failed to create entity for row ${i + 1}:`,
+            row,
+            error
+          );
         }
       }
     }
@@ -140,6 +150,9 @@ export class DataMapper {
       `Processing ${csvData.length} rows with concurrency: ${concurrency}`
     );
 
+    // Extract variable types once for all rows
+    const variableTypes = this.extractVariableTypes(mutation);
+
     // Split data into chunks for concurrent processing
     const chunks = this.chunkArray(csvData, concurrency);
     let processedCount = 0;
@@ -148,10 +161,14 @@ export class DataMapper {
     for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
       const chunk = chunks[chunkIndex];
       const promises = chunk.map(async (row) => {
-        const variables = this.mapCsvRowToVariables(row, mapping);
+        const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
 
         try {
-          const result = await this.client.executeMutation(mutation, variables, retryConfig);
+          const result = await this.client.executeMutation(
+            mutation,
+            variables,
+            retryConfig
+          );
           this.metrics.recordSuccess(entityName);
           return { success: true, result, row };
         } catch (error) {
@@ -166,7 +183,7 @@ export class DataMapper {
       // Count successes and failures in this chunk
       let chunkSuccesses = 0;
       let chunkFailures = 0;
-      
+
       results.forEach((result) => {
         if (result.status === "fulfilled") {
           const { success, error, row } = result.value;
@@ -189,7 +206,11 @@ export class DataMapper {
       // Show progress update (only in non-verbose mode)
       if (!this.verbose) {
         const progress = ((processedCount / totalRows) * 100).toFixed(1);
-        console.log(`📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${chunkIndex + 1}: ${chunkSuccesses} ✓, ${chunkFailures} ✗`);
+        console.log(
+          `📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${
+            chunkIndex + 1
+          }: ${chunkSuccesses} ✓, ${chunkFailures} ✗`
+        );
       }
     }
   }
@@ -204,19 +225,120 @@ export class DataMapper {
 
   private mapCsvRowToVariables(
     row: CsvRow,
-    mapping: Record<string, string>
+    mapping: Record<string, string>,
+    variableTypes: Record<string, string>
   ): Record<string, any> {
     const variables: Record<string, any> = {};
 
     for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
       if (row[csvColumn] !== undefined) {
-        variables[graphqlVar] = row[csvColumn];
+        const rawValue = row[csvColumn];
+        const type = variableTypes[graphqlVar];
+        variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
       }
     }
 
     return variables;
   }
 
+  private extractVariableTypes(mutation: string): Record<string, string> {
+    const types: Record<string, string> = {};
+
+    try {
+      const document: DocumentNode = parse(mutation);
+
+      // Find the operation (mutation/query) and extract variable definitions
+      for (const definition of document.definitions) {
+        if (
+          definition.kind === "OperationDefinition" &&
+          definition.variableDefinitions
+        ) {
+          for (const variableDef of definition.variableDefinitions) {
+            const varName = variableDef.variable.name.value;
+            const typeName = this.extractTypeName(variableDef);
+            if (typeName) {
+              types[varName] = typeName;
+            }
+          }
+        }
+      }
+    } catch (error) {
+      console.error("Error parsing GraphQL mutation:", error);
+    }
+
+    return types;
+  }
+
+  private extractTypeName(variableDef: VariableDefinitionNode): string | null {
+    const type = variableDef.type;
+
+    if (type.kind === "NonNullType") {
+      // Handle non-null types like String!
+      if (type.type.kind === "NamedType") {
+        return type.type.name.value;
+      }
+    } else if (type.kind === "NamedType") {
+      // Handle nullable types like String
+      return type.name.value;
+    }
+
+    return null;
+  }
+
+  private convertValue(value: string, type: string | undefined, varName: string): any {
+    if (!type) {
+      // No type information available, keep as string
+      return value;
+    }
+
+    const trimmedValue = value.trim();
+
+    switch (type) {
+      case "Int":
+        const intValue = Number(trimmedValue);
+        // Validate that it's a valid integer (no decimals, NaN, or Infinity)
+        if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {
+          console.warn(
+            `Warning: Cannot convert "${value}" to Int for variable $${varName}. Expected a valid integer. Using original value.`
+          );
+          return value;
+        }
+        return intValue;
+
+      case "Float":
+        const floatValue = Number(trimmedValue);
+        // Number() is more strict than parseFloat() - it requires the entire string to be valid
+        if (isNaN(floatValue) || !isFinite(floatValue)) {
+          console.warn(
+            `Warning: Cannot convert "${value}" to Float for variable $${varName}. Expected a valid number. Using original value.`
+          );
+          return value;
+        }
+        return floatValue;
+
+      case "Boolean":
+        const lowerValue = trimmedValue.toLowerCase();
+        if (lowerValue === "true" || lowerValue === "1") return true;
+        if (lowerValue === "false" || lowerValue === "0") return false;
+        console.warn(
+          `Warning: Cannot convert "${value}" to Boolean for variable $${varName}. Expected "true", "false", "1", or "0". Using original value.`
+        );
+        return value;
+
+      case "String":
+        return value;
+
+      default:
+        // Unknown scalar type - keep as string for safety
+        if (this.verbose) {
+          console.log(
+            `Unknown GraphQL type "${type}" for variable $${varName}. Keeping value as string.`
+          );
+        }
+        return value;
+    }
+  }
+
   getMetrics(): MetricsCollector {
     return this.metrics;
   }