@jackchuka/gql-ingest 1.4.0 → 2.0.0

package/src/mapper.test.ts

@@ -3,9 +3,18 @@ import path from "path";
 import { DataMapper } from "./mapper";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
+import { readCsvFile, DataReaderFactory } from "./readers";
 
 jest.mock("fs");
-jest.mock("./csv-reader");
+jest.mock("./readers", () => ({
+  ...jest.requireActual("./readers"),
+  readCsvFile: jest.fn(),
+  DataReaderFactory: {
+    getReader: jest.fn().mockReturnValue({
+      readFile: jest.fn(),
+    }),
+  },
+}));
 
 const mockFs = fs as jest.Mocked<typeof fs>;
 
@@ -114,8 +123,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createUser: { id: "123" },
@@ -129,8 +138,9 @@ describe("DataMapper", () => {
         path.resolve(testBasePath, "configs/test/mappings/users.json"),
         "utf8"
       );
-      expect(readCsvFile).toHaveBeenCalledWith(
-        path.resolve(testBasePath, "configs/test", "data/users.csv")
+      expect(DataReaderFactory.getReader).toHaveBeenCalledWith(
+        path.resolve(testBasePath, "configs/test", "data/users.csv"),
+        undefined
       );
       expect(mockFs.readFileSync).toHaveBeenCalledWith(
         path.resolve(testBasePath, "configs/test", "graphql/users.graphql"),
@@ -138,14 +148,22 @@ describe("DataMapper", () => {
       );
 
       expect(mockClient.executeMutation).toHaveBeenCalledTimes(2);
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "John",
-        email: "john@example.com",
-      }, undefined);
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "Jane",
-        email: "jane@example.com",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "John",
+          email: "john@example.com",
+        },
+        undefined
+      );
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "Jane",
+          email: "jane@example.com",
+        },
+        undefined
+      );
 
       consoleSpy.mockRestore();
     });
@@ -165,8 +183,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockRejectedValue(new Error("GraphQL error"));
 
@@ -210,8 +228,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createProduct: { id: "456" },
@@ -219,11 +237,15 @@ describe("DataMapper", () => {
 
       await dataMapper.processEntity("configs/test/mappings/products.json");
 
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "Widget",
-        price: "19.99",
-        sku: "W001",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "Widget",
+          price: "19.99",
+          sku: "W001",
+        },
+        undefined
+      );
     });
 
     it("should handle missing CSV columns gracefully", async () => {
@@ -248,8 +270,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createUser: { id: "789" },
@@ -257,10 +279,14 @@ describe("DataMapper", () => {
 
       await dataMapper.processEntity("configs/test/mappings/users.json");
 
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "John",
-        email: "john@example.com",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "John",
+          email: "john@example.com",
+        },
+        undefined
+      );
     });
 
     it("should call metrics methods during successful processing", async () => {
@@ -278,8 +304,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createUser: { id: "123" },
@@ -309,8 +335,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockRejectedValue(new Error("GraphQL error"));
 
@@ -365,8 +391,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createProduct: { id: "123" },
@@ -374,12 +400,16 @@ describe("DataMapper", () => {
 
       await dataMapper.processEntity("configs/test/mappings/products.json");
 
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "Widget",
-        price: 19.99,
-        quantity: 10,
-        active: true,
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "Widget",
+          price: 19.99,
+          quantity: 10,
+          active: true,
+        },
+        undefined
+      );
     });
 
     it("should handle invalid numeric conversions gracefully", async () => {
@@ -413,8 +443,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createProduct: { id: "123" },
@@ -424,11 +454,15 @@ describe("DataMapper", () => {
 
       await dataMapper.processEntity("configs/test/mappings/products.json");
 
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "Widget",
-        price: "invalid_price",
-        quantity: "invalid_quantity",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "Widget",
+          price: "invalid_price",
+          quantity: "invalid_quantity",
+        },
+        undefined
+      );
 
       expect(consoleSpy).toHaveBeenCalledWith(
         'Warning: Cannot convert "invalid_price" to Float for variable $price. Expected a valid number. Using original value.'
@@ -473,8 +507,8 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createProduct: { id: "123" },
@@ -485,15 +519,23 @@ describe("DataMapper", () => {
       await dataMapper.processEntity("configs/test/mappings/products.json");
 
       // Should keep invalid values as strings
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        int_field: "1.5",
-        float_field: "Infinity",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          int_field: "1.5",
+          float_field: "Infinity",
+        },
+        undefined
+      );
 
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        int_field: "not_a_number",
-        float_field: "1.2.3",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          int_field: "not_a_number",
+          float_field: "1.2.3",
+        },
+        undefined
+      );
 
       consoleSpy.mockRestore();
     });
@@ -527,24 +569,33 @@ describe("DataMapper", () => {
         .mockReturnValueOnce(JSON.stringify(mockConfig))
         .mockReturnValueOnce(mockMutation);
 
-      const { readCsvFile } = require("./csv-reader");
-      readCsvFile.mockResolvedValue(mockCsvData);
+      const { DataReaderFactory } = require("./readers");
+      DataReaderFactory.getReader().readFile.mockResolvedValue(mockCsvData);
 
       mockClient.executeMutation.mockResolvedValue({
         createProduct: { id: "123" },
       });
 
       // Create verbose mapper to test the logging
-      const verboseMapper = new DataMapper(mockClient, testBasePath, mockMetrics, true);
+      const verboseMapper = new DataMapper(
+        mockClient,
+        testBasePath,
+        mockMetrics,
+        true
+      );
       const consoleSpy = jest.spyOn(console, "log").mockImplementation();
 
       await verboseMapper.processEntity("configs/test/mappings/products.json");
 
       // Should keep custom scalar as string
-      expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
-        name: "Widget",
-        custom_field: "123",
-      }, undefined);
+      expect(mockClient.executeMutation).toHaveBeenCalledWith(
+        mockMutation,
+        {
+          name: "Widget",
+          custom_field: "123",
+        },
+        undefined
+      );
 
       expect(consoleSpy).toHaveBeenCalledWith(
         'Unknown GraphQL type "CustomScalar" for variable $custom_field. Keeping value as string.'

package/src/mapper.ts

@@ -1,15 +1,19 @@
 import fs from "fs";
 import path from "path";
 import { parse, DocumentNode, VariableDefinitionNode } from "graphql";
-import { readCsvFile, CsvRow } from "./csv-reader";
+import { DataReaderFactory, DataRow } from "./readers";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
 import { ParallelProcessingConfig, RetryConfig } from "./config";
 
 export interface MappingConfig {
-  csvFile: string;
+  // Legacy CSV support
+  csvFile?: string;
+  // New flexible data file support
+  dataFile?: string;
+  dataFormat?: string;
   graphqlFile: string;
-  mapping: Record<string, string>;
+  mapping: Record<string, string | any>;
 }
 
 export class DataMapper {
@@ -17,25 +21,55 @@ export class DataMapper {
   private basePath: string;
   private metrics: MetricsCollector;
   private verbose: boolean;
+  private formatOverride?: string;
 
   constructor(
     client: GraphQLClientWrapper,
     basePath: string = process.cwd(),
     metrics?: MetricsCollector,
-    verbose: boolean = false
+    verbose: boolean = false,
+    formatOverride?: string
   ) {
     this.client = client;
     this.basePath = basePath;
     this.metrics = metrics || new MetricsCollector();
     this.verbose = verbose;
+    this.formatOverride = formatOverride;
   }
 
-  discoverMappings(configDir: string): string[] {
+  discoverMappings(configDir: string, entityFilter?: string[]): string[] {
     const mappingsPath = path.resolve(this.basePath, configDir, "mappings");
 
     try {
       const files = fs.readdirSync(mappingsPath);
-      const jsonFiles = files.filter((file) => file.endsWith(".json")).sort(); // Alphabetical order for consistent processing
+      let jsonFiles = files.filter((file) => file.endsWith(".json"));
+
+      // Apply entity filter if provided
+      if (entityFilter && entityFilter.length > 0) {
+        const requestedEntities = new Set(entityFilter);
+        const foundEntities = new Set<string>();
+
+        jsonFiles = jsonFiles.filter((file) => {
+          const entityName = path.basename(file, ".json");
+          if (requestedEntities.has(entityName)) {
+            foundEntities.add(entityName);
+            return true;
+          }
+          return false;
+        });
+
+        // Check for requested entities that were not found
+        const notFound = entityFilter.filter((e) => !foundEntities.has(e));
+        if (notFound.length > 0) {
+          console.warn(
+            `Warning: The following entities were not found in mappings: ${notFound.join(
+              ", "
+            )}`
+          );
+        }
+      }
+
+      jsonFiles.sort(); // Alphabetical order for consistent processing
 
       console.log(
         `Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`
@@ -66,9 +100,20 @@ export class DataMapper {
     // Extract config directory (parent of mappings directory)
     const configDir = path.dirname(path.dirname(configFullPath));
 
-    // Read CSV data (relative to config directory)
-    const csvPath = path.resolve(configDir, config.csvFile);
-    const csvData = await readCsvFile(csvPath);
+    // Determine data file path (support both legacy csvFile and new dataFile)
+    const dataFile = config.dataFile || config.csvFile;
+    if (!dataFile) {
+      throw new Error(
+        `No data file specified in mapping config: ${configPath}`
+      );
+    }
+
+    const dataPath = path.resolve(configDir, dataFile);
+
+    // Get appropriate reader (prioritize CLI format override, then config format)
+    const format = this.formatOverride || config.dataFormat;
+    const reader = DataReaderFactory.getReader(dataPath, format);
+    const data = await reader.readFile(dataPath);
 
     // Read GraphQL mutation (relative to config directory)
     const graphqlPath = path.resolve(configDir, config.graphqlFile);
@@ -77,7 +122,7 @@ export class DataMapper {
     // Process rows with optional parallelization
     if (parallelConfig && parallelConfig.concurrency > 1) {
       await this.processRowsConcurrently(
-        csvData,
+        data,
         mutation,
         config.mapping,
         entityName,
@@ -86,7 +131,7 @@ export class DataMapper {
       );
     } else {
       await this.processRowsSequentially(
-        csvData,
+        data,
         mutation,
         config.mapping,
         entityName,
@@ -98,18 +143,18 @@ export class DataMapper {
   }
 
   private async processRowsSequentially(
-    csvData: CsvRow[],
+    data: DataRow[],
     mutation: string,
-    mapping: Record<string, string>,
+    mapping: Record<string, string | any>,
     entityName: string,
     retryConfig?: RetryConfig
   ): Promise<void> {
-    const totalRows = csvData.length;
+    const totalRows = data.length;
     const variableTypes = this.extractVariableTypes(mutation);
 
-    for (let i = 0; i < csvData.length; i++) {
-      const row = csvData[i];
-      const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
+    for (let i = 0; i < data.length; i++) {
+      const row = data[i];
+      const variables = this.mapRowToVariables(row, mapping, variableTypes);
 
       try {
         await this.client.executeMutation(mutation, variables, retryConfig);
@@ -138,30 +183,30 @@ export class DataMapper {
   }
 
   private async processRowsConcurrently(
-    csvData: CsvRow[],
+    data: DataRow[],
     mutation: string,
-    mapping: Record<string, string>,
+    mapping: Record<string, string | any>,
     entityName: string,
     parallelConfig: ParallelProcessingConfig,
     retryConfig?: RetryConfig
   ): Promise<void> {
     const concurrency = parallelConfig.concurrency;
     console.log(
-      `Processing ${csvData.length} rows with concurrency: ${concurrency}`
+      `Processing ${data.length} rows with concurrency: ${concurrency}`
     );
 
     // Extract variable types once for all rows
     const variableTypes = this.extractVariableTypes(mutation);
 
     // Split data into chunks for concurrent processing
-    const chunks = this.chunkArray(csvData, concurrency);
+    const chunks = this.chunkArray(data, concurrency);
     let processedCount = 0;
-    const totalRows = csvData.length;
+    const totalRows = data.length;
 
     for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
       const chunk = chunks[chunkIndex];
       const promises = chunk.map(async (row) => {
-        const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
+        const variables = this.mapRowToVariables(row, mapping, variableTypes);
 
         try {
           const result = await this.client.executeMutation(
@@ -223,24 +268,110 @@ export class DataMapper {
     return chunks;
   }
 
-  private mapCsvRowToVariables(
-    row: CsvRow,
-    mapping: Record<string, string>,
+  private mapRowToVariables(
+    row: DataRow,
+    mapping: Record<string, string | any>,
     variableTypes: Record<string, string>
   ): Record<string, any> {
     const variables: Record<string, any> = {};
 
-    for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
-      if (row[csvColumn] !== undefined) {
-        const rawValue = row[csvColumn];
+    for (const [graphqlVar, mappingValue] of Object.entries(mapping)) {
+      // Handle direct mapping for nested data (e.g., "input": "$")
+      if (mappingValue === "$") {
+        // Use the entire row as the variable value
+        variables[graphqlVar] = row;
+      }
+      // Handle path-based mapping for nested data (e.g., "input.name": "$.product.name")
+      else if (
+        typeof mappingValue === "string" &&
+        mappingValue.startsWith("$.")
+      ) {
+        const path = mappingValue.substring(2); // Remove '$.'
+        const value = this.getValueByPath(row, path);
+        if (value !== undefined) {
+          const type = variableTypes[graphqlVar];
+          variables[graphqlVar] = this.convertValue(value, type, graphqlVar);
+        }
+      }
+      // Handle traditional flat mapping (e.g., "name": "product_name")
+      else if (
+        typeof mappingValue === "string" &&
+        row[mappingValue] !== undefined
+      ) {
+        const rawValue = row[mappingValue];
         const type = variableTypes[graphqlVar];
         variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
       }
+      // Handle complex mapping object
+      else if (typeof mappingValue === "object" && mappingValue !== null) {
+        variables[graphqlVar] = this.mapNestedObject(
+          row,
+          mappingValue,
+          variableTypes
+        );
+      }
     }
 
     return variables;
   }
 
+  private getValueByPath(obj: any, path: string): any {
+    const parts = path.split(".");
+    let current = obj;
+
+    for (const part of parts) {
+      if (current && typeof current === "object" && part in current) {
+        current = current[part];
+      } else {
+        return undefined;
+      }
+    }
+
+    return current;
+  }
+
+  private mapNestedObject(
+    row: DataRow,
+    mappingObj: any,
+    variableTypes: Record<string, string>
+  ): any {
+    if (Array.isArray(mappingObj)) {
+      return mappingObj.map((item) =>
+        this.mapNestedObject(row, item, variableTypes)
+      );
+    }
+
+    if (typeof mappingObj === "object" && mappingObj !== null) {
+      const result: any = {};
+      for (const [key, value] of Object.entries(mappingObj)) {
+        if (typeof value === "string" && value.startsWith("$.")) {
+          const path = value.substring(2);
+          let fieldValue = this.getValueByPath(row, path);
+
+          // Handle special case for array fields (e.g., comma-separated values)
+          if (
+            key === "values" &&
+            typeof fieldValue === "string" &&
+            fieldValue.includes(",")
+          ) {
+            fieldValue = fieldValue.split(",").map((v) => v.trim());
+          }
+
+          result[key] = fieldValue;
+        } else if (typeof value === "string" && row[value] !== undefined) {
+          result[key] = row[value];
+        } else if (typeof value === "object") {
+          result[key] = this.mapNestedObject(row, value, variableTypes);
+        } else {
+          result[key] = value;
+        }
+      }
+      return result;
+    }
+
+    return mappingObj;
+  }
+
   private extractVariableTypes(mutation: string): Record<string, string> {
     const types: Record<string, string> = {};
 
@@ -285,9 +416,18 @@ export class DataMapper {
     return null;
   }
 
-  private convertValue(value: string, type: string | undefined, varName: string): any {
+  private convertValue(
+    value: any,
+    type: string | undefined,
+    varName: string
+  ): any {
     if (!type) {
-      // No type information available, keep as string
+      // No type information available, keep as is
+      return value;
+    }
+
+    // For non-string values (objects, arrays), return as is
+    if (typeof value !== "string") {
       return value;
     }
 
@@ -297,7 +437,11 @@ export class DataMapper {
       case "Int":
         const intValue = Number(trimmedValue);
         // Validate that it's a valid integer (no decimals, NaN, or Infinity)
-        if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {
+        if (
+          isNaN(intValue) ||
+          !isFinite(intValue) ||
+          !Number.isInteger(intValue)
+        ) {
           console.warn(
             `Warning: Cannot convert "${value}" to Int for variable $${varName}. Expected a valid integer. Using original value.`
           );