@jackchuka/gql-ingest 1.5.0 → 2.0.1

package/src/mapper.ts

@@ -1,15 +1,19 @@
 import fs from "fs";
 import path from "path";
 import { parse, DocumentNode, VariableDefinitionNode } from "graphql";
-import { readCsvFile, CsvRow } from "./csv-reader";
+import { DataReaderFactory, DataRow } from "./readers";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { MetricsCollector } from "./metrics";
 import { ParallelProcessingConfig, RetryConfig } from "./config";
 
 export interface MappingConfig {
-  csvFile: string;
+  // Legacy CSV support
+  csvFile?: string;
+  // New flexible data file support
+  dataFile?: string;
+  dataFormat?: string;
   graphqlFile: string;
-  mapping: Record<string, string>;
+  mapping: Record<string, string | any>;
 }
 
 export class DataMapper {
@@ -17,17 +21,20 @@ export class DataMapper {
   private basePath: string;
   private metrics: MetricsCollector;
   private verbose: boolean;
+  private formatOverride?: string;
 
   constructor(
     client: GraphQLClientWrapper,
     basePath: string = process.cwd(),
     metrics?: MetricsCollector,
-    verbose: boolean = false
+    verbose: boolean = false,
+    formatOverride?: string
   ) {
     this.client = client;
     this.basePath = basePath;
     this.metrics = metrics || new MetricsCollector();
     this.verbose = verbose;
+    this.formatOverride = formatOverride;
   }
 
   discoverMappings(configDir: string, entityFilter?: string[]): string[] {
@@ -93,9 +100,20 @@ export class DataMapper {
     // Extract config directory (parent of mappings directory)
     const configDir = path.dirname(path.dirname(configFullPath));
 
-    // Read CSV data (relative to config directory)
-    const csvPath = path.resolve(configDir, config.csvFile);
-    const csvData = await readCsvFile(csvPath);
+    // Determine data file path (support both legacy csvFile and new dataFile)
+    const dataFile = config.dataFile || config.csvFile;
+    if (!dataFile) {
+      throw new Error(
+        `No data file specified in mapping config: ${configPath}`
+      );
+    }
+
+    const dataPath = path.resolve(configDir, dataFile);
+
+    // Get appropriate reader (prioritize CLI format override, then config format)
+    const format = this.formatOverride || config.dataFormat;
+    const reader = DataReaderFactory.getReader(dataPath, format);
+    const data = await reader.readFile(dataPath);
 
     // Read GraphQL mutation (relative to config directory)
     const graphqlPath = path.resolve(configDir, config.graphqlFile);
@@ -104,7 +122,7 @@ export class DataMapper {
     // Process rows with optional parallelization
     if (parallelConfig && parallelConfig.concurrency > 1) {
       await this.processRowsConcurrently(
-        csvData,
+        data,
         mutation,
         config.mapping,
         entityName,
@@ -113,7 +131,7 @@ export class DataMapper {
       );
     } else {
       await this.processRowsSequentially(
-        csvData,
+        data,
         mutation,
         config.mapping,
         entityName,
@@ -125,18 +143,18 @@ export class DataMapper {
   }
 
   private async processRowsSequentially(
-    csvData: CsvRow[],
+    data: DataRow[],
     mutation: string,
-    mapping: Record<string, string>,
+    mapping: Record<string, string | any>,
     entityName: string,
     retryConfig?: RetryConfig
   ): Promise<void> {
-    const totalRows = csvData.length;
+    const totalRows = data.length;
     const variableTypes = this.extractVariableTypes(mutation);
 
-    for (let i = 0; i < csvData.length; i++) {
-      const row = csvData[i];
-      const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
+    for (let i = 0; i < data.length; i++) {
+      const row = data[i];
+      const variables = this.mapRowToVariables(row, mapping, variableTypes);
 
       try {
         await this.client.executeMutation(mutation, variables, retryConfig);
@@ -165,34 +183,30 @@ export class DataMapper {
   }
 
   private async processRowsConcurrently(
-    csvData: CsvRow[],
+    data: DataRow[],
     mutation: string,
-    mapping: Record<string, string>,
+    mapping: Record<string, string | any>,
     entityName: string,
     parallelConfig: ParallelProcessingConfig,
     retryConfig?: RetryConfig
   ): Promise<void> {
     const concurrency = parallelConfig.concurrency;
     console.log(
-      `Processing ${csvData.length} rows with concurrency: ${concurrency}`
+      `Processing ${data.length} rows with concurrency: ${concurrency}`
     );
 
     // Extract variable types once for all rows
     const variableTypes = this.extractVariableTypes(mutation);
 
     // Split data into chunks for concurrent processing
-    const chunks = this.chunkArray(csvData, concurrency);
+    const chunks = this.chunkArray(data, concurrency);
     let processedCount = 0;
-    const totalRows = csvData.length;
+    const totalRows = data.length;
 
     for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
       const chunk = chunks[chunkIndex];
       const promises = chunk.map(async (row) => {
-        const variables = this.mapCsvRowToVariables(
-          row,
-          mapping,
-          variableTypes
-        );
+        const variables = this.mapRowToVariables(row, mapping, variableTypes);
 
         try {
           const result = await this.client.executeMutation(
@@ -254,24 +268,110 @@ export class DataMapper {
     return chunks;
   }
 
-  private mapCsvRowToVariables(
-    row: CsvRow,
-    mapping: Record<string, string>,
+  private mapRowToVariables(
+    row: DataRow,
+    mapping: Record<string, string | any>,
     variableTypes: Record<string, string>
   ): Record<string, any> {
     const variables: Record<string, any> = {};
 
-    for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
-      if (row[csvColumn] !== undefined) {
-        const rawValue = row[csvColumn];
+    for (const [graphqlVar, mappingValue] of Object.entries(mapping)) {
+      // Handle direct mapping for nested data (e.g., "input": "$")
+      if (mappingValue === "$") {
+        // Use the entire row as the variable value
+        variables[graphqlVar] = row;
+      }
+      // Handle path-based mapping for nested data (e.g., "input.name": "$.product.name")
+      else if (
+        typeof mappingValue === "string" &&
+        mappingValue.startsWith("$.")
+      ) {
+        const path = mappingValue.substring(2); // Remove '$.'
+        const value = this.getValueByPath(row, path);
+        if (value !== undefined) {
+          const type = variableTypes[graphqlVar];
+          variables[graphqlVar] = this.convertValue(value, type, graphqlVar);
+        }
+      }
+      // Handle traditional flat mapping (e.g., "name": "product_name")
+      else if (
+        typeof mappingValue === "string" &&
+        row[mappingValue] !== undefined
+      ) {
+        const rawValue = row[mappingValue];
         const type = variableTypes[graphqlVar];
         variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
       }
+      // Handle complex mapping object
+      else if (typeof mappingValue === "object" && mappingValue !== null) {
+        variables[graphqlVar] = this.mapNestedObject(
+          row,
+          mappingValue,
+          variableTypes
+        );
+      }
     }
 
     return variables;
   }
 
+  private getValueByPath(obj: any, path: string): any {
+    const parts = path.split(".");
+    let current = obj;
+
+    for (const part of parts) {
+      if (current && typeof current === "object" && part in current) {
+        current = current[part];
+      } else {
+        return undefined;
+      }
+    }
+
+    return current;
+  }
+
+  private mapNestedObject(
+    row: DataRow,
+    mappingObj: any,
+    variableTypes: Record<string, string>
+  ): any {
+    if (Array.isArray(mappingObj)) {
+      return mappingObj.map((item) =>
+        this.mapNestedObject(row, item, variableTypes)
+      );
+    }
+
+    if (typeof mappingObj === "object" && mappingObj !== null) {
+      const result: any = {};
+      for (const [key, value] of Object.entries(mappingObj)) {
+        if (typeof value === "string" && value.startsWith("$.")) {
+          const path = value.substring(2);
+          let fieldValue = this.getValueByPath(row, path);
+
+          // Handle special case for array fields (e.g., comma-separated values)
+          if (
+            key === "values" &&
+            typeof fieldValue === "string" &&
+            fieldValue.includes(",")
+          ) {
+            fieldValue = fieldValue.split(",").map((v) => v.trim());
+          }
+
+          result[key] = fieldValue;
+        } else if (typeof value === "string" && row[value] !== undefined) {
+          result[key] = row[value];
+        } else if (typeof value === "object") {
+          result[key] = this.mapNestedObject(row, value, variableTypes);
+        } else {
+          result[key] = value;
+        }
+      }
+      return result;
+    }
+
+    return mappingObj;
+  }
+
   private extractVariableTypes(mutation: string): Record<string, string> {
     const types: Record<string, string> = {};
 
@@ -317,12 +417,17 @@ export class DataMapper {
   }
 
   private convertValue(
-    value: string,
+    value: any,
     type: string | undefined,
     varName: string
   ): any {
     if (!type) {
-      // No type information available, keep as string
+      // No type information available, keep as is
+      return value;
+    }
+
+    // For non-string values (objects, arrays), return as is
+    if (typeof value !== "string") {
       return value;
     }
 

package/src/{csv-reader.test.ts → readers/csv.test.ts}

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { readCsvFile } from "./csv-reader";
+import { readCsvFile } from "./csv";
 
 describe("CSV Reader", () => {
   const testDataDir = path.join(__dirname, "test-data");

package/src/readers/csv.ts

@@ -0,0 +1,29 @@
+import fs from "fs";
+import csv from "csv-parser";
+import { DataReader, DataRow } from "./data-reader";
+
+export interface CsvRow {
+  [key: string]: string;
+}
+
+export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
+  return new Promise((resolve, reject) => {
+    const results: CsvRow[] = [];
+
+    fs.createReadStream(filePath)
+      .pipe(csv())
+      .on("data", (data) => results.push(data))
+      .on("end", () => resolve(results))
+      .on("error", (error) => reject(error));
+  });
+}
+
+export class CsvReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["csv"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    return readCsvFile(filePath);
+  }
+}

package/src/readers/data-reader.test.ts

@@ -0,0 +1,104 @@
+import { DataReader, DataReaderFactory } from "./data-reader";
+
+class TestReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["test", "tst"];
+  }
+
+  async readFile(filePath: string): Promise<any[]> {
+    return [{ test: true }];
+  }
+}
+
+describe("DataReader", () => {
+  describe("canHandle", () => {
+    it("should check if reader can handle file based on extension", () => {
+      const reader = new TestReader();
+
+      expect(reader.canHandle("file.test")).toBe(true);
+      expect(reader.canHandle("file.tst")).toBe(true);
+      expect(reader.canHandle("path/to/file.test")).toBe(true);
+      expect(reader.canHandle("file.other")).toBe(false);
+      expect(reader.canHandle("file")).toBe(false);
+    });
+  });
+});
+
+describe("DataReaderFactory", () => {
+  beforeEach(() => {
+    // Clear the readers array before each test
+    (DataReaderFactory as any).readers = [];
+  });
+
+  describe("registerReader", () => {
+    it("should register a reader", () => {
+      const reader = new TestReader();
+      DataReaderFactory.registerReader(reader);
+
+      expect(DataReaderFactory.getSupportedFormats()).toContain("test");
+      expect(DataReaderFactory.getSupportedFormats()).toContain("tst");
+    });
+  });
+
+  describe("getReader", () => {
+    beforeEach(() => {
+      const reader = new TestReader();
+      DataReaderFactory.registerReader(reader);
+    });
+
+    it("should get reader by file extension", () => {
+      const reader = DataReaderFactory.getReader("file.test");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should get reader by format override", () => {
+      const reader = DataReaderFactory.getReader("file.other", "test");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should prioritize format override over file extension", () => {
+      const reader = DataReaderFactory.getReader("file.unknown", "tst");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should throw error when no reader found", () => {
+      expect(() => DataReaderFactory.getReader("file.unknown")).toThrow(
+        "No reader found for file: file.unknown"
+      );
+    });
+
+    it("should throw error when format specified but no reader found", () => {
+      expect(() => DataReaderFactory.getReader("file.txt", "unknown")).toThrow(
+        "No reader found for file: file.txt with format: unknown"
+      );
+    });
+  });
+
+  describe("getSupportedFormats", () => {
+    it("should return empty array when no readers registered", () => {
+      expect(DataReaderFactory.getSupportedFormats()).toEqual([]);
+    });
+
+    it("should return all supported formats", () => {
+      const reader1 = new TestReader();
+      DataReaderFactory.registerReader(reader1);
+
+      const formats = DataReaderFactory.getSupportedFormats();
+      expect(formats).toContain("test");
+      expect(formats).toContain("tst");
+      expect(formats.length).toBe(2);
+    });
+
+    it("should not duplicate formats", () => {
+      const reader1 = new TestReader();
+      const reader2 = new TestReader();
+      DataReaderFactory.registerReader(reader1);
+      DataReaderFactory.registerReader(reader2);
+
+      const formats = DataReaderFactory.getSupportedFormats();
+      expect(formats).toContain("test");
+      expect(formats).toContain("tst");
+      expect(formats.length).toBe(2);
+    });
+  });
+});

package/src/readers/data-reader.ts

@@ -0,0 +1,61 @@
+export interface DataRow {
+  [key: string]: any;
+}
+
+export abstract class DataReader {
+  abstract readFile(filePath: string): Promise<DataRow[]>;
+
+  /**
+   * Get the supported file extensions for this reader
+   */
+  abstract getSupportedExtensions(): string[];
+
+  /**
+   * Check if this reader can handle the given file
+   */
+  canHandle(filePath: string): boolean {
+    const extension = filePath.split(".").pop()?.toLowerCase();
+    return extension
+      ? this.getSupportedExtensions().includes(extension)
+      : false;
+  }
+}
+
+export class DataReaderFactory {
+  private static readers: DataReader[] = [];
+
+  static registerReader(reader: DataReader): void {
+    this.readers.push(reader);
+  }
+
+  static getReader(filePath: string, format?: string): DataReader {
+    // If format is specified, try to find reader by format
+    if (format) {
+      const reader = this.readers.find((r) =>
+        r.getSupportedExtensions().includes(format.toLowerCase())
+      );
+      if (reader) return reader;
+    }
+
+    // Otherwise, try to find reader by file extension
+    const reader = this.readers.find((r) => r.canHandle(filePath));
+
+    if (!reader) {
+      throw new Error(
+        `No reader found for file: ${filePath}${
+          format ? ` with format: ${format}` : ""
+        }`
+      );
+    }
+
+    return reader;
+  }
+
+  static getSupportedFormats(): string[] {
+    const formats = new Set<string>();
+    this.readers.forEach((reader) => {
+      reader.getSupportedExtensions().forEach((ext) => formats.add(ext));
+    });
+    return Array.from(formats);
+  }
+}

package/src/readers/index.ts

@@ -0,0 +1,18 @@
+export { DataReader, DataRow, DataReaderFactory } from "./data-reader";
+export { CsvReader, readCsvFile, CsvRow } from "./csv";
+export { JsonReader } from "./json";
+export { YamlReader } from "./yaml";
+export { JsonlReader } from "./jsonl";
+
+// Register all readers
+import { DataReaderFactory } from "./data-reader";
+import { CsvReader } from "./csv";
+import { JsonReader } from "./json";
+import { YamlReader } from "./yaml";
+import { JsonlReader } from "./jsonl";
+
+// Register readers on module load
+DataReaderFactory.registerReader(new CsvReader());
+DataReaderFactory.registerReader(new JsonReader());
+DataReaderFactory.registerReader(new YamlReader());
+DataReaderFactory.registerReader(new JsonlReader());

package/src/readers/json.test.ts

@@ -0,0 +1,80 @@
+import fs from "fs/promises";
+import { JsonReader } from "./json";
+
+jest.mock("fs/promises");
+
+describe("JsonReader", () => {
+  let reader: JsonReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+
+  beforeEach(() => {
+    reader = new JsonReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return json as supported extension", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["json"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .json files", () => {
+      expect(reader.canHandle("data.json")).toBe(true);
+      expect(reader.canHandle("path/to/file.json")).toBe(true);
+      expect(reader.canHandle("file.JSON")).toBe(true); // case insensitive
+    });
+
+    it("should return false for non-json files", () => {
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data.yaml")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse JSON array", async () => {
+      const mockData = [
+        { id: 1, name: "Item 1" },
+        { id: 2, name: "Item 2" },
+      ];
+      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
+
+      const result = await reader.readFile("data.json");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.json", "utf8");
+      expect(result).toEqual(mockData);
+    });
+
+    it("should wrap single object in array", async () => {
+      const mockData = { id: 1, name: "Item 1" };
+      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
+
+      const result = await reader.readFile("data.json");
+
+      expect(result).toEqual([mockData]);
+    });
+
+    it("should throw error for invalid JSON", async () => {
+      mockFs.readFile.mockResolvedValue("invalid json");
+
+      await expect(reader.readFile("data.json")).rejects.toThrow();
+    });
+
+    it("should throw error for null data", async () => {
+      mockFs.readFile.mockResolvedValue("null");
+
+      await expect(reader.readFile("data.json")).rejects.toThrow(
+        "Invalid JSON data structure in file: data.json. Expected array or object."
+      );
+    });
+
+    it("should throw error for primitive values", async () => {
+      mockFs.readFile.mockResolvedValue('"string value"');
+
+      await expect(reader.readFile("data.json")).rejects.toThrow(
+        "Invalid JSON data structure in file: data.json. Expected array or object."
+      );
+    });
+  });
+});

package/src/readers/json.ts

@@ -0,0 +1,27 @@
+import fs from "fs/promises";
+import { DataReader, DataRow } from "./data-reader";
+
+export class JsonReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["json"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const data = JSON.parse(content);
+
+    // If the data is already an array, return it
+    if (Array.isArray(data)) {
+      return data;
+    }
+
+    // If it's a single object, wrap it in an array
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+
+    throw new Error(
+      `Invalid JSON data structure in file: ${filePath}. Expected array or object.`
+    );
+  }
+}