@jackchuka/gql-ingest 1.4.0 → 2.0.0

package/src/metrics.ts

@@ -119,35 +119,43 @@ export class MetricsCollector {
     const duration = this.getDurationMs();
     const successRate = this.getSuccessRate();
     const avgRequestDuration = this.getAverageRequestDuration();
-    
+
     let summary = `\n📊 Processing Summary:\n`;
     summary += `   Total Processed: ${this.metrics.totalEntities}\n`;
     summary += `   ✓ Successes: ${this.metrics.totalSuccesses}\n`;
     summary += `   ✗ Failures: ${this.metrics.totalFailures}\n`;
     summary += `   Success Rate: ${successRate.toFixed(1)}%\n`;
     summary += `   Duration: ${(duration / 1000).toFixed(2)}s\n`;
-    
+
     if (this.metrics.requestDurations.length > 0) {
       summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
     }
-    
+
     if (this.metrics.retryAttempts > 0) {
       summary += `   Retry Attempts: ${this.metrics.retryAttempts}\n`;
       summary += `   Retry Successes: ${this.metrics.retrySuccesses}\n`;
       summary += `   Retry Failures: ${this.metrics.retryFailures}\n`;
     }
 
-    if (this.metrics.entityMetrics.size > 1) {
+    if (this.metrics.entityMetrics.size > 0) {
       summary += `\n📋 Per-Entity Breakdown:\n`;
       for (const [entityName, entityMetric] of this.metrics.entityMetrics) {
-        const entityTotal = entityMetric.successCount + entityMetric.failureCount;
-        const entityRate = entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
-        const entityDuration = entityMetric.endTime ? entityMetric.endTime - entityMetric.startTime : 0;
-        
-        summary += `   ${entityName}: ${entityTotal} total (${entityMetric.successCount} ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(1)}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
+        const entityTotal =
+          entityMetric.successCount + entityMetric.failureCount;
+        const entityRate =
+          entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
+        const entityDuration = entityMetric.endTime
+          ? entityMetric.endTime - entityMetric.startTime
+          : 0;
+
+        summary += `   ${entityName}: ${entityTotal} total (${
+          entityMetric.successCount
+        } ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(
+          1
+        )}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
       }
     }
 
     return summary;
   }
-}
+}

package/src/{csv-reader.test.ts → readers/csv.test.ts}

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { readCsvFile } from "./csv-reader";
+import { readCsvFile } from "./csv";
 
 describe("CSV Reader", () => {
   const testDataDir = path.join(__dirname, "test-data");

package/src/readers/csv.ts

@@ -0,0 +1,29 @@
+import fs from "fs";
+import csv from "csv-parser";
+import { DataReader, DataRow } from "./data-reader";
+
+export interface CsvRow {
+  [key: string]: string;
+}
+
+export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
+  return new Promise((resolve, reject) => {
+    const results: CsvRow[] = [];
+
+    fs.createReadStream(filePath)
+      .pipe(csv())
+      .on("data", (data) => results.push(data))
+      .on("end", () => resolve(results))
+      .on("error", (error) => reject(error));
+  });
+}
+
+export class CsvReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["csv"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    return readCsvFile(filePath);
+  }
+}

package/src/readers/data-reader.test.ts

@@ -0,0 +1,104 @@
+import { DataReader, DataReaderFactory } from "./data-reader";
+
+class TestReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["test", "tst"];
+  }
+
+  async readFile(filePath: string): Promise<any[]> {
+    return [{ test: true }];
+  }
+}
+
+describe("DataReader", () => {
+  describe("canHandle", () => {
+    it("should check if reader can handle file based on extension", () => {
+      const reader = new TestReader();
+
+      expect(reader.canHandle("file.test")).toBe(true);
+      expect(reader.canHandle("file.tst")).toBe(true);
+      expect(reader.canHandle("path/to/file.test")).toBe(true);
+      expect(reader.canHandle("file.other")).toBe(false);
+      expect(reader.canHandle("file")).toBe(false);
+    });
+  });
+});
+
+describe("DataReaderFactory", () => {
+  beforeEach(() => {
+    // Clear the readers array before each test
+    (DataReaderFactory as any).readers = [];
+  });
+
+  describe("registerReader", () => {
+    it("should register a reader", () => {
+      const reader = new TestReader();
+      DataReaderFactory.registerReader(reader);
+
+      expect(DataReaderFactory.getSupportedFormats()).toContain("test");
+      expect(DataReaderFactory.getSupportedFormats()).toContain("tst");
+    });
+  });
+
+  describe("getReader", () => {
+    beforeEach(() => {
+      const reader = new TestReader();
+      DataReaderFactory.registerReader(reader);
+    });
+
+    it("should get reader by file extension", () => {
+      const reader = DataReaderFactory.getReader("file.test");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should get reader by format override", () => {
+      const reader = DataReaderFactory.getReader("file.other", "test");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should prioritize format override over file extension", () => {
+      const reader = DataReaderFactory.getReader("file.unknown", "tst");
+      expect(reader).toBeInstanceOf(TestReader);
+    });
+
+    it("should throw error when no reader found", () => {
+      expect(() => DataReaderFactory.getReader("file.unknown")).toThrow(
+        "No reader found for file: file.unknown"
+      );
+    });
+
+    it("should throw error when format specified but no reader found", () => {
+      expect(() => DataReaderFactory.getReader("file.txt", "unknown")).toThrow(
+        "No reader found for file: file.txt with format: unknown"
+      );
+    });
+  });
+
+  describe("getSupportedFormats", () => {
+    it("should return empty array when no readers registered", () => {
+      expect(DataReaderFactory.getSupportedFormats()).toEqual([]);
+    });
+
+    it("should return all supported formats", () => {
+      const reader1 = new TestReader();
+      DataReaderFactory.registerReader(reader1);
+
+      const formats = DataReaderFactory.getSupportedFormats();
+      expect(formats).toContain("test");
+      expect(formats).toContain("tst");
+      expect(formats.length).toBe(2);
+    });
+
+    it("should not duplicate formats", () => {
+      const reader1 = new TestReader();
+      const reader2 = new TestReader();
+      DataReaderFactory.registerReader(reader1);
+      DataReaderFactory.registerReader(reader2);
+
+      const formats = DataReaderFactory.getSupportedFormats();
+      expect(formats).toContain("test");
+      expect(formats).toContain("tst");
+      expect(formats.length).toBe(2);
+    });
+  });
+});

package/src/readers/data-reader.ts

@@ -0,0 +1,61 @@
+export interface DataRow {
+  [key: string]: any;
+}
+
+export abstract class DataReader {
+  abstract readFile(filePath: string): Promise<DataRow[]>;
+
+  /**
+   * Get the supported file extensions for this reader
+   */
+  abstract getSupportedExtensions(): string[];
+
+  /**
+   * Check if this reader can handle the given file
+   */
+  canHandle(filePath: string): boolean {
+    const extension = filePath.split(".").pop()?.toLowerCase();
+    return extension
+      ? this.getSupportedExtensions().includes(extension)
+      : false;
+  }
+}
+
+export class DataReaderFactory {
+  private static readers: DataReader[] = [];
+
+  static registerReader(reader: DataReader): void {
+    this.readers.push(reader);
+  }
+
+  static getReader(filePath: string, format?: string): DataReader {
+    // If format is specified, try to find reader by format
+    if (format) {
+      const reader = this.readers.find((r) =>
+        r.getSupportedExtensions().includes(format.toLowerCase())
+      );
+      if (reader) return reader;
+    }
+
+    // Otherwise, try to find reader by file extension
+    const reader = this.readers.find((r) => r.canHandle(filePath));
+
+    if (!reader) {
+      throw new Error(
+        `No reader found for file: ${filePath}${
+          format ? ` with format: ${format}` : ""
+        }`
+      );
+    }
+
+    return reader;
+  }
+
+  static getSupportedFormats(): string[] {
+    const formats = new Set<string>();
+    this.readers.forEach((reader) => {
+      reader.getSupportedExtensions().forEach((ext) => formats.add(ext));
+    });
+    return Array.from(formats);
+  }
+}

package/src/readers/index.ts

@@ -0,0 +1,18 @@
+export { DataReader, DataRow, DataReaderFactory } from "./data-reader";
+export { CsvReader, readCsvFile, CsvRow } from "./csv";
+export { JsonReader } from "./json";
+export { YamlReader } from "./yaml";
+export { JsonlReader } from "./jsonl";
+
+// Register all readers
+import { DataReaderFactory } from "./data-reader";
+import { CsvReader } from "./csv";
+import { JsonReader } from "./json";
+import { YamlReader } from "./yaml";
+import { JsonlReader } from "./jsonl";
+
+// Register readers on module load
+DataReaderFactory.registerReader(new CsvReader());
+DataReaderFactory.registerReader(new JsonReader());
+DataReaderFactory.registerReader(new YamlReader());
+DataReaderFactory.registerReader(new JsonlReader());

package/src/readers/json.test.ts

@@ -0,0 +1,80 @@
+import fs from "fs/promises";
+import { JsonReader } from "./json";
+
+jest.mock("fs/promises");
+
+describe("JsonReader", () => {
+  let reader: JsonReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+
+  beforeEach(() => {
+    reader = new JsonReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return json as supported extension", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["json"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .json files", () => {
+      expect(reader.canHandle("data.json")).toBe(true);
+      expect(reader.canHandle("path/to/file.json")).toBe(true);
+      expect(reader.canHandle("file.JSON")).toBe(true); // case insensitive
+    });
+
+    it("should return false for non-json files", () => {
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data.yaml")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse JSON array", async () => {
+      const mockData = [
+        { id: 1, name: "Item 1" },
+        { id: 2, name: "Item 2" },
+      ];
+      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
+
+      const result = await reader.readFile("data.json");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.json", "utf8");
+      expect(result).toEqual(mockData);
+    });
+
+    it("should wrap single object in array", async () => {
+      const mockData = { id: 1, name: "Item 1" };
+      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
+
+      const result = await reader.readFile("data.json");
+
+      expect(result).toEqual([mockData]);
+    });
+
+    it("should throw error for invalid JSON", async () => {
+      mockFs.readFile.mockResolvedValue("invalid json");
+
+      await expect(reader.readFile("data.json")).rejects.toThrow();
+    });
+
+    it("should throw error for null data", async () => {
+      mockFs.readFile.mockResolvedValue("null");
+
+      await expect(reader.readFile("data.json")).rejects.toThrow(
+        "Invalid JSON data structure in file: data.json. Expected array or object."
+      );
+    });
+
+    it("should throw error for primitive values", async () => {
+      mockFs.readFile.mockResolvedValue('"string value"');
+
+      await expect(reader.readFile("data.json")).rejects.toThrow(
+        "Invalid JSON data structure in file: data.json. Expected array or object."
+      );
+    });
+  });
+});

package/src/readers/json.ts

@@ -0,0 +1,27 @@
+import fs from "fs/promises";
+import { DataReader, DataRow } from "./data-reader";
+
+export class JsonReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["json"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const data = JSON.parse(content);
+
+    // If the data is already an array, return it
+    if (Array.isArray(data)) {
+      return data;
+    }
+
+    // If it's a single object, wrap it in an array
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+
+    throw new Error(
+      `Invalid JSON data structure in file: ${filePath}. Expected array or object.`
+    );
+  }
+}

package/src/readers/jsonl.test.ts

@@ -0,0 +1,96 @@
+import fs from "fs/promises";
+import { JsonlReader } from "./jsonl";
+
+jest.mock("fs/promises");
+
+describe("JsonlReader", () => {
+  let reader: JsonlReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+
+  beforeEach(() => {
+    reader = new JsonlReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return jsonl and ndjson as supported extensions", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["jsonl", "ndjson"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .jsonl and .ndjson files", () => {
+      expect(reader.canHandle("data.jsonl")).toBe(true);
+      expect(reader.canHandle("data.ndjson")).toBe(true);
+      expect(reader.canHandle("path/to/file.jsonl")).toBe(true);
+    });
+
+    it("should return false for non-jsonl files", () => {
+      expect(reader.canHandle("data.json")).toBe(false);
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse JSONL file", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const line2 = { id: 2, name: "Item 2" };
+      const jsonlContent = `${JSON.stringify(line1)}\n${JSON.stringify(line2)}`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.jsonl", "utf8");
+      expect(result).toEqual([line1, line2]);
+    });
+
+    it("should handle empty lines", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const line2 = { id: 2, name: "Item 2" };
+      const jsonlContent = `${JSON.stringify(line1)}\n\n${JSON.stringify(
+        line2
+      )}\n`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([line1, line2]);
+    });
+
+    it("should handle single line", async () => {
+      const line = { id: 1, name: "Item 1" };
+      mockFs.readFile.mockResolvedValue(JSON.stringify(line));
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([line]);
+    });
+
+    it("should throw error for invalid JSON on specific line", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const jsonlContent = `${JSON.stringify(line1)}\ninvalid json\n`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      await expect(reader.readFile("data.jsonl")).rejects.toThrow(
+        "Invalid JSON at line 2 in file: data.jsonl"
+      );
+    });
+
+    it("should handle empty file", async () => {
+      mockFs.readFile.mockResolvedValue("");
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([]);
+    });
+
+    it("should handle file with only whitespace", async () => {
+      mockFs.readFile.mockResolvedValue("\n\n  \n\t\n");
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([]);
+    });
+  });
+});

package/src/readers/jsonl.ts

@@ -0,0 +1,28 @@
+import fs from "fs/promises";
+import { DataReader, DataRow } from "./data-reader";
+
+export class JsonlReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["jsonl", "ndjson"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const lines = content.split("\n").filter((line) => line.trim());
+
+    const results: DataRow[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      try {
+        const data = JSON.parse(lines[i]);
+        results.push(data);
+      } catch (error) {
+        throw new Error(
+          `Invalid JSON at line ${i + 1} in file: ${filePath}. Error: ${error}`
+        );
+      }
+    }
+
+    return results;
+  }
+}

package/src/readers/yaml.test.ts

@@ -0,0 +1,95 @@
+import fs from "fs/promises";
+import yaml from "js-yaml";
+import { YamlReader } from "./yaml";
+
+jest.mock("fs/promises");
+jest.mock("js-yaml");
+
+describe("YamlReader", () => {
+  let reader: YamlReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+  const mockYaml = yaml as jest.Mocked<typeof yaml>;
+
+  beforeEach(() => {
+    reader = new YamlReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return yaml and yml as supported extensions", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["yaml", "yml"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .yaml and .yml files", () => {
+      expect(reader.canHandle("data.yaml")).toBe(true);
+      expect(reader.canHandle("data.yml")).toBe(true);
+      expect(reader.canHandle("path/to/file.yaml")).toBe(true);
+    });
+
+    it("should return false for non-yaml files", () => {
+      expect(reader.canHandle("data.json")).toBe(false);
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse YAML array", async () => {
+      const mockData = [
+        { id: 1, name: "Item 1" },
+        { id: 2, name: "Item 2" },
+      ];
+      const yamlContent = "- id: 1\n  name: Item 1\n- id: 2\n  name: Item 2";
+      mockFs.readFile.mockResolvedValue(yamlContent);
+      mockYaml.load.mockReturnValue(mockData);
+
+      const result = await reader.readFile("data.yaml");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.yaml", "utf8");
+      expect(mockYaml.load).toHaveBeenCalledWith(yamlContent);
+      expect(result).toEqual(mockData);
+    });
+
+    it("should wrap single object in array", async () => {
+      const mockData = { id: 1, name: "Item 1" };
+      const yamlContent = "id: 1\nname: Item 1";
+      mockFs.readFile.mockResolvedValue(yamlContent);
+      mockYaml.load.mockReturnValue(mockData);
+
+      const result = await reader.readFile("data.yaml");
+
+      expect(result).toEqual([mockData]);
+    });
+
+    it("should throw error for invalid YAML", async () => {
+      mockFs.readFile.mockResolvedValue("invalid: yaml: content:");
+      mockYaml.load.mockImplementation(() => {
+        throw new Error("Invalid YAML");
+      });
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML"
+      );
+    });
+
+    it("should throw error for null data", async () => {
+      mockFs.readFile.mockResolvedValue("null");
+      mockYaml.load.mockReturnValue(null);
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML data structure in file: data.yaml. Expected array or object."
+      );
+    });
+
+    it("should throw error for primitive values", async () => {
+      mockFs.readFile.mockResolvedValue("string value");
+      mockYaml.load.mockReturnValue("string value");
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML data structure in file: data.yaml. Expected array or object."
+      );
+    });
+  });
+});

package/src/readers/yaml.ts

@@ -0,0 +1,28 @@
+import fs from "fs/promises";
+import yaml from "js-yaml";
+import { DataReader, DataRow } from "./data-reader";
+
+export class YamlReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["yaml", "yml"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const data = yaml.load(content);
+
+    // If the data is already an array, return it
+    if (Array.isArray(data)) {
+      return data;
+    }
+
+    // If it's a single object, wrap it in an array
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+
+    throw new Error(
+      `Invalid YAML data structure in file: ${filePath}. Expected array or object.`
+    );
+  }
+}

package/dist/csv-reader.d.ts

@@ -1,5 +0,0 @@
-export interface CsvRow {
-    [key: string]: string;
-}
-export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
-//# sourceMappingURL=csv-reader.d.ts.map

package/dist/csv-reader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../src/csv-reader.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUrE"}

package/dist/csv-reader.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=csv-reader.test.d.ts.map

package/dist/csv-reader.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv-reader.test.d.ts","sourceRoot":"","sources":["../src/csv-reader.test.ts"],"names":[],"mappings":""}

package/src/csv-reader.ts

@@ -1,18 +0,0 @@
-import fs from 'fs';
-import csv from 'csv-parser';
-
-export interface CsvRow {
-  [key: string]: string;
-}
-
-export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
-  return new Promise((resolve, reject) => {
-    const results: CsvRow[] = [];
-    
-    fs.createReadStream(filePath)
-      .pipe(csv())
-      .on('data', (data) => results.push(data))
-      .on('end', () => resolve(results))
-      .on('error', (error) => reject(error));
-  });
-}