@jackchuka/gql-ingest 2.1.0 → 2.2.0

package/src/metrics.ts

@@ -1,161 +0,0 @@
-export interface EntityMetrics {
-  entityName: string;
-  successCount: number;
-  failureCount: number;
-  startTime: number;
-  endTime?: number;
-}
-
-export interface ProcessingMetrics {
-  totalEntities: number;
-  totalSuccesses: number;
-  totalFailures: number;
-  entityMetrics: Map<string, EntityMetrics>;
-  requestDurations: number[];
-  retryAttempts: number;
-  retrySuccesses: number;
-  retryFailures: number;
-  startTime: number;
-  endTime?: number;
-}
-
-export class MetricsCollector {
-  private metrics: ProcessingMetrics;
-
-  constructor() {
-    this.metrics = {
-      totalEntities: 0,
-      totalSuccesses: 0,
-      totalFailures: 0,
-      entityMetrics: new Map(),
-      requestDurations: [],
-      retryAttempts: 0,
-      retrySuccesses: 0,
-      retryFailures: 0,
-      startTime: Date.now(),
-    };
-  }
-
-  startEntityProcessing(entityName: string): void {
-    if (!this.metrics.entityMetrics.has(entityName)) {
-      this.metrics.entityMetrics.set(entityName, {
-        entityName,
-        successCount: 0,
-        failureCount: 0,
-        startTime: Date.now(),
-      });
-    }
-  }
-
-  recordSuccess(entityName: string): void {
-    const entityMetric = this.metrics.entityMetrics.get(entityName);
-    if (entityMetric) {
-      entityMetric.successCount++;
-      this.metrics.totalSuccesses++;
-      this.metrics.totalEntities++;
-    }
-  }
-
-  recordFailure(entityName: string): void {
-    const entityMetric = this.metrics.entityMetrics.get(entityName);
-    if (entityMetric) {
-      entityMetric.failureCount++;
-      this.metrics.totalFailures++;
-      this.metrics.totalEntities++;
-    }
-  }
-
-  finishEntityProcessing(entityName: string): void {
-    const entityMetric = this.metrics.entityMetrics.get(entityName);
-    if (entityMetric) {
-      entityMetric.endTime = Date.now();
-    }
-  }
-
-  finishProcessing(): ProcessingMetrics {
-    this.metrics.endTime = Date.now();
-    return { ...this.metrics };
-  }
-
-  getEntityMetrics(entityName: string): EntityMetrics | undefined {
-    return this.metrics.entityMetrics.get(entityName);
-  }
-
-  getTotalProcessed(): number {
-    return this.metrics.totalEntities;
-  }
-
-  getSuccessRate(): number {
-    if (this.metrics.totalEntities === 0) return 0;
-    return (this.metrics.totalSuccesses / this.metrics.totalEntities) * 100;
-  }
-
-  recordRequestDuration(duration: number): void {
-    this.metrics.requestDurations.push(duration);
-  }
-
-  recordRetrySuccess(attempts: number): void {
-    this.metrics.retryAttempts += attempts;
-    this.metrics.retrySuccesses++;
-  }
-
-  recordRetryFailure(attempts: number): void {
-    this.metrics.retryAttempts += attempts;
-    this.metrics.retryFailures++;
-  }
-
-  getAverageRequestDuration(): number {
-    if (this.metrics.requestDurations.length === 0) return 0;
-    const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);
-    return sum / this.metrics.requestDurations.length;
-  }
-
-  getDurationMs(): number {
-    const endTime = this.metrics.endTime || Date.now();
-    return endTime - this.metrics.startTime;
-  }
-
-  generateSummary(): string {
-    const duration = this.getDurationMs();
-    const successRate = this.getSuccessRate();
-    const avgRequestDuration = this.getAverageRequestDuration();
-
-    let summary = `\n📊 Processing Summary:\n`;
-    summary += `   Total Processed: ${this.metrics.totalEntities}\n`;
-    summary += `   ✓ Successes: ${this.metrics.totalSuccesses}\n`;
-    summary += `   ✗ Failures: ${this.metrics.totalFailures}\n`;
-    summary += `   Success Rate: ${successRate.toFixed(1)}%\n`;
-    summary += `   Duration: ${(duration / 1000).toFixed(2)}s\n`;
-
-    if (this.metrics.requestDurations.length > 0) {
-      summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
-    }
-
-    if (this.metrics.retryAttempts > 0) {
-      summary += `   Retry Attempts: ${this.metrics.retryAttempts}\n`;
-      summary += `   Retry Successes: ${this.metrics.retrySuccesses}\n`;
-      summary += `   Retry Failures: ${this.metrics.retryFailures}\n`;
-    }
-
-    if (this.metrics.entityMetrics.size > 0) {
-      summary += `\n📋 Per-Entity Breakdown:\n`;
-      for (const [entityName, entityMetric] of this.metrics.entityMetrics) {
-        const entityTotal =
-          entityMetric.successCount + entityMetric.failureCount;
-        const entityRate =
-          entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
-        const entityDuration = entityMetric.endTime
-          ? entityMetric.endTime - entityMetric.startTime
-          : 0;
-
-        summary += `   ${entityName}: ${entityTotal} total (${
-          entityMetric.successCount
-        } ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(
-          1
-        )}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
-      }
-    }
-
-    return summary;
-  }
-}

package/src/readers/csv.test.ts

@@ -1,82 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { readCsvFile } from "./csv";
-
-describe("CSV Reader", () => {
-  const testDataDir = path.join(__dirname, "test-data");
-  const testCsvPath = path.join(testDataDir, "test.csv");
-
-  beforeAll(() => {
-    if (!fs.existsSync(testDataDir)) {
-      fs.mkdirSync(testDataDir, { recursive: true });
-    }
-  });
-
-  afterAll(() => {
-    if (fs.existsSync(testDataDir)) {
-      fs.rmSync(testDataDir, { recursive: true });
-    }
-  });
-
-  beforeEach(() => {
-    if (fs.existsSync(testCsvPath)) {
-      fs.unlinkSync(testCsvPath);
-    }
-  });
-
-  it("should read a simple CSV file", async () => {
-    const csvContent = "name,age\nJohn,30\nJane,25";
-    fs.writeFileSync(testCsvPath, csvContent);
-
-    const result = await readCsvFile(testCsvPath);
-
-    expect(result).toEqual([
-      { name: "John", age: "30" },
-      { name: "Jane", age: "25" },
-    ]);
-  });
-
-  it("should read CSV with special characters", async () => {
-    const csvContent =
-      'name,description\n"John Doe","A person with, comma"\n"Jane\'s Data","Quote test"';
-    fs.writeFileSync(testCsvPath, csvContent);
-
-    const result = await readCsvFile(testCsvPath);
-
-    expect(result).toEqual([
-      { name: "John Doe", description: "A person with, comma" },
-      { name: "Jane's Data", description: "Quote test" },
-    ]);
-  });
-
-  it("should handle empty CSV file", async () => {
-    const csvContent = "name,age\n";
-    fs.writeFileSync(testCsvPath, csvContent);
-
-    const result = await readCsvFile(testCsvPath);
-
-    expect(result).toEqual([]);
-  });
-
-  it("should handle CSV with only headers", async () => {
-    const csvContent = "name,age";
-    fs.writeFileSync(testCsvPath, csvContent);
-
-    const result = await readCsvFile(testCsvPath);
-
-    expect(result).toEqual([]);
-  });
-
-  it("should handle CSV with missing values", async () => {
-    const csvContent = "name,age,city\nJohn,30,\nJane,,Boston\n,25,NYC";
-    fs.writeFileSync(testCsvPath, csvContent);
-
-    const result = await readCsvFile(testCsvPath);
-
-    expect(result).toEqual([
-      { name: "John", age: "30", city: "" },
-      { name: "Jane", age: "", city: "Boston" },
-      { name: "", age: "25", city: "NYC" },
-    ]);
-  });
-});

package/src/readers/csv.ts

@@ -1,29 +0,0 @@
-import fs from "fs";
-import csv from "csv-parser";
-import { DataReader, DataRow } from "./data-reader";
-
-export interface CsvRow {
-  [key: string]: string;
-}
-
-export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
-  return new Promise((resolve, reject) => {
-    const results: CsvRow[] = [];
-
-    fs.createReadStream(filePath)
-      .pipe(csv())
-      .on("data", (data) => results.push(data))
-      .on("end", () => resolve(results))
-      .on("error", (error) => reject(error));
-  });
-}
-
-export class CsvReader extends DataReader {
-  getSupportedExtensions(): string[] {
-    return ["csv"];
-  }
-
-  async readFile(filePath: string): Promise<DataRow[]> {
-    return readCsvFile(filePath);
-  }
-}

package/src/readers/data-reader.test.ts

@@ -1,104 +0,0 @@
-import { DataReader, DataReaderFactory } from "./data-reader";
-
-class TestReader extends DataReader {
-  getSupportedExtensions(): string[] {
-    return ["test", "tst"];
-  }
-
-  async readFile(filePath: string): Promise<any[]> {
-    return [{ test: true }];
-  }
-}
-
-describe("DataReader", () => {
-  describe("canHandle", () => {
-    it("should check if reader can handle file based on extension", () => {
-      const reader = new TestReader();
-
-      expect(reader.canHandle("file.test")).toBe(true);
-      expect(reader.canHandle("file.tst")).toBe(true);
-      expect(reader.canHandle("path/to/file.test")).toBe(true);
-      expect(reader.canHandle("file.other")).toBe(false);
-      expect(reader.canHandle("file")).toBe(false);
-    });
-  });
-});
-
-describe("DataReaderFactory", () => {
-  beforeEach(() => {
-    // Clear the readers array before each test
-    (DataReaderFactory as any).readers = [];
-  });
-
-  describe("registerReader", () => {
-    it("should register a reader", () => {
-      const reader = new TestReader();
-      DataReaderFactory.registerReader(reader);
-
-      expect(DataReaderFactory.getSupportedFormats()).toContain("test");
-      expect(DataReaderFactory.getSupportedFormats()).toContain("tst");
-    });
-  });
-
-  describe("getReader", () => {
-    beforeEach(() => {
-      const reader = new TestReader();
-      DataReaderFactory.registerReader(reader);
-    });
-
-    it("should get reader by file extension", () => {
-      const reader = DataReaderFactory.getReader("file.test");
-      expect(reader).toBeInstanceOf(TestReader);
-    });
-
-    it("should get reader by format override", () => {
-      const reader = DataReaderFactory.getReader("file.other", "test");
-      expect(reader).toBeInstanceOf(TestReader);
-    });
-
-    it("should prioritize format override over file extension", () => {
-      const reader = DataReaderFactory.getReader("file.unknown", "tst");
-      expect(reader).toBeInstanceOf(TestReader);
-    });
-
-    it("should throw error when no reader found", () => {
-      expect(() => DataReaderFactory.getReader("file.unknown")).toThrow(
-        "No reader found for file: file.unknown"
-      );
-    });
-
-    it("should throw error when format specified but no reader found", () => {
-      expect(() => DataReaderFactory.getReader("file.txt", "unknown")).toThrow(
-        "No reader found for file: file.txt with format: unknown"
-      );
-    });
-  });
-
-  describe("getSupportedFormats", () => {
-    it("should return empty array when no readers registered", () => {
-      expect(DataReaderFactory.getSupportedFormats()).toEqual([]);
-    });
-
-    it("should return all supported formats", () => {
-      const reader1 = new TestReader();
-      DataReaderFactory.registerReader(reader1);
-
-      const formats = DataReaderFactory.getSupportedFormats();
-      expect(formats).toContain("test");
-      expect(formats).toContain("tst");
-      expect(formats.length).toBe(2);
-    });
-
-    it("should not duplicate formats", () => {
-      const reader1 = new TestReader();
-      const reader2 = new TestReader();
-      DataReaderFactory.registerReader(reader1);
-      DataReaderFactory.registerReader(reader2);
-
-      const formats = DataReaderFactory.getSupportedFormats();
-      expect(formats).toContain("test");
-      expect(formats).toContain("tst");
-      expect(formats.length).toBe(2);
-    });
-  });
-});

package/src/readers/data-reader.ts

@@ -1,61 +0,0 @@
-export interface DataRow {
-  [key: string]: any;
-}
-
-export abstract class DataReader {
-  abstract readFile(filePath: string): Promise<DataRow[]>;
-
-  /**
-   * Get the supported file extensions for this reader
-   */
-  abstract getSupportedExtensions(): string[];
-
-  /**
-   * Check if this reader can handle the given file
-   */
-  canHandle(filePath: string): boolean {
-    const extension = filePath.split(".").pop()?.toLowerCase();
-    return extension
-      ? this.getSupportedExtensions().includes(extension)
-      : false;
-  }
-}
-
-export class DataReaderFactory {
-  private static readers: DataReader[] = [];
-
-  static registerReader(reader: DataReader): void {
-    this.readers.push(reader);
-  }
-
-  static getReader(filePath: string, format?: string): DataReader {
-    // If format is specified, try to find reader by format
-    if (format) {
-      const reader = this.readers.find((r) =>
-        r.getSupportedExtensions().includes(format.toLowerCase())
-      );
-      if (reader) return reader;
-    }
-
-    // Otherwise, try to find reader by file extension
-    const reader = this.readers.find((r) => r.canHandle(filePath));
-
-    if (!reader) {
-      throw new Error(
-        `No reader found for file: ${filePath}${
-          format ? ` with format: ${format}` : ""
-        }`
-      );
-    }
-
-    return reader;
-  }
-
-  static getSupportedFormats(): string[] {
-    const formats = new Set<string>();
-    this.readers.forEach((reader) => {
-      reader.getSupportedExtensions().forEach((ext) => formats.add(ext));
-    });
-    return Array.from(formats);
-  }
-}

package/src/readers/index.ts

@@ -1,18 +0,0 @@
-export { DataReader, DataRow, DataReaderFactory } from "./data-reader";
-export { CsvReader, readCsvFile, CsvRow } from "./csv";
-export { JsonReader } from "./json";
-export { YamlReader } from "./yaml";
-export { JsonlReader } from "./jsonl";
-
-// Register all readers
-import { DataReaderFactory } from "./data-reader";
-import { CsvReader } from "./csv";
-import { JsonReader } from "./json";
-import { YamlReader } from "./yaml";
-import { JsonlReader } from "./jsonl";
-
-// Register readers on module load
-DataReaderFactory.registerReader(new CsvReader());
-DataReaderFactory.registerReader(new JsonReader());
-DataReaderFactory.registerReader(new YamlReader());
-DataReaderFactory.registerReader(new JsonlReader());

package/src/readers/json.test.ts

@@ -1,80 +0,0 @@
-import fs from "fs/promises";
-import { JsonReader } from "./json";
-
-jest.mock("fs/promises");
-
-describe("JsonReader", () => {
-  let reader: JsonReader;
-  const mockFs = fs as jest.Mocked<typeof fs>;
-
-  beforeEach(() => {
-    reader = new JsonReader();
-    jest.clearAllMocks();
-  });
-
-  describe("getSupportedExtensions", () => {
-    it("should return json as supported extension", () => {
-      expect(reader.getSupportedExtensions()).toEqual(["json"]);
-    });
-  });
-
-  describe("canHandle", () => {
-    it("should return true for .json files", () => {
-      expect(reader.canHandle("data.json")).toBe(true);
-      expect(reader.canHandle("path/to/file.json")).toBe(true);
-      expect(reader.canHandle("file.JSON")).toBe(true); // case insensitive
-    });
-
-    it("should return false for non-json files", () => {
-      expect(reader.canHandle("data.csv")).toBe(false);
-      expect(reader.canHandle("data.yaml")).toBe(false);
-      expect(reader.canHandle("data")).toBe(false);
-    });
-  });
-
-  describe("readFile", () => {
-    it("should read and parse JSON array", async () => {
-      const mockData = [
-        { id: 1, name: "Item 1" },
-        { id: 2, name: "Item 2" },
-      ];
-      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
-
-      const result = await reader.readFile("data.json");
-
-      expect(mockFs.readFile).toHaveBeenCalledWith("data.json", "utf8");
-      expect(result).toEqual(mockData);
-    });
-
-    it("should wrap single object in array", async () => {
-      const mockData = { id: 1, name: "Item 1" };
-      mockFs.readFile.mockResolvedValue(JSON.stringify(mockData));
-
-      const result = await reader.readFile("data.json");
-
-      expect(result).toEqual([mockData]);
-    });
-
-    it("should throw error for invalid JSON", async () => {
-      mockFs.readFile.mockResolvedValue("invalid json");
-
-      await expect(reader.readFile("data.json")).rejects.toThrow();
-    });
-
-    it("should throw error for null data", async () => {
-      mockFs.readFile.mockResolvedValue("null");
-
-      await expect(reader.readFile("data.json")).rejects.toThrow(
-        "Invalid JSON data structure in file: data.json. Expected array or object."
-      );
-    });
-
-    it("should throw error for primitive values", async () => {
-      mockFs.readFile.mockResolvedValue('"string value"');
-
-      await expect(reader.readFile("data.json")).rejects.toThrow(
-        "Invalid JSON data structure in file: data.json. Expected array or object."
-      );
-    });
-  });
-});

package/src/readers/json.ts

@@ -1,27 +0,0 @@
-import fs from "fs/promises";
-import { DataReader, DataRow } from "./data-reader";
-
-export class JsonReader extends DataReader {
-  getSupportedExtensions(): string[] {
-    return ["json"];
-  }
-
-  async readFile(filePath: string): Promise<DataRow[]> {
-    const content = await fs.readFile(filePath, "utf8");
-    const data = JSON.parse(content);
-
-    // If the data is already an array, return it
-    if (Array.isArray(data)) {
-      return data;
-    }
-
-    // If it's a single object, wrap it in an array
-    if (typeof data === "object" && data !== null) {
-      return [data];
-    }
-
-    throw new Error(
-      `Invalid JSON data structure in file: ${filePath}. Expected array or object.`
-    );
-  }
-}

package/src/readers/jsonl.test.ts

@@ -1,96 +0,0 @@
-import fs from "fs/promises";
-import { JsonlReader } from "./jsonl";
-
-jest.mock("fs/promises");
-
-describe("JsonlReader", () => {
-  let reader: JsonlReader;
-  const mockFs = fs as jest.Mocked<typeof fs>;
-
-  beforeEach(() => {
-    reader = new JsonlReader();
-    jest.clearAllMocks();
-  });
-
-  describe("getSupportedExtensions", () => {
-    it("should return jsonl and ndjson as supported extensions", () => {
-      expect(reader.getSupportedExtensions()).toEqual(["jsonl", "ndjson"]);
-    });
-  });
-
-  describe("canHandle", () => {
-    it("should return true for .jsonl and .ndjson files", () => {
-      expect(reader.canHandle("data.jsonl")).toBe(true);
-      expect(reader.canHandle("data.ndjson")).toBe(true);
-      expect(reader.canHandle("path/to/file.jsonl")).toBe(true);
-    });
-
-    it("should return false for non-jsonl files", () => {
-      expect(reader.canHandle("data.json")).toBe(false);
-      expect(reader.canHandle("data.csv")).toBe(false);
-      expect(reader.canHandle("data")).toBe(false);
-    });
-  });
-
-  describe("readFile", () => {
-    it("should read and parse JSONL file", async () => {
-      const line1 = { id: 1, name: "Item 1" };
-      const line2 = { id: 2, name: "Item 2" };
-      const jsonlContent = `${JSON.stringify(line1)}\n${JSON.stringify(line2)}`;
-      mockFs.readFile.mockResolvedValue(jsonlContent);
-
-      const result = await reader.readFile("data.jsonl");
-
-      expect(mockFs.readFile).toHaveBeenCalledWith("data.jsonl", "utf8");
-      expect(result).toEqual([line1, line2]);
-    });
-
-    it("should handle empty lines", async () => {
-      const line1 = { id: 1, name: "Item 1" };
-      const line2 = { id: 2, name: "Item 2" };
-      const jsonlContent = `${JSON.stringify(line1)}\n\n${JSON.stringify(
-        line2
-      )}\n`;
-      mockFs.readFile.mockResolvedValue(jsonlContent);
-
-      const result = await reader.readFile("data.jsonl");
-
-      expect(result).toEqual([line1, line2]);
-    });
-
-    it("should handle single line", async () => {
-      const line = { id: 1, name: "Item 1" };
-      mockFs.readFile.mockResolvedValue(JSON.stringify(line));
-
-      const result = await reader.readFile("data.jsonl");
-
-      expect(result).toEqual([line]);
-    });
-
-    it("should throw error for invalid JSON on specific line", async () => {
-      const line1 = { id: 1, name: "Item 1" };
-      const jsonlContent = `${JSON.stringify(line1)}\ninvalid json\n`;
-      mockFs.readFile.mockResolvedValue(jsonlContent);
-
-      await expect(reader.readFile("data.jsonl")).rejects.toThrow(
-        "Invalid JSON at line 2 in file: data.jsonl"
-      );
-    });
-
-    it("should handle empty file", async () => {
-      mockFs.readFile.mockResolvedValue("");
-
-      const result = await reader.readFile("data.jsonl");
-
-      expect(result).toEqual([]);
-    });
-
-    it("should handle file with only whitespace", async () => {
-      mockFs.readFile.mockResolvedValue("\n\n  \n\t\n");
-
-      const result = await reader.readFile("data.jsonl");
-
-      expect(result).toEqual([]);
-    });
-  });
-});

package/src/readers/jsonl.ts

@@ -1,28 +0,0 @@
-import fs from "fs/promises";
-import { DataReader, DataRow } from "./data-reader";
-
-export class JsonlReader extends DataReader {
-  getSupportedExtensions(): string[] {
-    return ["jsonl", "ndjson"];
-  }
-
-  async readFile(filePath: string): Promise<DataRow[]> {
-    const content = await fs.readFile(filePath, "utf8");
-    const lines = content.split("\n").filter((line) => line.trim());
-
-    const results: DataRow[] = [];
-
-    for (let i = 0; i < lines.length; i++) {
-      try {
-        const data = JSON.parse(lines[i]);
-        results.push(data);
-      } catch (error) {
-        throw new Error(
-          `Invalid JSON at line ${i + 1} in file: ${filePath}. Error: ${error}`
-        );
-      }
-    }
-
-    return results;
-  }
-}