@jackchuka/gql-ingest 1.5.0 → 2.0.1

package/src/readers/jsonl.test.ts

@@ -0,0 +1,96 @@
+import fs from "fs/promises";
+import { JsonlReader } from "./jsonl";
+
+jest.mock("fs/promises");
+
+describe("JsonlReader", () => {
+  let reader: JsonlReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+
+  beforeEach(() => {
+    reader = new JsonlReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return jsonl and ndjson as supported extensions", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["jsonl", "ndjson"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .jsonl and .ndjson files", () => {
+      expect(reader.canHandle("data.jsonl")).toBe(true);
+      expect(reader.canHandle("data.ndjson")).toBe(true);
+      expect(reader.canHandle("path/to/file.jsonl")).toBe(true);
+    });
+
+    it("should return false for non-jsonl files", () => {
+      expect(reader.canHandle("data.json")).toBe(false);
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse JSONL file", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const line2 = { id: 2, name: "Item 2" };
+      const jsonlContent = `${JSON.stringify(line1)}\n${JSON.stringify(line2)}`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.jsonl", "utf8");
+      expect(result).toEqual([line1, line2]);
+    });
+
+    it("should handle empty lines", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const line2 = { id: 2, name: "Item 2" };
+      const jsonlContent = `${JSON.stringify(line1)}\n\n${JSON.stringify(
+        line2
+      )}\n`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([line1, line2]);
+    });
+
+    it("should handle single line", async () => {
+      const line = { id: 1, name: "Item 1" };
+      mockFs.readFile.mockResolvedValue(JSON.stringify(line));
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([line]);
+    });
+
+    it("should throw error for invalid JSON on specific line", async () => {
+      const line1 = { id: 1, name: "Item 1" };
+      const jsonlContent = `${JSON.stringify(line1)}\ninvalid json\n`;
+      mockFs.readFile.mockResolvedValue(jsonlContent);
+
+      await expect(reader.readFile("data.jsonl")).rejects.toThrow(
+        "Invalid JSON at line 2 in file: data.jsonl"
+      );
+    });
+
+    it("should handle empty file", async () => {
+      mockFs.readFile.mockResolvedValue("");
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([]);
+    });
+
+    it("should handle file with only whitespace", async () => {
+      mockFs.readFile.mockResolvedValue("\n\n  \n\t\n");
+
+      const result = await reader.readFile("data.jsonl");
+
+      expect(result).toEqual([]);
+    });
+  });
+});

package/src/readers/jsonl.ts

@@ -0,0 +1,28 @@
+import fs from "fs/promises";
+import { DataReader, DataRow } from "./data-reader";
+
+export class JsonlReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["jsonl", "ndjson"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const lines = content.split("\n").filter((line) => line.trim());
+
+    const results: DataRow[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      try {
+        const data = JSON.parse(lines[i]);
+        results.push(data);
+      } catch (error) {
+        throw new Error(
+          `Invalid JSON at line ${i + 1} in file: ${filePath}. Error: ${error}`
+        );
+      }
+    }
+
+    return results;
+  }
+}

package/src/readers/yaml.test.ts

@@ -0,0 +1,95 @@
+import fs from "fs/promises";
+import yaml from "js-yaml";
+import { YamlReader } from "./yaml";
+
+jest.mock("fs/promises");
+jest.mock("js-yaml");
+
+describe("YamlReader", () => {
+  let reader: YamlReader;
+  const mockFs = fs as jest.Mocked<typeof fs>;
+  const mockYaml = yaml as jest.Mocked<typeof yaml>;
+
+  beforeEach(() => {
+    reader = new YamlReader();
+    jest.clearAllMocks();
+  });
+
+  describe("getSupportedExtensions", () => {
+    it("should return yaml and yml as supported extensions", () => {
+      expect(reader.getSupportedExtensions()).toEqual(["yaml", "yml"]);
+    });
+  });
+
+  describe("canHandle", () => {
+    it("should return true for .yaml and .yml files", () => {
+      expect(reader.canHandle("data.yaml")).toBe(true);
+      expect(reader.canHandle("data.yml")).toBe(true);
+      expect(reader.canHandle("path/to/file.yaml")).toBe(true);
+    });
+
+    it("should return false for non-yaml files", () => {
+      expect(reader.canHandle("data.json")).toBe(false);
+      expect(reader.canHandle("data.csv")).toBe(false);
+      expect(reader.canHandle("data")).toBe(false);
+    });
+  });
+
+  describe("readFile", () => {
+    it("should read and parse YAML array", async () => {
+      const mockData = [
+        { id: 1, name: "Item 1" },
+        { id: 2, name: "Item 2" },
+      ];
+      const yamlContent = "- id: 1\n  name: Item 1\n- id: 2\n  name: Item 2";
+      mockFs.readFile.mockResolvedValue(yamlContent);
+      mockYaml.load.mockReturnValue(mockData);
+
+      const result = await reader.readFile("data.yaml");
+
+      expect(mockFs.readFile).toHaveBeenCalledWith("data.yaml", "utf8");
+      expect(mockYaml.load).toHaveBeenCalledWith(yamlContent);
+      expect(result).toEqual(mockData);
+    });
+
+    it("should wrap single object in array", async () => {
+      const mockData = { id: 1, name: "Item 1" };
+      const yamlContent = "id: 1\nname: Item 1";
+      mockFs.readFile.mockResolvedValue(yamlContent);
+      mockYaml.load.mockReturnValue(mockData);
+
+      const result = await reader.readFile("data.yaml");
+
+      expect(result).toEqual([mockData]);
+    });
+
+    it("should throw error for invalid YAML", async () => {
+      mockFs.readFile.mockResolvedValue("invalid: yaml: content:");
+      mockYaml.load.mockImplementation(() => {
+        throw new Error("Invalid YAML");
+      });
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML"
+      );
+    });
+
+    it("should throw error for null data", async () => {
+      mockFs.readFile.mockResolvedValue("null");
+      mockYaml.load.mockReturnValue(null);
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML data structure in file: data.yaml. Expected array or object."
+      );
+    });
+
+    it("should throw error for primitive values", async () => {
+      mockFs.readFile.mockResolvedValue("string value");
+      mockYaml.load.mockReturnValue("string value");
+
+      await expect(reader.readFile("data.yaml")).rejects.toThrow(
+        "Invalid YAML data structure in file: data.yaml. Expected array or object."
+      );
+    });
+  });
+});

package/src/readers/yaml.ts

@@ -0,0 +1,28 @@
+import fs from "fs/promises";
+import yaml from "js-yaml";
+import { DataReader, DataRow } from "./data-reader";
+
+export class YamlReader extends DataReader {
+  getSupportedExtensions(): string[] {
+    return ["yaml", "yml"];
+  }
+
+  async readFile(filePath: string): Promise<DataRow[]> {
+    const content = await fs.readFile(filePath, "utf8");
+    const data = yaml.load(content);
+
+    // If the data is already an array, return it
+    if (Array.isArray(data)) {
+      return data;
+    }
+
+    // If it's a single object, wrap it in an array
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+
+    throw new Error(
+      `Invalid YAML data structure in file: ${filePath}. Expected array or object.`
+    );
+  }
+}

package/dist/csv-reader.d.ts

@@ -1,5 +0,0 @@
-export interface CsvRow {
-    [key: string]: string;
-}
-export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
-//# sourceMappingURL=csv-reader.d.ts.map

package/dist/csv-reader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../src/csv-reader.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUrE"}

package/dist/csv-reader.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=csv-reader.test.d.ts.map

package/dist/csv-reader.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv-reader.test.d.ts","sourceRoot":"","sources":["../src/csv-reader.test.ts"],"names":[],"mappings":""}

package/src/csv-reader.ts

@@ -1,18 +0,0 @@
-import fs from 'fs';
-import csv from 'csv-parser';
-
-export interface CsvRow {
-  [key: string]: string;
-}
-
-export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
-  return new Promise((resolve, reject) => {
-    const results: CsvRow[] = [];
-    
-    fs.createReadStream(filePath)
-      .pipe(csv())
-      .on('data', (data) => results.push(data))
-      .on('end', () => resolve(results))
-      .on('error', (error) => reject(error));
-  });
-}