@jackchuka/gql-ingest 1.0.2 → 1.2.0

package/src/dependency-resolver.ts

@@ -0,0 +1,98 @@
+export interface DependencyGraph {
+  [entityName: string]: string[];
+}
+
+export interface ExecutionWave {
+  entities: string[];
+  wave: number;
+}
+
+export class DependencyResolver {
+  private dependencies: DependencyGraph;
+  private entities: string[];
+
+  constructor(entities: string[], dependencies: DependencyGraph = {}) {
+    this.entities = entities;
+    this.dependencies = dependencies;
+  }
+
+  resolveExecutionOrder(): ExecutionWave[] {
+    const waves: ExecutionWave[] = [];
+    const processed = new Set<string>();
+    const inProgress = new Set<string>();
+    let waveNumber = 0;
+
+    while (processed.size < this.entities.length) {
+      const currentWave: string[] = [];
+
+      for (const entity of this.entities) {
+        if (processed.has(entity) || inProgress.has(entity)) {
+          continue;
+        }
+
+        const deps = this.dependencies[entity] || [];
+        const canProcess = deps.every((dep) => processed.has(dep));
+
+        if (canProcess) {
+          currentWave.push(entity);
+          inProgress.add(entity);
+        }
+      }
+
+      if (currentWave.length === 0) {
+        const remaining = this.entities.filter((e) => !processed.has(e));
+        throw new Error(
+          `Circular dependency detected or missing dependencies for entities: ${remaining.join(
+            ", "
+          )}`
+        );
+      }
+
+      waves.push({
+        entities: currentWave,
+        wave: waveNumber++,
+      });
+
+      currentWave.forEach((entity) => {
+        processed.add(entity);
+        inProgress.delete(entity);
+      });
+    }
+
+    return waves;
+  }
+
+  validateDependencies(): string[] {
+    const errors: string[] = [];
+    const entitySet = new Set(this.entities);
+
+    for (const [entity, deps] of Object.entries(this.dependencies)) {
+      if (!entitySet.has(entity)) {
+        errors.push(
+          `Entity '${entity}' has dependencies but is not in the entity list`
+        );
+        continue;
+      }
+
+      for (const dep of deps) {
+        if (!entitySet.has(dep)) {
+          errors.push(
+            `Entity '${entity}' depends on '${dep}' which does not exist`
+          );
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  getDependents(entityName: string): string[] {
+    return Object.entries(this.dependencies)
+      .filter(([_, deps]) => deps.includes(entityName))
+      .map(([entity, _]) => entity);
+  }
+
+  getDependencies(entityName: string): string[] {
+    return this.dependencies[entityName] || [];
+  }
+}

package/src/graphql-client.ts

@@ -1,20 +1,46 @@
 import { GraphQLClient } from 'graphql-request';
+import { MetricsCollector } from './metrics';
 
 export class GraphQLClientWrapper {
   private client: GraphQLClient;
+  private metrics?: MetricsCollector;
+  private verbose: boolean;
 
-  constructor(endpoint: string, headers?: Record<string, string>) {
+  constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose: boolean = false) {
     this.client = new GraphQLClient(endpoint, {
       headers: headers || {}
     });
+    this.metrics = metrics;
+    this.verbose = verbose;
   }
 
   async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
+    const startTime = Date.now();
+    
     try {
       const result = await this.client.request(mutation, variables);
+      
+      if (this.metrics) {
+        const duration = Date.now() - startTime;
+        this.metrics.recordRequestDuration(duration);
+      }
+      
+      if (this.verbose) {
+        console.log(`✓ GraphQL request completed in ${Date.now() - startTime}ms:`, result);
+      }
+      
       return result;
     } catch (error) {
-      console.error('GraphQL mutation failed:', error);
+      if (this.metrics) {
+        const duration = Date.now() - startTime;
+        this.metrics.recordRequestDuration(duration);
+      }
+      
+      if (this.verbose) {
+        console.error(`✗ GraphQL request failed in ${Date.now() - startTime}ms:`, error);
+      } else {
+        console.error('GraphQL mutation failed:', error);
+      }
       throw error;
     }
   }

package/src/mapper.test.ts

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import { DataMapper } from "./mapper";
 import { GraphQLClientWrapper } from "./graphql-client";
+import { MetricsCollector } from "./metrics";
 
 jest.mock("fs");
 jest.mock("./csv-reader");
@@ -10,6 +11,7 @@ const mockFs = fs as jest.Mocked<typeof fs>;
 
 describe("DataMapper", () => {
   let mockClient: jest.Mocked<GraphQLClientWrapper>;
+  let mockMetrics: jest.Mocked<MetricsCollector>;
   let dataMapper: DataMapper;
   const testBasePath = "/test/base/path";
 
@@ -19,7 +21,15 @@ describe("DataMapper", () => {
       setHeaders: jest.fn(),
     } as any;
 
-    dataMapper = new DataMapper(mockClient, testBasePath);
+    mockMetrics = {
+      startEntityProcessing: jest.fn(),
+      recordSuccess: jest.fn(),
+      recordFailure: jest.fn(),
+      finishEntityProcessing: jest.fn(),
+      getMetrics: jest.fn(),
+    } as any;
+
+    dataMapper = new DataMapper(mockClient, testBasePath, mockMetrics);
   });
 
   afterEach(() => {
@@ -165,7 +175,7 @@ describe("DataMapper", () => {
       await dataMapper.processEntity("configs/test/mappings/users.json");
 
       expect(consoleSpy).toHaveBeenCalledWith(
-        "✗ Failed to create entity for row:",
+        "✗ Failed to create entity for row 1:",
         { user_name: "John" },
         expect.any(Error)
       );
@@ -252,5 +262,74 @@ describe("DataMapper", () => {
         email: "john@example.com",
       });
     });
+
+    it("should call metrics methods during successful processing", async () => {
+      const mockConfig = {
+        csvFile: "data/users.csv",
+        graphqlFile: "graphql/users.graphql",
+        mapping: { name: "user_name" },
+      };
+
+      const mockCsvData = [{ user_name: "John" }, { user_name: "Jane" }];
+      const mockMutation =
+        "mutation CreateUser($name: String!) { createUser(input: { name: $name }) { id } }";
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockResolvedValue({
+        createUser: { id: "123" },
+      });
+
+      await dataMapper.processEntity("configs/test/mappings/users.json");
+
+      expect(mockMetrics.startEntityProcessing).toHaveBeenCalledWith("users");
+      expect(mockMetrics.recordSuccess).toHaveBeenCalledTimes(2);
+      expect(mockMetrics.recordSuccess).toHaveBeenCalledWith("users");
+      expect(mockMetrics.finishEntityProcessing).toHaveBeenCalledWith("users");
+      expect(mockMetrics.recordFailure).not.toHaveBeenCalled();
+    });
+
+    it("should call metrics methods during failed processing", async () => {
+      const mockConfig = {
+        csvFile: "data/users.csv",
+        graphqlFile: "graphql/users.graphql",
+        mapping: { name: "user_name" },
+      };
+
+      const mockCsvData = [{ user_name: "John" }];
+      const mockMutation =
+        "mutation CreateUser($name: String!) { createUser(input: { name: $name }) { id } }";
+
+      mockFs.readFileSync
+        .mockReturnValueOnce(JSON.stringify(mockConfig))
+        .mockReturnValueOnce(mockMutation);
+
+      const { readCsvFile } = require("./csv-reader");
+      readCsvFile.mockResolvedValue(mockCsvData);
+
+      mockClient.executeMutation.mockRejectedValue(new Error("GraphQL error"));
+
+      const consoleSpy = jest.spyOn(console, "error").mockImplementation();
+
+      await dataMapper.processEntity("configs/test/mappings/users.json");
+
+      expect(mockMetrics.startEntityProcessing).toHaveBeenCalledWith("users");
+      expect(mockMetrics.recordFailure).toHaveBeenCalledTimes(1);
+      expect(mockMetrics.recordFailure).toHaveBeenCalledWith("users");
+      expect(mockMetrics.finishEntityProcessing).toHaveBeenCalledWith("users");
+      expect(mockMetrics.recordSuccess).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should expose metrics through getMetrics method", () => {
+      const metrics = dataMapper.getMetrics();
+      expect(metrics).toBe(mockMetrics);
+    });
   });
 });

package/src/mapper.ts

@@ -1,7 +1,9 @@
-import fs from 'fs';
-import path from 'path';
-import { readCsvFile, CsvRow } from './csv-reader';
-import { GraphQLClientWrapper } from './graphql-client';
+import fs from "fs";
+import path from "path";
+import { readCsvFile, CsvRow } from "./csv-reader";
+import { GraphQLClientWrapper } from "./graphql-client";
+import { MetricsCollector } from "./metrics";
+import { ParallelProcessingConfig } from "./config";
 
 export interface MappingConfig {
   csvFile: string;
@@ -12,69 +14,205 @@ export interface MappingConfig {
 export class DataMapper {
   private client: GraphQLClientWrapper;
   private basePath: string;
+  private metrics: MetricsCollector;
+  private verbose: boolean;
 
-  constructor(client: GraphQLClientWrapper, basePath: string = process.cwd()) {
+  constructor(
+    client: GraphQLClientWrapper,
+    basePath: string = process.cwd(),
+    metrics?: MetricsCollector,
+    verbose: boolean = false
+  ) {
     this.client = client;
     this.basePath = basePath;
+    this.metrics = metrics || new MetricsCollector();
+    this.verbose = verbose;
   }
 
   discoverMappings(configDir: string): string[] {
-    const mappingsPath = path.resolve(this.basePath, configDir, 'mappings');
-    
+    const mappingsPath = path.resolve(this.basePath, configDir, "mappings");
+
     try {
       const files = fs.readdirSync(mappingsPath);
-      const jsonFiles = files
-        .filter(file => file.endsWith('.json'))
-        .sort(); // Alphabetical order for consistent processing
-      
-      console.log(`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(', ')}`);
-      return jsonFiles.map(file => path.join(configDir, 'mappings', file));
+      const jsonFiles = files.filter((file) => file.endsWith(".json")).sort(); // Alphabetical order for consistent processing
+
+      console.log(
+        `Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`
+      );
+      return jsonFiles.map((file) => path.join(configDir, "mappings", file));
     } catch (error) {
       console.error(`Error reading mappings directory ${mappingsPath}:`, error);
       return [];
     }
   }
 
-  async processEntity(configPath: string): Promise<void> {
+  async processEntity(
+    configPath: string,
+    parallelConfig?: ParallelProcessingConfig
+  ): Promise<void> {
+    const entityName = path.basename(configPath, ".json");
     console.log(`Processing entity: ${configPath}`);
-    
+
+    this.metrics.startEntityProcessing(entityName);
+
     // Read mapping configuration
     const configFullPath = path.resolve(this.basePath, configPath);
-    const config: MappingConfig = JSON.parse(fs.readFileSync(configFullPath, 'utf8'));
-    
+    const config: MappingConfig = JSON.parse(
+      fs.readFileSync(configFullPath, "utf8")
+    );
+
     // Extract config directory (parent of mappings directory)
     const configDir = path.dirname(path.dirname(configFullPath));
-    
+
     // Read CSV data (relative to config directory)
     const csvPath = path.resolve(configDir, config.csvFile);
     const csvData = await readCsvFile(csvPath);
-    
+
     // Read GraphQL mutation (relative to config directory)
     const graphqlPath = path.resolve(configDir, config.graphqlFile);
-    const mutation = fs.readFileSync(graphqlPath, 'utf8');
+    const mutation = fs.readFileSync(graphqlPath, "utf8");
+
+    // Process rows with optional parallelization
+    if (parallelConfig && parallelConfig.concurrency > 1) {
+      await this.processRowsConcurrently(
+        csvData,
+        mutation,
+        config.mapping,
+        entityName,
+        parallelConfig
+      );
+    } else {
+      await this.processRowsSequentially(
+        csvData,
+        mutation,
+        config.mapping,
+        entityName
+      );
+    }
+
+    this.metrics.finishEntityProcessing(entityName);
+  }
+
+  private async processRowsSequentially(
+    csvData: CsvRow[],
+    mutation: string,
+    mapping: Record<string, string>,
+    entityName: string
+  ): Promise<void> {
+    const totalRows = csvData.length;
     
-    // Process each row
-    for (const row of csvData) {
-      const variables = this.mapCsvRowToVariables(row, config.mapping);
-      
+    for (let i = 0; i < csvData.length; i++) {
+      const row = csvData[i];
+      const variables = this.mapCsvRowToVariables(row, mapping);
+
       try {
-        const result = await this.client.executeMutation(mutation, variables);
-        console.log(`✓ Created entity with result:`, result);
+        await this.client.executeMutation(mutation, variables);
+        this.metrics.recordSuccess(entityName);
+        
+        // Show progress every 10% or at the end (only in non-verbose mode)
+        if (!this.verbose && ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1)) {
+          const progress = (((i + 1) / totalRows) * 100).toFixed(1);
+          console.log(`📊 Progress: ${i + 1}/${totalRows} (${progress}%) ✓`);
+        }
       } catch (error) {
-        console.error(`✗ Failed to create entity for row:`, row, error);
+        this.metrics.recordFailure(entityName);
+        if (!this.verbose) {
+          console.error(`✗ Failed to create entity for row ${i + 1}:`, row, error);
+        }
       }
     }
   }
 
-  private mapCsvRowToVariables(row: CsvRow, mapping: Record<string, string>): Record<string, any> {
+  private async processRowsConcurrently(
+    csvData: CsvRow[],
+    mutation: string,
+    mapping: Record<string, string>,
+    entityName: string,
+    parallelConfig: ParallelProcessingConfig
+  ): Promise<void> {
+    const concurrency = parallelConfig.concurrency;
+    console.log(
+      `Processing ${csvData.length} rows with concurrency: ${concurrency}`
+    );
+
+    // Split data into chunks for concurrent processing
+    const chunks = this.chunkArray(csvData, concurrency);
+    let processedCount = 0;
+    const totalRows = csvData.length;
+
+    for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
+      const chunk = chunks[chunkIndex];
+      const promises = chunk.map(async (row) => {
+        const variables = this.mapCsvRowToVariables(row, mapping);
+
+        try {
+          const result = await this.client.executeMutation(mutation, variables);
+          this.metrics.recordSuccess(entityName);
+          return { success: true, result, row };
+        } catch (error) {
+          this.metrics.recordFailure(entityName);
+          return { success: false, error, row };
+        }
+      });
+
+      const results = await Promise.allSettled(promises);
+      processedCount += chunk.length;
+
+      // Count successes and failures in this chunk
+      let chunkSuccesses = 0;
+      let chunkFailures = 0;
+      
+      results.forEach((result) => {
+        if (result.status === "fulfilled") {
+          const { success, error, row } = result.value;
+          if (success) {
+            chunkSuccesses++;
+          } else {
+            chunkFailures++;
+            if (!this.verbose) {
+              console.error(`✗ Failed to create entity for row:`, row, error);
+            }
+          }
+        } else {
+          chunkFailures++;
+          if (!this.verbose) {
+            console.error(`✗ Promise rejected:`, result.reason);
+          }
+        }
+      });
+
+      // Show progress update (only in non-verbose mode)
+      if (!this.verbose) {
+        const progress = ((processedCount / totalRows) * 100).toFixed(1);
+        console.log(`📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${chunkIndex + 1}: ${chunkSuccesses} ✓, ${chunkFailures} ✗`);
+      }
+    }
+  }
+
+  private chunkArray<T>(array: T[], chunkSize: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+
+  private mapCsvRowToVariables(
+    row: CsvRow,
+    mapping: Record<string, string>
+  ): Record<string, any> {
     const variables: Record<string, any> = {};
-    
+
     for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
       if (row[csvColumn] !== undefined) {
         variables[graphqlVar] = row[csvColumn];
       }
     }
-    
+
     return variables;
   }
-}
+
+  getMetrics(): MetricsCollector {
+    return this.metrics;
+  }
+}