@jackchuka/gql-ingest 1.0.2 → 1.2.0

package/src/cli.ts

@@ -1,6 +1,20 @@
 import { Command } from "commander";
 import { GraphQLClientWrapper } from "./graphql-client";
 import { DataMapper } from "./mapper";
+import { MetricsCollector } from "./metrics";
+import { loadConfig, getEntityConfig } from "./config";
+import { DependencyResolver } from "./dependency-resolver";
+import { basename } from "path";
+
+// Utility function to chunk array into smaller arrays
+function chunkArray<T>(array: T[], chunkSize: number): T[][] {
+  if (chunkSize <= 0) return [array];
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    chunks.push(array.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
 
 const program = new Command();
 
@@ -21,6 +35,7 @@ program
     "-h, --headers <headers>",
     "JSON string of headers to include in requests"
   )
+  .option("-v, --verbose", "Show detailed request results and responses")
   .action(async (options) => {
     try {
       console.log("Starting seed data generation...");
@@ -28,11 +43,27 @@ program
       // Parse headers if provided
       const headers = options.headers ? JSON.parse(options.headers) : {};
 
+      // Initialize metrics collector
+      const metrics = new MetricsCollector();
+
       // Initialize GraphQL client
-      const client = new GraphQLClientWrapper(options.endpoint, headers);
+      const client = new GraphQLClientWrapper(
+        options.endpoint,
+        headers,
+        metrics,
+        options.verbose
+      );
+
+      // Load configuration
+      const config = loadConfig(options.config);
 
       // Initialize data mapper
-      const mapper = new DataMapper(client);
+      const mapper = new DataMapper(
+        client,
+        process.cwd(),
+        metrics,
+        options.verbose
+      );
 
       // Discover all mapping files dynamically
       const mappingPaths = mapper.discoverMappings(options.config);
@@ -42,19 +73,92 @@ program
         return;
       }
 
-      for (const configPath of mappingPaths) {
-        try {
-          await mapper.processEntity(configPath);
-        } catch (error) {
-          console.warn(`Warning: Could not process ${configPath}:`, error);
-        }
+      // Extract entity names from mapping paths
+      const entityNames = mappingPaths.map((path) => basename(path, ".json"));
+
+      // Setup dependency resolver
+      const resolver = new DependencyResolver(
+        entityNames,
+        config.entityDependencies
+      );
+
+      // Validate dependencies
+      const validationErrors = resolver.validateDependencies();
+      if (validationErrors.length > 0) {
+        console.error("Dependency validation errors:");
+        validationErrors.forEach((error) => console.error(`  - ${error}`));
+        process.exit(1);
+      }
+
+      // Process entities in dependency-aware waves
+      if (config.parallelProcessing.entityConcurrency === 1) {
+        await processEntitiesSequentially(mappingPaths, mapper, config);
+      } else {
+        await processEntitiesInWaves(mappingPaths, resolver, mapper, config);
       }
 
-      console.log("Seed data generation completed!");
+      metrics.finishProcessing();
+      console.log(metrics.generateSummary());
     } catch (error) {
       console.error("Error:", error);
       process.exit(1);
     }
   });
 
+async function processEntitiesSequentially(
+  mappingPaths: string[],
+  mapper: DataMapper,
+  config: ReturnType<typeof loadConfig>
+): Promise<void> {
+  for (const configPath of mappingPaths) {
+    try {
+      const entityName = basename(configPath, ".json");
+      const entityConfig = getEntityConfig(entityName, config);
+      await mapper.processEntity(configPath, entityConfig);
+    } catch (error) {
+      console.warn(`Warning: Could not process ${configPath}:`, error);
+    }
+  }
+}
+
+async function processEntitiesInWaves(
+  mappingPaths: string[],
+  resolver: DependencyResolver,
+  mapper: DataMapper,
+  config: ReturnType<typeof loadConfig>
+): Promise<void> {
+  const waves = resolver.resolveExecutionOrder();
+  const pathMap = new Map(
+    mappingPaths.map((path) => [basename(path, ".json"), path])
+  );
+
+  console.log(`Processing ${waves.length} dependency waves...`);
+
+  for (const wave of waves) {
+    console.log(
+      `Wave ${wave.wave + 1}: Processing entities [${wave.entities.join(", ")}]`
+    );
+
+    // Process entities in controlled batches based on entityConcurrency
+    const entityConcurrency = config.parallelProcessing.entityConcurrency;
+    const chunks = chunkArray(wave.entities, entityConcurrency);
+    
+    for (const chunk of chunks) {
+      const entityPromises = chunk.map(async (entityName) => {
+        const configPath = pathMap.get(entityName);
+        if (configPath) {
+          try {
+            const entityConfig = getEntityConfig(entityName, config);
+            await mapper.processEntity(configPath, entityConfig);
+          } catch (error) {
+            console.warn(`Warning: Could not process ${configPath}:`, error);
+          }
+        }
+      });
+
+      await Promise.allSettled(entityPromises);
+    }
+  }
+}
+
 program.parse();

package/src/config.test.ts

@@ -0,0 +1,204 @@
+import fs from "fs";
+import path from "path";
+import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
+
+jest.mock("fs");
+const mockFs = fs as jest.Mocked<typeof fs>;
+
+describe("Configuration", () => {
+  const testConfigDir = "/test/config";
+  const configPath = path.join(testConfigDir, "config.yaml");
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("loadConfig", () => {
+    it("should return default config when no config.yaml exists", () => {
+      mockFs.existsSync.mockReturnValue(false);
+      const consoleSpy = jest.spyOn(console, "log").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        "No config.yaml found, using default sequential processing"
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should load and merge YAML configuration", () => {
+      const yamlContent = `
+parallelProcessing:
+  concurrency: 5
+  entityConcurrency: 3
+
+entityConfig:
+  users:
+    concurrency: 2
+    preserveRowOrder: true
+
+entityDependencies:
+  products: ["users"]
+`;
+
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue(yamlContent);
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config.parallelProcessing.concurrency).toBe(5);
+      expect(config.parallelProcessing.entityConcurrency).toBe(3);
+      expect(config.entityConfig.users.concurrency).toBe(2);
+      expect(config.entityConfig.users.preserveRowOrder).toBe(true);
+      expect(config.entityDependencies.products).toEqual(["users"]);
+    });
+
+    it("should merge partial configuration with defaults", () => {
+      const yamlContent = `
+parallelProcessing:
+  concurrency: 10
+entityConfig:
+  products:
+    concurrency: 20
+`;
+
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue(yamlContent);
+
+      const config = loadConfig(testConfigDir);
+
+      // Should merge with defaults
+      expect(config.parallelProcessing.concurrency).toBe(10);
+      expect(config.parallelProcessing.entityConcurrency).toBe(1); // default
+      expect(config.parallelProcessing.preserveRowOrder).toBe(true); // default
+    });
+
+    it("should handle invalid YAML gracefully", () => {
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue("invalid: yaml: content: [");
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("Warning: Failed to parse config.yaml")
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should handle file read errors gracefully", () => {
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockImplementation(() => {
+        throw new Error("File read error");
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("Warning: Failed to parse config.yaml")
+      );
+
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe("getEntityConfig", () => {
+    const globalConfig = {
+      parallelProcessing: {
+        concurrency: 10,
+        entityConcurrency: 3,
+        preserveRowOrder: false,
+      },
+      entityConfig: {
+        users: {
+          concurrency: 2,
+          preserveRowOrder: true,
+        },
+        products: {
+          concurrency: 20,
+        },
+      },
+      entityDependencies: {},
+    };
+
+    it("should return global config for entity without overrides", () => {
+      const entityConfig = getEntityConfig("orders", globalConfig);
+
+      expect(entityConfig).toEqual(globalConfig.parallelProcessing);
+    });
+
+    it("should merge entity overrides with global config", () => {
+      const entityConfig = getEntityConfig("products", globalConfig);
+
+      expect(entityConfig.concurrency).toBe(20); // overridden
+      expect(entityConfig.entityConcurrency).toBe(3); // from global
+      expect(entityConfig.preserveRowOrder).toBe(false); // from global
+    });
+
+    it("should apply preserveRowOrder constraint", () => {
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("users", globalConfig);
+
+      expect(entityConfig.concurrency).toBe(1); // forced to 1
+      expect(entityConfig.preserveRowOrder).toBe(true);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        "Entity 'users': preserveRowOrder=true forces concurrency=1 (was 2)"
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should not apply constraint when concurrency is already 1", () => {
+      const config = {
+        ...globalConfig,
+        entityConfig: {
+          ...globalConfig.entityConfig,
+          sequential: {
+            concurrency: 1,
+            preserveRowOrder: true,
+          },
+        },
+      };
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("sequential", config);
+
+      expect(entityConfig.concurrency).toBe(1);
+      expect(consoleSpy).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should not apply constraint when preserveRowOrder is false", () => {
+      const config = {
+        ...globalConfig,
+        entityConfig: {
+          ...globalConfig.entityConfig,
+          bulk: {
+            concurrency: 50,
+            preserveRowOrder: false,
+          },
+        },
+      };
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("bulk", config);
+
+      expect(entityConfig.concurrency).toBe(50);
+      expect(consoleSpy).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+  });
+});

package/src/config.ts

@@ -0,0 +1,90 @@
+import fs from "fs";
+import path from "path";
+import * as yaml from "js-yaml";
+
+export interface ParallelProcessingConfig {
+  concurrency: number;
+  entityConcurrency: number;
+  preserveRowOrder: boolean;
+}
+
+export interface EntityConfig {
+  concurrency?: number;
+  preserveRowOrder?: boolean;
+}
+
+export interface ProcessingConfig {
+  parallelProcessing: ParallelProcessingConfig;
+  entityConfig: Record<string, EntityConfig>;
+  entityDependencies: Record<string, string[]>;
+}
+
+export interface FullConfig extends ProcessingConfig {
+  // Future: additional config sections can be added here
+}
+
+export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
+  concurrency: 1,
+  entityConcurrency: 1,
+  preserveRowOrder: true,
+};
+
+export const DEFAULT_CONFIG: ProcessingConfig = {
+  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
+  entityConfig: {},
+  entityDependencies: {},
+};
+
+export function loadConfig(configDir: string): ProcessingConfig {
+  const configPath = path.join(configDir, "config.yaml");
+
+  try {
+    if (!fs.existsSync(configPath)) {
+      console.log("No config.yaml found, using default sequential processing");
+      return DEFAULT_CONFIG;
+    }
+
+    const fileContents = fs.readFileSync(configPath, "utf8");
+    const yamlConfig = yaml.load(fileContents) as Partial<FullConfig>;
+
+    return mergeWithDefaults(yamlConfig);
+  } catch (error) {
+    console.warn(
+      `Warning: Failed to parse config.yaml: ${error}. Using defaults.`
+    );
+    return DEFAULT_CONFIG;
+  }
+}
+
+function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
+  return {
+    parallelProcessing: {
+      ...DEFAULT_PARALLEL_CONFIG,
+      ...(yamlConfig.parallelProcessing || {}),
+    },
+    entityConfig: yamlConfig.entityConfig || {},
+    entityDependencies: yamlConfig.entityDependencies || {},
+  };
+}
+
+export function getEntityConfig(
+  entityName: string,
+  globalConfig: ProcessingConfig
+): ParallelProcessingConfig {
+  const entityOverrides = globalConfig.entityConfig[entityName] || {};
+
+  let finalConfig = {
+    ...globalConfig.parallelProcessing,
+    ...entityOverrides,
+  };
+
+  // Apply constraint: preserveRowOrder forces concurrency = 1
+  if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {
+    console.warn(
+      `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`
+    );
+    finalConfig.concurrency = 1;
+  }
+
+  return finalConfig;
+}

package/src/dependency-resolver.test.ts

@@ -0,0 +1,197 @@
+import { DependencyResolver } from "./dependency-resolver";
+
+describe("DependencyResolver", () => {
+  describe("resolveExecutionOrder", () => {
+    it("should handle entities with no dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {};
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(1);
+      expect(waves[0].wave).toBe(0);
+      expect(waves[0].entities).toEqual(["users", "products", "orders"]);
+    });
+
+    it("should resolve simple linear dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(3);
+      expect(waves[0].entities).toEqual(["users"]);
+      expect(waves[1].entities).toEqual(["products"]);
+      expect(waves[2].entities).toEqual(["orders"]);
+    });
+
+    it("should resolve complex dependency graph", () => {
+      const entities = ["users", "categories", "products", "orders", "reviews"];
+      const dependencies = {
+        products: ["users", "categories"],
+        orders: ["users", "products"],
+        reviews: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(3);
+
+      // Wave 0: users and categories (no dependencies)
+      expect(waves[0].entities.sort()).toEqual(["categories", "users"]);
+
+      // Wave 1: products (depends on users and categories)
+      expect(waves[1].entities).toEqual(["products"]);
+
+      // Wave 2: orders and reviews (both depend on users and products)
+      expect(waves[2].entities.sort()).toEqual(["orders", "reviews"]);
+    });
+
+    it("should handle entities with multiple dependencies", () => {
+      const entities = ["a", "b", "c", "d"];
+      const dependencies = {
+        c: ["a", "b"],
+        d: ["a", "b"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(2);
+      expect(waves[0].entities.sort()).toEqual(["a", "b"]);
+      expect(waves[1].entities.sort()).toEqual(["c", "d"]);
+    });
+
+    it("should detect circular dependencies", () => {
+      const entities = ["a", "b", "c"];
+      const dependencies = {
+        a: ["b"],
+        b: ["c"],
+        c: ["a"], // circular
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(() => resolver.resolveExecutionOrder()).toThrow(
+        "Circular dependency detected or missing dependencies for entities: a, b, c"
+      );
+    });
+
+    it("should detect missing dependencies", () => {
+      const entities = ["a", "b"];
+      const dependencies = {
+        a: ["missing"],
+        b: ["a"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(() => resolver.resolveExecutionOrder()).toThrow(
+        "Circular dependency detected or missing dependencies for entities: a, b"
+      );
+    });
+  });
+
+  describe("validateDependencies", () => {
+    it("should return no errors for valid dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toHaveLength(0);
+    });
+
+    it("should detect entity with dependencies not in entity list", () => {
+      const entities = ["users", "products"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["products"], // orders not in entities list
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toContain(
+        "Entity 'orders' has dependencies but is not in the entity list"
+      );
+    });
+
+    it("should detect dependencies on non-existent entities", () => {
+      const entities = ["users", "products"];
+      const dependencies = {
+        products: ["users", "categories"], // categories doesn't exist
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toContain(
+        "Entity 'products' depends on 'categories' which does not exist"
+      );
+    });
+
+    it("should detect multiple validation errors", () => {
+      const entities = ["users"];
+      const dependencies = {
+        products: ["categories"], // products not in list, categories doesn't exist
+        orders: ["missing"], // orders not in list, missing doesn't exist
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toHaveLength(2);
+      expect(errors).toContain(
+        "Entity 'products' has dependencies but is not in the entity list"
+      );
+      expect(errors).toContain(
+        "Entity 'orders' has dependencies but is not in the entity list"
+      );
+    });
+  });
+
+  describe("getDependents", () => {
+    it("should return entities that depend on the given entity", () => {
+      const entities = ["users", "products", "orders", "reviews"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+        reviews: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const usersDependents = resolver.getDependents("users");
+      const productsDependents = resolver.getDependents("products");
+      const ordersDependents = resolver.getDependents("orders");
+
+      expect(usersDependents.sort()).toEqual(["orders", "products", "reviews"]);
+      expect(productsDependents.sort()).toEqual(["orders", "reviews"]);
+      expect(ordersDependents).toEqual([]);
+    });
+  });
+
+  describe("getDependencies", () => {
+    it("should return dependencies for the given entity", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(resolver.getDependencies("users")).toEqual([]);
+      expect(resolver.getDependencies("products")).toEqual(["users"]);
+      expect(resolver.getDependencies("orders")).toEqual(["users", "products"]);
+      expect(resolver.getDependencies("nonexistent")).toEqual([]);
+    });
+  });
+});