@jackchuka/gql-ingest 1.0.2 → 1.1.0

package/src/config.test.ts

@@ -0,0 +1,205 @@
+import fs from "fs";
+import path from "path";
+import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
+
+jest.mock("fs");
+const mockFs = fs as jest.Mocked<typeof fs>;
+
+describe("Configuration", () => {
+  const testConfigDir = "/test/config";
+  const configPath = path.join(testConfigDir, "config.yaml");
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("loadConfig", () => {
+    it("should return default config when no config.yaml exists", () => {
+      mockFs.existsSync.mockReturnValue(false);
+      const consoleSpy = jest.spyOn(console, "log").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        "No config.yaml found, using default sequential processing"
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should load and merge YAML configuration", () => {
+      const yamlContent = `
+parallelProcessing:
+  concurrency: 5
+  enableEntityParallelization: true
+
+entityConfig:
+  users:
+    concurrency: 2
+    preserveRowOrder: true
+
+entityDependencies:
+  products: ["users"]
+`;
+
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue(yamlContent);
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config.parallelProcessing.concurrency).toBe(5);
+      expect(config.parallelProcessing.enableEntityParallelization).toBe(true);
+      expect(config.entityConfig.users.concurrency).toBe(2);
+      expect(config.entityConfig.users.preserveRowOrder).toBe(true);
+      expect(config.entityDependencies.products).toEqual(["users"]);
+    });
+
+    it("should merge partial configuration with defaults", () => {
+      const yamlContent = `
+parallelProcessing:
+  concurrency: 10
+entityConfig:
+  products:
+    concurrency: 20
+`;
+
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue(yamlContent);
+
+      const config = loadConfig(testConfigDir);
+
+      // Should merge with defaults
+      expect(config.parallelProcessing.concurrency).toBe(10);
+      expect(config.parallelProcessing.enableEntityParallelization).toBe(false); // default
+      expect(config.parallelProcessing.preserveRowOrder).toBe(true); // default
+    });
+
+    it("should handle invalid YAML gracefully", () => {
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockReturnValue("invalid: yaml: content: [");
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("Warning: Failed to parse config.yaml")
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should handle file read errors gracefully", () => {
+      mockFs.existsSync.mockReturnValue(true);
+      mockFs.readFileSync.mockImplementation(() => {
+        throw new Error("File read error");
+      });
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const config = loadConfig(testConfigDir);
+
+      expect(config).toEqual(DEFAULT_CONFIG);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining("Warning: Failed to parse config.yaml")
+      );
+
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe("getEntityConfig", () => {
+    const globalConfig = {
+      parallelProcessing: {
+        concurrency: 10,
+        enableEntityParallelization: true,
+        preserveRowOrder: false,
+        preserveEntityOrder: false,
+      },
+      entityConfig: {
+        users: {
+          concurrency: 2,
+          preserveRowOrder: true,
+        },
+        products: {
+          concurrency: 20,
+        },
+      },
+      entityDependencies: {},
+    };
+
+    it("should return global config for entity without overrides", () => {
+      const entityConfig = getEntityConfig("orders", globalConfig);
+
+      expect(entityConfig).toEqual(globalConfig.parallelProcessing);
+    });
+
+    it("should merge entity overrides with global config", () => {
+      const entityConfig = getEntityConfig("products", globalConfig);
+
+      expect(entityConfig.concurrency).toBe(20); // overridden
+      expect(entityConfig.enableEntityParallelization).toBe(true); // from global
+      expect(entityConfig.preserveRowOrder).toBe(false); // from global
+    });
+
+    it("should apply preserveRowOrder constraint", () => {
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("users", globalConfig);
+
+      expect(entityConfig.concurrency).toBe(1); // forced to 1
+      expect(entityConfig.preserveRowOrder).toBe(true);
+      expect(consoleSpy).toHaveBeenCalledWith(
+        "Entity 'users': preserveRowOrder=true forces concurrency=1 (was 2)"
+      );
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should not apply constraint when concurrency is already 1", () => {
+      const config = {
+        ...globalConfig,
+        entityConfig: {
+          ...globalConfig.entityConfig,
+          sequential: {
+            concurrency: 1,
+            preserveRowOrder: true,
+          },
+        },
+      };
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("sequential", config);
+
+      expect(entityConfig.concurrency).toBe(1);
+      expect(consoleSpy).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+
+    it("should not apply constraint when preserveRowOrder is false", () => {
+      const config = {
+        ...globalConfig,
+        entityConfig: {
+          ...globalConfig.entityConfig,
+          bulk: {
+            concurrency: 50,
+            preserveRowOrder: false,
+          },
+        },
+      };
+
+      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
+
+      const entityConfig = getEntityConfig("bulk", config);
+
+      expect(entityConfig.concurrency).toBe(50);
+      expect(consoleSpy).not.toHaveBeenCalled();
+
+      consoleSpy.mockRestore();
+    });
+  });
+});

package/src/config.ts

@@ -0,0 +1,92 @@
+import fs from "fs";
+import path from "path";
+import * as yaml from "js-yaml";
+
+export interface ParallelProcessingConfig {
+  concurrency: number;
+  enableEntityParallelization: boolean;
+  preserveRowOrder: boolean;
+  preserveEntityOrder: boolean;
+}
+
+export interface EntityConfig {
+  concurrency?: number;
+  preserveRowOrder?: boolean;
+}
+
+export interface ProcessingConfig {
+  parallelProcessing: ParallelProcessingConfig;
+  entityConfig: Record<string, EntityConfig>;
+  entityDependencies: Record<string, string[]>;
+}
+
+export interface FullConfig extends ProcessingConfig {
+  // Future: additional config sections can be added here
+}
+
+export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
+  concurrency: 1,
+  enableEntityParallelization: false,
+  preserveRowOrder: true,
+  preserveEntityOrder: true,
+};
+
+export const DEFAULT_CONFIG: ProcessingConfig = {
+  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
+  entityConfig: {},
+  entityDependencies: {},
+};
+
+export function loadConfig(configDir: string): ProcessingConfig {
+  const configPath = path.join(configDir, "config.yaml");
+
+  try {
+    if (!fs.existsSync(configPath)) {
+      console.log("No config.yaml found, using default sequential processing");
+      return DEFAULT_CONFIG;
+    }
+
+    const fileContents = fs.readFileSync(configPath, "utf8");
+    const yamlConfig = yaml.load(fileContents) as Partial<FullConfig>;
+
+    return mergeWithDefaults(yamlConfig);
+  } catch (error) {
+    console.warn(
+      `Warning: Failed to parse config.yaml: ${error}. Using defaults.`
+    );
+    return DEFAULT_CONFIG;
+  }
+}
+
+function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
+  return {
+    parallelProcessing: {
+      ...DEFAULT_PARALLEL_CONFIG,
+      ...(yamlConfig.parallelProcessing || {}),
+    },
+    entityConfig: yamlConfig.entityConfig || {},
+    entityDependencies: yamlConfig.entityDependencies || {},
+  };
+}
+
+export function getEntityConfig(
+  entityName: string,
+  globalConfig: ProcessingConfig
+): ParallelProcessingConfig {
+  const entityOverrides = globalConfig.entityConfig[entityName] || {};
+
+  let finalConfig = {
+    ...globalConfig.parallelProcessing,
+    ...entityOverrides,
+  };
+
+  // Apply constraint: preserveRowOrder forces concurrency = 1
+  if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {
+    console.warn(
+      `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`
+    );
+    finalConfig.concurrency = 1;
+  }
+
+  return finalConfig;
+}

package/src/dependency-resolver.test.ts

@@ -0,0 +1,197 @@
+import { DependencyResolver } from "./dependency-resolver";
+
+describe("DependencyResolver", () => {
+  describe("resolveExecutionOrder", () => {
+    it("should handle entities with no dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {};
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(1);
+      expect(waves[0].wave).toBe(0);
+      expect(waves[0].entities).toEqual(["users", "products", "orders"]);
+    });
+
+    it("should resolve simple linear dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(3);
+      expect(waves[0].entities).toEqual(["users"]);
+      expect(waves[1].entities).toEqual(["products"]);
+      expect(waves[2].entities).toEqual(["orders"]);
+    });
+
+    it("should resolve complex dependency graph", () => {
+      const entities = ["users", "categories", "products", "orders", "reviews"];
+      const dependencies = {
+        products: ["users", "categories"],
+        orders: ["users", "products"],
+        reviews: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(3);
+
+      // Wave 0: users and categories (no dependencies)
+      expect(waves[0].entities.sort()).toEqual(["categories", "users"]);
+
+      // Wave 1: products (depends on users and categories)
+      expect(waves[1].entities).toEqual(["products"]);
+
+      // Wave 2: orders and reviews (both depend on users and products)
+      expect(waves[2].entities.sort()).toEqual(["orders", "reviews"]);
+    });
+
+    it("should handle entities with multiple dependencies", () => {
+      const entities = ["a", "b", "c", "d"];
+      const dependencies = {
+        c: ["a", "b"],
+        d: ["a", "b"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const waves = resolver.resolveExecutionOrder();
+
+      expect(waves).toHaveLength(2);
+      expect(waves[0].entities.sort()).toEqual(["a", "b"]);
+      expect(waves[1].entities.sort()).toEqual(["c", "d"]);
+    });
+
+    it("should detect circular dependencies", () => {
+      const entities = ["a", "b", "c"];
+      const dependencies = {
+        a: ["b"],
+        b: ["c"],
+        c: ["a"], // circular
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(() => resolver.resolveExecutionOrder()).toThrow(
+        "Circular dependency detected or missing dependencies for entities: a, b, c"
+      );
+    });
+
+    it("should detect missing dependencies", () => {
+      const entities = ["a", "b"];
+      const dependencies = {
+        a: ["missing"],
+        b: ["a"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(() => resolver.resolveExecutionOrder()).toThrow(
+        "Circular dependency detected or missing dependencies for entities: a, b"
+      );
+    });
+  });
+
+  describe("validateDependencies", () => {
+    it("should return no errors for valid dependencies", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toHaveLength(0);
+    });
+
+    it("should detect entity with dependencies not in entity list", () => {
+      const entities = ["users", "products"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["products"], // orders not in entities list
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toContain(
+        "Entity 'orders' has dependencies but is not in the entity list"
+      );
+    });
+
+    it("should detect dependencies on non-existent entities", () => {
+      const entities = ["users", "products"];
+      const dependencies = {
+        products: ["users", "categories"], // categories doesn't exist
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toContain(
+        "Entity 'products' depends on 'categories' which does not exist"
+      );
+    });
+
+    it("should detect multiple validation errors", () => {
+      const entities = ["users"];
+      const dependencies = {
+        products: ["categories"], // products not in list, categories doesn't exist
+        orders: ["missing"], // orders not in list, missing doesn't exist
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const errors = resolver.validateDependencies();
+
+      expect(errors).toHaveLength(2);
+      expect(errors).toContain(
+        "Entity 'products' has dependencies but is not in the entity list"
+      );
+      expect(errors).toContain(
+        "Entity 'orders' has dependencies but is not in the entity list"
+      );
+    });
+  });
+
+  describe("getDependents", () => {
+    it("should return entities that depend on the given entity", () => {
+      const entities = ["users", "products", "orders", "reviews"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+        reviews: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      const usersDependents = resolver.getDependents("users");
+      const productsDependents = resolver.getDependents("products");
+      const ordersDependents = resolver.getDependents("orders");
+
+      expect(usersDependents.sort()).toEqual(["orders", "products", "reviews"]);
+      expect(productsDependents.sort()).toEqual(["orders", "reviews"]);
+      expect(ordersDependents).toEqual([]);
+    });
+  });
+
+  describe("getDependencies", () => {
+    it("should return dependencies for the given entity", () => {
+      const entities = ["users", "products", "orders"];
+      const dependencies = {
+        products: ["users"],
+        orders: ["users", "products"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies);
+
+      expect(resolver.getDependencies("users")).toEqual([]);
+      expect(resolver.getDependencies("products")).toEqual(["users"]);
+      expect(resolver.getDependencies("orders")).toEqual(["users", "products"]);
+      expect(resolver.getDependencies("nonexistent")).toEqual([]);
+    });
+  });
+});

package/src/dependency-resolver.ts

@@ -0,0 +1,98 @@
+export interface DependencyGraph {
+  [entityName: string]: string[];
+}
+
+export interface ExecutionWave {
+  entities: string[];
+  wave: number;
+}
+
+export class DependencyResolver {
+  private dependencies: DependencyGraph;
+  private entities: string[];
+
+  constructor(entities: string[], dependencies: DependencyGraph = {}) {
+    this.entities = entities;
+    this.dependencies = dependencies;
+  }
+
+  resolveExecutionOrder(): ExecutionWave[] {
+    const waves: ExecutionWave[] = [];
+    const processed = new Set<string>();
+    const inProgress = new Set<string>();
+    let waveNumber = 0;
+
+    while (processed.size < this.entities.length) {
+      const currentWave: string[] = [];
+
+      for (const entity of this.entities) {
+        if (processed.has(entity) || inProgress.has(entity)) {
+          continue;
+        }
+
+        const deps = this.dependencies[entity] || [];
+        const canProcess = deps.every((dep) => processed.has(dep));
+
+        if (canProcess) {
+          currentWave.push(entity);
+          inProgress.add(entity);
+        }
+      }
+
+      if (currentWave.length === 0) {
+        const remaining = this.entities.filter((e) => !processed.has(e));
+        throw new Error(
+          `Circular dependency detected or missing dependencies for entities: ${remaining.join(
+            ", "
+          )}`
+        );
+      }
+
+      waves.push({
+        entities: currentWave,
+        wave: waveNumber++,
+      });
+
+      currentWave.forEach((entity) => {
+        processed.add(entity);
+        inProgress.delete(entity);
+      });
+    }
+
+    return waves;
+  }
+
+  validateDependencies(): string[] {
+    const errors: string[] = [];
+    const entitySet = new Set(this.entities);
+
+    for (const [entity, deps] of Object.entries(this.dependencies)) {
+      if (!entitySet.has(entity)) {
+        errors.push(
+          `Entity '${entity}' has dependencies but is not in the entity list`
+        );
+        continue;
+      }
+
+      for (const dep of deps) {
+        if (!entitySet.has(dep)) {
+          errors.push(
+            `Entity '${entity}' depends on '${dep}' which does not exist`
+          );
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  getDependents(entityName: string): string[] {
+    return Object.entries(this.dependencies)
+      .filter(([_, deps]) => deps.includes(entityName))
+      .map(([entity, _]) => entity);
+  }
+
+  getDependencies(entityName: string): string[] {
+    return this.dependencies[entityName] || [];
+  }
+}

package/src/graphql-client.ts

@@ -1,20 +1,46 @@
 import { GraphQLClient } from 'graphql-request';
+import { MetricsCollector } from './metrics';
 
 export class GraphQLClientWrapper {
   private client: GraphQLClient;
+  private metrics?: MetricsCollector;
+  private verbose: boolean;
 
-  constructor(endpoint: string, headers?: Record<string, string>) {
+  constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose: boolean = false) {
     this.client = new GraphQLClient(endpoint, {
       headers: headers || {}
     });
+    this.metrics = metrics;
+    this.verbose = verbose;
   }
 
   async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
+    const startTime = Date.now();
+    
     try {
       const result = await this.client.request(mutation, variables);
+      
+      if (this.metrics) {
+        const duration = Date.now() - startTime;
+        this.metrics.recordRequestDuration(duration);
+      }
+      
+      if (this.verbose) {
+        console.log(`✓ GraphQL request completed in ${Date.now() - startTime}ms:`, result);
+      }
+      
       return result;
     } catch (error) {
-      console.error('GraphQL mutation failed:', error);
+      if (this.metrics) {
+        const duration = Date.now() - startTime;
+        this.metrics.recordRequestDuration(duration);
+      }
+      
+      if (this.verbose) {
+        console.error(`✗ GraphQL request failed in ${Date.now() - startTime}ms:`, error);
+      } else {
+        console.error('GraphQL mutation failed:', error);
+      }
       throw error;
     }
   }