@jackchuka/gql-ingest 2.1.0 → 2.2.1

package/src/cli.ts

@@ -1,187 +0,0 @@
-import { Command } from "commander";
-import { GraphQLClientWrapper } from "./graphql-client";
-import { DataMapper } from "./mapper";
-import { MetricsCollector } from "./metrics";
-import { loadConfig, getEntityConfig, getRetryConfig } from "./config";
-import { DependencyResolver } from "./dependency-resolver";
-import { basename } from "path";
-
-// Utility function to chunk array into smaller arrays
-function chunkArray<T>(array: T[], chunkSize: number): T[][] {
-  if (chunkSize <= 0) return [array];
-  const chunks: T[][] = [];
-  for (let i = 0; i < array.length; i += chunkSize) {
-    chunks.push(array.slice(i, i + chunkSize));
-  }
-  return chunks;
-}
-
-const program = new Command();
-
-program
-  .name("gql-ingest")
-  .description(
-    "A CLI tool for ingesting data from CSV files into a GraphQL API"
-  )
-  .version(require("../package.json").version);
-
-program
-  .requiredOption("-e, --endpoint <url>", "GraphQL endpoint URL")
-  .requiredOption(
-    "-c, --config <path>",
-    "Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)"
-  )
-  .option(
-    "-n, --entities <entities>",
-    "Comma-separated list of specific entities to process (e.g., users,products)"
-  )
-  .option(
-    "-h, --headers <headers>",
-    "JSON string of headers to include in requests"
-  )
-  .option("-v, --verbose", "Show detailed request results and responses")
-  .option(
-    "-f, --format <format>",
-    "Override data format detection (csv, json, yaml, jsonl)"
-  )
-  .action(async (options) => {
-    try {
-      console.log("Starting seed data generation...");
-
-      // Parse headers if provided
-      const headers = options.headers ? JSON.parse(options.headers) : {};
-
-      // Initialize metrics collector
-      const metrics = new MetricsCollector();
-
-      // Initialize GraphQL client
-      const client = new GraphQLClientWrapper(
-        options.endpoint,
-        headers,
-        metrics,
-        options.verbose
-      );
-
-      // Load configuration
-      const config = loadConfig(options.config);
-
-      // Initialize data mapper
-      const mapper = new DataMapper(
-        client,
-        process.cwd(),
-        metrics,
-        options.verbose,
-        options.format
-      );
-
-      // Parse entities filter if provided
-      const entityFilter = options.entities
-        ? options.entities.split(",").map((e: string) => e.trim())
-        : undefined;
-
-      // Discover all mapping files dynamically
-      const mappingPaths = mapper.discoverMappings(
-        options.config,
-        entityFilter
-      );
-
-      if (mappingPaths.length === 0) {
-        const filterMsg = entityFilter
-          ? ` matching entities: ${entityFilter.join(", ")}`
-          : "";
-        console.warn(
-          `No mapping files found in ${options.config}/mappings${filterMsg}`
-        );
-        return;
-      }
-
-      // Extract entity names from mapping paths
-      const entityNames = mappingPaths.map((path) => basename(path, ".json"));
-
-      // Filter dependencies to only include those relevant to selected entities
-      const relevantDependencies: Record<string, string[]> = {};
-      if (config.entityDependencies) {
-        for (const entity of entityNames) {
-          if (config.entityDependencies[entity]) {
-            relevantDependencies[entity] = config.entityDependencies[entity];
-          }
-        }
-      }
-
-      // Setup dependency resolver with filtered dependencies
-      const resolver = new DependencyResolver(
-        entityNames,
-        relevantDependencies,
-        !!entityFilter // Allow partial resolution when using --entities
-      );
-
-      // Validate dependencies
-      const validationErrors = resolver.validateDependencies();
-      if (validationErrors.length > 0) {
-        if (entityFilter) {
-          // When using --entities flag, show warnings instead of errors
-          console.warn("\n⚠️  Warning: Dependency validation issues:");
-          validationErrors.forEach((error) => console.warn(`  - ${error}`));
-          console.warn(
-            "This may cause errors if the dependent data doesn't already exist.\n"
-          );
-        } else {
-          // Strict validation when processing all entities
-          console.error("Dependency validation errors:");
-          validationErrors.forEach((error) => console.error(`  - ${error}`));
-          process.exit(1);
-        }
-      }
-
-      await processEntitiesInWaves(mappingPaths, resolver, mapper, config);
-
-      metrics.finishProcessing();
-      console.log(metrics.generateSummary());
-    } catch (error) {
-      console.error("Error:", error);
-      process.exit(1);
-    }
-  });
-
-async function processEntitiesInWaves(
-  mappingPaths: string[],
-  resolver: DependencyResolver,
-  mapper: DataMapper,
-  config: ReturnType<typeof loadConfig>
-): Promise<void> {
-  const waves = resolver.resolveExecutionOrder();
-  const pathMap = new Map(
-    mappingPaths.map((path) => [basename(path, ".json"), path])
-  );
-
-  console.log(`Processing ${waves.length} dependency waves...`);
-
-  for (const wave of waves) {
-    console.log(
-      `Wave ${wave.wave + 1}: Processing entities [${wave.entities.join(", ")}]`
-    );
-
-    // Process entities in controlled batches based on entityConcurrency
-    const entityConcurrency = config.parallelProcessing.entityConcurrency;
-    const chunks = chunkArray(wave.entities, entityConcurrency);
-
-    for (const chunk of chunks) {
-      const entityPromises = chunk.map(async (entityName) => {
-        const configPath = pathMap.get(entityName);
-        if (configPath) {
-          try {
-            const entityConfig = getEntityConfig(entityName, config);
-            const retryConfig = getRetryConfig(entityName, config);
-            await mapper.processEntity(configPath, entityConfig, retryConfig);
-          } catch (error) {
-            console.warn(`Warning: Could not process ${configPath}:`, error);
-          }
-        }
-      });
-
-      await Promise.allSettled(entityPromises);
-    }
-  }
-}
-
-program.parse();

package/src/config.test.ts

@@ -1,272 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { loadConfig, getEntityConfig, getRetryConfig, DEFAULT_CONFIG } from "./config";
-
-jest.mock("fs");
-const mockFs = fs as jest.Mocked<typeof fs>;
-
-describe("Configuration", () => {
-  const testConfigDir = "/test/config";
-  const configPath = path.join(testConfigDir, "config.yaml");
-
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  describe("loadConfig", () => {
-    it("should return default config when no config.yaml exists", () => {
-      mockFs.existsSync.mockReturnValue(false);
-      const consoleSpy = jest.spyOn(console, "log").mockImplementation();
-
-      const config = loadConfig(testConfigDir);
-
-      expect(config).toEqual(DEFAULT_CONFIG);
-      expect(consoleSpy).toHaveBeenCalledWith(
-        "No config.yaml found, using default sequential processing"
-      );
-
-      consoleSpy.mockRestore();
-    });
-
-    it("should load and merge YAML configuration", () => {
-      const yamlContent = `
-parallelProcessing:
-  concurrency: 5
-  entityConcurrency: 3
-
-entityConfig:
-  users:
-    concurrency: 2
-    preserveRowOrder: true
-
-entityDependencies:
-  products: ["users"]
-`;
-
-      mockFs.existsSync.mockReturnValue(true);
-      mockFs.readFileSync.mockReturnValue(yamlContent);
-
-      const config = loadConfig(testConfigDir);
-
-      expect(config.parallelProcessing.concurrency).toBe(5);
-      expect(config.parallelProcessing.entityConcurrency).toBe(3);
-      expect(config.entityConfig.users.concurrency).toBe(2);
-      expect(config.entityConfig.users.preserveRowOrder).toBe(true);
-      expect(config.entityDependencies.products).toEqual(["users"]);
-    });
-
-    it("should merge partial configuration with defaults", () => {
-      const yamlContent = `
-parallelProcessing:
-  concurrency: 10
-entityConfig:
-  products:
-    concurrency: 20
-`;
-
-      mockFs.existsSync.mockReturnValue(true);
-      mockFs.readFileSync.mockReturnValue(yamlContent);
-
-      const config = loadConfig(testConfigDir);
-
-      // Should merge with defaults
-      expect(config.parallelProcessing.concurrency).toBe(10);
-      expect(config.parallelProcessing.entityConcurrency).toBe(1); // default
-      expect(config.parallelProcessing.preserveRowOrder).toBe(true); // default
-    });
-
-    it("should handle invalid YAML gracefully", () => {
-      mockFs.existsSync.mockReturnValue(true);
-      mockFs.readFileSync.mockReturnValue("invalid: yaml: content: [");
-
-      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
-
-      const config = loadConfig(testConfigDir);
-
-      expect(config).toEqual(DEFAULT_CONFIG);
-      expect(consoleSpy).toHaveBeenCalledWith(
-        expect.stringContaining("Warning: Failed to parse config.yaml")
-      );
-
-      consoleSpy.mockRestore();
-    });
-
-    it("should handle file read errors gracefully", () => {
-      mockFs.existsSync.mockReturnValue(true);
-      mockFs.readFileSync.mockImplementation(() => {
-        throw new Error("File read error");
-      });
-
-      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
-
-      const config = loadConfig(testConfigDir);
-
-      expect(config).toEqual(DEFAULT_CONFIG);
-      expect(consoleSpy).toHaveBeenCalledWith(
-        expect.stringContaining("Warning: Failed to parse config.yaml")
-      );
-
-      consoleSpy.mockRestore();
-    });
-  });
-
-  describe("getEntityConfig", () => {
-    const globalConfig = {
-      retry: {
-        maxAttempts: 3,
-        baseDelay: 1000,
-        maxDelay: 30000,
-        exponentialBackoff: true,
-        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
-      },
-      parallelProcessing: {
-        concurrency: 10,
-        entityConcurrency: 3,
-        preserveRowOrder: false,
-      },
-      entityConfig: {
-        users: {
-          concurrency: 2,
-          preserveRowOrder: true,
-        },
-        products: {
-          concurrency: 20,
-        },
-      },
-      entityDependencies: {},
-    };
-
-    it("should return global config for entity without overrides", () => {
-      const entityConfig = getEntityConfig("orders", globalConfig);
-
-      expect(entityConfig).toEqual(globalConfig.parallelProcessing);
-    });
-
-    it("should merge entity overrides with global config", () => {
-      const entityConfig = getEntityConfig("products", globalConfig);
-
-      expect(entityConfig.concurrency).toBe(20); // overridden
-      expect(entityConfig.entityConcurrency).toBe(3); // from global
-      expect(entityConfig.preserveRowOrder).toBe(false); // from global
-    });
-
-    it("should apply preserveRowOrder constraint", () => {
-      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
-
-      const entityConfig = getEntityConfig("users", globalConfig);
-
-      expect(entityConfig.concurrency).toBe(1); // forced to 1
-      expect(entityConfig.preserveRowOrder).toBe(true);
-      expect(consoleSpy).toHaveBeenCalledWith(
-        "Entity 'users': preserveRowOrder=true forces concurrency=1 (was 2)"
-      );
-
-      consoleSpy.mockRestore();
-    });
-
-    it("should not apply constraint when concurrency is already 1", () => {
-      const config = {
-        ...globalConfig,
-        entityConfig: {
-          ...globalConfig.entityConfig,
-          sequential: {
-            concurrency: 1,
-            preserveRowOrder: true,
-          },
-        },
-      };
-
-      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
-
-      const entityConfig = getEntityConfig("sequential", config);
-
-      expect(entityConfig.concurrency).toBe(1);
-      expect(consoleSpy).not.toHaveBeenCalled();
-
-      consoleSpy.mockRestore();
-    });
-
-    it("should not apply constraint when preserveRowOrder is false", () => {
-      const config = {
-        ...globalConfig,
-        entityConfig: {
-          ...globalConfig.entityConfig,
-          bulk: {
-            concurrency: 50,
-            preserveRowOrder: false,
-          },
-        },
-      };
-
-      const consoleSpy = jest.spyOn(console, "warn").mockImplementation();
-
-      const entityConfig = getEntityConfig("bulk", config);
-
-      expect(entityConfig.concurrency).toBe(50);
-      expect(consoleSpy).not.toHaveBeenCalled();
-
-      consoleSpy.mockRestore();
-    });
-  });
-
-  describe("getRetryConfig", () => {
-    const globalConfig = {
-      retry: {
-        maxAttempts: 3,
-        baseDelay: 1000,
-        maxDelay: 30000,
-        exponentialBackoff: true,
-        retryableStatusCodes: [408, 429, 500, 502, 503, 504],
-      },
-      parallelProcessing: {
-        concurrency: 10,
-        entityConcurrency: 3,
-        preserveRowOrder: false,
-      },
-      entityConfig: {
-        important: {
-          retry: {
-            maxAttempts: 5,
-            baseDelay: 500,
-          },
-        },
-        fast: {
-          retry: {
-            maxAttempts: 1,
-          },
-        },
-      },
-      entityDependencies: {},
-    };
-
-    it("should return global retry config for entity without overrides", () => {
-      const retryConfig = getRetryConfig("regular", globalConfig);
-
-      expect(retryConfig).toEqual(globalConfig.retry);
-    });
-
-    it("should merge entity retry overrides with global config", () => {
-      const retryConfig = getRetryConfig("important", globalConfig);
-
-      expect(retryConfig.maxAttempts).toBe(5); // overridden
-      expect(retryConfig.baseDelay).toBe(500); // overridden
-      expect(retryConfig.maxDelay).toBe(30000); // from global
-      expect(retryConfig.exponentialBackoff).toBe(true); // from global
-      expect(retryConfig.retryableStatusCodes).toEqual([408, 429, 500, 502, 503, 504]); // from global
-    });
-
-    it("should handle partial retry overrides", () => {
-      const retryConfig = getRetryConfig("fast", globalConfig);
-
-      expect(retryConfig.maxAttempts).toBe(1); // overridden
-      expect(retryConfig.baseDelay).toBe(1000); // from global
-      expect(retryConfig.maxDelay).toBe(30000); // from global
-    });
-
-    it("should handle entity with no retry config", () => {
-      const retryConfig = getRetryConfig("undefined-entity", globalConfig);
-
-      expect(retryConfig).toEqual(globalConfig.retry);
-    });
-  });
-});

package/src/config.ts

@@ -1,125 +0,0 @@
-import fs from "fs";
-import path from "path";
-import * as yaml from "js-yaml";
-
-export interface ParallelProcessingConfig {
-  concurrency: number;
-  entityConcurrency: number;
-  preserveRowOrder: boolean;
-}
-
-export interface RetryConfig {
-  maxAttempts: number;
-  baseDelay: number;
-  maxDelay: number;
-  exponentialBackoff: boolean;
-  retryableStatusCodes: number[];
-}
-
-export interface EntityConfig {
-  concurrency?: number;
-  preserveRowOrder?: boolean;
-  retry?: Partial<RetryConfig>;
-}
-
-export interface ProcessingConfig {
-  retry: RetryConfig;
-  parallelProcessing: ParallelProcessingConfig;
-  entityConfig: Record<string, EntityConfig>;
-  entityDependencies: Record<string, string[]>;
-}
-
-export interface FullConfig extends ProcessingConfig {
-  // Future: additional config sections can be added here
-}
-
-export const DEFAULT_RETRY_CONFIG: RetryConfig = {
-  maxAttempts: 3,
-  baseDelay: 1000,
-  maxDelay: 30000,
-  exponentialBackoff: true,
-  retryableStatusCodes: [408, 429, 500, 502, 503, 504],
-};
-
-export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
-  concurrency: 1,
-  entityConcurrency: 1,
-  preserveRowOrder: true,
-};
-
-export const DEFAULT_CONFIG: ProcessingConfig = {
-  retry: DEFAULT_RETRY_CONFIG,
-  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
-  entityConfig: {},
-  entityDependencies: {},
-};
-
-export function loadConfig(configDir: string): ProcessingConfig {
-  const configPath = path.join(configDir, "config.yaml");
-
-  try {
-    if (!fs.existsSync(configPath)) {
-      console.log("No config.yaml found, using default sequential processing");
-      return DEFAULT_CONFIG;
-    }
-
-    const fileContents = fs.readFileSync(configPath, "utf8");
-    const yamlConfig = yaml.load(fileContents) as Partial<FullConfig>;
-
-    return mergeWithDefaults(yamlConfig);
-  } catch (error) {
-    console.warn(
-      `Warning: Failed to parse config.yaml: ${error}. Using defaults.`
-    );
-    return DEFAULT_CONFIG;
-  }
-}
-
-function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
-  return {
-    retry: {
-      ...DEFAULT_RETRY_CONFIG,
-      ...(yamlConfig.retry || {}),
-    },
-    parallelProcessing: {
-      ...DEFAULT_PARALLEL_CONFIG,
-      ...(yamlConfig.parallelProcessing || {}),
-    },
-    entityConfig: yamlConfig.entityConfig || {},
-    entityDependencies: yamlConfig.entityDependencies || {},
-  };
-}
-
-export function getEntityConfig(
-  entityName: string,
-  globalConfig: ProcessingConfig
-): ParallelProcessingConfig {
-  const entityOverrides = globalConfig.entityConfig[entityName] || {};
-
-  let finalConfig = {
-    ...globalConfig.parallelProcessing,
-    ...entityOverrides,
-  };
-
-  // Apply constraint: preserveRowOrder forces concurrency = 1
-  if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {
-    console.warn(
-      `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`
-    );
-    finalConfig.concurrency = 1;
-  }
-
-  return finalConfig;
-}
-
-export function getRetryConfig(
-  entityName: string,
-  globalConfig: ProcessingConfig
-): RetryConfig {
-  const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
-
-  return {
-    ...globalConfig.retry,
-    ...entityOverrides,
-  };
-}