@jackchuka/gql-ingest 2.2.1 → 3.0.0

package/dist/index.js

@@ -0,0 +1,1379 @@
+// src/lib/gql-ingest.ts
+import { EventEmitter } from "events";
+
+// src/lib/graphql-client.ts
+import { GraphQLClient } from "graphql-request";
+
+// src/lib/config.ts
+import fs from "fs";
+import path from "path";
+import * as yaml from "js-yaml";
+
+// src/lib/logger.ts
+var noopLogger = {
+  debug: () => {
+  },
+  info: () => {
+  },
+  warn: () => {
+  },
+  error: () => {
+  }
+};
+function createConsoleLogger() {
+  return {
+    debug: (msg, ...args) => console.debug(`[gql-ingest] ${msg}`, ...args),
+    info: (msg, ...args) => console.info(`[gql-ingest] ${msg}`, ...args),
+    warn: (msg, ...args) => console.warn(`[gql-ingest] ${msg}`, ...args),
+    error: (msg, ...args) => console.error(`[gql-ingest] ${msg}`, ...args)
+  };
+}
+function createDefaultLogger(verbose = false) {
+  return verbose ? createConsoleLogger() : noopLogger;
+}
+
+// src/lib/config.ts
+var DEFAULT_RETRY_CONFIG = {
+  maxAttempts: 3,
+  baseDelay: 1e3,
+  maxDelay: 3e4,
+  exponentialBackoff: true,
+  retryableStatusCodes: [408, 429, 500, 502, 503, 504]
+};
+var DEFAULT_PARALLEL_CONFIG = {
+  concurrency: 1,
+  entityConcurrency: 1,
+  preserveRowOrder: true
+};
+var DEFAULT_CONFIG = {
+  retry: DEFAULT_RETRY_CONFIG,
+  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
+  entityConfig: {},
+  entityDependencies: {}
+};
+function loadConfig(configDir, logger = noopLogger) {
+  const configPath = path.join(configDir, "config.yaml");
+  try {
+    if (!fs.existsSync(configPath)) {
+      logger.info("No config.yaml found, using default sequential processing");
+      return DEFAULT_CONFIG;
+    }
+    const fileContents = fs.readFileSync(configPath, "utf8");
+    const yamlConfig = yaml.load(fileContents);
+    return mergeWithDefaults(yamlConfig);
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    logger.warn(`Warning: Failed to parse config.yaml: ${errorMessage}. Using defaults.`);
+    return DEFAULT_CONFIG;
+  }
+}
+function mergeWithDefaults(yamlConfig) {
+  return {
+    retry: {
+      ...DEFAULT_RETRY_CONFIG,
+      ...yamlConfig.retry
+    },
+    parallelProcessing: {
+      ...DEFAULT_PARALLEL_CONFIG,
+      ...yamlConfig.parallelProcessing
+    },
+    entityConfig: yamlConfig.entityConfig || {},
+    entityDependencies: yamlConfig.entityDependencies || {}
+  };
+}
+function getEntityConfig(entityName, globalConfig, logger = noopLogger) {
+  const entityOverrides = globalConfig.entityConfig[entityName] || {};
+  let finalConfig = {
+    ...globalConfig.parallelProcessing,
+    ...entityOverrides
+  };
+  if (finalConfig.preserveRowOrder && finalConfig.concurrency > 1) {
+    logger.warn(
+      `Entity '${entityName}': preserveRowOrder=true forces concurrency=1 (was ${finalConfig.concurrency})`
+    );
+    finalConfig.concurrency = 1;
+  }
+  return finalConfig;
+}
+function getRetryConfig(entityName, globalConfig) {
+  const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
+  return {
+    ...globalConfig.retry,
+    ...entityOverrides
+  };
+}
+
+// src/lib/graphql-client.ts
+var GraphQLClientWrapper = class {
+  constructor(endpoint, headers, metrics, logger = noopLogger) {
+    this.client = new GraphQLClient(endpoint, {
+      headers: headers || {}
+    });
+    this.metrics = metrics;
+    this.logger = logger;
+  }
+  async executeMutation(mutation, variables, retryConfig, signal) {
+    const config = retryConfig || DEFAULT_RETRY_CONFIG;
+    let lastError;
+    const totalStartTime = Date.now();
+    for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
+      if (signal?.aborted) {
+        throw new Error(`Request aborted: ${signal.reason || "cancelled"}`);
+      }
+      const attemptStartTime = Date.now();
+      try {
+        const result = await this.client.request(mutation, variables);
+        if (this.metrics) {
+          const duration = Date.now() - attemptStartTime;
+          this.metrics.recordRequestDuration(duration);
+          if (attempt > 0) {
+            this.metrics.recordRetrySuccess(attempt);
+          }
+        }
+        const totalDuration2 = Date.now() - totalStartTime;
+        const retryInfo = attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : "";
+        this.logger.debug(`\u2713 GraphQL request completed in ${totalDuration2}ms${retryInfo}`, result);
+        return result;
+      } catch (error) {
+        lastError = error;
+        const duration = Date.now() - attemptStartTime;
+        if (this.metrics) {
+          this.metrics.recordRequestDuration(duration);
+        }
+        if (attempt === config.maxAttempts - 1) {
+          if (this.metrics && attempt > 0) {
+            this.metrics.recordRetryFailure(attempt);
+          }
+          break;
+        }
+        if (!this.isRetryableError(error, config)) {
+          this.logger.error(
+            `\u2717 GraphQL request failed with non-retryable error in ${duration}ms`,
+            error
+          );
+          throw error;
+        }
+        const delay = this.calculateDelay(attempt, config);
+        this.logger.debug(
+          `\u23F3 GraphQL request failed (attempt ${attempt + 1}/${config.maxAttempts}), retrying in ${delay}ms...`
+        );
+        await this.sleepWithSignal(delay, signal);
+      }
+    }
+    const totalDuration = Date.now() - totalStartTime;
+    this.logger.error(
+      `\u2717 GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms`,
+      lastError
+    );
+    throw lastError;
+  }
+  isRetryableError(error, config) {
+    if (!error.response) {
+      return true;
+    }
+    const status = error.response.status;
+    return config.retryableStatusCodes.includes(status);
+  }
+  calculateDelay(attempt, config) {
+    if (!config.exponentialBackoff) {
+      return config.baseDelay;
+    }
+    const exponentialDelay = config.baseDelay * Math.pow(2, attempt);
+    const cappedDelay = Math.min(exponentialDelay, config.maxDelay);
+    const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);
+    return Math.max(0, cappedDelay + jitter);
+  }
+  /**
+   * Sleep that can be interrupted by abort signal
+   */
+  sleepWithSignal(ms, signal) {
+    return new Promise((resolve, reject) => {
+      if (signal?.aborted) {
+        reject(new Error(`Request aborted: ${signal.reason || "cancelled"}`));
+        return;
+      }
+      const timeoutId = setTimeout(resolve, ms);
+      if (signal) {
+        const abortHandler = () => {
+          clearTimeout(timeoutId);
+          reject(new Error(`Request aborted: ${signal.reason || "cancelled"}`));
+        };
+        signal.addEventListener("abort", abortHandler, { once: true });
+      }
+    });
+  }
+  setHeaders(headers) {
+    this.client.setHeaders(headers);
+  }
+};
+
+// src/lib/mapper.ts
+import fs6 from "fs";
+import path2 from "path";
+import { parse } from "graphql";
+
+// src/readers/data-reader.ts
+var DataReader = class {
+  /**
+   * Check if this reader can handle the given file
+   */
+  canHandle(filePath) {
+    const extension = filePath.split(".").pop()?.toLowerCase();
+    return extension ? this.getSupportedExtensions().includes(extension) : false;
+  }
+};
+var DataReaderFactory = class {
+  static {
+    this.readers = [];
+  }
+  static registerReader(reader) {
+    this.readers.push(reader);
+  }
+  static getReader(filePath, format) {
+    if (format) {
+      const reader2 = this.readers.find(
+        (r) => r.getSupportedExtensions().includes(format.toLowerCase())
+      );
+      if (reader2) return reader2;
+    }
+    const reader = this.readers.find((r) => r.canHandle(filePath));
+    if (!reader) {
+      throw new Error(
+        `No reader found for file: ${filePath}${format ? ` with format: ${format}` : ""}`
+      );
+    }
+    return reader;
+  }
+  static getSupportedFormats() {
+    const formats = /* @__PURE__ */ new Set();
+    this.readers.forEach((reader) => {
+      reader.getSupportedExtensions().forEach((ext) => formats.add(ext));
+    });
+    return Array.from(formats);
+  }
+};
+
+// src/readers/csv.ts
+import fs2 from "fs";
+import csv from "csv-parser";
+var CsvReader = class extends DataReader {
+  getSupportedExtensions() {
+    return ["csv"];
+  }
+  async readFile(filePath) {
+    return new Promise((resolve, reject) => {
+      const results = [];
+      fs2.createReadStream(filePath).pipe(csv()).on("data", (data) => results.push(data)).on("end", () => resolve(results)).on("error", (error) => reject(error));
+    });
+  }
+};
+
+// src/readers/json.ts
+import fs3 from "fs/promises";
+var JsonReader = class extends DataReader {
+  getSupportedExtensions() {
+    return ["json"];
+  }
+  async readFile(filePath) {
+    const content = await fs3.readFile(filePath, "utf8");
+    const data = JSON.parse(content);
+    if (Array.isArray(data)) {
+      return data;
+    }
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+    throw new Error(`Invalid JSON data structure in file: ${filePath}. Expected array or object.`);
+  }
+};
+
+// src/readers/yaml.ts
+import fs4 from "fs/promises";
+import yaml2 from "js-yaml";
+var YamlReader = class extends DataReader {
+  getSupportedExtensions() {
+    return ["yaml", "yml"];
+  }
+  async readFile(filePath) {
+    const content = await fs4.readFile(filePath, "utf8");
+    const data = yaml2.load(content);
+    if (Array.isArray(data)) {
+      return data;
+    }
+    if (typeof data === "object" && data !== null) {
+      return [data];
+    }
+    throw new Error(`Invalid YAML data structure in file: ${filePath}. Expected array or object.`);
+  }
+};
+
+// src/readers/jsonl.ts
+import fs5 from "fs/promises";
+var JsonlReader = class extends DataReader {
+  getSupportedExtensions() {
+    return ["jsonl", "ndjson"];
+  }
+  async readFile(filePath) {
+    const content = await fs5.readFile(filePath, "utf8");
+    const lines = content.split("\n").filter((line) => line.trim());
+    const results = [];
+    for (let i = 0; i < lines.length; i++) {
+      try {
+        const data = JSON.parse(lines[i]);
+        results.push(data);
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(
+          `Invalid JSON at line ${i + 1} in file: ${filePath}. Error: ${errorMessage}`
+        );
+      }
+    }
+    return results;
+  }
+};
+
+// src/readers/index.ts
+DataReaderFactory.registerReader(new CsvReader());
+DataReaderFactory.registerReader(new JsonReader());
+DataReaderFactory.registerReader(new YamlReader());
+DataReaderFactory.registerReader(new JsonlReader());
+
+// src/lib/metrics.ts
+var MetricsCollector = class {
+  constructor() {
+    this.metrics = {
+      totalEntities: 0,
+      totalSuccesses: 0,
+      totalFailures: 0,
+      entityMetrics: /* @__PURE__ */ new Map(),
+      requestDurations: [],
+      retryAttempts: 0,
+      retrySuccesses: 0,
+      retryFailures: 0,
+      startTime: Date.now()
+    };
+  }
+  startEntityProcessing(entityName) {
+    if (!this.metrics.entityMetrics.has(entityName)) {
+      this.metrics.entityMetrics.set(entityName, {
+        entityName,
+        successCount: 0,
+        failureCount: 0,
+        startTime: Date.now()
+      });
+    }
+  }
+  recordSuccess(entityName) {
+    const entityMetric = this.metrics.entityMetrics.get(entityName);
+    if (entityMetric) {
+      entityMetric.successCount++;
+      this.metrics.totalSuccesses++;
+      this.metrics.totalEntities++;
+    }
+  }
+  recordFailure(entityName) {
+    const entityMetric = this.metrics.entityMetrics.get(entityName);
+    if (entityMetric) {
+      entityMetric.failureCount++;
+      this.metrics.totalFailures++;
+      this.metrics.totalEntities++;
+    }
+  }
+  finishEntityProcessing(entityName) {
+    const entityMetric = this.metrics.entityMetrics.get(entityName);
+    if (entityMetric) {
+      entityMetric.endTime = Date.now();
+    }
+  }
+  finishProcessing() {
+    this.metrics.endTime = Date.now();
+    return { ...this.metrics };
+  }
+  getEntityMetrics(entityName) {
+    return this.metrics.entityMetrics.get(entityName);
+  }
+  getMetrics() {
+    const endTime = this.metrics.endTime || Date.now();
+    const totalDuration = endTime - this.metrics.startTime;
+    const entities = {};
+    for (const [name, metrics] of this.metrics.entityMetrics) {
+      const entityEndTime = metrics.endTime || Date.now();
+      entities[name] = {
+        rowsProcessed: metrics.successCount + metrics.failureCount,
+        successfulRows: metrics.successCount,
+        failedRows: metrics.failureCount,
+        duration: entityEndTime - metrics.startTime
+      };
+    }
+    return {
+      totalRows: this.metrics.totalEntities,
+      successfulOperations: this.metrics.totalSuccesses,
+      failedOperations: this.metrics.totalFailures,
+      totalDuration,
+      entities
+    };
+  }
+  getTotalProcessed() {
+    return this.metrics.totalEntities;
+  }
+  getSuccessRate() {
+    if (this.metrics.totalEntities === 0) return 0;
+    return this.metrics.totalSuccesses / this.metrics.totalEntities * 100;
+  }
+  recordRequestDuration(duration) {
+    this.metrics.requestDurations.push(duration);
+  }
+  recordRetrySuccess(attempts) {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retrySuccesses++;
+  }
+  recordRetryFailure(attempts) {
+    this.metrics.retryAttempts += attempts;
+    this.metrics.retryFailures++;
+  }
+  getAverageRequestDuration() {
+    if (this.metrics.requestDurations.length === 0) return 0;
+    const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);
+    return sum / this.metrics.requestDurations.length;
+  }
+  getDurationMs() {
+    const endTime = this.metrics.endTime || Date.now();
+    return endTime - this.metrics.startTime;
+  }
+  generateSummary() {
+    const duration = this.getDurationMs();
+    const successRate = this.getSuccessRate();
+    const avgRequestDuration = this.getAverageRequestDuration();
+    let summary = `
+\u{1F4CA} Processing Summary:
+`;
+    summary += `   Total Processed: ${this.metrics.totalEntities}
+`;
+    summary += `   \u2713 Successes: ${this.metrics.totalSuccesses}
+`;
+    summary += `   \u2717 Failures: ${this.metrics.totalFailures}
+`;
+    summary += `   Success Rate: ${successRate.toFixed(1)}%
+`;
+    summary += `   Duration: ${(duration / 1e3).toFixed(2)}s
+`;
+    if (this.metrics.requestDurations.length > 0) {
+      summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms
+`;
+    }
+    if (this.metrics.retryAttempts > 0) {
+      summary += `   Retry Attempts: ${this.metrics.retryAttempts}
+`;
+      summary += `   Retry Successes: ${this.metrics.retrySuccesses}
+`;
+      summary += `   Retry Failures: ${this.metrics.retryFailures}
+`;
+    }
+    if (this.metrics.entityMetrics.size > 0) {
+      summary += `
+\u{1F4CB} Per-Entity Breakdown:
+`;
+      for (const [entityName, entityMetric] of this.metrics.entityMetrics) {
+        const entityTotal = entityMetric.successCount + entityMetric.failureCount;
+        const entityRate = entityTotal > 0 ? entityMetric.successCount / entityTotal * 100 : 0;
+        const entityDuration = entityMetric.endTime ? entityMetric.endTime - entityMetric.startTime : 0;
+        summary += `   ${entityName}: ${entityTotal} total (${entityMetric.successCount} \u2713, ${entityMetric.failureCount} \u2717) - ${entityRate.toFixed(
+          1
+        )}% success - ${(entityDuration / 1e3).toFixed(2)}s
+`;
+      }
+    }
+    return summary;
+  }
+};
+
+// src/lib/mapper.ts
+var DataMapper = class {
+  constructor(client, basePath = process.cwd(), metrics, logger = noopLogger, formatOverride) {
+    this.client = client;
+    this.basePath = basePath;
+    this.metrics = metrics || new MetricsCollector();
+    this.logger = logger;
+    this.formatOverride = formatOverride;
+  }
+  discoverMappings(configDir, entityFilter) {
+    const mappingsPath = path2.resolve(this.basePath, configDir, "mappings");
+    try {
+      const files = fs6.readdirSync(mappingsPath);
+      let jsonFiles = files.filter((file) => file.endsWith(".json"));
+      if (entityFilter && entityFilter.length > 0) {
+        const requestedEntities = new Set(entityFilter);
+        const foundEntities = /* @__PURE__ */ new Set();
+        jsonFiles = jsonFiles.filter((file) => {
+          const entityName = path2.basename(file, ".json");
+          if (requestedEntities.has(entityName)) {
+            foundEntities.add(entityName);
+            return true;
+          }
+          return false;
+        });
+        const notFound = entityFilter.filter((e) => !foundEntities.has(e));
+        if (notFound.length > 0) {
+          this.logger.warn(
+            `Warning: The following entities were not found in mappings: ${notFound.join(", ")}`
+          );
+        }
+      }
+      jsonFiles.sort();
+      this.logger.info(`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`);
+      return jsonFiles.map((file) => path2.join(configDir, "mappings", file));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger.error(`Error reading mappings directory ${mappingsPath}: ${message}`);
+      return [];
+    }
+  }
+  /**
+   * Process an entity (backward-compatible method)
+   */
+  async processEntity(configPath, parallelConfig, retryConfig) {
+    return this.processEntityWithEvents(configPath, parallelConfig, retryConfig);
+  }
+  /**
+   * Process an entity with event callbacks and abort support
+   */
+  async processEntityWithEvents(configPath, parallelConfig, retryConfig, signal, callbacks) {
+    const entityName = path2.basename(configPath, ".json");
+    const entityStartTime = Date.now();
+    this.logger.info(`Processing entity: ${configPath}`);
+    this.metrics.startEntityProcessing(entityName);
+    const configFullPath = path2.resolve(this.basePath, configPath);
+    const config = JSON.parse(fs6.readFileSync(configFullPath, "utf8"));
+    const configDir = path2.dirname(path2.dirname(configFullPath));
+    const dataFile = config.dataFile || config.csvFile;
+    if (!dataFile) {
+      throw new Error(`No data file specified in mapping config: ${configPath}`);
+    }
+    const dataPath = path2.resolve(configDir, dataFile);
+    const format = this.formatOverride || config.dataFormat;
+    const reader = DataReaderFactory.getReader(dataPath, format);
+    const data = await reader.readFile(dataPath);
+    const graphqlPath = path2.resolve(configDir, config.graphqlFile);
+    const mutation = fs6.readFileSync(graphqlPath, "utf8");
+    callbacks?.onEntityStart?.({
+      entityName,
+      mappingPath: configPath,
+      totalRows: data.length
+    });
+    if (parallelConfig && parallelConfig.concurrency > 1) {
+      await this.processRowsConcurrentlyWithEvents(
+        data,
+        mutation,
+        config.mapping,
+        entityName,
+        parallelConfig,
+        retryConfig,
+        signal,
+        callbacks
+      );
+    } else {
+      await this.processRowsSequentiallyWithEvents(
+        data,
+        mutation,
+        config.mapping,
+        entityName,
+        retryConfig,
+        signal,
+        callbacks
+      );
+    }
+    this.metrics.finishEntityProcessing(entityName);
+    const internalMetrics = this.metrics.getEntityMetrics(entityName);
+    const duration = Date.now() - entityStartTime;
+    const entityMetrics = internalMetrics ? {
+      rowsProcessed: internalMetrics.successCount + internalMetrics.failureCount,
+      successfulRows: internalMetrics.successCount,
+      failedRows: internalMetrics.failureCount,
+      duration
+    } : {
+      rowsProcessed: 0,
+      successfulRows: 0,
+      failedRows: 0,
+      duration
+    };
+    callbacks?.onEntityComplete?.({
+      entityName,
+      metrics: entityMetrics,
+      success: entityMetrics.failedRows === 0,
+      durationMs: duration
+    });
+  }
+  async processRowsSequentiallyWithEvents(data, mutation, mapping, entityName, retryConfig, signal, callbacks) {
+    const totalRows = data.length;
+    const variableTypes = this.extractVariableTypes(mutation);
+    for (let i = 0; i < data.length; i++) {
+      if (signal?.aborted) {
+        this.logger.info(`Processing cancelled at row ${i + 1}/${totalRows}`);
+        return;
+      }
+      const row = data[i];
+      const variables = this.mapRowToVariables(row, mapping, variableTypes);
+      const rowStartTime = Date.now();
+      try {
+        const result = await this.client.executeMutation(mutation, variables, retryConfig, signal);
+        this.metrics.recordSuccess(entityName);
+        callbacks?.onRowSuccess?.({
+          entityName,
+          rowIndex: i,
+          row,
+          result,
+          durationMs: Date.now() - rowStartTime
+        });
+        if ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1) {
+          const progress = ((i + 1) / totalRows * 100).toFixed(1);
+          this.logger.info(`\u{1F4CA} Progress: ${i + 1}/${totalRows} (${progress}%) \u2713`);
+        }
+      } catch (error) {
+        if (signal?.aborted) {
+          this.logger.info(`Processing cancelled at row ${i + 1}/${totalRows}`);
+          return;
+        }
+        this.metrics.recordFailure(entityName);
+        this.logger.error(`\u2717 Failed to create entity for row ${i + 1}`, row, error);
+        callbacks?.onRowFailure?.({
+          entityName,
+          rowIndex: i,
+          row,
+          error: error instanceof Error ? error : new Error(String(error)),
+          retryAttempts: retryConfig?.maxAttempts ?? 3
+        });
+      }
+    }
+  }
+  async processRowsConcurrentlyWithEvents(data, mutation, mapping, entityName, parallelConfig, retryConfig, signal, callbacks) {
+    const concurrency = parallelConfig.concurrency;
+    this.logger.info(`Processing ${data.length} rows with concurrency: ${concurrency}`);
+    const variableTypes = this.extractVariableTypes(mutation);
+    const chunks = this.chunkArray(data, concurrency);
+    let processedCount = 0;
+    const totalRows = data.length;
+    for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
+      if (signal?.aborted) {
+        this.logger.info(`Processing cancelled at chunk ${chunkIndex + 1}/${chunks.length}`);
+        return;
+      }
+      const chunk = chunks[chunkIndex];
+      const chunkStartIndex = chunkIndex * concurrency;
+      const promises = chunk.map(async (row, index) => {
+        const rowIndex = chunkStartIndex + index;
+        const variables = this.mapRowToVariables(row, mapping, variableTypes);
+        const rowStartTime = Date.now();
+        try {
+          const result = await this.client.executeMutation(
+            mutation,
+            variables,
+            retryConfig,
+            signal
+          );
+          this.metrics.recordSuccess(entityName);
+          callbacks?.onRowSuccess?.({
+            entityName,
+            rowIndex,
+            row,
+            result,
+            durationMs: Date.now() - rowStartTime
+          });
+          return { success: true, result, row, rowIndex };
+        } catch (error) {
+          this.metrics.recordFailure(entityName);
+          callbacks?.onRowFailure?.({
+            entityName,
+            rowIndex,
+            row,
+            error: error instanceof Error ? error : new Error(String(error)),
+            retryAttempts: retryConfig?.maxAttempts ?? 3
+          });
+          return { success: false, error, row, rowIndex };
+        }
+      });
+      const results = await Promise.allSettled(promises);
+      processedCount += chunk.length;
+      if (signal?.aborted) {
+        this.logger.info(`Processing cancelled after chunk ${chunkIndex + 1}/${chunks.length}`);
+        return;
+      }
+      let chunkSuccesses = 0;
+      let chunkFailures = 0;
+      results.forEach((result) => {
+        if (result.status === "fulfilled") {
+          const { success, error, row } = result.value;
+          if (success) {
+            chunkSuccesses++;
+          } else {
+            chunkFailures++;
+            this.logger.error(`\u2717 Failed to create entity for row`, row, error);
+          }
+        } else {
+          chunkFailures++;
+          this.logger.error(`\u2717 Promise rejected: ${result.reason}`);
+        }
+      });
+      const progress = (processedCount / totalRows * 100).toFixed(1);
+      this.logger.info(
+        `\u{1F4CA} Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${chunkIndex + 1}: ${chunkSuccesses} \u2713, ${chunkFailures} \u2717`
+      );
+    }
+  }
+  chunkArray(array, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+  mapRowToVariables(row, mapping, variableTypes) {
+    const variables = {};
+    for (const [graphqlVar, mappingValue] of Object.entries(mapping)) {
+      if (mappingValue === "$") {
+        variables[graphqlVar] = row;
+      } else if (typeof mappingValue === "string" && mappingValue.startsWith("$.")) {
+        const path3 = mappingValue.substring(2);
+        const value = this.getValueByPath(row, path3);
+        if (value !== void 0) {
+          const type = variableTypes[graphqlVar];
+          variables[graphqlVar] = this.convertValue(value, type, graphqlVar);
+        }
+      } else if (typeof mappingValue === "string" && row[mappingValue] !== void 0) {
+        const rawValue = row[mappingValue];
+        const type = variableTypes[graphqlVar];
+        variables[graphqlVar] = this.convertValue(rawValue, type, graphqlVar);
+      } else if (typeof mappingValue === "object" && mappingValue !== null) {
+        variables[graphqlVar] = this.mapNestedObject(row, mappingValue, variableTypes);
+      }
+    }
+    return variables;
+  }
+  getValueByPath(obj, path3) {
+    const parts = path3.split(".");
+    let current = obj;
+    for (const part of parts) {
+      if (current && typeof current === "object" && part in current) {
+        current = current[part];
+      } else {
+        return void 0;
+      }
+    }
+    return current;
+  }
+  mapNestedObject(row, mappingObj, variableTypes) {
+    if (Array.isArray(mappingObj)) {
+      return mappingObj.map((item) => this.mapNestedObject(row, item, variableTypes));
+    }
+    if (typeof mappingObj === "object" && mappingObj !== null) {
+      const result = {};
+      for (const [key, value] of Object.entries(mappingObj)) {
+        if (typeof value === "string" && value.startsWith("$.")) {
+          const path3 = value.substring(2);
+          let fieldValue = this.getValueByPath(row, path3);
+          if (key === "values" && typeof fieldValue === "string" && fieldValue.includes(",")) {
+            fieldValue = fieldValue.split(",").map((v) => v.trim());
+          }
+          result[key] = fieldValue;
+        } else if (typeof value === "string" && row[value] !== void 0) {
+          result[key] = row[value];
+        } else if (typeof value === "object") {
+          result[key] = this.mapNestedObject(row, value, variableTypes);
+        } else {
+          result[key] = value;
+        }
+      }
+      return result;
+    }
+    return mappingObj;
+  }
+  extractVariableTypes(mutation) {
+    const types = {};
+    try {
+      const document = parse(mutation);
+      for (const definition of document.definitions) {
+        if (definition.kind === "OperationDefinition" && definition.variableDefinitions) {
+          for (const variableDef of definition.variableDefinitions) {
+            const varName = variableDef.variable.name.value;
+            const typeName = this.extractTypeName(variableDef);
+            if (typeName) {
+              types[varName] = typeName;
+            }
+          }
+        }
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger.error(`Error parsing GraphQL mutation: ${message}`);
+    }
+    return types;
+  }
+  extractTypeName(variableDef) {
+    const type = variableDef.type;
+    if (type.kind === "NonNullType") {
+      if (type.type.kind === "NamedType") {
+        return type.type.name.value;
+      }
+    } else if (type.kind === "NamedType") {
+      return type.name.value;
+    }
+    return null;
+  }
+  convertValue(value, type, varName) {
+    if (!type) {
+      return value;
+    }
+    if (typeof value !== "string") {
+      return value;
+    }
+    const trimmedValue = value.trim();
+    switch (type) {
+      case "Int":
+        const intValue = Number(trimmedValue);
+        if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {
+          this.logger.warn(
+            `Warning: Cannot convert "${value}" to Int for variable $${varName}. Expected a valid integer. Using original value.`
+          );
+          return value;
+        }
+        return intValue;
+      case "Float":
+        const floatValue = Number(trimmedValue);
+        if (isNaN(floatValue) || !isFinite(floatValue)) {
+          this.logger.warn(
+            `Warning: Cannot convert "${value}" to Float for variable $${varName}. Expected a valid number. Using original value.`
+          );
+          return value;
+        }
+        return floatValue;
+      case "Boolean":
+        const lowerValue = trimmedValue.toLowerCase();
+        if (lowerValue === "true" || lowerValue === "1") return true;
+        if (lowerValue === "false" || lowerValue === "0") return false;
+        this.logger.warn(
+          `Warning: Cannot convert "${value}" to Boolean for variable $${varName}. Expected "true", "false", "1", or "0". Using original value.`
+        );
+        return value;
+      case "String":
+        return value;
+      default:
+        this.logger.debug(
+          `Unknown GraphQL type "${type}" for variable $${varName}. Keeping value as string.`
+        );
+        return value;
+    }
+  }
+  getMetrics() {
+    return this.metrics;
+  }
+};
+
+// src/lib/dependency-resolver.ts
+var DependencyResolver = class {
+  constructor(entities, dependencies = {}, allowPartialResolution = false) {
+    this.entities = entities;
+    this.dependencies = dependencies;
+    this.allowPartialResolution = allowPartialResolution;
+  }
+  resolveExecutionOrder() {
+    const waves = [];
+    const processed = /* @__PURE__ */ new Set();
+    const inProgress = /* @__PURE__ */ new Set();
+    let waveNumber = 0;
+    while (processed.size < this.entities.length) {
+      const currentWave = [];
+      for (const entity of this.entities) {
+        if (processed.has(entity) || inProgress.has(entity)) {
+          continue;
+        }
+        const deps = this.dependencies[entity] || [];
+        const canProcess = deps.every(
+          (dep) => processed.has(dep) || this.allowPartialResolution && !this.entities.includes(dep)
+        );
+        if (canProcess) {
+          currentWave.push(entity);
+          inProgress.add(entity);
+        }
+      }
+      if (currentWave.length === 0) {
+        const remaining = this.entities.filter((e) => !processed.has(e));
+        throw new Error(
+          `Circular dependency detected or missing dependencies for entities: ${remaining.join(
+            ", "
+          )}`
+        );
+      }
+      waves.push({
+        entities: currentWave,
+        wave: waveNumber++
+      });
+      currentWave.forEach((entity) => {
+        processed.add(entity);
+        inProgress.delete(entity);
+      });
+    }
+    return waves;
+  }
+  validateDependencies() {
+    const errors = [];
+    const entitySet = new Set(this.entities);
+    for (const [entity, deps] of Object.entries(this.dependencies)) {
+      if (!entitySet.has(entity)) {
+        errors.push(`Entity '${entity}' has dependencies but is not in the entity list`);
+        continue;
+      }
+      for (const dep of deps) {
+        if (!entitySet.has(dep)) {
+          errors.push(`Entity '${entity}' depends on '${dep}' which does not exist`);
+        }
+      }
+    }
+    return errors;
+  }
+  getDependents(entityName) {
+    return Object.entries(this.dependencies).filter(([_, deps]) => deps.includes(entityName)).map(([entity, _]) => entity);
+  }
+  getDependencies(entityName) {
+    return this.dependencies[entityName] || [];
+  }
+};
+
+// src/lib/gql-ingest.ts
+import { basename } from "path";
+
+// src/lib/events.ts
+var DEFAULT_EVENT_OPTIONS = {
+  emitRowEvents: true,
+  progressInterval: 1e3,
+  emitProgressEvents: true
+};
+
+// src/lib/gql-ingest.ts
+var GQLIngest = class extends EventEmitter {
+  constructor(options) {
+    super();
+    // Cancellation state
+    this.abortController = null;
+    this.isProcessing = false;
+    this.startTime = 0;
+    this.progressIntervalId = null;
+    // Processing state for progress tracking
+    this.currentWave = 0;
+    this.totalWaves = 0;
+    this.entitiesCompleted = 0;
+    this.totalEntities = 0;
+    this.endpoint = options.endpoint;
+    this.headers = options.headers || {};
+    this.logger = options.logger ?? noopLogger;
+    this.formatOverride = options.formatOverride;
+    this.eventOptions = { ...DEFAULT_EVENT_OPTIONS, ...options.eventOptions };
+    this.metrics = new MetricsCollector();
+    this.client = new GraphQLClientWrapper(this.endpoint, this.headers, this.metrics, this.logger);
+    this.mapper = new DataMapper(
+      this.client,
+      process.cwd(),
+      this.metrics,
+      this.logger,
+      this.formatOverride
+    );
+  }
+  /**
+   * Cancel the current ingestion process
+   * @param reason Optional reason for cancellation
+   */
+  cancel(reason = "User requested cancellation") {
+    if (this.abortController && this.isProcessing) {
+      this.abortController.abort(reason);
+    }
+  }
+  /**
+   * Check if processing is currently in progress
+   */
+  get processing() {
+    return this.isProcessing;
+  }
+  /**
+   * Safely emit an event, catching any errors from listeners
+   */
+  safeEmit(event, payload) {
+    try {
+      return this.emit(event, payload);
+    } catch (error) {
+      this.logger.error(`Error in event listener for '${String(event)}':`, error);
+      return false;
+    }
+  }
+  /**
+   * Start the progress interval timer
+   */
+  startProgressInterval() {
+    if (!this.eventOptions.emitProgressEvents) return;
+    this.progressIntervalId = setInterval(() => {
+      this.emitProgressEvent();
+    }, this.eventOptions.progressInterval);
+  }
+  /**
+   * Stop the progress interval timer
+   */
+  stopProgressInterval() {
+    if (this.progressIntervalId) {
+      clearInterval(this.progressIntervalId);
+      this.progressIntervalId = null;
+    }
+  }
+  /**
+   * Emit a progress event with current state
+   */
+  emitProgressEvent() {
+    const metrics = this.metrics.getMetrics();
+    const payload = {
+      currentWave: this.currentWave,
+      totalWaves: this.totalWaves,
+      entitiesCompleted: this.entitiesCompleted,
+      totalEntities: this.totalEntities,
+      rowsProcessed: metrics.totalRows,
+      successfulRows: metrics.successfulOperations,
+      failedRows: metrics.failedOperations,
+      progressPercent: this.totalEntities > 0 ? this.entitiesCompleted / this.totalEntities * 100 : 0,
+      elapsedMs: Date.now() - this.startTime
+    };
+    this.safeEmit("progress", payload);
+  }
+  /**
+   * Handle cancellation and emit event
+   */
+  handleCancellation(reason) {
+    this.stopProgressInterval();
+    const payload = {
+      reason: reason || "Cancelled",
+      metrics: this.metrics.getMetrics(),
+      currentEntity: this.currentEntity,
+      elapsedMs: Date.now() - this.startTime
+    };
+    this.safeEmit("cancelled", payload);
+    return {
+      metrics: this.metrics.getMetrics(),
+      success: false,
+      cancelled: true,
+      errors: [`Operation cancelled: ${reason}`]
+    };
+  }
+  /**
+   * Combine multiple AbortSignals into one
+   */
+  combineSignals(...signals) {
+    const controller = new AbortController();
+    for (const signal of signals) {
+      if (signal.aborted) {
+        controller.abort(signal.reason);
+        break;
+      }
+      signal.addEventListener("abort", () => controller.abort(signal.reason), {
+        once: true
+      });
+    }
+    return controller.signal;
+  }
+  /**
+   * Ingest data from a configuration directory
+   * @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
+   * @param options Optional ingestion options
+   * @returns Promise with ingestion result
+   */
+  async ingest(configPath, options) {
+    const errors = [];
+    this.abortController = new AbortController();
+    const signal = options?.signal ? this.combineSignals(options.signal, this.abortController.signal) : this.abortController.signal;
+    this.isProcessing = true;
+    this.startTime = Date.now();
+    this.entitiesCompleted = 0;
+    this.currentWave = 0;
+    this.currentEntity = void 0;
+    try {
+      if (signal.aborted) {
+        return this.handleCancellation(signal.reason);
+      }
+      this.metrics = new MetricsCollector();
+      this.client = new GraphQLClientWrapper(
+        this.endpoint,
+        this.headers,
+        this.metrics,
+        this.logger
+      );
+      this.mapper = new DataMapper(
+        this.client,
+        process.cwd(),
+        this.metrics,
+        this.logger,
+        options?.format ?? this.formatOverride
+      );
+      const config = loadConfig(configPath, this.logger);
+      let entityFilter;
+      if (options?.entities) {
+        if (typeof options.entities === "string") {
+          entityFilter = options.entities.split(",").map((e) => e.trim());
+        } else {
+          entityFilter = options.entities;
+        }
+      }
+      const mappingPaths = this.mapper.discoverMappings(configPath, entityFilter);
+      if (mappingPaths.length === 0) {
+        const filterMsg = entityFilter ? ` matching entities: ${entityFilter.join(", ")}` : "";
+        const warning = `No mapping files found in ${configPath}/mappings${filterMsg}`;
+        this.logger.warn(warning);
+        return {
+          metrics: this.metrics.getMetrics(),
+          success: false,
+          errors: [warning]
+        };
+      }
+      const entityNames = mappingPaths.map((path3) => basename(path3, ".json"));
+      this.totalEntities = entityNames.length;
+      const relevantDependencies = {};
+      if (config.entityDependencies) {
+        for (const entity of entityNames) {
+          if (config.entityDependencies[entity]) {
+            relevantDependencies[entity] = config.entityDependencies[entity];
+          }
+        }
+      }
+      const resolver = new DependencyResolver(
+        entityNames,
+        relevantDependencies,
+        !!entityFilter
+        // Allow partial resolution when using --entities
+      );
+      const validationErrors = resolver.validateDependencies();
+      if (validationErrors.length > 0) {
+        if (entityFilter) {
+          this.logger.warn("\n\u26A0\uFE0F  Warning: Dependency validation issues:");
+          validationErrors.forEach((error) => this.logger.warn(`  - ${error}`));
+          this.logger.warn("This may cause errors if the dependent data doesn't already exist.\n");
+        } else {
+          this.logger.error("Dependency validation errors:");
+          validationErrors.forEach((error) => {
+            this.logger.error(`  - ${error}`);
+            errors.push(error);
+          });
+          return {
+            metrics: this.metrics.getMetrics(),
+            success: false,
+            errors
+          };
+        }
+      }
+      const waves = resolver.resolveExecutionOrder();
+      this.totalWaves = waves.length;
+      const startedPayload = {
+        configPath,
+        totalEntities: entityNames.length,
+        entityNames,
+        totalWaves: waves.length,
+        startTime: this.startTime
+      };
+      this.safeEmit("started", startedPayload);
+      this.startProgressInterval();
+      await this.processEntitiesInWaves(
+        mappingPaths,
+        resolver,
+        this.mapper,
+        config,
+        this.logger,
+        signal
+      );
+      if (signal.aborted) {
+        return this.handleCancellation(signal.reason);
+      }
+      this.metrics.finishProcessing();
+      this.stopProgressInterval();
+      const finalMetrics = this.metrics.getMetrics();
+      const allSuccessful = finalMetrics.failedOperations === 0;
+      const finishedPayload = {
+        metrics: finalMetrics,
+        durationMs: Date.now() - this.startTime,
+        allSuccessful
+      };
+      this.safeEmit("finished", finishedPayload);
+      return {
+        metrics: finalMetrics,
+        success: true
+      };
+    } catch (error) {
+      this.stopProgressInterval();
+      if (signal.aborted) {
+        return this.handleCancellation(signal.reason);
+      }
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.logger.error(`Error: ${errorMessage}`);
+      errors.push(errorMessage);
+      const erroredPayload = {
+        error: error instanceof Error ? error : new Error(String(error)),
+        metrics: this.metrics.getMetrics(),
+        currentEntity: this.currentEntity,
+        elapsedMs: Date.now() - this.startTime
+      };
+      this.safeEmit("errored", erroredPayload);
+      return {
+        metrics: this.metrics.getMetrics(),
+        success: false,
+        errors
+      };
+    } finally {
+      this.isProcessing = false;
+      this.abortController = null;
+      this.stopProgressInterval();
+    }
+  }
+  /**
+   * Ingest specific entities from a configuration directory
+   * @param configPath Path to configuration directory
+   * @param entities Array of entity names to process
+   * @returns Promise with ingestion result
+   */
+  async ingestEntities(configPath, entities) {
+    return this.ingest(configPath, { entities });
+  }
+  /**
+   * Get current processing metrics
+   * @returns Current metrics
+   */
+  getMetrics() {
+    return this.metrics.getMetrics();
+  }
+  /**
+   * Get a summary of the current metrics
+   * @returns Formatted summary string
+   */
+  getMetricsSummary() {
+    return this.metrics.generateSummary();
+  }
+  /**
+   * Get the GraphQL client for advanced usage
+   * @returns GraphQL client wrapper
+   */
+  getClient() {
+    return this.client;
+  }
+  /**
+   * Get the data mapper for advanced usage
+   * @returns Data mapper
+   */
+  getMapper() {
+    return this.mapper;
+  }
+  /**
+   * Set the logger instance
+   * @param logger Logger instance to use
+   */
+  setLogger(logger) {
+    this.logger = logger;
+    this.client = new GraphQLClientWrapper(this.endpoint, this.headers, this.metrics, logger);
+    this.mapper = new DataMapper(
+      this.client,
+      process.cwd(),
+      this.metrics,
+      logger,
+      this.formatOverride
+    );
+  }
+  /**
+   * Update headers for GraphQL requests
+   * @param headers New headers to use
+   */
+  setHeaders(headers) {
+    this.headers = headers;
+    this.client = new GraphQLClientWrapper(this.endpoint, headers, this.metrics, this.logger);
+    this.mapper = new DataMapper(
+      this.client,
+      process.cwd(),
+      this.metrics,
+      this.logger,
+      this.formatOverride
+    );
+  }
+  /**
+   * Process entities in dependency-aware waves with abort support
+   */
+  async processEntitiesInWaves(mappingPaths, resolver, mapper, config, logger, signal) {
+    const waves = resolver.resolveExecutionOrder();
+    const pathMap = new Map(mappingPaths.map((path3) => [basename(path3, ".json"), path3]));
+    logger.info(`Processing ${waves.length} dependency waves...`);
+    for (const wave of waves) {
+      if (signal?.aborted) {
+        return;
+      }
+      this.currentWave = wave.wave;
+      logger.info(`Wave ${wave.wave + 1}: Processing entities [${wave.entities.join(", ")}]`);
+      const entityConcurrency = config.parallelProcessing.entityConcurrency;
+      const chunks = this.chunkArray(wave.entities, entityConcurrency);
+      for (const chunk of chunks) {
+        if (signal?.aborted) {
+          return;
+        }
+        const entityPromises = chunk.map(async (entityName) => {
+          if (signal?.aborted) {
+            return;
+          }
+          const configPath = pathMap.get(entityName);
+          if (configPath) {
+            this.currentEntity = entityName;
+            try {
+              const entityConfig = getEntityConfig(entityName, config, logger);
+              const retryConfig = getRetryConfig(entityName, config);
+              await mapper.processEntityWithEvents(configPath, entityConfig, retryConfig, signal, {
+                onEntityStart: (payload) => this.safeEmit("entityStart", {
+                  ...payload,
+                  waveIndex: wave.wave
+                }),
+                onEntityComplete: (payload) => {
+                  this.entitiesCompleted++;
+                  this.safeEmit("entityComplete", payload);
+                },
+                onRowSuccess: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowSuccess", payload) : void 0,
+                onRowFailure: this.eventOptions.emitRowEvents ? (payload) => this.safeEmit("rowFailure", payload) : void 0
+              });
+            } catch (error) {
+              const message = error instanceof Error ? error.message : String(error);
+              logger.warn(`Warning: Could not process ${configPath}: ${message}`);
+            }
+          }
+        });
+        await Promise.allSettled(entityPromises);
+      }
+    }
+  }
+  /**
+   * Utility function to chunk array into smaller arrays
+   */
+  chunkArray(array, chunkSize) {
+    if (chunkSize <= 0) return [array];
+    const chunks = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+};
+export {
+  CsvReader,
+  DEFAULT_CONFIG,
+  DEFAULT_EVENT_OPTIONS,
+  DEFAULT_PARALLEL_CONFIG,
+  DEFAULT_RETRY_CONFIG,
+  DataMapper,
+  DataReader,
+  DataReaderFactory,
+  DependencyResolver,
+  GQLIngest,
+  GraphQLClientWrapper,
+  JsonReader,
+  JsonlReader,
+  MetricsCollector,
+  YamlReader,
+  createConsoleLogger,
+  createDefaultLogger,
+  getEntityConfig,
+  getRetryConfig,
+  loadConfig,
+  noopLogger
+};
+//# sourceMappingURL=index.js.map