@jackchuka/gql-ingest 3.1.8 → 4.0.0

package/dist/cli/templates/index.d.ts

@@ -25,5 +25,5 @@ export declare function generateConfigYaml(basePath: string, force: boolean, log
 export declare function generateEntityFiles(basePath: string, entityName: string, options: EntityTemplateOptions, logger: Logger, force?: boolean): Promise<void>;
 export declare function toPascalCase(str: string): string;
 export declare function validateEntityName(name: string): boolean;
-export declare function ensureDirectories(basePath: string, logger: Logger): void;
+export declare function ensureDirectories(basePath: string, _logger: Logger): void;
 //# sourceMappingURL=index.d.ts.map

package/dist/cli/templates/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cli/templates/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG1C,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAI3D,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAE/D;AAED,eAAO,MAAM,mBAAmB;;;;;;;;;;;;EAKtB,CAAC;AAEX,eAAO,MAAM,mBAAmB,EAAE,UAAkB,CAAC;AAErD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,UAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAYf;AA+DD,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,qBAAqB,EAC9B,MAAM,EAAE,MAAM,EACd,KAAK,UAAQ,GACZ,OAAO,CAAC,IAAI,CAAC,CAgCf;AA8DD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CASxE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cli/templates/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG1C,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAI3D,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAE/D;AAED,eAAO,MAAM,mBAAmB;;;;;;;;;;;;EAKtB,CAAC;AAEX,eAAO,MAAM,mBAAmB,EAAE,UAAkB,CAAC;AAErD,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,UAAkB,GACzB,OAAO,CAAC,IAAI,CAAC,CAYf;AA+DD,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAWf;AAED,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,qBAAqB,EAC9B,MAAM,EAAE,MAAM,EACd,KAAK,UAAQ,GACZ,OAAO,CAAC,IAAI,CAAC,CAmCf;AA+DD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAIzE"}

package/dist/index.js

@@ -12,7 +12,6 @@ import { GraphQLClient } from "graphql-request";
 
 // src/lib/config.ts
 import fs from "fs";
-import path from "path";
 import * as yaml from "js-yaml";
 
 // src/lib/logger.ts
@@ -13876,23 +13875,24 @@ var DEFAULT_CONFIG = {
   entityConfig: {},
   entityDependencies: {}
 };
-function loadConfig(configDir, logger = noopLogger) {
-  const configPath = path.join(configDir, "config.yaml");
+function loadConfig(configFile, logger = noopLogger) {
+  if (!configFile) {
+    logger.info("No config file provided, using defaults");
+    return DEFAULT_CONFIG;
+  }
   try {
-    if (!fs.existsSync(configPath)) {
-      logger.info("No config.yaml found, using default sequential processing");
-      return DEFAULT_CONFIG;
-    }
-    const fileContents = fs.readFileSync(configPath, "utf8");
-    const yamlConfig = yaml.load(fileContents);
-    if (yamlConfig === null || typeof yamlConfig !== "object" || Array.isArray(yamlConfig)) {
-      logger.warn("Warning: config.yaml is not a valid object. Using defaults.");
-      return DEFAULT_CONFIG;
-    }
+    const content = fs.readFileSync(configFile, "utf-8");
+    const yamlConfig = yaml.load(content);
+    logger.info(`Loaded config from ${configFile}`);
     return mergeWithDefaults(yamlConfig);
   } catch (error48) {
-    const errorMessage = error48 instanceof Error ? error48.message : String(error48);
-    logger.warn(`Warning: Failed to parse config.yaml: ${errorMessage}. Using defaults.`);
+    const isNotFound = error48 instanceof Error && "code" in error48 && // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
+    error48.code === "ENOENT";
+    if (isNotFound) {
+      logger.info(`Config file not found at ${configFile}, using defaults`);
+    } else {
+      logger.warn(`Failed to parse ${configFile}, using defaults`);
+    }
     return DEFAULT_CONFIG;
   }
 }
@@ -14038,7 +14038,7 @@ var GraphQLClientWrapper = class {
 
 // src/lib/mapper.ts
 import fs6 from "fs";
-import path2 from "path";
+import path from "path";
 import { parse as parse3, Kind } from "graphql";
 
 // src/readers/data-reader.ts
@@ -14322,38 +14322,6 @@ var DataMapper = class {
     this.logger = logger;
     this.formatOverride = formatOverride;
   }
-  discoverMappings(configDir, entityFilter) {
-    const mappingsPath = path2.resolve(this.basePath, configDir, "mappings");
-    try {
-      const files = fs6.readdirSync(mappingsPath);
-      let jsonFiles = files.filter((file2) => file2.endsWith(".json"));
-      if (entityFilter && entityFilter.length > 0) {
-        const requestedEntities = new Set(entityFilter);
-        const foundEntities = /* @__PURE__ */ new Set();
-        jsonFiles = jsonFiles.filter((file2) => {
-          const entityName = path2.basename(file2, ".json");
-          if (requestedEntities.has(entityName)) {
-            foundEntities.add(entityName);
-            return true;
-          }
-          return false;
-        });
-        const notFound = entityFilter.filter((e) => !foundEntities.has(e));
-        if (notFound.length > 0) {
-          this.logger.warn(
-            `Warning: The following entities were not found in mappings: ${notFound.join(", ")}`
-          );
-        }
-      }
-      jsonFiles.sort();
-      this.logger.info(`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`);
-      return jsonFiles.map((file2) => path2.join(configDir, "mappings", file2));
-    } catch (error48) {
-      const message = error48 instanceof Error ? error48.message : String(error48);
-      this.logger.error(`Error reading mappings directory ${mappingsPath}: ${message}`);
-      return [];
-    }
-  }
   /**
    * Process an entity (backward-compatible method)
    */
@@ -14364,22 +14332,18 @@ var DataMapper = class {
    * Process an entity with event callbacks and abort support
    */
   async processEntityWithEvents(configPath, parallelConfig, retryConfig, signal, callbacks) {
-    const entityName = path2.basename(configPath, ".json");
     const entityStartTime = Date.now();
-    this.logger.info(`Processing entity: ${configPath}`);
-    this.metrics.startEntityProcessing(entityName);
-    const configFullPath = path2.resolve(this.basePath, configPath);
+    const configFullPath = path.resolve(this.basePath, configPath);
     const config2 = JSON.parse(fs6.readFileSync(configFullPath, "utf8"));
-    const configDir = path2.dirname(path2.dirname(configFullPath));
-    const dataFile = config2.dataFile || config2.csvFile;
-    if (!dataFile) {
-      throw new Error(`No data file specified in mapping config: ${configPath}`);
-    }
-    const dataPath = path2.resolve(configDir, dataFile);
+    const entityName = config2.name;
+    this.logger.info(`Processing entity: ${entityName}`);
+    this.metrics.startEntityProcessing(entityName);
+    const entityDir = path.dirname(configFullPath);
+    const dataPath = path.resolve(entityDir, config2.dataFile);
     const format = this.formatOverride || config2.dataFormat;
     const reader = DataReaderFactory.getReader(dataPath, format);
     const data = await reader.readFile(dataPath);
-    const graphqlPath = path2.resolve(configDir, config2.graphqlFile);
+    const graphqlPath = path.resolve(entityDir, config2.graphqlFile);
     const mutation = fs6.readFileSync(graphqlPath, "utf8");
     callbacks?.onEntityStart?.({
       entityName,
@@ -14558,8 +14522,8 @@ var DataMapper = class {
       if (mappingValue === "$") {
         variables[graphqlVar] = row;
       } else if (typeof mappingValue === "string" && mappingValue.startsWith("$.")) {
-        const path3 = mappingValue.substring(2);
-        const value = this.getValueByPath(row, path3);
+        const dataPath = mappingValue.substring(2);
+        const value = this.getValueByPath(row, dataPath);
         if (value !== void 0) {
           const type = variableTypes[graphqlVar];
           variables[graphqlVar] = this.convertValue(value, type, graphqlVar);
@@ -14574,8 +14538,8 @@ var DataMapper = class {
     }
     return variables;
   }
-  getValueByPath(obj, path3) {
-    const parts = path3.split(".");
+  getValueByPath(obj, dataPath) {
+    const parts = dataPath.split(".");
     let current = obj;
     for (const part of parts) {
       if (current && typeof current === "object" && part in current) {
@@ -14594,8 +14558,8 @@ var DataMapper = class {
       const result = {};
       for (const [key, value] of Object.entries(mappingObj)) {
         if (typeof value === "string" && value.startsWith("$.")) {
-          const path3 = value.substring(2);
-          let fieldValue = this.getValueByPath(row, path3);
+          const dataPath = value.substring(2);
+          let fieldValue = this.getValueByPath(row, dataPath);
           if (key === "values" && typeof fieldValue === "string" && fieldValue.includes(",")) {
             fieldValue = fieldValue.split(",").map((v) => v.trim());
           }
@@ -14764,7 +14728,8 @@ var DependencyResolver = class {
 };
 
 // src/lib/gql-ingest.ts
-import { basename } from "path";
+import fs7 from "fs";
+import path2 from "path";
 
 // src/lib/events.ts
 var DEFAULT_EVENT_OPTIONS = {
@@ -14900,12 +14865,12 @@ var GQLIngest = class extends EventEmitter {
     return controller.signal;
   }
   /**
-   * Ingest data from a configuration directory
-   * @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
+   * Ingest entity files
+   * @param entityFiles Array of paths to entity JSON mapping files
    * @param options Optional ingestion options
    * @returns Promise with ingestion result
    */
-  async ingest(configPath, options) {
+  async ingest(entityFiles, options) {
     const errors = [];
     this.abortController = new AbortController();
     const signal = options?.signal ? this.combineSignals(options.signal, this.abortController.signal) : this.abortController.signal;
@@ -14932,19 +14897,9 @@ var GQLIngest = class extends EventEmitter {
         this.logger,
         options?.format ?? this.formatOverride
       );
-      const config2 = loadConfig(configPath, this.logger);
-      let entityFilter;
-      if (options?.entities) {
-        if (typeof options.entities === "string") {
-          entityFilter = options.entities.split(",").map((e) => e.trim());
-        } else {
-          entityFilter = options.entities;
-        }
-      }
-      const mappingPaths = this.mapper.discoverMappings(configPath, entityFilter);
-      if (mappingPaths.length === 0) {
-        const filterMsg = entityFilter ? ` matching entities: ${entityFilter.join(", ")}` : "";
-        const warning = `No mapping files found in ${configPath}/mappings${filterMsg}`;
+      const config2 = loadConfig(options?.config, this.logger);
+      if (entityFiles.length === 0) {
+        const warning = "No entity files provided";
         this.logger.warn(warning);
         return {
           metrics: this.metrics.getMetrics(),
@@ -14952,7 +14907,21 @@ var GQLIngest = class extends EventEmitter {
           errors: [warning]
         };
       }
-      const entityNames = mappingPaths.map((path3) => basename(path3, ".json"));
+      const entityFilter = options?.entities ? new Set(options.entities) : null;
+      const entityNames = [];
+      const pathMap = /* @__PURE__ */ new Map();
+      for (const file2 of entityFiles) {
+        const fullPath = path2.resolve(process.cwd(), file2);
+        const entityConfig = JSON.parse(fs7.readFileSync(fullPath, "utf8"));
+        if (!entityConfig.name) {
+          throw new Error(`Missing "name" field in entity file: ${file2}`);
+        }
+        if (entityFilter && !entityFilter.has(entityConfig.name)) {
+          continue;
+        }
+        entityNames.push(entityConfig.name);
+        pathMap.set(entityConfig.name, file2);
+      }
       this.totalEntities = entityNames.length;
       const relevantDependencies = {};
       if (config2.entityDependencies) {
@@ -14962,35 +14931,23 @@ var GQLIngest = class extends EventEmitter {
           }
         }
       }
-      const resolver = new DependencyResolver(
-        entityNames,
-        relevantDependencies,
-        !!entityFilter
-        // Allow partial resolution when using --entities
-      );
+      const resolver = new DependencyResolver(entityNames, relevantDependencies, false);
       const validationErrors = resolver.validateDependencies();
       if (validationErrors.length > 0) {
-        if (entityFilter) {
-          this.logger.warn("\n\u26A0\uFE0F  Warning: Dependency validation issues:");
-          validationErrors.forEach((error48) => this.logger.warn(`  - ${error48}`));
-          this.logger.warn("This may cause errors if the dependent data doesn't already exist.\n");
-        } else {
-          this.logger.error("Dependency validation errors:");
-          validationErrors.forEach((error48) => {
-            this.logger.error(`  - ${error48}`);
-            errors.push(error48);
-          });
-          return {
-            metrics: this.metrics.getMetrics(),
-            success: false,
-            errors
-          };
-        }
+        this.logger.error("Dependency validation errors:");
+        validationErrors.forEach((error48) => {
+          this.logger.error(`  - ${error48}`);
+          errors.push(error48);
+        });
+        return {
+          metrics: this.metrics.getMetrics(),
+          success: false,
+          errors
+        };
       }
       const waves = resolver.resolveExecutionOrder();
       this.totalWaves = waves.length;
       const startedPayload = {
-        configPath,
         totalEntities: entityNames.length,
         entityNames,
         totalWaves: waves.length,
@@ -14999,7 +14956,7 @@ var GQLIngest = class extends EventEmitter {
       this.safeEmit("started", startedPayload);
       this.startProgressInterval();
       await this.processEntitiesInWaves(
-        mappingPaths,
+        pathMap,
         resolver,
         this.mapper,
         config2,
@@ -15049,15 +15006,6 @@ var GQLIngest = class extends EventEmitter {
       this.stopProgressInterval();
     }
   }
-  /**
-   * Ingest specific entities from a configuration directory
-   * @param configPath Path to configuration directory
-   * @param entities Array of entity names to process
-   * @returns Promise with ingestion result
-   */
-  async ingestEntities(configPath, entities) {
-    return this.ingest(configPath, { entities });
-  }
   /**
    * Get current processing metrics
    * @returns Current metrics
@@ -15119,9 +15067,8 @@ var GQLIngest = class extends EventEmitter {
   /**
    * Process entities in dependency-aware waves with abort support
    */
-  async processEntitiesInWaves(mappingPaths, resolver, mapper, config2, logger, signal) {
+  async processEntitiesInWaves(pathMap, resolver, mapper, config2, logger, signal) {
     const waves = resolver.resolveExecutionOrder();
-    const pathMap = new Map(mappingPaths.map((path3) => [basename(path3, ".json"), path3]));
     logger.info(`Processing ${waves.length} dependency waves...`);
     for (const wave of waves) {
       if (signal?.aborted) {