@jackchuka/gql-ingest 1.4.0 → 1.5.0

package/dist/dependency-resolver.d.ts

@@ -8,7 +8,8 @@ export interface ExecutionWave {
 export declare class DependencyResolver {
     private dependencies;
     private entities;
-    constructor(entities: string[], dependencies?: DependencyGraph);
+    private allowPartialResolution;
+    constructor(entities: string[], dependencies?: DependencyGraph, allowPartialResolution?: boolean);
     resolveExecutionOrder(): ExecutionWave[];
     validateDependencies(): string[];
     getDependents(entityName: string): string[];

package/dist/dependency-resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;gBAEf,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,GAAE,eAAoB;IAKlE,qBAAqB,IAAI,aAAa,EAAE;IA8CxC,oBAAoB,IAAI,MAAM,EAAE;IAwBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
+{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAE5B,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,GAAE,eAAoB,EAAE,sBAAsB,GAAE,OAAe;IAM3G,qBAAqB,IAAI,aAAa,EAAE;IAgDxC,oBAAoB,IAAI,MAAM,EAAE;IAwBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}

package/dist/mapper.d.ts

@@ -12,7 +12,7 @@ export declare class DataMapper {
     private metrics;
     private verbose;
     constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean);
-    discoverMappings(configDir: string): string[];
+    discoverMappings(configDir: string, entityFilter?: string[]): string[];
     processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
     private processRowsSequentially;
     private processRowsConcurrently;

package/dist/mapper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAsDpB,UAAU,IAAI,gBAAgB;CAG/B"}
+{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IA4ChE,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IAkFrC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IA8DpB,UAAU,IAAI,gBAAgB;CAG/B"}

package/dist/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAmC1B"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CA2C1B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jackchuka/gql-ingest",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "A CLI tool for ingesting data from CSV files into a GraphQL API",
   "type": "module",
   "main": "dist/cli.js",

package/src/cli.ts

@@ -31,6 +31,10 @@ program
     "-c, --config <path>",
     "Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)"
   )
+  .option(
+    "-n, --entities <entities>",
+    "Comma-separated list of specific entities to process (e.g., users,products)"
+  )
   .option(
     "-h, --headers <headers>",
     "JSON string of headers to include in requests"
@@ -65,29 +69,61 @@ program
         options.verbose
       );
 
+      // Parse entities filter if provided
+      const entityFilter = options.entities
+        ? options.entities.split(",").map((e: string) => e.trim())
+        : undefined;
+
       // Discover all mapping files dynamically
-      const mappingPaths = mapper.discoverMappings(options.config);
+      const mappingPaths = mapper.discoverMappings(
+        options.config,
+        entityFilter
+      );
 
       if (mappingPaths.length === 0) {
-        console.warn(`No mapping files found in ${options.config}/mappings`);
+        const filterMsg = entityFilter
+          ? ` matching entities: ${entityFilter.join(", ")}`
+          : "";
+        console.warn(
+          `No mapping files found in ${options.config}/mappings${filterMsg}`
+        );
         return;
       }
 
       // Extract entity names from mapping paths
       const entityNames = mappingPaths.map((path) => basename(path, ".json"));
 
-      // Setup dependency resolver
+      // Filter dependencies to only include those relevant to selected entities
+      const relevantDependencies: Record<string, string[]> = {};
+      if (config.entityDependencies) {
+        for (const entity of entityNames) {
+          if (config.entityDependencies[entity]) {
+            relevantDependencies[entity] = config.entityDependencies[entity];
+          }
+        }
+      }
+
+      // Setup dependency resolver with filtered dependencies
       const resolver = new DependencyResolver(
         entityNames,
-        config.entityDependencies
+        relevantDependencies,
+        !!entityFilter // Allow partial resolution when using --entities
       );
 
       // Validate dependencies
       const validationErrors = resolver.validateDependencies();
       if (validationErrors.length > 0) {
-        console.error("Dependency validation errors:");
-        validationErrors.forEach((error) => console.error(`  - ${error}`));
-        process.exit(1);
+        if (entityFilter) {
+          // When using --entities flag, show warnings instead of errors
+          console.warn("\n⚠️  Warning: Dependency validation issues:");
+          validationErrors.forEach((error) => console.warn(`  - ${error}`));
+          console.warn("This may cause errors if the dependent data doesn't already exist.\n");
+        } else {
+          // Strict validation when processing all entities
+          console.error("Dependency validation errors:");
+          validationErrors.forEach((error) => console.error(`  - ${error}`));
+          process.exit(1);
+        }
       }
 
       // Process entities in dependency-aware waves

package/src/dependency-resolver.test.ts

@@ -88,12 +88,26 @@ describe("DependencyResolver", () => {
         a: ["missing"],
         b: ["a"],
       };
-      const resolver = new DependencyResolver(entities, dependencies);
+      const resolver = new DependencyResolver(entities, dependencies, false);
 
       expect(() => resolver.resolveExecutionOrder()).toThrow(
         "Circular dependency detected or missing dependencies for entities: a, b"
       );
     });
+
+    it("should allow entities with dependencies not in the entity list when partial resolution is enabled", () => {
+      const entities = ["a", "b"];
+      const dependencies = {
+        a: ["missing"],
+        b: ["a"],
+      };
+      const resolver = new DependencyResolver(entities, dependencies, true);
+
+      const waves = resolver.resolveExecutionOrder();
+      expect(waves).toHaveLength(2);
+      expect(waves[0].entities).toEqual(["a"]);
+      expect(waves[1].entities).toEqual(["b"]);
+    });
   });
 
   describe("validateDependencies", () => {

package/src/dependency-resolver.ts

@@ -10,10 +10,12 @@ export interface ExecutionWave {
 export class DependencyResolver {
   private dependencies: DependencyGraph;
   private entities: string[];
+  private allowPartialResolution: boolean;
 
-  constructor(entities: string[], dependencies: DependencyGraph = {}) {
+  constructor(entities: string[], dependencies: DependencyGraph = {}, allowPartialResolution: boolean = false) {
     this.entities = entities;
     this.dependencies = dependencies;
+    this.allowPartialResolution = allowPartialResolution;
   }
 
   resolveExecutionOrder(): ExecutionWave[] {
@@ -31,7 +33,9 @@ export class DependencyResolver {
         }
 
         const deps = this.dependencies[entity] || [];
-        const canProcess = deps.every((dep) => processed.has(dep));
+        const canProcess = deps.every((dep) => 
+          processed.has(dep) || (this.allowPartialResolution && !this.entities.includes(dep))
+        );
 
         if (canProcess) {
           currentWave.push(entity);

package/src/mapper.ts

@@ -30,12 +30,39 @@ export class DataMapper {
     this.verbose = verbose;
   }
 
-  discoverMappings(configDir: string): string[] {
+  discoverMappings(configDir: string, entityFilter?: string[]): string[] {
     const mappingsPath = path.resolve(this.basePath, configDir, "mappings");
 
     try {
       const files = fs.readdirSync(mappingsPath);
-      const jsonFiles = files.filter((file) => file.endsWith(".json")).sort(); // Alphabetical order for consistent processing
+      let jsonFiles = files.filter((file) => file.endsWith(".json"));
+
+      // Apply entity filter if provided
+      if (entityFilter && entityFilter.length > 0) {
+        const requestedEntities = new Set(entityFilter);
+        const foundEntities = new Set<string>();
+
+        jsonFiles = jsonFiles.filter((file) => {
+          const entityName = path.basename(file, ".json");
+          if (requestedEntities.has(entityName)) {
+            foundEntities.add(entityName);
+            return true;
+          }
+          return false;
+        });
+
+        // Check for requested entities that were not found
+        const notFound = entityFilter.filter((e) => !foundEntities.has(e));
+        if (notFound.length > 0) {
+          console.warn(
+            `Warning: The following entities were not found in mappings: ${notFound.join(
+              ", "
+            )}`
+          );
+        }
+      }
+
+      jsonFiles.sort(); // Alphabetical order for consistent processing
 
       console.log(
         `Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`
@@ -161,7 +188,11 @@ export class DataMapper {
     for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
       const chunk = chunks[chunkIndex];
       const promises = chunk.map(async (row) => {
-        const variables = this.mapCsvRowToVariables(row, mapping, variableTypes);
+        const variables = this.mapCsvRowToVariables(
+          row,
+          mapping,
+          variableTypes
+        );
 
         try {
           const result = await this.client.executeMutation(
@@ -285,7 +316,11 @@ export class DataMapper {
     return null;
   }
 
-  private convertValue(value: string, type: string | undefined, varName: string): any {
+  private convertValue(
+    value: string,
+    type: string | undefined,
+    varName: string
+  ): any {
     if (!type) {
       // No type information available, keep as string
       return value;
@@ -297,7 +332,11 @@ export class DataMapper {
       case "Int":
         const intValue = Number(trimmedValue);
         // Validate that it's a valid integer (no decimals, NaN, or Infinity)
-        if (isNaN(intValue) || !isFinite(intValue) || !Number.isInteger(intValue)) {
+        if (
+          isNaN(intValue) ||
+          !isFinite(intValue) ||
+          !Number.isInteger(intValue)
+        ) {
           console.warn(
             `Warning: Cannot convert "${value}" to Int for variable $${varName}. Expected a valid integer. Using original value.`
           );

package/src/metrics.ts

@@ -119,35 +119,43 @@ export class MetricsCollector {
     const duration = this.getDurationMs();
     const successRate = this.getSuccessRate();
     const avgRequestDuration = this.getAverageRequestDuration();
-    
+
     let summary = `\n📊 Processing Summary:\n`;
     summary += `   Total Processed: ${this.metrics.totalEntities}\n`;
     summary += `   ✓ Successes: ${this.metrics.totalSuccesses}\n`;
     summary += `   ✗ Failures: ${this.metrics.totalFailures}\n`;
     summary += `   Success Rate: ${successRate.toFixed(1)}%\n`;
     summary += `   Duration: ${(duration / 1000).toFixed(2)}s\n`;
-    
+
     if (this.metrics.requestDurations.length > 0) {
       summary += `   Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
     }
-    
+
     if (this.metrics.retryAttempts > 0) {
       summary += `   Retry Attempts: ${this.metrics.retryAttempts}\n`;
       summary += `   Retry Successes: ${this.metrics.retrySuccesses}\n`;
       summary += `   Retry Failures: ${this.metrics.retryFailures}\n`;
     }
 
-    if (this.metrics.entityMetrics.size > 1) {
+    if (this.metrics.entityMetrics.size > 0) {
       summary += `\n📋 Per-Entity Breakdown:\n`;
       for (const [entityName, entityMetric] of this.metrics.entityMetrics) {
-        const entityTotal = entityMetric.successCount + entityMetric.failureCount;
-        const entityRate = entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
-        const entityDuration = entityMetric.endTime ? entityMetric.endTime - entityMetric.startTime : 0;
-        
-        summary += `   ${entityName}: ${entityTotal} total (${entityMetric.successCount} ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(1)}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
+        const entityTotal =
+          entityMetric.successCount + entityMetric.failureCount;
+        const entityRate =
+          entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
+        const entityDuration = entityMetric.endTime
+          ? entityMetric.endTime - entityMetric.startTime
+          : 0;
+
+        summary += `   ${entityName}: ${entityTotal} total (${
+          entityMetric.successCount
+        } ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(
+          1
+        )}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
       }
     }
 
     return summary;
   }
-}
+}