@jackchuka/gql-ingest 2.2.2 → 3.1.0

package/README.md

@@ -3,7 +3,7 @@
 [![npm version](https://badge.fury.io/js/%40jackchuka%2Fgql-ingest.svg)](https://badge.fury.io/js/%40jackchuka%2Fgql-ingest)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-A TypeScript CLI tool that reads data from multiple formats (CSV, JSON, YAML, JSONL) and ingests it into GraphQL APIs through configurable mutations.
+A TypeScript library and CLI tool that reads data from multiple formats (CSV, JSON, YAML, JSONL) and ingests it into GraphQL APIs through configurable mutations.
 
 ## Features
 
@@ -16,6 +16,8 @@ A TypeScript CLI tool that reads data from multiple formats (CSV, JSON, YAML, JS
 - ✅ Entity-level and row-level concurrency control
 - ✅ **Retry capabilities** with exponential backoff and configurable error handling
 - ✅ Comprehensive metrics and progress tracking
+- ✅ **Event-based progress monitoring** with real-time callbacks
+- ✅ **Cancellation support** via AbortController pattern
 
 ## Installation
 
@@ -38,55 +40,280 @@ pnpm install
 pnpm run build
 ```
 
+## Quick Start
+
+Initialize a new configuration and start ingesting data in minutes:
+
+```bash
+# Create a new configuration directory
+gql-ingest init ./my-config
+
+# Add a new entity
+gql-ingest add users -p ./my-config -f json --fields "id,name,email"
+
+# Run ingestion
+gql-ingest -e https://your-api.com/graphql -c ./my-config
+```
+
 ## Usage
 
-### CLI Options
+### CLI Commands
+
+#### Initialize Configuration
+
+Create a new configuration directory with example files:
+
+```bash
+gql-ingest init [path] [options]
+
+Options:
+  --no-example  Skip creating example entity files
+  --no-config   Skip creating config.yaml
+  -f, --force   Overwrite existing files
+  -q, --quiet   Suppress output
+```
+
+This creates:
+
+- `data/` - Data files directory
+- `graphql/` - GraphQL mutation files
+- `mappings/` - Mapping configuration files
+- `config.yaml` - Processing configuration
+- Example entity files (by default)
+
+#### Add Entity
+
+Add a new entity to an existing configuration:
+
+```bash
+gql-ingest add <entity-name> [options]
+
+Options:
+  -p, --path <path>      Config directory path (default: current directory)
+  -f, --format <format>  Data format (csv, json, yaml, jsonl)
+  --fields <fields>      Comma-separated field names
+  --mutation <name>      GraphQL mutation name
+  --no-interactive       Skip prompts, use defaults only
+  -q, --quiet            Suppress output
+```
+
+Interactive mode prompts for format, fields, and mutation name. Use `--no-interactive` with flags for CI/CD.
+
+#### Run Ingestion
+
+Ingest data from configuration into GraphQL API:
 
 ```bash
 gql-ingest [options]
 
 Options:
-  -V, --version            output the version number
   -e, --endpoint <url>     GraphQL endpoint URL (required)
   -c, --config <path>      Path to configuration directory (required)
-  -n, --entities <list>    Comma-separated list of specific entities to process
-  -h, --headers <headers>  JSON string of headers to include in requests
-  -f, --format <format>    Override data format detection (csv, json, yaml, jsonl)
-  -v, --verbose            Show detailed request results and responses
-  --help                   display help for command
+  -n, --entities <list>    Comma-separated list of entities to process
+  -h, --headers <headers>  JSON string of headers
+  -f, --format <format>    Override data format detection
+  -q, --quiet              Suppress output
 ```
 
-### Examples
+### CLI Examples
 
 ```bash
 # Basic usage
-npx @jackchuka/gql-ingest \
-  --endpoint https://your-graphql-api.com/graphql \
-  --config ./examples/demo
+gql-ingest \
+  -e https://your-graphql-api.com/graphql \
+  -c ./examples/demo
 
 # With authentication headers
-npx @jackchuka/gql-ingest \
-  --endpoint https://your-graphql-api.com/graphql \
-  --config ./examples/demo \
-  --headers '{"Authorization": "Bearer YOUR_TOKEN"}'
-
-# With custom headers
-npx @jackchuka/gql-ingest \
-  --endpoint https://api.example.com/graphql \
-  --config ./my-config \
-  --headers '{"X-API-Key": "your-api-key", "Content-Type": "application/json"}'
+gql-ingest \
+  -e https://your-graphql-api.com/graphql \
+  -c ./examples/demo \
+  -h '{"Authorization": "Bearer YOUR_TOKEN"}'
 
 # Process specific entities only
-npx @jackchuka/gql-ingest \
-  --endpoint https://your-graphql-api.com/graphql \
-  --config ./examples/demo \
-  --entities users,products
-
-# Process a single entity
-npx @jackchuka/gql-ingest \
-  --endpoint https://your-graphql-api.com/graphql \
-  --config ./examples/demo \
-  --entities items
+gql-ingest \
+  -e https://your-graphql-api.com/graphql \
+  -c ./examples/demo \
+  -n users,products
+```
+
+### Programmatic API
+
+GQL Ingest provides a full programmatic API for integration into your Node.js applications.
+
+#### Installation for API Usage
+
+```bash
+npm install @jackchuka/gql-ingest
+```
+
+#### Basic API Usage
+
+```typescript
+import { GQLIngest, createConsoleLogger } from "@jackchuka/gql-ingest";
+
+// Initialize the client
+const client = new GQLIngest({
+  endpoint: "https://your-graphql-api.com/graphql",
+  headers: {
+    Authorization: "Bearer YOUR_TOKEN",
+  },
+  logger: createConsoleLogger({ prefix: "my-app" }), // Optional: enable logging with prefix
+});
+
+// Ingest all data from a configuration
+const result = await client.ingest("./config");
+
+// Check if ingestion was successful
+if (result.success) {
+  console.log("Ingestion completed successfully");
+  console.log("Metrics:", result.metrics);
+} else {
+  console.error("Ingestion failed:", result.errors);
+}
+```
+
+#### Processing Specific Entities
+
+```typescript
+// Process only specific entities
+const result = await client.ingestEntities("./config", ["users", "products"]);
+
+// Or using the ingest method with options
+const result = await client.ingest("./config", {
+  entities: ["users", "products"],
+  format: "csv", // Optional: override format detection
+});
+```
+
+#### Advanced API Usage
+
+For more control, you can access the underlying components directly:
+
+```typescript
+import {
+  GraphQLClientWrapper,
+  DataMapper,
+  DependencyResolver,
+  MetricsCollector,
+  loadConfig,
+  createConsoleLogger,
+} from "@jackchuka/gql-ingest";
+
+// Create your own custom workflow
+const logger = createConsoleLogger();
+const metrics = new MetricsCollector();
+const client = new GraphQLClientWrapper(endpoint, headers, metrics, logger);
+const mapper = new DataMapper(client, basePath, metrics, logger);
+
+// Load configuration
+const config = loadConfig("./config");
+
+// Process entities with custom logic
+// ... your custom implementation
+```
+
+#### API Methods
+
+**GQLIngest Class Methods:**
+
+- `constructor(options: GQLIngestOptions)` - Initialize the client
+- `ingest(configPath: string, options?: IngestOptions)` - Ingest data from a configuration
+- `ingestEntities(configPath: string, entities: string[])` - Process specific entities
+- `getMetrics()` - Get current processing metrics
+- `getMetricsSummary()` - Get formatted metrics summary
+- `setLogger(logger: Logger)` - Set custom logger
+- `setHeaders(headers: Record<string, string>)` - Update request headers
+- `cancel(reason?: string)` - Cancel in-progress ingestion
+- `processing` - Property indicating if ingestion is in progress
+
+#### Event-Based Progress Monitoring
+
+GQLIngest extends EventEmitter, enabling real-time progress tracking and cancellation:
+
+```typescript
+import { GQLIngest } from "@jackchuka/gql-ingest";
+
+const client = new GQLIngest({
+  endpoint: "https://your-api.com/graphql",
+  eventOptions: {
+    emitRowEvents: true, // Emit events for each row
+    emitProgressEvents: true, // Emit periodic progress
+    progressInterval: 1000, // Progress every 1 second
+  },
+});
+
+// Listen for events
+client.on("started", (p) => console.log(`Starting ${p.totalEntities} entities`));
+client.on("progress", (p) => console.log(`${p.progressPercent.toFixed(1)}% complete`));
+client.on("entityStart", (p) => console.log(`Processing ${p.entityName}`));
+client.on("entityComplete", (p) =>
+  console.log(`${p.entityName}: ${p.metrics.successfulRows} rows`),
+);
+client.on("rowSuccess", (p) => console.log(`Row ${p.rowIndex} OK`));
+client.on("rowFailure", (p) => console.error(`Row ${p.rowIndex} failed: ${p.error.message}`));
+client.on("finished", (p) => console.log(`Done in ${p.durationMs}ms`));
+client.on("errored", (p) => console.error(`Error: ${p.error.message}`));
+client.on("cancelled", (p) => console.log(`Cancelled: ${p.reason}`));
+
+// Handle graceful shutdown
+process.on("SIGINT", () => client.cancel("User interrupted"));
+
+await client.ingest("./config");
+```
+
+**Available Events:**
+
+| Event            | When Emitted             | Key Payload Fields                                |
+| ---------------- | ------------------------ | ------------------------------------------------- |
+| `started`        | Ingestion begins         | `configPath`, `entityNames`, `totalWaves`         |
+| `progress`       | Periodic interval        | `progressPercent`, `successfulRows`, `failedRows` |
+| `entityStart`    | Entity processing begins | `entityName`, `totalRows`, `waveIndex`            |
+| `entityComplete` | Entity processing ends   | `entityName`, `metrics`, `success`                |
+| `rowSuccess`     | Row mutation succeeds    | `entityName`, `rowIndex`, `row`, `result`         |
+| `rowFailure`     | Row mutation fails       | `entityName`, `rowIndex`, `error`                 |
+| `cancelled`      | Processing cancelled     | `reason`, `metrics`, `elapsedMs`                  |
+| `finished`       | Processing completes     | `metrics`, `durationMs`, `allSuccessful`          |
+| `errored`        | Fatal error occurs       | `error`, `metrics`, `elapsedMs`                   |
+
+#### Cancellation Support
+
+Cancel in-progress ingestion using the `cancel()` method or external AbortController:
+
+```typescript
+// Method 1: Using cancel()
+const client = new GQLIngest({ endpoint: "..." });
+process.on("SIGINT", () => client.cancel("User interrupted"));
+await client.ingest("./config");
+
+// Method 2: Using external AbortController
+const controller = new AbortController();
+setTimeout(() => controller.abort("Timeout"), 60000);
+await client.ingest("./config", { signal: controller.signal });
+```
+
+#### TypeScript Support
+
+Full TypeScript support is included with comprehensive type definitions:
+
+```typescript
+import type {
+  GQLIngestOptions,
+  IngestOptions,
+  IngestResult,
+  ProcessingMetrics,
+  EntityMetrics,
+  // Event types
+  EventOptions,
+  StartedEventPayload,
+  ProgressEventPayload,
+  EntityStartEventPayload,
+  EntityCompleteEventPayload,
+  RowSuccessEventPayload,
+  RowFailureEventPayload,
+  CancelledEventPayload,
+  FinishedEventPayload,
+  ErroredEventPayload,
+} from "@jackchuka/gql-ingest";
 ```
 
 ## Parallel Processing 🚀

package/dist/cli/commands/add.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerAddCommand(program: Command): void;
+//# sourceMappingURL=add.d.ts.map

package/dist/cli/commands/add.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"add.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/add.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAepC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkGzD"}

package/dist/cli/commands/init.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerInitCommand(program: Command): void;
+//# sourceMappingURL=init.d.ts.map

package/dist/cli/commands/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAuC1D"}

package/dist/cli/index.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":""}