@jackchuka/gql-ingest 3.1.9 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ import { type Config, type ParallelProcessingConfig, type RetryConfig } from "./
3
3
  export type { Config, ParallelProcessingConfig, RetryConfig, EntityConfig } from "./config-schema";
4
4
  export { DEFAULT_RETRY_CONFIG, DEFAULT_PARALLEL_CONFIG } from "./config-schema";
5
5
  export declare const DEFAULT_CONFIG: Config;
6
- export declare function loadConfig(configDir: string, logger?: Logger): Config;
6
+ export declare function loadConfig(configFile?: string, logger?: Logger): Config;
7
7
  export declare function getEntityConfig(entityName: string, globalConfig: Config, logger?: Logger): ParallelProcessingConfig;
8
8
  export declare function getRetryConfig(entityName: string, globalConfig: Config): RetryConfig;
9
9
  //# sourceMappingURL=config.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EAAE,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEnG,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,eAAO,MAAM,cAAc,EAAE,MAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,GAAE,MAAmB,GAAG,MAAM,CAuBjF;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAmB,GAC1B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,WAAW,CAOpF"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EAAE,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEnG,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,eAAO,MAAM,cAAc,EAAE,MAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,MAAM,GAAE,MAAmB,GAAG,MAAM,CAyBnF;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAmB,GAC1B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,WAAW,CAOpF"}
@@ -8,8 +8,6 @@ export type GQLIngestEventType = "started" | "progress" | "entityStart" | "entit
8
8
  * Payload for 'started' event - emitted when ingestion begins
9
9
  */
10
10
  export interface StartedEventPayload {
11
- /** Path to configuration directory */
12
- configPath: string;
13
11
  /** Total number of entities to process */
14
12
  totalEntities: number;
15
13
  /** Names of entities that will be processed */
@@ -1 +1 @@
1
- {"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/lib/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,UAAU,GACV,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,EAAE,aAAa,CAAC;IACvB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,CAAC,mBAAmB,CAAC,CAAC;IAC/B,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IACjC,WAAW,EAAE,CAAC,uBAAuB,CAAC,CAAC;IACvC,cAAc,EAAE,CAAC,0BAA0B,CAAC,CAAC;IAC7C,UAAU,EAAE,CAAC,sBAAsB,CAAC,CAAC;IACrC,UAAU,EAAE,CAAC,sBAAsB,CAAC,CAAC;IACrC,SAAS,EAAE,CAAC,qBAAqB,CAAC,CAAC;IACnC,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IACjC,OAAO,EAAE,CAAC,mBAAmB,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,YAAY,CAIxD,CAAC"}
1
+ {"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/lib/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,UAAU,GACV,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,EAAE,aAAa,CAAC;IACvB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,CAAC,mBAAmB,CAAC,CAAC;IAC/B,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IACjC,WAAW,EAAE,CAAC,uBAAuB,CAAC,CAAC;IACvC,cAAc,EAAE,CAAC,0BAA0B,CAAC,CAAC;IAC7C,UAAU,EAAE,CAAC,sBAAsB,CAAC,CAAC;IACrC,UAAU,EAAE,CAAC,sBAAsB,CAAC,CAAC;IACrC,SAAS,EAAE,CAAC,qBAAqB,CAAC,CAAC;IACnC,QAAQ,EAAE,CAAC,oBAAoB,CAAC,CAAC;IACjC,OAAO,EAAE,CAAC,mBAAmB,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,YAAY,CAIxD,CAAC"}
@@ -23,10 +23,12 @@ export interface GQLIngestOptions {
23
23
  * Options for ingesting data
24
24
  */
25
25
  export interface IngestOptions {
26
- /** Comma-separated list or array of specific entities to process */
27
- entities?: string[] | string;
26
+ /** Path to config.yaml for retry/parallelism/dependency settings */
27
+ config?: string;
28
28
  /** Override data format detection for this operation */
29
29
  format?: string;
30
+ /** Filter to only process entities with these names */
31
+ entities?: string[];
30
32
  /** AbortSignal for external cancellation */
31
33
  signal?: AbortSignal;
32
34
  }
@@ -56,6 +58,7 @@ export declare class GQLIngest extends EventEmitter<GQLIngestEventMap> {
56
58
  private client;
57
59
  private mapper;
58
60
  private eventOptions;
61
+ private outputStore;
59
62
  private abortController;
60
63
  private isProcessing;
61
64
  private startTime;
@@ -100,19 +103,12 @@ export declare class GQLIngest extends EventEmitter<GQLIngestEventMap> {
100
103
  */
101
104
  private combineSignals;
102
105
  /**
103
- * Ingest data from a configuration directory
104
- * @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
106
+ * Ingest entity files
107
+ * @param entityFiles Array of paths to entity JSON mapping files
105
108
  * @param options Optional ingestion options
106
109
  * @returns Promise with ingestion result
107
110
  */
108
- ingest(configPath: string, options?: IngestOptions): Promise<IngestResult>;
109
- /**
110
- * Ingest specific entities from a configuration directory
111
- * @param configPath Path to configuration directory
112
- * @param entities Array of entity names to process
113
- * @returns Promise with ingestion result
114
- */
115
- ingestEntities(configPath: string, entities: string[]): Promise<IngestResult>;
111
+ ingest(entityFiles: string[], options?: IngestOptions): Promise<IngestResult>;
116
112
  /**
117
113
  * Get current processing metrics
118
114
  * @returns Current metrics
@@ -1 +1 @@
1
- {"version":3,"file":"gql-ingest.d.ts","sourceRoot":"","sources":["../../src/lib/gql-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAoB,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAGhE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,OAAO,EACL,iBAAiB,EACjB,YAAY,EAOb,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC7B,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,YAAY,CAAC,iBAAiB,CAAC;IAC5D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAyB;IAG7C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,kBAAkB,CAA+C;IAGzE,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,aAAa,CAAqB;gBAE9B,OAAO,EAAE,gBAAgB;IAoBrC;;;OAGG;IACH,MAAM,CAAC,MAAM,SAAgC,GAAG,IAAI;IAMpD;;OAEG;IACH,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;OAEG;IACH,OAAO,CAAC,QAAQ;IAahB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAgMhF;;;;;OAKG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAInF;;;OAGG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;OAGG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;;OAGG;IACH,SAAS,IAAI,UAAU;IAIvB;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY/B;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAYjD;;OAEG;YACW,sBAAsB;IA2EpC;;OAEG;IACH,OAAO,CAAC,UAAU;CAQnB"}
1
+ {"version":3,"file":"gql-ingest.d.ts","sourceRoot":"","sources":["../../src/lib/gql-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAe,MAAM,UAAU,CAAC;AACnD,OAAO,EAAoB,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAGhE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAI9C,OAAO,EACL,iBAAiB,EACjB,YAAY,EAOb,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,YAAY,CAAC,iBAAiB,CAAC;IAC5D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAyB;IAE7C,OAAO,CAAC,WAAW,CAA0B;IAG7C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,kBAAkB,CAA+C;IAGzE,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,aAAa,CAAqB;gBAE9B,OAAO,EAAE,gBAAgB;IAoBrC;;;OAGG;IACH,MAAM,CAAC,MAAM,SAAgC,GAAG,IAAI;IAMpD;;OAEG;IACH,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;OAEG;IACH,OAAO,CAAC,QAAQ;IAahB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;OAKG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAqLnF;;;OAGG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;OAGG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;;OAGG;IACH,SAAS,IAAI,UAAU;IAIvB;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY/B;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAYjD;;OAEG;YACW,sBAAsB;IAkFpC;;OAEG;IACH,OAAO,CAAC,UAAU;CAQnB"}
@@ -3,12 +3,20 @@ import { MetricsCollector } from "./metrics";
3
3
  import { ParallelProcessingConfig, RetryConfig } from "./config";
4
4
  import { Logger } from "./logger";
5
5
  import { EntityStartEventPayload, EntityCompleteEventPayload, RowSuccessEventPayload, RowFailureEventPayload } from "./events";
6
+ export interface OutputCaptureConfig {
7
+ /** JSONPath into the input row for the lookup key (e.g., "$.legalName") */
8
+ key: string;
9
+ /** Map of field names to JSONPaths into the mutation response */
10
+ fields: Record<string, string>;
11
+ }
12
+ export type OutputStore = Map<string, Map<string, Record<string, unknown>>>;
6
13
  export interface MappingConfig {
7
- csvFile?: string;
8
- dataFile?: string;
14
+ name: string;
15
+ dataFile: string;
9
16
  dataFormat?: string;
10
17
  graphqlFile: string;
11
18
  mapping: Record<string, unknown>;
19
+ outputCapture?: OutputCaptureConfig;
12
20
  }
13
21
  /**
14
22
  * Callbacks for entity processing events
@@ -26,7 +34,6 @@ export declare class DataMapper {
26
34
  private logger;
27
35
  private formatOverride?;
28
36
  constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, logger?: Logger, formatOverride?: string);
29
- discoverMappings(configDir: string, entityFilter?: string[]): string[];
30
37
  /**
31
38
  * Process an entity (backward-compatible method)
32
39
  */
@@ -34,13 +41,17 @@ export declare class DataMapper {
34
41
  /**
35
42
  * Process an entity with event callbacks and abort support
36
43
  */
37
- processEntityWithEvents(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig, signal?: AbortSignal, callbacks?: EntityProcessingCallbacks): Promise<void>;
44
+ processEntityWithEvents(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig, signal?: AbortSignal, callbacks?: EntityProcessingCallbacks, outputStore?: OutputStore): Promise<void>;
38
45
  private processRowsSequentiallyWithEvents;
39
46
  private processRowsConcurrentlyWithEvents;
40
47
  private chunkArray;
41
48
  private mapRowToVariables;
42
49
  private getValueByPath;
43
50
  private mapNestedObject;
51
+ private stripJsonPathPrefix;
52
+ private isRefMapping;
53
+ private resolveRef;
54
+ private captureOutput;
44
55
  private extractVariableTypes;
45
56
  private extractTypeName;
46
57
  private convertValue;
@@ -1 +1 @@
1
- {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../../src/lib/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,UAAU,CAAC;AAElB,MAAM,WAAW,aAAa;IAE5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE,WAAW,CAAC,KAAK,IAAI,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACzD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;CAC1D;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB,EAC3B,cAAc,CAAC,EAAE,MAAM;IASzB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAyCtE;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;IAIhB;;OAEG;IACG,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,EACpB,SAAS,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,IAAI,CAAC;YAyFF,iCAAiC;YA+DjC,iCAAiC;IA6G/C,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IAqCzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IAoCvB,OAAO,CAAC,oBAAoB;IA0B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAyDpB,UAAU,IAAI,gBAAgB;CAG/B"}
1
+ {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../../src/lib/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,UAAU,CAAC;AAElB,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,iEAAiE;IACjE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAS5E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE,WAAW,CAAC,KAAK,IAAI,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACzD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;CAC1D;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB,EAC3B,cAAc,CAAC,EAAE,MAAM;IASzB;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;IAIhB;;OAEG;IACG,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,EACpB,SAAS,CAAC,EAAE,yBAAyB,EACrC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YAqFF,iCAAiC;YAmEjC,iCAAiC;IAiH/C,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IAwCzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IA2CvB,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,UAAU;IAwClB,OAAO,CAAC,aAAa;IAyCrB,OAAO,CAAC,oBAAoB;IA0B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAyDpB,UAAU,IAAI,gBAAgB;CAG/B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jackchuka/gql-ingest",
3
- "version": "3.1.9",
3
+ "version": "4.1.0",
4
4
  "description": "A CLI tool for ingesting data from files into a GraphQL API. Supports CSV, JSON, JSONL, and YAML file formats.",
5
5
  "keywords": [
6
6
  "api",
@@ -37,6 +37,7 @@
37
37
  "graphql": "16.13.2",
38
38
  "graphql-request": "^7.2.0",
39
39
  "js-yaml": "^4.1.0",
40
+ "tinyglobby": "^0.2.15",
40
41
  "zod": "^4.3.5"
41
42
  },
42
43
  "devDependencies": {