@jackchuka/gql-ingest 1.4.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +162 -6
  3. package/bin/cli.js +55 -52
  4. package/dist/dependency-resolver.d.ts +2 -1
  5. package/dist/dependency-resolver.d.ts.map +1 -1
  6. package/dist/mapper.d.ts +10 -5
  7. package/dist/mapper.d.ts.map +1 -1
  8. package/dist/metrics.d.ts.map +1 -1
  9. package/dist/readers/csv.d.ts +10 -0
  10. package/dist/readers/csv.d.ts.map +1 -0
  11. package/dist/readers/csv.test.d.ts +2 -0
  12. package/dist/readers/csv.test.d.ts.map +1 -0
  13. package/dist/readers/data-reader.d.ts +21 -0
  14. package/dist/readers/data-reader.d.ts.map +1 -0
  15. package/dist/readers/data-reader.test.d.ts +2 -0
  16. package/dist/readers/data-reader.test.d.ts.map +1 -0
  17. package/dist/readers/index.d.ts +6 -0
  18. package/dist/readers/index.d.ts.map +1 -0
  19. package/dist/readers/json.d.ts +6 -0
  20. package/dist/readers/json.d.ts.map +1 -0
  21. package/dist/readers/json.test.d.ts +2 -0
  22. package/dist/readers/json.test.d.ts.map +1 -0
  23. package/dist/readers/jsonl.d.ts +6 -0
  24. package/dist/readers/jsonl.d.ts.map +1 -0
  25. package/dist/readers/jsonl.test.d.ts +2 -0
  26. package/dist/readers/jsonl.test.d.ts.map +1 -0
  27. package/dist/readers/yaml.d.ts +6 -0
  28. package/dist/readers/yaml.d.ts.map +1 -0
  29. package/dist/readers/yaml.test.d.ts +2 -0
  30. package/dist/readers/yaml.test.d.ts.map +1 -0
  31. package/package.json +1 -1
  32. package/src/cli.ts +49 -8
  33. package/src/dependency-resolver.test.ts +15 -1
  34. package/src/dependency-resolver.ts +6 -2
  35. package/src/graphql-client.test.ts +19 -4
  36. package/src/mapper.test.ts +115 -64
  37. package/src/mapper.ts +176 -32
  38. package/src/metrics.ts +18 -10
  39. package/src/{csv-reader.test.ts → readers/csv.test.ts} +1 -1
  40. package/src/readers/csv.ts +29 -0
  41. package/src/readers/data-reader.test.ts +104 -0
  42. package/src/readers/data-reader.ts +61 -0
  43. package/src/readers/index.ts +18 -0
  44. package/src/readers/json.test.ts +80 -0
  45. package/src/readers/json.ts +27 -0
  46. package/src/readers/jsonl.test.ts +96 -0
  47. package/src/readers/jsonl.ts +28 -0
  48. package/src/readers/yaml.test.ts +95 -0
  49. package/src/readers/yaml.ts +28 -0
  50. package/dist/csv-reader.d.ts +0 -5
  51. package/dist/csv-reader.d.ts.map +0 -1
  52. package/dist/csv-reader.test.d.ts +0 -2
  53. package/dist/csv-reader.test.d.ts.map +0 -1
  54. package/src/csv-reader.ts +0 -18
@@ -8,7 +8,8 @@ export interface ExecutionWave {
8
8
  export declare class DependencyResolver {
9
9
  private dependencies;
10
10
  private entities;
11
- constructor(entities: string[], dependencies?: DependencyGraph);
11
+ private allowPartialResolution;
12
+ constructor(entities: string[], dependencies?: DependencyGraph, allowPartialResolution?: boolean);
12
13
  resolveExecutionOrder(): ExecutionWave[];
13
14
  validateDependencies(): string[];
14
15
  getDependents(entityName: string): string[];
@@ -1 +1 @@
1
- {"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;gBAEf,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,GAAE,eAAoB;IAKlE,qBAAqB,IAAI,aAAa,EAAE;IA8CxC,oBAAoB,IAAI,MAAM,EAAE;IAwBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
1
+ {"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAE5B,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,GAAE,eAAoB,EAAE,sBAAsB,GAAE,OAAe;IAM3G,qBAAqB,IAAI,aAAa,EAAE;IAgDxC,oBAAoB,IAAI,MAAM,EAAE;IAwBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
package/dist/mapper.d.ts CHANGED
@@ -2,22 +2,27 @@ import { GraphQLClientWrapper } from "./graphql-client";
2
2
  import { MetricsCollector } from "./metrics";
3
3
  import { ParallelProcessingConfig, RetryConfig } from "./config";
4
4
  export interface MappingConfig {
5
- csvFile: string;
5
+ csvFile?: string;
6
+ dataFile?: string;
7
+ dataFormat?: string;
6
8
  graphqlFile: string;
7
- mapping: Record<string, string>;
9
+ mapping: Record<string, string | any>;
8
10
  }
9
11
  export declare class DataMapper {
10
12
  private client;
11
13
  private basePath;
12
14
  private metrics;
13
15
  private verbose;
14
- constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean);
15
- discoverMappings(configDir: string): string[];
16
+ private formatOverride?;
17
+ constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean, formatOverride?: string);
18
+ discoverMappings(configDir: string, entityFilter?: string[]): string[];
16
19
  processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
17
20
  private processRowsSequentially;
18
21
  private processRowsConcurrently;
19
22
  private chunkArray;
20
- private mapCsvRowToVariables;
23
+ private mapRowToVariables;
24
+ private getValueByPath;
25
+ private mapNestedObject;
21
26
  private extractVariableTypes;
22
27
  private extractTypeName;
23
28
  private convertValue;
@@ -1 +1 @@
1
- {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAsDpB,UAAU,IAAI,gBAAgB;CAG/B"}
1
+ {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAE5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC;CACvC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe,EACxB,cAAc,CAAC,EAAE,MAAM;IASzB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IA4ChE,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YAyDF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IA+CzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IA0CvB,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAmEpB,UAAU,IAAI,gBAAgB;CAG/B"}
@@ -1 +1 @@
1
- {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAmC1B"}
1
+ {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CA2C1B"}
@@ -0,0 +1,10 @@
1
+ import { DataReader, DataRow } from "./data-reader";
2
+ export interface CsvRow {
3
+ [key: string]: string;
4
+ }
5
+ export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
6
+ export declare class CsvReader extends DataReader {
7
+ getSupportedExtensions(): string[];
8
+ readFile(filePath: string): Promise<DataRow[]>;
9
+ }
10
+ //# sourceMappingURL=csv.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/readers/csv.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUrE;AAED,qBAAa,SAAU,SAAQ,UAAU;IACvC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAGrD"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=csv.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv.test.d.ts","sourceRoot":"","sources":["../../src/readers/csv.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,21 @@
1
+ export interface DataRow {
2
+ [key: string]: any;
3
+ }
4
+ export declare abstract class DataReader {
5
+ abstract readFile(filePath: string): Promise<DataRow[]>;
6
+ /**
7
+ * Get the supported file extensions for this reader
8
+ */
9
+ abstract getSupportedExtensions(): string[];
10
+ /**
11
+ * Check if this reader can handle the given file
12
+ */
13
+ canHandle(filePath: string): boolean;
14
+ }
15
+ export declare class DataReaderFactory {
16
+ private static readers;
17
+ static registerReader(reader: DataReader): void;
18
+ static getReader(filePath: string, format?: string): DataReader;
19
+ static getSupportedFormats(): string[];
20
+ }
21
+ //# sourceMappingURL=data-reader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"data-reader.d.ts","sourceRoot":"","sources":["../../src/readers/data-reader.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,8BAAsB,UAAU;IAC9B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvD;;OAEG;IACH,QAAQ,CAAC,sBAAsB,IAAI,MAAM,EAAE;IAE3C;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;CAMrC;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAoB;IAE1C,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI/C,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU;IAuB/D,MAAM,CAAC,mBAAmB,IAAI,MAAM,EAAE;CAOvC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=data-reader.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"data-reader.test.d.ts","sourceRoot":"","sources":["../../src/readers/data-reader.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,6 @@
1
+ export { DataReader, DataRow, DataReaderFactory } from "./data-reader";
2
+ export { CsvReader, readCsvFile, CsvRow } from "./csv";
3
+ export { JsonReader } from "./json";
4
+ export { YamlReader } from "./yaml";
5
+ export { JsonlReader } from "./jsonl";
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/readers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC"}
@@ -0,0 +1,6 @@
1
+ import { DataReader, DataRow } from "./data-reader";
2
+ export declare class JsonReader extends DataReader {
3
+ getSupportedExtensions(): string[];
4
+ readFile(filePath: string): Promise<DataRow[]>;
5
+ }
6
+ //# sourceMappingURL=json.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/readers/json.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,UAAW,SAAQ,UAAU;IACxC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAkBrD"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=json.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json.test.d.ts","sourceRoot":"","sources":["../../src/readers/json.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,6 @@
1
+ import { DataReader, DataRow } from "./data-reader";
2
+ export declare class JsonlReader extends DataReader {
3
+ getSupportedExtensions(): string[];
4
+ readFile(filePath: string): Promise<DataRow[]>;
5
+ }
6
+ //# sourceMappingURL=jsonl.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,WAAY,SAAQ,UAAU;IACzC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAmBrD"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=jsonl.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jsonl.test.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,6 @@
1
+ import { DataReader, DataRow } from "./data-reader";
2
+ export declare class YamlReader extends DataReader {
3
+ getSupportedExtensions(): string[];
4
+ readFile(filePath: string): Promise<DataRow[]>;
5
+ }
6
+ //# sourceMappingURL=yaml.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"yaml.d.ts","sourceRoot":"","sources":["../../src/readers/yaml.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,UAAW,SAAQ,UAAU;IACxC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAkBrD"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=yaml.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"yaml.test.d.ts","sourceRoot":"","sources":["../../src/readers/yaml.test.ts"],"names":[],"mappings":""}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jackchuka/gql-ingest",
3
- "version": "1.4.0",
3
+ "version": "2.0.0",
4
4
  "description": "A CLI tool for ingesting data from CSV files into a GraphQL API",
5
5
  "type": "module",
6
6
  "main": "dist/cli.js",
package/src/cli.ts CHANGED
@@ -31,11 +31,19 @@ program
31
31
  "-c, --config <path>",
32
32
  "Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)"
33
33
  )
34
+ .option(
35
+ "-n, --entities <entities>",
36
+ "Comma-separated list of specific entities to process (e.g., users,products)"
37
+ )
34
38
  .option(
35
39
  "-h, --headers <headers>",
36
40
  "JSON string of headers to include in requests"
37
41
  )
38
42
  .option("-v, --verbose", "Show detailed request results and responses")
43
+ .option(
44
+ "-f, --format <format>",
45
+ "Override data format detection (csv, json, yaml, jsonl)"
46
+ )
39
47
  .action(async (options) => {
40
48
  try {
41
49
  console.log("Starting seed data generation...");
@@ -62,32 +70,65 @@ program
62
70
  client,
63
71
  process.cwd(),
64
72
  metrics,
65
- options.verbose
73
+ options.verbose,
74
+ options.format
66
75
  );
67
76
 
77
+ // Parse entities filter if provided
78
+ const entityFilter = options.entities
79
+ ? options.entities.split(",").map((e: string) => e.trim())
80
+ : undefined;
81
+
68
82
  // Discover all mapping files dynamically
69
- const mappingPaths = mapper.discoverMappings(options.config);
83
+ const mappingPaths = mapper.discoverMappings(
84
+ options.config,
85
+ entityFilter
86
+ );
70
87
 
71
88
  if (mappingPaths.length === 0) {
72
- console.warn(`No mapping files found in ${options.config}/mappings`);
89
+ const filterMsg = entityFilter
90
+ ? ` matching entities: ${entityFilter.join(", ")}`
91
+ : "";
92
+ console.warn(
93
+ `No mapping files found in ${options.config}/mappings${filterMsg}`
94
+ );
73
95
  return;
74
96
  }
75
97
 
76
98
  // Extract entity names from mapping paths
77
99
  const entityNames = mappingPaths.map((path) => basename(path, ".json"));
78
100
 
79
- // Setup dependency resolver
101
+ // Filter dependencies to only include those relevant to selected entities
102
+ const relevantDependencies: Record<string, string[]> = {};
103
+ if (config.entityDependencies) {
104
+ for (const entity of entityNames) {
105
+ if (config.entityDependencies[entity]) {
106
+ relevantDependencies[entity] = config.entityDependencies[entity];
107
+ }
108
+ }
109
+ }
110
+
111
+ // Setup dependency resolver with filtered dependencies
80
112
  const resolver = new DependencyResolver(
81
113
  entityNames,
82
- config.entityDependencies
114
+ relevantDependencies,
115
+ !!entityFilter // Allow partial resolution when using --entities
83
116
  );
84
117
 
85
118
  // Validate dependencies
86
119
  const validationErrors = resolver.validateDependencies();
87
120
  if (validationErrors.length > 0) {
88
- console.error("Dependency validation errors:");
89
- validationErrors.forEach((error) => console.error(` - ${error}`));
90
- process.exit(1);
121
+ if (entityFilter) {
122
+ // When using --entities flag, show warnings instead of errors
123
+ console.warn("\n⚠️ Warning: Dependency validation issues:");
124
+ validationErrors.forEach((error) => console.warn(` - ${error}`));
125
+ console.warn("This may cause errors if the dependent data doesn't already exist.\n");
126
+ } else {
127
+ // Strict validation when processing all entities
128
+ console.error("Dependency validation errors:");
129
+ validationErrors.forEach((error) => console.error(` - ${error}`));
130
+ process.exit(1);
131
+ }
91
132
  }
92
133
 
93
134
  // Process entities in dependency-aware waves
@@ -88,12 +88,26 @@ describe("DependencyResolver", () => {
88
88
  a: ["missing"],
89
89
  b: ["a"],
90
90
  };
91
- const resolver = new DependencyResolver(entities, dependencies);
91
+ const resolver = new DependencyResolver(entities, dependencies, false);
92
92
 
93
93
  expect(() => resolver.resolveExecutionOrder()).toThrow(
94
94
  "Circular dependency detected or missing dependencies for entities: a, b"
95
95
  );
96
96
  });
97
+
98
+ it("should allow entities with dependencies not in the entity list when partial resolution is enabled", () => {
99
+ const entities = ["a", "b"];
100
+ const dependencies = {
101
+ a: ["missing"],
102
+ b: ["a"],
103
+ };
104
+ const resolver = new DependencyResolver(entities, dependencies, true);
105
+
106
+ const waves = resolver.resolveExecutionOrder();
107
+ expect(waves).toHaveLength(2);
108
+ expect(waves[0].entities).toEqual(["a"]);
109
+ expect(waves[1].entities).toEqual(["b"]);
110
+ });
97
111
  });
98
112
 
99
113
  describe("validateDependencies", () => {
@@ -10,10 +10,12 @@ export interface ExecutionWave {
10
10
  export class DependencyResolver {
11
11
  private dependencies: DependencyGraph;
12
12
  private entities: string[];
13
+ private allowPartialResolution: boolean;
13
14
 
14
- constructor(entities: string[], dependencies: DependencyGraph = {}) {
15
+ constructor(entities: string[], dependencies: DependencyGraph = {}, allowPartialResolution: boolean = false) {
15
16
  this.entities = entities;
16
17
  this.dependencies = dependencies;
18
+ this.allowPartialResolution = allowPartialResolution;
17
19
  }
18
20
 
19
21
  resolveExecutionOrder(): ExecutionWave[] {
@@ -31,7 +33,9 @@ export class DependencyResolver {
31
33
  }
32
34
 
33
35
  const deps = this.dependencies[entity] || [];
34
- const canProcess = deps.every((dep) => processed.has(dep));
36
+ const canProcess = deps.every((dep) =>
37
+ processed.has(dep) || (this.allowPartialResolution && !this.entities.includes(dep))
38
+ );
35
39
 
36
40
  if (canProcess) {
37
41
  currentWave.push(entity);
@@ -63,7 +63,10 @@ describe("GraphQLClientWrapper", () => {
63
63
  clientWrapper.executeMutation(mutation, variables)
64
64
  ).rejects.toThrow("GraphQL error");
65
65
 
66
- expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed after 3 attempts:", error);
66
+ expect(consoleSpy).toHaveBeenCalledWith(
67
+ "GraphQL mutation failed after 3 attempts:",
68
+ error
69
+ );
67
70
 
68
71
  consoleSpy.mockRestore();
69
72
  });
@@ -96,7 +99,11 @@ describe("GraphQLClientWrapper", () => {
96
99
  .mockRejectedValueOnce(serverError)
97
100
  .mockResolvedValueOnce({ data: { result: "success" } });
98
101
 
99
- const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
102
+ const result = await clientWrapper.executeMutation(
103
+ mutation,
104
+ variables,
105
+ retryConfig
106
+ );
100
107
 
101
108
  expect(mockRequest).toHaveBeenCalledTimes(3);
102
109
  expect(result).toEqual({ data: { result: "success" } });
@@ -143,7 +150,11 @@ describe("GraphQLClientWrapper", () => {
143
150
  .mockRejectedValueOnce(networkError)
144
151
  .mockResolvedValueOnce({ data: { result: "success" } });
145
152
 
146
- const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
153
+ const result = await clientWrapper.executeMutation(
154
+ mutation,
155
+ variables,
156
+ retryConfig
157
+ );
147
158
 
148
159
  expect(mockRequest).toHaveBeenCalledTimes(2);
149
160
  expect(result).toEqual({ data: { result: "success" } });
@@ -192,7 +203,11 @@ describe("GraphQLClientWrapper", () => {
192
203
  .mockResolvedValueOnce({ data: { result: "success" } });
193
204
 
194
205
  const startTime = Date.now();
195
- const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
206
+ const result = await clientWrapper.executeMutation(
207
+ mutation,
208
+ variables,
209
+ retryConfig
210
+ );
196
211
  const totalTime = Date.now() - startTime;
197
212
 
198
213
  expect(mockRequest).toHaveBeenCalledTimes(3);