@jackchuka/gql-ingest 1.4.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +162 -6
- package/bin/cli.js +55 -52
- package/dist/dependency-resolver.d.ts +2 -1
- package/dist/dependency-resolver.d.ts.map +1 -1
- package/dist/mapper.d.ts +10 -5
- package/dist/mapper.d.ts.map +1 -1
- package/dist/metrics.d.ts.map +1 -1
- package/dist/readers/csv.d.ts +10 -0
- package/dist/readers/csv.d.ts.map +1 -0
- package/dist/readers/csv.test.d.ts +2 -0
- package/dist/readers/csv.test.d.ts.map +1 -0
- package/dist/readers/data-reader.d.ts +21 -0
- package/dist/readers/data-reader.d.ts.map +1 -0
- package/dist/readers/data-reader.test.d.ts +2 -0
- package/dist/readers/data-reader.test.d.ts.map +1 -0
- package/dist/readers/index.d.ts +6 -0
- package/dist/readers/index.d.ts.map +1 -0
- package/dist/readers/json.d.ts +6 -0
- package/dist/readers/json.d.ts.map +1 -0
- package/dist/readers/json.test.d.ts +2 -0
- package/dist/readers/json.test.d.ts.map +1 -0
- package/dist/readers/jsonl.d.ts +6 -0
- package/dist/readers/jsonl.d.ts.map +1 -0
- package/dist/readers/jsonl.test.d.ts +2 -0
- package/dist/readers/jsonl.test.d.ts.map +1 -0
- package/dist/readers/yaml.d.ts +6 -0
- package/dist/readers/yaml.d.ts.map +1 -0
- package/dist/readers/yaml.test.d.ts +2 -0
- package/dist/readers/yaml.test.d.ts.map +1 -0
- package/package.json +1 -1
- package/src/cli.ts +49 -8
- package/src/dependency-resolver.test.ts +15 -1
- package/src/dependency-resolver.ts +6 -2
- package/src/graphql-client.test.ts +19 -4
- package/src/mapper.test.ts +115 -64
- package/src/mapper.ts +176 -32
- package/src/metrics.ts +18 -10
- package/src/{csv-reader.test.ts → readers/csv.test.ts} +1 -1
- package/src/readers/csv.ts +29 -0
- package/src/readers/data-reader.test.ts +104 -0
- package/src/readers/data-reader.ts +61 -0
- package/src/readers/index.ts +18 -0
- package/src/readers/json.test.ts +80 -0
- package/src/readers/json.ts +27 -0
- package/src/readers/jsonl.test.ts +96 -0
- package/src/readers/jsonl.ts +28 -0
- package/src/readers/yaml.test.ts +95 -0
- package/src/readers/yaml.ts +28 -0
- package/dist/csv-reader.d.ts +0 -5
- package/dist/csv-reader.d.ts.map +0 -1
- package/dist/csv-reader.test.d.ts +0 -2
- package/dist/csv-reader.test.d.ts.map +0 -1
- package/src/csv-reader.ts +0 -18
|
@@ -8,7 +8,8 @@ export interface ExecutionWave {
|
|
|
8
8
|
export declare class DependencyResolver {
|
|
9
9
|
private dependencies;
|
|
10
10
|
private entities;
|
|
11
|
-
|
|
11
|
+
private allowPartialResolution;
|
|
12
|
+
constructor(entities: string[], dependencies?: DependencyGraph, allowPartialResolution?: boolean);
|
|
12
13
|
resolveExecutionOrder(): ExecutionWave[];
|
|
13
14
|
validateDependencies(): string[];
|
|
14
15
|
getDependents(entityName: string): string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;
|
|
1
|
+
{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../src/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAE5B,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,GAAE,eAAoB,EAAE,sBAAsB,GAAE,OAAe;IAM3G,qBAAqB,IAAI,aAAa,EAAE;IAgDxC,oBAAoB,IAAI,MAAM,EAAE;IAwBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
|
package/dist/mapper.d.ts
CHANGED
|
@@ -2,22 +2,27 @@ import { GraphQLClientWrapper } from "./graphql-client";
|
|
|
2
2
|
import { MetricsCollector } from "./metrics";
|
|
3
3
|
import { ParallelProcessingConfig, RetryConfig } from "./config";
|
|
4
4
|
export interface MappingConfig {
|
|
5
|
-
csvFile
|
|
5
|
+
csvFile?: string;
|
|
6
|
+
dataFile?: string;
|
|
7
|
+
dataFormat?: string;
|
|
6
8
|
graphqlFile: string;
|
|
7
|
-
mapping: Record<string, string>;
|
|
9
|
+
mapping: Record<string, string | any>;
|
|
8
10
|
}
|
|
9
11
|
export declare class DataMapper {
|
|
10
12
|
private client;
|
|
11
13
|
private basePath;
|
|
12
14
|
private metrics;
|
|
13
15
|
private verbose;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
+
private formatOverride?;
|
|
17
|
+
constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean, formatOverride?: string);
|
|
18
|
+
discoverMappings(configDir: string, entityFilter?: string[]): string[];
|
|
16
19
|
processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
|
|
17
20
|
private processRowsSequentially;
|
|
18
21
|
private processRowsConcurrently;
|
|
19
22
|
private chunkArray;
|
|
20
|
-
private
|
|
23
|
+
private mapRowToVariables;
|
|
24
|
+
private getValueByPath;
|
|
25
|
+
private mapNestedObject;
|
|
21
26
|
private extractVariableTypes;
|
|
22
27
|
private extractTypeName;
|
|
23
28
|
private convertValue;
|
package/dist/mapper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;
|
|
1
|
+
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAE5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC;CACvC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe,EACxB,cAAc,CAAC,EAAE,MAAM;IASzB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IA4ChE,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YAyDF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IA+CzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IA0CvB,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAmEpB,UAAU,IAAI,gBAAgB;CAG/B"}
|
package/dist/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CA2C1B"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { DataReader, DataRow } from "./data-reader";
|
|
2
|
+
export interface CsvRow {
|
|
3
|
+
[key: string]: string;
|
|
4
|
+
}
|
|
5
|
+
export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
|
|
6
|
+
export declare class CsvReader extends DataReader {
|
|
7
|
+
getSupportedExtensions(): string[];
|
|
8
|
+
readFile(filePath: string): Promise<DataRow[]>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/readers/csv.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUrE;AAED,qBAAa,SAAU,SAAQ,UAAU;IACvC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAGrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.test.d.ts","sourceRoot":"","sources":["../../src/readers/csv.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface DataRow {
|
|
2
|
+
[key: string]: any;
|
|
3
|
+
}
|
|
4
|
+
export declare abstract class DataReader {
|
|
5
|
+
abstract readFile(filePath: string): Promise<DataRow[]>;
|
|
6
|
+
/**
|
|
7
|
+
* Get the supported file extensions for this reader
|
|
8
|
+
*/
|
|
9
|
+
abstract getSupportedExtensions(): string[];
|
|
10
|
+
/**
|
|
11
|
+
* Check if this reader can handle the given file
|
|
12
|
+
*/
|
|
13
|
+
canHandle(filePath: string): boolean;
|
|
14
|
+
}
|
|
15
|
+
export declare class DataReaderFactory {
|
|
16
|
+
private static readers;
|
|
17
|
+
static registerReader(reader: DataReader): void;
|
|
18
|
+
static getReader(filePath: string, format?: string): DataReader;
|
|
19
|
+
static getSupportedFormats(): string[];
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=data-reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data-reader.d.ts","sourceRoot":"","sources":["../../src/readers/data-reader.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,8BAAsB,UAAU;IAC9B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvD;;OAEG;IACH,QAAQ,CAAC,sBAAsB,IAAI,MAAM,EAAE;IAE3C;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;CAMrC;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAoB;IAE1C,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI/C,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU;IAuB/D,MAAM,CAAC,mBAAmB,IAAI,MAAM,EAAE;CAOvC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data-reader.test.d.ts","sourceRoot":"","sources":["../../src/readers/data-reader.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { DataReader, DataRow, DataReaderFactory } from "./data-reader";
|
|
2
|
+
export { CsvReader, readCsvFile, CsvRow } from "./csv";
|
|
3
|
+
export { JsonReader } from "./json";
|
|
4
|
+
export { YamlReader } from "./yaml";
|
|
5
|
+
export { JsonlReader } from "./jsonl";
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/readers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/readers/json.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,UAAW,SAAQ,UAAU;IACxC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAkBrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.test.d.ts","sourceRoot":"","sources":["../../src/readers/json.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,WAAY,SAAQ,UAAU;IACzC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAmBrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.test.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"yaml.d.ts","sourceRoot":"","sources":["../../src/readers/yaml.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpD,qBAAa,UAAW,SAAQ,UAAU;IACxC,sBAAsB,IAAI,MAAM,EAAE;IAI5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAkBrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"yaml.test.d.ts","sourceRoot":"","sources":["../../src/readers/yaml.test.ts"],"names":[],"mappings":""}
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -31,11 +31,19 @@ program
|
|
|
31
31
|
"-c, --config <path>",
|
|
32
32
|
"Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)"
|
|
33
33
|
)
|
|
34
|
+
.option(
|
|
35
|
+
"-n, --entities <entities>",
|
|
36
|
+
"Comma-separated list of specific entities to process (e.g., users,products)"
|
|
37
|
+
)
|
|
34
38
|
.option(
|
|
35
39
|
"-h, --headers <headers>",
|
|
36
40
|
"JSON string of headers to include in requests"
|
|
37
41
|
)
|
|
38
42
|
.option("-v, --verbose", "Show detailed request results and responses")
|
|
43
|
+
.option(
|
|
44
|
+
"-f, --format <format>",
|
|
45
|
+
"Override data format detection (csv, json, yaml, jsonl)"
|
|
46
|
+
)
|
|
39
47
|
.action(async (options) => {
|
|
40
48
|
try {
|
|
41
49
|
console.log("Starting seed data generation...");
|
|
@@ -62,32 +70,65 @@ program
|
|
|
62
70
|
client,
|
|
63
71
|
process.cwd(),
|
|
64
72
|
metrics,
|
|
65
|
-
options.verbose
|
|
73
|
+
options.verbose,
|
|
74
|
+
options.format
|
|
66
75
|
);
|
|
67
76
|
|
|
77
|
+
// Parse entities filter if provided
|
|
78
|
+
const entityFilter = options.entities
|
|
79
|
+
? options.entities.split(",").map((e: string) => e.trim())
|
|
80
|
+
: undefined;
|
|
81
|
+
|
|
68
82
|
// Discover all mapping files dynamically
|
|
69
|
-
const mappingPaths = mapper.discoverMappings(
|
|
83
|
+
const mappingPaths = mapper.discoverMappings(
|
|
84
|
+
options.config,
|
|
85
|
+
entityFilter
|
|
86
|
+
);
|
|
70
87
|
|
|
71
88
|
if (mappingPaths.length === 0) {
|
|
72
|
-
|
|
89
|
+
const filterMsg = entityFilter
|
|
90
|
+
? ` matching entities: ${entityFilter.join(", ")}`
|
|
91
|
+
: "";
|
|
92
|
+
console.warn(
|
|
93
|
+
`No mapping files found in ${options.config}/mappings${filterMsg}`
|
|
94
|
+
);
|
|
73
95
|
return;
|
|
74
96
|
}
|
|
75
97
|
|
|
76
98
|
// Extract entity names from mapping paths
|
|
77
99
|
const entityNames = mappingPaths.map((path) => basename(path, ".json"));
|
|
78
100
|
|
|
79
|
-
//
|
|
101
|
+
// Filter dependencies to only include those relevant to selected entities
|
|
102
|
+
const relevantDependencies: Record<string, string[]> = {};
|
|
103
|
+
if (config.entityDependencies) {
|
|
104
|
+
for (const entity of entityNames) {
|
|
105
|
+
if (config.entityDependencies[entity]) {
|
|
106
|
+
relevantDependencies[entity] = config.entityDependencies[entity];
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Setup dependency resolver with filtered dependencies
|
|
80
112
|
const resolver = new DependencyResolver(
|
|
81
113
|
entityNames,
|
|
82
|
-
|
|
114
|
+
relevantDependencies,
|
|
115
|
+
!!entityFilter // Allow partial resolution when using --entities
|
|
83
116
|
);
|
|
84
117
|
|
|
85
118
|
// Validate dependencies
|
|
86
119
|
const validationErrors = resolver.validateDependencies();
|
|
87
120
|
if (validationErrors.length > 0) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
121
|
+
if (entityFilter) {
|
|
122
|
+
// When using --entities flag, show warnings instead of errors
|
|
123
|
+
console.warn("\n⚠️ Warning: Dependency validation issues:");
|
|
124
|
+
validationErrors.forEach((error) => console.warn(` - ${error}`));
|
|
125
|
+
console.warn("This may cause errors if the dependent data doesn't already exist.\n");
|
|
126
|
+
} else {
|
|
127
|
+
// Strict validation when processing all entities
|
|
128
|
+
console.error("Dependency validation errors:");
|
|
129
|
+
validationErrors.forEach((error) => console.error(` - ${error}`));
|
|
130
|
+
process.exit(1);
|
|
131
|
+
}
|
|
91
132
|
}
|
|
92
133
|
|
|
93
134
|
// Process entities in dependency-aware waves
|
|
@@ -88,12 +88,26 @@ describe("DependencyResolver", () => {
|
|
|
88
88
|
a: ["missing"],
|
|
89
89
|
b: ["a"],
|
|
90
90
|
};
|
|
91
|
-
const resolver = new DependencyResolver(entities, dependencies);
|
|
91
|
+
const resolver = new DependencyResolver(entities, dependencies, false);
|
|
92
92
|
|
|
93
93
|
expect(() => resolver.resolveExecutionOrder()).toThrow(
|
|
94
94
|
"Circular dependency detected or missing dependencies for entities: a, b"
|
|
95
95
|
);
|
|
96
96
|
});
|
|
97
|
+
|
|
98
|
+
it("should allow entities with dependencies not in the entity list when partial resolution is enabled", () => {
|
|
99
|
+
const entities = ["a", "b"];
|
|
100
|
+
const dependencies = {
|
|
101
|
+
a: ["missing"],
|
|
102
|
+
b: ["a"],
|
|
103
|
+
};
|
|
104
|
+
const resolver = new DependencyResolver(entities, dependencies, true);
|
|
105
|
+
|
|
106
|
+
const waves = resolver.resolveExecutionOrder();
|
|
107
|
+
expect(waves).toHaveLength(2);
|
|
108
|
+
expect(waves[0].entities).toEqual(["a"]);
|
|
109
|
+
expect(waves[1].entities).toEqual(["b"]);
|
|
110
|
+
});
|
|
97
111
|
});
|
|
98
112
|
|
|
99
113
|
describe("validateDependencies", () => {
|
|
@@ -10,10 +10,12 @@ export interface ExecutionWave {
|
|
|
10
10
|
export class DependencyResolver {
|
|
11
11
|
private dependencies: DependencyGraph;
|
|
12
12
|
private entities: string[];
|
|
13
|
+
private allowPartialResolution: boolean;
|
|
13
14
|
|
|
14
|
-
constructor(entities: string[], dependencies: DependencyGraph = {}) {
|
|
15
|
+
constructor(entities: string[], dependencies: DependencyGraph = {}, allowPartialResolution: boolean = false) {
|
|
15
16
|
this.entities = entities;
|
|
16
17
|
this.dependencies = dependencies;
|
|
18
|
+
this.allowPartialResolution = allowPartialResolution;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
resolveExecutionOrder(): ExecutionWave[] {
|
|
@@ -31,7 +33,9 @@ export class DependencyResolver {
|
|
|
31
33
|
}
|
|
32
34
|
|
|
33
35
|
const deps = this.dependencies[entity] || [];
|
|
34
|
-
const canProcess = deps.every((dep) =>
|
|
36
|
+
const canProcess = deps.every((dep) =>
|
|
37
|
+
processed.has(dep) || (this.allowPartialResolution && !this.entities.includes(dep))
|
|
38
|
+
);
|
|
35
39
|
|
|
36
40
|
if (canProcess) {
|
|
37
41
|
currentWave.push(entity);
|
|
@@ -63,7 +63,10 @@ describe("GraphQLClientWrapper", () => {
|
|
|
63
63
|
clientWrapper.executeMutation(mutation, variables)
|
|
64
64
|
).rejects.toThrow("GraphQL error");
|
|
65
65
|
|
|
66
|
-
expect(consoleSpy).toHaveBeenCalledWith(
|
|
66
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
67
|
+
"GraphQL mutation failed after 3 attempts:",
|
|
68
|
+
error
|
|
69
|
+
);
|
|
67
70
|
|
|
68
71
|
consoleSpy.mockRestore();
|
|
69
72
|
});
|
|
@@ -96,7 +99,11 @@ describe("GraphQLClientWrapper", () => {
|
|
|
96
99
|
.mockRejectedValueOnce(serverError)
|
|
97
100
|
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
98
101
|
|
|
99
|
-
const result = await clientWrapper.executeMutation(
|
|
102
|
+
const result = await clientWrapper.executeMutation(
|
|
103
|
+
mutation,
|
|
104
|
+
variables,
|
|
105
|
+
retryConfig
|
|
106
|
+
);
|
|
100
107
|
|
|
101
108
|
expect(mockRequest).toHaveBeenCalledTimes(3);
|
|
102
109
|
expect(result).toEqual({ data: { result: "success" } });
|
|
@@ -143,7 +150,11 @@ describe("GraphQLClientWrapper", () => {
|
|
|
143
150
|
.mockRejectedValueOnce(networkError)
|
|
144
151
|
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
145
152
|
|
|
146
|
-
const result = await clientWrapper.executeMutation(
|
|
153
|
+
const result = await clientWrapper.executeMutation(
|
|
154
|
+
mutation,
|
|
155
|
+
variables,
|
|
156
|
+
retryConfig
|
|
157
|
+
);
|
|
147
158
|
|
|
148
159
|
expect(mockRequest).toHaveBeenCalledTimes(2);
|
|
149
160
|
expect(result).toEqual({ data: { result: "success" } });
|
|
@@ -192,7 +203,11 @@ describe("GraphQLClientWrapper", () => {
|
|
|
192
203
|
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
193
204
|
|
|
194
205
|
const startTime = Date.now();
|
|
195
|
-
const result = await clientWrapper.executeMutation(
|
|
206
|
+
const result = await clientWrapper.executeMutation(
|
|
207
|
+
mutation,
|
|
208
|
+
variables,
|
|
209
|
+
retryConfig
|
|
210
|
+
);
|
|
196
211
|
const totalTime = Date.now() - startTime;
|
|
197
212
|
|
|
198
213
|
expect(mockRequest).toHaveBeenCalledTimes(3);
|