@jackchuka/gql-ingest 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -31
- package/bin/cli.js +220 -41
- package/dist/config.d.ts +11 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/graphql-client.d.ts +6 -2
- package/dist/graphql-client.d.ts.map +1 -1
- package/dist/mapper.d.ts +5 -2
- package/dist/mapper.d.ts.map +1 -1
- package/dist/metrics.d.ts +5 -0
- package/dist/metrics.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/cli.ts +6 -4
- package/src/config.test.ts +69 -1
- package/src/config.ts +35 -0
- package/src/graphql-client.test.ts +127 -1
- package/src/graphql-client.ts +132 -32
- package/src/mapper.test.ts +225 -4
- package/src/mapper.ts +145 -18
- package/src/metrics.ts +22 -0
package/dist/config.d.ts
CHANGED
|
@@ -3,19 +3,30 @@ export interface ParallelProcessingConfig {
|
|
|
3
3
|
entityConcurrency: number;
|
|
4
4
|
preserveRowOrder: boolean;
|
|
5
5
|
}
|
|
6
|
+
export interface RetryConfig {
|
|
7
|
+
maxAttempts: number;
|
|
8
|
+
baseDelay: number;
|
|
9
|
+
maxDelay: number;
|
|
10
|
+
exponentialBackoff: boolean;
|
|
11
|
+
retryableStatusCodes: number[];
|
|
12
|
+
}
|
|
6
13
|
export interface EntityConfig {
|
|
7
14
|
concurrency?: number;
|
|
8
15
|
preserveRowOrder?: boolean;
|
|
16
|
+
retry?: Partial<RetryConfig>;
|
|
9
17
|
}
|
|
10
18
|
export interface ProcessingConfig {
|
|
19
|
+
retry: RetryConfig;
|
|
11
20
|
parallelProcessing: ParallelProcessingConfig;
|
|
12
21
|
entityConfig: Record<string, EntityConfig>;
|
|
13
22
|
entityDependencies: Record<string, string[]>;
|
|
14
23
|
}
|
|
15
24
|
export interface FullConfig extends ProcessingConfig {
|
|
16
25
|
}
|
|
26
|
+
export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
|
|
17
27
|
export declare const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig;
|
|
18
28
|
export declare const DEFAULT_CONFIG: ProcessingConfig;
|
|
19
29
|
export declare function loadConfig(configDir: string): ProcessingConfig;
|
|
20
30
|
export declare function getEntityConfig(entityName: string, globalConfig: ProcessingConfig): ParallelProcessingConfig;
|
|
31
|
+
export declare function getRetryConfig(entityName: string, globalConfig: ProcessingConfig): RetryConfig;
|
|
21
32
|
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,WAAW,CAAC;IACnB,kBAAkB,EAAE,wBAAwB,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC3C,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAW,SAAQ,gBAAgB;CAEnD;AAED,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,gBAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAmB9D;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,GAC7B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAC5B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,GAC7B,WAAW,CAOb"}
|
package/dist/graphql-client.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import { MetricsCollector } from
|
|
1
|
+
import { MetricsCollector } from "./metrics";
|
|
2
|
+
import { RetryConfig } from "./config";
|
|
2
3
|
export declare class GraphQLClientWrapper {
|
|
3
4
|
private client;
|
|
4
5
|
private metrics?;
|
|
5
6
|
private verbose;
|
|
6
7
|
constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose?: boolean);
|
|
7
|
-
executeMutation(mutation: string, variables: Record<string, any
|
|
8
|
+
executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig): Promise<any>;
|
|
9
|
+
private isRetryableError;
|
|
10
|
+
private calculateDelay;
|
|
11
|
+
private sleep;
|
|
8
12
|
setHeaders(headers: Record<string, string>): void;
|
|
9
13
|
}
|
|
10
14
|
//# sourceMappingURL=graphql-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../src/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../src/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAwB,WAAW,EAAE,MAAM,UAAU,CAAC;AAE7D,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,OAAO,CAAU;gBAGvB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IASpB,eAAe,CACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9B,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,GAAG,CAAC;IA6Ff,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,KAAK;IAIb,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
|
package/dist/mapper.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { GraphQLClientWrapper } from "./graphql-client";
|
|
2
2
|
import { MetricsCollector } from "./metrics";
|
|
3
|
-
import { ParallelProcessingConfig } from "./config";
|
|
3
|
+
import { ParallelProcessingConfig, RetryConfig } from "./config";
|
|
4
4
|
export interface MappingConfig {
|
|
5
5
|
csvFile: string;
|
|
6
6
|
graphqlFile: string;
|
|
@@ -13,11 +13,14 @@ export declare class DataMapper {
|
|
|
13
13
|
private verbose;
|
|
14
14
|
constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean);
|
|
15
15
|
discoverMappings(configDir: string): string[];
|
|
16
|
-
processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig): Promise<void>;
|
|
16
|
+
processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
|
|
17
17
|
private processRowsSequentially;
|
|
18
18
|
private processRowsConcurrently;
|
|
19
19
|
private chunkArray;
|
|
20
20
|
private mapCsvRowToVariables;
|
|
21
|
+
private extractVariableTypes;
|
|
22
|
+
private extractTypeName;
|
|
23
|
+
private convertValue;
|
|
21
24
|
getMetrics(): MetricsCollector;
|
|
22
25
|
}
|
|
23
26
|
//# sourceMappingURL=mapper.d.ts.map
|
package/dist/mapper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAsDpB,UAAU,IAAI,gBAAgB;CAG/B"}
|
package/dist/metrics.d.ts
CHANGED
|
@@ -11,6 +11,9 @@ export interface ProcessingMetrics {
|
|
|
11
11
|
totalFailures: number;
|
|
12
12
|
entityMetrics: Map<string, EntityMetrics>;
|
|
13
13
|
requestDurations: number[];
|
|
14
|
+
retryAttempts: number;
|
|
15
|
+
retrySuccesses: number;
|
|
16
|
+
retryFailures: number;
|
|
14
17
|
startTime: number;
|
|
15
18
|
endTime?: number;
|
|
16
19
|
}
|
|
@@ -26,6 +29,8 @@ export declare class MetricsCollector {
|
|
|
26
29
|
getTotalProcessed(): number;
|
|
27
30
|
getSuccessRate(): number;
|
|
28
31
|
recordRequestDuration(duration: number): void;
|
|
32
|
+
recordRetrySuccess(attempts: number): void;
|
|
33
|
+
recordRetryFailure(attempts: number): void;
|
|
29
34
|
getAverageRequestDuration(): number;
|
|
30
35
|
getDurationMs(): number;
|
|
31
36
|
generateSummary(): string;
|
package/dist/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAmC1B"}
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Command } from "commander";
|
|
|
2
2
|
import { GraphQLClientWrapper } from "./graphql-client";
|
|
3
3
|
import { DataMapper } from "./mapper";
|
|
4
4
|
import { MetricsCollector } from "./metrics";
|
|
5
|
-
import { loadConfig, getEntityConfig } from "./config";
|
|
5
|
+
import { loadConfig, getEntityConfig, getRetryConfig } from "./config";
|
|
6
6
|
import { DependencyResolver } from "./dependency-resolver";
|
|
7
7
|
import { basename } from "path";
|
|
8
8
|
|
|
@@ -114,7 +114,8 @@ async function processEntitiesSequentially(
|
|
|
114
114
|
try {
|
|
115
115
|
const entityName = basename(configPath, ".json");
|
|
116
116
|
const entityConfig = getEntityConfig(entityName, config);
|
|
117
|
-
|
|
117
|
+
const retryConfig = getRetryConfig(entityName, config);
|
|
118
|
+
await mapper.processEntity(configPath, entityConfig, retryConfig);
|
|
118
119
|
} catch (error) {
|
|
119
120
|
console.warn(`Warning: Could not process ${configPath}:`, error);
|
|
120
121
|
}
|
|
@@ -142,14 +143,15 @@ async function processEntitiesInWaves(
|
|
|
142
143
|
// Process entities in controlled batches based on entityConcurrency
|
|
143
144
|
const entityConcurrency = config.parallelProcessing.entityConcurrency;
|
|
144
145
|
const chunks = chunkArray(wave.entities, entityConcurrency);
|
|
145
|
-
|
|
146
|
+
|
|
146
147
|
for (const chunk of chunks) {
|
|
147
148
|
const entityPromises = chunk.map(async (entityName) => {
|
|
148
149
|
const configPath = pathMap.get(entityName);
|
|
149
150
|
if (configPath) {
|
|
150
151
|
try {
|
|
151
152
|
const entityConfig = getEntityConfig(entityName, config);
|
|
152
|
-
|
|
153
|
+
const retryConfig = getRetryConfig(entityName, config);
|
|
154
|
+
await mapper.processEntity(configPath, entityConfig, retryConfig);
|
|
153
155
|
} catch (error) {
|
|
154
156
|
console.warn(`Warning: Could not process ${configPath}:`, error);
|
|
155
157
|
}
|
package/src/config.test.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
|
-
import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
|
|
3
|
+
import { loadConfig, getEntityConfig, getRetryConfig, DEFAULT_CONFIG } from "./config";
|
|
4
4
|
|
|
5
5
|
jest.mock("fs");
|
|
6
6
|
const mockFs = fs as jest.Mocked<typeof fs>;
|
|
@@ -112,6 +112,13 @@ entityConfig:
|
|
|
112
112
|
|
|
113
113
|
describe("getEntityConfig", () => {
|
|
114
114
|
const globalConfig = {
|
|
115
|
+
retry: {
|
|
116
|
+
maxAttempts: 3,
|
|
117
|
+
baseDelay: 1000,
|
|
118
|
+
maxDelay: 30000,
|
|
119
|
+
exponentialBackoff: true,
|
|
120
|
+
retryableStatusCodes: [408, 429, 500, 502, 503, 504],
|
|
121
|
+
},
|
|
115
122
|
parallelProcessing: {
|
|
116
123
|
concurrency: 10,
|
|
117
124
|
entityConcurrency: 3,
|
|
@@ -201,4 +208,65 @@ entityConfig:
|
|
|
201
208
|
consoleSpy.mockRestore();
|
|
202
209
|
});
|
|
203
210
|
});
|
|
211
|
+
|
|
212
|
+
describe("getRetryConfig", () => {
|
|
213
|
+
const globalConfig = {
|
|
214
|
+
retry: {
|
|
215
|
+
maxAttempts: 3,
|
|
216
|
+
baseDelay: 1000,
|
|
217
|
+
maxDelay: 30000,
|
|
218
|
+
exponentialBackoff: true,
|
|
219
|
+
retryableStatusCodes: [408, 429, 500, 502, 503, 504],
|
|
220
|
+
},
|
|
221
|
+
parallelProcessing: {
|
|
222
|
+
concurrency: 10,
|
|
223
|
+
entityConcurrency: 3,
|
|
224
|
+
preserveRowOrder: false,
|
|
225
|
+
},
|
|
226
|
+
entityConfig: {
|
|
227
|
+
important: {
|
|
228
|
+
retry: {
|
|
229
|
+
maxAttempts: 5,
|
|
230
|
+
baseDelay: 500,
|
|
231
|
+
},
|
|
232
|
+
},
|
|
233
|
+
fast: {
|
|
234
|
+
retry: {
|
|
235
|
+
maxAttempts: 1,
|
|
236
|
+
},
|
|
237
|
+
},
|
|
238
|
+
},
|
|
239
|
+
entityDependencies: {},
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
it("should return global retry config for entity without overrides", () => {
|
|
243
|
+
const retryConfig = getRetryConfig("regular", globalConfig);
|
|
244
|
+
|
|
245
|
+
expect(retryConfig).toEqual(globalConfig.retry);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("should merge entity retry overrides with global config", () => {
|
|
249
|
+
const retryConfig = getRetryConfig("important", globalConfig);
|
|
250
|
+
|
|
251
|
+
expect(retryConfig.maxAttempts).toBe(5); // overridden
|
|
252
|
+
expect(retryConfig.baseDelay).toBe(500); // overridden
|
|
253
|
+
expect(retryConfig.maxDelay).toBe(30000); // from global
|
|
254
|
+
expect(retryConfig.exponentialBackoff).toBe(true); // from global
|
|
255
|
+
expect(retryConfig.retryableStatusCodes).toEqual([408, 429, 500, 502, 503, 504]); // from global
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it("should handle partial retry overrides", () => {
|
|
259
|
+
const retryConfig = getRetryConfig("fast", globalConfig);
|
|
260
|
+
|
|
261
|
+
expect(retryConfig.maxAttempts).toBe(1); // overridden
|
|
262
|
+
expect(retryConfig.baseDelay).toBe(1000); // from global
|
|
263
|
+
expect(retryConfig.maxDelay).toBe(30000); // from global
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
it("should handle entity with no retry config", () => {
|
|
267
|
+
const retryConfig = getRetryConfig("undefined-entity", globalConfig);
|
|
268
|
+
|
|
269
|
+
expect(retryConfig).toEqual(globalConfig.retry);
|
|
270
|
+
});
|
|
271
|
+
});
|
|
204
272
|
});
|
package/src/config.ts
CHANGED
|
@@ -8,12 +8,22 @@ export interface ParallelProcessingConfig {
|
|
|
8
8
|
preserveRowOrder: boolean;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
+
export interface RetryConfig {
|
|
12
|
+
maxAttempts: number;
|
|
13
|
+
baseDelay: number;
|
|
14
|
+
maxDelay: number;
|
|
15
|
+
exponentialBackoff: boolean;
|
|
16
|
+
retryableStatusCodes: number[];
|
|
17
|
+
}
|
|
18
|
+
|
|
11
19
|
export interface EntityConfig {
|
|
12
20
|
concurrency?: number;
|
|
13
21
|
preserveRowOrder?: boolean;
|
|
22
|
+
retry?: Partial<RetryConfig>;
|
|
14
23
|
}
|
|
15
24
|
|
|
16
25
|
export interface ProcessingConfig {
|
|
26
|
+
retry: RetryConfig;
|
|
17
27
|
parallelProcessing: ParallelProcessingConfig;
|
|
18
28
|
entityConfig: Record<string, EntityConfig>;
|
|
19
29
|
entityDependencies: Record<string, string[]>;
|
|
@@ -23,6 +33,14 @@ export interface FullConfig extends ProcessingConfig {
|
|
|
23
33
|
// Future: additional config sections can be added here
|
|
24
34
|
}
|
|
25
35
|
|
|
36
|
+
export const DEFAULT_RETRY_CONFIG: RetryConfig = {
|
|
37
|
+
maxAttempts: 3,
|
|
38
|
+
baseDelay: 1000,
|
|
39
|
+
maxDelay: 30000,
|
|
40
|
+
exponentialBackoff: true,
|
|
41
|
+
retryableStatusCodes: [408, 429, 500, 502, 503, 504],
|
|
42
|
+
};
|
|
43
|
+
|
|
26
44
|
export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
|
|
27
45
|
concurrency: 1,
|
|
28
46
|
entityConcurrency: 1,
|
|
@@ -30,6 +48,7 @@ export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
|
|
|
30
48
|
};
|
|
31
49
|
|
|
32
50
|
export const DEFAULT_CONFIG: ProcessingConfig = {
|
|
51
|
+
retry: DEFAULT_RETRY_CONFIG,
|
|
33
52
|
parallelProcessing: DEFAULT_PARALLEL_CONFIG,
|
|
34
53
|
entityConfig: {},
|
|
35
54
|
entityDependencies: {},
|
|
@@ -58,6 +77,10 @@ export function loadConfig(configDir: string): ProcessingConfig {
|
|
|
58
77
|
|
|
59
78
|
function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
|
|
60
79
|
return {
|
|
80
|
+
retry: {
|
|
81
|
+
...DEFAULT_RETRY_CONFIG,
|
|
82
|
+
...(yamlConfig.retry || {}),
|
|
83
|
+
},
|
|
61
84
|
parallelProcessing: {
|
|
62
85
|
...DEFAULT_PARALLEL_CONFIG,
|
|
63
86
|
...(yamlConfig.parallelProcessing || {}),
|
|
@@ -88,3 +111,15 @@ export function getEntityConfig(
|
|
|
88
111
|
|
|
89
112
|
return finalConfig;
|
|
90
113
|
}
|
|
114
|
+
|
|
115
|
+
export function getRetryConfig(
|
|
116
|
+
entityName: string,
|
|
117
|
+
globalConfig: ProcessingConfig
|
|
118
|
+
): RetryConfig {
|
|
119
|
+
const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
...globalConfig.retry,
|
|
123
|
+
...entityOverrides,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
@@ -63,7 +63,7 @@ describe("GraphQLClientWrapper", () => {
|
|
|
63
63
|
clientWrapper.executeMutation(mutation, variables)
|
|
64
64
|
).rejects.toThrow("GraphQL error");
|
|
65
65
|
|
|
66
|
-
expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed:", error);
|
|
66
|
+
expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed after 3 attempts:", error);
|
|
67
67
|
|
|
68
68
|
consoleSpy.mockRestore();
|
|
69
69
|
});
|
|
@@ -75,4 +75,130 @@ describe("GraphQLClientWrapper", () => {
|
|
|
75
75
|
|
|
76
76
|
expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
|
|
77
77
|
});
|
|
78
|
+
|
|
79
|
+
describe("retry functionality", () => {
|
|
80
|
+
it("should retry on retryable status codes", async () => {
|
|
81
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
82
|
+
const variables = { name: "John" };
|
|
83
|
+
const retryConfig = {
|
|
84
|
+
maxAttempts: 3,
|
|
85
|
+
baseDelay: 100,
|
|
86
|
+
maxDelay: 1000,
|
|
87
|
+
exponentialBackoff: false,
|
|
88
|
+
retryableStatusCodes: [500],
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const serverError = new Error("Server Error");
|
|
92
|
+
(serverError as any).response = { status: 500 };
|
|
93
|
+
|
|
94
|
+
mockRequest
|
|
95
|
+
.mockRejectedValueOnce(serverError)
|
|
96
|
+
.mockRejectedValueOnce(serverError)
|
|
97
|
+
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
98
|
+
|
|
99
|
+
const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
|
|
100
|
+
|
|
101
|
+
expect(mockRequest).toHaveBeenCalledTimes(3);
|
|
102
|
+
expect(result).toEqual({ data: { result: "success" } });
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("should not retry on non-retryable status codes", async () => {
|
|
106
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
107
|
+
const variables = { name: "John" };
|
|
108
|
+
const retryConfig = {
|
|
109
|
+
maxAttempts: 3,
|
|
110
|
+
baseDelay: 100,
|
|
111
|
+
maxDelay: 1000,
|
|
112
|
+
exponentialBackoff: false,
|
|
113
|
+
retryableStatusCodes: [500],
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
const clientError = new Error("Bad Request");
|
|
117
|
+
(clientError as any).response = { status: 400 };
|
|
118
|
+
|
|
119
|
+
mockRequest.mockRejectedValueOnce(clientError);
|
|
120
|
+
|
|
121
|
+
await expect(
|
|
122
|
+
clientWrapper.executeMutation(mutation, variables, retryConfig)
|
|
123
|
+
).rejects.toThrow("Bad Request");
|
|
124
|
+
|
|
125
|
+
expect(mockRequest).toHaveBeenCalledTimes(1);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("should retry on network errors (no response)", async () => {
|
|
129
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
130
|
+
const variables = { name: "John" };
|
|
131
|
+
const retryConfig = {
|
|
132
|
+
maxAttempts: 2,
|
|
133
|
+
baseDelay: 100,
|
|
134
|
+
maxDelay: 1000,
|
|
135
|
+
exponentialBackoff: false,
|
|
136
|
+
retryableStatusCodes: [500],
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const networkError = new Error("Network Error");
|
|
140
|
+
// No response property = network error
|
|
141
|
+
|
|
142
|
+
mockRequest
|
|
143
|
+
.mockRejectedValueOnce(networkError)
|
|
144
|
+
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
145
|
+
|
|
146
|
+
const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
|
|
147
|
+
|
|
148
|
+
expect(mockRequest).toHaveBeenCalledTimes(2);
|
|
149
|
+
expect(result).toEqual({ data: { result: "success" } });
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it("should fail after max attempts are exhausted", async () => {
|
|
153
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
154
|
+
const variables = { name: "John" };
|
|
155
|
+
const retryConfig = {
|
|
156
|
+
maxAttempts: 2,
|
|
157
|
+
baseDelay: 100,
|
|
158
|
+
maxDelay: 1000,
|
|
159
|
+
exponentialBackoff: false,
|
|
160
|
+
retryableStatusCodes: [500],
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const serverError = new Error("Server Error");
|
|
164
|
+
(serverError as any).response = { status: 500 };
|
|
165
|
+
|
|
166
|
+
mockRequest.mockRejectedValue(serverError);
|
|
167
|
+
|
|
168
|
+
await expect(
|
|
169
|
+
clientWrapper.executeMutation(mutation, variables, retryConfig)
|
|
170
|
+
).rejects.toThrow("Server Error");
|
|
171
|
+
|
|
172
|
+
expect(mockRequest).toHaveBeenCalledTimes(2);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it("should use exponential backoff when configured", async () => {
|
|
176
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
177
|
+
const variables = { name: "John" };
|
|
178
|
+
const retryConfig = {
|
|
179
|
+
maxAttempts: 3,
|
|
180
|
+
baseDelay: 100,
|
|
181
|
+
maxDelay: 1000,
|
|
182
|
+
exponentialBackoff: true,
|
|
183
|
+
retryableStatusCodes: [500],
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
const serverError = new Error("Server Error");
|
|
187
|
+
(serverError as any).response = { status: 500 };
|
|
188
|
+
|
|
189
|
+
mockRequest
|
|
190
|
+
.mockRejectedValueOnce(serverError)
|
|
191
|
+
.mockRejectedValueOnce(serverError)
|
|
192
|
+
.mockResolvedValueOnce({ data: { result: "success" } });
|
|
193
|
+
|
|
194
|
+
const startTime = Date.now();
|
|
195
|
+
const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
|
|
196
|
+
const totalTime = Date.now() - startTime;
|
|
197
|
+
|
|
198
|
+
expect(mockRequest).toHaveBeenCalledTimes(3);
|
|
199
|
+
expect(result).toEqual({ data: { result: "success" } });
|
|
200
|
+
// Should have some delay for retries (base + exponential)
|
|
201
|
+
expect(totalTime).toBeGreaterThan(200); // At least 100ms base + 200ms exponential
|
|
202
|
+
});
|
|
203
|
+
});
|
|
78
204
|
});
|
package/src/graphql-client.ts
CHANGED
|
@@ -1,51 +1,151 @@
|
|
|
1
|
-
import { GraphQLClient } from
|
|
2
|
-
import { MetricsCollector } from
|
|
1
|
+
import { GraphQLClient } from "graphql-request";
|
|
2
|
+
import { MetricsCollector } from "./metrics";
|
|
3
|
+
import { DEFAULT_RETRY_CONFIG, RetryConfig } from "./config";
|
|
3
4
|
|
|
4
5
|
export class GraphQLClientWrapper {
|
|
5
6
|
private client: GraphQLClient;
|
|
6
7
|
private metrics?: MetricsCollector;
|
|
7
8
|
private verbose: boolean;
|
|
8
9
|
|
|
9
|
-
constructor(
|
|
10
|
+
constructor(
|
|
11
|
+
endpoint: string,
|
|
12
|
+
headers?: Record<string, string>,
|
|
13
|
+
metrics?: MetricsCollector,
|
|
14
|
+
verbose: boolean = false
|
|
15
|
+
) {
|
|
10
16
|
this.client = new GraphQLClient(endpoint, {
|
|
11
|
-
headers: headers || {}
|
|
17
|
+
headers: headers || {},
|
|
12
18
|
});
|
|
13
19
|
this.metrics = metrics;
|
|
14
20
|
this.verbose = verbose;
|
|
15
21
|
}
|
|
16
22
|
|
|
17
|
-
async executeMutation(
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
23
|
+
async executeMutation(
|
|
24
|
+
mutation: string,
|
|
25
|
+
variables: Record<string, any>,
|
|
26
|
+
retryConfig?: RetryConfig
|
|
27
|
+
): Promise<any> {
|
|
28
|
+
const config = retryConfig || DEFAULT_RETRY_CONFIG;
|
|
29
|
+
|
|
30
|
+
let lastError: any;
|
|
31
|
+
const totalStartTime = Date.now();
|
|
32
|
+
|
|
33
|
+
for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
|
|
34
|
+
const attemptStartTime = Date.now();
|
|
35
|
+
|
|
36
|
+
try {
|
|
37
|
+
const result = await this.client.request(mutation, variables);
|
|
38
|
+
|
|
39
|
+
if (this.metrics) {
|
|
40
|
+
const duration = Date.now() - attemptStartTime;
|
|
41
|
+
this.metrics.recordRequestDuration(duration);
|
|
42
|
+
if (attempt > 0) {
|
|
43
|
+
this.metrics.recordRetrySuccess(attempt);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (this.verbose) {
|
|
48
|
+
const totalDuration = Date.now() - totalStartTime;
|
|
49
|
+
const retryInfo =
|
|
50
|
+
attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : "";
|
|
51
|
+
console.log(
|
|
52
|
+
`✓ GraphQL request completed in ${totalDuration}ms${retryInfo}:`,
|
|
53
|
+
result
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return result;
|
|
58
|
+
} catch (error: any) {
|
|
59
|
+
lastError = error;
|
|
60
|
+
const duration = Date.now() - attemptStartTime;
|
|
61
|
+
|
|
62
|
+
if (this.metrics) {
|
|
63
|
+
this.metrics.recordRequestDuration(duration);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Check if this is the last attempt
|
|
67
|
+
if (attempt === config.maxAttempts - 1) {
|
|
68
|
+
if (this.metrics && attempt > 0) {
|
|
69
|
+
this.metrics.recordRetryFailure(attempt);
|
|
70
|
+
}
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Check if error is retryable
|
|
75
|
+
if (!this.isRetryableError(error, config)) {
|
|
76
|
+
if (this.verbose) {
|
|
77
|
+
console.error(
|
|
78
|
+
`✗ GraphQL request failed with non-retryable error in ${duration}ms:`,
|
|
79
|
+
error
|
|
80
|
+
);
|
|
81
|
+
} else {
|
|
82
|
+
console.error("GraphQL mutation failed (non-retryable):", error);
|
|
83
|
+
}
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Calculate delay
|
|
88
|
+
const delay = this.calculateDelay(attempt, config);
|
|
89
|
+
|
|
90
|
+
if (this.verbose) {
|
|
91
|
+
console.log(
|
|
92
|
+
`⏳ GraphQL request failed (attempt ${attempt + 1}/${
|
|
93
|
+
config.maxAttempts
|
|
94
|
+
}), retrying in ${delay}ms...`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Wait before retry
|
|
99
|
+
await this.sleep(delay);
|
|
43
100
|
}
|
|
44
|
-
throw error;
|
|
45
101
|
}
|
|
102
|
+
|
|
103
|
+
// All retries exhausted
|
|
104
|
+
if (this.verbose) {
|
|
105
|
+
const totalDuration = Date.now() - totalStartTime;
|
|
106
|
+
console.error(
|
|
107
|
+
`✗ GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms:`,
|
|
108
|
+
lastError
|
|
109
|
+
);
|
|
110
|
+
} else {
|
|
111
|
+
console.error(
|
|
112
|
+
`GraphQL mutation failed after ${config.maxAttempts} attempts:`,
|
|
113
|
+
lastError
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
throw lastError;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private isRetryableError(error: any, config: RetryConfig): boolean {
|
|
121
|
+
// Network errors (no response)
|
|
122
|
+
if (!error.response) {
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Check HTTP status codes
|
|
127
|
+
const status = error.response.status;
|
|
128
|
+
return config.retryableStatusCodes.includes(status);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private calculateDelay(attempt: number, config: RetryConfig): number {
|
|
132
|
+
if (!config.exponentialBackoff) {
|
|
133
|
+
return config.baseDelay;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const exponentialDelay = config.baseDelay * Math.pow(2, attempt);
|
|
137
|
+
const cappedDelay = Math.min(exponentialDelay, config.maxDelay);
|
|
138
|
+
|
|
139
|
+
// Add jitter (±20% randomization)
|
|
140
|
+
const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);
|
|
141
|
+
return Math.max(0, cappedDelay + jitter);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private sleep(ms: number): Promise<void> {
|
|
145
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
46
146
|
}
|
|
47
147
|
|
|
48
148
|
setHeaders(headers: Record<string, string>) {
|
|
49
149
|
this.client.setHeaders(headers);
|
|
50
150
|
}
|
|
51
|
-
}
|
|
151
|
+
}
|