@jackchuka/gql-ingest 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/config.d.ts CHANGED
@@ -3,19 +3,30 @@ export interface ParallelProcessingConfig {
3
3
  entityConcurrency: number;
4
4
  preserveRowOrder: boolean;
5
5
  }
6
+ export interface RetryConfig {
7
+ maxAttempts: number;
8
+ baseDelay: number;
9
+ maxDelay: number;
10
+ exponentialBackoff: boolean;
11
+ retryableStatusCodes: number[];
12
+ }
6
13
  export interface EntityConfig {
7
14
  concurrency?: number;
8
15
  preserveRowOrder?: boolean;
16
+ retry?: Partial<RetryConfig>;
9
17
  }
10
18
  export interface ProcessingConfig {
19
+ retry: RetryConfig;
11
20
  parallelProcessing: ParallelProcessingConfig;
12
21
  entityConfig: Record<string, EntityConfig>;
13
22
  entityDependencies: Record<string, string[]>;
14
23
  }
15
24
  export interface FullConfig extends ProcessingConfig {
16
25
  }
26
+ export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
17
27
  export declare const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig;
18
28
  export declare const DEFAULT_CONFIG: ProcessingConfig;
19
29
  export declare function loadConfig(configDir: string): ProcessingConfig;
20
30
  export declare function getEntityConfig(entityName: string, globalConfig: ProcessingConfig): ParallelProcessingConfig;
31
+ export declare function getRetryConfig(entityName: string, globalConfig: ProcessingConfig): RetryConfig;
21
32
  //# sourceMappingURL=config.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,wBAAwB,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC3C,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAW,SAAQ,gBAAgB;CAEnD;AAED,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,gBAI5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAmB9D;AAaD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,GAC7B,wBAAwB,CAiB1B"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,WAAW,CAAC;IACnB,kBAAkB,EAAE,wBAAwB,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC3C,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAW,SAAQ,gBAAgB;CAEnD;AAED,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,gBAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAmB9D;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,GAC7B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAC5B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,gBAAgB,GAC7B,WAAW,CAOb"}
@@ -1,10 +1,14 @@
1
- import { MetricsCollector } from './metrics';
1
+ import { MetricsCollector } from "./metrics";
2
+ import { RetryConfig } from "./config";
2
3
  export declare class GraphQLClientWrapper {
3
4
  private client;
4
5
  private metrics?;
5
6
  private verbose;
6
7
  constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose?: boolean);
7
- executeMutation(mutation: string, variables: Record<string, any>): Promise<any>;
8
+ executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig): Promise<any>;
9
+ private isRetryableError;
10
+ private calculateDelay;
11
+ private sleep;
8
12
  setHeaders(headers: Record<string, string>): void;
9
13
  }
10
14
  //# sourceMappingURL=graphql-client.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../src/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,OAAO,CAAU;gBAEb,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,gBAAgB,EAAE,OAAO,GAAE,OAAe;IAQ9G,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC;IA+BrF,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
1
+ {"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../src/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAwB,WAAW,EAAE,MAAM,UAAU,CAAC;AAE7D,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,OAAO,CAAU;gBAGvB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IASpB,eAAe,CACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9B,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,GAAG,CAAC;IA6Ff,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,KAAK;IAIb,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
package/dist/mapper.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { GraphQLClientWrapper } from "./graphql-client";
2
2
  import { MetricsCollector } from "./metrics";
3
- import { ParallelProcessingConfig } from "./config";
3
+ import { ParallelProcessingConfig, RetryConfig } from "./config";
4
4
  export interface MappingConfig {
5
5
  csvFile: string;
6
6
  graphqlFile: string;
@@ -13,11 +13,14 @@ export declare class DataMapper {
13
13
  private verbose;
14
14
  constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, verbose?: boolean);
15
15
  discoverMappings(configDir: string): string[];
16
- processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig): Promise<void>;
16
+ processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
17
17
  private processRowsSequentially;
18
18
  private processRowsConcurrently;
19
19
  private chunkArray;
20
20
  private mapCsvRowToVariables;
21
+ private extractVariableTypes;
22
+ private extractTypeName;
23
+ private convertValue;
21
24
  getMetrics(): MetricsCollector;
22
25
  }
23
26
  //# sourceMappingURL=mapper.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,MAAM,UAAU,CAAC;AAEpD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,GACxC,OAAO,CAAC,IAAI,CAAC;YA4CF,uBAAuB;YA8BvB,uBAAuB;IAkErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAe5B,UAAU,IAAI,gBAAgB;CAG/B"}
1
+ {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,OAAO,CAAU;gBAGvB,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,GAAE,OAAe;IAQ1B,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;YA8CF,uBAAuB;YAwCvB,uBAAuB;IA8ErC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAsDpB,UAAU,IAAI,gBAAgB;CAG/B"}
package/dist/metrics.d.ts CHANGED
@@ -11,6 +11,9 @@ export interface ProcessingMetrics {
11
11
  totalFailures: number;
12
12
  entityMetrics: Map<string, EntityMetrics>;
13
13
  requestDurations: number[];
14
+ retryAttempts: number;
15
+ retrySuccesses: number;
16
+ retryFailures: number;
14
17
  startTime: number;
15
18
  endTime?: number;
16
19
  }
@@ -26,6 +29,8 @@ export declare class MetricsCollector {
26
29
  getTotalProcessed(): number;
27
30
  getSuccessRate(): number;
28
31
  recordRequestDuration(duration: number): void;
32
+ recordRetrySuccess(attempts: number): void;
33
+ recordRetryFailure(attempts: number): void;
29
34
  getAverageRequestDuration(): number;
30
35
  getDurationMs(): number;
31
36
  generateSummary(): string;
@@ -1 +1 @@
1
- {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAanC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CA6B1B"}
1
+ {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAoB;;IAgBnC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,iBAAiB;IAKrC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI/D,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAmC1B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jackchuka/gql-ingest",
3
- "version": "1.2.0",
3
+ "version": "1.4.0",
4
4
  "description": "A CLI tool for ingesting data from CSV files into a GraphQL API",
5
5
  "type": "module",
6
6
  "main": "dist/cli.js",
package/src/cli.ts CHANGED
@@ -2,7 +2,7 @@ import { Command } from "commander";
2
2
  import { GraphQLClientWrapper } from "./graphql-client";
3
3
  import { DataMapper } from "./mapper";
4
4
  import { MetricsCollector } from "./metrics";
5
- import { loadConfig, getEntityConfig } from "./config";
5
+ import { loadConfig, getEntityConfig, getRetryConfig } from "./config";
6
6
  import { DependencyResolver } from "./dependency-resolver";
7
7
  import { basename } from "path";
8
8
 
@@ -114,7 +114,8 @@ async function processEntitiesSequentially(
114
114
  try {
115
115
  const entityName = basename(configPath, ".json");
116
116
  const entityConfig = getEntityConfig(entityName, config);
117
- await mapper.processEntity(configPath, entityConfig);
117
+ const retryConfig = getRetryConfig(entityName, config);
118
+ await mapper.processEntity(configPath, entityConfig, retryConfig);
118
119
  } catch (error) {
119
120
  console.warn(`Warning: Could not process ${configPath}:`, error);
120
121
  }
@@ -142,14 +143,15 @@ async function processEntitiesInWaves(
142
143
  // Process entities in controlled batches based on entityConcurrency
143
144
  const entityConcurrency = config.parallelProcessing.entityConcurrency;
144
145
  const chunks = chunkArray(wave.entities, entityConcurrency);
145
-
146
+
146
147
  for (const chunk of chunks) {
147
148
  const entityPromises = chunk.map(async (entityName) => {
148
149
  const configPath = pathMap.get(entityName);
149
150
  if (configPath) {
150
151
  try {
151
152
  const entityConfig = getEntityConfig(entityName, config);
152
- await mapper.processEntity(configPath, entityConfig);
153
+ const retryConfig = getRetryConfig(entityName, config);
154
+ await mapper.processEntity(configPath, entityConfig, retryConfig);
153
155
  } catch (error) {
154
156
  console.warn(`Warning: Could not process ${configPath}:`, error);
155
157
  }
@@ -1,6 +1,6 @@
1
1
  import fs from "fs";
2
2
  import path from "path";
3
- import { loadConfig, getEntityConfig, DEFAULT_CONFIG } from "./config";
3
+ import { loadConfig, getEntityConfig, getRetryConfig, DEFAULT_CONFIG } from "./config";
4
4
 
5
5
  jest.mock("fs");
6
6
  const mockFs = fs as jest.Mocked<typeof fs>;
@@ -112,6 +112,13 @@ entityConfig:
112
112
 
113
113
  describe("getEntityConfig", () => {
114
114
  const globalConfig = {
115
+ retry: {
116
+ maxAttempts: 3,
117
+ baseDelay: 1000,
118
+ maxDelay: 30000,
119
+ exponentialBackoff: true,
120
+ retryableStatusCodes: [408, 429, 500, 502, 503, 504],
121
+ },
115
122
  parallelProcessing: {
116
123
  concurrency: 10,
117
124
  entityConcurrency: 3,
@@ -201,4 +208,65 @@ entityConfig:
201
208
  consoleSpy.mockRestore();
202
209
  });
203
210
  });
211
+
212
+ describe("getRetryConfig", () => {
213
+ const globalConfig = {
214
+ retry: {
215
+ maxAttempts: 3,
216
+ baseDelay: 1000,
217
+ maxDelay: 30000,
218
+ exponentialBackoff: true,
219
+ retryableStatusCodes: [408, 429, 500, 502, 503, 504],
220
+ },
221
+ parallelProcessing: {
222
+ concurrency: 10,
223
+ entityConcurrency: 3,
224
+ preserveRowOrder: false,
225
+ },
226
+ entityConfig: {
227
+ important: {
228
+ retry: {
229
+ maxAttempts: 5,
230
+ baseDelay: 500,
231
+ },
232
+ },
233
+ fast: {
234
+ retry: {
235
+ maxAttempts: 1,
236
+ },
237
+ },
238
+ },
239
+ entityDependencies: {},
240
+ };
241
+
242
+ it("should return global retry config for entity without overrides", () => {
243
+ const retryConfig = getRetryConfig("regular", globalConfig);
244
+
245
+ expect(retryConfig).toEqual(globalConfig.retry);
246
+ });
247
+
248
+ it("should merge entity retry overrides with global config", () => {
249
+ const retryConfig = getRetryConfig("important", globalConfig);
250
+
251
+ expect(retryConfig.maxAttempts).toBe(5); // overridden
252
+ expect(retryConfig.baseDelay).toBe(500); // overridden
253
+ expect(retryConfig.maxDelay).toBe(30000); // from global
254
+ expect(retryConfig.exponentialBackoff).toBe(true); // from global
255
+ expect(retryConfig.retryableStatusCodes).toEqual([408, 429, 500, 502, 503, 504]); // from global
256
+ });
257
+
258
+ it("should handle partial retry overrides", () => {
259
+ const retryConfig = getRetryConfig("fast", globalConfig);
260
+
261
+ expect(retryConfig.maxAttempts).toBe(1); // overridden
262
+ expect(retryConfig.baseDelay).toBe(1000); // from global
263
+ expect(retryConfig.maxDelay).toBe(30000); // from global
264
+ });
265
+
266
+ it("should handle entity with no retry config", () => {
267
+ const retryConfig = getRetryConfig("undefined-entity", globalConfig);
268
+
269
+ expect(retryConfig).toEqual(globalConfig.retry);
270
+ });
271
+ });
204
272
  });
package/src/config.ts CHANGED
@@ -8,12 +8,22 @@ export interface ParallelProcessingConfig {
8
8
  preserveRowOrder: boolean;
9
9
  }
10
10
 
11
+ export interface RetryConfig {
12
+ maxAttempts: number;
13
+ baseDelay: number;
14
+ maxDelay: number;
15
+ exponentialBackoff: boolean;
16
+ retryableStatusCodes: number[];
17
+ }
18
+
11
19
  export interface EntityConfig {
12
20
  concurrency?: number;
13
21
  preserveRowOrder?: boolean;
22
+ retry?: Partial<RetryConfig>;
14
23
  }
15
24
 
16
25
  export interface ProcessingConfig {
26
+ retry: RetryConfig;
17
27
  parallelProcessing: ParallelProcessingConfig;
18
28
  entityConfig: Record<string, EntityConfig>;
19
29
  entityDependencies: Record<string, string[]>;
@@ -23,6 +33,14 @@ export interface FullConfig extends ProcessingConfig {
23
33
  // Future: additional config sections can be added here
24
34
  }
25
35
 
36
+ export const DEFAULT_RETRY_CONFIG: RetryConfig = {
37
+ maxAttempts: 3,
38
+ baseDelay: 1000,
39
+ maxDelay: 30000,
40
+ exponentialBackoff: true,
41
+ retryableStatusCodes: [408, 429, 500, 502, 503, 504],
42
+ };
43
+
26
44
  export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
27
45
  concurrency: 1,
28
46
  entityConcurrency: 1,
@@ -30,6 +48,7 @@ export const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig = {
30
48
  };
31
49
 
32
50
  export const DEFAULT_CONFIG: ProcessingConfig = {
51
+ retry: DEFAULT_RETRY_CONFIG,
33
52
  parallelProcessing: DEFAULT_PARALLEL_CONFIG,
34
53
  entityConfig: {},
35
54
  entityDependencies: {},
@@ -58,6 +77,10 @@ export function loadConfig(configDir: string): ProcessingConfig {
58
77
 
59
78
  function mergeWithDefaults(yamlConfig: Partial<FullConfig>): ProcessingConfig {
60
79
  return {
80
+ retry: {
81
+ ...DEFAULT_RETRY_CONFIG,
82
+ ...(yamlConfig.retry || {}),
83
+ },
61
84
  parallelProcessing: {
62
85
  ...DEFAULT_PARALLEL_CONFIG,
63
86
  ...(yamlConfig.parallelProcessing || {}),
@@ -88,3 +111,15 @@ export function getEntityConfig(
88
111
 
89
112
  return finalConfig;
90
113
  }
114
+
115
+ export function getRetryConfig(
116
+ entityName: string,
117
+ globalConfig: ProcessingConfig
118
+ ): RetryConfig {
119
+ const entityOverrides = globalConfig.entityConfig[entityName]?.retry || {};
120
+
121
+ return {
122
+ ...globalConfig.retry,
123
+ ...entityOverrides,
124
+ };
125
+ }
@@ -63,7 +63,7 @@ describe("GraphQLClientWrapper", () => {
63
63
  clientWrapper.executeMutation(mutation, variables)
64
64
  ).rejects.toThrow("GraphQL error");
65
65
 
66
- expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed:", error);
66
+ expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed after 3 attempts:", error);
67
67
 
68
68
  consoleSpy.mockRestore();
69
69
  });
@@ -75,4 +75,130 @@ describe("GraphQLClientWrapper", () => {
75
75
 
76
76
  expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
77
77
  });
78
+
79
+ describe("retry functionality", () => {
80
+ it("should retry on retryable status codes", async () => {
81
+ const mutation = "mutation { createUser(name: $name) { id } }";
82
+ const variables = { name: "John" };
83
+ const retryConfig = {
84
+ maxAttempts: 3,
85
+ baseDelay: 100,
86
+ maxDelay: 1000,
87
+ exponentialBackoff: false,
88
+ retryableStatusCodes: [500],
89
+ };
90
+
91
+ const serverError = new Error("Server Error");
92
+ (serverError as any).response = { status: 500 };
93
+
94
+ mockRequest
95
+ .mockRejectedValueOnce(serverError)
96
+ .mockRejectedValueOnce(serverError)
97
+ .mockResolvedValueOnce({ data: { result: "success" } });
98
+
99
+ const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
100
+
101
+ expect(mockRequest).toHaveBeenCalledTimes(3);
102
+ expect(result).toEqual({ data: { result: "success" } });
103
+ });
104
+
105
+ it("should not retry on non-retryable status codes", async () => {
106
+ const mutation = "mutation { createUser(name: $name) { id } }";
107
+ const variables = { name: "John" };
108
+ const retryConfig = {
109
+ maxAttempts: 3,
110
+ baseDelay: 100,
111
+ maxDelay: 1000,
112
+ exponentialBackoff: false,
113
+ retryableStatusCodes: [500],
114
+ };
115
+
116
+ const clientError = new Error("Bad Request");
117
+ (clientError as any).response = { status: 400 };
118
+
119
+ mockRequest.mockRejectedValueOnce(clientError);
120
+
121
+ await expect(
122
+ clientWrapper.executeMutation(mutation, variables, retryConfig)
123
+ ).rejects.toThrow("Bad Request");
124
+
125
+ expect(mockRequest).toHaveBeenCalledTimes(1);
126
+ });
127
+
128
+ it("should retry on network errors (no response)", async () => {
129
+ const mutation = "mutation { createUser(name: $name) { id } }";
130
+ const variables = { name: "John" };
131
+ const retryConfig = {
132
+ maxAttempts: 2,
133
+ baseDelay: 100,
134
+ maxDelay: 1000,
135
+ exponentialBackoff: false,
136
+ retryableStatusCodes: [500],
137
+ };
138
+
139
+ const networkError = new Error("Network Error");
140
+ // No response property = network error
141
+
142
+ mockRequest
143
+ .mockRejectedValueOnce(networkError)
144
+ .mockResolvedValueOnce({ data: { result: "success" } });
145
+
146
+ const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
147
+
148
+ expect(mockRequest).toHaveBeenCalledTimes(2);
149
+ expect(result).toEqual({ data: { result: "success" } });
150
+ });
151
+
152
+ it("should fail after max attempts are exhausted", async () => {
153
+ const mutation = "mutation { createUser(name: $name) { id } }";
154
+ const variables = { name: "John" };
155
+ const retryConfig = {
156
+ maxAttempts: 2,
157
+ baseDelay: 100,
158
+ maxDelay: 1000,
159
+ exponentialBackoff: false,
160
+ retryableStatusCodes: [500],
161
+ };
162
+
163
+ const serverError = new Error("Server Error");
164
+ (serverError as any).response = { status: 500 };
165
+
166
+ mockRequest.mockRejectedValue(serverError);
167
+
168
+ await expect(
169
+ clientWrapper.executeMutation(mutation, variables, retryConfig)
170
+ ).rejects.toThrow("Server Error");
171
+
172
+ expect(mockRequest).toHaveBeenCalledTimes(2);
173
+ });
174
+
175
+ it("should use exponential backoff when configured", async () => {
176
+ const mutation = "mutation { createUser(name: $name) { id } }";
177
+ const variables = { name: "John" };
178
+ const retryConfig = {
179
+ maxAttempts: 3,
180
+ baseDelay: 100,
181
+ maxDelay: 1000,
182
+ exponentialBackoff: true,
183
+ retryableStatusCodes: [500],
184
+ };
185
+
186
+ const serverError = new Error("Server Error");
187
+ (serverError as any).response = { status: 500 };
188
+
189
+ mockRequest
190
+ .mockRejectedValueOnce(serverError)
191
+ .mockRejectedValueOnce(serverError)
192
+ .mockResolvedValueOnce({ data: { result: "success" } });
193
+
194
+ const startTime = Date.now();
195
+ const result = await clientWrapper.executeMutation(mutation, variables, retryConfig);
196
+ const totalTime = Date.now() - startTime;
197
+
198
+ expect(mockRequest).toHaveBeenCalledTimes(3);
199
+ expect(result).toEqual({ data: { result: "success" } });
200
+ // Should have some delay for retries (base + exponential)
201
+ expect(totalTime).toBeGreaterThan(200); // At least 100ms base + 200ms exponential
202
+ });
203
+ });
78
204
  });
@@ -1,51 +1,151 @@
1
- import { GraphQLClient } from 'graphql-request';
2
- import { MetricsCollector } from './metrics';
1
+ import { GraphQLClient } from "graphql-request";
2
+ import { MetricsCollector } from "./metrics";
3
+ import { DEFAULT_RETRY_CONFIG, RetryConfig } from "./config";
3
4
 
4
5
  export class GraphQLClientWrapper {
5
6
  private client: GraphQLClient;
6
7
  private metrics?: MetricsCollector;
7
8
  private verbose: boolean;
8
9
 
9
- constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose: boolean = false) {
10
+ constructor(
11
+ endpoint: string,
12
+ headers?: Record<string, string>,
13
+ metrics?: MetricsCollector,
14
+ verbose: boolean = false
15
+ ) {
10
16
  this.client = new GraphQLClient(endpoint, {
11
- headers: headers || {}
17
+ headers: headers || {},
12
18
  });
13
19
  this.metrics = metrics;
14
20
  this.verbose = verbose;
15
21
  }
16
22
 
17
- async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
18
- const startTime = Date.now();
19
-
20
- try {
21
- const result = await this.client.request(mutation, variables);
22
-
23
- if (this.metrics) {
24
- const duration = Date.now() - startTime;
25
- this.metrics.recordRequestDuration(duration);
26
- }
27
-
28
- if (this.verbose) {
29
- console.log(`✓ GraphQL request completed in ${Date.now() - startTime}ms:`, result);
30
- }
31
-
32
- return result;
33
- } catch (error) {
34
- if (this.metrics) {
35
- const duration = Date.now() - startTime;
36
- this.metrics.recordRequestDuration(duration);
37
- }
38
-
39
- if (this.verbose) {
40
- console.error(`✗ GraphQL request failed in ${Date.now() - startTime}ms:`, error);
41
- } else {
42
- console.error('GraphQL mutation failed:', error);
23
+ async executeMutation(
24
+ mutation: string,
25
+ variables: Record<string, any>,
26
+ retryConfig?: RetryConfig
27
+ ): Promise<any> {
28
+ const config = retryConfig || DEFAULT_RETRY_CONFIG;
29
+
30
+ let lastError: any;
31
+ const totalStartTime = Date.now();
32
+
33
+ for (let attempt = 0; attempt < config.maxAttempts; attempt++) {
34
+ const attemptStartTime = Date.now();
35
+
36
+ try {
37
+ const result = await this.client.request(mutation, variables);
38
+
39
+ if (this.metrics) {
40
+ const duration = Date.now() - attemptStartTime;
41
+ this.metrics.recordRequestDuration(duration);
42
+ if (attempt > 0) {
43
+ this.metrics.recordRetrySuccess(attempt);
44
+ }
45
+ }
46
+
47
+ if (this.verbose) {
48
+ const totalDuration = Date.now() - totalStartTime;
49
+ const retryInfo =
50
+ attempt > 0 ? ` (succeeded on attempt ${attempt + 1})` : "";
51
+ console.log(
52
+ `✓ GraphQL request completed in ${totalDuration}ms${retryInfo}:`,
53
+ result
54
+ );
55
+ }
56
+
57
+ return result;
58
+ } catch (error: any) {
59
+ lastError = error;
60
+ const duration = Date.now() - attemptStartTime;
61
+
62
+ if (this.metrics) {
63
+ this.metrics.recordRequestDuration(duration);
64
+ }
65
+
66
+ // Check if this is the last attempt
67
+ if (attempt === config.maxAttempts - 1) {
68
+ if (this.metrics && attempt > 0) {
69
+ this.metrics.recordRetryFailure(attempt);
70
+ }
71
+ break;
72
+ }
73
+
74
+ // Check if error is retryable
75
+ if (!this.isRetryableError(error, config)) {
76
+ if (this.verbose) {
77
+ console.error(
78
+ `✗ GraphQL request failed with non-retryable error in ${duration}ms:`,
79
+ error
80
+ );
81
+ } else {
82
+ console.error("GraphQL mutation failed (non-retryable):", error);
83
+ }
84
+ throw error;
85
+ }
86
+
87
+ // Calculate delay
88
+ const delay = this.calculateDelay(attempt, config);
89
+
90
+ if (this.verbose) {
91
+ console.log(
92
+ `⏳ GraphQL request failed (attempt ${attempt + 1}/${
93
+ config.maxAttempts
94
+ }), retrying in ${delay}ms...`
95
+ );
96
+ }
97
+
98
+ // Wait before retry
99
+ await this.sleep(delay);
43
100
  }
44
- throw error;
45
101
  }
102
+
103
+ // All retries exhausted
104
+ if (this.verbose) {
105
+ const totalDuration = Date.now() - totalStartTime;
106
+ console.error(
107
+ `✗ GraphQL request failed after ${config.maxAttempts} attempts in ${totalDuration}ms:`,
108
+ lastError
109
+ );
110
+ } else {
111
+ console.error(
112
+ `GraphQL mutation failed after ${config.maxAttempts} attempts:`,
113
+ lastError
114
+ );
115
+ }
116
+
117
+ throw lastError;
118
+ }
119
+
120
+ private isRetryableError(error: any, config: RetryConfig): boolean {
121
+ // Network errors (no response)
122
+ if (!error.response) {
123
+ return true;
124
+ }
125
+
126
+ // Check HTTP status codes
127
+ const status = error.response.status;
128
+ return config.retryableStatusCodes.includes(status);
129
+ }
130
+
131
+ private calculateDelay(attempt: number, config: RetryConfig): number {
132
+ if (!config.exponentialBackoff) {
133
+ return config.baseDelay;
134
+ }
135
+
136
+ const exponentialDelay = config.baseDelay * Math.pow(2, attempt);
137
+ const cappedDelay = Math.min(exponentialDelay, config.maxDelay);
138
+
139
+ // Add jitter (±20% randomization)
140
+ const jitter = cappedDelay * 0.2 * (Math.random() - 0.5);
141
+ return Math.max(0, cappedDelay + jitter);
142
+ }
143
+
144
+ private sleep(ms: number): Promise<void> {
145
+ return new Promise((resolve) => setTimeout(resolve, ms));
46
146
  }
47
147
 
48
148
  setHeaders(headers: Record<string, string>) {
49
149
  this.client.setHeaders(headers);
50
150
  }
51
- }
151
+ }