@jackchuka/gql-ingest 2.2.2 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +259 -32
  2. package/dist/cli/commands/add.d.ts +3 -0
  3. package/dist/cli/commands/add.d.ts.map +1 -0
  4. package/dist/cli/commands/init.d.ts +3 -0
  5. package/dist/cli/commands/init.d.ts.map +1 -0
  6. package/dist/cli/index.d.ts +2 -0
  7. package/dist/cli/index.d.ts.map +1 -0
  8. package/dist/cli/index.js +288 -0
  9. package/dist/cli/templates/index.d.ts +14 -0
  10. package/dist/cli/templates/index.d.ts.map +1 -0
  11. package/dist/index.d.ts +16 -0
  12. package/dist/index.d.ts.map +1 -0
  13. package/dist/index.js +15204 -0
  14. package/dist/index.js.map +7 -0
  15. package/dist/lib/config-schema.d.ts +103 -0
  16. package/dist/lib/config-schema.d.ts.map +1 -0
  17. package/dist/lib/config.d.ts +9 -0
  18. package/dist/lib/config.d.ts.map +1 -0
  19. package/dist/lib/dependency-resolver.d.ts.map +1 -0
  20. package/dist/lib/events.d.ts +167 -0
  21. package/dist/lib/events.d.ts.map +1 -0
  22. package/dist/lib/gql-ingest.d.ts +155 -0
  23. package/dist/lib/gql-ingest.d.ts.map +1 -0
  24. package/dist/{graphql-client.d.ts → lib/graphql-client.d.ts} +8 -4
  25. package/dist/lib/graphql-client.d.ts.map +1 -0
  26. package/dist/lib/index.d.ts +8 -0
  27. package/dist/lib/index.d.ts.map +1 -0
  28. package/dist/lib/logger.d.ts +29 -0
  29. package/dist/lib/logger.d.ts.map +1 -0
  30. package/dist/lib/mapper.d.ts +49 -0
  31. package/dist/lib/mapper.d.ts.map +1 -0
  32. package/dist/{metrics.d.ts → lib/metrics.d.ts} +22 -8
  33. package/dist/lib/metrics.d.ts.map +1 -0
  34. package/dist/readers/csv.d.ts +0 -4
  35. package/dist/readers/csv.d.ts.map +1 -1
  36. package/dist/readers/index.d.ts +1 -1
  37. package/dist/readers/index.d.ts.map +1 -1
  38. package/package.json +19 -7
  39. package/dist/cli.d.ts +0 -2
  40. package/dist/cli.d.ts.map +0 -1
  41. package/dist/cli.js +0 -237
  42. package/dist/config.d.ts +0 -32
  43. package/dist/config.d.ts.map +0 -1
  44. package/dist/config.test.d.ts +0 -2
  45. package/dist/config.test.d.ts.map +0 -1
  46. package/dist/dependency-resolver.d.ts.map +0 -1
  47. package/dist/dependency-resolver.test.d.ts +0 -2
  48. package/dist/dependency-resolver.test.d.ts.map +0 -1
  49. package/dist/graphql-client.d.ts.map +0 -1
  50. package/dist/graphql-client.test.d.ts +0 -2
  51. package/dist/graphql-client.test.d.ts.map +0 -1
  52. package/dist/mapper.d.ts +0 -31
  53. package/dist/mapper.d.ts.map +0 -1
  54. package/dist/mapper.test.d.ts +0 -2
  55. package/dist/mapper.test.d.ts.map +0 -1
  56. package/dist/metrics.d.ts.map +0 -1
  57. package/dist/metrics.test.d.ts +0 -2
  58. package/dist/metrics.test.d.ts.map +0 -1
  59. package/dist/readers/csv.test.d.ts +0 -2
  60. package/dist/readers/csv.test.d.ts.map +0 -1
  61. package/dist/readers/data-reader.test.d.ts +0 -2
  62. package/dist/readers/data-reader.test.d.ts.map +0 -1
  63. package/dist/readers/json.test.d.ts +0 -2
  64. package/dist/readers/json.test.d.ts.map +0 -1
  65. package/dist/readers/jsonl.test.d.ts +0 -2
  66. package/dist/readers/jsonl.test.d.ts.map +0 -1
  67. package/dist/readers/yaml.test.d.ts +0 -2
  68. package/dist/readers/yaml.test.d.ts.map +0 -1
  69. /package/dist/{dependency-resolver.d.ts → lib/dependency-resolver.d.ts} +0 -0
@@ -0,0 +1,103 @@
1
+ import { z } from "zod/v4";
2
+ export declare const RetryConfigSchema: z.ZodObject<{
3
+ maxAttempts: z.ZodNumber;
4
+ baseDelay: z.ZodNumber;
5
+ maxDelay: z.ZodNumber;
6
+ exponentialBackoff: z.ZodBoolean;
7
+ retryableStatusCodes: z.ZodArray<z.ZodNumber>;
8
+ }, z.core.$strip>;
9
+ export declare const ParallelProcessingConfigSchema: z.ZodObject<{
10
+ concurrency: z.ZodNumber;
11
+ entityConcurrency: z.ZodNumber;
12
+ preserveRowOrder: z.ZodBoolean;
13
+ }, z.core.$strip>;
14
+ export declare const EntityConfigSchema: z.ZodObject<{
15
+ concurrency: z.ZodOptional<z.ZodNumber>;
16
+ preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
17
+ retry: z.ZodOptional<z.ZodObject<{
18
+ maxAttempts: z.ZodOptional<z.ZodNumber>;
19
+ baseDelay: z.ZodOptional<z.ZodNumber>;
20
+ maxDelay: z.ZodOptional<z.ZodNumber>;
21
+ exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
22
+ retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
23
+ }, z.core.$strip>>;
24
+ }, z.core.$strip>;
25
+ export declare const ConfigSchema: z.ZodObject<{
26
+ retry: z.ZodObject<{
27
+ maxAttempts: z.ZodNumber;
28
+ baseDelay: z.ZodNumber;
29
+ maxDelay: z.ZodNumber;
30
+ exponentialBackoff: z.ZodBoolean;
31
+ retryableStatusCodes: z.ZodArray<z.ZodNumber>;
32
+ }, z.core.$strip>;
33
+ parallelProcessing: z.ZodObject<{
34
+ concurrency: z.ZodNumber;
35
+ entityConcurrency: z.ZodNumber;
36
+ preserveRowOrder: z.ZodBoolean;
37
+ }, z.core.$strip>;
38
+ entityConfig: z.ZodRecord<z.ZodString, z.ZodObject<{
39
+ concurrency: z.ZodOptional<z.ZodNumber>;
40
+ preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
41
+ retry: z.ZodOptional<z.ZodObject<{
42
+ maxAttempts: z.ZodOptional<z.ZodNumber>;
43
+ baseDelay: z.ZodOptional<z.ZodNumber>;
44
+ maxDelay: z.ZodOptional<z.ZodNumber>;
45
+ exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
46
+ retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
47
+ }, z.core.$strip>>;
48
+ }, z.core.$strip>>;
49
+ entityDependencies: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>;
50
+ }, z.core.$strip>;
51
+ export type RetryConfig = z.infer<typeof RetryConfigSchema>;
52
+ export type ParallelProcessingConfig = z.infer<typeof ParallelProcessingConfigSchema>;
53
+ export type EntityConfig = z.infer<typeof EntityConfigSchema>;
54
+ export type Config = z.infer<typeof ConfigSchema>;
55
+ export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
56
+ export declare const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig;
57
+ export declare const CONFIG_TEMPLATE: {
58
+ schema: {
59
+ parallelProcessing: z.ZodObject<{
60
+ concurrency: z.ZodNumber;
61
+ entityConcurrency: z.ZodNumber;
62
+ preserveRowOrder: z.ZodBoolean;
63
+ }, z.core.$strip>;
64
+ retry: z.ZodObject<{
65
+ maxAttempts: z.ZodNumber;
66
+ baseDelay: z.ZodNumber;
67
+ maxDelay: z.ZodNumber;
68
+ exponentialBackoff: z.ZodBoolean;
69
+ retryableStatusCodes: z.ZodArray<z.ZodNumber>;
70
+ }, z.core.$strip>;
71
+ entityConfig: z.ZodRecord<z.ZodString, z.ZodObject<{
72
+ concurrency: z.ZodOptional<z.ZodNumber>;
73
+ preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
74
+ retry: z.ZodOptional<z.ZodObject<{
75
+ maxAttempts: z.ZodOptional<z.ZodNumber>;
76
+ baseDelay: z.ZodOptional<z.ZodNumber>;
77
+ maxDelay: z.ZodOptional<z.ZodNumber>;
78
+ exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
79
+ retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
80
+ }, z.core.$strip>>;
81
+ }, z.core.$strip>>;
82
+ entityDependencies: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>;
83
+ };
84
+ defaults: {
85
+ parallelProcessing: {
86
+ concurrency: number;
87
+ entityConcurrency: number;
88
+ preserveRowOrder: boolean;
89
+ };
90
+ retry: {
91
+ maxAttempts: number;
92
+ baseDelay: number;
93
+ maxDelay: number;
94
+ exponentialBackoff: boolean;
95
+ retryableStatusCodes: number[];
96
+ };
97
+ };
98
+ examples: {
99
+ entityConfig: Record<string, EntityConfig>;
100
+ entityDependencies: Record<string, string[]>;
101
+ };
102
+ };
103
+ //# sourceMappingURL=config-schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config-schema.d.ts","sourceRoot":"","sources":["../../src/lib/config-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAE3B,eAAO,MAAM,iBAAiB;;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,8BAA8B;;;;iBAIzC,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;iBAI7B,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;iBASvB,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAGlD,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAGF,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAkBnB,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC;4BAG5B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;;CAEhC,CAAC"}
@@ -0,0 +1,9 @@
1
+ import { Logger } from "./logger";
2
+ import { type Config, type ParallelProcessingConfig, type RetryConfig } from "./config-schema";
3
+ export type { Config, ParallelProcessingConfig, RetryConfig, EntityConfig } from "./config-schema";
4
+ export { DEFAULT_RETRY_CONFIG, DEFAULT_PARALLEL_CONFIG } from "./config-schema";
5
+ export declare const DEFAULT_CONFIG: Config;
6
+ export declare function loadConfig(configDir: string, logger?: Logger): Config;
7
+ export declare function getEntityConfig(entityName: string, globalConfig: Config, logger?: Logger): ParallelProcessingConfig;
8
+ export declare function getRetryConfig(entityName: string, globalConfig: Config): RetryConfig;
9
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EAAE,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEnG,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,eAAO,MAAM,cAAc,EAAE,MAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,GAAE,MAAmB,GAAG,MAAM,CAkBjF;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAmB,GAC1B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,WAAW,CAOpF"}
@@ -0,0 +1 @@
1
+ {"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../../src/lib/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAGtC,QAAQ,EAAE,MAAM,EAAE,EAClB,YAAY,GAAE,eAAoB,EAClC,sBAAsB,GAAE,OAAe;IAOzC,qBAAqB,IAAI,aAAa,EAAE;IAiDxC,oBAAoB,IAAI,MAAM,EAAE;IAoBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
@@ -0,0 +1,167 @@
1
+ import { ProcessingMetrics, EntityMetrics } from "./metrics";
2
+ import { DataRow } from "../readers";
3
+ /**
4
+ * Event types emitted by GQLIngest during processing
5
+ */
6
+ export type GQLIngestEventType = "started" | "progress" | "entityStart" | "entityComplete" | "rowSuccess" | "rowFailure" | "cancelled" | "finished" | "errored";
7
+ /**
8
+ * Payload for 'started' event - emitted when ingestion begins
9
+ */
10
+ export interface StartedEventPayload {
11
+ /** Path to configuration directory */
12
+ configPath: string;
13
+ /** Total number of entities to process */
14
+ totalEntities: number;
15
+ /** Names of entities that will be processed */
16
+ entityNames: string[];
17
+ /** Number of dependency waves */
18
+ totalWaves: number;
19
+ /** Timestamp when ingestion started */
20
+ startTime: number;
21
+ }
22
+ /**
23
+ * Payload for 'progress' event - emitted periodically during processing
24
+ */
25
+ export interface ProgressEventPayload {
26
+ /** Current wave number (0-indexed) */
27
+ currentWave: number;
28
+ /** Total number of waves */
29
+ totalWaves: number;
30
+ /** Number of entities completed */
31
+ entitiesCompleted: number;
32
+ /** Total number of entities */
33
+ totalEntities: number;
34
+ /** Total rows processed so far */
35
+ rowsProcessed: number;
36
+ /** Number of successful row operations */
37
+ successfulRows: number;
38
+ /** Number of failed row operations */
39
+ failedRows: number;
40
+ /** Progress percentage (0-100) */
41
+ progressPercent: number;
42
+ /** Elapsed time in milliseconds */
43
+ elapsedMs: number;
44
+ }
45
+ /**
46
+ * Payload for 'entityStart' event - emitted when an entity begins processing
47
+ */
48
+ export interface EntityStartEventPayload {
49
+ /** Name of the entity being processed */
50
+ entityName: string;
51
+ /** Path to the entity's mapping configuration */
52
+ mappingPath: string;
53
+ /** Total number of rows to process for this entity */
54
+ totalRows: number;
55
+ /** Wave number this entity belongs to */
56
+ waveIndex: number;
57
+ }
58
+ /**
59
+ * Payload for 'entityComplete' event - emitted when an entity finishes processing
60
+ */
61
+ export interface EntityCompleteEventPayload {
62
+ /** Name of the entity that completed */
63
+ entityName: string;
64
+ /** Metrics for this entity */
65
+ metrics: EntityMetrics;
66
+ /** Whether all rows were successful */
67
+ success: boolean;
68
+ /** Duration in milliseconds */
69
+ durationMs: number;
70
+ }
71
+ /**
72
+ * Payload for 'rowSuccess' event - emitted for each successful row mutation
73
+ */
74
+ export interface RowSuccessEventPayload {
75
+ /** Entity name */
76
+ entityName: string;
77
+ /** Row index (0-indexed) */
78
+ rowIndex: number;
79
+ /** The original row data */
80
+ row: DataRow;
81
+ /** GraphQL mutation result */
82
+ result: unknown;
83
+ /** Duration of the mutation in milliseconds */
84
+ durationMs: number;
85
+ }
86
+ /**
87
+ * Payload for 'rowFailure' event - emitted for each failed row mutation
88
+ */
89
+ export interface RowFailureEventPayload {
90
+ /** Entity name */
91
+ entityName: string;
92
+ /** Row index (0-indexed) */
93
+ rowIndex: number;
94
+ /** The original row data */
95
+ row: DataRow;
96
+ /** Error that occurred */
97
+ error: Error;
98
+ /** Number of retry attempts made */
99
+ retryAttempts: number;
100
+ }
101
+ /**
102
+ * Payload for 'cancelled' event - emitted when processing is cancelled
103
+ */
104
+ export interface CancelledEventPayload {
105
+ /** Reason for cancellation */
106
+ reason: string;
107
+ /** Metrics at time of cancellation */
108
+ metrics: ProcessingMetrics;
109
+ /** Entity being processed when cancelled (if any) */
110
+ currentEntity?: string;
111
+ /** How long processing ran before cancellation */
112
+ elapsedMs: number;
113
+ }
114
+ /**
115
+ * Payload for 'finished' event - emitted when processing completes successfully
116
+ */
117
+ export interface FinishedEventPayload {
118
+ /** Final processing metrics */
119
+ metrics: ProcessingMetrics;
120
+ /** Total duration in milliseconds */
121
+ durationMs: number;
122
+ /** Whether all entities succeeded */
123
+ allSuccessful: boolean;
124
+ }
125
+ /**
126
+ * Payload for 'errored' event - emitted when a fatal error occurs
127
+ */
128
+ export interface ErroredEventPayload {
129
+ /** The error that occurred */
130
+ error: Error;
131
+ /** Metrics at time of error */
132
+ metrics: ProcessingMetrics;
133
+ /** Entity being processed when error occurred (if any) */
134
+ currentEntity?: string;
135
+ /** How long processing ran before error */
136
+ elapsedMs: number;
137
+ }
138
+ /**
139
+ * Map of event types to their payload types
140
+ */
141
+ export interface GQLIngestEventMap {
142
+ started: StartedEventPayload;
143
+ progress: ProgressEventPayload;
144
+ entityStart: EntityStartEventPayload;
145
+ entityComplete: EntityCompleteEventPayload;
146
+ rowSuccess: RowSuccessEventPayload;
147
+ rowFailure: RowFailureEventPayload;
148
+ cancelled: CancelledEventPayload;
149
+ finished: FinishedEventPayload;
150
+ errored: ErroredEventPayload;
151
+ }
152
+ /**
153
+ * Options for configuring event emission behavior
154
+ */
155
+ export interface EventOptions {
156
+ /** Emit rowSuccess/rowFailure events (can be verbose for large datasets). Default: true */
157
+ emitRowEvents?: boolean;
158
+ /** Interval for progress events in milliseconds. Default: 1000 (1 second) */
159
+ progressInterval?: number;
160
+ /** Emit progress events. Default: true */
161
+ emitProgressEvents?: boolean;
162
+ }
163
+ /**
164
+ * Default event options
165
+ */
166
+ export declare const DEFAULT_EVENT_OPTIONS: Required<EventOptions>;
167
+ //# sourceMappingURL=events.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/lib/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,UAAU,GACV,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,EAAE,aAAa,CAAC;IACvB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,WAAW,EAAE,uBAAuB,CAAC;IACrC,cAAc,EAAE,0BAA0B,CAAC;IAC3C,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,qBAAqB,CAAC;IACjC,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,mBAAmB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,YAAY,CAIxD,CAAC"}
@@ -0,0 +1,155 @@
1
+ import { EventEmitter } from "events";
2
+ import { GraphQLClientWrapper } from "./graphql-client";
3
+ import { DataMapper } from "./mapper";
4
+ import { ProcessingMetrics } from "./metrics";
5
+ import { Logger } from "./logger";
6
+ import { EventOptions } from "./events";
7
+ /**
8
+ * Options for initializing GQLIngest client
9
+ */
10
+ export interface GQLIngestOptions {
11
+ /** GraphQL endpoint URL */
12
+ endpoint: string;
13
+ /** Optional headers to include in GraphQL requests */
14
+ headers?: Record<string, string>;
15
+ /** Logger instance. Defaults to silent no-op logger. */
16
+ logger?: Logger;
17
+ /** Override data format detection (csv, json, yaml, jsonl) */
18
+ formatOverride?: string;
19
+ /** Event emission options */
20
+ eventOptions?: EventOptions;
21
+ }
22
+ /**
23
+ * Options for ingesting data
24
+ */
25
+ export interface IngestOptions {
26
+ /** Comma-separated list or array of specific entities to process */
27
+ entities?: string[] | string;
28
+ /** Override data format detection for this operation */
29
+ format?: string;
30
+ /** AbortSignal for external cancellation */
31
+ signal?: AbortSignal;
32
+ }
33
+ /**
34
+ * Result of an ingestion operation
35
+ */
36
+ export interface IngestResult {
37
+ /** Processing metrics */
38
+ metrics: ProcessingMetrics;
39
+ /** Whether the operation was successful */
40
+ success: boolean;
41
+ /** Any errors that occurred */
42
+ errors?: string[];
43
+ /** Whether the operation was cancelled */
44
+ cancelled?: boolean;
45
+ }
46
+ /**
47
+ * Main class for programmatic access to gql-ingest functionality.
48
+ * Extends EventEmitter for progress monitoring and cancellation support.
49
+ */
50
+ export declare class GQLIngest extends EventEmitter {
51
+ private endpoint;
52
+ private headers;
53
+ private logger;
54
+ private formatOverride?;
55
+ private metrics;
56
+ private client;
57
+ private mapper;
58
+ private eventOptions;
59
+ private abortController;
60
+ private isProcessing;
61
+ private startTime;
62
+ private progressIntervalId;
63
+ private currentWave;
64
+ private totalWaves;
65
+ private entitiesCompleted;
66
+ private totalEntities;
67
+ private currentEntity;
68
+ constructor(options: GQLIngestOptions);
69
+ /**
70
+ * Cancel the current ingestion process
71
+ * @param reason Optional reason for cancellation
72
+ */
73
+ cancel(reason?: string): void;
74
+ /**
75
+ * Check if processing is currently in progress
76
+ */
77
+ get processing(): boolean;
78
+ /**
79
+ * Safely emit an event, catching any errors from listeners
80
+ */
81
+ private safeEmit;
82
+ /**
83
+ * Start the progress interval timer
84
+ */
85
+ private startProgressInterval;
86
+ /**
87
+ * Stop the progress interval timer
88
+ */
89
+ private stopProgressInterval;
90
+ /**
91
+ * Emit a progress event with current state
92
+ */
93
+ private emitProgressEvent;
94
+ /**
95
+ * Handle cancellation and emit event
96
+ */
97
+ private handleCancellation;
98
+ /**
99
+ * Combine multiple AbortSignals into one
100
+ */
101
+ private combineSignals;
102
+ /**
103
+ * Ingest data from a configuration directory
104
+ * @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
105
+ * @param options Optional ingestion options
106
+ * @returns Promise with ingestion result
107
+ */
108
+ ingest(configPath: string, options?: IngestOptions): Promise<IngestResult>;
109
+ /**
110
+ * Ingest specific entities from a configuration directory
111
+ * @param configPath Path to configuration directory
112
+ * @param entities Array of entity names to process
113
+ * @returns Promise with ingestion result
114
+ */
115
+ ingestEntities(configPath: string, entities: string[]): Promise<IngestResult>;
116
+ /**
117
+ * Get current processing metrics
118
+ * @returns Current metrics
119
+ */
120
+ getMetrics(): ProcessingMetrics;
121
+ /**
122
+ * Get a summary of the current metrics
123
+ * @returns Formatted summary string
124
+ */
125
+ getMetricsSummary(): string;
126
+ /**
127
+ * Get the GraphQL client for advanced usage
128
+ * @returns GraphQL client wrapper
129
+ */
130
+ getClient(): GraphQLClientWrapper;
131
+ /**
132
+ * Get the data mapper for advanced usage
133
+ * @returns Data mapper
134
+ */
135
+ getMapper(): DataMapper;
136
+ /**
137
+ * Set the logger instance
138
+ * @param logger Logger instance to use
139
+ */
140
+ setLogger(logger: Logger): void;
141
+ /**
142
+ * Update headers for GraphQL requests
143
+ * @param headers New headers to use
144
+ */
145
+ setHeaders(headers: Record<string, string>): void;
146
+ /**
147
+ * Process entities in dependency-aware waves with abort support
148
+ */
149
+ private processEntitiesInWaves;
150
+ /**
151
+ * Utility function to chunk array into smaller arrays
152
+ */
153
+ private chunkArray;
154
+ }
155
+ //# sourceMappingURL=gql-ingest.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gql-ingest.d.ts","sourceRoot":"","sources":["../../src/lib/gql-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAoB,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAGhE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,OAAO,EAEL,YAAY,EAOb,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC7B,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,YAAY;IACzC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAyB;IAG7C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,kBAAkB,CAA+C;IAGzE,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,aAAa,CAAqB;gBAE9B,OAAO,EAAE,gBAAgB;IAoBrC;;;OAGG;IACH,MAAM,CAAC,MAAM,SAAgC,GAAG,IAAI;IAMpD;;OAEG;IACH,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;OAEG;IACH,OAAO,CAAC,QAAQ;IAYhB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAgMhF;;;;;OAKG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAInF;;;OAGG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;OAGG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;;OAGG;IACH,SAAS,IAAI,UAAU;IAIvB;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY/B;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAYjD;;OAEG;YACW,sBAAsB;IA2EpC;;OAEG;IACH,OAAO,CAAC,UAAU;CAQnB"}
@@ -1,14 +1,18 @@
1
1
  import { MetricsCollector } from "./metrics";
2
2
  import { RetryConfig } from "./config";
3
+ import { Logger } from "./logger";
3
4
  export declare class GraphQLClientWrapper {
4
5
  private client;
5
6
  private metrics?;
6
- private verbose;
7
- constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, verbose?: boolean);
8
- executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig): Promise<any>;
7
+ private logger;
8
+ constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, logger?: Logger);
9
+ executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig, signal?: AbortSignal): Promise<any>;
9
10
  private isRetryableError;
10
11
  private calculateDelay;
11
- private sleep;
12
+ /**
13
+ * Sleep that can be interrupted by abort signal
14
+ */
15
+ private sleepWithSignal;
12
16
  setHeaders(headers: Record<string, string>): void;
13
17
  }
14
18
  //# sourceMappingURL=graphql-client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../../src/lib/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAwB,WAAW,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGrB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB;IASvB,eAAe,CACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9B,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,CAAC;IA4Ef,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,cAAc;IAatB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
@@ -0,0 +1,8 @@
1
+ export * from "./logger";
2
+ export * from "./metrics";
3
+ export * from "./config";
4
+ export * from "./dependency-resolver";
5
+ export * from "./graphql-client";
6
+ export * from "./mapper";
7
+ export * from "./gql-ingest";
8
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lib/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,uBAAuB,CAAC;AACtC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC"}
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Logger interface for configurable logging
3
+ *
4
+ * Libraries should be quiet by default unless explicitly configured.
5
+ */
6
+ export interface Logger {
7
+ debug: (message: string, ...args: unknown[]) => void;
8
+ info: (message: string, ...args: unknown[]) => void;
9
+ warn: (message: string, ...args: unknown[]) => void;
10
+ error: (message: string, ...args: unknown[]) => void;
11
+ }
12
+ /**
13
+ * No-op logger that silently discards all log messages.
14
+ * Used as the default logger to keep the library quiet.
15
+ */
16
+ export declare const noopLogger: Logger;
17
+ export interface ConsoleLoggerOptions {
18
+ prefix?: string;
19
+ }
20
+ /**
21
+ * Creates a console logger with optional prefix.
22
+ */
23
+ export declare function createConsoleLogger(options?: ConsoleLoggerOptions): Logger;
24
+ /**
25
+ * Creates a default logger based on the verbose flag.
26
+ * Returns no-op logger by default, console logger when verbose is true.
27
+ */
28
+ export declare function createDefaultLogger(verbose?: boolean): Logger;
29
+ //# sourceMappingURL=logger.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACrD,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACpD,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACpD,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;CACtD;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,CAU9E;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,UAAQ,GAAG,MAAM,CAE3D"}
@@ -0,0 +1,49 @@
1
+ import { GraphQLClientWrapper } from "./graphql-client";
2
+ import { MetricsCollector } from "./metrics";
3
+ import { ParallelProcessingConfig, RetryConfig } from "./config";
4
+ import { Logger } from "./logger";
5
+ import { EntityStartEventPayload, EntityCompleteEventPayload, RowSuccessEventPayload, RowFailureEventPayload } from "./events";
6
+ export interface MappingConfig {
7
+ csvFile?: string;
8
+ dataFile?: string;
9
+ dataFormat?: string;
10
+ graphqlFile: string;
11
+ mapping: Record<string, unknown>;
12
+ }
13
+ /**
14
+ * Callbacks for entity processing events
15
+ */
16
+ export interface EntityProcessingCallbacks {
17
+ onEntityStart?: (payload: Omit<EntityStartEventPayload, "waveIndex">) => void;
18
+ onEntityComplete?: (payload: EntityCompleteEventPayload) => void;
19
+ onRowSuccess?: (payload: RowSuccessEventPayload) => void;
20
+ onRowFailure?: (payload: RowFailureEventPayload) => void;
21
+ }
22
+ export declare class DataMapper {
23
+ private client;
24
+ private basePath;
25
+ private metrics;
26
+ private logger;
27
+ private formatOverride?;
28
+ constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, logger?: Logger, formatOverride?: string);
29
+ discoverMappings(configDir: string, entityFilter?: string[]): string[];
30
+ /**
31
+ * Process an entity (backward-compatible method)
32
+ */
33
+ processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
34
+ /**
35
+ * Process an entity with event callbacks and abort support
36
+ */
37
+ processEntityWithEvents(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig, signal?: AbortSignal, callbacks?: EntityProcessingCallbacks): Promise<void>;
38
+ private processRowsSequentiallyWithEvents;
39
+ private processRowsConcurrentlyWithEvents;
40
+ private chunkArray;
41
+ private mapRowToVariables;
42
+ private getValueByPath;
43
+ private mapNestedObject;
44
+ private extractVariableTypes;
45
+ private extractTypeName;
46
+ private convertValue;
47
+ getMetrics(): MetricsCollector;
48
+ }
49
+ //# sourceMappingURL=mapper.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../../src/lib/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,UAAU,CAAC;AAElB,MAAM,WAAW,aAAa;IAE5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE,WAAW,CAAC,KAAK,IAAI,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACzD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;CAC1D;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB,EAC3B,cAAc,CAAC,EAAE,MAAM;IASzB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAyCtE;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;IAIhB;;OAEG;IACG,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,EACpB,SAAS,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,IAAI,CAAC;YAyFF,iCAAiC;YA+DjC,iCAAiC;IA6G/C,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IAqCzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IAoCvB,OAAO,CAAC,oBAAoB;IA0B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAyDpB,UAAU,IAAI,gBAAgB;CAG/B"}
@@ -1,15 +1,21 @@
1
1
  export interface EntityMetrics {
2
- entityName: string;
3
- successCount: number;
4
- failureCount: number;
5
- startTime: number;
6
- endTime?: number;
2
+ rowsProcessed: number;
3
+ successfulRows: number;
4
+ failedRows: number;
5
+ duration: number;
7
6
  }
8
7
  export interface ProcessingMetrics {
8
+ totalRows: number;
9
+ successfulOperations: number;
10
+ failedOperations: number;
11
+ totalDuration: number;
12
+ entities: Record<string, EntityMetrics>;
13
+ }
14
+ export interface InternalMetrics {
9
15
  totalEntities: number;
10
16
  totalSuccesses: number;
11
17
  totalFailures: number;
12
- entityMetrics: Map<string, EntityMetrics>;
18
+ entityMetrics: Map<string, InternalEntityMetrics>;
13
19
  requestDurations: number[];
14
20
  retryAttempts: number;
15
21
  retrySuccesses: number;
@@ -17,6 +23,13 @@ export interface ProcessingMetrics {
17
23
  startTime: number;
18
24
  endTime?: number;
19
25
  }
26
+ export interface InternalEntityMetrics {
27
+ entityName: string;
28
+ successCount: number;
29
+ failureCount: number;
30
+ startTime: number;
31
+ endTime?: number;
32
+ }
20
33
  export declare class MetricsCollector {
21
34
  private metrics;
22
35
  constructor();
@@ -24,8 +37,9 @@ export declare class MetricsCollector {
24
37
  recordSuccess(entityName: string): void;
25
38
  recordFailure(entityName: string): void;
26
39
  finishEntityProcessing(entityName: string): void;
27
- finishProcessing(): ProcessingMetrics;
28
- getEntityMetrics(entityName: string): EntityMetrics | undefined;
40
+ finishProcessing(): InternalMetrics;
41
+ getEntityMetrics(entityName: string): InternalEntityMetrics | undefined;
42
+ getMetrics(): ProcessingMetrics;
29
43
  getTotalProcessed(): number;
30
44
  getSuccessRate(): number;
31
45
  recordRequestDuration(duration: number): void;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/lib/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAClD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAkB;;IAgBjC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,eAAe;IAKnC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAIvE,UAAU,IAAI,iBAAiB;IAwB/B,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAyC1B"}
@@ -1,8 +1,4 @@
1
1
  import { DataReader, DataRow } from "./data-reader";
2
- export interface CsvRow {
3
- [key: string]: string;
4
- }
5
- export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
6
2
  export declare class CsvReader extends DataReader {
7
3
  getSupportedExtensions(): string[];
8
4
  readFile(filePath: string): Promise<DataRow[]>;