@jackchuka/gql-ingest 2.2.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +259 -32
- package/dist/cli/commands/add.d.ts +3 -0
- package/dist/cli/commands/add.d.ts.map +1 -0
- package/dist/cli/commands/init.d.ts +3 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +288 -0
- package/dist/cli/templates/index.d.ts +14 -0
- package/dist/cli/templates/index.d.ts.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15204 -0
- package/dist/index.js.map +7 -0
- package/dist/lib/config-schema.d.ts +103 -0
- package/dist/lib/config-schema.d.ts.map +1 -0
- package/dist/lib/config.d.ts +9 -0
- package/dist/lib/config.d.ts.map +1 -0
- package/dist/lib/dependency-resolver.d.ts.map +1 -0
- package/dist/lib/events.d.ts +167 -0
- package/dist/lib/events.d.ts.map +1 -0
- package/dist/lib/gql-ingest.d.ts +155 -0
- package/dist/lib/gql-ingest.d.ts.map +1 -0
- package/dist/{graphql-client.d.ts → lib/graphql-client.d.ts} +8 -4
- package/dist/lib/graphql-client.d.ts.map +1 -0
- package/dist/lib/index.d.ts +8 -0
- package/dist/lib/index.d.ts.map +1 -0
- package/dist/lib/logger.d.ts +29 -0
- package/dist/lib/logger.d.ts.map +1 -0
- package/dist/lib/mapper.d.ts +49 -0
- package/dist/lib/mapper.d.ts.map +1 -0
- package/dist/{metrics.d.ts → lib/metrics.d.ts} +22 -8
- package/dist/lib/metrics.d.ts.map +1 -0
- package/dist/readers/csv.d.ts +0 -4
- package/dist/readers/csv.d.ts.map +1 -1
- package/dist/readers/index.d.ts +1 -1
- package/dist/readers/index.d.ts.map +1 -1
- package/package.json +19 -7
- package/dist/cli.d.ts +0 -2
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -237
- package/dist/config.d.ts +0 -32
- package/dist/config.d.ts.map +0 -1
- package/dist/config.test.d.ts +0 -2
- package/dist/config.test.d.ts.map +0 -1
- package/dist/dependency-resolver.d.ts.map +0 -1
- package/dist/dependency-resolver.test.d.ts +0 -2
- package/dist/dependency-resolver.test.d.ts.map +0 -1
- package/dist/graphql-client.d.ts.map +0 -1
- package/dist/graphql-client.test.d.ts +0 -2
- package/dist/graphql-client.test.d.ts.map +0 -1
- package/dist/mapper.d.ts +0 -31
- package/dist/mapper.d.ts.map +0 -1
- package/dist/mapper.test.d.ts +0 -2
- package/dist/mapper.test.d.ts.map +0 -1
- package/dist/metrics.d.ts.map +0 -1
- package/dist/metrics.test.d.ts +0 -2
- package/dist/metrics.test.d.ts.map +0 -1
- package/dist/readers/csv.test.d.ts +0 -2
- package/dist/readers/csv.test.d.ts.map +0 -1
- package/dist/readers/data-reader.test.d.ts +0 -2
- package/dist/readers/data-reader.test.d.ts.map +0 -1
- package/dist/readers/json.test.d.ts +0 -2
- package/dist/readers/json.test.d.ts.map +0 -1
- package/dist/readers/jsonl.test.d.ts +0 -2
- package/dist/readers/jsonl.test.d.ts.map +0 -1
- package/dist/readers/yaml.test.d.ts +0 -2
- package/dist/readers/yaml.test.d.ts.map +0 -1
- /package/dist/{dependency-resolver.d.ts → lib/dependency-resolver.d.ts} +0 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { z } from "zod/v4";
|
|
2
|
+
export declare const RetryConfigSchema: z.ZodObject<{
|
|
3
|
+
maxAttempts: z.ZodNumber;
|
|
4
|
+
baseDelay: z.ZodNumber;
|
|
5
|
+
maxDelay: z.ZodNumber;
|
|
6
|
+
exponentialBackoff: z.ZodBoolean;
|
|
7
|
+
retryableStatusCodes: z.ZodArray<z.ZodNumber>;
|
|
8
|
+
}, z.core.$strip>;
|
|
9
|
+
export declare const ParallelProcessingConfigSchema: z.ZodObject<{
|
|
10
|
+
concurrency: z.ZodNumber;
|
|
11
|
+
entityConcurrency: z.ZodNumber;
|
|
12
|
+
preserveRowOrder: z.ZodBoolean;
|
|
13
|
+
}, z.core.$strip>;
|
|
14
|
+
export declare const EntityConfigSchema: z.ZodObject<{
|
|
15
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
16
|
+
preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
|
|
17
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
18
|
+
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
19
|
+
baseDelay: z.ZodOptional<z.ZodNumber>;
|
|
20
|
+
maxDelay: z.ZodOptional<z.ZodNumber>;
|
|
21
|
+
exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
|
|
22
|
+
retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
23
|
+
}, z.core.$strip>>;
|
|
24
|
+
}, z.core.$strip>;
|
|
25
|
+
export declare const ConfigSchema: z.ZodObject<{
|
|
26
|
+
retry: z.ZodObject<{
|
|
27
|
+
maxAttempts: z.ZodNumber;
|
|
28
|
+
baseDelay: z.ZodNumber;
|
|
29
|
+
maxDelay: z.ZodNumber;
|
|
30
|
+
exponentialBackoff: z.ZodBoolean;
|
|
31
|
+
retryableStatusCodes: z.ZodArray<z.ZodNumber>;
|
|
32
|
+
}, z.core.$strip>;
|
|
33
|
+
parallelProcessing: z.ZodObject<{
|
|
34
|
+
concurrency: z.ZodNumber;
|
|
35
|
+
entityConcurrency: z.ZodNumber;
|
|
36
|
+
preserveRowOrder: z.ZodBoolean;
|
|
37
|
+
}, z.core.$strip>;
|
|
38
|
+
entityConfig: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
39
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
40
|
+
preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
|
|
41
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
42
|
+
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
43
|
+
baseDelay: z.ZodOptional<z.ZodNumber>;
|
|
44
|
+
maxDelay: z.ZodOptional<z.ZodNumber>;
|
|
45
|
+
exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
|
|
46
|
+
retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
47
|
+
}, z.core.$strip>>;
|
|
48
|
+
}, z.core.$strip>>;
|
|
49
|
+
entityDependencies: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>;
|
|
50
|
+
}, z.core.$strip>;
|
|
51
|
+
export type RetryConfig = z.infer<typeof RetryConfigSchema>;
|
|
52
|
+
export type ParallelProcessingConfig = z.infer<typeof ParallelProcessingConfigSchema>;
|
|
53
|
+
export type EntityConfig = z.infer<typeof EntityConfigSchema>;
|
|
54
|
+
export type Config = z.infer<typeof ConfigSchema>;
|
|
55
|
+
export declare const DEFAULT_RETRY_CONFIG: RetryConfig;
|
|
56
|
+
export declare const DEFAULT_PARALLEL_CONFIG: ParallelProcessingConfig;
|
|
57
|
+
export declare const CONFIG_TEMPLATE: {
|
|
58
|
+
schema: {
|
|
59
|
+
parallelProcessing: z.ZodObject<{
|
|
60
|
+
concurrency: z.ZodNumber;
|
|
61
|
+
entityConcurrency: z.ZodNumber;
|
|
62
|
+
preserveRowOrder: z.ZodBoolean;
|
|
63
|
+
}, z.core.$strip>;
|
|
64
|
+
retry: z.ZodObject<{
|
|
65
|
+
maxAttempts: z.ZodNumber;
|
|
66
|
+
baseDelay: z.ZodNumber;
|
|
67
|
+
maxDelay: z.ZodNumber;
|
|
68
|
+
exponentialBackoff: z.ZodBoolean;
|
|
69
|
+
retryableStatusCodes: z.ZodArray<z.ZodNumber>;
|
|
70
|
+
}, z.core.$strip>;
|
|
71
|
+
entityConfig: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
72
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
preserveRowOrder: z.ZodOptional<z.ZodBoolean>;
|
|
74
|
+
retry: z.ZodOptional<z.ZodObject<{
|
|
75
|
+
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
76
|
+
baseDelay: z.ZodOptional<z.ZodNumber>;
|
|
77
|
+
maxDelay: z.ZodOptional<z.ZodNumber>;
|
|
78
|
+
exponentialBackoff: z.ZodOptional<z.ZodBoolean>;
|
|
79
|
+
retryableStatusCodes: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
|
|
80
|
+
}, z.core.$strip>>;
|
|
81
|
+
}, z.core.$strip>>;
|
|
82
|
+
entityDependencies: z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>;
|
|
83
|
+
};
|
|
84
|
+
defaults: {
|
|
85
|
+
parallelProcessing: {
|
|
86
|
+
concurrency: number;
|
|
87
|
+
entityConcurrency: number;
|
|
88
|
+
preserveRowOrder: boolean;
|
|
89
|
+
};
|
|
90
|
+
retry: {
|
|
91
|
+
maxAttempts: number;
|
|
92
|
+
baseDelay: number;
|
|
93
|
+
maxDelay: number;
|
|
94
|
+
exponentialBackoff: boolean;
|
|
95
|
+
retryableStatusCodes: number[];
|
|
96
|
+
};
|
|
97
|
+
};
|
|
98
|
+
examples: {
|
|
99
|
+
entityConfig: Record<string, EntityConfig>;
|
|
100
|
+
entityDependencies: Record<string, string[]>;
|
|
101
|
+
};
|
|
102
|
+
};
|
|
103
|
+
//# sourceMappingURL=config-schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-schema.d.ts","sourceRoot":"","sources":["../../src/lib/config-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAE3B,eAAO,MAAM,iBAAiB;;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,8BAA8B;;;;iBAIzC,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;iBAI7B,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;iBASvB,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAGlD,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,wBAIrC,CAAC;AAGF,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAkBnB,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC;4BAG5B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;;CAEhC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Logger } from "./logger";
|
|
2
|
+
import { type Config, type ParallelProcessingConfig, type RetryConfig } from "./config-schema";
|
|
3
|
+
export type { Config, ParallelProcessingConfig, RetryConfig, EntityConfig } from "./config-schema";
|
|
4
|
+
export { DEFAULT_RETRY_CONFIG, DEFAULT_PARALLEL_CONFIG } from "./config-schema";
|
|
5
|
+
export declare const DEFAULT_CONFIG: Config;
|
|
6
|
+
export declare function loadConfig(configDir: string, logger?: Logger): Config;
|
|
7
|
+
export declare function getEntityConfig(entityName: string, globalConfig: Config, logger?: Logger): ParallelProcessingConfig;
|
|
8
|
+
export declare function getRetryConfig(entityName: string, globalConfig: Config): RetryConfig;
|
|
9
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EAGL,KAAK,MAAM,EACX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EAAE,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEnG,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAEhF,eAAO,MAAM,cAAc,EAAE,MAK5B,CAAC;AAEF,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,GAAE,MAAmB,GAAG,MAAM,CAkBjF;AAiBD,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,GAAE,MAAmB,GAC1B,wBAAwB,CAiB1B;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,WAAW,CAOpF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dependency-resolver.d.ts","sourceRoot":"","sources":["../../src/lib/dependency-resolver.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,sBAAsB,CAAU;gBAGtC,QAAQ,EAAE,MAAM,EAAE,EAClB,YAAY,GAAE,eAAoB,EAClC,sBAAsB,GAAE,OAAe;IAOzC,qBAAqB,IAAI,aAAa,EAAE;IAiDxC,oBAAoB,IAAI,MAAM,EAAE;IAoBhC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;IAM3C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAG9C"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { ProcessingMetrics, EntityMetrics } from "./metrics";
|
|
2
|
+
import { DataRow } from "../readers";
|
|
3
|
+
/**
|
|
4
|
+
* Event types emitted by GQLIngest during processing
|
|
5
|
+
*/
|
|
6
|
+
export type GQLIngestEventType = "started" | "progress" | "entityStart" | "entityComplete" | "rowSuccess" | "rowFailure" | "cancelled" | "finished" | "errored";
|
|
7
|
+
/**
|
|
8
|
+
* Payload for 'started' event - emitted when ingestion begins
|
|
9
|
+
*/
|
|
10
|
+
export interface StartedEventPayload {
|
|
11
|
+
/** Path to configuration directory */
|
|
12
|
+
configPath: string;
|
|
13
|
+
/** Total number of entities to process */
|
|
14
|
+
totalEntities: number;
|
|
15
|
+
/** Names of entities that will be processed */
|
|
16
|
+
entityNames: string[];
|
|
17
|
+
/** Number of dependency waves */
|
|
18
|
+
totalWaves: number;
|
|
19
|
+
/** Timestamp when ingestion started */
|
|
20
|
+
startTime: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Payload for 'progress' event - emitted periodically during processing
|
|
24
|
+
*/
|
|
25
|
+
export interface ProgressEventPayload {
|
|
26
|
+
/** Current wave number (0-indexed) */
|
|
27
|
+
currentWave: number;
|
|
28
|
+
/** Total number of waves */
|
|
29
|
+
totalWaves: number;
|
|
30
|
+
/** Number of entities completed */
|
|
31
|
+
entitiesCompleted: number;
|
|
32
|
+
/** Total number of entities */
|
|
33
|
+
totalEntities: number;
|
|
34
|
+
/** Total rows processed so far */
|
|
35
|
+
rowsProcessed: number;
|
|
36
|
+
/** Number of successful row operations */
|
|
37
|
+
successfulRows: number;
|
|
38
|
+
/** Number of failed row operations */
|
|
39
|
+
failedRows: number;
|
|
40
|
+
/** Progress percentage (0-100) */
|
|
41
|
+
progressPercent: number;
|
|
42
|
+
/** Elapsed time in milliseconds */
|
|
43
|
+
elapsedMs: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Payload for 'entityStart' event - emitted when an entity begins processing
|
|
47
|
+
*/
|
|
48
|
+
export interface EntityStartEventPayload {
|
|
49
|
+
/** Name of the entity being processed */
|
|
50
|
+
entityName: string;
|
|
51
|
+
/** Path to the entity's mapping configuration */
|
|
52
|
+
mappingPath: string;
|
|
53
|
+
/** Total number of rows to process for this entity */
|
|
54
|
+
totalRows: number;
|
|
55
|
+
/** Wave number this entity belongs to */
|
|
56
|
+
waveIndex: number;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Payload for 'entityComplete' event - emitted when an entity finishes processing
|
|
60
|
+
*/
|
|
61
|
+
export interface EntityCompleteEventPayload {
|
|
62
|
+
/** Name of the entity that completed */
|
|
63
|
+
entityName: string;
|
|
64
|
+
/** Metrics for this entity */
|
|
65
|
+
metrics: EntityMetrics;
|
|
66
|
+
/** Whether all rows were successful */
|
|
67
|
+
success: boolean;
|
|
68
|
+
/** Duration in milliseconds */
|
|
69
|
+
durationMs: number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Payload for 'rowSuccess' event - emitted for each successful row mutation
|
|
73
|
+
*/
|
|
74
|
+
export interface RowSuccessEventPayload {
|
|
75
|
+
/** Entity name */
|
|
76
|
+
entityName: string;
|
|
77
|
+
/** Row index (0-indexed) */
|
|
78
|
+
rowIndex: number;
|
|
79
|
+
/** The original row data */
|
|
80
|
+
row: DataRow;
|
|
81
|
+
/** GraphQL mutation result */
|
|
82
|
+
result: unknown;
|
|
83
|
+
/** Duration of the mutation in milliseconds */
|
|
84
|
+
durationMs: number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Payload for 'rowFailure' event - emitted for each failed row mutation
|
|
88
|
+
*/
|
|
89
|
+
export interface RowFailureEventPayload {
|
|
90
|
+
/** Entity name */
|
|
91
|
+
entityName: string;
|
|
92
|
+
/** Row index (0-indexed) */
|
|
93
|
+
rowIndex: number;
|
|
94
|
+
/** The original row data */
|
|
95
|
+
row: DataRow;
|
|
96
|
+
/** Error that occurred */
|
|
97
|
+
error: Error;
|
|
98
|
+
/** Number of retry attempts made */
|
|
99
|
+
retryAttempts: number;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Payload for 'cancelled' event - emitted when processing is cancelled
|
|
103
|
+
*/
|
|
104
|
+
export interface CancelledEventPayload {
|
|
105
|
+
/** Reason for cancellation */
|
|
106
|
+
reason: string;
|
|
107
|
+
/** Metrics at time of cancellation */
|
|
108
|
+
metrics: ProcessingMetrics;
|
|
109
|
+
/** Entity being processed when cancelled (if any) */
|
|
110
|
+
currentEntity?: string;
|
|
111
|
+
/** How long processing ran before cancellation */
|
|
112
|
+
elapsedMs: number;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Payload for 'finished' event - emitted when processing completes successfully
|
|
116
|
+
*/
|
|
117
|
+
export interface FinishedEventPayload {
|
|
118
|
+
/** Final processing metrics */
|
|
119
|
+
metrics: ProcessingMetrics;
|
|
120
|
+
/** Total duration in milliseconds */
|
|
121
|
+
durationMs: number;
|
|
122
|
+
/** Whether all entities succeeded */
|
|
123
|
+
allSuccessful: boolean;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Payload for 'errored' event - emitted when a fatal error occurs
|
|
127
|
+
*/
|
|
128
|
+
export interface ErroredEventPayload {
|
|
129
|
+
/** The error that occurred */
|
|
130
|
+
error: Error;
|
|
131
|
+
/** Metrics at time of error */
|
|
132
|
+
metrics: ProcessingMetrics;
|
|
133
|
+
/** Entity being processed when error occurred (if any) */
|
|
134
|
+
currentEntity?: string;
|
|
135
|
+
/** How long processing ran before error */
|
|
136
|
+
elapsedMs: number;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Map of event types to their payload types
|
|
140
|
+
*/
|
|
141
|
+
export interface GQLIngestEventMap {
|
|
142
|
+
started: StartedEventPayload;
|
|
143
|
+
progress: ProgressEventPayload;
|
|
144
|
+
entityStart: EntityStartEventPayload;
|
|
145
|
+
entityComplete: EntityCompleteEventPayload;
|
|
146
|
+
rowSuccess: RowSuccessEventPayload;
|
|
147
|
+
rowFailure: RowFailureEventPayload;
|
|
148
|
+
cancelled: CancelledEventPayload;
|
|
149
|
+
finished: FinishedEventPayload;
|
|
150
|
+
errored: ErroredEventPayload;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Options for configuring event emission behavior
|
|
154
|
+
*/
|
|
155
|
+
export interface EventOptions {
|
|
156
|
+
/** Emit rowSuccess/rowFailure events (can be verbose for large datasets). Default: true */
|
|
157
|
+
emitRowEvents?: boolean;
|
|
158
|
+
/** Interval for progress events in milliseconds. Default: 1000 (1 second) */
|
|
159
|
+
progressInterval?: number;
|
|
160
|
+
/** Emit progress events. Default: true */
|
|
161
|
+
emitProgressEvents?: boolean;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Default event options
|
|
165
|
+
*/
|
|
166
|
+
export declare const DEFAULT_EVENT_OPTIONS: Required<EventOptions>;
|
|
167
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/lib/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,SAAS,GACT,UAAU,GACV,aAAa,GACb,gBAAgB,GAChB,YAAY,GACZ,YAAY,GACZ,WAAW,GACX,UAAU,GACV,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,EAAE,aAAa,CAAC;IACvB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,GAAG,EAAE,OAAO,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,WAAW,EAAE,uBAAuB,CAAC;IACrC,cAAc,EAAE,0BAA0B,CAAC;IAC3C,UAAU,EAAE,sBAAsB,CAAC;IACnC,UAAU,EAAE,sBAAsB,CAAC;IACnC,SAAS,EAAE,qBAAqB,CAAC;IACjC,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,mBAAmB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,QAAQ,CAAC,YAAY,CAIxD,CAAC"}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { EventEmitter } from "events";
|
|
2
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
3
|
+
import { DataMapper } from "./mapper";
|
|
4
|
+
import { ProcessingMetrics } from "./metrics";
|
|
5
|
+
import { Logger } from "./logger";
|
|
6
|
+
import { EventOptions } from "./events";
|
|
7
|
+
/**
|
|
8
|
+
* Options for initializing GQLIngest client
|
|
9
|
+
*/
|
|
10
|
+
export interface GQLIngestOptions {
|
|
11
|
+
/** GraphQL endpoint URL */
|
|
12
|
+
endpoint: string;
|
|
13
|
+
/** Optional headers to include in GraphQL requests */
|
|
14
|
+
headers?: Record<string, string>;
|
|
15
|
+
/** Logger instance. Defaults to silent no-op logger. */
|
|
16
|
+
logger?: Logger;
|
|
17
|
+
/** Override data format detection (csv, json, yaml, jsonl) */
|
|
18
|
+
formatOverride?: string;
|
|
19
|
+
/** Event emission options */
|
|
20
|
+
eventOptions?: EventOptions;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Options for ingesting data
|
|
24
|
+
*/
|
|
25
|
+
export interface IngestOptions {
|
|
26
|
+
/** Comma-separated list or array of specific entities to process */
|
|
27
|
+
entities?: string[] | string;
|
|
28
|
+
/** Override data format detection for this operation */
|
|
29
|
+
format?: string;
|
|
30
|
+
/** AbortSignal for external cancellation */
|
|
31
|
+
signal?: AbortSignal;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Result of an ingestion operation
|
|
35
|
+
*/
|
|
36
|
+
export interface IngestResult {
|
|
37
|
+
/** Processing metrics */
|
|
38
|
+
metrics: ProcessingMetrics;
|
|
39
|
+
/** Whether the operation was successful */
|
|
40
|
+
success: boolean;
|
|
41
|
+
/** Any errors that occurred */
|
|
42
|
+
errors?: string[];
|
|
43
|
+
/** Whether the operation was cancelled */
|
|
44
|
+
cancelled?: boolean;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Main class for programmatic access to gql-ingest functionality.
|
|
48
|
+
* Extends EventEmitter for progress monitoring and cancellation support.
|
|
49
|
+
*/
|
|
50
|
+
export declare class GQLIngest extends EventEmitter {
|
|
51
|
+
private endpoint;
|
|
52
|
+
private headers;
|
|
53
|
+
private logger;
|
|
54
|
+
private formatOverride?;
|
|
55
|
+
private metrics;
|
|
56
|
+
private client;
|
|
57
|
+
private mapper;
|
|
58
|
+
private eventOptions;
|
|
59
|
+
private abortController;
|
|
60
|
+
private isProcessing;
|
|
61
|
+
private startTime;
|
|
62
|
+
private progressIntervalId;
|
|
63
|
+
private currentWave;
|
|
64
|
+
private totalWaves;
|
|
65
|
+
private entitiesCompleted;
|
|
66
|
+
private totalEntities;
|
|
67
|
+
private currentEntity;
|
|
68
|
+
constructor(options: GQLIngestOptions);
|
|
69
|
+
/**
|
|
70
|
+
* Cancel the current ingestion process
|
|
71
|
+
* @param reason Optional reason for cancellation
|
|
72
|
+
*/
|
|
73
|
+
cancel(reason?: string): void;
|
|
74
|
+
/**
|
|
75
|
+
* Check if processing is currently in progress
|
|
76
|
+
*/
|
|
77
|
+
get processing(): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Safely emit an event, catching any errors from listeners
|
|
80
|
+
*/
|
|
81
|
+
private safeEmit;
|
|
82
|
+
/**
|
|
83
|
+
* Start the progress interval timer
|
|
84
|
+
*/
|
|
85
|
+
private startProgressInterval;
|
|
86
|
+
/**
|
|
87
|
+
* Stop the progress interval timer
|
|
88
|
+
*/
|
|
89
|
+
private stopProgressInterval;
|
|
90
|
+
/**
|
|
91
|
+
* Emit a progress event with current state
|
|
92
|
+
*/
|
|
93
|
+
private emitProgressEvent;
|
|
94
|
+
/**
|
|
95
|
+
* Handle cancellation and emit event
|
|
96
|
+
*/
|
|
97
|
+
private handleCancellation;
|
|
98
|
+
/**
|
|
99
|
+
* Combine multiple AbortSignals into one
|
|
100
|
+
*/
|
|
101
|
+
private combineSignals;
|
|
102
|
+
/**
|
|
103
|
+
* Ingest data from a configuration directory
|
|
104
|
+
* @param configPath Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)
|
|
105
|
+
* @param options Optional ingestion options
|
|
106
|
+
* @returns Promise with ingestion result
|
|
107
|
+
*/
|
|
108
|
+
ingest(configPath: string, options?: IngestOptions): Promise<IngestResult>;
|
|
109
|
+
/**
|
|
110
|
+
* Ingest specific entities from a configuration directory
|
|
111
|
+
* @param configPath Path to configuration directory
|
|
112
|
+
* @param entities Array of entity names to process
|
|
113
|
+
* @returns Promise with ingestion result
|
|
114
|
+
*/
|
|
115
|
+
ingestEntities(configPath: string, entities: string[]): Promise<IngestResult>;
|
|
116
|
+
/**
|
|
117
|
+
* Get current processing metrics
|
|
118
|
+
* @returns Current metrics
|
|
119
|
+
*/
|
|
120
|
+
getMetrics(): ProcessingMetrics;
|
|
121
|
+
/**
|
|
122
|
+
* Get a summary of the current metrics
|
|
123
|
+
* @returns Formatted summary string
|
|
124
|
+
*/
|
|
125
|
+
getMetricsSummary(): string;
|
|
126
|
+
/**
|
|
127
|
+
* Get the GraphQL client for advanced usage
|
|
128
|
+
* @returns GraphQL client wrapper
|
|
129
|
+
*/
|
|
130
|
+
getClient(): GraphQLClientWrapper;
|
|
131
|
+
/**
|
|
132
|
+
* Get the data mapper for advanced usage
|
|
133
|
+
* @returns Data mapper
|
|
134
|
+
*/
|
|
135
|
+
getMapper(): DataMapper;
|
|
136
|
+
/**
|
|
137
|
+
* Set the logger instance
|
|
138
|
+
* @param logger Logger instance to use
|
|
139
|
+
*/
|
|
140
|
+
setLogger(logger: Logger): void;
|
|
141
|
+
/**
|
|
142
|
+
* Update headers for GraphQL requests
|
|
143
|
+
* @param headers New headers to use
|
|
144
|
+
*/
|
|
145
|
+
setHeaders(headers: Record<string, string>): void;
|
|
146
|
+
/**
|
|
147
|
+
* Process entities in dependency-aware waves with abort support
|
|
148
|
+
*/
|
|
149
|
+
private processEntitiesInWaves;
|
|
150
|
+
/**
|
|
151
|
+
* Utility function to chunk array into smaller arrays
|
|
152
|
+
*/
|
|
153
|
+
private chunkArray;
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=gql-ingest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gql-ingest.d.ts","sourceRoot":"","sources":["../../src/lib/gql-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAoB,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAGhE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,OAAO,EAEL,YAAY,EAOb,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC7B,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yBAAyB;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,SAAU,SAAQ,YAAY;IACzC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAyB;IAG7C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,kBAAkB,CAA+C;IAGzE,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,aAAa,CAAqB;gBAE9B,OAAO,EAAE,gBAAgB;IAoBrC;;;OAGG;IACH,MAAM,CAAC,MAAM,SAAgC,GAAG,IAAI;IAMpD;;OAEG;IACH,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;OAEG;IACH,OAAO,CAAC,QAAQ;IAYhB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;OAKG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAgMhF;;;;;OAKG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAInF;;;OAGG;IACH,UAAU,IAAI,iBAAiB;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;;OAGG;IACH,SAAS,IAAI,oBAAoB;IAIjC;;;OAGG;IACH,SAAS,IAAI,UAAU;IAIvB;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAY/B;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAYjD;;OAEG;YACW,sBAAsB;IA2EpC;;OAEG;IACH,OAAO,CAAC,UAAU;CAQnB"}
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import { MetricsCollector } from "./metrics";
|
|
2
2
|
import { RetryConfig } from "./config";
|
|
3
|
+
import { Logger } from "./logger";
|
|
3
4
|
export declare class GraphQLClientWrapper {
|
|
4
5
|
private client;
|
|
5
6
|
private metrics?;
|
|
6
|
-
private
|
|
7
|
-
constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector,
|
|
8
|
-
executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig): Promise<any>;
|
|
7
|
+
private logger;
|
|
8
|
+
constructor(endpoint: string, headers?: Record<string, string>, metrics?: MetricsCollector, logger?: Logger);
|
|
9
|
+
executeMutation(mutation: string, variables: Record<string, any>, retryConfig?: RetryConfig, signal?: AbortSignal): Promise<any>;
|
|
9
10
|
private isRetryableError;
|
|
10
11
|
private calculateDelay;
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Sleep that can be interrupted by abort signal
|
|
14
|
+
*/
|
|
15
|
+
private sleepWithSignal;
|
|
12
16
|
setHeaders(headers: Record<string, string>): void;
|
|
13
17
|
}
|
|
14
18
|
//# sourceMappingURL=graphql-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../../src/lib/graphql-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAwB,WAAW,EAAE,MAAM,UAAU,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAE9C,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGrB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB;IASvB,eAAe,CACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9B,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,CAAC;IA4Ef,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,cAAc;IAatB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lib/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,uBAAuB,CAAC;AACtC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logger interface for configurable logging
|
|
3
|
+
*
|
|
4
|
+
* Libraries should be quiet by default unless explicitly configured.
|
|
5
|
+
*/
|
|
6
|
+
export interface Logger {
|
|
7
|
+
debug: (message: string, ...args: unknown[]) => void;
|
|
8
|
+
info: (message: string, ...args: unknown[]) => void;
|
|
9
|
+
warn: (message: string, ...args: unknown[]) => void;
|
|
10
|
+
error: (message: string, ...args: unknown[]) => void;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* No-op logger that silently discards all log messages.
|
|
14
|
+
* Used as the default logger to keep the library quiet.
|
|
15
|
+
*/
|
|
16
|
+
export declare const noopLogger: Logger;
|
|
17
|
+
export interface ConsoleLoggerOptions {
|
|
18
|
+
prefix?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Creates a console logger with optional prefix.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createConsoleLogger(options?: ConsoleLoggerOptions): Logger;
|
|
24
|
+
/**
|
|
25
|
+
* Creates a default logger based on the verbose flag.
|
|
26
|
+
* Returns no-op logger by default, console logger when verbose is true.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createDefaultLogger(verbose?: boolean): Logger;
|
|
29
|
+
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACrD,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACpD,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IACpD,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;CACtD;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,CAU9E;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,UAAQ,GAAG,MAAM,CAE3D"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
2
|
+
import { MetricsCollector } from "./metrics";
|
|
3
|
+
import { ParallelProcessingConfig, RetryConfig } from "./config";
|
|
4
|
+
import { Logger } from "./logger";
|
|
5
|
+
import { EntityStartEventPayload, EntityCompleteEventPayload, RowSuccessEventPayload, RowFailureEventPayload } from "./events";
|
|
6
|
+
export interface MappingConfig {
|
|
7
|
+
csvFile?: string;
|
|
8
|
+
dataFile?: string;
|
|
9
|
+
dataFormat?: string;
|
|
10
|
+
graphqlFile: string;
|
|
11
|
+
mapping: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Callbacks for entity processing events
|
|
15
|
+
*/
|
|
16
|
+
export interface EntityProcessingCallbacks {
|
|
17
|
+
onEntityStart?: (payload: Omit<EntityStartEventPayload, "waveIndex">) => void;
|
|
18
|
+
onEntityComplete?: (payload: EntityCompleteEventPayload) => void;
|
|
19
|
+
onRowSuccess?: (payload: RowSuccessEventPayload) => void;
|
|
20
|
+
onRowFailure?: (payload: RowFailureEventPayload) => void;
|
|
21
|
+
}
|
|
22
|
+
export declare class DataMapper {
|
|
23
|
+
private client;
|
|
24
|
+
private basePath;
|
|
25
|
+
private metrics;
|
|
26
|
+
private logger;
|
|
27
|
+
private formatOverride?;
|
|
28
|
+
constructor(client: GraphQLClientWrapper, basePath?: string, metrics?: MetricsCollector, logger?: Logger, formatOverride?: string);
|
|
29
|
+
discoverMappings(configDir: string, entityFilter?: string[]): string[];
|
|
30
|
+
/**
|
|
31
|
+
* Process an entity (backward-compatible method)
|
|
32
|
+
*/
|
|
33
|
+
processEntity(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig): Promise<void>;
|
|
34
|
+
/**
|
|
35
|
+
* Process an entity with event callbacks and abort support
|
|
36
|
+
*/
|
|
37
|
+
processEntityWithEvents(configPath: string, parallelConfig?: ParallelProcessingConfig, retryConfig?: RetryConfig, signal?: AbortSignal, callbacks?: EntityProcessingCallbacks): Promise<void>;
|
|
38
|
+
private processRowsSequentiallyWithEvents;
|
|
39
|
+
private processRowsConcurrentlyWithEvents;
|
|
40
|
+
private chunkArray;
|
|
41
|
+
private mapRowToVariables;
|
|
42
|
+
private getValueByPath;
|
|
43
|
+
private mapNestedObject;
|
|
44
|
+
private extractVariableTypes;
|
|
45
|
+
private extractTypeName;
|
|
46
|
+
private convertValue;
|
|
47
|
+
getMetrics(): MetricsCollector;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=mapper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../../src/lib/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,MAAM,EAAc,MAAM,UAAU,CAAC;AAC9C,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACvB,MAAM,UAAU,CAAC;AAElB,MAAM,WAAW,aAAa;IAE5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE,WAAW,CAAC,KAAK,IAAI,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACzD,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,IAAI,CAAC;CAC1D;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAC,CAAS;gBAG9B,MAAM,EAAE,oBAAoB,EAC5B,QAAQ,GAAE,MAAsB,EAChC,OAAO,CAAC,EAAE,gBAAgB,EAC1B,MAAM,GAAE,MAAmB,EAC3B,cAAc,CAAC,EAAE,MAAM;IASzB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAyCtE;;OAEG;IACG,aAAa,CACjB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,GACxB,OAAO,CAAC,IAAI,CAAC;IAIhB;;OAEG;IACG,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,wBAAwB,EACzC,WAAW,CAAC,EAAE,WAAW,EACzB,MAAM,CAAC,EAAE,WAAW,EACpB,SAAS,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,IAAI,CAAC;YAyFF,iCAAiC;YA+DjC,iCAAiC;IA6G/C,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,iBAAiB;IAqCzB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IAoCvB,OAAO,CAAC,oBAAoB;IA0B5B,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,YAAY;IAyDpB,UAAU,IAAI,gBAAgB;CAG/B"}
|
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
export interface EntityMetrics {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
endTime?: number;
|
|
2
|
+
rowsProcessed: number;
|
|
3
|
+
successfulRows: number;
|
|
4
|
+
failedRows: number;
|
|
5
|
+
duration: number;
|
|
7
6
|
}
|
|
8
7
|
export interface ProcessingMetrics {
|
|
8
|
+
totalRows: number;
|
|
9
|
+
successfulOperations: number;
|
|
10
|
+
failedOperations: number;
|
|
11
|
+
totalDuration: number;
|
|
12
|
+
entities: Record<string, EntityMetrics>;
|
|
13
|
+
}
|
|
14
|
+
export interface InternalMetrics {
|
|
9
15
|
totalEntities: number;
|
|
10
16
|
totalSuccesses: number;
|
|
11
17
|
totalFailures: number;
|
|
12
|
-
entityMetrics: Map<string,
|
|
18
|
+
entityMetrics: Map<string, InternalEntityMetrics>;
|
|
13
19
|
requestDurations: number[];
|
|
14
20
|
retryAttempts: number;
|
|
15
21
|
retrySuccesses: number;
|
|
@@ -17,6 +23,13 @@ export interface ProcessingMetrics {
|
|
|
17
23
|
startTime: number;
|
|
18
24
|
endTime?: number;
|
|
19
25
|
}
|
|
26
|
+
export interface InternalEntityMetrics {
|
|
27
|
+
entityName: string;
|
|
28
|
+
successCount: number;
|
|
29
|
+
failureCount: number;
|
|
30
|
+
startTime: number;
|
|
31
|
+
endTime?: number;
|
|
32
|
+
}
|
|
20
33
|
export declare class MetricsCollector {
|
|
21
34
|
private metrics;
|
|
22
35
|
constructor();
|
|
@@ -24,8 +37,9 @@ export declare class MetricsCollector {
|
|
|
24
37
|
recordSuccess(entityName: string): void;
|
|
25
38
|
recordFailure(entityName: string): void;
|
|
26
39
|
finishEntityProcessing(entityName: string): void;
|
|
27
|
-
finishProcessing():
|
|
28
|
-
getEntityMetrics(entityName: string):
|
|
40
|
+
finishProcessing(): InternalMetrics;
|
|
41
|
+
getEntityMetrics(entityName: string): InternalEntityMetrics | undefined;
|
|
42
|
+
getMetrics(): ProcessingMetrics;
|
|
29
43
|
getTotalProcessed(): number;
|
|
30
44
|
getSuccessRate(): number;
|
|
31
45
|
recordRequestDuration(duration: number): void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/lib/metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAClD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAkB;;IAgBjC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/C,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IASvC,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhD,gBAAgB,IAAI,eAAe;IAKnC,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAIvE,UAAU,IAAI,iBAAiB;IAwB/B,iBAAiB,IAAI,MAAM;IAI3B,cAAc,IAAI,MAAM;IAKxB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAI7C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK1C,yBAAyB,IAAI,MAAM;IAMnC,aAAa,IAAI,MAAM;IAKvB,eAAe,IAAI,MAAM;CAyC1B"}
|
package/dist/readers/csv.d.ts
CHANGED
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import { DataReader, DataRow } from "./data-reader";
|
|
2
|
-
export interface CsvRow {
|
|
3
|
-
[key: string]: string;
|
|
4
|
-
}
|
|
5
|
-
export declare function readCsvFile(filePath: string): Promise<CsvRow[]>;
|
|
6
2
|
export declare class CsvReader extends DataReader {
|
|
7
3
|
getSupportedExtensions(): string[];
|
|
8
4
|
readFile(filePath: string): Promise<DataRow[]>;
|