@jackchuka/gql-ingest 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +150 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/csv-reader.d.ts +5 -0
- package/dist/csv-reader.d.ts.map +1 -0
- package/dist/csv-reader.test.d.ts +2 -0
- package/dist/csv-reader.test.d.ts.map +1 -0
- package/dist/graphql-client.d.ts +7 -0
- package/dist/graphql-client.d.ts.map +1 -0
- package/dist/graphql-client.test.d.ts +2 -0
- package/dist/graphql-client.test.d.ts.map +1 -0
- package/dist/mapper.d.ts +15 -0
- package/dist/mapper.d.ts.map +1 -0
- package/dist/mapper.test.d.ts +2 -0
- package/dist/mapper.test.d.ts.map +1 -0
- package/package.json +49 -0
- package/src/cli.ts +60 -0
- package/src/csv-reader.test.ts +82 -0
- package/src/csv-reader.ts +18 -0
- package/src/graphql-client.test.ts +78 -0
- package/src/graphql-client.ts +25 -0
- package/src/mapper.test.ts +256 -0
- package/src/mapper.ts +80 -0
package/README.md
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# GQL Ingest
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/js/%40jackchuka%2Fgql-ingest)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
A TypeScript CLI tool that reads CSV files and ingests data into GraphQL APIs through configurable mutations.
|
|
7
|
+
|
|
8
|
+
## Features
|
|
9
|
+
|
|
10
|
+
- ✅ External GraphQL mutation definitions (separate .graphql files)
|
|
11
|
+
- ✅ CSV-to-GraphQL variable mapping via JSON configuration
|
|
12
|
+
- ✅ Configurable GraphQL endpoint and headers
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
### For End Users
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
# Install globally
|
|
20
|
+
npm install -g @jackchuka/gql-ingest
|
|
21
|
+
|
|
22
|
+
# Or use with npx (no installation required)
|
|
23
|
+
npx @jackchuka/gql-ingest --endpoint <url> --config <path>
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
### For Development
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
git clone https://github.com/jackchuka/gql-ingest.git
|
|
30
|
+
cd gql-ingest
|
|
31
|
+
npm install
|
|
32
|
+
npm run build
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Usage
|
|
36
|
+
|
|
37
|
+
### CLI Options
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
gql-ingest [options]
|
|
41
|
+
|
|
42
|
+
Options:
|
|
43
|
+
-V, --version output the version number
|
|
44
|
+
-e, --endpoint <url> GraphQL endpoint URL (required)
|
|
45
|
+
-c, --config <path> Path to configuration directory (required)
|
|
46
|
+
-h, --headers <headers> JSON string of headers to include in requests
|
|
47
|
+
--help display help for command
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### Examples
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
# Basic usage
|
|
54
|
+
npx @jackchuka/gql-ingest \
|
|
55
|
+
--endpoint https://your-graphql-api.com/graphql \
|
|
56
|
+
--config ./examples/demo
|
|
57
|
+
|
|
58
|
+
# With authentication headers
|
|
59
|
+
npx @jackchuka/gql-ingest \
|
|
60
|
+
--endpoint https://your-graphql-api.com/graphql \
|
|
61
|
+
--config ./examples/demo \
|
|
62
|
+
--headers '{"Authorization": "Bearer YOUR_TOKEN"}'
|
|
63
|
+
|
|
64
|
+
# With custom headers
|
|
65
|
+
npx @jackchuka/gql-ingest \
|
|
66
|
+
--endpoint https://api.example.com/graphql \
|
|
67
|
+
--config ./my-config \
|
|
68
|
+
--headers '{"X-API-Key": "your-api-key", "Content-Type": "application/json"}'
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Configuration
|
|
72
|
+
|
|
73
|
+
The `--config` flag points to a configuration directory containing three subdirectories:
|
|
74
|
+
|
|
75
|
+
- `data/` - CSV files with actual data
|
|
76
|
+
- `graphql/` - GraphQL mutation definitions
|
|
77
|
+
- `mappings/` - JSON files that map CSV columns to GraphQL variables
|
|
78
|
+
|
|
79
|
+
Each entity has three corresponding files across these directories with matching names.
|
|
80
|
+
|
|
81
|
+
### Example Configuration
|
|
82
|
+
|
|
83
|
+
**examples/demo/mappings/items.json**:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{
|
|
87
|
+
"csvFile": "data/items.csv",
|
|
88
|
+
"graphqlFile": "graphql/items.graphql",
|
|
89
|
+
"mapping": {
|
|
90
|
+
"name": "item_name",
|
|
91
|
+
"sku": "item_sku"
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**examples/demo/data/items.csv**:
|
|
97
|
+
|
|
98
|
+
```csv
|
|
99
|
+
item_name,item_sku
|
|
100
|
+
Item1,item-1-sku
|
|
101
|
+
Item2,item-2-sku
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
**examples/demo/graphql/items.graphql**:
|
|
105
|
+
|
|
106
|
+
```graphql
|
|
107
|
+
mutation CreateItem($name: String!, $sku: String!) {
|
|
108
|
+
createItem(input: { name: $name, sku: $sku }) {
|
|
109
|
+
id
|
|
110
|
+
name
|
|
111
|
+
sku
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Development
|
|
117
|
+
|
|
118
|
+
### Scripts
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
npm run build # Build CLI bundle with esbuild
|
|
122
|
+
npm run build:types # Generate TypeScript declarations
|
|
123
|
+
npm run build:all # Build bundle + types
|
|
124
|
+
npm run dev # Run in development mode
|
|
125
|
+
npm run test # Run test suite
|
|
126
|
+
npm run test:watch # Run tests in watch mode
|
|
127
|
+
npm run test:coverage # Run tests with coverage report
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Testing
|
|
131
|
+
|
|
132
|
+
The project includes comprehensive unit tests for all modules:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
npm test # Run all tests
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## How It Works
|
|
139
|
+
|
|
140
|
+
1. **Discovery**: The tool scans the `mappings/` directory for `.json` files
|
|
141
|
+
2. **Processing**: For each mapping file:
|
|
142
|
+
- Reads the corresponding CSV data file
|
|
143
|
+
- Loads the GraphQL mutation definition
|
|
144
|
+
- Maps CSV columns to GraphQL variables
|
|
145
|
+
- Executes the mutation for each CSV row
|
|
146
|
+
3. **Error Handling**: Failed mutations are logged but don't stop processing
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
MIT
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.d.ts","sourceRoot":"","sources":["../src/csv-reader.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUrE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-reader.test.d.ts","sourceRoot":"","sources":["../src/csv-reader.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare class GraphQLClientWrapper {
|
|
2
|
+
private client;
|
|
3
|
+
constructor(endpoint: string, headers?: Record<string, string>);
|
|
4
|
+
executeMutation(mutation: string, variables: Record<string, any>): Promise<any>;
|
|
5
|
+
setHeaders(headers: Record<string, string>): void;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=graphql-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graphql-client.d.ts","sourceRoot":"","sources":["../src/graphql-client.ts"],"names":[],"mappings":"AAEA,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAgB;gBAElB,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAMxD,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC;IAUrF,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAG3C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graphql-client.test.d.ts","sourceRoot":"","sources":["../src/graphql-client.test.ts"],"names":[],"mappings":""}
|
package/dist/mapper.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { GraphQLClientWrapper } from './graphql-client';
|
|
2
|
+
export interface MappingConfig {
|
|
3
|
+
csvFile: string;
|
|
4
|
+
graphqlFile: string;
|
|
5
|
+
mapping: Record<string, string>;
|
|
6
|
+
}
|
|
7
|
+
export declare class DataMapper {
|
|
8
|
+
private client;
|
|
9
|
+
private basePath;
|
|
10
|
+
constructor(client: GraphQLClientWrapper, basePath?: string);
|
|
11
|
+
discoverMappings(configDir: string): string[];
|
|
12
|
+
processEntity(configPath: string): Promise<void>;
|
|
13
|
+
private mapCsvRowToVariables;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=mapper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../src/mapper.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,oBAAoB,EAAE,QAAQ,GAAE,MAAsB;IAK1E,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;IAiBvC,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BtD,OAAO,CAAC,oBAAoB;CAW7B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mapper.test.d.ts","sourceRoot":"","sources":["../src/mapper.test.ts"],"names":[],"mappings":""}
|
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@jackchuka/gql-ingest",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "A CLI tool for ingesting data from CSV files into a GraphQL API",
|
|
5
|
+
"main": "dist/cli.js",
|
|
6
|
+
"bin": {
|
|
7
|
+
"gql-ingest": "dist/cli.js"
|
|
8
|
+
},
|
|
9
|
+
"scripts": {
|
|
10
|
+
"build": "node esbuild.config.js",
|
|
11
|
+
"build:types": "tsc --emitDeclarationOnly",
|
|
12
|
+
"build:all": "npm run build && npm run build:types",
|
|
13
|
+
"dev": "ts-node src/cli.ts",
|
|
14
|
+
"test": "jest",
|
|
15
|
+
"test:watch": "jest --watch",
|
|
16
|
+
"prepublishOnly": "npm run build:all"
|
|
17
|
+
},
|
|
18
|
+
"keywords": [
|
|
19
|
+
"graphql",
|
|
20
|
+
"cli",
|
|
21
|
+
"csv",
|
|
22
|
+
"ingest",
|
|
23
|
+
"api"
|
|
24
|
+
],
|
|
25
|
+
"author": "jackchuka",
|
|
26
|
+
"license": "MIT",
|
|
27
|
+
"repository": {
|
|
28
|
+
"type": "git",
|
|
29
|
+
"url": "https://github.com/jackchuka/gql-ingest.git"
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"commander": "^14.0.0",
|
|
33
|
+
"csv-parser": "^3.0.0",
|
|
34
|
+
"graphql-request": "^7.2.0"
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"@types/jest": "^30.0.0",
|
|
38
|
+
"@types/node": "^24.0.4",
|
|
39
|
+
"esbuild": "^0.25.5",
|
|
40
|
+
"jest": "^30.0.3",
|
|
41
|
+
"ts-jest": "^29.4.0",
|
|
42
|
+
"ts-node": "^10.9.0",
|
|
43
|
+
"typescript": "^5.3.0"
|
|
44
|
+
},
|
|
45
|
+
"files": [
|
|
46
|
+
"dist/**/*",
|
|
47
|
+
"src/**/*"
|
|
48
|
+
]
|
|
49
|
+
}
|
package/src/cli.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
3
|
+
import { DataMapper } from "./mapper";
|
|
4
|
+
|
|
5
|
+
const program = new Command();
|
|
6
|
+
|
|
7
|
+
program
|
|
8
|
+
.name("gql-ingest")
|
|
9
|
+
.description(
|
|
10
|
+
"A CLI tool for ingesting data from CSV files into a GraphQL API"
|
|
11
|
+
)
|
|
12
|
+
.version(require("../package.json").version);
|
|
13
|
+
|
|
14
|
+
program
|
|
15
|
+
.requiredOption("-e, --endpoint <url>", "GraphQL endpoint URL")
|
|
16
|
+
.requiredOption(
|
|
17
|
+
"-c, --config <path>",
|
|
18
|
+
"Path to configuration directory (containing data/, graphql/, mappings/ subdirectories)"
|
|
19
|
+
)
|
|
20
|
+
.option(
|
|
21
|
+
"-h, --headers <headers>",
|
|
22
|
+
"JSON string of headers to include in requests"
|
|
23
|
+
)
|
|
24
|
+
.action(async (options) => {
|
|
25
|
+
try {
|
|
26
|
+
console.log("Starting seed data generation...");
|
|
27
|
+
|
|
28
|
+
// Parse headers if provided
|
|
29
|
+
const headers = options.headers ? JSON.parse(options.headers) : {};
|
|
30
|
+
|
|
31
|
+
// Initialize GraphQL client
|
|
32
|
+
const client = new GraphQLClientWrapper(options.endpoint, headers);
|
|
33
|
+
|
|
34
|
+
// Initialize data mapper
|
|
35
|
+
const mapper = new DataMapper(client);
|
|
36
|
+
|
|
37
|
+
// Discover all mapping files dynamically
|
|
38
|
+
const mappingPaths = mapper.discoverMappings(options.config);
|
|
39
|
+
|
|
40
|
+
if (mappingPaths.length === 0) {
|
|
41
|
+
console.warn(`No mapping files found in ${options.config}/mappings`);
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
for (const configPath of mappingPaths) {
|
|
46
|
+
try {
|
|
47
|
+
await mapper.processEntity(configPath);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
console.warn(`Warning: Could not process ${configPath}:`, error);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log("Seed data generation completed!");
|
|
54
|
+
} catch (error) {
|
|
55
|
+
console.error("Error:", error);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
program.parse();
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { readCsvFile } from "./csv-reader";
|
|
4
|
+
|
|
5
|
+
describe("CSV Reader", () => {
|
|
6
|
+
const testDataDir = path.join(__dirname, "test-data");
|
|
7
|
+
const testCsvPath = path.join(testDataDir, "test.csv");
|
|
8
|
+
|
|
9
|
+
beforeAll(() => {
|
|
10
|
+
if (!fs.existsSync(testDataDir)) {
|
|
11
|
+
fs.mkdirSync(testDataDir, { recursive: true });
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
afterAll(() => {
|
|
16
|
+
if (fs.existsSync(testDataDir)) {
|
|
17
|
+
fs.rmSync(testDataDir, { recursive: true });
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
if (fs.existsSync(testCsvPath)) {
|
|
23
|
+
fs.unlinkSync(testCsvPath);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("should read a simple CSV file", async () => {
|
|
28
|
+
const csvContent = "name,age\nJohn,30\nJane,25";
|
|
29
|
+
fs.writeFileSync(testCsvPath, csvContent);
|
|
30
|
+
|
|
31
|
+
const result = await readCsvFile(testCsvPath);
|
|
32
|
+
|
|
33
|
+
expect(result).toEqual([
|
|
34
|
+
{ name: "John", age: "30" },
|
|
35
|
+
{ name: "Jane", age: "25" },
|
|
36
|
+
]);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("should read CSV with special characters", async () => {
|
|
40
|
+
const csvContent =
|
|
41
|
+
'name,description\n"John Doe","A person with, comma"\n"Jane\'s Data","Quote test"';
|
|
42
|
+
fs.writeFileSync(testCsvPath, csvContent);
|
|
43
|
+
|
|
44
|
+
const result = await readCsvFile(testCsvPath);
|
|
45
|
+
|
|
46
|
+
expect(result).toEqual([
|
|
47
|
+
{ name: "John Doe", description: "A person with, comma" },
|
|
48
|
+
{ name: "Jane's Data", description: "Quote test" },
|
|
49
|
+
]);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("should handle empty CSV file", async () => {
|
|
53
|
+
const csvContent = "name,age\n";
|
|
54
|
+
fs.writeFileSync(testCsvPath, csvContent);
|
|
55
|
+
|
|
56
|
+
const result = await readCsvFile(testCsvPath);
|
|
57
|
+
|
|
58
|
+
expect(result).toEqual([]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("should handle CSV with only headers", async () => {
|
|
62
|
+
const csvContent = "name,age";
|
|
63
|
+
fs.writeFileSync(testCsvPath, csvContent);
|
|
64
|
+
|
|
65
|
+
const result = await readCsvFile(testCsvPath);
|
|
66
|
+
|
|
67
|
+
expect(result).toEqual([]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("should handle CSV with missing values", async () => {
|
|
71
|
+
const csvContent = "name,age,city\nJohn,30,\nJane,,Boston\n,25,NYC";
|
|
72
|
+
fs.writeFileSync(testCsvPath, csvContent);
|
|
73
|
+
|
|
74
|
+
const result = await readCsvFile(testCsvPath);
|
|
75
|
+
|
|
76
|
+
expect(result).toEqual([
|
|
77
|
+
{ name: "John", age: "30", city: "" },
|
|
78
|
+
{ name: "Jane", age: "", city: "Boston" },
|
|
79
|
+
{ name: "", age: "25", city: "NYC" },
|
|
80
|
+
]);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import csv from 'csv-parser';
|
|
3
|
+
|
|
4
|
+
export interface CsvRow {
|
|
5
|
+
[key: string]: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export async function readCsvFile(filePath: string): Promise<CsvRow[]> {
|
|
9
|
+
return new Promise((resolve, reject) => {
|
|
10
|
+
const results: CsvRow[] = [];
|
|
11
|
+
|
|
12
|
+
fs.createReadStream(filePath)
|
|
13
|
+
.pipe(csv())
|
|
14
|
+
.on('data', (data) => results.push(data))
|
|
15
|
+
.on('end', () => resolve(results))
|
|
16
|
+
.on('error', (error) => reject(error));
|
|
17
|
+
});
|
|
18
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
2
|
+
|
|
3
|
+
const mockRequest = jest.fn();
|
|
4
|
+
const mockSetHeaders = jest.fn();
|
|
5
|
+
|
|
6
|
+
jest.mock("graphql-request", () => ({
|
|
7
|
+
GraphQLClient: jest.fn().mockImplementation(() => ({
|
|
8
|
+
request: mockRequest,
|
|
9
|
+
setHeaders: mockSetHeaders,
|
|
10
|
+
})),
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
describe("GraphQLClientWrapper", () => {
|
|
14
|
+
let clientWrapper: GraphQLClientWrapper;
|
|
15
|
+
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
jest.clearAllMocks();
|
|
18
|
+
clientWrapper = new GraphQLClientWrapper("https://api.example.com/graphql");
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it("should create GraphQLClient with endpoint and default headers", () => {
|
|
22
|
+
const { GraphQLClient } = require("graphql-request");
|
|
23
|
+
expect(GraphQLClient).toHaveBeenCalledWith(
|
|
24
|
+
"https://api.example.com/graphql",
|
|
25
|
+
{ headers: {} }
|
|
26
|
+
);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("should create GraphQLClient with custom headers", () => {
|
|
30
|
+
const headers = { Authorization: "Bearer token123" };
|
|
31
|
+
new GraphQLClientWrapper("https://api.example.com/graphql", headers);
|
|
32
|
+
|
|
33
|
+
const { GraphQLClient } = require("graphql-request");
|
|
34
|
+
expect(GraphQLClient).toHaveBeenCalledWith(
|
|
35
|
+
"https://api.example.com/graphql",
|
|
36
|
+
{ headers }
|
|
37
|
+
);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("should execute mutation successfully", async () => {
|
|
41
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
42
|
+
const variables = { name: "John" };
|
|
43
|
+
const expectedResult = { createUser: { id: "123" } };
|
|
44
|
+
|
|
45
|
+
mockRequest.mockResolvedValue(expectedResult);
|
|
46
|
+
|
|
47
|
+
const result = await clientWrapper.executeMutation(mutation, variables);
|
|
48
|
+
|
|
49
|
+
expect(mockRequest).toHaveBeenCalledWith(mutation, variables);
|
|
50
|
+
expect(result).toEqual(expectedResult);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("should handle GraphQL errors", async () => {
|
|
54
|
+
const mutation = "mutation { createUser(name: $name) { id } }";
|
|
55
|
+
const variables = { name: "John" };
|
|
56
|
+
const error = new Error("GraphQL error");
|
|
57
|
+
|
|
58
|
+
mockRequest.mockRejectedValue(error);
|
|
59
|
+
|
|
60
|
+
const consoleSpy = jest.spyOn(console, "error").mockImplementation();
|
|
61
|
+
|
|
62
|
+
await expect(
|
|
63
|
+
clientWrapper.executeMutation(mutation, variables)
|
|
64
|
+
).rejects.toThrow("GraphQL error");
|
|
65
|
+
|
|
66
|
+
expect(consoleSpy).toHaveBeenCalledWith("GraphQL mutation failed:", error);
|
|
67
|
+
|
|
68
|
+
consoleSpy.mockRestore();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("should set headers on the client", () => {
|
|
72
|
+
const newHeaders = { "X-API-Key": "api123" };
|
|
73
|
+
|
|
74
|
+
clientWrapper.setHeaders(newHeaders);
|
|
75
|
+
|
|
76
|
+
expect(mockSetHeaders).toHaveBeenCalledWith(newHeaders);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { GraphQLClient } from 'graphql-request';
|
|
2
|
+
|
|
3
|
+
export class GraphQLClientWrapper {
|
|
4
|
+
private client: GraphQLClient;
|
|
5
|
+
|
|
6
|
+
constructor(endpoint: string, headers?: Record<string, string>) {
|
|
7
|
+
this.client = new GraphQLClient(endpoint, {
|
|
8
|
+
headers: headers || {}
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async executeMutation(mutation: string, variables: Record<string, any>): Promise<any> {
|
|
13
|
+
try {
|
|
14
|
+
const result = await this.client.request(mutation, variables);
|
|
15
|
+
return result;
|
|
16
|
+
} catch (error) {
|
|
17
|
+
console.error('GraphQL mutation failed:', error);
|
|
18
|
+
throw error;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
setHeaders(headers: Record<string, string>) {
|
|
23
|
+
this.client.setHeaders(headers);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { DataMapper } from "./mapper";
|
|
4
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
5
|
+
|
|
6
|
+
jest.mock("fs");
|
|
7
|
+
jest.mock("./csv-reader");
|
|
8
|
+
|
|
9
|
+
const mockFs = fs as jest.Mocked<typeof fs>;
|
|
10
|
+
|
|
11
|
+
describe("DataMapper", () => {
|
|
12
|
+
let mockClient: jest.Mocked<GraphQLClientWrapper>;
|
|
13
|
+
let dataMapper: DataMapper;
|
|
14
|
+
const testBasePath = "/test/base/path";
|
|
15
|
+
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
mockClient = {
|
|
18
|
+
executeMutation: jest.fn(),
|
|
19
|
+
setHeaders: jest.fn(),
|
|
20
|
+
} as any;
|
|
21
|
+
|
|
22
|
+
dataMapper = new DataMapper(mockClient, testBasePath);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
jest.clearAllMocks();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("discoverMappings", () => {
|
|
30
|
+
it("should discover mapping files in alphabetical order", () => {
|
|
31
|
+
const mockFiles = ["users.json", "items.json", "orders.json"];
|
|
32
|
+
mockFs.readdirSync.mockReturnValue(mockFiles as any);
|
|
33
|
+
|
|
34
|
+
const consoleSpy = jest.spyOn(console, "log").mockImplementation();
|
|
35
|
+
|
|
36
|
+
const result = dataMapper.discoverMappings("configs/test");
|
|
37
|
+
|
|
38
|
+
expect(mockFs.readdirSync).toHaveBeenCalledWith(
|
|
39
|
+
path.resolve(testBasePath, "configs/test", "mappings")
|
|
40
|
+
);
|
|
41
|
+
expect(result).toEqual([
|
|
42
|
+
"configs/test/mappings/items.json",
|
|
43
|
+
"configs/test/mappings/orders.json",
|
|
44
|
+
"configs/test/mappings/users.json",
|
|
45
|
+
]);
|
|
46
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
47
|
+
"Discovered 3 mapping files: items.json, orders.json, users.json"
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
consoleSpy.mockRestore();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("should filter only JSON files", () => {
|
|
54
|
+
const mockFiles = ["users.json", "items.txt", "orders.json", "readme.md"];
|
|
55
|
+
mockFs.readdirSync.mockReturnValue(mockFiles as any);
|
|
56
|
+
|
|
57
|
+
const result = dataMapper.discoverMappings("configs/test");
|
|
58
|
+
|
|
59
|
+
expect(result).toEqual([
|
|
60
|
+
"configs/test/mappings/orders.json",
|
|
61
|
+
"configs/test/mappings/users.json",
|
|
62
|
+
]);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("should handle directory read errors", () => {
|
|
66
|
+
mockFs.readdirSync.mockImplementation(() => {
|
|
67
|
+
throw new Error("Directory not found");
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
const consoleSpy = jest.spyOn(console, "error").mockImplementation();
|
|
71
|
+
|
|
72
|
+
const result = dataMapper.discoverMappings("configs/nonexistent");
|
|
73
|
+
|
|
74
|
+
expect(result).toEqual([]);
|
|
75
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
76
|
+
expect.stringContaining("Error reading mappings directory"),
|
|
77
|
+
expect.any(Error)
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
consoleSpy.mockRestore();
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
describe("processEntity", () => {
|
|
85
|
+
it("should process entity successfully", async () => {
|
|
86
|
+
const mockConfig = {
|
|
87
|
+
csvFile: "data/users.csv",
|
|
88
|
+
graphqlFile: "graphql/users.graphql",
|
|
89
|
+
mapping: {
|
|
90
|
+
name: "user_name",
|
|
91
|
+
email: "user_email",
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const mockCsvData = [
|
|
96
|
+
{ user_name: "John", user_email: "john@example.com" },
|
|
97
|
+
{ user_name: "Jane", user_email: "jane@example.com" },
|
|
98
|
+
];
|
|
99
|
+
|
|
100
|
+
const mockMutation =
|
|
101
|
+
"mutation CreateUser($name: String!, $email: String!) { createUser(input: { name: $name, email: $email }) { id } }";
|
|
102
|
+
|
|
103
|
+
mockFs.readFileSync
|
|
104
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
105
|
+
.mockReturnValueOnce(mockMutation);
|
|
106
|
+
|
|
107
|
+
const { readCsvFile } = require("./csv-reader");
|
|
108
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
109
|
+
|
|
110
|
+
mockClient.executeMutation.mockResolvedValue({
|
|
111
|
+
createUser: { id: "123" },
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
const consoleSpy = jest.spyOn(console, "log").mockImplementation();
|
|
115
|
+
|
|
116
|
+
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
117
|
+
|
|
118
|
+
expect(mockFs.readFileSync).toHaveBeenCalledWith(
|
|
119
|
+
path.resolve(testBasePath, "configs/test/mappings/users.json"),
|
|
120
|
+
"utf8"
|
|
121
|
+
);
|
|
122
|
+
expect(readCsvFile).toHaveBeenCalledWith(
|
|
123
|
+
path.resolve(testBasePath, "configs/test", "data/users.csv")
|
|
124
|
+
);
|
|
125
|
+
expect(mockFs.readFileSync).toHaveBeenCalledWith(
|
|
126
|
+
path.resolve(testBasePath, "configs/test", "graphql/users.graphql"),
|
|
127
|
+
"utf8"
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
expect(mockClient.executeMutation).toHaveBeenCalledTimes(2);
|
|
131
|
+
expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
|
|
132
|
+
name: "John",
|
|
133
|
+
email: "john@example.com",
|
|
134
|
+
});
|
|
135
|
+
expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
|
|
136
|
+
name: "Jane",
|
|
137
|
+
email: "jane@example.com",
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
consoleSpy.mockRestore();
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("should handle GraphQL execution errors gracefully", async () => {
|
|
144
|
+
const mockConfig = {
|
|
145
|
+
csvFile: "data/users.csv",
|
|
146
|
+
graphqlFile: "graphql/users.graphql",
|
|
147
|
+
mapping: { name: "user_name" },
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const mockCsvData = [{ user_name: "John" }];
|
|
151
|
+
const mockMutation =
|
|
152
|
+
"mutation CreateUser($name: String!) { createUser(input: { name: $name }) { id } }";
|
|
153
|
+
|
|
154
|
+
mockFs.readFileSync
|
|
155
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
156
|
+
.mockReturnValueOnce(mockMutation);
|
|
157
|
+
|
|
158
|
+
const { readCsvFile } = require("./csv-reader");
|
|
159
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
160
|
+
|
|
161
|
+
mockClient.executeMutation.mockRejectedValue(new Error("GraphQL error"));
|
|
162
|
+
|
|
163
|
+
const consoleSpy = jest.spyOn(console, "error").mockImplementation();
|
|
164
|
+
|
|
165
|
+
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
166
|
+
|
|
167
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
168
|
+
"✗ Failed to create entity for row:",
|
|
169
|
+
{ user_name: "John" },
|
|
170
|
+
expect.any(Error)
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
consoleSpy.mockRestore();
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("should map CSV columns to GraphQL variables correctly", async () => {
|
|
177
|
+
const mockConfig = {
|
|
178
|
+
csvFile: "data/products.csv",
|
|
179
|
+
graphqlFile: "graphql/products.graphql",
|
|
180
|
+
mapping: {
|
|
181
|
+
name: "product_name",
|
|
182
|
+
price: "product_price",
|
|
183
|
+
sku: "product_sku",
|
|
184
|
+
},
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
const mockCsvData = [
|
|
188
|
+
{
|
|
189
|
+
product_name: "Widget",
|
|
190
|
+
product_price: "19.99",
|
|
191
|
+
product_sku: "W001",
|
|
192
|
+
extra_column: "ignored",
|
|
193
|
+
},
|
|
194
|
+
];
|
|
195
|
+
|
|
196
|
+
const mockMutation =
|
|
197
|
+
"mutation CreateProduct($name: String!, $price: String!, $sku: String!) { createProduct(input: { name: $name, price: $price, sku: $sku }) { id } }";
|
|
198
|
+
|
|
199
|
+
mockFs.readFileSync
|
|
200
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
201
|
+
.mockReturnValueOnce(mockMutation);
|
|
202
|
+
|
|
203
|
+
const { readCsvFile } = require("./csv-reader");
|
|
204
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
205
|
+
|
|
206
|
+
mockClient.executeMutation.mockResolvedValue({
|
|
207
|
+
createProduct: { id: "456" },
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
await dataMapper.processEntity("configs/test/mappings/products.json");
|
|
211
|
+
|
|
212
|
+
expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
|
|
213
|
+
name: "Widget",
|
|
214
|
+
price: "19.99",
|
|
215
|
+
sku: "W001",
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it("should handle missing CSV columns gracefully", async () => {
|
|
220
|
+
const mockConfig = {
|
|
221
|
+
csvFile: "data/users.csv",
|
|
222
|
+
graphqlFile: "graphql/users.graphql",
|
|
223
|
+
mapping: {
|
|
224
|
+
name: "user_name",
|
|
225
|
+
email: "user_email",
|
|
226
|
+
phone: "user_phone",
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const mockCsvData = [
|
|
231
|
+
{ user_name: "John", user_email: "john@example.com" },
|
|
232
|
+
];
|
|
233
|
+
|
|
234
|
+
const mockMutation =
|
|
235
|
+
"mutation CreateUser($name: String!, $email: String, $phone: String) { createUser(input: { name: $name, email: $email, phone: $phone }) { id } }";
|
|
236
|
+
|
|
237
|
+
mockFs.readFileSync
|
|
238
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
239
|
+
.mockReturnValueOnce(mockMutation);
|
|
240
|
+
|
|
241
|
+
const { readCsvFile } = require("./csv-reader");
|
|
242
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
243
|
+
|
|
244
|
+
mockClient.executeMutation.mockResolvedValue({
|
|
245
|
+
createUser: { id: "789" },
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
249
|
+
|
|
250
|
+
expect(mockClient.executeMutation).toHaveBeenCalledWith(mockMutation, {
|
|
251
|
+
name: "John",
|
|
252
|
+
email: "john@example.com",
|
|
253
|
+
});
|
|
254
|
+
});
|
|
255
|
+
});
|
|
256
|
+
});
|
package/src/mapper.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { readCsvFile, CsvRow } from './csv-reader';
|
|
4
|
+
import { GraphQLClientWrapper } from './graphql-client';
|
|
5
|
+
|
|
6
|
+
export interface MappingConfig {
|
|
7
|
+
csvFile: string;
|
|
8
|
+
graphqlFile: string;
|
|
9
|
+
mapping: Record<string, string>;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export class DataMapper {
|
|
13
|
+
private client: GraphQLClientWrapper;
|
|
14
|
+
private basePath: string;
|
|
15
|
+
|
|
16
|
+
constructor(client: GraphQLClientWrapper, basePath: string = process.cwd()) {
|
|
17
|
+
this.client = client;
|
|
18
|
+
this.basePath = basePath;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
discoverMappings(configDir: string): string[] {
|
|
22
|
+
const mappingsPath = path.resolve(this.basePath, configDir, 'mappings');
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
const files = fs.readdirSync(mappingsPath);
|
|
26
|
+
const jsonFiles = files
|
|
27
|
+
.filter(file => file.endsWith('.json'))
|
|
28
|
+
.sort(); // Alphabetical order for consistent processing
|
|
29
|
+
|
|
30
|
+
console.log(`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(', ')}`);
|
|
31
|
+
return jsonFiles.map(file => path.join(configDir, 'mappings', file));
|
|
32
|
+
} catch (error) {
|
|
33
|
+
console.error(`Error reading mappings directory ${mappingsPath}:`, error);
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async processEntity(configPath: string): Promise<void> {
|
|
39
|
+
console.log(`Processing entity: ${configPath}`);
|
|
40
|
+
|
|
41
|
+
// Read mapping configuration
|
|
42
|
+
const configFullPath = path.resolve(this.basePath, configPath);
|
|
43
|
+
const config: MappingConfig = JSON.parse(fs.readFileSync(configFullPath, 'utf8'));
|
|
44
|
+
|
|
45
|
+
// Extract config directory (parent of mappings directory)
|
|
46
|
+
const configDir = path.dirname(path.dirname(configFullPath));
|
|
47
|
+
|
|
48
|
+
// Read CSV data (relative to config directory)
|
|
49
|
+
const csvPath = path.resolve(configDir, config.csvFile);
|
|
50
|
+
const csvData = await readCsvFile(csvPath);
|
|
51
|
+
|
|
52
|
+
// Read GraphQL mutation (relative to config directory)
|
|
53
|
+
const graphqlPath = path.resolve(configDir, config.graphqlFile);
|
|
54
|
+
const mutation = fs.readFileSync(graphqlPath, 'utf8');
|
|
55
|
+
|
|
56
|
+
// Process each row
|
|
57
|
+
for (const row of csvData) {
|
|
58
|
+
const variables = this.mapCsvRowToVariables(row, config.mapping);
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
const result = await this.client.executeMutation(mutation, variables);
|
|
62
|
+
console.log(`✓ Created entity with result:`, result);
|
|
63
|
+
} catch (error) {
|
|
64
|
+
console.error(`✗ Failed to create entity for row:`, row, error);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private mapCsvRowToVariables(row: CsvRow, mapping: Record<string, string>): Record<string, any> {
|
|
70
|
+
const variables: Record<string, any> = {};
|
|
71
|
+
|
|
72
|
+
for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
|
|
73
|
+
if (row[csvColumn] !== undefined) {
|
|
74
|
+
variables[graphqlVar] = row[csvColumn];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return variables;
|
|
79
|
+
}
|
|
80
|
+
}
|