@jackchuka/gql-ingest 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +48 -1
- package/dist/config.d.ts +22 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.test.d.ts +2 -0
- package/dist/config.test.d.ts.map +1 -0
- package/dist/dependency-resolver.d.ts +17 -0
- package/dist/dependency-resolver.d.ts.map +1 -0
- package/dist/dependency-resolver.test.d.ts +2 -0
- package/dist/dependency-resolver.test.d.ts.map +1 -0
- package/dist/graphql-client.d.ts +4 -1
- package/dist/graphql-client.d.ts.map +1 -1
- package/dist/mapper.d.ts +11 -3
- package/dist/mapper.d.ts.map +1 -1
- package/dist/metrics.d.ts +33 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.test.d.ts +2 -0
- package/dist/metrics.test.d.ts.map +1 -0
- package/package.json +5 -2
- package/src/cli.ts +113 -9
- package/src/config.test.ts +205 -0
- package/src/config.ts +92 -0
- package/src/dependency-resolver.test.ts +197 -0
- package/src/dependency-resolver.ts +98 -0
- package/src/graphql-client.ts +28 -2
- package/src/mapper.test.ts +81 -2
- package/src/mapper.ts +169 -31
- package/src/metrics.test.ts +207 -0
- package/src/metrics.ts +131 -0
package/src/mapper.test.ts
CHANGED
|
@@ -2,6 +2,7 @@ import fs from "fs";
|
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { DataMapper } from "./mapper";
|
|
4
4
|
import { GraphQLClientWrapper } from "./graphql-client";
|
|
5
|
+
import { MetricsCollector } from "./metrics";
|
|
5
6
|
|
|
6
7
|
jest.mock("fs");
|
|
7
8
|
jest.mock("./csv-reader");
|
|
@@ -10,6 +11,7 @@ const mockFs = fs as jest.Mocked<typeof fs>;
|
|
|
10
11
|
|
|
11
12
|
describe("DataMapper", () => {
|
|
12
13
|
let mockClient: jest.Mocked<GraphQLClientWrapper>;
|
|
14
|
+
let mockMetrics: jest.Mocked<MetricsCollector>;
|
|
13
15
|
let dataMapper: DataMapper;
|
|
14
16
|
const testBasePath = "/test/base/path";
|
|
15
17
|
|
|
@@ -19,7 +21,15 @@ describe("DataMapper", () => {
|
|
|
19
21
|
setHeaders: jest.fn(),
|
|
20
22
|
} as any;
|
|
21
23
|
|
|
22
|
-
|
|
24
|
+
mockMetrics = {
|
|
25
|
+
startEntityProcessing: jest.fn(),
|
|
26
|
+
recordSuccess: jest.fn(),
|
|
27
|
+
recordFailure: jest.fn(),
|
|
28
|
+
finishEntityProcessing: jest.fn(),
|
|
29
|
+
getMetrics: jest.fn(),
|
|
30
|
+
} as any;
|
|
31
|
+
|
|
32
|
+
dataMapper = new DataMapper(mockClient, testBasePath, mockMetrics);
|
|
23
33
|
});
|
|
24
34
|
|
|
25
35
|
afterEach(() => {
|
|
@@ -165,7 +175,7 @@ describe("DataMapper", () => {
|
|
|
165
175
|
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
166
176
|
|
|
167
177
|
expect(consoleSpy).toHaveBeenCalledWith(
|
|
168
|
-
"✗ Failed to create entity for row:",
|
|
178
|
+
"✗ Failed to create entity for row 1:",
|
|
169
179
|
{ user_name: "John" },
|
|
170
180
|
expect.any(Error)
|
|
171
181
|
);
|
|
@@ -252,5 +262,74 @@ describe("DataMapper", () => {
|
|
|
252
262
|
email: "john@example.com",
|
|
253
263
|
});
|
|
254
264
|
});
|
|
265
|
+
|
|
266
|
+
it("should call metrics methods during successful processing", async () => {
|
|
267
|
+
const mockConfig = {
|
|
268
|
+
csvFile: "data/users.csv",
|
|
269
|
+
graphqlFile: "graphql/users.graphql",
|
|
270
|
+
mapping: { name: "user_name" },
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
const mockCsvData = [{ user_name: "John" }, { user_name: "Jane" }];
|
|
274
|
+
const mockMutation =
|
|
275
|
+
"mutation CreateUser($name: String!) { createUser(input: { name: $name }) { id } }";
|
|
276
|
+
|
|
277
|
+
mockFs.readFileSync
|
|
278
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
279
|
+
.mockReturnValueOnce(mockMutation);
|
|
280
|
+
|
|
281
|
+
const { readCsvFile } = require("./csv-reader");
|
|
282
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
283
|
+
|
|
284
|
+
mockClient.executeMutation.mockResolvedValue({
|
|
285
|
+
createUser: { id: "123" },
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
289
|
+
|
|
290
|
+
expect(mockMetrics.startEntityProcessing).toHaveBeenCalledWith("users");
|
|
291
|
+
expect(mockMetrics.recordSuccess).toHaveBeenCalledTimes(2);
|
|
292
|
+
expect(mockMetrics.recordSuccess).toHaveBeenCalledWith("users");
|
|
293
|
+
expect(mockMetrics.finishEntityProcessing).toHaveBeenCalledWith("users");
|
|
294
|
+
expect(mockMetrics.recordFailure).not.toHaveBeenCalled();
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it("should call metrics methods during failed processing", async () => {
|
|
298
|
+
const mockConfig = {
|
|
299
|
+
csvFile: "data/users.csv",
|
|
300
|
+
graphqlFile: "graphql/users.graphql",
|
|
301
|
+
mapping: { name: "user_name" },
|
|
302
|
+
};
|
|
303
|
+
|
|
304
|
+
const mockCsvData = [{ user_name: "John" }];
|
|
305
|
+
const mockMutation =
|
|
306
|
+
"mutation CreateUser($name: String!) { createUser(input: { name: $name }) { id } }";
|
|
307
|
+
|
|
308
|
+
mockFs.readFileSync
|
|
309
|
+
.mockReturnValueOnce(JSON.stringify(mockConfig))
|
|
310
|
+
.mockReturnValueOnce(mockMutation);
|
|
311
|
+
|
|
312
|
+
const { readCsvFile } = require("./csv-reader");
|
|
313
|
+
readCsvFile.mockResolvedValue(mockCsvData);
|
|
314
|
+
|
|
315
|
+
mockClient.executeMutation.mockRejectedValue(new Error("GraphQL error"));
|
|
316
|
+
|
|
317
|
+
const consoleSpy = jest.spyOn(console, "error").mockImplementation();
|
|
318
|
+
|
|
319
|
+
await dataMapper.processEntity("configs/test/mappings/users.json");
|
|
320
|
+
|
|
321
|
+
expect(mockMetrics.startEntityProcessing).toHaveBeenCalledWith("users");
|
|
322
|
+
expect(mockMetrics.recordFailure).toHaveBeenCalledTimes(1);
|
|
323
|
+
expect(mockMetrics.recordFailure).toHaveBeenCalledWith("users");
|
|
324
|
+
expect(mockMetrics.finishEntityProcessing).toHaveBeenCalledWith("users");
|
|
325
|
+
expect(mockMetrics.recordSuccess).not.toHaveBeenCalled();
|
|
326
|
+
|
|
327
|
+
consoleSpy.mockRestore();
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
it("should expose metrics through getMetrics method", () => {
|
|
331
|
+
const metrics = dataMapper.getMetrics();
|
|
332
|
+
expect(metrics).toBe(mockMetrics);
|
|
333
|
+
});
|
|
255
334
|
});
|
|
256
335
|
});
|
package/src/mapper.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import fs from
|
|
2
|
-
import path from
|
|
3
|
-
import { readCsvFile, CsvRow } from
|
|
4
|
-
import { GraphQLClientWrapper } from
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { readCsvFile, CsvRow } from "./csv-reader";
|
|
4
|
+
import { GraphQLClientWrapper } from "./graphql-client";
|
|
5
|
+
import { MetricsCollector } from "./metrics";
|
|
6
|
+
import { ParallelProcessingConfig } from "./config";
|
|
5
7
|
|
|
6
8
|
export interface MappingConfig {
|
|
7
9
|
csvFile: string;
|
|
@@ -12,69 +14,205 @@ export interface MappingConfig {
|
|
|
12
14
|
export class DataMapper {
|
|
13
15
|
private client: GraphQLClientWrapper;
|
|
14
16
|
private basePath: string;
|
|
17
|
+
private metrics: MetricsCollector;
|
|
18
|
+
private verbose: boolean;
|
|
15
19
|
|
|
16
|
-
constructor(
|
|
20
|
+
constructor(
|
|
21
|
+
client: GraphQLClientWrapper,
|
|
22
|
+
basePath: string = process.cwd(),
|
|
23
|
+
metrics?: MetricsCollector,
|
|
24
|
+
verbose: boolean = false
|
|
25
|
+
) {
|
|
17
26
|
this.client = client;
|
|
18
27
|
this.basePath = basePath;
|
|
28
|
+
this.metrics = metrics || new MetricsCollector();
|
|
29
|
+
this.verbose = verbose;
|
|
19
30
|
}
|
|
20
31
|
|
|
21
32
|
discoverMappings(configDir: string): string[] {
|
|
22
|
-
const mappingsPath = path.resolve(this.basePath, configDir,
|
|
23
|
-
|
|
33
|
+
const mappingsPath = path.resolve(this.basePath, configDir, "mappings");
|
|
34
|
+
|
|
24
35
|
try {
|
|
25
36
|
const files = fs.readdirSync(mappingsPath);
|
|
26
|
-
const jsonFiles = files
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return jsonFiles.map(file => path.join(configDir,
|
|
37
|
+
const jsonFiles = files.filter((file) => file.endsWith(".json")).sort(); // Alphabetical order for consistent processing
|
|
38
|
+
|
|
39
|
+
console.log(
|
|
40
|
+
`Discovered ${jsonFiles.length} mapping files: ${jsonFiles.join(", ")}`
|
|
41
|
+
);
|
|
42
|
+
return jsonFiles.map((file) => path.join(configDir, "mappings", file));
|
|
32
43
|
} catch (error) {
|
|
33
44
|
console.error(`Error reading mappings directory ${mappingsPath}:`, error);
|
|
34
45
|
return [];
|
|
35
46
|
}
|
|
36
47
|
}
|
|
37
48
|
|
|
38
|
-
async processEntity(
|
|
49
|
+
async processEntity(
|
|
50
|
+
configPath: string,
|
|
51
|
+
parallelConfig?: ParallelProcessingConfig
|
|
52
|
+
): Promise<void> {
|
|
53
|
+
const entityName = path.basename(configPath, ".json");
|
|
39
54
|
console.log(`Processing entity: ${configPath}`);
|
|
40
|
-
|
|
55
|
+
|
|
56
|
+
this.metrics.startEntityProcessing(entityName);
|
|
57
|
+
|
|
41
58
|
// Read mapping configuration
|
|
42
59
|
const configFullPath = path.resolve(this.basePath, configPath);
|
|
43
|
-
const config: MappingConfig = JSON.parse(
|
|
44
|
-
|
|
60
|
+
const config: MappingConfig = JSON.parse(
|
|
61
|
+
fs.readFileSync(configFullPath, "utf8")
|
|
62
|
+
);
|
|
63
|
+
|
|
45
64
|
// Extract config directory (parent of mappings directory)
|
|
46
65
|
const configDir = path.dirname(path.dirname(configFullPath));
|
|
47
|
-
|
|
66
|
+
|
|
48
67
|
// Read CSV data (relative to config directory)
|
|
49
68
|
const csvPath = path.resolve(configDir, config.csvFile);
|
|
50
69
|
const csvData = await readCsvFile(csvPath);
|
|
51
|
-
|
|
70
|
+
|
|
52
71
|
// Read GraphQL mutation (relative to config directory)
|
|
53
72
|
const graphqlPath = path.resolve(configDir, config.graphqlFile);
|
|
54
|
-
const mutation = fs.readFileSync(graphqlPath,
|
|
73
|
+
const mutation = fs.readFileSync(graphqlPath, "utf8");
|
|
74
|
+
|
|
75
|
+
// Process rows with optional parallelization
|
|
76
|
+
if (parallelConfig && parallelConfig.concurrency > 1) {
|
|
77
|
+
await this.processRowsConcurrently(
|
|
78
|
+
csvData,
|
|
79
|
+
mutation,
|
|
80
|
+
config.mapping,
|
|
81
|
+
entityName,
|
|
82
|
+
parallelConfig
|
|
83
|
+
);
|
|
84
|
+
} else {
|
|
85
|
+
await this.processRowsSequentially(
|
|
86
|
+
csvData,
|
|
87
|
+
mutation,
|
|
88
|
+
config.mapping,
|
|
89
|
+
entityName
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
this.metrics.finishEntityProcessing(entityName);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private async processRowsSequentially(
|
|
97
|
+
csvData: CsvRow[],
|
|
98
|
+
mutation: string,
|
|
99
|
+
mapping: Record<string, string>,
|
|
100
|
+
entityName: string
|
|
101
|
+
): Promise<void> {
|
|
102
|
+
const totalRows = csvData.length;
|
|
55
103
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
const variables = this.mapCsvRowToVariables(row,
|
|
59
|
-
|
|
104
|
+
for (let i = 0; i < csvData.length; i++) {
|
|
105
|
+
const row = csvData[i];
|
|
106
|
+
const variables = this.mapCsvRowToVariables(row, mapping);
|
|
107
|
+
|
|
60
108
|
try {
|
|
61
|
-
|
|
62
|
-
|
|
109
|
+
await this.client.executeMutation(mutation, variables);
|
|
110
|
+
this.metrics.recordSuccess(entityName);
|
|
111
|
+
|
|
112
|
+
// Show progress every 10% or at the end (only in non-verbose mode)
|
|
113
|
+
if (!this.verbose && ((i + 1) % Math.max(1, Math.floor(totalRows / 10)) === 0 || i === totalRows - 1)) {
|
|
114
|
+
const progress = (((i + 1) / totalRows) * 100).toFixed(1);
|
|
115
|
+
console.log(`📊 Progress: ${i + 1}/${totalRows} (${progress}%) ✓`);
|
|
116
|
+
}
|
|
63
117
|
} catch (error) {
|
|
64
|
-
|
|
118
|
+
this.metrics.recordFailure(entityName);
|
|
119
|
+
if (!this.verbose) {
|
|
120
|
+
console.error(`✗ Failed to create entity for row ${i + 1}:`, row, error);
|
|
121
|
+
}
|
|
65
122
|
}
|
|
66
123
|
}
|
|
67
124
|
}
|
|
68
125
|
|
|
69
|
-
private
|
|
126
|
+
private async processRowsConcurrently(
|
|
127
|
+
csvData: CsvRow[],
|
|
128
|
+
mutation: string,
|
|
129
|
+
mapping: Record<string, string>,
|
|
130
|
+
entityName: string,
|
|
131
|
+
parallelConfig: ParallelProcessingConfig
|
|
132
|
+
): Promise<void> {
|
|
133
|
+
const concurrency = parallelConfig.concurrency;
|
|
134
|
+
console.log(
|
|
135
|
+
`Processing ${csvData.length} rows with concurrency: ${concurrency}`
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
// Split data into chunks for concurrent processing
|
|
139
|
+
const chunks = this.chunkArray(csvData, concurrency);
|
|
140
|
+
let processedCount = 0;
|
|
141
|
+
const totalRows = csvData.length;
|
|
142
|
+
|
|
143
|
+
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
|
|
144
|
+
const chunk = chunks[chunkIndex];
|
|
145
|
+
const promises = chunk.map(async (row) => {
|
|
146
|
+
const variables = this.mapCsvRowToVariables(row, mapping);
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
const result = await this.client.executeMutation(mutation, variables);
|
|
150
|
+
this.metrics.recordSuccess(entityName);
|
|
151
|
+
return { success: true, result, row };
|
|
152
|
+
} catch (error) {
|
|
153
|
+
this.metrics.recordFailure(entityName);
|
|
154
|
+
return { success: false, error, row };
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
const results = await Promise.allSettled(promises);
|
|
159
|
+
processedCount += chunk.length;
|
|
160
|
+
|
|
161
|
+
// Count successes and failures in this chunk
|
|
162
|
+
let chunkSuccesses = 0;
|
|
163
|
+
let chunkFailures = 0;
|
|
164
|
+
|
|
165
|
+
results.forEach((result) => {
|
|
166
|
+
if (result.status === "fulfilled") {
|
|
167
|
+
const { success, error, row } = result.value;
|
|
168
|
+
if (success) {
|
|
169
|
+
chunkSuccesses++;
|
|
170
|
+
} else {
|
|
171
|
+
chunkFailures++;
|
|
172
|
+
if (!this.verbose) {
|
|
173
|
+
console.error(`✗ Failed to create entity for row:`, row, error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
} else {
|
|
177
|
+
chunkFailures++;
|
|
178
|
+
if (!this.verbose) {
|
|
179
|
+
console.error(`✗ Promise rejected:`, result.reason);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// Show progress update (only in non-verbose mode)
|
|
185
|
+
if (!this.verbose) {
|
|
186
|
+
const progress = ((processedCount / totalRows) * 100).toFixed(1);
|
|
187
|
+
console.log(`📊 Progress: ${processedCount}/${totalRows} (${progress}%) - Chunk ${chunkIndex + 1}: ${chunkSuccesses} ✓, ${chunkFailures} ✗`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
private chunkArray<T>(array: T[], chunkSize: number): T[][] {
|
|
193
|
+
const chunks: T[][] = [];
|
|
194
|
+
for (let i = 0; i < array.length; i += chunkSize) {
|
|
195
|
+
chunks.push(array.slice(i, i + chunkSize));
|
|
196
|
+
}
|
|
197
|
+
return chunks;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private mapCsvRowToVariables(
|
|
201
|
+
row: CsvRow,
|
|
202
|
+
mapping: Record<string, string>
|
|
203
|
+
): Record<string, any> {
|
|
70
204
|
const variables: Record<string, any> = {};
|
|
71
|
-
|
|
205
|
+
|
|
72
206
|
for (const [graphqlVar, csvColumn] of Object.entries(mapping)) {
|
|
73
207
|
if (row[csvColumn] !== undefined) {
|
|
74
208
|
variables[graphqlVar] = row[csvColumn];
|
|
75
209
|
}
|
|
76
210
|
}
|
|
77
|
-
|
|
211
|
+
|
|
78
212
|
return variables;
|
|
79
213
|
}
|
|
80
|
-
|
|
214
|
+
|
|
215
|
+
getMetrics(): MetricsCollector {
|
|
216
|
+
return this.metrics;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { MetricsCollector } from './metrics';
|
|
2
|
+
|
|
3
|
+
describe('MetricsCollector', () => {
|
|
4
|
+
let metrics: MetricsCollector;
|
|
5
|
+
|
|
6
|
+
beforeEach(() => {
|
|
7
|
+
metrics = new MetricsCollector();
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
describe('initialization', () => {
|
|
11
|
+
it('should initialize with zero counts', () => {
|
|
12
|
+
expect(metrics.getTotalProcessed()).toBe(0);
|
|
13
|
+
expect(metrics.getSuccessRate()).toBe(0);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('should have a start time', () => {
|
|
17
|
+
expect(metrics.getDurationMs()).toBeGreaterThanOrEqual(0);
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe('entity processing', () => {
|
|
22
|
+
it('should start entity processing', () => {
|
|
23
|
+
metrics.startEntityProcessing('testEntity');
|
|
24
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
25
|
+
|
|
26
|
+
expect(entityMetric).toBeDefined();
|
|
27
|
+
expect(entityMetric?.entityName).toBe('testEntity');
|
|
28
|
+
expect(entityMetric?.successCount).toBe(0);
|
|
29
|
+
expect(entityMetric?.failureCount).toBe(0);
|
|
30
|
+
expect(entityMetric?.startTime).toBeGreaterThan(0);
|
|
31
|
+
expect(entityMetric?.endTime).toBeUndefined();
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('should not create duplicate entity metrics', () => {
|
|
35
|
+
metrics.startEntityProcessing('testEntity');
|
|
36
|
+
metrics.startEntityProcessing('testEntity');
|
|
37
|
+
|
|
38
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
39
|
+
expect(entityMetric).toBeDefined();
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('should finish entity processing', () => {
|
|
43
|
+
metrics.startEntityProcessing('testEntity');
|
|
44
|
+
metrics.finishEntityProcessing('testEntity');
|
|
45
|
+
|
|
46
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
47
|
+
expect(entityMetric?.endTime).toBeGreaterThan(0);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe('success and failure recording', () => {
|
|
52
|
+
beforeEach(() => {
|
|
53
|
+
metrics.startEntityProcessing('testEntity');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('should record successes', () => {
|
|
57
|
+
metrics.recordSuccess('testEntity');
|
|
58
|
+
metrics.recordSuccess('testEntity');
|
|
59
|
+
|
|
60
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
61
|
+
expect(entityMetric?.successCount).toBe(2);
|
|
62
|
+
expect(entityMetric?.failureCount).toBe(0);
|
|
63
|
+
expect(metrics.getTotalProcessed()).toBe(2);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('should record failures', () => {
|
|
67
|
+
metrics.recordFailure('testEntity');
|
|
68
|
+
metrics.recordFailure('testEntity');
|
|
69
|
+
|
|
70
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
71
|
+
expect(entityMetric?.successCount).toBe(0);
|
|
72
|
+
expect(entityMetric?.failureCount).toBe(2);
|
|
73
|
+
expect(metrics.getTotalProcessed()).toBe(2);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('should record mixed results', () => {
|
|
77
|
+
metrics.recordSuccess('testEntity');
|
|
78
|
+
metrics.recordFailure('testEntity');
|
|
79
|
+
metrics.recordSuccess('testEntity');
|
|
80
|
+
|
|
81
|
+
const entityMetric = metrics.getEntityMetrics('testEntity');
|
|
82
|
+
expect(entityMetric?.successCount).toBe(2);
|
|
83
|
+
expect(entityMetric?.failureCount).toBe(1);
|
|
84
|
+
expect(metrics.getTotalProcessed()).toBe(3);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('should handle unknown entity gracefully', () => {
|
|
88
|
+
expect(() => {
|
|
89
|
+
metrics.recordSuccess('unknownEntity');
|
|
90
|
+
}).not.toThrow();
|
|
91
|
+
|
|
92
|
+
expect(metrics.getTotalProcessed()).toBe(0);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe('multiple entities', () => {
|
|
97
|
+
beforeEach(() => {
|
|
98
|
+
metrics.startEntityProcessing('entity1');
|
|
99
|
+
metrics.startEntityProcessing('entity2');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('should track multiple entities separately', () => {
|
|
103
|
+
metrics.recordSuccess('entity1');
|
|
104
|
+
metrics.recordSuccess('entity1');
|
|
105
|
+
metrics.recordFailure('entity2');
|
|
106
|
+
|
|
107
|
+
const entity1Metrics = metrics.getEntityMetrics('entity1');
|
|
108
|
+
const entity2Metrics = metrics.getEntityMetrics('entity2');
|
|
109
|
+
|
|
110
|
+
expect(entity1Metrics?.successCount).toBe(2);
|
|
111
|
+
expect(entity1Metrics?.failureCount).toBe(0);
|
|
112
|
+
expect(entity2Metrics?.successCount).toBe(0);
|
|
113
|
+
expect(entity2Metrics?.failureCount).toBe(1);
|
|
114
|
+
expect(metrics.getTotalProcessed()).toBe(3);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe('success rate calculation', () => {
|
|
119
|
+
beforeEach(() => {
|
|
120
|
+
metrics.startEntityProcessing('testEntity');
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('should calculate 100% success rate', () => {
|
|
124
|
+
metrics.recordSuccess('testEntity');
|
|
125
|
+
metrics.recordSuccess('testEntity');
|
|
126
|
+
|
|
127
|
+
expect(metrics.getSuccessRate()).toBe(100);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it('should calculate 0% success rate', () => {
|
|
131
|
+
metrics.recordFailure('testEntity');
|
|
132
|
+
metrics.recordFailure('testEntity');
|
|
133
|
+
|
|
134
|
+
expect(metrics.getSuccessRate()).toBe(0);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it('should calculate partial success rate', () => {
|
|
138
|
+
metrics.recordSuccess('testEntity');
|
|
139
|
+
metrics.recordFailure('testEntity');
|
|
140
|
+
metrics.recordFailure('testEntity');
|
|
141
|
+
|
|
142
|
+
expect(metrics.getSuccessRate()).toBeCloseTo(33.3, 1);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('should return 0% for no processed items', () => {
|
|
146
|
+
expect(metrics.getSuccessRate()).toBe(0);
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
describe('summary generation', () => {
|
|
151
|
+
it('should generate summary for empty metrics', () => {
|
|
152
|
+
const summary = metrics.generateSummary();
|
|
153
|
+
|
|
154
|
+
expect(summary).toContain('Total Processed: 0');
|
|
155
|
+
expect(summary).toContain('Successes: 0');
|
|
156
|
+
expect(summary).toContain('Failures: 0');
|
|
157
|
+
expect(summary).toContain('Success Rate: 0.0%');
|
|
158
|
+
expect(summary).toContain('Duration:');
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it('should generate summary with single entity', () => {
|
|
162
|
+
metrics.startEntityProcessing('testEntity');
|
|
163
|
+
metrics.recordSuccess('testEntity');
|
|
164
|
+
metrics.recordFailure('testEntity');
|
|
165
|
+
metrics.finishEntityProcessing('testEntity');
|
|
166
|
+
|
|
167
|
+
const summary = metrics.generateSummary();
|
|
168
|
+
|
|
169
|
+
expect(summary).toContain('Total Processed: 2');
|
|
170
|
+
expect(summary).toContain('Successes: 1');
|
|
171
|
+
expect(summary).toContain('Failures: 1');
|
|
172
|
+
expect(summary).toContain('Success Rate: 50.0%');
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('should generate summary with multiple entities', () => {
|
|
176
|
+
metrics.startEntityProcessing('entity1');
|
|
177
|
+
metrics.recordSuccess('entity1');
|
|
178
|
+
metrics.finishEntityProcessing('entity1');
|
|
179
|
+
|
|
180
|
+
metrics.startEntityProcessing('entity2');
|
|
181
|
+
metrics.recordFailure('entity2');
|
|
182
|
+
metrics.finishEntityProcessing('entity2');
|
|
183
|
+
|
|
184
|
+
const summary = metrics.generateSummary();
|
|
185
|
+
|
|
186
|
+
expect(summary).toContain('Total Processed: 2');
|
|
187
|
+
expect(summary).toContain('Per-Entity Breakdown');
|
|
188
|
+
expect(summary).toContain('entity1:');
|
|
189
|
+
expect(summary).toContain('entity2:');
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
describe('finish processing', () => {
|
|
194
|
+
it('should set end time and return metrics', () => {
|
|
195
|
+
metrics.startEntityProcessing('testEntity');
|
|
196
|
+
metrics.recordSuccess('testEntity');
|
|
197
|
+
|
|
198
|
+
const finalMetrics = metrics.finishProcessing();
|
|
199
|
+
|
|
200
|
+
expect(finalMetrics.endTime).toBeGreaterThan(0);
|
|
201
|
+
expect(finalMetrics.totalEntities).toBe(1);
|
|
202
|
+
expect(finalMetrics.totalSuccesses).toBe(1);
|
|
203
|
+
expect(finalMetrics.totalFailures).toBe(0);
|
|
204
|
+
expect(finalMetrics.entityMetrics.size).toBe(1);
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
});
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
export interface EntityMetrics {
|
|
2
|
+
entityName: string;
|
|
3
|
+
successCount: number;
|
|
4
|
+
failureCount: number;
|
|
5
|
+
startTime: number;
|
|
6
|
+
endTime?: number;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface ProcessingMetrics {
|
|
10
|
+
totalEntities: number;
|
|
11
|
+
totalSuccesses: number;
|
|
12
|
+
totalFailures: number;
|
|
13
|
+
entityMetrics: Map<string, EntityMetrics>;
|
|
14
|
+
requestDurations: number[];
|
|
15
|
+
startTime: number;
|
|
16
|
+
endTime?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class MetricsCollector {
|
|
20
|
+
private metrics: ProcessingMetrics;
|
|
21
|
+
|
|
22
|
+
constructor() {
|
|
23
|
+
this.metrics = {
|
|
24
|
+
totalEntities: 0,
|
|
25
|
+
totalSuccesses: 0,
|
|
26
|
+
totalFailures: 0,
|
|
27
|
+
entityMetrics: new Map(),
|
|
28
|
+
requestDurations: [],
|
|
29
|
+
startTime: Date.now(),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
startEntityProcessing(entityName: string): void {
|
|
34
|
+
if (!this.metrics.entityMetrics.has(entityName)) {
|
|
35
|
+
this.metrics.entityMetrics.set(entityName, {
|
|
36
|
+
entityName,
|
|
37
|
+
successCount: 0,
|
|
38
|
+
failureCount: 0,
|
|
39
|
+
startTime: Date.now(),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
recordSuccess(entityName: string): void {
|
|
45
|
+
const entityMetric = this.metrics.entityMetrics.get(entityName);
|
|
46
|
+
if (entityMetric) {
|
|
47
|
+
entityMetric.successCount++;
|
|
48
|
+
this.metrics.totalSuccesses++;
|
|
49
|
+
this.metrics.totalEntities++;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
recordFailure(entityName: string): void {
|
|
54
|
+
const entityMetric = this.metrics.entityMetrics.get(entityName);
|
|
55
|
+
if (entityMetric) {
|
|
56
|
+
entityMetric.failureCount++;
|
|
57
|
+
this.metrics.totalFailures++;
|
|
58
|
+
this.metrics.totalEntities++;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
finishEntityProcessing(entityName: string): void {
|
|
63
|
+
const entityMetric = this.metrics.entityMetrics.get(entityName);
|
|
64
|
+
if (entityMetric) {
|
|
65
|
+
entityMetric.endTime = Date.now();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
finishProcessing(): ProcessingMetrics {
|
|
70
|
+
this.metrics.endTime = Date.now();
|
|
71
|
+
return { ...this.metrics };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
getEntityMetrics(entityName: string): EntityMetrics | undefined {
|
|
75
|
+
return this.metrics.entityMetrics.get(entityName);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
getTotalProcessed(): number {
|
|
79
|
+
return this.metrics.totalEntities;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
getSuccessRate(): number {
|
|
83
|
+
if (this.metrics.totalEntities === 0) return 0;
|
|
84
|
+
return (this.metrics.totalSuccesses / this.metrics.totalEntities) * 100;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
recordRequestDuration(duration: number): void {
|
|
88
|
+
this.metrics.requestDurations.push(duration);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
getAverageRequestDuration(): number {
|
|
92
|
+
if (this.metrics.requestDurations.length === 0) return 0;
|
|
93
|
+
const sum = this.metrics.requestDurations.reduce((a, b) => a + b, 0);
|
|
94
|
+
return sum / this.metrics.requestDurations.length;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
getDurationMs(): number {
|
|
98
|
+
const endTime = this.metrics.endTime || Date.now();
|
|
99
|
+
return endTime - this.metrics.startTime;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
generateSummary(): string {
|
|
103
|
+
const duration = this.getDurationMs();
|
|
104
|
+
const successRate = this.getSuccessRate();
|
|
105
|
+
const avgRequestDuration = this.getAverageRequestDuration();
|
|
106
|
+
|
|
107
|
+
let summary = `\n📊 Processing Summary:\n`;
|
|
108
|
+
summary += ` Total Processed: ${this.metrics.totalEntities}\n`;
|
|
109
|
+
summary += ` ✓ Successes: ${this.metrics.totalSuccesses}\n`;
|
|
110
|
+
summary += ` ✗ Failures: ${this.metrics.totalFailures}\n`;
|
|
111
|
+
summary += ` Success Rate: ${successRate.toFixed(1)}%\n`;
|
|
112
|
+
summary += ` Duration: ${(duration / 1000).toFixed(2)}s\n`;
|
|
113
|
+
|
|
114
|
+
if (this.metrics.requestDurations.length > 0) {
|
|
115
|
+
summary += ` Avg Request Time: ${avgRequestDuration.toFixed(0)}ms\n`;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (this.metrics.entityMetrics.size > 1) {
|
|
119
|
+
summary += `\n📋 Per-Entity Breakdown:\n`;
|
|
120
|
+
for (const [entityName, entityMetric] of this.metrics.entityMetrics) {
|
|
121
|
+
const entityTotal = entityMetric.successCount + entityMetric.failureCount;
|
|
122
|
+
const entityRate = entityTotal > 0 ? (entityMetric.successCount / entityTotal) * 100 : 0;
|
|
123
|
+
const entityDuration = entityMetric.endTime ? entityMetric.endTime - entityMetric.startTime : 0;
|
|
124
|
+
|
|
125
|
+
summary += ` ${entityName}: ${entityTotal} total (${entityMetric.successCount} ✓, ${entityMetric.failureCount} ✗) - ${entityRate.toFixed(1)}% success - ${(entityDuration / 1000).toFixed(2)}s\n`;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return summary;
|
|
130
|
+
}
|
|
131
|
+
}
|