@nu-art/ts-common 0.201.1 → 0.201.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/modules/CSVModule.d.ts +1 -0
- package/modules/CSVModule.js +14 -0
- package/package.json +1 -1
package/modules/CSVModule.d.ts
CHANGED
|
@@ -33,6 +33,7 @@ declare class CSVModule_Class extends Module<Config> {
|
|
|
33
33
|
readCsvFromFile<T extends Partial<StringMap>>(inputFile: string, readOptions?: ReadOptions): Promise<T[]>;
|
|
34
34
|
readCsvFromBuffer<T extends Partial<StringMap>>(buffer: Buffer, readOptions?: ReadOptions): Promise<T[]>;
|
|
35
35
|
readCsvFromStream<T extends Partial<StringMap>>(stream: Readable, readOptions?: ReadOptions): Promise<T[]>;
|
|
36
|
+
forEachCsvRowFromStream<T extends Partial<StringMap>>(stream: Readable, callback: (instance: T) => Promise<void>, readOptions?: ReadOptions, queueCount?: number): Promise<void>;
|
|
36
37
|
private createReadParserOptions;
|
|
37
38
|
}
|
|
38
39
|
export declare const CSVModule: CSVModule_Class;
|
package/modules/CSVModule.js
CHANGED
|
@@ -35,6 +35,7 @@ const fs_1 = require("fs");
|
|
|
35
35
|
const module_1 = require("../core/module");
|
|
36
36
|
const stream_1 = require("stream");
|
|
37
37
|
const csvParser = require("csv-parser");
|
|
38
|
+
const queue_1 = require("../utils/queue");
|
|
38
39
|
const DefaultConfig = {
|
|
39
40
|
options: {
|
|
40
41
|
fieldSeparator: ',',
|
|
@@ -98,6 +99,19 @@ class CSVModule_Class extends module_1.Module {
|
|
|
98
99
|
});
|
|
99
100
|
});
|
|
100
101
|
}
|
|
102
|
+
forEachCsvRowFromStream(stream, callback, readOptions = {}, queueCount = 5) {
|
|
103
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
104
|
+
return new Promise((resolve, reject) => {
|
|
105
|
+
const instancesQueue = new queue_1.Queue('instancesQueue');
|
|
106
|
+
instancesQueue.setParallelCount(queueCount);
|
|
107
|
+
stream
|
|
108
|
+
.pipe(csvParser(this.createReadParserOptions(readOptions)))
|
|
109
|
+
.on('data', (instance) => instancesQueue.addItem(() => callback(instance)))
|
|
110
|
+
.on('error', (err) => reject(err))
|
|
111
|
+
.on('end', () => resolve());
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
}
|
|
101
115
|
createReadParserOptions(readOptions) {
|
|
102
116
|
return {
|
|
103
117
|
mapHeaders: (args) => {
|