@forzalabs/remora 1.0.10 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/definitions/json_schemas/consumer-schema.json +1 -2
- package/definitions/json_schemas/producer-schema.json +1 -2
- package/definitions/json_schemas/source-schema.json +1 -2
- package/executors/ConsumerExecutor.js +9 -5
- package/executors/ExecutorOrchestrator.js +3 -2
- package/executors/ExecutorProgress.js +9 -7
- package/package.json +1 -1
|
@@ -71,13 +71,13 @@ class ConsumerExecutorClass {
|
|
|
71
71
|
return fs_1.default.createWriteStream(workPath);
|
|
72
72
|
};
|
|
73
73
|
this.processRecord = (options) => {
|
|
74
|
-
var _a;
|
|
74
|
+
var _a, _b;
|
|
75
75
|
const { consumer, fields, dimensions, producer, record, requestOptions } = options;
|
|
76
|
-
//
|
|
76
|
+
// Map to consumer fields and apply consumer field logic
|
|
77
77
|
for (const field of fields) {
|
|
78
78
|
const { cField } = field;
|
|
79
79
|
const fieldKey = (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key;
|
|
80
|
-
//
|
|
80
|
+
// Set the fixed default value for the field, or throw error if not present in the producer
|
|
81
81
|
const dimension = dimensions.find(x => x.name === cField.key);
|
|
82
82
|
if (!dimension) {
|
|
83
83
|
if (cField.fixed && Algo_1.default.hasVal(cField.default))
|
|
@@ -85,12 +85,16 @@ class ConsumerExecutorClass {
|
|
|
85
85
|
else
|
|
86
86
|
throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map(x => x.name).join(', ')})`);
|
|
87
87
|
}
|
|
88
|
-
//
|
|
88
|
+
// Change the name of the dimension
|
|
89
89
|
if (cField.alias && cField.alias !== dimension.name) {
|
|
90
90
|
record[cField.alias] = record[dimension.name];
|
|
91
91
|
delete record[dimension.name];
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
}
|
|
94
|
+
// Transformations need to be applied after the mapping since they might refer to other fields with their new names
|
|
95
|
+
for (const field of fields) {
|
|
96
|
+
const { cField } = field;
|
|
97
|
+
const fieldKey = (_b = cField.alias) !== null && _b !== void 0 ? _b : cField.key;
|
|
94
98
|
if (cField.transform)
|
|
95
99
|
record[fieldKey] = TransformationEngine_1.default.applyTransformations(record[fieldKey], cField.transform, cField, record);
|
|
96
100
|
}
|
|
@@ -76,7 +76,8 @@ class ExecutorOrchestratorClass {
|
|
|
76
76
|
const firstLine = (yield DriverHelper_1.default.quickReadFile(response.files[0].fullUri, 1))[0];
|
|
77
77
|
const header = ProducerExecutor_1.default.processHeader(firstLine, prod);
|
|
78
78
|
const prodDimensions = ProducerExecutor_1.default.reconcileHeader(header, prod);
|
|
79
|
-
|
|
79
|
+
const totalFiles = response.files.length;
|
|
80
|
+
for (const [fileIndex, file] of response.files.entries()) {
|
|
80
81
|
const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
|
|
81
82
|
const workerThreads = [];
|
|
82
83
|
for (const chunk of chunks) {
|
|
@@ -92,7 +93,7 @@ class ExecutorOrchestratorClass {
|
|
|
92
93
|
workerId: workerId,
|
|
93
94
|
options: options
|
|
94
95
|
};
|
|
95
|
-
_progress.register((currentWorkerIndex + 1).toString());
|
|
96
|
+
_progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
|
|
96
97
|
workersId.push(workerId);
|
|
97
98
|
workerThreads.push(this._executorPool.exec('executor', [workerData], {
|
|
98
99
|
on: payload => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
|
|
@@ -6,11 +6,12 @@ class ExecutorProgress {
|
|
|
6
6
|
this._FPS = 2;
|
|
7
7
|
this._lastRenderTime = 0;
|
|
8
8
|
this._lastRenderedLines = -1;
|
|
9
|
-
this.register = (name) => {
|
|
10
|
-
this.workers[name] = 0;
|
|
9
|
+
this.register = (name, producerName, fileIndex, totalFiles) => {
|
|
10
|
+
this.workers[name] = { progress: 0, producerName, fileIndex, totalFiles };
|
|
11
11
|
};
|
|
12
12
|
this.update = (name, value) => {
|
|
13
|
-
this.workers[name]
|
|
13
|
+
if (this.workers[name])
|
|
14
|
+
this.workers[name].progress = value;
|
|
14
15
|
const now = Date.now();
|
|
15
16
|
const interval = 1000 / this._FPS;
|
|
16
17
|
if (now - this._lastRenderTime >= interval) {
|
|
@@ -20,7 +21,7 @@ class ExecutorProgress {
|
|
|
20
21
|
};
|
|
21
22
|
this.complete = () => {
|
|
22
23
|
for (const key of Object.keys(this.workers)) {
|
|
23
|
-
this.workers[key] = 1;
|
|
24
|
+
this.workers[key].progress = 1;
|
|
24
25
|
}
|
|
25
26
|
this.render();
|
|
26
27
|
};
|
|
@@ -35,13 +36,14 @@ class ExecutorProgress {
|
|
|
35
36
|
}
|
|
36
37
|
this._lastRenderedLines = 0;
|
|
37
38
|
for (const key of Object.keys(this.workers)) {
|
|
38
|
-
const
|
|
39
|
-
const percentage = Math.min(100, Math.max(0,
|
|
39
|
+
const worker = this.workers[key];
|
|
40
|
+
const percentage = Math.min(100, Math.max(0, worker.progress * 100));
|
|
40
41
|
const barWidth = 30;
|
|
41
42
|
const filledWidth = Math.floor((percentage / 100) * barWidth);
|
|
42
43
|
const emptyWidth = barWidth - filledWidth;
|
|
43
44
|
const bar = '#'.repeat(filledWidth) + '-'.repeat(emptyWidth);
|
|
44
|
-
|
|
45
|
+
const fileInfo = worker.totalFiles > 1 ? ` [${worker.fileIndex + 1}/${worker.totalFiles}]` : '';
|
|
46
|
+
console.log(`Worker ${key.padStart(2, '0')}: [${bar}] ${percentage.toFixed(2)}% (${worker.producerName}${fileInfo})`);
|
|
45
47
|
this._lastRenderedLines++;
|
|
46
48
|
}
|
|
47
49
|
};
|