@forzalabs/remora 0.0.25 → 0.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/Constants.js
CHANGED
|
@@ -76,7 +76,6 @@ class ExecutionEnvironment {
|
|
|
76
76
|
break;
|
|
77
77
|
}
|
|
78
78
|
case 'post-process-json': {
|
|
79
|
-
// this._fetchedData = PostProcessor.doProjection(this._consumer, this._fetchedData)
|
|
80
79
|
const myProdData = this._getIntermidiate(planStep);
|
|
81
80
|
const processedData = PostProcessor_1.default.doProjection(this._consumer, myProdData);
|
|
82
81
|
this._storeIntermidiate(planStep, processedData);
|
|
@@ -87,7 +86,6 @@ class ExecutionEnvironment {
|
|
|
87
86
|
(0, Affirm_1.default)(Array.isArray(this._resultingData), 'Invalid data type, must be an array');
|
|
88
87
|
(0, Affirm_1.default)(planStep.producer, `Invalid producer in csv-to-json step`);
|
|
89
88
|
const csv = this._getIntermidiate(planStep);
|
|
90
|
-
// this._fetchedData = ParseManager.csvToJson(csv, planStep.producer)
|
|
91
89
|
const jsonData = ParseManager_1.default.csvToJson(csv, planStep.producer);
|
|
92
90
|
this._storeIntermidiate(planStep, jsonData);
|
|
93
91
|
break;
|
|
@@ -31,15 +31,13 @@ class JoinEngineClass {
|
|
|
31
31
|
// Extract field names from SQL condition like ${P.id} = ${orders.user_id}
|
|
32
32
|
const regex = /\${([^}]+)}/g;
|
|
33
33
|
const matches = Array.from(sql.matchAll(regex));
|
|
34
|
-
if (matches.length !== 2)
|
|
34
|
+
if (matches.length !== 2)
|
|
35
35
|
throw new Error(`Invalid join condition: ${sql}. Expected format: \${P.field} = \${producer.field}`);
|
|
36
|
-
}
|
|
37
36
|
const [left, right] = matches.map(m => m[1]);
|
|
38
37
|
const [leftProducer, leftField] = left.split('.');
|
|
39
38
|
const [rightProducer, rightField] = right.split('.');
|
|
40
|
-
if (!leftField || !rightField)
|
|
39
|
+
if (!leftField || !rightField)
|
|
41
40
|
throw new Error(`Invalid join condition: ${sql}. Both sides must specify a field name after the dot.`);
|
|
42
|
-
}
|
|
43
41
|
// Replace P with actual producer name
|
|
44
42
|
const actualLeftProducer = leftProducer === 'P' ? producer.name : leftProducer;
|
|
45
43
|
const actualRightProducer = rightProducer === 'P' ? producer.name : rightProducer;
|
|
@@ -55,9 +53,8 @@ class JoinEngineClass {
|
|
|
55
53
|
};
|
|
56
54
|
this.findProducerData = (producerName, producedData) => {
|
|
57
55
|
const data = producedData.find(pd => pd.producerKey === producerName);
|
|
58
|
-
if (!data)
|
|
56
|
+
if (!data)
|
|
59
57
|
throw new Error(`No data found for producer: ${producerName}`);
|
|
60
|
-
}
|
|
61
58
|
return data.data;
|
|
62
59
|
};
|
|
63
60
|
this.createLookupMap = (data, key) => {
|
|
@@ -80,11 +77,9 @@ class JoinEngineClass {
|
|
|
80
77
|
var _a;
|
|
81
78
|
(0, Affirm_1.default)(consumer, 'Invalid consumer');
|
|
82
79
|
(0, Affirm_1.default)(producedData, 'Invalid produced data');
|
|
83
|
-
if (consumer.producers.length <= 1)
|
|
80
|
+
if (consumer.producers.length <= 1)
|
|
84
81
|
return this.findProducerData(consumer.producers[0].name, producedData);
|
|
85
|
-
}
|
|
86
82
|
// Start with the first producer's data
|
|
87
|
-
// let result = this.findProducerData(consumer.producers[0].name, producedData)
|
|
88
83
|
let result = [];
|
|
89
84
|
const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
|
|
90
85
|
const consumerColumns = ConsumerEngine_1.default.compile(consumer);
|
|
@@ -9,6 +9,7 @@ const TypeCaster_1 = __importDefault(require("./TypeCaster"));
|
|
|
9
9
|
class TransformationEngineClass {
|
|
10
10
|
constructor() {
|
|
11
11
|
this.apply = (consumer, data) => {
|
|
12
|
+
var _a;
|
|
12
13
|
(0, Affirm_1.default)(consumer, 'Invalid consumer');
|
|
13
14
|
Affirm_1.default.hasValue(data, 'Invalid data');
|
|
14
15
|
const fieldsToTransform = consumer.fields.filter(field => Algo_1.default.hasVal(field.transform));
|
|
@@ -18,18 +19,19 @@ class TransformationEngineClass {
|
|
|
18
19
|
for (const field of fieldsToTransform) {
|
|
19
20
|
if (!field.transform)
|
|
20
21
|
continue;
|
|
21
|
-
const
|
|
22
|
+
const fieldKey = (_a = field.alias) !== null && _a !== void 0 ? _a : field.key;
|
|
23
|
+
const value = record[fieldKey];
|
|
22
24
|
if (!Algo_1.default.hasVal(value) && Algo_1.default.hasVal(field.default))
|
|
23
|
-
record[
|
|
25
|
+
record[fieldKey] = field.default;
|
|
24
26
|
else if (!Algo_1.default.hasVal(value))
|
|
25
27
|
continue;
|
|
26
28
|
try {
|
|
27
|
-
record[
|
|
29
|
+
record[fieldKey] = this.applyTransformations(value, field.transform, field);
|
|
28
30
|
}
|
|
29
31
|
catch (error) {
|
|
30
32
|
switch (field.onError) {
|
|
31
33
|
case 'set_default':
|
|
32
|
-
record[
|
|
34
|
+
record[fieldKey] = field.default;
|
|
33
35
|
break;
|
|
34
36
|
case 'skip':
|
|
35
37
|
break;
|