@forzalabs/remora 0.0.50-nasco.3 → 0.0.51-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.50-nasco',
+    cliVersion: '0.0.51-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -79,6 +79,10 @@
                             "code"
                         ],
                         "additionalProperties": false
+                    },
+                    "union": {
+                        "type": "boolean",
+                        "description": "Merges the data from the various producers in a single dataset. They must have the same output dimensions. If true, then you can't set any joins on any producer, since all producers are merged in a single dataset."
                     }
                 },
                 "required": [

package/engines/transform/JoinEngine.js

@@ -92,6 +92,8 @@ class JoinEngineClass {
             (0, Affirm_1.default)(producedData, 'Invalid produced data');
             if (consumer.producers.length <= 1)
                 return this.findProducerData(consumer.producers[0].name, producedData);
+            if (consumer.producers.some(x => x.union))
+                return yield this.union(consumer, producedData);
             const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
             const consumerColumns = ConsumerEngine_1.default.compile(consumer);
             // Create a new dataset for the joined result
@@ -132,6 +134,21 @@ class JoinEngineClass {
             }
             return resultDataset;
         });
+        this.union = (consumer, producedData) => __awaiter(this, void 0, void 0, function* () {
+            const getDimensionsKey = (ds) => ds.getDimensions().map(x => x.name.trim()).join(';').trim();
+            const mainDataset = producedData[0].dataset;
+            const mainDimKey = getDimensionsKey(mainDataset);
+            const otherProducedData = producedData.slice(1);
+            for (const prodData of otherProducedData) {
+                const prodDimKey = getDimensionsKey(prodData.dataset);
+                if (mainDimKey !== prodDimKey)
+                    throw new Error(`On consumer "${consumer.name}", can't union the dataset "${prodData.dataset['_name']}" (producer: ${prodData.producerKey}) because the dimensions are different from the main dataset "${mainDataset['_name']}" (producer: ${producedData[0].producerKey}). "${mainDimKey}" != "${prodDimKey}"`);
+                yield prodData.dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    yield mainDataset.append(batch);
+                }));
+            }
+            return mainDataset;
+        });
         this.performStreamingJoin = (leftDataset, rightLookup, condition, relationship, consumerColumns, resultDataset) => __awaiter(this, void 0, void 0, function* () {
             const joinedRecords = [];
             const batchSize = leftDataset.getBatchSize();

package/engines/validation/Validator.js

@@ -90,6 +90,12 @@ class ValidatorClass {
                     errors.push('No producers found');
                 if (producers.some(x => !x))
                     errors.push(`Invalid producer found in consumer "${consumer.name}"`);
+                if (consumer.producers.length > 0) {
+                    const withJoins = consumer.producers.filter(x => (Algo_1.default.hasVal(x.joins) && x.joins.length > 0) || !x.union);
+                    const withUnions = consumer.producers.filter(x => x.union === true);
+                    if (withJoins.length > 0 && withUnions.length)
+                        errors.push(`Multiple producers in consumer have mixed "joins" and "union": you can either have multiple producers with "joins" or multiple producers with "union", but not both (joins: ${withJoins.map(x => x.name).join(', ')}; unions: ${withUnions.map(x => x.name).join(', ')})`);
+                }
                 // Validation on sources
                 const sources = producers.map(x => Environment_1.default.getSource(x.source));
                 if (sources.length === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.50-nasco.3",
+    "version": "0.0.51-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,