@forzalabs/remora 0.0.49-nasco.3 → 0.0.51-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.49-nasco',
+    cliVersion: '0.0.51-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/core/Algo.js

@@ -150,6 +150,7 @@ const algo = {
         (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
         return Math.max(...arr);
     },
-    replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace)
+    replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
+    deepClone: (data) => JSON.parse(JSON.stringify(data))
 };
 exports.default = algo;

package/definitions/json_schemas/consumer-schema.json

@@ -79,6 +79,10 @@
                             "code"
                         ],
                         "additionalProperties": false
+                    },
+                    "union": {
+                        "type": "boolean",
+                        "description": "Merges the data from the various producers in a single dataset. They must have the same output dimensions. If true, then you can't set any joins on any producer, since all producers are merged in a single dataset."
                     }
                 },
                 "required": [

package/engines/consumer/PostProcessor.js

@@ -33,9 +33,7 @@ class PostProcessorClass {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            let newDataset = yield this.dropDimensions(dataset, consumer);
-            newDataset = this.updateDimensions(newDataset, consumer);
-            newDataset = yield this.reorderDimensions(newDataset, consumer);
+            let newDataset = yield dataset.wholeUpdateDimensions(fields);
             newDataset = yield newDataset.map(record => {
                 var _a, _b;
                 for (const field of fields) {
@@ -53,38 +51,6 @@ class PostProcessorClass {
             });
             return newDataset;
         });
-        this.updateDimensions = (dataset, consumer) => {
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            dataset.updateDimensions(fields);
-            return dataset;
-        };
-        this.dropDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            const initialDimensions = dataset.getDimensions();
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            const dimensionsToKeep = new Set();
-            // First, identify which dimensions the consumer actually wants
-            for (const field of fields) {
-                const { key } = field.cField;
-                dimensionsToKeep.add(key);
-            }
-            // Create a copy to iterate over since we'll be modifying the original
-            const dimensionsCopy = [...initialDimensions];
-            const dimensionsToDrop = [];
-            for (const dim of dimensionsCopy) {
-                if (!dimensionsToKeep.has(dim.name)) {
-                    // This dimension is not wanted by the consumer, collect it for dropping
-                    dimensionsToDrop.push(dim.name);
-                }
-            }
-            // Drop all unwanted dimensions in a single optimized operation
-            if (dimensionsToDrop.length > 0)
-                yield dataset.dropDimensions(dimensionsToDrop);
-            return dataset;
-        });
-        this.reorderDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            return yield dataset.reorderDimensions(fields.map(x => { var _a; return ((_a = x.cField.alias) !== null && _a !== void 0 ? _a : x.cField.key); }));
-        });
         /**
          * Gets an array of objects (with potentially nested fields) and unpacks them to an array of objects with no nested fields
          * If some nested keys are lists, then a logic similar to a SQL JOIN is used and rows are duplicated

package/engines/dataset/Dataset.js

@@ -34,6 +34,7 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 class Dataset {
     constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
+        this._pipeline = [];
         this.getPath = () => this._path;
         this.setPath = (path) => {
             this._path = path;
@@ -122,13 +123,14 @@ class Dataset {
             const rl = (0, readline_1.createInterface)({ input: readStream, crlfDelay: Infinity });
             let batch = [];
             let lineCount = 0;
+            const dimensions = Algo_1.default.deepClone(this._dimensions);
             try {
                 for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
                     _c = rl_1_1.value;
                     _d = false;
                     const line = _c;
                     try {
-                        const record = new DatasetRecord_1.default(line, this._dimensions, this._delimiter);
+                        const record = new DatasetRecord_1.default(line, dimensions, this._delimiter);
                         batch.push(record);
                         lineCount++;
                         if (batch.length >= this._batchSize) {
@@ -520,6 +522,7 @@ class Dataset {
             if (batch.length > 0) {
                 yield processor(batch, batchIndex);
             }
+            this._iterations++;
             this._finishOperation('stream-batches');
         });
         /**
@@ -679,68 +682,52 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.updateDimensions = (fields) => {
-            this._startOperation('update-dimensions');
-            for (const field of fields) {
-                const { cField: { key, alias, hidden, fixed, default: defaultValue } } = field;
-                const currentDim = this._dimensions.find(x => x.name === key);
-                if (currentDim) {
-                    currentDim.name = alias !== null && alias !== void 0 ? alias : key;
-                    currentDim.hidden = hidden;
-                }
-                else if (fixed && Algo_1.default.hasVal(defaultValue)) {
-                    this._dimensions.push({
-                        hidden: hidden,
-                        index: this._dimensions.length,
-                        key: key,
-                        name: alias !== null && alias !== void 0 ? alias : key
-                    });
-                }
-                else {
-                    throw new Error(`Trying to update the dataset dimension "${(alias !== null && alias !== void 0 ? alias : key)}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-                }
+        /**
+         * - remove dimension
+         * - rename a dimension
+         * - change hidden flag
+         * - move a dimension
+         */
+        this.wholeUpdateDimensions = (fields) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            let updates = [];
+            // Add all the updates
+            for (let i = 0; i < fields.length; i++) {
+                const { cField } = fields[i];
+                const currentMatch = structuredClone(this._dimensions.find(x => x.name === cField.key));
+                updates.push({
+                    currentDimension: currentMatch,
+                    newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
+                    newHidden: cField.hidden,
+                    newPosition: i,
+                    toDelete: false
+                });
             }
-            this._finishOperation('update-dimensions');
-            return this;
-        };
-        this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
-            if (dimensionNames.length === 0)
-                return this;
-            this._startOperation('drop-dimensions');
-            const toRemove = this._dimensions
-                .filter(x => dimensionNames.includes(x.name))
-                .sort((a, b) => b.index - a.index);
-            const toRemoveNames = toRemove.map(x => x.name);
-            yield this.map(record => record.dropDimensions(toRemoveNames));
-            this._dimensions = this._dimensions
-                .filter(x => !dimensionNames.includes(x.name))
-                .map((x, i) => (Object.assign(Object.assign({}, x), { index: i })));
-            this._finishOperation('drop-dimensions');
-            return this;
-        });
-        this.reorderDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
-            if (dimensionNames.length === 0)
+            // Add all the updates to remove dimensions
+            for (const dim of this._dimensions) {
+                if (!updates.find(x => { var _a; return ((_a = x.currentDimension) === null || _a === void 0 ? void 0 : _a.name) === dim.name; }))
+                    updates.push({ currentDimension: dim, toDelete: true });
+            }
+            // Now keep only the updates that actually change something
+            updates = updates.filter(x => x.toDelete
+                || !x.currentDimension
+                || (x.currentDimension && (x.currentDimension.name !== x.newName
+                    || (Algo_1.default.hasVal(x.newHidden) && x.newHidden !== x.currentDimension.hidden)
+                    || x.newPosition !== x.currentDimension.index)));
+            if (updates.length === 0)
                 return this;
-            this._startOperation('reorder-dimensions');
-            // Validate that all provided dimension names exist
-            const existingNames = this._dimensions.map(d => d.name);
-            const missingDimensions = dimensionNames.filter(name => !existingNames.includes(name));
-            (0, Affirm_1.default)(missingDimensions.length === 0, `Cannot reorder dimensions. Missing dimensions: ${missingDimensions.join(', ')}`);
-            // Validate that all existing dimensions are included
-            const extraDimensions = existingNames.filter(name => !dimensionNames.includes(name));
-            (0, Affirm_1.default)(extraDimensions.length === 0, `Cannot reorder dimensions. All existing dimensions must be included. Missing: ${extraDimensions.join(', ')}`);
-            const movements = dimensionNames.map((name, index) => {
-                const dim = this._dimensions.find(x => x.name === name);
-                const newDim = structuredClone(dim);
-                newDim.index = index;
-                return { newDimension: newDim, oldDimension: dim };
+            let updatedDimensions = null;
+            const newDataset = yield this.map(record => {
+                for (const update of updates) {
+                    record.wholeUpdateDimension(update);
+                }
+                record._dimensions.sort((a, b) => a.index - b.index);
+                if (!updatedDimensions)
+                    updatedDimensions = record._dimensions;
+                return record;
             });
-            // Update metadata
-            this._dimensions = movements.map(x => x.newDimension);
-            // Reorder the data in the file using streaming approach
-            yield this.map(record => record.reorderDimensions(movements));
-            this._finishOperation('reorder-dimensions');
-            return this;
+            this._dimensions = updatedDimensions;
+            return newDataset;
         });
         this.print = (...args_1) => __awaiter(this, [...args_1], void 0, function* (count = 3, full = false) {
             console.log(`DS ${this._name} (${this._size} | ${this._iterations})`);
@@ -862,6 +849,7 @@ class Dataset {
         this._size = 0;
         this._iterations = 0;
         this._operations = [];
+        this._pipeline = [];
         const datasetName = this._name
             .replace(/[^a-zA-Z0-9_-]/g, '_')
             .replace(/_{2,}/g, '_')

package/engines/dataset/DatasetRecord.js

@@ -6,44 +6,45 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
 class DatasetRecord {
     constructor(row, dimensions, delimiter) {
+        this.parse = (row, delimiter, dimensions) => {
+            if (!this.isEmpty() && dimensions.length > 0) {
+                const parts = row.split(delimiter);
+                for (let i = 0; i < dimensions.length; i++) {
+                    const dim = dimensions[i];
+                    this._value[dim.name] = parts[i];
+                }
+            }
+        };
         this.stringify = () => this._dimensions.map(x => this._value[x.name]).join(this._delimiter);
         this.isEmpty = () => { var _a; return ((_a = this._row) === null || _a === void 0 ? void 0 : _a.trim().length) === 0; };
         this.getRaw = () => this._row;
         this.getValue = (dimension) => this._value[dimension];
         this.setValue = (dimension, value) => this._value[dimension] = value;
-        this.dropDimensions = (dimensionNames) => {
-            if (dimensionNames.length === 0)
-                return this;
-            // Remove the dimensions from the internal value storage
-            for (const dimensionName of dimensionNames)
-                delete this._value[dimensionName];
-            // Filter out the dropped dimensions from the dimensions array
-            this._dimensions = this._dimensions.filter(dim => !dimensionNames.includes(dim.name));
-            // Update the indices of remaining dimensions
-            this._dimensions.forEach((dim, index) => {
-                dim.index = index;
-            });
-            return this;
-        };
-        this.reorderDimensions = (movements) => {
-            if (movements.length === 0)
-                return this;
-            // Parse the raw row again with the new dimension order
-            const parts = this._row.split(this._delimiter);
-            const newValue = {};
-            // Create the new dimensions array and remap values
-            this._dimensions = [];
-            for (const move of movements) {
-                const { newDimension, oldDimension } = move;
-                // Add the dimension with its new index
-                this._dimensions.push(newDimension);
-                // Remap the value from the old position to the new dimension name
-                if (parts.length > oldDimension.index) {
-                    newValue[newDimension.name] = parts[oldDimension.index];
-                }
+        this.wholeUpdateDimension = (update) => {
+            var _a;
+            if (update.toDelete) {
+                // To remove
+                delete this._value[update.currentDimension.name];
+                this._dimensions = this._dimensions.filter(x => x.key !== update.currentDimension.name);
+            }
+            else if (!update.currentDimension) {
+                // To create (at the right position)
+                const newDimension = { index: update.newPosition, key: update.newName, name: update.newName, hidden: update.newHidden };
+                this._value[newDimension.name] = null;
+                this._dimensions = [...this._dimensions, newDimension];
+            }
+            else {
+                // Change: name, hidden, position
+                const index = this._dimensions.findIndex(x => x.key === update.currentDimension.name);
+                const currentDim = this._dimensions[index];
+                const updatedDim = { name: update.newName, key: (_a = currentDim.key) !== null && _a !== void 0 ? _a : update.newName, hidden: update.newHidden, index: update.newPosition };
+                this._value[updatedDim.name] = this._value[currentDim.name];
+                if (updatedDim.name !== currentDim.name)
+                    delete this._value[currentDim.name];
+                const newDimensions = [...this._dimensions];
+                newDimensions.splice(index, 1, updatedDim);
+                this._dimensions = newDimensions;
             }
-            // Update the value mapping
-            this._value = newValue;
             return this;
         };
         this.toJSON = () => {
@@ -70,16 +71,10 @@ class DatasetRecord {
             return line;
         };
         this._row = row;
-        this._dimensions = dimensions.sort((a, b) => a.index - b.index);
+        this._dimensions = dimensions;
         this._delimiter = delimiter;
         this._value = {};
-        if (!this.isEmpty() && this._dimensions.length > 0) {
-            const parts = row.split(delimiter);
-            for (let i = 0; i < dimensions.length; i++) {
-                const dim = dimensions[i];
-                this._value[dim.name] = parts[i];
-            }
-        }
+        this.parse(row, delimiter, this._dimensions);
     }
 }
 exports.default = DatasetRecord;

package/engines/execution/ExecutionEnvironment.js

@@ -17,7 +17,6 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
 const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
 const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
@@ -70,27 +69,6 @@ class ExecutionEnvironment {
                             this._storeIntermidiate(planStep, dataset);
                             break;
                         }
-                        case 'read-file-whole': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-whole step`);
-                            const fileData = yield ProducerEngine_1.default.readFile(planStep.producer, { readmode: 'all' });
-                            this._storeIntermidiate(planStep, fileData.dataset);
-                            break;
-                        }
-                        case 'read-file-lines': {
-                            (0, Affirm_1.default)(planStep.lines, `Invalid lines in read-file-lines step`);
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
-                            const { producer, lines: { from, to } } = planStep;
-                            const fileData = yield ProducerEngine_1.default.readFile(producer, { readmode: 'lines', lines: { from, to } });
-                            this._storeIntermidiate(planStep, fileData.dataset);
-                            break;
-                        }
-                        case 'download-file-locally': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in download-file-locally step`);
-                            const { producer } = planStep;
-                            const readRes = yield ProducerEngine_1.default.readFile(producer, { readmode: 'download' });
-                            this._storeIntermidiate(planStep, readRes.dataset);
-                            break;
-                        }
                         case 'load-dataset': {
                             (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
                             const { producer } = planStep;
@@ -171,14 +149,14 @@ class ExecutionEnvironment {
                         }
                         default: throw new Error(`Invalid execution plan step type "${planStep.type}"`);
                     }
-                    Logger_1.default.log(`Completed step: ${planStep.type}`);
+                    Logger_1.default.log(`\tCompleted step: ${planStep.type}`);
                 }
             }
             catch (error) {
                 const ds = (_a = this._resultingDataset) !== null && _a !== void 0 ? _a : this._getIntermidiate(currentStep);
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getSize()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
-                Logger_1.default.log(`Error at step ${currentStep.type}:\n\t${error}`);
+                Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
                 throw error;
             }
             Logger_1.default.log(`Completed execution of consumer:\n\tSize: ${result._stats.size}\n\tCycles: ${result._stats.cycles}\n\tTime: ${result._stats.elapsedMS}\n\tOperations: ${Logger_1.default.formatList(result._stats.operations)}`);

package/engines/transform/JoinEngine.js

@@ -92,6 +92,8 @@ class JoinEngineClass {
             (0, Affirm_1.default)(producedData, 'Invalid produced data');
             if (consumer.producers.length <= 1)
                 return this.findProducerData(consumer.producers[0].name, producedData);
+            if (consumer.producers.some(x => x.union))
+                return yield this.union(consumer, producedData);
             const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
             const consumerColumns = ConsumerEngine_1.default.compile(consumer);
             // Create a new dataset for the joined result
@@ -132,6 +134,21 @@ class JoinEngineClass {
             }
             return resultDataset;
         });
+        this.union = (consumer, producedData) => __awaiter(this, void 0, void 0, function* () {
+            const getDimensionsKey = (ds) => ds.getDimensions().map(x => x.name.trim()).join(';').trim();
+            const mainDataset = producedData[0].dataset;
+            const mainDimKey = getDimensionsKey(mainDataset);
+            const otherProducedData = producedData.slice(1);
+            for (const prodData of otherProducedData) {
+                const prodDimKey = getDimensionsKey(prodData.dataset);
+                if (mainDimKey !== prodDimKey)
+                    throw new Error(`On consumer "${consumer.name}", can't union the dataset "${prodData.dataset['_name']}" (producer: ${prodData.producerKey}) because the dimensions are different from the main dataset "${mainDataset['_name']}" (producer: ${producedData[0].producerKey}). "${mainDimKey}" != "${prodDimKey}"`);
+                yield prodData.dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    yield mainDataset.append(batch);
+                }));
+            }
+            return mainDataset;
+        });
         this.performStreamingJoin = (leftDataset, rightLookup, condition, relationship, consumerColumns, resultDataset) => __awaiter(this, void 0, void 0, function* () {
             const joinedRecords = [];
             const batchSize = leftDataset.getBatchSize();

package/engines/validation/Validator.js

@@ -90,6 +90,12 @@ class ValidatorClass {
                     errors.push('No producers found');
                 if (producers.some(x => !x))
                     errors.push(`Invalid producer found in consumer "${consumer.name}"`);
+                if (consumer.producers.length > 0) {
+                    const withJoins = consumer.producers.filter(x => (Algo_1.default.hasVal(x.joins) && x.joins.length > 0) || !x.union);
+                    const withUnions = consumer.producers.filter(x => x.union === true);
+                    if (withJoins.length > 0 && withUnions.length)
+                        errors.push(`Multiple producers in consumer have mixed "joins" and "union": you can either have multiple producers with "joins" or multiple producers with "union", but not both (joins: ${withJoins.map(x => x.name).join(', ')}; unions: ${withUnions.map(x => x.name).join(', ')})`);
+                }
                 // Validation on sources
                 const sources = producers.map(x => Environment_1.default.getSource(x.source));
                 if (sources.length === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.49-nasco.3",
+    "version": "0.0.51-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,6 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
+        "dev": "clear && npx tsx scripts/dev.ts",
         "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",