@forzalabs/remora 0.0.48-nasco.3 → 0.0.50-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.48-nasco',
+    cliVersion: '0.0.50-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/core/Algo.js

@@ -150,6 +150,7 @@ const algo = {
         (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
         return Math.max(...arr);
     },
-    replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace)
+    replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),
+    deepClone: (data) => JSON.parse(JSON.stringify(data))
 };
 exports.default = algo;

package/drivers/DriverHelper.js

@@ -24,9 +24,12 @@ const readline_1 = require("readline");
 const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
 const Logger_1 = __importDefault(require("../helper/Logger"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
 const DriverHelper = {
-    appendToUnifiedFile: (stream, fileKey, destinationPath, append, headerLine, fileType) => __awaiter(void 0, void 0, void 0, function* () {
-        const shouldValidateHeader = fileType === 'CSV' || fileType === 'TXT';
+    appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
+        (0, Affirm_1.default)(options, 'Invalid options');
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow } = options;
+        const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
         const headerValidationTransform = new stream_1.Transform({
@@ -36,7 +39,6 @@ const DriverHelper = {
                     const lines = chunkStr.split('\n');
                     if (isFirstLine && lines.length > 0) {
                         const firstLine = lines[0];
-                        // Validate header only for CSV and TXT files
                         if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
                             const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
                             Logger_1.default.log(msg);

package/drivers/LocalDriver.js

@@ -83,12 +83,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -102,12 +105,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
@@ -122,12 +128,20 @@ class LocalSourceDriver {
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
                 // Copy and validate header in a single stream pass
                 const readStream = fs.createReadStream(sourceFilePath);
-                return DriverHelper_1.default.appendToUnifiedFile(readStream, fileKey, dataset.getPath(), appendMode, headerLine, file.fileType);
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream: readStream,
+                    fileKey,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
+                });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 // Get header line from the first file
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 // Copy files sequentially to avoid file conflicts
@@ -222,7 +236,7 @@ class LocalSourceDriver {
             }
             return lines;
         });
-        this._get = (request, index) => __awaiter(this, void 0, void 0, function* () {
+        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
             const { fileKey, fileType, options } = request;
             let lineFrom, lineTo, sheetName, hasHeaderRow;
             if (options) {
@@ -257,6 +271,12 @@ class LocalSourceDriver {
                         lines = yield this._readXmlLines(fileUrl);
                     break;
             }
+            const firstLine = lines[0];
+            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                Logger_1.default.log(msg);
+                throw new Error(msg);
+            }
             // If this is not the first file read in a pattern match AND the file type has an header,
             // then I need to remove the header from the resulting lines or the header will be duplicated
             if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {

package/drivers/S3Driver.js

@@ -31,6 +31,7 @@ const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -235,11 +236,20 @@ class S3SourceDriver {
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
                 const stream = response.Body;
-                return DriverHelper_1.default.appendToUnifiedFile(stream, fileUrl, dataset.getPath(), appendMode, headerLine, file.fileType);
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream,
+                    fileKey: fileUrl,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
+                });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 // Get header line from the first file
                 const firstFileCommand = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,

package/engines/consumer/PostProcessor.js

@@ -33,9 +33,7 @@ class PostProcessorClass {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            let newDataset = yield this.dropDimensions(dataset, consumer);
-            newDataset = this.updateDimensions(newDataset, consumer);
-            newDataset = yield this.reorderDimensions(newDataset, consumer);
+            let newDataset = yield dataset.wholeUpdateDimensions(fields);
             newDataset = yield newDataset.map(record => {
                 var _a, _b;
                 for (const field of fields) {
@@ -53,38 +51,6 @@ class PostProcessorClass {
             });
             return newDataset;
         });
-        this.updateDimensions = (dataset, consumer) => {
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            dataset.updateDimensions(fields);
-            return dataset;
-        };
-        this.dropDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            const initialDimensions = dataset.getDimensions();
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            const dimensionsToKeep = new Set();
-            // First, identify which dimensions the consumer actually wants
-            for (const field of fields) {
-                const { key } = field.cField;
-                dimensionsToKeep.add(key);
-            }
-            // Create a copy to iterate over since we'll be modifying the original
-            const dimensionsCopy = [...initialDimensions];
-            const dimensionsToDrop = [];
-            for (const dim of dimensionsCopy) {
-                if (!dimensionsToKeep.has(dim.name)) {
-                    // This dimension is not wanted by the consumer, collect it for dropping
-                    dimensionsToDrop.push(dim.name);
-                }
-            }
-            // Drop all unwanted dimensions in a single optimized operation
-            if (dimensionsToDrop.length > 0)
-                yield dataset.dropDimensions(dimensionsToDrop);
-            return dataset;
-        });
-        this.reorderDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            return yield dataset.reorderDimensions(fields.map(x => { var _a; return ((_a = x.cField.alias) !== null && _a !== void 0 ? _a : x.cField.key); }));
-        });
         /**
          * Gets an array of objects (with potentially nested fields) and unpacks them to an array of objects with no nested fields
          * If some nested keys are lists, then a logic similar to a SQL JOIN is used and rows are duplicated

package/engines/dataset/Dataset.js

@@ -34,6 +34,7 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 class Dataset {
     constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
+        this._pipeline = [];
         this.getPath = () => this._path;
         this.setPath = (path) => {
             this._path = path;
@@ -122,13 +123,14 @@ class Dataset {
             const rl = (0, readline_1.createInterface)({ input: readStream, crlfDelay: Infinity });
             let batch = [];
             let lineCount = 0;
+            const dimensions = Algo_1.default.deepClone(this._dimensions);
             try {
                 for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
                     _c = rl_1_1.value;
                     _d = false;
                     const line = _c;
                     try {
-                        const record = new DatasetRecord_1.default(line, this._dimensions, this._delimiter);
+                        const record = new DatasetRecord_1.default(line, dimensions, this._delimiter);
                         batch.push(record);
                         lineCount++;
                         if (batch.length >= this._batchSize) {
@@ -520,6 +522,7 @@ class Dataset {
             if (batch.length > 0) {
                 yield processor(batch, batchIndex);
             }
+            this._iterations++;
             this._finishOperation('stream-batches');
         });
         /**
@@ -679,68 +682,52 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.updateDimensions = (fields) => {
-            this._startOperation('update-dimensions');
-            for (const field of fields) {
-                const { cField: { key, alias, hidden, fixed, default: defaultValue } } = field;
-                const currentDim = this._dimensions.find(x => x.name === key);
-                if (currentDim) {
-                    currentDim.name = alias !== null && alias !== void 0 ? alias : key;
-                    currentDim.hidden = hidden;
-                }
-                else if (fixed && Algo_1.default.hasVal(defaultValue)) {
-                    this._dimensions.push({
-                        hidden: hidden,
-                        index: this._dimensions.length,
-                        key: key,
-                        name: alias !== null && alias !== void 0 ? alias : key
-                    });
-                }
-                else {
-                    throw new Error(`Trying to update the dataset dimension "${(alias !== null && alias !== void 0 ? alias : key)}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-                }
+        /**
+         * - remove dimension
+         * - rename a dimension
+         * - change hidden flag
+         * - move a dimension
+         */
+        this.wholeUpdateDimensions = (fields) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            let updates = [];
+            // Add all the updates
+            for (let i = 0; i < fields.length; i++) {
+                const { cField } = fields[i];
+                const currentMatch = structuredClone(this._dimensions.find(x => x.name === cField.key));
+                updates.push({
+                    currentDimension: currentMatch,
+                    newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
+                    newHidden: cField.hidden,
+                    newPosition: i,
+                    toDelete: false
+                });
             }
-            this._finishOperation('update-dimensions');
-            return this;
-        };
-        this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
-            if (dimensionNames.length === 0)
-                return this;
-            this._startOperation('drop-dimensions');
-            const toRemove = this._dimensions
-                .filter(x => dimensionNames.includes(x.name))
-                .sort((a, b) => b.index - a.index);
-            const toRemoveNames = toRemove.map(x => x.name);
-            yield this.map(record => record.dropDimensions(toRemoveNames));
-            this._dimensions = this._dimensions
-                .filter(x => !dimensionNames.includes(x.name))
-                .map((x, i) => (Object.assign(Object.assign({}, x), { index: i })));
-            this._finishOperation('drop-dimensions');
-            return this;
-        });
-        this.reorderDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
-            if (dimensionNames.length === 0)
+            // Add all the updates to remove dimensions
+            for (const dim of this._dimensions) {
+                if (!updates.find(x => { var _a; return ((_a = x.currentDimension) === null || _a === void 0 ? void 0 : _a.name) === dim.name; }))
+                    updates.push({ currentDimension: dim, toDelete: true });
+            }
+            // Now keep only the updates that actually change something
+            updates = updates.filter(x => x.toDelete
+                || !x.currentDimension
+                || (x.currentDimension && (x.currentDimension.name !== x.newName
+                    || (Algo_1.default.hasVal(x.newHidden) && x.newHidden !== x.currentDimension.hidden)
+                    || x.newPosition !== x.currentDimension.index)));
+            if (updates.length === 0)
                 return this;
-            this._startOperation('reorder-dimensions');
-            // Validate that all provided dimension names exist
-            const existingNames = this._dimensions.map(d => d.name);
-            const missingDimensions = dimensionNames.filter(name => !existingNames.includes(name));
-            (0, Affirm_1.default)(missingDimensions.length === 0, `Cannot reorder dimensions. Missing dimensions: ${missingDimensions.join(', ')}`);
-            // Validate that all existing dimensions are included
-            const extraDimensions = existingNames.filter(name => !dimensionNames.includes(name));
-            (0, Affirm_1.default)(extraDimensions.length === 0, `Cannot reorder dimensions. All existing dimensions must be included. Missing: ${extraDimensions.join(', ')}`);
-            const movements = dimensionNames.map((name, index) => {
-                const dim = this._dimensions.find(x => x.name === name);
-                const newDim = structuredClone(dim);
-                newDim.index = index;
-                return { newDimension: newDim, oldDimension: dim };
+            let updatedDimensions = null;
+            const newDataset = yield this.map(record => {
+                for (const update of updates) {
+                    record.wholeUpdateDimension(update);
+                }
+                record._dimensions.sort((a, b) => a.index - b.index);
+                if (!updatedDimensions)
+                    updatedDimensions = record._dimensions;
+                return record;
             });
-            // Update metadata
-            this._dimensions = movements.map(x => x.newDimension);
-            // Reorder the data in the file using streaming approach
-            yield this.map(record => record.reorderDimensions(movements));
-            this._finishOperation('reorder-dimensions');
-            return this;
+            this._dimensions = updatedDimensions;
+            return newDataset;
         });
         this.print = (...args_1) => __awaiter(this, [...args_1], void 0, function* (count = 3, full = false) {
             console.log(`DS ${this._name} (${this._size} | ${this._iterations})`);
@@ -862,6 +849,7 @@ class Dataset {
         this._size = 0;
         this._iterations = 0;
         this._operations = [];
+        this._pipeline = [];
         const datasetName = this._name
             .replace(/[^a-zA-Z0-9_-]/g, '_')
             .replace(/_{2,}/g, '_')

package/engines/dataset/DatasetRecord.js

@@ -6,44 +6,45 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
 class DatasetRecord {
     constructor(row, dimensions, delimiter) {
+        this.parse = (row, delimiter, dimensions) => {
+            if (!this.isEmpty() && dimensions.length > 0) {
+                const parts = row.split(delimiter);
+                for (let i = 0; i < dimensions.length; i++) {
+                    const dim = dimensions[i];
+                    this._value[dim.name] = parts[i];
+                }
+            }
+        };
         this.stringify = () => this._dimensions.map(x => this._value[x.name]).join(this._delimiter);
         this.isEmpty = () => { var _a; return ((_a = this._row) === null || _a === void 0 ? void 0 : _a.trim().length) === 0; };
         this.getRaw = () => this._row;
         this.getValue = (dimension) => this._value[dimension];
         this.setValue = (dimension, value) => this._value[dimension] = value;
-        this.dropDimensions = (dimensionNames) => {
-            if (dimensionNames.length === 0)
-                return this;
-            // Remove the dimensions from the internal value storage
-            for (const dimensionName of dimensionNames)
-                delete this._value[dimensionName];
-            // Filter out the dropped dimensions from the dimensions array
-            this._dimensions = this._dimensions.filter(dim => !dimensionNames.includes(dim.name));
-            // Update the indices of remaining dimensions
-            this._dimensions.forEach((dim, index) => {
-                dim.index = index;
-            });
-            return this;
-        };
-        this.reorderDimensions = (movements) => {
-            if (movements.length === 0)
-                return this;
-            // Parse the raw row again with the new dimension order
-            const parts = this._row.split(this._delimiter);
-            const newValue = {};
-            // Create the new dimensions array and remap values
-            this._dimensions = [];
-            for (const move of movements) {
-                const { newDimension, oldDimension } = move;
-                // Add the dimension with its new index
-                this._dimensions.push(newDimension);
-                // Remap the value from the old position to the new dimension name
-                if (parts.length > oldDimension.index) {
-                    newValue[newDimension.name] = parts[oldDimension.index];
-                }
+        this.wholeUpdateDimension = (update) => {
+            var _a;
+            if (update.toDelete) {
+                // To remove
+                delete this._value[update.currentDimension.name];
+                this._dimensions = this._dimensions.filter(x => x.key !== update.currentDimension.name);
+            }
+            else if (!update.currentDimension) {
+                // To create (at the right position)
+                const newDimension = { index: update.newPosition, key: update.newName, name: update.newName, hidden: update.newHidden };
+                this._value[newDimension.name] = null;
+                this._dimensions = [...this._dimensions, newDimension];
+            }
+            else {
+                // Change: name, hidden, position
+                const index = this._dimensions.findIndex(x => x.key === update.currentDimension.name);
+                const currentDim = this._dimensions[index];
+                const updatedDim = { name: update.newName, key: (_a = currentDim.key) !== null && _a !== void 0 ? _a : update.newName, hidden: update.newHidden, index: update.newPosition };
+                this._value[updatedDim.name] = this._value[currentDim.name];
+                if (updatedDim.name !== currentDim.name)
+                    delete this._value[currentDim.name];
+                const newDimensions = [...this._dimensions];
+                newDimensions.splice(index, 1, updatedDim);
+                this._dimensions = newDimensions;
             }
-            // Update the value mapping
-            this._value = newValue;
             return this;
         };
         this.toJSON = () => {
@@ -70,16 +71,10 @@ class DatasetRecord {
             return line;
         };
         this._row = row;
-        this._dimensions = dimensions.sort((a, b) => a.index - b.index);
+        this._dimensions = dimensions;
         this._delimiter = delimiter;
         this._value = {};
-        if (!this.isEmpty() && this._dimensions.length > 0) {
-            const parts = row.split(delimiter);
-            for (let i = 0; i < dimensions.length; i++) {
-                const dim = dimensions[i];
-                this._value[dim.name] = parts[i];
-            }
-        }
+        this.parse(row, delimiter, this._dimensions);
     }
 }
 exports.default = DatasetRecord;

package/engines/execution/ExecutionEnvironment.js

@@ -17,7 +17,6 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
 const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
 const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
@@ -70,27 +69,6 @@ class ExecutionEnvironment {
                             this._storeIntermidiate(planStep, dataset);
                             break;
                         }
-                        case 'read-file-whole': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-whole step`);
-                            const fileData = yield ProducerEngine_1.default.readFile(planStep.producer, { readmode: 'all' });
-                            this._storeIntermidiate(planStep, fileData.dataset);
-                            break;
-                        }
-                        case 'read-file-lines': {
-                            (0, Affirm_1.default)(planStep.lines, `Invalid lines in read-file-lines step`);
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
-                            const { producer, lines: { from, to } } = planStep;
-                            const fileData = yield ProducerEngine_1.default.readFile(producer, { readmode: 'lines', lines: { from, to } });
-                            this._storeIntermidiate(planStep, fileData.dataset);
-                            break;
-                        }
-                        case 'download-file-locally': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in download-file-locally step`);
-                            const { producer } = planStep;
-                            const readRes = yield ProducerEngine_1.default.readFile(producer, { readmode: 'download' });
-                            this._storeIntermidiate(planStep, readRes.dataset);
-                            break;
-                        }
                         case 'load-dataset': {
                             (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
                             const { producer } = planStep;
@@ -171,14 +149,14 @@ class ExecutionEnvironment {
                         }
                         default: throw new Error(`Invalid execution plan step type "${planStep.type}"`);
                     }
-                    Logger_1.default.log(`Completed step: ${planStep.type}`);
+                    Logger_1.default.log(`\tCompleted step: ${planStep.type}`);
                 }
             }
             catch (error) {
                 const ds = (_a = this._resultingDataset) !== null && _a !== void 0 ? _a : this._getIntermidiate(currentStep);
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getSize()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
-                Logger_1.default.log(`Error at step ${currentStep.type}:\n\t${error}`);
+                Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
                 throw error;
             }
             Logger_1.default.log(`Completed execution of consumer:\n\tSize: ${result._stats.size}\n\tCycles: ${result._stats.cycles}\n\tTime: ${result._stats.elapsedMS}\n\tOperations: ${Logger_1.default.formatList(result._stats.operations)}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.48-nasco.3",
+    "version": "0.0.50-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,6 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
+        "dev": "clear && npx tsx scripts/dev.ts",
         "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",