@forzalabs/remora 0.0.22 → 0.0.24

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.22',
+    cliVersion: '0.0.24',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/actions/run.js

@@ -26,9 +26,16 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
         console.log();
         const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
         const user = UserManager_1.default.getUser();
-        const consumersToExecute = (consumerName && consumerName.length > 0)
-            ? [Environment_1.default.getConsumer(consumerName)]
-            : Environment_1.default._env.consumers;
+        const consumersToExecute = [];
+        if (consumerName && consumerName.length > 0) {
+            const cons = Environment_1.default.getConsumer(consumerName);
+            if (!cons)
+                throw new Error(`Consumer with name "${consumerName}" was not found.`);
+            consumersToExecute.push(cons);
+        }
+        else {
+            consumersToExecute.push(...Environment_1.default._env.consumers);
+        }
         const results = [];
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];

package/engines/ParseManager.js

@@ -11,16 +11,15 @@ class ParseManagerClass {
         this.csvToJson = (csv, producer) => {
             (0, Affirm_1.default)(csv, 'Invalid csv content');
             Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
-            const fileRows = csv.split('\n');
-            return this.csvLinesToJson(fileRows, producer);
+            return this.csvLinesToJson(csv, producer);
         };
-        this.csvLinesToJson = (lines, producer) => {
+        this.csvLinesToJson = (lines, producer, discover) => {
             var _a;
             (0, Affirm_1.default)(lines, 'Invalid csv lines');
             Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
             const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
             const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
-            const headerColumns = this._extractHeader(lines[0], delimiterChar, producer);
+            const headerColumns = this._extractHeader(lines[0], delimiterChar, producer, discover);
             const result = [];
             for (const row of rows) {
                 const rowObject = {};
@@ -32,7 +31,7 @@ class ParseManagerClass {
             }
             return result;
         };
-        this._extractHeader = (headerLine, delimiter, producer) => {
+        this._extractHeader = (headerLine, delimiter, producer, discover) => {
             var _a;
             (0, Affirm_1.default)(headerLine, 'Invalid CSV header line');
             (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
@@ -40,6 +39,9 @@ class ParseManagerClass {
             const source = Environment_1.default.getSource(producer.source);
             const columns = FileCompiler_1.default.compileProducer(producer, source);
             const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
+            if (discover)
+                columns.push(...headerColumns.map(x => ({ nameInProducer: x })));
             const csvColumns = [];
             for (const pColumn of columns) {
                 const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;

package/engines/ProducerEngine.js

@@ -111,7 +111,7 @@ class ProducerEngineClass {
                     throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
             }
         });
-        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10) {
+        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10, discover = false) {
             var _a, _b, _c;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
@@ -132,7 +132,7 @@ class ProducerEngineClass {
                 case 'aws-s3': {
                     const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
                     if (((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'CSV') {
-                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer);
+                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer, discover);
                     }
                     else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
                         // With JSON or JSONL the readFile function already parses the strings

package/engines/ai/DeveloperEngine.js

@@ -21,7 +21,7 @@ class DeveloperEngineClass {
         this.discover = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
-            const sampleData = yield ProducerEngine_1.default.readSampleData(producer);
+            const sampleData = yield ProducerEngine_1.default.readSampleData(producer, 10, true);
             (0, Affirm_1.default)(sampleData, 'Discover process failed: no result found');
             const typeDefinitions = this.extractFieldTypes(sampleData);
             const mappedProducer = {

package/engines/consumer/PostProcessor.js

@@ -145,10 +145,16 @@ class PostProcessorClass {
             return unpackedData;
         };
         this._getFieldValue = (record, field) => {
+            var _a, _b, _c;
             const fieldValue = record[field.cField.key];
-            if (Algo_1.default.hasVal(fieldValue))
-                return fieldValue;
-            else if (!Algo_1.default.hasVal(fieldValue) && Algo_1.default.hasVal(field.cField.default))
+            if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {
+                const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
+                if (fieldType === 'number' && typeof fieldValue === 'string' && fieldValue.length === 0)
+                    return (_c = field.cField.default) !== null && _c !== void 0 ? _c : fieldValue;
+                else
+                    return fieldValue;
+            }
+            else if ((!Algo_1.default.hasVal(fieldValue) || isNaN(fieldValue)) && Algo_1.default.hasVal(field.cField.default))
                 return field.cField.default;
             else
                 return fieldValue;

package/engines/execution/ExecutionEnvironment.js

@@ -83,7 +83,7 @@ class ExecutionEnvironment {
                         (0, Affirm_1.default)(this._fetchedData, 'Invalid data');
                         (0, Affirm_1.default)(Array.isArray(this._fetchedData), 'Invalid data type, must be an array');
                         (0, Affirm_1.default)(planStep.producer, `Invalid producer in csv-to-json step`);
-                        const csv = this._fetchedData[0];
+                        const csv = this._fetchedData;
                         this._fetchedData = ParseManager_1.default.csvToJson(csv, planStep.producer);
                         break;
                     }

package/engines/validation/Validator.js

@@ -130,9 +130,16 @@ class ValidatorClass {
                     const errors = [];
                     const trxsFields = fields.filter(x => x.transform);
                     for (const field of trxsFields) {
-                        const transformations = Object.keys(field.transform);
-                        if (transformations.length !== 1)
-                            errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${transformations.length}`);
+                        const trxToValidate = [];
+                        if (Array.isArray(field.transform))
+                            trxToValidate.push(...field.transform);
+                        else
+                            trxToValidate.push(field.transform);
+                        for (const trans of trxToValidate) {
+                            const trxKeys = Object.keys(trans);
+                            if (trxKeys.length !== 1)
+                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}`);
+                        }
                     }
                     return errors;
                 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.22",
+    "version": "0.0.24",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,