@forzalabs/remora 0.0.44-nasco.3 → 0.0.46-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.40-nasco',
+    cliVersion: '0.0.46-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -173,6 +173,10 @@
                     },
                     "default": {
                         "description": "Default value of the field if it is missing (or on error if specified)"
+                    },
+                    "hidden": {
+                        "type": "boolean",
+                        "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
                     }
                 },
                 "required": [
@@ -428,6 +432,10 @@
                 },
                 "default": {
                     "description": "Default value of the field if it is missing (or on error if specified)"
+                },
+                "hidden": {
+                    "type": "boolean",
+                    "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
                 }
             },
             "required": [
@@ -732,6 +740,36 @@
                     },
                     "required": ["append"],
                     "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "combine_fields": {
+                            "type": "object",
+                            "properties": {
+                                "fields": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "description": "Array of field names to combine",
+                                    "minItems": 1
+                                },
+                                "separator": {
+                                    "type": "string",
+                                    "description": "Optional separator between fields (default: empty string)"
+                                },
+                                "template": {
+                                    "type": "string",
+                                    "description": "Optional template string with placeholders like '{field1} - {field2}'"
+                                }
+                            },
+                            "required": ["fields"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["combine_fields"],
+                    "additionalProperties": false
                 }
             ]
         }

package/definitions/json_schemas/producer-schema.json

@@ -70,7 +70,8 @@
                             "mask",
                             "crypt",
                             "random",
-                            "seeded-random"
+                            "seeded-random",
+                            "none"
                         ],
                         "description": "Masking type to apply to this dimension. 'hash' replaces with a hashed value. 'mask' replaces characters with a mask character. 'crypt' encrypts the value. 'random' replaces with a random value. 'seeded-random' replaces with a random value generated from a seed. You can use environment variables by using the {your-env-var} notation",
                         "examples": [

package/engines/CryptoEngine.js

@@ -18,6 +18,8 @@ class CryptoEngineClass {
                     throw new Error('Not implemented yet');
                 case 'mask':
                     throw new Error('Not implemented yet');
+                case 'none':
+                    return `${fieldReference} AS "${fieldName}"`;
                 default:
                     throw new Error('This type is not ');
             }
@@ -55,6 +57,8 @@ class CryptoEngineClass {
                     throw new Error('Not implemented yet');
                 case 'mask':
                     throw new Error('Not implemented yet');
+                case 'none':
+                    return value;
                 default:
                     throw new Error(`This type doesn't exist`);
             }

package/engines/consumer/PostProcessor.js

@@ -34,7 +34,7 @@ class PostProcessorClass {
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
             let newDataset = yield this.dropDimensions(dataset, consumer);
-            newDataset = this.renameDimensions(newDataset, consumer);
+            newDataset = this.updateDimensions(newDataset, consumer);
             newDataset = yield this.reorderDimensions(newDataset, consumer);
             newDataset = yield newDataset.map(record => {
                 var _a, _b;
@@ -53,16 +53,15 @@ class PostProcessorClass {
             });
             return newDataset;
         });
-        this.renameDimensions = (dataset, consumer) => {
+        this.updateDimensions = (dataset, consumer) => {
             const dimensions = dataset.getDimensions();
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
             for (const dim of dimensions) {
                 // This dimension is wanted by the consumer, check if it needs renaming
                 const consumerField = fields.find(x => x.cField.key === dim.name);
                 if (consumerField) {
-                    const { cField: { key, alias } } = consumerField;
-                    if (key !== alias && Algo_1.default.hasVal(alias) && alias.length > 0)
-                        dataset.renameDimension(key, alias);
+                    const { cField: { key, alias, hidden } } = consumerField;
+                    dataset.updateDimension(key, { name: alias, hidden: hidden });
                 }
             }
             return dataset;
@@ -206,7 +205,8 @@ class PostProcessorClass {
                     const newDimensions = expectedFieldNames.map((key, index) => ({
                         name: key,
                         key: key,
-                        index: index
+                        index: index,
+                        hidden: null
                     }));
                     // Create the row string
                     const values = newDimensions.map(dim => {
@@ -227,7 +227,8 @@ class PostProcessorClass {
                 const newDimensions = columns.map((col, index) => ({
                     name: col.nameInProducer,
                     key: col.nameInProducer,
-                    index: index
+                    index: index,
+                    hidden: null
                 }));
                 // Update the dataset dimensions
                 resDataset['_dimensions'] = newDimensions;

package/engines/dataset/Dataset.js

@@ -678,12 +678,17 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.renameDimension = (currentName, newName) => {
-            const dimension = this._dimensions.find(x => x.name === currentName);
-            (0, Affirm_1.default)(dimension, `Trying to rename the dataset dimension "${currentName}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-            this._startOperation('rename-dimension');
-            dimension.name = newName;
-            this._finishOperation('rename-dimension');
+        this.updateDimension = (dimensionName, newValues) => {
+            const dimension = this._dimensions.find(x => x.name === dimensionName);
+            (0, Affirm_1.default)(dimension, `Trying to update the dataset dimension "${dimensionName}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
+            this._startOperation('update-dimension');
+            const { hidden, name } = newValues;
+            if (name && name.length > 0)
+                dimension.name = name;
+            if (hidden)
+                dimension.hidden = hidden;
+            this._finishOperation('update-dimension');
+            return this;
         };
         this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
             if (dimensionNames.length === 0)

package/engines/dataset/DatasetManager.js

@@ -49,7 +49,7 @@ class DatasetManagerClass {
                     const headerLine = firstLine;
                     const rawDimensions = ParseManager_1.default._extractHeader(headerLine, delimiterChar, producer, discover);
                     return {
-                        dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index })),
+                        dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
                         delimiter: delimiterChar
                     };
                 }
@@ -64,7 +64,7 @@ class DatasetManagerClass {
                         const columnKey = (_b = pColumn.aliasInProducer) !== null && _b !== void 0 ? _b : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                        dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer });
+                        dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer, hidden: null });
                     }
                     const delimiterChar = (_c = file.delimiter) !== null && _c !== void 0 ? _c : ',';
                     return { dimensions, delimiter: delimiterChar };
@@ -76,7 +76,7 @@ class DatasetManagerClass {
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
                         return {
-                            dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i }); }),
+                            dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i, hidden: null }); }),
                             delimiter: delimiterChar
                         };
                     }
@@ -84,7 +84,7 @@ class DatasetManagerClass {
                         const delimiterChar = (_e = producer.settings.delimiter) !== null && _e !== void 0 ? _e : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
-                            dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index })),
+                            dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
                             delimiter: delimiterChar
                         };
                     }

package/engines/dataset/DatasetRecord.js

@@ -46,10 +46,25 @@ class DatasetRecord {
             this._value = newValue;
             return this;
         };
-        this.toJSON = () => JSON.stringify(this._value);
+        this.toJSON = () => {
+            if (this._dimensions.some(x => x.hidden)) {
+                // remove the not wanted dimension
+                const clonedValue = structuredClone(this._value);
+                for (const dim of this._dimensions) {
+                    if (dim.hidden)
+                        delete clonedValue[dim.name];
+                }
+                return JSON.stringify(clonedValue);
+            }
+            else {
+                return JSON.stringify(this._value);
+            }
+        };
         this.toCSV = (delimiter) => {
             const myDelimtier = delimiter !== null && delimiter !== void 0 ? delimiter : this._delimiter;
+            // remove the not wanted dimension
             const line = this._dimensions
+                .filter(x => !x.hidden)
                 .map(x => { var _a, _b; return `"${Algo_1.default.replaceAll((_b = (_a = this._value[x.name]) === null || _a === void 0 ? void 0 : _a.toString()) !== null && _b !== void 0 ? _b : '', '"', '""')}"`; })
                 .join(myDelimtier);
             return line;

package/engines/execution/ExecutionEnvironment.js

@@ -123,6 +123,7 @@ class ExecutionEnvironment {
                         }
                         case 'export-file': {
                             (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
+                            (0, Affirm_1.default)(this._resultingDataset, 'Invalid resulting dataset in export-file step');
                             const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingDataset);
                             result.fileUri = res;
                             break;

package/engines/execution/RequestExecutor.js

@@ -21,7 +21,6 @@ class RequestExecutorClass {
         this.execute = (dataset, request) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(dataset, 'Invalid data');
             (0, Affirm_1.default)(request, 'Invalid request');
-            (0, Affirm_1.default)(Array.isArray(dataset), `Invalid data type: should be an array`);
             if (request.filters)
                 dataset = yield this.applyFilters(dataset, request.filters);
             if (request.order)

package/engines/producer/ProducerManager.js

@@ -16,7 +16,8 @@ class ProducerManagerClass {
                 'mask',
                 'crypt',
                 'random',
-                'seeded-random'
+                'seeded-random',
+                'none'
             ];
             (0, Affirm_1.default)(allowedMaskValues.includes(mask), `Mask value "${mask}" in dimension "${dimension.name}" is not a valid value.`);
             return mask;

package/engines/transform/JoinEngine.js

@@ -103,7 +103,8 @@ class JoinEngineClass {
             const resultDimensions = consumerColumns.map((col, index) => ({
                 name: col.consumerAlias || col.consumerKey,
                 key: col.consumerAlias || col.consumerKey,
-                index
+                index,
+                hidden: null
             }));
             // Initialize the result dataset with proper dimensions
             resultDataset.getDimensions().length = 0;

package/engines/transform/TransformationEngine.js

@@ -31,10 +31,13 @@ class TransformationEngineClass {
                     const value = record.getValue(fieldKey);
                     if (!Algo_1.default.hasVal(value) && Algo_1.default.hasVal(field.default))
                         record.setValue(fieldKey, field.default);
+                    else if (!Algo_1.default.hasVal(value) && this.isFieldCombinationTransformation(field.transform))
+                        // For field combination transformations, we don't skip null values as they might combine with other fields
+                        continue;
                     else if (!Algo_1.default.hasVal(value))
                         continue;
                     try {
-                        record.setValue(fieldKey, this.applyTransformations(value, field.transform, field));
+                        record.setValue(fieldKey, this.applyTransformations(value, field.transform, field, record));
                     }
                     catch (error) {
                         switch (field.onError) {
@@ -52,13 +55,19 @@ class TransformationEngineClass {
                 return record;
             });
         });
-        this.applyTransformations = (value, transformations, field) => {
+        this.isFieldCombinationTransformation = (transformation) => {
+            if (Array.isArray(transformation)) {
+                return transformation.some(t => this.isFieldCombinationTransformation(t));
+            }
+            return 'combine_fields' in transformation;
+        };
+        this.applyTransformations = (value, transformations, field, record) => {
             var _a;
             if (Array.isArray(transformations)) {
                 // Process array transformations without creating intermediate arrays
                 let result = value;
                 for (const transform of transformations) {
-                    result = this.applyTransformations(result, transform, field);
+                    result = this.applyTransformations(result, transform, field, record);
                 }
                 return result;
             }
@@ -190,6 +199,30 @@ class TransformationEngineClass {
                 return transformations.prepend + TypeCaster_1.default.cast(value, 'string');
             if ('append' in transformations)
                 return TypeCaster_1.default.cast(value, 'string') + transformations.append;
+            if ('combine_fields' in transformations) {
+                if (!record) {
+                    throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
+                }
+                const { fields, separator = '', template } = transformations.combine_fields;
+                // Get values from the specified fields
+                const fieldValues = fields.map(fieldName => {
+                    const fieldValue = record.getValue(fieldName);
+                    return fieldValue !== null && fieldValue !== undefined ? String(fieldValue) : '';
+                });
+                // If template is provided, use it for formatting
+                if (template) {
+                    let result = template;
+                    for (let i = 0; i < fields.length; i++) {
+                        const placeholder = `{${fields[i]}}`;
+                        result = result.replace(new RegExp(placeholder, 'g'), fieldValues[i]);
+                    }
+                    return result;
+                }
+                else {
+                    // Otherwise, join with separator
+                    return fieldValues.join(separator);
+                }
+            }
             if ('conditional' in transformations) {
                 for (const clause of transformations.conditional.clauses) {
                     if (this.evaluateCondition(value, clause.if)) {

package/engines/validation/Validator.js

@@ -138,6 +138,7 @@ class ValidatorClass {
                     return errors;
                 };
                 const validateTransformations = (fields) => {
+                    var _a;
                     const errors = [];
                     const trxsFields = fields.filter(x => x.transform);
                     for (const field of trxsFields) {
@@ -149,7 +150,17 @@ class ValidatorClass {
                         for (const trans of trxToValidate) {
                             const trxKeys = Object.keys(trans);
                             if (trxKeys.length !== 1)
-                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}`);
+                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}.`);
+                            if ('combine_fields' in trans) {
+                                const { combine_fields } = trans;
+                                if (!combine_fields.fields || combine_fields.fields.length === 0)
+                                    errors.push(`The "combine_field" transformation is missing the "fields" property ("${field.key}").`);
+                                const missingFieldsInConsumer = combine_fields.fields
+                                    .map(x => ({ field: x, found: fields.find(k => { var _a; return ((_a = k.alias) !== null && _a !== void 0 ? _a : k.key) === x; }) }))
+                                    .filter(x => !x.found);
+                                if (missingFieldsInConsumer.length > 0)
+                                    errors.push(`The requested field(s) for a transformation is missing in the consumer -> missing field(s): "${missingFieldsInConsumer.map(x => x.field).join(', ')}"; field transformation: "${(_a = field.alias) !== null && _a !== void 0 ? _a : field.key}";`);
+                            }
                         }
                     }
                     return errors;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.44-nasco.3",
+    "version": "0.0.46-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,