@forzalabs/remora 0.0.45-nasco.3 → 0.0.47-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.45-nasco',
+    cliVersion: '0.0.47-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -173,6 +173,14 @@
                     },
                     "default": {
                         "description": "Default value of the field if it is missing (or on error if specified)"
+                    },
+                    "hidden": {
+                        "type": "boolean",
+                        "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                    },
+                    "fixed": {
+                        "type": "boolean",
+                        "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                     }
                 },
                 "required": [
@@ -428,6 +436,14 @@
                 },
                 "default": {
                     "description": "Default value of the field if it is missing (or on error if specified)"
+                },
+                "hidden": {
+                    "type": "boolean",
+                    "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                },
+                "fixed": {
+                    "type": "boolean",
+                    "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                 }
             },
             "required": [
@@ -732,6 +748,36 @@
                     },
                     "required": ["append"],
                     "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "combine_fields": {
+                            "type": "object",
+                            "properties": {
+                                "fields": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "description": "Array of field names to combine",
+                                    "minItems": 1
+                                },
+                                "separator": {
+                                    "type": "string",
+                                    "description": "Optional separator between fields (default: empty string)"
+                                },
+                                "template": {
+                                    "type": "string",
+                                    "description": "Optional template string with placeholders like '{field1} - {field2}'"
+                                }
+                            },
+                            "required": ["fields"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["combine_fields"],
+                    "additionalProperties": false
                 }
             ]
         }

package/drivers/LocalDriver.js

@@ -63,6 +63,7 @@ const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); //
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -131,6 +132,7 @@ class LocalSourceDriver {
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
                     yield copyLocally(allFileKeys[i], i > 0); // Append mode for subsequent files

package/engines/consumer/ConsumerManager.js

@@ -138,7 +138,7 @@ class ConsumerManagerClass {
             if (!column) {
                 // If the consumer doesn't find the field in the producer but has a default value AND set_default onError
                 // then instead of failing, create a placeholder column for the producer
-                if (field.onError === 'set_default' && Algo_1.default.hasVal(field.default)) {
+                if (field.fixed === true && Algo_1.default.hasVal(field.default)) {
                     column = {
                         aliasInProducer: field.key,
                         nameInProducer: (_a = field.alias) !== null && _a !== void 0 ? _a : field.key,

package/engines/consumer/PostProcessor.js

@@ -34,7 +34,7 @@ class PostProcessorClass {
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
             let newDataset = yield this.dropDimensions(dataset, consumer);
-            newDataset = this.renameDimensions(newDataset, consumer);
+            newDataset = this.updateDimensions(newDataset, consumer);
             newDataset = yield this.reorderDimensions(newDataset, consumer);
             newDataset = yield newDataset.map(record => {
                 var _a, _b;
@@ -53,18 +53,9 @@ class PostProcessorClass {
             });
             return newDataset;
         });
-        this.renameDimensions = (dataset, consumer) => {
-            const dimensions = dataset.getDimensions();
+        this.updateDimensions = (dataset, consumer) => {
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            for (const dim of dimensions) {
-                // This dimension is wanted by the consumer, check if it needs renaming
-                const consumerField = fields.find(x => x.cField.key === dim.name);
-                if (consumerField) {
-                    const { cField: { key, alias } } = consumerField;
-                    if (key !== alias && Algo_1.default.hasVal(alias) && alias.length > 0)
-                        dataset.renameDimension(key, alias);
-                }
-            }
+            dataset.updateDimensions(fields);
             return dataset;
         };
         this.dropDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
@@ -206,7 +197,8 @@ class PostProcessorClass {
                     const newDimensions = expectedFieldNames.map((key, index) => ({
                         name: key,
                         key: key,
-                        index: index
+                        index: index,
+                        hidden: null
                     }));
                     // Create the row string
                     const values = newDimensions.map(dim => {
@@ -227,7 +219,8 @@ class PostProcessorClass {
                 const newDimensions = columns.map((col, index) => ({
                     name: col.nameInProducer,
                     key: col.nameInProducer,
-                    index: index
+                    index: index,
+                    hidden: null
                 }));
                 // Update the dataset dimensions
                 resDataset['_dimensions'] = newDimensions;
@@ -236,7 +229,9 @@ class PostProcessorClass {
         });
         this._getFieldValue = (record, field) => {
             var _a, _b, _c;
-            const { key, alias } = field.cField;
+            const { key, alias, fixed, default: defaultValue } = field.cField;
+            if (fixed && Algo_1.default.hasVal(defaultValue))
+                return defaultValue;
             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
             const fieldValue = record.getValue(fieldKey);
             if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {

package/engines/dataset/Dataset.js

@@ -31,6 +31,7 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const XMLParser_1 = __importDefault(require("../parsing/XMLParser"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
 class Dataset {
     constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
         this.getPath = () => this._path;
@@ -678,12 +679,29 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.renameDimension = (currentName, newName) => {
-            const dimension = this._dimensions.find(x => x.name === currentName);
-            (0, Affirm_1.default)(dimension, `Trying to rename the dataset dimension "${currentName}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-            this._startOperation('rename-dimension');
-            dimension.name = newName;
-            this._finishOperation('rename-dimension');
+        this.updateDimensions = (fields) => {
+            this._startOperation('update-dimensions');
+            for (const field of fields) {
+                const { cField: { key, alias, hidden, fixed, default: defaultValue } } = field;
+                const currentDim = this._dimensions.find(x => x.name === key);
+                if (currentDim) {
+                    currentDim.name = alias !== null && alias !== void 0 ? alias : key;
+                    currentDim.hidden = hidden;
+                }
+                else if (fixed && Algo_1.default.hasVal(defaultValue)) {
+                    this._dimensions.push({
+                        hidden: hidden,
+                        index: this._dimensions.length,
+                        key: key,
+                        name: alias !== null && alias !== void 0 ? alias : key
+                    });
+                }
+                else {
+                    throw new Error(`Trying to update the dataset dimension "${(alias !== null && alias !== void 0 ? alias : key)}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
+                }
+            }
+            this._finishOperation('update-dimensions');
+            return this;
         };
         this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {
             if (dimensionNames.length === 0)

package/engines/dataset/DatasetManager.js

@@ -49,7 +49,7 @@ class DatasetManagerClass {
                     const headerLine = firstLine;
                     const rawDimensions = ParseManager_1.default._extractHeader(headerLine, delimiterChar, producer, discover);
                     return {
-                        dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index })),
+                        dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
                         delimiter: delimiterChar
                     };
                 }
@@ -64,7 +64,7 @@ class DatasetManagerClass {
                         const columnKey = (_b = pColumn.aliasInProducer) !== null && _b !== void 0 ? _b : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
-                        dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer });
+                        dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer, hidden: null });
                     }
                     const delimiterChar = (_c = file.delimiter) !== null && _c !== void 0 ? _c : ',';
                     return { dimensions, delimiter: delimiterChar };
@@ -76,7 +76,7 @@ class DatasetManagerClass {
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
                         return {
-                            dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i }); }),
+                            dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i, hidden: null }); }),
                             delimiter: delimiterChar
                         };
                     }
@@ -84,7 +84,7 @@ class DatasetManagerClass {
                         const delimiterChar = (_e = producer.settings.delimiter) !== null && _e !== void 0 ? _e : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
-                            dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index })),
+                            dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),
                             delimiter: delimiterChar
                         };
                     }

package/engines/dataset/DatasetRecord.js

@@ -46,10 +46,25 @@ class DatasetRecord {
             this._value = newValue;
             return this;
         };
-        this.toJSON = () => JSON.stringify(this._value);
+        this.toJSON = () => {
+            if (this._dimensions.some(x => x.hidden)) {
+                // remove the not wanted dimension
+                const clonedValue = structuredClone(this._value);
+                for (const dim of this._dimensions) {
+                    if (dim.hidden)
+                        delete clonedValue[dim.name];
+                }
+                return JSON.stringify(clonedValue);
+            }
+            else {
+                return JSON.stringify(this._value);
+            }
+        };
         this.toCSV = (delimiter) => {
             const myDelimtier = delimiter !== null && delimiter !== void 0 ? delimiter : this._delimiter;
+            // remove the not wanted dimension
             const line = this._dimensions
+                .filter(x => !x.hidden)
                 .map(x => { var _a, _b; return `"${Algo_1.default.replaceAll((_b = (_a = this._value[x.name]) === null || _a === void 0 ? void 0 : _a.toString()) !== null && _b !== void 0 ? _b : '', '"', '""')}"`; })
                 .join(myDelimtier);
             return line;

package/engines/execution/ExecutionEnvironment.js

@@ -123,6 +123,7 @@ class ExecutionEnvironment {
                         }
                         case 'export-file': {
                             (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
+                            (0, Affirm_1.default)(this._resultingDataset, 'Invalid resulting dataset in export-file step');
                             const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingDataset);
                             result.fileUri = res;
                             break;

package/engines/transform/JoinEngine.js

@@ -103,7 +103,8 @@ class JoinEngineClass {
             const resultDimensions = consumerColumns.map((col, index) => ({
                 name: col.consumerAlias || col.consumerKey,
                 key: col.consumerAlias || col.consumerKey,
-                index
+                index,
+                hidden: null
             }));
             // Initialize the result dataset with proper dimensions
             resultDataset.getDimensions().length = 0;

package/engines/transform/TransformationEngine.js

@@ -31,10 +31,13 @@ class TransformationEngineClass {
                     const value = record.getValue(fieldKey);
                     if (!Algo_1.default.hasVal(value) && Algo_1.default.hasVal(field.default))
                         record.setValue(fieldKey, field.default);
+                    else if (!Algo_1.default.hasVal(value) && this.isFieldCombinationTransformation(field.transform))
+                        // For field combination transformations, we don't skip null values as they might combine with other fields
+                        continue;
                     else if (!Algo_1.default.hasVal(value))
                         continue;
                     try {
-                        record.setValue(fieldKey, this.applyTransformations(value, field.transform, field));
+                        record.setValue(fieldKey, this.applyTransformations(value, field.transform, field, record));
                     }
                     catch (error) {
                         switch (field.onError) {
@@ -52,13 +55,19 @@ class TransformationEngineClass {
                 return record;
             });
         });
-        this.applyTransformations = (value, transformations, field) => {
+        this.isFieldCombinationTransformation = (transformation) => {
+            if (Array.isArray(transformation)) {
+                return transformation.some(t => this.isFieldCombinationTransformation(t));
+            }
+            return 'combine_fields' in transformation;
+        };
+        this.applyTransformations = (value, transformations, field, record) => {
             var _a;
             if (Array.isArray(transformations)) {
                 // Process array transformations without creating intermediate arrays
                 let result = value;
                 for (const transform of transformations) {
-                    result = this.applyTransformations(result, transform, field);
+                    result = this.applyTransformations(result, transform, field, record);
                 }
                 return result;
             }
@@ -190,6 +199,30 @@ class TransformationEngineClass {
                 return transformations.prepend + TypeCaster_1.default.cast(value, 'string');
             if ('append' in transformations)
                 return TypeCaster_1.default.cast(value, 'string') + transformations.append;
+            if ('combine_fields' in transformations) {
+                if (!record) {
+                    throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
+                }
+                const { fields, separator = '', template } = transformations.combine_fields;
+                // Get values from the specified fields
+                const fieldValues = fields.map(fieldName => {
+                    const fieldValue = record.getValue(fieldName);
+                    return fieldValue !== null && fieldValue !== undefined ? String(fieldValue) : '';
+                });
+                // If template is provided, use it for formatting
+                if (template) {
+                    let result = template;
+                    for (let i = 0; i < fields.length; i++) {
+                        const placeholder = `{${fields[i]}}`;
+                        result = result.replace(new RegExp(placeholder, 'g'), fieldValues[i]);
+                    }
+                    return result;
+                }
+                else {
+                    // Otherwise, join with separator
+                    return fieldValues.join(separator);
+                }
+            }
             if ('conditional' in transformations) {
                 for (const clause of transformations.conditional.clauses) {
                     if (this.evaluateCondition(value, clause.if)) {

package/engines/validation/Validator.js

@@ -138,6 +138,7 @@ class ValidatorClass {
                     return errors;
                 };
                 const validateTransformations = (fields) => {
+                    var _a;
                     const errors = [];
                     const trxsFields = fields.filter(x => x.transform);
                     for (const field of trxsFields) {
@@ -149,7 +150,17 @@ class ValidatorClass {
                         for (const trans of trxToValidate) {
                             const trxKeys = Object.keys(trans);
                             if (trxKeys.length !== 1)
-                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}`);
+                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}.`);
+                            if ('combine_fields' in trans) {
+                                const { combine_fields } = trans;
+                                if (!combine_fields.fields || combine_fields.fields.length === 0)
+                                    errors.push(`The "combine_field" transformation is missing the "fields" property ("${field.key}").`);
+                                const missingFieldsInConsumer = combine_fields.fields
+                                    .map(x => ({ field: x, found: fields.find(k => { var _a; return ((_a = k.alias) !== null && _a !== void 0 ? _a : k.key) === x; }) }))
+                                    .filter(x => !x.found);
+                                if (missingFieldsInConsumer.length > 0)
+                                    errors.push(`The requested field(s) for a transformation is missing in the consumer -> missing field(s): "${missingFieldsInConsumer.map(x => x.field).join(', ')}"; field transformation: "${(_a = field.alias) !== null && _a !== void 0 ? _a : field.key}";`);
+                            }
                         }
                     }
                     return errors;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.45-nasco.3",
+    "version": "0.0.47-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,