@forzalabs/remora 0.0.46-nasco.3 → 0.0.47-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.46-nasco',
+    cliVersion: '0.0.47-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -177,6 +177,10 @@
                     "hidden": {
                         "type": "boolean",
                         "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                    },
+                    "fixed": {
+                        "type": "boolean",
+                        "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                     }
                 },
                 "required": [
@@ -436,6 +440,10 @@
                 "hidden": {
                     "type": "boolean",
                     "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                },
+                "fixed": {
+                    "type": "boolean",
+                    "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                 }
             },
             "required": [

package/drivers/LocalDriver.js

@@ -63,6 +63,7 @@ const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); //
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -131,6 +132,7 @@ class LocalSourceDriver {
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
                     yield copyLocally(allFileKeys[i], i > 0); // Append mode for subsequent files

package/engines/consumer/ConsumerManager.js

@@ -138,7 +138,7 @@ class ConsumerManagerClass {
             if (!column) {
                 // If the consumer doesn't find the field in the producer but has a default value AND set_default onError
                 // then instead of failing, create a placeholder column for the producer
-                if (field.onError === 'set_default' && Algo_1.default.hasVal(field.default)) {
+                if (field.fixed === true && Algo_1.default.hasVal(field.default)) {
                     column = {
                         aliasInProducer: field.key,
                         nameInProducer: (_a = field.alias) !== null && _a !== void 0 ? _a : field.key,

package/engines/consumer/PostProcessor.js

@@ -54,16 +54,8 @@ class PostProcessorClass {
             return newDataset;
         });
         this.updateDimensions = (dataset, consumer) => {
-            const dimensions = dataset.getDimensions();
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            for (const dim of dimensions) {
-                // This dimension is wanted by the consumer, check if it needs renaming
-                const consumerField = fields.find(x => x.cField.key === dim.name);
-                if (consumerField) {
-                    const { cField: { key, alias, hidden } } = consumerField;
-                    dataset.updateDimension(key, { name: alias, hidden: hidden });
-                }
-            }
+            dataset.updateDimensions(fields);
             return dataset;
         };
         this.dropDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
@@ -237,7 +229,9 @@ class PostProcessorClass {
         });
         this._getFieldValue = (record, field) => {
             var _a, _b, _c;
-            const { key, alias } = field.cField;
+            const { key, alias, fixed, default: defaultValue } = field.cField;
+            if (fixed && Algo_1.default.hasVal(defaultValue))
+                return defaultValue;
             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
             const fieldValue = record.getValue(fieldKey);
             if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {

package/engines/dataset/Dataset.js

@@ -31,6 +31,7 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const XMLParser_1 = __importDefault(require("../parsing/XMLParser"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
 class Dataset {
     constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
         this.getPath = () => this._path;
@@ -678,16 +679,28 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.updateDimension = (dimensionName, newValues) => {
-            const dimension = this._dimensions.find(x => x.name === dimensionName);
-            (0, Affirm_1.default)(dimension, `Trying to update the dataset dimension "${dimensionName}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-            this._startOperation('update-dimension');
-            const { hidden, name } = newValues;
-            if (name && name.length > 0)
-                dimension.name = name;
-            if (hidden)
-                dimension.hidden = hidden;
-            this._finishOperation('update-dimension');
+        this.updateDimensions = (fields) => {
+            this._startOperation('update-dimensions');
+            for (const field of fields) {
+                const { cField: { key, alias, hidden, fixed, default: defaultValue } } = field;
+                const currentDim = this._dimensions.find(x => x.name === key);
+                if (currentDim) {
+                    currentDim.name = alias !== null && alias !== void 0 ? alias : key;
+                    currentDim.hidden = hidden;
+                }
+                else if (fixed && Algo_1.default.hasVal(defaultValue)) {
+                    this._dimensions.push({
+                        hidden: hidden,
+                        index: this._dimensions.length,
+                        key: key,
+                        name: alias !== null && alias !== void 0 ? alias : key
+                    });
+                }
+                else {
+                    throw new Error(`Trying to update the dataset dimension "${(alias !== null && alias !== void 0 ? alias : key)}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
+                }
+            }
+            this._finishOperation('update-dimensions');
             return this;
         };
         this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.46-nasco.3",
+    "version": "0.0.47-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,