@forzalabs/remora 0.0.46-nasco.3 → 0.0.48-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.46-nasco',
+    cliVersion: '0.0.48-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/consumer-schema.json

@@ -177,6 +177,10 @@
                     "hidden": {
                         "type": "boolean",
                         "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                    },
+                    "fixed": {
+                        "type": "boolean",
+                        "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                     }
                 },
                 "required": [
@@ -436,6 +440,10 @@
                 "hidden": {
                     "type": "boolean",
                     "description": "If set, the field is kept and used during processing, but omitted when exporting the data"
+                },
+                "fixed": {
+                    "type": "boolean",
+                    "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                 }
             },
             "required": [

package/drivers/DriverHelper.js

@@ -0,0 +1,86 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const promises_1 = require("stream/promises");
+const fs_1 = require("fs");
+const Logger_1 = __importDefault(require("../helper/Logger"));
+const DriverHelper = {
+    appendToUnifiedFile: (stream, fileKey, destinationPath, append, headerLine, fileType) => __awaiter(void 0, void 0, void 0, function* () {
+        const shouldValidateHeader = fileType === 'CSV' || fileType === 'TXT';
+        let isFirstLine = true;
+        let hasValidatedHeader = shouldValidateHeader ? false : true;
+        const headerValidationTransform = new stream_1.Transform({
+            transform(chunk, encoding, callback) {
+                if (!hasValidatedHeader) {
+                    const chunkStr = chunk.toString();
+                    const lines = chunkStr.split('\n');
+                    if (isFirstLine && lines.length > 0) {
+                        const firstLine = lines[0];
+                        // Validate header only for CSV and TXT files
+                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                            Logger_1.default.log(msg);
+                            return callback(new Error(msg));
+                        }
+                        hasValidatedHeader = true;
+                        isFirstLine = false;
+                    }
+                }
+                callback(null, chunk);
+            }
+        });
+        const writeOptions = append ? { flags: 'a' } : {};
+        const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);
+        yield (0, promises_1.pipeline)(stream, headerValidationTransform, writeStream);
+    }),
+    quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const fileStream = (0, fs_1.createReadStream)(filePath);
+        const rl = (0, readline_1.createInterface)({ input: fileStream, crlfDelay: Infinity });
+        const lines = [];
+        let counter = 0;
+        try {
+            for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                _c = rl_1_1.value;
+                _d = false;
+                const line = _c;
+                lines.push(line);
+                counter++;
+                if (counter >= lineCount) {
+                    break;
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        rl.close();
+        fileStream.close();
+        return lines;
+    })
+};
+exports.default = DriverHelper;

package/drivers/LocalDriver.js

@@ -63,6 +63,8 @@ const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); //
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -115,30 +117,28 @@ class LocalSourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const copyLocally = (fileKey_1, ...args_1) => __awaiter(this, [fileKey_1, ...args_1], void 0, function* (fileKey, appendMode = false) {
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false) {
                 const sourceFilePath = path_1.default.join(this._path, fileKey);
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-                const writeOptions = appendMode ? { flags: 'a' } : {};
+                // Copy and validate header in a single stream pass
                 const readStream = fs.createReadStream(sourceFilePath);
-                const writeStream = fs.createWriteStream(dataset.getPath(), writeOptions);
-                return new Promise((resolve, reject) => {
-                    readStream.pipe(writeStream);
-                    writeStream.on('finish', resolve);
-                    writeStream.on('error', reject);
-                    readStream.on('error', reject);
-                });
+                return DriverHelper_1.default.appendToUnifiedFile(readStream, fileKey, dataset.getPath(), appendMode, headerLine, file.fileType);
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                // Get header line from the first file
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield copyLocally(allFileKeys[i], i > 0); // Append mode for subsequent files
+                    yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
                 return dataset;
             }
             else {
-                yield copyLocally(fileKey);
+                // For single file, no header validation needed
+                yield copyLocally(fileKey, '', false);
                 return dataset;
             }
         });

package/drivers/S3Driver.js

@@ -29,9 +29,8 @@ const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
-const promises_1 = require("stream/promises");
-const fs_1 = require("fs");
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -227,7 +226,8 @@ class S3SourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const downloadLocally = (fileUrl_1, ...args_1) => __awaiter(this, [fileUrl_1, ...args_1], void 0, function* (fileUrl, appendMode = false) {
+            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false) {
+                // Download and validate header in a single stream pass
                 const command = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
                     Key: fileUrl
@@ -235,23 +235,55 @@ class S3SourceDriver {
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
                 const stream = response.Body;
-                const writeOptions = appendMode ? { flags: 'a' } : {};
-                yield (0, promises_1.pipeline)(stream, (0, fs_1.createWriteStream)(dataset.getPath(), writeOptions));
+                return DriverHelper_1.default.appendToUnifiedFile(stream, fileUrl, dataset.getPath(), appendMode, headerLine, file.fileType);
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
+                // Get header line from the first file
+                const firstFileCommand = new client_s3_1.GetObjectCommand({
+                    Bucket: this._bucketName,
+                    Key: allFileKeys[0]
+                });
+                const firstFileResponse = yield this._client.send(firstFileCommand);
+                (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
+                const firstFileStream = firstFileResponse.Body;
+                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield downloadLocally(allFileKeys[i], i > 0); // Append mode for subsequent files
+                    yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
                 return dataset;
             }
             else {
-                yield downloadLocally(fileKey);
+                // For single file, no header validation needed
+                yield downloadLocally(fileKey, '');
                 return dataset;
             }
         });
+        this.getFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            let firstLine = '';
+            try {
+                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                    _c = rl_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    firstLine = line;
+                    break;
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            rl.close();
+            return firstLine;
+        });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
@@ -276,7 +308,7 @@ class S3SourceDriver {
             }
         });
         this._readLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
+            var _a, e_2, _b, _c;
             const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
             const lines = [];
             let lineCounter = 0;
@@ -300,19 +332,19 @@ class S3SourceDriver {
                     }
                 }
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
             finally {
                 try {
                     if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
                 }
-                finally { if (e_1) throw e_1.error; }
+                finally { if (e_2) throw e_2.error; }
             }
             reader.close();
             return lines;
         });
         this._readExcelLines = (stream, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
             var _a, stream_1, stream_1_1;
-            var _b, e_2, _c, _d;
+            var _b, e_3, _c, _d;
             (0, Affirm_1.default)(sheetName, `Invalid sheetname`);
             const chunks = [];
             try {
@@ -323,12 +355,12 @@ class S3SourceDriver {
                     chunks.push(chunk);
                 }
             }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
             finally {
                 try {
                     if (!_a && !_b && (_c = stream_1.return)) yield _c.call(stream_1);
                 }
-                finally { if (e_2) throw e_2.error; }
+                finally { if (e_3) throw e_3.error; }
             }
             const buffer = Buffer.concat(chunks);
             const excel = xlsx_1.default.read(buffer, { type: 'buffer' });
@@ -343,7 +375,7 @@ class S3SourceDriver {
         });
         this._readXmlLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
             var _a, stream_2, stream_2_1;
-            var _b, e_3, _c, _d;
+            var _b, e_4, _c, _d;
             const chunks = [];
             try {
                 for (_a = true, stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), _b = stream_2_1.done, !_b; _a = true) {
@@ -353,12 +385,12 @@ class S3SourceDriver {
                     chunks.push(chunk);
                 }
             }
-            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            catch (e_4_1) { e_4 = { error: e_4_1 }; }
             finally {
                 try {
                     if (!_a && !_b && (_c = stream_2.return)) yield _c.call(stream_2);
                 }
-                finally { if (e_3) throw e_3.error; }
+                finally { if (e_4) throw e_4.error; }
             }
             const buffer = Buffer.concat(chunks);
             const jsonData = XMLParser_1.default.xmlToJson(buffer);

package/engines/consumer/ConsumerManager.js

@@ -138,7 +138,7 @@ class ConsumerManagerClass {
             if (!column) {
                 // If the consumer doesn't find the field in the producer but has a default value AND set_default onError
                 // then instead of failing, create a placeholder column for the producer
-                if (field.onError === 'set_default' && Algo_1.default.hasVal(field.default)) {
+                if (field.fixed === true && Algo_1.default.hasVal(field.default)) {
                     column = {
                         aliasInProducer: field.key,
                         nameInProducer: (_a = field.alias) !== null && _a !== void 0 ? _a : field.key,

package/engines/consumer/PostProcessor.js

@@ -54,16 +54,8 @@ class PostProcessorClass {
             return newDataset;
         });
         this.updateDimensions = (dataset, consumer) => {
-            const dimensions = dataset.getDimensions();
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            for (const dim of dimensions) {
-                // This dimension is wanted by the consumer, check if it needs renaming
-                const consumerField = fields.find(x => x.cField.key === dim.name);
-                if (consumerField) {
-                    const { cField: { key, alias, hidden } } = consumerField;
-                    dataset.updateDimension(key, { name: alias, hidden: hidden });
-                }
-            }
+            dataset.updateDimensions(fields);
             return dataset;
         };
         this.dropDimensions = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
@@ -237,7 +229,9 @@ class PostProcessorClass {
         });
         this._getFieldValue = (record, field) => {
             var _a, _b, _c;
-            const { key, alias } = field.cField;
+            const { key, alias, fixed, default: defaultValue } = field.cField;
+            if (fixed && Algo_1.default.hasVal(defaultValue))
+                return defaultValue;
             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
             const fieldValue = record.getValue(fieldKey);
             if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {

package/engines/dataset/Dataset.js

@@ -31,6 +31,7 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const XMLParser_1 = __importDefault(require("../parsing/XMLParser"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
 class Dataset {
     constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
         this.getPath = () => this._path;
@@ -678,16 +679,28 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
-        this.updateDimension = (dimensionName, newValues) => {
-            const dimension = this._dimensions.find(x => x.name === dimensionName);
-            (0, Affirm_1.default)(dimension, `Trying to update the dataset dimension "${dimensionName}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
-            this._startOperation('update-dimension');
-            const { hidden, name } = newValues;
-            if (name && name.length > 0)
-                dimension.name = name;
-            if (hidden)
-                dimension.hidden = hidden;
-            this._finishOperation('update-dimension');
+        this.updateDimensions = (fields) => {
+            this._startOperation('update-dimensions');
+            for (const field of fields) {
+                const { cField: { key, alias, hidden, fixed, default: defaultValue } } = field;
+                const currentDim = this._dimensions.find(x => x.name === key);
+                if (currentDim) {
+                    currentDim.name = alias !== null && alias !== void 0 ? alias : key;
+                    currentDim.hidden = hidden;
+                }
+                else if (fixed && Algo_1.default.hasVal(defaultValue)) {
+                    this._dimensions.push({
+                        hidden: hidden,
+                        index: this._dimensions.length,
+                        key: key,
+                        name: alias !== null && alias !== void 0 ? alias : key
+                    });
+                }
+                else {
+                    throw new Error(`Trying to update the dataset dimension "${(alias !== null && alias !== void 0 ? alias : key)}", but none was found (${this._dimensions.map(x => x.name).join(', ')})`);
+                }
+            }
+            this._finishOperation('update-dimensions');
             return this;
         };
         this.dropDimensions = (dimensionNames) => __awaiter(this, void 0, void 0, function* () {

package/engines/execution/ExecutionPlanner.js

@@ -85,6 +85,8 @@ class ExecutionPlannerClas {
                 plan.push(...this._planProducer(producers[0], options));
             else
                 plan.push(...(producers.flatMap(x => this._planProducer(x, options))));
+            // I technically don't need this, but I keep it to merge all the datasets to a single one
+            // so the other steps of the plan can work with a single dataset variable
             plan.push({ type: 'join-producers-data' });
             if (consumer.filters && consumer.filters.length > 0)
                 plan.push({ type: 'apply-consumer-filters-on-JSON' });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.46-nasco.3",
+    "version": "0.0.48-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,