@forzalabs/remora 0.2.4 → 0.2.5

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.2.4',
+    cliVersion: '0.2.5',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,

package/actions/mock.js

@@ -0,0 +1,43 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mock = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const DeveloperEngine_1 = __importDefault(require("../engines/ai/DeveloperEngine"));
+const compile_1 = require("./compile");
+const mock = (producerName, records) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        (0, compile_1.compile)();
+        console.log(); // needed for newline
+        const spinner = (0, ora_1.default)(chalk_1.default.blue(`Generating ${records} mock records for producer "${producerName}"...`)).start();
+        const producer = Environment_1.default.getProducer(producerName);
+        if (!producer) {
+            spinner.fail(chalk_1.default.red(`Producer "${producerName}" not found.`));
+            process.exit(1);
+        }
+        const result = yield DeveloperEngine_1.default.createMockData(producer, records);
+        spinner.succeed(`Mock data generated successfully`);
+        console.log(chalk_1.default.green('\n✅ Mock data generation complete!'));
+        console.log(chalk_1.default.blue('📁 File path: ') + chalk_1.default.cyan(result.filePath));
+        console.log(chalk_1.default.blue('📊 Records generated: ') + chalk_1.default.cyan(result.recordCount.toString()));
+        process.exit(0);
+    }
+    catch (err) {
+        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during mock data generation:'), err instanceof Error ? err.message : String(err));
+        process.exit(1);
+    }
+});
+exports.mock = mock;

package/actions/run.js

@@ -20,7 +20,7 @@ const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEn
 const compile_1 = require("./compile");
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
-const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
+const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
         console.log(); // needed for newline
@@ -32,6 +32,13 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
                 throw new Error(`Consumer with name "${consumerName}" was not found.`);
             consumersToExecute.push(cons);
         }
+        else if ((options === null || options === void 0 ? void 0 : options.project) && options.project.length > 0) {
+            const projectConsumers = Environment_1.default._env.consumers.filter(c => { var _a; return ((_a = c.project) === null || _a === void 0 ? void 0 : _a.toLowerCase()) === options.project.toLowerCase(); });
+            if (projectConsumers.length === 0) {
+                throw new Error(`No consumers found for project "${options.project}".`);
+            }
+            consumersToExecute.push(...projectConsumers);
+        }
         else {
             consumersToExecute.push(...Environment_1.default._env.consumers);
         }

package/definitions/json_schemas/consumer-schema.json

@@ -842,6 +842,264 @@
                     },
                     "required": ["combine_fields"],
                     "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Apply conditional logic to transform values based on comparison conditions",
+                    "properties": {
+                        "conditional": {
+                            "type": "object",
+                            "properties": {
+                                "clauses": {
+                                    "type": "array",
+                                    "description": "Array of if-then clauses evaluated in order. First matching clause wins.",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "if": {
+                                                "$ref": "#/definitions/comparisonCondition"
+                                            },
+                                            "then": {
+                                                "description": "The value to return if the condition matches",
+                                                "oneOf": [
+                                                    { "type": "string" },
+                                                    { "type": "number" },
+                                                    { "type": "boolean" }
+                                                ]
+                                            }
+                                        },
+                                        "required": ["if", "then"],
+                                        "additionalProperties": false
+                                    },
+                                    "minItems": 1
+                                },
+                                "else": {
+                                    "description": "Default value if no clause matches. If not specified, the original value is kept.",
+                                    "oneOf": [
+                                        { "type": "string" },
+                                        { "type": "number" },
+                                        { "type": "boolean" },
+                                        { "type": "null" }
+                                    ]
+                                }
+                            },
+                            "required": ["clauses"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["conditional"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "mask": {
+                            "type": "string",
+                            "enum": ["hash", "mask", "crypt", "random", "seeded-random", "none"],
+                            "description": "Apply masking to the field value for data privacy. Options: 'hash' (one-way hash), 'mask' (replace with asterisks), 'crypt' (reversible encryption), 'random' (random value), 'seeded-random' (consistent random value per input), 'none' (no masking)"
+                        }
+                    },
+                    "required": ["mask"],
+                    "additionalProperties": false
+                }
+            ]
+        },
+        "comparisonCondition": {
+            "description": "A condition to compare a field value against. Exactly one comparison operator must be specified.",
+            "oneOf": [
+                {
+                    "type": "object",
+                    "description": "Check if the numeric value is greater than the specified number",
+                    "properties": {
+                        "greater_than": {
+                            "type": "number",
+                            "description": "The value must be greater than this number"
+                        }
+                    },
+                    "required": ["greater_than"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the numeric value is greater than or equal to the specified number",
+                    "properties": {
+                        "greater_than_or_equal": {
+                            "type": "number",
+                            "description": "The value must be greater than or equal to this number"
+                        }
+                    },
+                    "required": ["greater_than_or_equal"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the numeric value is less than the specified number",
+                    "properties": {
+                        "less_than": {
+                            "type": "number",
+                            "description": "The value must be less than this number"
+                        }
+                    },
+                    "required": ["less_than"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the numeric value is less than or equal to the specified number",
+                    "properties": {
+                        "less_than_or_equal": {
+                            "type": "number",
+                            "description": "The value must be less than or equal to this number"
+                        }
+                    },
+                    "required": ["less_than_or_equal"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value equals the specified value (strict equality)",
+                    "properties": {
+                        "equals": {
+                            "description": "The value must equal this exactly",
+                            "oneOf": [
+                                { "type": "string" },
+                                { "type": "number" },
+                                { "type": "boolean" }
+                            ]
+                        }
+                    },
+                    "required": ["equals"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value does not equal the specified value",
+                    "properties": {
+                        "not_equals": {
+                            "description": "The value must not equal this",
+                            "oneOf": [
+                                { "type": "string" },
+                                { "type": "number" },
+                                { "type": "boolean" }
+                            ]
+                        }
+                    },
+                    "required": ["not_equals"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value is in the specified list of values",
+                    "properties": {
+                        "in": {
+                            "type": "array",
+                            "description": "The value must be one of these values",
+                            "items": {
+                                "oneOf": [
+                                    { "type": "string" },
+                                    { "type": "number" },
+                                    { "type": "boolean" }
+                                ]
+                            },
+                            "minItems": 1
+                        }
+                    },
+                    "required": ["in"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value is not in the specified list of values",
+                    "properties": {
+                        "not_in": {
+                            "type": "array",
+                            "description": "The value must not be any of these values",
+                            "items": {
+                                "oneOf": [
+                                    { "type": "string" },
+                                    { "type": "number" },
+                                    { "type": "boolean" }
+                                ]
+                            },
+                            "minItems": 1
+                        }
+                    },
+                    "required": ["not_in"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the string value starts with the specified prefix",
+                    "properties": {
+                        "starts_with": {
+                            "type": "string",
+                            "description": "The string value must start with this prefix"
+                        }
+                    },
+                    "required": ["starts_with"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the string value ends with the specified suffix",
+                    "properties": {
+                        "ends_with": {
+                            "type": "string",
+                            "description": "The string value must end with this suffix"
+                        }
+                    },
+                    "required": ["ends_with"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the string value contains the specified substring",
+                    "properties": {
+                        "contains": {
+                            "type": "string",
+                            "description": "The string value must contain this substring"
+                        }
+                    },
+                    "required": ["contains"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the string value does not contain the specified substring",
+                    "properties": {
+                        "not_contains": {
+                            "type": "string",
+                            "description": "The string value must not contain this substring"
+                        }
+                    },
+                    "required": ["not_contains"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value is empty (null, undefined, or blank string)",
+                    "properties": {
+                        "is_empty": {
+                            "type": "boolean",
+                            "const": true,
+                            "description": "Returns true if the value is null, undefined, or an empty/whitespace-only string"
+                        }
+                    },
+                    "required": ["is_empty"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "description": "Check if the value is not empty (has content)",
+                    "properties": {
+                        "is_not_empty": {
+                            "type": "boolean",
+                            "const": true,
+                            "description": "Returns true if the value is not null, not undefined, and not an empty/whitespace-only string"
+                        }
+                    },
+                    "required": ["is_not_empty"],
+                    "additionalProperties": false
                 }
             ]
         },

package/definitions/json_schemas/producer-schema.json

@@ -145,7 +145,8 @@
                         "TXT",
                         "XLS",
                         "XLSX",
-                        "XML"
+                        "XML",
+                        "PARQUET"
                     ],
                     "description": "The type of file to read"
                 },

package/drivers/DriverHelper.js

@@ -25,6 +25,10 @@ const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
+const path_1 = __importDefault(require("path"));
+const Constants_1 = __importDefault(require("../Constants"));
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser"));
 const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
@@ -33,7 +37,7 @@ const DriverHelper = {
         const keys = (fileType === 'JSON' || fileType === 'JSONL')
             ? Object.keys(JSON.parse(headerLine))
             : [];
-        const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
+        const shouldValidateHeader = fileType === 'CSV' || fileType === 'XLS' || (fileType === 'TXT' && hasHeaderRow === true);
         // When sourceFilename is set, the headerLine includes $source_filename at the end.
         // For validation, we need to compare against the original header without this suffix.
         const originalHeaderLine = sourceFilename
@@ -150,9 +154,14 @@ const DriverHelper = {
         return lineCount;
     }),
     quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
-        var _a, e_1, _b, _c;
         const fileStream = (0, fs_1.createReadStream)(filePath);
-        const rl = (0, readline_1.createInterface)({ input: fileStream, crlfDelay: Infinity });
+        const lines = yield DriverHelper.quickReadStream(fileStream, lineCount);
+        fileStream.close();
+        return lines;
+    }),
+    quickReadStream: (stream, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const rl = (0, readline_1.createInterface)({ input: stream, crlfDelay: Infinity });
         const lines = [];
         let counter = 0;
         try {
@@ -175,8 +184,42 @@ const DriverHelper = {
             finally { if (e_1) throw e_1.error; }
         }
         rl.close();
-        fileStream.close();
         return lines;
+    }),
+    setHeaderFromFile: (fileKey, file, filePath, dataset) => __awaiter(void 0, void 0, void 0, function* () {
+        (0, Affirm_1.default)(filePath, 'Invalid path');
+        (0, Affirm_1.default)(fileKey, 'Invalid fileKey');
+        (0, Affirm_1.default)(file, 'Invalid File');
+        let headerLine;
+        switch (file.fileType) {
+            case 'XLS':
+            case 'XLSX':
+                headerLine = yield XLSParser_1.default.getHeaderXls(path_1.default.join(filePath, fileKey), file.sheetName);
+                if (file.includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
+                dataset.setFirstLine(headerLine);
+                break;
+            case 'XML':
+                // using a differnt logic for encoded type xls and xlsx
+                headerLine = (yield XMLParser_1.default.readXmlLines(path_1.default.join(filePath, fileKey)))[0];
+                dataset.setFirstLine(headerLine);
+                break;
+            case 'CSV':
+            case 'JSON':
+            case 'JSONL':
+            case 'TXT':
+                // Get header line from the first file
+                headerLine = (yield DriverHelper.quickReadFile(path_1.default.join(filePath, fileKey), 1))[0];
+                // If including source filename, append a placeholder column name to the header
+                if (file.includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
+                dataset.setFirstLine(headerLine);
+                break;
+            default:
+                throw new Error(`the fileType "${file.fileType}" is not implemented yet`);
+        }
     })
 };
 exports.default = DriverHelper;

package/drivers/LocalDriver.js

@@ -60,12 +60,12 @@ const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
+const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
-const Constants_1 = __importDefault(require("../Constants"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -125,13 +125,18 @@ class LocalSourceDriver {
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
             const includeSourceFilename = file.includeSourceFilename === true;
-            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename) {
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename, stream) {
                 const sourceFilePath = path_1.default.join(this._path, fileKey);
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
                 // Copy and validate header in a single stream pass
                 const readStream = fs.createReadStream(sourceFilePath);
+                let streamToUse = readStream;
+                if (['XLS', 'XLSX'].includes(file.fileType))
+                    streamToUse = stream;
+                else
+                    streamToUse = readStream;
                 return DriverHelper_1.default.appendToUnifiedFile({
-                    stream: readStream,
+                    stream: streamToUse,
                     fileKey,
                     destinationPath: dataset.getPath(),
                     append: appendMode,
@@ -142,38 +147,45 @@ class LocalSourceDriver {
                     sourceFilename
                 });
             });
+            // this function copy the local file on the temporary file and retrive the number of line.
+            const handleFileAndGetLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
+                let totalLineCount;
+                let streamXLS;
+                switch (fileType) {
+                    case 'XLS':
+                    case 'XLSX':
+                        streamXLS = (yield XLSParser_1.default.getStreamXls(path_1.default.join(this._path, fileKey), file.sheetName));
+                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename, streamXLS);
+                        break;
+                    default:
+                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), false, sourceFilename);
+                        break;
+                }
+                return totalLineCount;
+            });
             const { fileKey } = file;
+            let totalLineCount = 0;
+            let sourceFilename;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
+                yield DriverHelper_1.default.setHeaderFromFile(allFileKeys[0], file, this._path, dataset);
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
-                // Get header line from the first file
-                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
-                // If including source filename, append a placeholder column name to the header
-                if (includeSourceFilename) {
-                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
-                }
-                dataset.setFirstLine(headerLine);
-                let totalLineCount = 0;
+                totalLineCount = 0;
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
                     const currentFileKey = allFileKeys[i];
                     // Pass the filename (just the basename) if includeSourceFilename is enabled
                     const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
-                    totalLineCount += yield copyLocally(currentFileKey, headerLine, i > 0, sourceFilename); // Append mode for subsequent files
+                    totalLineCount += yield handleFileAndGetLineCount(currentFileKey, true, file.fileType, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
-                // For single file, include the filename if configured
-                let headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
-                if (includeSourceFilename) {
-                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
-                }
-                dataset.setFirstLine(headerLine);
-                const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
-                const totalLineCount = yield copyLocally(fileKey, headerLine, false, sourceFilename);
+                sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                yield DriverHelper_1.default.setHeaderFromFile(fileKey, file, this._path, dataset);
+                totalLineCount = (yield handleFileAndGetLineCount(fileKey, false, file.fileType, sourceFilename));
                 dataset.setCount(totalLineCount);
                 return dataset;
             }

package/drivers/S3Driver.js

@@ -34,6 +34,7 @@ const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const Constants_1 = __importDefault(require("../Constants"));
+const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -224,7 +225,16 @@ class S3SourceDriver {
                 });
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
-                const stream = response.Body;
+                let stream;
+                switch (file.fileType) {
+                    case 'XLS':
+                    case 'XLSX':
+                        stream = yield XLSParser_1.default.parseXLSStream(response.Body, file.sheetName);
+                        break;
+                    default:
+                        stream = response.Body;
+                        break;
+                }
                 return DriverHelper_1.default.appendToUnifiedFile({
                     stream,
                     fileKey: fileUrl,
@@ -238,6 +248,45 @@ class S3SourceDriver {
                 });
             });
             const { fileKey } = file;
+            const setFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
+                var _a, e_1, _b, _c;
+                const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+                let firstLine = '';
+                switch (file.fileType) {
+                    case 'XLSX':
+                    case 'XLS':
+                        firstLine = yield XLSParser_1.default.getHeaderXlsFromStream(stream, file.sheetName);
+                        break;
+                    case 'CSV':
+                    case 'JSON':
+                    case 'JSONL':
+                    case 'TXT':
+                        try {
+                            for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                                _c = rl_1_1.value;
+                                _d = false;
+                                const line = _c;
+                                firstLine = line;
+                                break;
+                            }
+                        }
+                        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                        finally {
+                            try {
+                                if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                            }
+                            finally { if (e_1) throw e_1.error; }
+                        }
+                        rl.close();
+                        break;
+                }
+                // If including source filename, append a placeholder column name to the header
+                if (file.includeSourceFilename) {
+                    firstLine = firstLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
+                dataset.setFirstLine(firstLine);
+                return firstLine;
+            });
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
@@ -250,12 +299,7 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
-                // If including source filename, append a placeholder column name to the header
-                if (includeSourceFilename) {
-                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
-                }
-                dataset.setFirstLine(headerLine);
+                const headerLine = yield setFirstLineFromStream(firstFileStream);
                 let totalLineCount = 0;
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
@@ -276,12 +320,7 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
-                // If including source filename, append a placeholder column name to the header
-                if (includeSourceFilename) {
-                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
-                }
-                dataset.setFirstLine(headerLine);
+                const headerLine = yield setFirstLineFromStream(firstFileStream);
                 // Pass the filename if includeSourceFilename is enabled
                 const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
                 const totalLineCount = yield downloadLocally(fileKey, headerLine, false, sourceFilename);
@@ -289,29 +328,6 @@ class S3SourceDriver {
                 return dataset;
             }
         });
-        this.getFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            let firstLine = '';
-            try {
-                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
-                    _c = rl_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    firstLine = line;
-                    break;
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            rl.close();
-            return firstLine;
-        });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');

package/engines/ai/DeveloperEngine.js

@@ -14,6 +14,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const ProducerEngine_1 = __importDefault(require("../producer/ProducerEngine"));
+const Environment_1 = __importDefault(require("../Environment"));
 const path_1 = __importDefault(require("path"));
 const promises_1 = __importDefault(require("fs/promises"));
 const dayjs_1 = __importDefault(require("dayjs"));
@@ -327,6 +328,169 @@ class DeveloperEngineClass {
                 reason: 'No PHI/PII patterns detected'
             };
         };
+        this.createMockData = (producer, records) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            (0, Affirm_1.default)(records > 0, 'Record count must be greater than 0');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            (0, Affirm_1.default)(source.engine === 'local', `Mock data generation only supports local file-based producers. Source engine "${source.engine}" is not supported.`);
+            const { fileKey, fileType, delimiter } = producer.settings;
+            (0, Affirm_1.default)(fileKey, 'Producer must have a fileKey setting for mock data generation');
+            (0, Affirm_1.default)(fileType, 'Producer must have a fileType setting for mock data generation');
+            // Generate mock records
+            const mockRecords = this.generateMockRecords(producer.dimensions, records);
+            // Get the file path
+            const basePath = source.authentication.path || process.cwd();
+            const filePath = path_1.default.join(basePath, fileKey.replace('%', 'mock'));
+            // Ensure directory exists
+            yield promises_1.default.mkdir(path_1.default.dirname(filePath), { recursive: true });
+            // Write to file based on type
+            const content = this.formatMockData(mockRecords, fileType, delimiter);
+            yield promises_1.default.writeFile(filePath, content, 'utf-8');
+            return { filePath, recordCount: records };
+        });
+        this.generateMockRecords = (dimensions, count) => {
+            const records = [];
+            for (let i = 0; i < count; i++) {
+                const record = {};
+                for (const dimension of dimensions) {
+                    // Skip sourceFilename dimensions as they are auto-populated
+                    if (dimension.sourceFilename)
+                        continue;
+                    record[dimension.name] = this.generateMockValue(dimension, i);
+                }
+                records.push(record);
+            }
+            return records;
+        };
+        this.generateMockValue = (dimension, index) => {
+            const { name, type } = dimension;
+            const nameLower = name.toLowerCase();
+            // Generate contextual mock data based on field name patterns
+            if (this.matchesPattern(nameLower, ['id', 'identifier', 'key', 'pk'])) {
+                return `${index + 1}`;
+            }
+            if (this.matchesPattern(nameLower, ['first_name', 'firstname', 'fname', 'given_name'])) {
+                return this.pickRandom(['John', 'Jane', 'Michael', 'Sarah', 'David', 'Emily', 'Robert', 'Lisa', 'James', 'Mary']);
+            }
+            if (this.matchesPattern(nameLower, ['last_name', 'lastname', 'lname', 'surname', 'family_name'])) {
+                return this.pickRandom(['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia', 'Miller', 'Davis', 'Martinez', 'Wilson']);
+            }
+            if (this.matchesPattern(nameLower, ['name', 'full_name', 'fullname'])) {
+                const firstNames = ['John', 'Jane', 'Michael', 'Sarah', 'David', 'Emily'];
+                const lastNames = ['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia'];
+                return `${this.pickRandom(firstNames)} ${this.pickRandom(lastNames)}`;
+            }
+            if (this.matchesPattern(nameLower, ['email', 'mail'])) {
+                return `user${index + 1}@example.com`;
+            }
+            if (this.matchesPattern(nameLower, ['phone', 'telephone', 'mobile', 'cell'])) {
+                return `555-${String(Math.floor(Math.random() * 900) + 100).padStart(3, '0')}-${String(Math.floor(Math.random() * 9000) + 1000).padStart(4, '0')}`;
+            }
+            if (this.matchesPattern(nameLower, ['address', 'street', 'addr'])) {
+                const streets = ['Main St', 'Oak Ave', 'Elm Dr', 'Pine Rd', 'Maple Ln', 'Cedar Blvd'];
+                return `${Math.floor(Math.random() * 9999) + 1} ${this.pickRandom(streets)}`;
+            }
+            if (this.matchesPattern(nameLower, ['city', 'town'])) {
+                return this.pickRandom(['New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego']);
+            }
+            if (this.matchesPattern(nameLower, ['state', 'province'])) {
+                return this.pickRandom(['CA', 'TX', 'FL', 'NY', 'PA', 'IL', 'OH', 'GA', 'NC', 'MI']);
+            }
+            if (this.matchesPattern(nameLower, ['zip', 'postal', 'zipcode'])) {
+                return String(Math.floor(Math.random() * 90000) + 10000);
+            }
+            if (this.matchesPattern(nameLower, ['country'])) {
+                return this.pickRandom(['USA', 'Canada', 'UK', 'Germany', 'France', 'Australia']);
+            }
+            if (this.matchesPattern(nameLower, ['age', 'years'])) {
+                return Math.floor(Math.random() * 80) + 18;
+            }
+            if (this.matchesPattern(nameLower, ['sex', 'gender'])) {
+                return this.pickRandom(['M', 'F', 'Male', 'Female']);
+            }
+            if (this.matchesPattern(nameLower, ['birth', 'dob', 'birthdate'])) {
+                const year = Math.floor(Math.random() * 60) + 1940;
+                const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, '0');
+                const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, '0');
+                return `${year}-${month}-${day}`;
+            }
+            if (this.matchesPattern(nameLower, ['date', 'created', 'updated', 'timestamp'])) {
+                const year = Math.floor(Math.random() * 5) + 2020;
+                const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, '0');
+                const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, '0');
+                return `${year}-${month}-${day}`;
+            }
+            if (this.matchesPattern(nameLower, ['amount', 'price', 'cost', 'total', 'balance'])) {
+                return (Math.random() * 1000).toFixed(2);
+            }
+            if (this.matchesPattern(nameLower, ['quantity', 'count', 'qty'])) {
+                return Math.floor(Math.random() * 100) + 1;
+            }
+            if (this.matchesPattern(nameLower, ['status'])) {
+                return this.pickRandom(['active', 'inactive', 'pending', 'completed', 'cancelled']);
+            }
+            if (this.matchesPattern(nameLower, ['type', 'category'])) {
+                return this.pickRandom(['TypeA', 'TypeB', 'TypeC', 'TypeD']);
+            }
+            if (this.matchesPattern(nameLower, ['description', 'desc', 'notes', 'comment'])) {
+                return `Sample description for record ${index + 1}`;
+            }
+            // Fall back to type-based generation
+            return this.generateValueByType(type, index);
+        };
+        this.matchesPattern = (fieldName, patterns) => {
+            return patterns.some(pattern => fieldName.includes(pattern));
+        };
+        this.pickRandom = (arr) => {
+            return arr[Math.floor(Math.random() * arr.length)];
+        };
+        this.generateValueByType = (type, index) => {
+            switch (type) {
+                case 'string':
+                    return `value_${index + 1}`;
+                case 'number':
+                    return Math.floor(Math.random() * 1000);
+                case 'boolean':
+                    return Math.random() > 0.5;
+                case 'datetime': {
+                    const year = Math.floor(Math.random() * 5) + 2020;
+                    const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, '0');
+                    const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, '0');
+                    return `${year}-${month}-${day}`;
+                }
+                default:
+                    return `value_${index + 1}`;
+            }
+        };
+        this.formatMockData = (records, fileType, delimiter) => {
+            switch (fileType) {
+                case 'JSON':
+                    return JSON.stringify(records, null, 2);
+                case 'JSONL':
+                    return records.map(r => JSON.stringify(r)).join('\n');
+                case 'CSV':
+                case 'TXT': {
+                    const delim = delimiter || ',';
+                    if (records.length === 0)
+                        return '';
+                    const headers = Object.keys(records[0]);
+                    const headerLine = headers.join(delim);
+                    const dataLines = records.map(record => headers.map(h => {
+                        const val = record[h];
+                        const strVal = val === null || val === undefined ? '' : String(val);
+                        // Escape delimiter and quotes in values
+                        if (strVal.includes(delim) || strVal.includes('"') || strVal.includes('\n')) {
+                            return `"${strVal.replace(/"/g, '""')}"`;
+                        }
+                        return strVal;
+                    }).join(delim));
+                    return [headerLine, ...dataLines].join('\n');
+                }
+                default:
+                    throw new Error(`Unsupported file type for mock data generation: ${fileType}`);
+            }
+        };
     }
 }
 const DeveloperEngine = new DeveloperEngineClass();

package/engines/consumer/PostProcessor.js

@@ -46,8 +46,9 @@ class PostProcessorClass {
                     }
                     record.sortDimensions();
                 }
-                if (!updatedDimensions)
+                if (!updatedDimensions) {
                     updatedDimensions = record._dimensions;
+                }
                 // Finally apply the rules and changes of the consumer fields to the record
                 for (const field of fields) {
                     const { key, alias } = field.cField;
@@ -70,6 +71,11 @@ class PostProcessorClass {
                     fakeRecord.wholeUpdateDimension(update);
                 updatedDimensions = fakeRecord._dimensions;
             }
+            // Validate that dimensions have sequential indexes with no gaps
+            const indexes = updatedDimensions.map(d => d.index).sort((a, b) => a - b);
+            for (let i = 0; i < indexes.length; i++) {
+                (0, Affirm_1.default)(indexes[i] === i, `Missing or duplicate dimension index: expected index ${i} but found ${indexes[i]}. See dimension updates applied on consumer "${consumer.name}".`);
+            }
             dataset.setDimensions(updatedDimensions);
             return dataset;
         });

package/engines/dataset/DatasetManager.js

@@ -69,6 +69,8 @@ class DatasetManagerClass {
             (0, Affirm_1.default)(producer, `Invalid producer`);
             const file = dsFile;
             switch (file.fileType) {
+                case 'XLS':
+                case 'XLSX':
                 case 'CSV': {
                     const delimiterChar = (_a = file.delimiter) !== null && _a !== void 0 ? _a : ',';
                     const headerLine = firstLine;
@@ -86,18 +88,12 @@ class DatasetManagerClass {
                 }
                 case 'PARQUET':
                 case 'JSONL':
+                case 'XML':
                 case 'JSON': {
                     const source = Environment_1.default.getSource(producer.source);
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
-                    // const columnsWithDot = columns.filter(x => x.aliasInProducer.includes('.'))
-                    // if (columnsWithDot.length > 0) {
-                    //     console.log(columns, keys, 'PAPAPAPP')
-                    //     for (const colWithDot of columnsWithDot) {
-                    //         console.log(colWithDot)
-                    //     }
-                    // }
                     // If includeSourceFilename is enabled, the driver has added $source_filename column
                     // We need to add it to the keys list so dimensions can reference it
                     const includeSourceFilename = file.includeSourceFilename === true;
@@ -211,12 +207,6 @@ class DatasetManagerClass {
                         };
                     }
                 }
-                case 'XLS':
-                    break;
-                case 'XLSX':
-                    break;
-                case 'XML':
-                    break;
             }
         });
         this.computeDimensionsUpdates = (dataset, consumer) => {

package/engines/dataset/DatasetRecord.js

@@ -11,8 +11,7 @@ class DatasetRecord {
         this.parse = (row, delimiter, dimensions) => {
             if (!this.isEmpty() && dimensions.length > 0) {
                 const parts = CSVParser_1.default.parseRow(row, delimiter);
-                for (let i = 0; i < dimensions.length; i++) {
-                    const dim = dimensions[i];
+                for (const dim of dimensions) {
                     // Use dim.index to get the correct column from the file, not the loop index
                     this._value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.type, dim.format);
                 }
@@ -53,8 +52,8 @@ class DatasetRecord {
                     index: update.newPosition,
                     key: update.newName,
                     name: update.newName,
-                    hidden: update.newHidden,
-                    type: (_b = (_a = update.currentDimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string'
+                    type: (_b = (_a = update.currentDimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string',
+                    hidden: update.newHidden
                 };
                 this._value[newDimension.name] = null;
                 this._dimensions = [...this._dimensions, newDimension];

package/engines/parsing/XLSParser.js

@@ -0,0 +1,87 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const DriverHelper_1 = __importDefault(require("../../drivers/DriverHelper"));
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const xlsx_1 = __importDefault(require("xlsx"));
+class XLSParserClass {
+    constructor() {
+        this.getStreamXls = (filePath, sheetName) => __awaiter(this, void 0, void 0, function* () {
+            const fileXls = xlsx_1.default.readFile(filePath);
+            (0, Affirm_1.default)(fileXls.SheetNames.length > 0, 'The Excel file has no sheets.');
+            const targetSheetName = yield this.getTargetSheetName(fileXls, sheetName);
+            const workSheet = fileXls.Sheets[targetSheetName];
+            const xlsStream = xlsx_1.default.stream.to_csv(workSheet);
+            return xlsStream;
+        });
+        this.getHeaderXls = (filePath, sheetName) => __awaiter(this, void 0, void 0, function* () {
+            const header = yield DriverHelper_1.default.quickReadStream(yield this.getStreamXls(filePath, sheetName), 2);
+            return header[0];
+        });
+        this.getHeaderXlsFromStream = (stream, sheetName) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            let header;
+            const workbook = xlsx_1.default.read(stream);
+            const targetSheetName = yield this.getTargetSheetName(workbook, sheetName);
+            const xlsxStream = xlsx_1.default.stream.to_csv(workbook.Sheets[targetSheetName]);
+            try {
+                for (var _d = true, xlsxStream_1 = __asyncValues(xlsxStream), xlsxStream_1_1; xlsxStream_1_1 = yield xlsxStream_1.next(), _a = xlsxStream_1_1.done, !_a; _d = true) {
+                    _c = xlsxStream_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (line != '') {
+                        header = line;
+                        break;
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = xlsxStream_1.return)) yield _b.call(xlsxStream_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            (0, Affirm_1.default)(header, `the file received from the stream is empty`);
+            return header;
+        });
+        this.getTargetSheetName = (workbook, sheetName) => __awaiter(this, void 0, void 0, function* () {
+            let targetSheetName = sheetName ? sheetName : workbook.SheetNames[0];
+            if (!targetSheetName) {
+                (0, Affirm_1.default)(workbook.SheetNames.length > 0, 'The Excel file has no sheets.');
+                targetSheetName = workbook.SheetNames[0];
+            }
+            else {
+                (0, Affirm_1.default)(workbook.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${workbook.SheetNames.join(', ')})`);
+            }
+            return targetSheetName;
+        });
+        this.parseXLSStream = (xlsStream, sheetName) => __awaiter(this, void 0, void 0, function* () {
+            const workbook = xlsx_1.default.read(xlsStream);
+            const targetSheetName = yield this.getTargetSheetName(workbook, sheetName);
+            const workSheet = workbook.Sheets[targetSheetName];
+            const csvStream = xlsx_1.default.stream.to_csv(workSheet);
+            return csvStream;
+        });
+    }
+}
+const XLSParser = new XLSParserClass();
+exports.default = XLSParser;

package/engines/parsing/XMLParser.js

@@ -1,10 +1,54 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const fast_xml_parser_1 = require("fast-xml-parser");
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const fs = __importStar(require("fs"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
 const DEFAULT_OPTIONS = {
     attributeNamePrefix: '@_',
     ignoreAttributes: false,
@@ -52,6 +96,18 @@ class XMLParserClass {
                 throw new Error('Failed to parse XML data.');
             }
         };
+        // TO DO: improve performances for larger file using streams instead of an array of string
+        this.readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = fs.readFileSync(fileUri, 'utf-8');
+            const jsonData = XMLParser.xmlToJson(fileContent);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            // Assuming jsonData is an array of objects, where each object is a record.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
         this._parser = new fast_xml_parser_1.XMLParser(Object.assign(Object.assign({}, DEFAULT_OPTIONS), options)); // Use 'as any' if type issues persist with library
     }
 }

package/engines/transform/TransformationEngine.js

@@ -100,12 +100,36 @@ class TransformationEngineClass {
                     throw new Error(`Cannot multiply non-numeric value in field '${field.key}'`);
                 return num * transformations.multiply;
             }
+            if ('multiplyBy' in transformations) {
+                if (!record) {
+                    throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
+                }
+                const { fields } = transformations.multiplyBy;
+                const fieldValues = fields.map(fieldName => {
+                    const fieldValue = record.getValue(fieldName);
+                    return fieldValue !== null && fieldValue !== undefined ? TypeCaster_1.default.cast(fieldValue, 'number') : 1;
+                });
+                const product = fieldValues.reduce((accumulator, value) => accumulator * value, 1);
+                return product;
+            }
             if ('add' in transformations) {
                 const num = TypeCaster_1.default.cast(value, 'number');
                 if (isNaN(num))
                     throw new Error(`Cannot add to non-numeric value in field '${field.key}'`);
                 return num + transformations.add;
             }
+            if ('addBy' in transformations) {
+                if (!record) {
+                    throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
+                }
+                const { fields } = transformations.addBy;
+                const fieldValues = fields.map(fieldName => {
+                    const fieldValue = record.getValue(fieldName);
+                    return fieldValue !== null && fieldValue !== undefined ? TypeCaster_1.default.cast(fieldValue, 'number') : 1;
+                });
+                const sum = fieldValues.reduce((accumulator, value) => accumulator + value);
+                return sum;
+            }
             if ('extract' in transformations) {
                 const date = TypeCaster_1.default.cast(value, 'date');
                 if (isNaN(date.getTime()))
@@ -277,6 +301,24 @@ class TransformationEngineClass {
             if ('not_in' in condition) {
                 return !condition.not_in.includes(value);
             }
+            if ('starts_with' in condition) {
+                return TypeCaster_1.default.cast(value, 'string').startsWith(condition.starts_with);
+            }
+            if ('ends_with' in condition) {
+                return TypeCaster_1.default.cast(value, 'string').endsWith(condition.ends_with);
+            }
+            if ('contains' in condition) {
+                return TypeCaster_1.default.cast(value, 'string').includes(condition.contains);
+            }
+            if ('not_contains' in condition) {
+                return !TypeCaster_1.default.cast(value, 'string').includes(condition.not_contains);
+            }
+            if ('is_empty' in condition) {
+                return value === null || value === undefined || TypeCaster_1.default.cast(value, 'string').trim() === '';
+            }
+            if ('is_not_empty' in condition) {
+                return value !== null && value !== undefined && TypeCaster_1.default.cast(value, 'string').trim() !== '';
+            }
             return false;
         };
         this.applyMasking = (value, maskType, field) => {

package/index.js

@@ -19,6 +19,7 @@ const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
 const Runtime_1 = __importDefault(require("./helper/Runtime"));
 const automap_1 = require("./actions/automap");
 const sample_1 = require("./actions/sample");
+const mock_1 = require("./actions/mock");
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
 // Validate the remora licence
@@ -66,8 +67,9 @@ program
     .action(deploy_1.deploy);
 program
     .command('run [name]')
-    .description('Execute consumers. Optionally specify a single consumers name to run.')
-    .action(run_1.run);
+    .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
+    .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
+    .action((name, options) => (0, run_1.run)(name, options));
 program
     .command('discover')
     .description('Discover the data shape of a producer and automatically create the resource for it.')
@@ -100,4 +102,16 @@ program
     }
     (0, sample_1.sample)(name, sampleSize);
 });
+program
+    .command('mock <producer>')
+    .description('Generate mock data for a producer based on its dimensions')
+    .argument('<records>', 'Number of mock records to generate')
+    .action((producer, records) => {
+    const recordCount = parseInt(records, 10);
+    if (isNaN(recordCount) || recordCount <= 0) {
+        console.error('Record count must be a positive number');
+        process.exit(1);
+    }
+    (0, mock_1.mock)(producer, recordCount);
+});
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.2.4",
+    "version": "0.2.5",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -10,6 +10,7 @@
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
         "dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
+        "dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
         "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",