@forzalabs/remora 0.0.28 → 0.0.30

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.28',
+    cliVersion: '0.0.30',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/producer-schema.json

@@ -68,9 +68,11 @@
                         "enum": [
                             "hash",
                             "mask",
-                            "crypt"
+                            "crypt",
+                            "random",
+                            "seeded-random"
                         ],
-                        "description": "Masking type to apply to this dimension"
+                        "description": "Masking type to apply to this dimension. 'hash' replaces with a hashed value. 'mask' replaces characters with a mask character. 'crypt' encrypts the value. 'random' replaces with a random value. 'seeded-random' replaces with a random value generated from a seed."
                     }
                 },
                 "required": [
@@ -135,7 +137,8 @@
                         "CSV", 
                         "TXT",
                         "XLS",
-                        "XLSX"
+                        "XLSX",
+                        "XML"
                     ],
                     "description": "The type of file to read"
                 },

package/drivers/LocalDriver.js

@@ -59,6 +59,7 @@ const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -85,6 +86,8 @@ class LocalSourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(fileUrl, options === null || options === void 0 ? void 0 : options.sheetName);
+                case 'XML':
+                    return yield this._readXmlLines(fileUrl);
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -106,6 +109,8 @@ class LocalSourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
+                case 'XML':
+                    return yield this._readXmlLines(fileUrl, lineFrom, lineTo);
             }
         });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
@@ -169,6 +174,17 @@ class LocalSourceDriver {
             else
                 return lines;
         });
+        this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = fs.readFileSync(fileUri, 'utf-8');
+            const jsonData = XMLParser_1.default.xmlToJson(fileContent);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            // Assuming jsonData is an array of objects, where each object is a record.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
     }
 }
 exports.LocalSourceDriver = LocalSourceDriver;

package/drivers/S3Driver.js

@@ -26,6 +26,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const readline_1 = __importDefault(require("readline"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -149,6 +150,8 @@ class S3SourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(stream, options === null || options === void 0 ? void 0 : options.sheetName);
+                case 'XML':
+                    return yield this._readXmlLines(stream);
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -172,6 +175,8 @@ class S3SourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(stream, sheetName, lineFrom, lineTo);
+                case 'XML':
+                    return yield this._readXmlLines(stream, lineFrom, lineTo);
             }
         });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
@@ -255,6 +260,34 @@ class S3SourceDriver {
             else
                 return lines;
         });
+        this._readXmlLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, stream_2, stream_2_1;
+            var _b, e_3, _c, _d;
+            const chunks = [];
+            try {
+                for (_a = true, stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), _b = stream_2_1.done, !_b; _a = true) {
+                    _d = stream_2_1.value;
+                    _a = false;
+                    const chunk = _d;
+                    chunks.push(chunk);
+                }
+            }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            finally {
+                try {
+                    if (!_a && !_b && (_c = stream_2.return)) yield _c.call(stream_2);
+                }
+                finally { if (e_3) throw e_3.error; }
+            }
+            const buffer = Buffer.concat(chunks);
+            const jsonData = XMLParser_1.default.xmlToJson(buffer);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
     }
 }
 exports.S3SourceDriver = S3SourceDriver;

package/engines/CryptoEngine.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const crypto_1 = __importDefault(require("crypto"));
 const Algo_1 = __importDefault(require("../core/Algo"));
+const RandomEngine_1 = __importDefault(require("./RandomEngine"));
 class CryptoEngineClass {
     constructor() {
         this.hashQuery = (maskType, fieldReference, fieldName) => {
@@ -24,7 +25,7 @@ class CryptoEngineClass {
         this.valueToHash = (value) => {
             return crypto_1.default.createHash('sha256').update(JSON.stringify(value)).digest('hex');
         };
-        this.hashValue = (maskType, value) => {
+        this.hashValue = (maskType, value, valueType) => {
             if (!Algo_1.default.hasVal(value))
                 return value;
             if (!Algo_1.default.hasVal(maskType))
@@ -32,6 +33,24 @@ class CryptoEngineClass {
             switch (maskType) {
                 case 'hash':
                     return this.valueToHash(value);
+                case 'random': {
+                    switch (valueType) {
+                        case 'datetime': return RandomEngine_1.default.rngDate();
+                        case 'number': return RandomEngine_1.default.rng();
+                        case 'string': return this.valueToHash(value);
+                        default:
+                            throw new Error('Not implemented yet');
+                    }
+                }
+                case 'seeded-random': {
+                    switch (valueType) {
+                        case 'datetime': return RandomEngine_1.default.sRngDate(value);
+                        case 'number': return RandomEngine_1.default.sRng(value);
+                        case 'string': return this.valueToHash(value);
+                        default:
+                            throw new Error('Not implemented yet');
+                    }
+                }
                 case 'crypt':
                     throw new Error('Not implemented yet');
                 case 'mask':

package/engines/ProducerEngine.js

@@ -19,7 +19,7 @@ const Environment_1 = __importDefault(require("./Environment"));
 const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
 const SQLCompiler_1 = __importDefault(require("./sql/SQLCompiler"));
 const SQLUtils_1 = __importDefault(require("./sql/SQLUtils"));
-const ParseManager_1 = __importDefault(require("./ParseManager"));
+const ParseManager_1 = __importDefault(require("./parsing/ParseManager"));
 class ProducerEngineClass {
     constructor() {
         this.compile = (producer) => {
@@ -107,9 +107,27 @@ class ProducerEngineClass {
                 case 'JSONL':
                 case 'JSON': {
                     if (lines.length === 1) {
-                        lines = lines[0].split('\n');
+                        // Attempt to handle cases where a single line might contain multiple JSON objects separated by newlines
+                        // Or if the entire file content is a single JSON array stringified.
+                        try {
+                            const parsedAsArray = JSON.parse(lines[0]);
+                            if (Array.isArray(parsedAsArray)) {
+                                return { data: parsedAsArray, dataType: 'array-of-json' };
+                            }
+                        }
+                        catch (error) {
+                            // If parsing as array fails, proceed to split by newline
+                            console.warn('Failed to parse single line as JSON array, splitting by newline:', error);
+                        }
+                        lines = lines[0].split('\\n');
                     }
-                    const json = lines.map(x => JSON.parse(x));
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
+                    return { data: json, dataType: 'array-of-json' };
+                }
+                case 'XML': {
+                    // The driver's _readXmlLines method now returns an array of JSON strings.
+                    // Each string needs to be parsed into a JSON object.
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }
                 default:
@@ -155,6 +173,19 @@ class ProducerEngineClass {
                             sampleData = sampleData.slice(0, sampleSize);
                             break;
                         }
+                        case 'XML': {
+                            // fileData.data from readFile for XML is now an array of JSON objects
+                            if (fileData.dataType === 'array-of-json' && Array.isArray(fileData.data)) {
+                                sampleData = fileData.data;
+                            }
+                            else {
+                                // Fallback or error handling if data is not in expected format
+                                console.warn('Unexpected data format for XML in readSampleData');
+                                sampleData = [];
+                            }
+                            sampleData = sampleData.slice(0, sampleSize);
+                            break;
+                        }
                         default: {
                             sampleData = fileData.data;
                             break;

package/engines/RandomEngine.js

@@ -0,0 +1,35 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const seedrandom_1 = __importDefault(require("seedrandom"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+class RandomEngineClass {
+    constructor() {
+        this.rng = (min, max) => {
+            const rng = Math.random;
+            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
+                return Math.floor(rng() * (max - min + 1)) + min;
+            else
+                return rng();
+        };
+        this.sRng = (seed, min, max) => {
+            const rng = (0, seedrandom_1.default)(String(seed));
+            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
+                return Math.floor(rng() * (max - min + 1)) + min;
+            else
+                return rng();
+        };
+        this.rngDate = (min, max) => {
+            const randomNumber = this.rng(min, max);
+            return new Date(randomNumber).toJSON();
+        };
+        this.sRngDate = (seed, min, max) => {
+            const randomNumber = this.sRng(seed, min, max);
+            return new Date(randomNumber).toJSON();
+        };
+    }
+}
+const RandomEngine = new RandomEngineClass();
+exports.default = RandomEngine;

package/engines/consumer/PostProcessor.js

@@ -33,15 +33,16 @@ class PostProcessorClass {
                     const groups = Algo_1.default.groupBy(items, groupingRule.groupingKey);
                     const projections = [];
                     groups.forEach(gItems => {
-                        var _a;
+                        var _a, _b, _c;
                         const projected = {};
                         const first = gItems[0];
                         for (const field of allFields) {
                             const { key, alias, grouping } = field.cField;
                             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
                             const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
+                            const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                             if (!field.cField.grouping) {
-                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey]);
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey], fieldType);
                             }
                             else {
                                 const { subFields } = grouping;
@@ -64,7 +65,7 @@ class PostProcessorClass {
                             const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                             const fieldValue = this._getFieldValue(x, field);
                             if (Algo_1.default.hasVal(maskType))
-                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue);
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue, fieldType);
                             else
                                 projected[fieldKey] = TypeCaster_1.default.cast(fieldValue, fieldType);
                         }
@@ -87,9 +88,10 @@ class PostProcessorClass {
             const columns = FileCompiler_1.default.compileProducer(producer, source);
             (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
             const unpackDimension = (item, dimension) => {
-                var _a;
+                var _a, _b, _c;
                 const { nameInProducer, aliasInProducer } = dimension;
                 const maskType = (_a = dimension.dimension.mask) !== null && _a !== void 0 ? _a : undefined;
+                const fieldType = (_c = (_b = dimension.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
                 const keys = aliasInProducer.split('.');
                 let prevValue = item;
                 for (const key of keys) {
@@ -114,7 +116,7 @@ class PostProcessorClass {
                             prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[key];
                     }
                 }
-                prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue);
+                prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue, fieldType);
                 const res = { [nameInProducer]: prevValue };
                 return res;
             };

package/engines/execution/ExecutionEnvironment.js

@@ -17,7 +17,7 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
 const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const ParseManager_1 = __importDefault(require("../ParseManager"));
+const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const ProducerEngine_1 = __importDefault(require("../ProducerEngine"));
 const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));

package/engines/execution/ExecutionPlanner.js

@@ -104,7 +104,8 @@ class ExecutionPlannerClas {
                         plan.push({ type: 'read-file-lines', producer, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
                     else
                         plan.push({ type: 'read-file-whole', producer });
-                    if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'CSV')
+                    const fileType = (_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase();
+                    if (fileType === 'CSV' || fileType === 'TXT' || fileType === 'XLS' || fileType === 'XLSX')
                         plan.push({ type: 'csv-to-json', producer });
                     if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
                         plan.push({ type: 'nested-field-unpacking', producer });

package/engines/parsing/ParseManager.js

@@ -0,0 +1,75 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+class ParseManagerClass {
+    constructor() {
+        this.csvToJson = (csv, producer) => {
+            (0, Affirm_1.default)(csv, 'Invalid csv content');
+            Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
+            return this.csvLinesToJson(csv, producer);
+        };
+        this.csvLinesToJson = (lines, producer, discover) => {
+            var _a;
+            (0, Affirm_1.default)(lines, 'Invalid csv lines');
+            Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
+            const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
+            const { header, records } = this._getClassifiedRows(lines, delimiterChar, producer);
+            const headerColumns = this._extractHeader(header, delimiterChar, producer, discover);
+            const rows = records.map(x => x.split(delimiterChar).map(k => k.trim()));
+            const result = [];
+            for (const row of rows) {
+                const rowObject = {};
+                for (let i = 0; i < headerColumns.length; i++) {
+                    const column = headerColumns[i];
+                    rowObject[column.saveAs] = row[column.index];
+                }
+                result.push(rowObject);
+            }
+            return result;
+        };
+        this._getClassifiedRows = (lines, delimiterChar, producer) => {
+            if (producer.settings.fileType === 'TXT' && !producer.settings.hasHeaderRow) {
+                // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
+                const source = Environment_1.default.getSource(producer.source);
+                const columns = FileCompiler_1.default.compileProducer(producer, source);
+                return {
+                    header: columns.map(x => x.nameInProducer).join(delimiterChar),
+                    records: lines
+                };
+            }
+            else {
+                return {
+                    header: lines[0],
+                    records: lines.slice(1)
+                };
+            }
+        };
+        this._extractHeader = (headerLine, delimiter, producer, discover) => {
+            var _a;
+            (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
+            (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            const columns = FileCompiler_1.default.compileProducer(producer, source);
+            const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
+            if (discover)
+                columns.push(...headerColumns.map(x => ({ nameInProducer: x })));
+            const csvColumns = [];
+            for (const pColumn of columns) {
+                const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
+                const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
+                (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+                csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
+            }
+            return csvColumns;
+        };
+    }
+}
+const ParseManager = new ParseManagerClass();
+exports.default = ParseManager;

package/engines/parsing/XMLParser.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fast_xml_parser_1 = require("fast-xml-parser");
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const DEFAULT_OPTIONS = {
+    attributeNamePrefix: '@_',
+    ignoreAttributes: false,
+    parseAttributeValue: true,
+    parseTagValue: true,
+    trimValues: true,
+    isArray: () => {
+        return false;
+    }
+};
+class XMLParserClass {
+    constructor(options) {
+        this.xmlToJson = (xmlData) => {
+            (0, Affirm_1.default)(xmlData, 'XML data cannot be empty');
+            try {
+                const parsedData = this._parser.parse(xmlData);
+                if (typeof parsedData === 'object' && parsedData !== null) {
+                    const rootKeys = Object.keys(parsedData);
+                    if (rootKeys.length === 1) {
+                        const potentialArray = parsedData[rootKeys[0]];
+                        if (Array.isArray(potentialArray)) {
+                            return potentialArray;
+                        }
+                        if (typeof potentialArray === 'object' && potentialArray !== null) {
+                            const innerKeys = Object.keys(potentialArray);
+                            if (innerKeys.length === 1 && Array.isArray(potentialArray[innerKeys[0]])) {
+                                return potentialArray[innerKeys[0]];
+                            }
+                            if (Array.isArray(potentialArray)) {
+                                return potentialArray;
+                            }
+                            const values = Object.values(potentialArray).filter(Array.isArray);
+                            if (values.length === 1 && Array.isArray(values[0])) {
+                                return values[0];
+                            }
+                            return [potentialArray];
+                        }
+                        return [potentialArray];
+                    }
+                }
+                return parsedData;
+            }
+            catch (error) {
+                console.error('Error parsing XML:', error);
+                throw new Error('Failed to parse XML data.');
+            }
+        };
+        this._parser = new fast_xml_parser_1.XMLParser(Object.assign(Object.assign({}, DEFAULT_OPTIONS), options)); // Use 'as any' if type issues persist with library
+    }
+}
+const XMLParser = new XMLParserClass();
+exports.default = XMLParser;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.28",
+    "version": "0.0.30",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -43,6 +43,7 @@
         "commander": "^10.0.0",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.3",
+        "fast-xml-parser": "^5.2.3",
         "fs-extra": "^11.1.0",
         "inquirer": "^8.2.5",
         "json-schema": "^0.4.0",
@@ -54,6 +55,7 @@
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
+        "seedrandom": "^3.0.5",
         "xlsx": "^0.18.5",
         "zod": "^3.24.2"
     },