@forzalabs/remora 0.0.29 → 0.0.30

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.29',
+    cliVersion: '0.0.30',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/definitions/json_schemas/producer-schema.json

@@ -137,7 +137,8 @@
                         "CSV", 
                         "TXT",
                         "XLS",
-                        "XLSX"
+                        "XLSX",
+                        "XML"
                     ],
                     "description": "The type of file to read"
                 },

package/drivers/LocalDriver.js

@@ -59,6 +59,7 @@ const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -85,6 +86,8 @@ class LocalSourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(fileUrl, options === null || options === void 0 ? void 0 : options.sheetName);
+                case 'XML':
+                    return yield this._readXmlLines(fileUrl);
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -106,6 +109,8 @@ class LocalSourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
+                case 'XML':
+                    return yield this._readXmlLines(fileUrl, lineFrom, lineTo);
             }
         });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
@@ -169,6 +174,17 @@ class LocalSourceDriver {
             else
                 return lines;
         });
+        this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = fs.readFileSync(fileUri, 'utf-8');
+            const jsonData = XMLParser_1.default.xmlToJson(fileContent);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            // Assuming jsonData is an array of objects, where each object is a record.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
     }
 }
 exports.LocalSourceDriver = LocalSourceDriver;

package/drivers/S3Driver.js

@@ -26,6 +26,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const readline_1 = __importDefault(require("readline"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -149,6 +150,8 @@ class S3SourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(stream, options === null || options === void 0 ? void 0 : options.sheetName);
+                case 'XML':
+                    return yield this._readXmlLines(stream);
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -172,6 +175,8 @@ class S3SourceDriver {
                 case 'XLS':
                 case 'XLSX':
                     return yield this._readExcelLines(stream, sheetName, lineFrom, lineTo);
+                case 'XML':
+                    return yield this._readXmlLines(stream, lineFrom, lineTo);
             }
         });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
@@ -255,6 +260,34 @@ class S3SourceDriver {
             else
                 return lines;
         });
+        this._readXmlLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, stream_2, stream_2_1;
+            var _b, e_3, _c, _d;
+            const chunks = [];
+            try {
+                for (_a = true, stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), _b = stream_2_1.done, !_b; _a = true) {
+                    _d = stream_2_1.value;
+                    _a = false;
+                    const chunk = _d;
+                    chunks.push(chunk);
+                }
+            }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            finally {
+                try {
+                    if (!_a && !_b && (_c = stream_2.return)) yield _c.call(stream_2);
+                }
+                finally { if (e_3) throw e_3.error; }
+            }
+            const buffer = Buffer.concat(chunks);
+            const jsonData = XMLParser_1.default.xmlToJson(buffer);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
     }
 }
 exports.S3SourceDriver = S3SourceDriver;

package/engines/ProducerEngine.js

@@ -19,7 +19,7 @@ const Environment_1 = __importDefault(require("./Environment"));
 const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
 const SQLCompiler_1 = __importDefault(require("./sql/SQLCompiler"));
 const SQLUtils_1 = __importDefault(require("./sql/SQLUtils"));
-const ParseManager_1 = __importDefault(require("./ParseManager"));
+const ParseManager_1 = __importDefault(require("./parsing/ParseManager"));
 class ProducerEngineClass {
     constructor() {
         this.compile = (producer) => {
@@ -107,9 +107,27 @@ class ProducerEngineClass {
                 case 'JSONL':
                 case 'JSON': {
                     if (lines.length === 1) {
-                        lines = lines[0].split('\n');
+                        // Attempt to handle cases where a single line might contain multiple JSON objects separated by newlines
+                        // Or if the entire file content is a single JSON array stringified.
+                        try {
+                            const parsedAsArray = JSON.parse(lines[0]);
+                            if (Array.isArray(parsedAsArray)) {
+                                return { data: parsedAsArray, dataType: 'array-of-json' };
+                            }
+                        }
+                        catch (error) {
+                            // If parsing as array fails, proceed to split by newline
+                            console.warn('Failed to parse single line as JSON array, splitting by newline:', error);
+                        }
+                        lines = lines[0].split('\\n');
                     }
-                    const json = lines.map(x => JSON.parse(x));
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
+                    return { data: json, dataType: 'array-of-json' };
+                }
+                case 'XML': {
+                    // The driver's _readXmlLines method now returns an array of JSON strings.
+                    // Each string needs to be parsed into a JSON object.
+                    const json = lines.filter(line => line.trim() !== '').map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }
                 default:
@@ -155,6 +173,19 @@ class ProducerEngineClass {
                             sampleData = sampleData.slice(0, sampleSize);
                             break;
                         }
+                        case 'XML': {
+                            // fileData.data from readFile for XML is now an array of JSON objects
+                            if (fileData.dataType === 'array-of-json' && Array.isArray(fileData.data)) {
+                                sampleData = fileData.data;
+                            }
+                            else {
+                                // Fallback or error handling if data is not in expected format
+                                console.warn('Unexpected data format for XML in readSampleData');
+                                sampleData = [];
+                            }
+                            sampleData = sampleData.slice(0, sampleSize);
+                            break;
+                        }
                         default: {
                             sampleData = fileData.data;
                             break;

package/engines/execution/ExecutionEnvironment.js

@@ -17,7 +17,7 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
 const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const ParseManager_1 = __importDefault(require("../ParseManager"));
+const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const ProducerEngine_1 = __importDefault(require("../ProducerEngine"));
 const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));

package/engines/parsing/ParseManager.js

@@ -0,0 +1,75 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+class ParseManagerClass {
+    constructor() {
+        this.csvToJson = (csv, producer) => {
+            (0, Affirm_1.default)(csv, 'Invalid csv content');
+            Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
+            return this.csvLinesToJson(csv, producer);
+        };
+        this.csvLinesToJson = (lines, producer, discover) => {
+            var _a;
+            (0, Affirm_1.default)(lines, 'Invalid csv lines');
+            Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
+            const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
+            const { header, records } = this._getClassifiedRows(lines, delimiterChar, producer);
+            const headerColumns = this._extractHeader(header, delimiterChar, producer, discover);
+            const rows = records.map(x => x.split(delimiterChar).map(k => k.trim()));
+            const result = [];
+            for (const row of rows) {
+                const rowObject = {};
+                for (let i = 0; i < headerColumns.length; i++) {
+                    const column = headerColumns[i];
+                    rowObject[column.saveAs] = row[column.index];
+                }
+                result.push(rowObject);
+            }
+            return result;
+        };
+        this._getClassifiedRows = (lines, delimiterChar, producer) => {
+            if (producer.settings.fileType === 'TXT' && !producer.settings.hasHeaderRow) {
+                // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
+                const source = Environment_1.default.getSource(producer.source);
+                const columns = FileCompiler_1.default.compileProducer(producer, source);
+                return {
+                    header: columns.map(x => x.nameInProducer).join(delimiterChar),
+                    records: lines
+                };
+            }
+            else {
+                return {
+                    header: lines[0],
+                    records: lines.slice(1)
+                };
+            }
+        };
+        this._extractHeader = (headerLine, delimiter, producer, discover) => {
+            var _a;
+            (0, Affirm_1.default)(headerLine, `Invalid CSV header line for producer "${producer.name}"`);
+            (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            const columns = FileCompiler_1.default.compileProducer(producer, source);
+            const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
+            if (discover)
+                columns.push(...headerColumns.map(x => ({ nameInProducer: x })));
+            const csvColumns = [];
+            for (const pColumn of columns) {
+                const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
+                const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
+                (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+                csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
+            }
+            return csvColumns;
+        };
+    }
+}
+const ParseManager = new ParseManagerClass();
+exports.default = ParseManager;

package/engines/parsing/XMLParser.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fast_xml_parser_1 = require("fast-xml-parser");
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const DEFAULT_OPTIONS = {
+    attributeNamePrefix: '@_',
+    ignoreAttributes: false,
+    parseAttributeValue: true,
+    parseTagValue: true,
+    trimValues: true,
+    isArray: () => {
+        return false;
+    }
+};
+class XMLParserClass {
+    constructor(options) {
+        this.xmlToJson = (xmlData) => {
+            (0, Affirm_1.default)(xmlData, 'XML data cannot be empty');
+            try {
+                const parsedData = this._parser.parse(xmlData);
+                if (typeof parsedData === 'object' && parsedData !== null) {
+                    const rootKeys = Object.keys(parsedData);
+                    if (rootKeys.length === 1) {
+                        const potentialArray = parsedData[rootKeys[0]];
+                        if (Array.isArray(potentialArray)) {
+                            return potentialArray;
+                        }
+                        if (typeof potentialArray === 'object' && potentialArray !== null) {
+                            const innerKeys = Object.keys(potentialArray);
+                            if (innerKeys.length === 1 && Array.isArray(potentialArray[innerKeys[0]])) {
+                                return potentialArray[innerKeys[0]];
+                            }
+                            if (Array.isArray(potentialArray)) {
+                                return potentialArray;
+                            }
+                            const values = Object.values(potentialArray).filter(Array.isArray);
+                            if (values.length === 1 && Array.isArray(values[0])) {
+                                return values[0];
+                            }
+                            return [potentialArray];
+                        }
+                        return [potentialArray];
+                    }
+                }
+                return parsedData;
+            }
+            catch (error) {
+                console.error('Error parsing XML:', error);
+                throw new Error('Failed to parse XML data.');
+            }
+        };
+        this._parser = new fast_xml_parser_1.XMLParser(Object.assign(Object.assign({}, DEFAULT_OPTIONS), options)); // Use 'as any' if type issues persist with library
+    }
+}
+const XMLParser = new XMLParserClass();
+exports.default = XMLParser;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.29",
+    "version": "0.0.30",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -43,6 +43,7 @@
         "commander": "^10.0.0",
         "cross-env": "^7.0.3",
         "dotenv": "^16.0.3",
+        "fast-xml-parser": "^5.2.3",
         "fs-extra": "^11.1.0",
         "inquirer": "^8.2.5",
         "json-schema": "^0.4.0",