@forzalabs/remora 0.0.47-nasco.3 → 0.0.49-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.47-nasco',
+    cliVersion: '0.0.49-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/DriverHelper.js

@@ -0,0 +1,88 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const promises_1 = require("stream/promises");
+const fs_1 = require("fs");
+const Logger_1 = __importDefault(require("../helper/Logger"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const DriverHelper = {
+    appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
+        (0, Affirm_1.default)(options, 'Invalid options');
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow } = options;
+        const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
+        let isFirstLine = true;
+        let hasValidatedHeader = shouldValidateHeader ? false : true;
+        const headerValidationTransform = new stream_1.Transform({
+            transform(chunk, encoding, callback) {
+                if (!hasValidatedHeader) {
+                    const chunkStr = chunk.toString();
+                    const lines = chunkStr.split('\n');
+                    if (isFirstLine && lines.length > 0) {
+                        const firstLine = lines[0];
+                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                            Logger_1.default.log(msg);
+                            return callback(new Error(msg));
+                        }
+                        hasValidatedHeader = true;
+                        isFirstLine = false;
+                    }
+                }
+                callback(null, chunk);
+            }
+        });
+        const writeOptions = append ? { flags: 'a' } : {};
+        const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);
+        yield (0, promises_1.pipeline)(stream, headerValidationTransform, writeStream);
+    }),
+    quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const fileStream = (0, fs_1.createReadStream)(filePath);
+        const rl = (0, readline_1.createInterface)({ input: fileStream, crlfDelay: Infinity });
+        const lines = [];
+        let counter = 0;
+        try {
+            for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                _c = rl_1_1.value;
+                _d = false;
+                const line = _c;
+                lines.push(line);
+                counter++;
+                if (counter >= lineCount) {
+                    break;
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        rl.close();
+        fileStream.close();
+        return lines;
+    })
+};
+exports.default = DriverHelper;

package/drivers/LocalDriver.js

@@ -64,6 +64,7 @@ const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -82,12 +83,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -101,12 +105,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
@@ -116,31 +123,36 @@ class LocalSourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const copyLocally = (fileKey_1, ...args_1) => __awaiter(this, [fileKey_1, ...args_1], void 0, function* (fileKey, appendMode = false) {
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false) {
                 const sourceFilePath = path_1.default.join(this._path, fileKey);
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-                const writeOptions = appendMode ? { flags: 'a' } : {};
+                // Copy and validate header in a single stream pass
                 const readStream = fs.createReadStream(sourceFilePath);
-                const writeStream = fs.createWriteStream(dataset.getPath(), writeOptions);
-                return new Promise((resolve, reject) => {
-                    readStream.pipe(writeStream);
-                    writeStream.on('finish', resolve);
-                    writeStream.on('error', reject);
-                    readStream.on('error', reject);
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream: readStream,
+                    fileKey,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
                 });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
+                // Get header line from the first file
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield copyLocally(allFileKeys[i], i > 0); // Append mode for subsequent files
+                    yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
                 return dataset;
             }
             else {
-                yield copyLocally(fileKey);
+                // For single file, no header validation needed
+                yield copyLocally(fileKey, '', false);
                 return dataset;
             }
         });
@@ -224,7 +236,7 @@ class LocalSourceDriver {
             }
             return lines;
         });
-        this._get = (request, index) => __awaiter(this, void 0, void 0, function* () {
+        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
             const { fileKey, fileType, options } = request;
             let lineFrom, lineTo, sheetName, hasHeaderRow;
             if (options) {
@@ -259,6 +271,12 @@ class LocalSourceDriver {
                         lines = yield this._readXmlLines(fileUrl);
                     break;
             }
+            const firstLine = lines[0];
+            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                Logger_1.default.log(msg);
+                throw new Error(msg);
+            }
             // If this is not the first file read in a pattern match AND the file type has an header,
             // then I need to remove the header from the resulting lines or the header will be duplicated
             if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {

package/drivers/S3Driver.js

@@ -29,9 +29,9 @@ const xlsx_1 = __importDefault(require("xlsx"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
-const promises_1 = require("stream/promises");
-const fs_1 = require("fs");
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
+const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -227,7 +227,8 @@ class S3SourceDriver {
             (0, Affirm_1.default)(file, 'Invalid dataset file');
             (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
             (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const downloadLocally = (fileUrl_1, ...args_1) => __awaiter(this, [fileUrl_1, ...args_1], void 0, function* (fileUrl, appendMode = false) {
+            const downloadLocally = (fileUrl_1, headerLine_1, ...args_1) => __awaiter(this, [fileUrl_1, headerLine_1, ...args_1], void 0, function* (fileUrl, headerLine, appendMode = false) {
+                // Download and validate header in a single stream pass
                 const command = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,
                     Key: fileUrl
@@ -235,23 +236,64 @@ class S3SourceDriver {
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
                 const stream = response.Body;
-                const writeOptions = appendMode ? { flags: 'a' } : {};
-                yield (0, promises_1.pipeline)(stream, (0, fs_1.createWriteStream)(dataset.getPath(), writeOptions));
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream,
+                    fileKey: fileUrl,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
+                });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
+                // Get header line from the first file
+                const firstFileCommand = new client_s3_1.GetObjectCommand({
+                    Bucket: this._bucketName,
+                    Key: allFileKeys[0]
+                });
+                const firstFileResponse = yield this._client.send(firstFileCommand);
+                (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
+                const firstFileStream = firstFileResponse.Body;
+                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield downloadLocally(allFileKeys[i], i > 0); // Append mode for subsequent files
+                    yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
                 return dataset;
             }
             else {
-                yield downloadLocally(fileKey);
+                // For single file, no header validation needed
+                yield downloadLocally(fileKey, '');
                 return dataset;
             }
         });
+        this.getFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            let firstLine = '';
+            try {
+                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                    _c = rl_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    firstLine = line;
+                    break;
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            rl.close();
+            return firstLine;
+        });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
@@ -276,7 +318,7 @@ class S3SourceDriver {
             }
         });
         this._readLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
+            var _a, e_2, _b, _c;
             const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
             const lines = [];
             let lineCounter = 0;
@@ -300,19 +342,19 @@ class S3SourceDriver {
                     }
                 }
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
             finally {
                 try {
                     if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
                 }
-                finally { if (e_1) throw e_1.error; }
+                finally { if (e_2) throw e_2.error; }
             }
             reader.close();
             return lines;
         });
         this._readExcelLines = (stream, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
             var _a, stream_1, stream_1_1;
-            var _b, e_2, _c, _d;
+            var _b, e_3, _c, _d;
             (0, Affirm_1.default)(sheetName, `Invalid sheetname`);
             const chunks = [];
             try {
@@ -323,12 +365,12 @@ class S3SourceDriver {
                     chunks.push(chunk);
                 }
             }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
             finally {
                 try {
                     if (!_a && !_b && (_c = stream_1.return)) yield _c.call(stream_1);
                 }
-                finally { if (e_2) throw e_2.error; }
+                finally { if (e_3) throw e_3.error; }
             }
             const buffer = Buffer.concat(chunks);
             const excel = xlsx_1.default.read(buffer, { type: 'buffer' });
@@ -343,7 +385,7 @@ class S3SourceDriver {
         });
         this._readXmlLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
             var _a, stream_2, stream_2_1;
-            var _b, e_3, _c, _d;
+            var _b, e_4, _c, _d;
             const chunks = [];
             try {
                 for (_a = true, stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), _b = stream_2_1.done, !_b; _a = true) {
@@ -353,12 +395,12 @@ class S3SourceDriver {
                     chunks.push(chunk);
                 }
             }
-            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            catch (e_4_1) { e_4 = { error: e_4_1 }; }
             finally {
                 try {
                     if (!_a && !_b && (_c = stream_2.return)) yield _c.call(stream_2);
                 }
-                finally { if (e_3) throw e_3.error; }
+                finally { if (e_4) throw e_4.error; }
             }
             const buffer = Buffer.concat(chunks);
             const jsonData = XMLParser_1.default.xmlToJson(buffer);

package/engines/execution/ExecutionPlanner.js

@@ -85,6 +85,8 @@ class ExecutionPlannerClas {
                 plan.push(...this._planProducer(producers[0], options));
             else
                 plan.push(...(producers.flatMap(x => this._planProducer(x, options))));
+            // I technically don't need this, but I keep it to merge all the datasets to a single one
+            // so the other steps of the plan can work with a single dataset variable
             plan.push({ type: 'join-producers-data' });
             if (consumer.filters && consumer.filters.length > 0)
                 plan.push({ type: 'apply-consumer-filters-on-JSON' });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.47-nasco.3",
+    "version": "0.0.49-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,