@forzalabs/remora 0.0.48-nasco.3 → 0.0.49-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.48-nasco',
+    cliVersion: '0.0.49-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/DriverHelper.js

@@ -24,9 +24,12 @@ const readline_1 = require("readline");
 const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
 const Logger_1 = __importDefault(require("../helper/Logger"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
 const DriverHelper = {
-    appendToUnifiedFile: (stream, fileKey, destinationPath, append, headerLine, fileType) => __awaiter(void 0, void 0, void 0, function* () {
-        const shouldValidateHeader = fileType === 'CSV' || fileType === 'TXT';
+    appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
+        (0, Affirm_1.default)(options, 'Invalid options');
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow } = options;
+        const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
         const headerValidationTransform = new stream_1.Transform({
@@ -36,7 +39,6 @@ const DriverHelper = {
                     const lines = chunkStr.split('\n');
                     if (isFirstLine && lines.length > 0) {
                         const firstLine = lines[0];
-                        // Validate header only for CSV and TXT files
                         if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
                             const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
                             Logger_1.default.log(msg);

package/drivers/LocalDriver.js

@@ -83,12 +83,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
@@ -102,12 +105,15 @@ class LocalSourceDriver {
             const { fileKey } = request;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), i));
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
                 const results = yield Promise.all(promises);
                 return results.flat();
             }
             else {
-                return yield this._get(request);
+                return yield this._get(request, '');
             }
         });
         this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
@@ -122,12 +128,20 @@ class LocalSourceDriver {
                 (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
                 // Copy and validate header in a single stream pass
                 const readStream = fs.createReadStream(sourceFilePath);
-                return DriverHelper_1.default.appendToUnifiedFile(readStream, fileKey, dataset.getPath(), appendMode, headerLine, file.fileType);
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream: readStream,
+                    fileKey,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
+                });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 // Get header line from the first file
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 // Copy files sequentially to avoid file conflicts
@@ -222,7 +236,7 @@ class LocalSourceDriver {
             }
             return lines;
         });
-        this._get = (request, index) => __awaiter(this, void 0, void 0, function* () {
+        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
             const { fileKey, fileType, options } = request;
             let lineFrom, lineTo, sheetName, hasHeaderRow;
             if (options) {
@@ -257,6 +271,12 @@ class LocalSourceDriver {
                         lines = yield this._readXmlLines(fileUrl);
                     break;
             }
+            const firstLine = lines[0];
+            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                Logger_1.default.log(msg);
+                throw new Error(msg);
+            }
             // If this is not the first file read in a pattern match AND the file type has an header,
             // then I need to remove the header from the resulting lines or the header will be duplicated
             if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {

package/drivers/S3Driver.js

@@ -31,6 +31,7 @@ const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -235,11 +236,20 @@ class S3SourceDriver {
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
                 const stream = response.Body;
-                return DriverHelper_1.default.appendToUnifiedFile(stream, fileUrl, dataset.getPath(), appendMode, headerLine, file.fileType);
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream,
+                    fileKey: fileUrl,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow
+                });
             });
             const { fileKey } = file;
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 // Get header line from the first file
                 const firstFileCommand = new client_s3_1.GetObjectCommand({
                     Bucket: this._bucketName,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.48-nasco.3",
+    "version": "0.0.49-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,