@forzalabs/remora 0.2.6 → 1.0.0

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.2.6',
+    cliVersion: '1.0.0',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,
@@ -19,7 +19,15 @@ const CONSTANTS = {
         MIN_RUNTIME_HEAP_MB: 4000,
         RECOMMENDED_RUNTIME_HEAP_MB: 8000,
         INDICATIVE_THREAD_LINE_COUNT: 750000,
-        MAX_THREAD_COUNT: 8
+        MAX_THREAD_COUNT: 8,
+        /**
+         * Minimum file size to consider parallel processing (10 MB)
+         */
+        MIN_FILE_SIZE_FOR_PARALLEL: 10 * 1024 * 1024,
+        /**
+         * Minimum chunk size per worker to justify overhead (2 MB)
+         */
+        MIN_CHUNK_SIZE: 2 * 1024 * 1024
     }
 };
 exports.default = CONSTANTS;

package/actions/debug.js

@@ -19,6 +19,7 @@ const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
 const compile_1 = require("./compile");
 const debug = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    void options;
     try {
         (0, compile_1.compile)();
         console.log('\n');

package/actions/deploy.js

@@ -54,6 +54,7 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
                 const version = Constants_1.default.workerVersion;
                 const workerAPI = `${host}/cli/v${version}/uploaddeployment`;
                 const formData = new FormData();
+                // @ts-ignore
                 const blob = new Blob([zipBuffer], { type: 'application/zip' });
                 formData.append('remora_config', blob, 'temp_deployment.zip'); // Updated to match the actual file name
                 const apiKey = process.env.REMORA_LICENCE_KEY;

package/actions/run.js

@@ -14,17 +14,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.run = void 0;
 const chalk_1 = __importDefault(require("chalk"));
-const ora_1 = __importDefault(require("ora"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
-const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEngine"));
 const compile_1 = require("./compile");
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
+const ExecutorOrchestrator_1 = __importDefault(require("../executors/ExecutorOrchestrator"));
 const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
-        console.log(); // needed for newline
-        const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
+        console.log(chalk_1.default.blue('Running consumer(s)...\n')); // needed for newline
         const consumersToExecute = [];
         if (consumerName && consumerName.length > 0) {
             const cons = Environment_1.default.getConsumer(consumerName);
@@ -52,7 +50,14 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
                     console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
                     process.exit(1);
                 }
-                const response = yield ConsumerEngine_1.default.execute(consumer, {}, { _id: check.customer, name: check.customer, type: 'licence' }, { invokedBy: 'CLI' });
+                const response = yield ExecutorOrchestrator_1.default.launch({
+                    consumer,
+                    details: {
+                        invokedBy: 'CLI',
+                        user: { _id: check.customer, name: check.customer, type: 'licence' }
+                    },
+                    logProgress: true
+                });
                 results.push({ success: true, consumer, response });
             }
             catch (error) {
@@ -62,15 +67,14 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
                     console.log(myErr.stack);
             }
         }
-        spinner.succeed('All consumers have been executed:');
         results.forEach(({ response, consumer, success, error }) => {
             if (success) {
-                const { _stats: { size, elapsedMS } } = response;
-                const rowCount = size;
+                const { elapsedMS, outputCount, resultUri } = response;
+                const rowCount = outputCount;
                 const duration = Helper_1.default.formatDuration(elapsedMS);
                 const performanceInfo = chalk_1.default.gray(` (${rowCount} rows, ${duration})`);
-                if (response.fileUri)
-                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.fileUri}`) + performanceInfo);
+                if (resultUri)
+                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${resultUri}`) + performanceInfo);
                 else
                     console.log(chalk_1.default.green(`• Consumer ${consumer.name} `) + performanceInfo);
             }
@@ -82,11 +86,11 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
         const successfulResults = results.filter(x => x.success);
         const totalRows = successfulResults.reduce((sum, result) => {
             var _a, _b;
-            return sum + ((_b = (_a = result.response) === null || _a === void 0 ? void 0 : _a._stats.size) !== null && _b !== void 0 ? _b : 0);
+            return sum + ((_b = (_a = result.response) === null || _a === void 0 ? void 0 : _a.outputCount) !== null && _b !== void 0 ? _b : 0);
         }, 0);
         const totalDuration = successfulResults.reduce((sum, result) => {
-            var _a, _b;
-            return sum + (((_b = (_a = result.response) === null || _a === void 0 ? void 0 : _a._stats) === null || _b === void 0 ? void 0 : _b.elapsedMS) || 0);
+            var _a;
+            return sum + (((_a = result.response) === null || _a === void 0 ? void 0 : _a.elapsedMS) || 0);
         }, 0);
         const totalsInfo = chalk_1.default.gray(` (${totalRows} rows, ${Helper_1.default.formatDuration(totalDuration)})`);
         if (results.some(x => !x.success))

package/actions/sample.js

@@ -93,7 +93,7 @@ const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, v
     const mappedData = rawSampleData.map(record => {
         const mappedRecord = new DatasetRecord_1.default('', [], record._delimiter);
         consumer.fields.forEach(field => {
-            if (field.key !== '*' && !field.grouping) {
+            if (field.key !== '*') {
                 const sourceValue = record.getValue(field.key);
                 const outputKey = field.alias || field.key;
                 mappedRecord.setValue(outputKey, sourceValue);

package/core/Algo.js

@@ -122,13 +122,15 @@ const algo = {
     },
     mean: (numbers) => {
         (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
-        (0, Affirm_1.default)(numbers.length > 0, 'Array must be non-empty');
+        if (numbers.length === 0)
+            return 0;
         const total = algo.sum(numbers);
         return total / numbers.length;
     },
     sum: (numbers) => {
         (0, Affirm_1.default)(algo.hasVal(numbers), 'Array must not be null or undefined');
-        (0, Affirm_1.default)(numbers.length > 0, 'Array must be non-empty');
+        if (numbers.length === 0)
+            return 0;
         let total = 0;
         for (let i = 0; i < numbers.length; i++) {
             total += numbers[i];
@@ -142,12 +144,14 @@ const algo = {
     },
     min: (arr) => {
         (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
-        (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
+        if (arr.length === 0)
+            return 0;
         return Math.min(...arr);
     },
     max: (arr) => {
         (0, Affirm_1.default)(algo.hasVal(arr), 'Array must not be null or undefined');
-        (0, Affirm_1.default)(arr.length > 0, 'Array must be non-empty');
+        if (arr.length === 0)
+            return 0;
         return Math.max(...arr);
     },
     replaceAll: (text, search, replace) => text.replace(new RegExp(search, 'g'), replace),

package/definitions/ExecutorDefinitions.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/json_schemas/consumer-schema.json

@@ -845,7 +845,7 @@
                 },
                 {
                     "type": "object",
-                    "description": "Apply conditional logic to transform values based on comparison conditions",
+                    "description": "Apply conditional logic to transform values based on comparison conditions.",
                     "properties": {
                         "conditional": {
                             "type": "object",

package/definitions/json_schemas/producer-schema.json

@@ -148,7 +148,7 @@
                         "XML",
                         "PARQUET"
                     ],
-                    "description": "The type of file to read"
+                    "description": "The type of file to read."
                 },
                 "delimiter": {
                     "type": "string",

package/definitions/temp.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/drivers/DeltaShareDriver.js

@@ -177,6 +177,10 @@ class DeltaShareSourceDriver {
                 .map(x => JSON.parse(x));
             return deltaLines;
         });
+        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+            void producer;
+            throw new Error('DeltaShareSourceDriver.ready is not supported: Delta Sharing does not support readiness checks');
+        });
     }
 }
 exports.default = DeltaShareSourceDriver;

package/drivers/DriverFactory.js

@@ -12,23 +12,24 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const LocalDriver_1 = require("./LocalDriver");
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
-const S3Driver_1 = require("./S3Driver");
 const DeltaShareDriver_1 = __importDefault(require("./DeltaShareDriver"));
-const HttpApiDriver_1 = require("./HttpApiDriver");
+const HttpApiDriver_1 = __importDefault(require("./HttpApiDriver"));
+const LocalSourceDriver_1 = __importDefault(require("./files/LocalSourceDriver"));
+const LocalDestinationDriver_1 = __importDefault(require("./files/LocalDestinationDriver"));
+const S3SourceDriver_1 = __importDefault(require("./s3/S3SourceDriver"));
+const S3DestinationDriver_1 = __importDefault(require("./s3/S3DestinationDriver"));
 class DriverFactoryClass {
     constructor() {
         this.instantiateSource = (source) => __awaiter(this, void 0, void 0, function* () {
             switch (source.engine) {
-                // TODO: implement all the other engines
                 case 'aws-redshift': {
                     const driver = new RedshiftDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'aws-s3': {
-                    const driver = new S3Driver_1.S3SourceDriver();
+                    const driver = new S3SourceDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }
@@ -38,12 +39,12 @@ class DriverFactoryClass {
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.LocalSourceDriver();
+                    const driver = new LocalSourceDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'http-api': {
-                    const driver = new HttpApiDriver_1.HttpApiSourceDriver();
+                    const driver = new HttpApiDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }
@@ -52,14 +53,13 @@ class DriverFactoryClass {
         });
         this.instantiateDestination = (source) => __awaiter(this, void 0, void 0, function* () {
             switch (source.engine) {
-                // TODO: implement all the other engines
                 case 'aws-s3': {
-                    const driver = new S3Driver_1.S3DestinationDriver();
+                    const driver = new S3DestinationDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }
                 case 'local': {
-                    const driver = new LocalDriver_1.LocalDestinationDriver();
+                    const driver = new LocalDestinationDriver_1.default();
                     yield driver.init(source);
                     return driver;
                 }

package/drivers/DriverHelper.js

@@ -23,12 +23,12 @@ const stream_1 = require("stream");
 const readline_1 = require("readline");
 const promises_1 = require("stream/promises");
 const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
-const path_1 = __importDefault(require("path"));
-const Constants_1 = __importDefault(require("../Constants"));
 const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser"));
+const Constants_1 = __importDefault(require("../Constants"));
 const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
@@ -37,7 +37,7 @@ const DriverHelper = {
         const keys = (fileType === 'JSON' || fileType === 'JSONL')
             ? Object.keys(JSON.parse(headerLine))
             : [];
-        const shouldValidateHeader = fileType === 'CSV' || fileType === 'XLS' || (fileType === 'TXT' && hasHeaderRow === true);
+        const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         // When sourceFilename is set, the headerLine includes $source_filename at the end.
         // For validation, we need to compare against the original header without this suffix.
         const originalHeaderLine = sourceFilename
@@ -154,14 +154,9 @@ const DriverHelper = {
         return lineCount;
     }),
     quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
-        const fileStream = (0, fs_1.createReadStream)(filePath);
-        const lines = yield DriverHelper.quickReadStream(fileStream, lineCount);
-        fileStream.close();
-        return lines;
-    }),
-    quickReadStream: (stream, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
         var _a, e_1, _b, _c;
-        const rl = (0, readline_1.createInterface)({ input: stream, crlfDelay: Infinity });
+        const fileStream = (0, fs_1.createReadStream)(filePath);
+        const rl = (0, readline_1.createInterface)({ input: fileStream, crlfDelay: Infinity });
         const lines = [];
         let counter = 0;
         try {
@@ -184,6 +179,7 @@ const DriverHelper = {
             finally { if (e_1) throw e_1.error; }
         }
         rl.close();
+        fileStream.close();
         return lines;
     }),
     setHeaderFromFile: (fileKey, file, filePath, dataset) => __awaiter(void 0, void 0, void 0, function* () {
@@ -220,6 +216,33 @@ const DriverHelper = {
             default:
                 throw new Error(`the fileType "${file.fileType}" is not implemented yet`);
         }
+    }),
+    quickReadStream: (stream, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, e_2, _b, _c;
+        const rl = (0, readline_1.createInterface)({ input: stream, crlfDelay: Infinity });
+        const lines = [];
+        let counter = 0;
+        try {
+            for (var _d = true, rl_2 = __asyncValues(rl), rl_2_1; rl_2_1 = yield rl_2.next(), _a = rl_2_1.done, !_a; _d = true) {
+                _c = rl_2_1.value;
+                _d = false;
+                const line = _c;
+                lines.push(line);
+                counter++;
+                if (counter >= lineCount) {
+                    break;
+                }
+            }
+        }
+        catch (e_2_1) { e_2 = { error: e_2_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = rl_2.return)) yield _b.call(rl_2);
+            }
+            finally { if (e_2) throw e_2.error; }
+        }
+        rl.close();
+        return lines;
     })
 };
 exports.default = DriverHelper;

package/drivers/HttpApiDriver.js

@@ -198,6 +198,10 @@ class HttpApiSourceDriver {
             }
             return itemsData;
         };
+        this.ready = (producer) => {
+            void producer;
+            throw new Error('Not implemented yet');
+        };
     }
 }
 exports.HttpApiSourceDriver = HttpApiSourceDriver;

package/drivers/LocalDriver.js

@@ -59,13 +59,14 @@ const readline_1 = __importDefault(require("readline"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const xlsx_1 = __importDefault(require("xlsx"));
-const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
+const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser"));
 const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
 const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
+const stream_1 = require("stream");
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -147,8 +148,7 @@ class LocalSourceDriver {
                     sourceFilename
                 });
             });
-            // this function copy the local file on the temporary file and retrive the number of line.
-            const handleFileAndGetLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
+            const getTotalLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
                 let totalLineCount;
                 let streamXLS;
                 switch (fileType) {
@@ -158,7 +158,7 @@ class LocalSourceDriver {
                         totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename, streamXLS);
                         break;
                     default:
-                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename);
+                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), false, sourceFilename);
                         break;
                 }
                 return totalLineCount;
@@ -177,7 +177,7 @@ class LocalSourceDriver {
                     const currentFileKey = allFileKeys[i];
                     // Pass the filename (just the basename) if includeSourceFilename is enabled
                     const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
-                    totalLineCount += yield handleFileAndGetLineCount(currentFileKey, i > 0, file.fileType, sourceFilename); // Append mode for subsequent files
+                    totalLineCount += yield getTotalLineCount(currentFileKey, true, file.fileType, sourceFilename); // Append mode for subsequent files
                 }
                 dataset.setCount(totalLineCount);
                 return dataset;
@@ -185,7 +185,7 @@ class LocalSourceDriver {
             else {
                 sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
                 yield DriverHelper_1.default.setHeaderFromFile(fileKey, file, this._path, dataset);
-                totalLineCount = (yield handleFileAndGetLineCount(fileKey, false, file.fileType, sourceFilename));
+                totalLineCount = (yield getTotalLineCount(fileKey, false, file.fileType, sourceFilename));
                 dataset.setCount(totalLineCount);
                 return dataset;
             }
@@ -376,6 +376,24 @@ class LocalSourceDriver {
             }
             fs.renameSync(sourceFilePath, destinationFilePath);
         };
+        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const { fileKey } = producer.settings;
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                const allFilePaths = allFileKeys.map(x => path_1.default.join(this._path, x));
+                const readStreams = allFilePaths.map(x => fs.createReadStream(x));
+                let pass = new stream_1.PassThrough();
+                for (const [index, stream] of readStreams.entries())
+                    pass = stream.pipe(pass, { end: index === readStreams.length - 1 });
+                return pass;
+            }
+            else {
+                const sourceFilePath = path_1.default.join(this._path, fileKey);
+                const readStream = fs.createReadStream(sourceFilePath);
+                return readStream;
+            }
+        });
     }
 }
 exports.LocalSourceDriver = LocalSourceDriver;
@@ -470,6 +488,55 @@ class LocalDestinationDriver {
             const fileContent = yield s3Driver.downloadFile(sourceFileKey);
             yield this.saveFile(destinationFileKey, fileContent);
         });
+        this.ready = (destinationPath) => __awaiter(this, void 0, void 0, function* () {
+            return fs.createWriteStream(destinationPath);
+        });
+        this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const toFilePath = path_1.default.join(this._path, toName);
+                fs.renameSync(fromPath, toFilePath);
+                return { bucket: '', key: toFilePath, res: true };
+            }
+            catch (error) {
+                Logger_1.default.error(error);
+                return { bucket: '', key: '', res: false };
+            }
+        });
+        this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_2, _b, _c;
+            try {
+                const toFilePath = path_1.default.join(this._path, toName);
+                const decoder = new TextDecoder();
+                const reader = fs.createReadStream(fromPath);
+                const writer = fs.createWriteStream(toFilePath);
+                try {
+                    for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
+                        _c = reader_2_1.value;
+                        _d = false;
+                        const chunk = _c;
+                        const decoded = decoder.decode(chunk);
+                        const lines = decoded.split('\n');
+                        for (const line of lines) {
+                            writer.write(transform(line) + '\n');
+                        }
+                    }
+                }
+                catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                finally {
+                    try {
+                        if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
+                    }
+                    finally { if (e_2) throw e_2.error; }
+                }
+                writer.close();
+                reader.close();
+                return { bucket: '', key: toFilePath, res: true };
+            }
+            catch (error) {
+                Logger_1.default.error(error);
+                return { bucket: '', key: '', res: false };
+            }
+        });
     }
 }
 exports.LocalDestinationDriver = LocalDestinationDriver;

package/drivers/RedshiftDriver.js

@@ -175,6 +175,10 @@ class RedshiftDriver {
             }
             return records;
         };
+        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+            void producer;
+            throw new Error('Not implemented yet');
+        });
     }
 }
 exports.default = RedshiftDriver;

package/drivers/S3Driver.js

@@ -34,7 +34,6 @@ const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 const Logger_1 = __importDefault(require("../helper/Logger"));
 const Constants_1 = __importDefault(require("../Constants"));
-const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
 class S3DestinationDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -225,16 +224,7 @@ class S3SourceDriver {
                 });
                 const response = yield this._client.send(command);
                 (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
-                let stream;
-                switch (file.fileType) {
-                    case 'XLS':
-                    case 'XLSX':
-                        stream = yield XLSParser_1.default.parseXLSStream(response.Body, file.sheetName);
-                        break;
-                    default:
-                        stream = response.Body;
-                        break;
-                }
+                const stream = response.Body;
                 return DriverHelper_1.default.appendToUnifiedFile({
                     stream,
                     fileKey: fileUrl,
@@ -248,45 +238,6 @@ class S3SourceDriver {
                 });
             });
             const { fileKey } = file;
-            const setFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
-                var _a, e_1, _b, _c;
-                const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-                let firstLine = '';
-                switch (file.fileType) {
-                    case 'XLSX':
-                    case 'XLS':
-                        firstLine = yield XLSParser_1.default.getHeaderXlsFromStream(stream, file.sheetName);
-                        break;
-                    case 'CSV':
-                    case 'JSON':
-                    case 'JSONL':
-                    case 'TXT':
-                        try {
-                            for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
-                                _c = rl_1_1.value;
-                                _d = false;
-                                const line = _c;
-                                firstLine = line;
-                                break;
-                            }
-                        }
-                        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                        finally {
-                            try {
-                                if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
-                            }
-                            finally { if (e_1) throw e_1.error; }
-                        }
-                        rl.close();
-                        break;
-                }
-                // If including source filename, append a placeholder column name to the header
-                if (file.includeSourceFilename) {
-                    firstLine = firstLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
-                }
-                dataset.setFirstLine(firstLine);
-                return firstLine;
-            });
             if (fileKey.includes('%')) {
                 const allFileKeys = yield this.listFiles(fileKey);
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
@@ -299,7 +250,12 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield setFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
+                dataset.setFirstLine(headerLine);
                 let totalLineCount = 0;
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
@@ -320,7 +276,12 @@ class S3SourceDriver {
                 const firstFileResponse = yield this._client.send(firstFileCommand);
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
-                const headerLine = yield setFirstLineFromStream(firstFileStream);
+                let headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                // If including source filename, append a placeholder column name to the header
+                if (includeSourceFilename) {
+                    headerLine = headerLine + dataset.getDelimiter() + Constants_1.default.SOURCE_FILENAME_COLUMN;
+                }
+                dataset.setFirstLine(headerLine);
                 // Pass the filename if includeSourceFilename is enabled
                 const sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
                 const totalLineCount = yield downloadLocally(fileKey, headerLine, false, sourceFilename);
@@ -328,6 +289,29 @@ class S3SourceDriver {
                 return dataset;
             }
         });
+        this.getFirstLineFromStream = (stream) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const rl = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            let firstLine = '';
+            try {
+                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                    _c = rl_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    firstLine = line;
+                    break;
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            rl.close();
+            return firstLine;
+        });
         this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
             var _a;
             (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');