@forzalabs/remora 0.2.6 → 1.0.0

package/drivers/files/LocalDestinationDriver.js

@@ -0,0 +1,200 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const promises_1 = __importDefault(require("fs/promises"));
+const readline_1 = __importDefault(require("readline"));
+const path_1 = __importDefault(require("path"));
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const FileExporter_1 = __importDefault(require("../../engines/file/FileExporter"));
+const Logger_1 = __importDefault(require("../../helper/Logger"));
+class LocalDestinationDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, `Invalid source`);
+            const fileURL = source.authentication['path'];
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
+            const exist = fs.existsSync(fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
+            this._path = source.authentication['path'];
+            return this;
+        });
+        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(options, 'Invalid upload options');
+            (0, Affirm_1.default)(options.name, 'File name is required');
+            (0, Affirm_1.default)(options.content != null, 'File content is required');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder))
+                    fs.mkdirSync(folder, { recursive: true });
+                const filePath = path_1.default.join(folder, options.name);
+                fs.writeFileSync(filePath, options.content);
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
+            }
+        });
+        this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(options, `Invalid upload options`);
+            const { dataset, name, recordProjection } = options;
+            (0, Affirm_1.default)(dataset, 'No streaming dataset');
+            (0, Affirm_1.default)(name, 'No filename provided for upload stream');
+            (0, Affirm_1.default)(recordProjection, 'No recordProjection for upload stream');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder))
+                    fs.mkdirSync(folder, { recursive: true });
+                const filePath = path_1.default.join(folder, options.name);
+                fs.writeFileSync(filePath, '');
+                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
+                    for (const chunk of chunks)
+                        fs.appendFileSync(filePath, chunk);
+                }));
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                // Clean up the partial file if it exists
+                const filePath = path_1.default.join(folder, options.name);
+                if (fs.existsSync(filePath)) {
+                    try {
+                        fs.unlinkSync(filePath);
+                    }
+                    catch (cleanupError) {
+                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
+                        throw cleanupError;
+                    }
+                }
+                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
+            }
+        });
+        this.saveFile = (fileKey, content) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            const filePath = path_1.default.join(this._path, fileKey);
+            const directory = path_1.default.dirname(filePath);
+            // Create directory if it doesn't exist
+            if (!fs.existsSync(directory)) {
+                fs.mkdirSync(directory, { recursive: true });
+            }
+            fs.writeFileSync(filePath, content);
+            return Promise.resolve();
+        };
+        this.copyFromLocal = (sourceFilePath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFilePath, 'Invalid source file path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const destinationFilePath = path_1.default.join(this._path, destinationFileKey);
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            // Ensure destination directory exists
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.copyFileSync(sourceFilePath, destinationFilePath);
+        };
+        this.copyFromS3 = (s3Driver, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = yield s3Driver.downloadFile(sourceFileKey);
+            yield this.saveFile(destinationFileKey, fileContent);
+        });
+        this.ready = (destinationPath) => __awaiter(this, void 0, void 0, function* () {
+            return fs.createWriteStream(destinationPath);
+        });
+        this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const toFilePath = path_1.default.join(this._path, toName);
+                yield promises_1.default.rename(fromPath, toFilePath);
+                return { bucket: '', key: toFilePath, res: true };
+            }
+            catch (error) {
+                Logger_1.default.error(error);
+                return { bucket: '', key: '', res: false };
+            }
+        });
+        this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            try {
+                const toFilePath = path_1.default.join(this._path, toName);
+                const reader = fs.createReadStream(fromPath);
+                const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
+                const writer = fs.createWriteStream(toFilePath);
+                try {
+                    for (var _d = true, lineReader_1 = __asyncValues(lineReader), lineReader_1_1; lineReader_1_1 = yield lineReader_1.next(), _a = lineReader_1_1.done, !_a; _d = true) {
+                        _c = lineReader_1_1.value;
+                        _d = false;
+                        const line = _c;
+                        writer.write(transform(line) + '\n');
+                    }
+                }
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (!_d && !_a && (_b = lineReader_1.return)) yield _b.call(lineReader_1);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                }
+                writer.close();
+                reader.close();
+                return { bucket: '', key: toFilePath, res: true };
+            }
+            catch (error) {
+                Logger_1.default.error(error);
+                return { bucket: '', key: '', res: false };
+            }
+        });
+    }
+}
+exports.default = LocalDestinationDriver;

package/drivers/files/LocalSourceDriver.js

@@ -0,0 +1,394 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path_1 = __importDefault(require("path"));
+const readline_1 = __importDefault(require("readline"));
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const xlsx_1 = __importDefault(require("xlsx"));
+const XMLParser_1 = __importDefault(require("../../engines/parsing/XMLParser")); // Added XMLParser import
+const XLSParser_1 = __importDefault(require("../../engines/parsing/XLSParser"));
+const Helper_1 = __importDefault(require("../../helper/Helper"));
+const ParseHelper_1 = __importDefault(require("../../engines/parsing/ParseHelper"));
+const Logger_1 = __importDefault(require("../../helper/Logger"));
+const DriverHelper_1 = __importDefault(require("../DriverHelper"));
+class LocalSourceDriver {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            const fileURL = source.authentication['path'];
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
+            const exist = fs.existsSync(fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
+            this._path = source.authentication['path'];
+            return this;
+        });
+        this.readAll = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(request, `Invalid download request`);
+            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
+            (0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
+            const { fileKey } = request;
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
+                const results = yield Promise.all(promises);
+                return results.flat();
+            }
+            else {
+                return yield this._get(request, '');
+            }
+        });
+        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(request, 'Invalid read options');
+            (0, Affirm_1.default)(request.fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(request.fileType, `Invalid file type`);
+            (0, Affirm_1.default)(request.options, `Invalid request options`);
+            Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
+            Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
+            const { fileKey } = request;
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
+                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
+                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
+                const results = yield Promise.all(promises);
+                return results.flat();
+            }
+            else {
+                return yield this._get(request, '');
+            }
+        });
+        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(dataset, `Invalid dataset`);
+            const file = dataset.getFile();
+            (0, Affirm_1.default)(file, 'Invalid dataset file');
+            (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(file.fileType, `Invalid file type`);
+            const includeSourceFilename = file.includeSourceFilename === true;
+            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename, stream) {
+                const sourceFilePath = path_1.default.join(this._path, fileKey);
+                (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
+                // Copy and validate header in a single stream pass
+                const readStream = fs.createReadStream(sourceFilePath);
+                let streamToUse = readStream;
+                if (['XLS', 'XLSX'].includes(file.fileType))
+                    streamToUse = stream;
+                else
+                    streamToUse = readStream;
+                return DriverHelper_1.default.appendToUnifiedFile({
+                    stream: streamToUse,
+                    fileKey,
+                    destinationPath: dataset.getPath(),
+                    append: appendMode,
+                    headerLine,
+                    fileType: file.fileType,
+                    hasHeaderRow: file.hasHeaderRow,
+                    delimiter: dataset.getDelimiter(),
+                    sourceFilename
+                });
+            });
+            // this function copy the local file on the temporary file and retrive the number of line.
+            const handleFileAndGetLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
+                let totalLineCount;
+                let streamXLS;
+                switch (fileType) {
+                    case 'XLS':
+                    case 'XLSX':
+                        streamXLS = (yield XLSParser_1.default.getStreamXls(path_1.default.join(this._path, fileKey), file.sheetName));
+                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename, streamXLS);
+                        break;
+                    default:
+                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename);
+                        break;
+                }
+                return totalLineCount;
+            });
+            const { fileKey } = file;
+            let totalLineCount = 0;
+            let sourceFilename;
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                yield DriverHelper_1.default.setHeaderFromFile(allFileKeys[0], file, this._path, dataset);
+                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
+                Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
+                totalLineCount = 0;
+                // Copy files sequentially to avoid file conflicts
+                for (let i = 0; i < allFileKeys.length; i++) {
+                    const currentFileKey = allFileKeys[i];
+                    // Pass the filename (just the basename) if includeSourceFilename is enabled
+                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
+                    totalLineCount += yield handleFileAndGetLineCount(currentFileKey, i > 0, file.fileType, sourceFilename); // Append mode for subsequent files
+                }
+                dataset.setCount(totalLineCount);
+                return dataset;
+            }
+            else {
+                sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
+                yield DriverHelper_1.default.setHeaderFromFile(fileKey, file, this._path, dataset);
+                totalLineCount = (yield handleFileAndGetLineCount(fileKey, false, file.fileType, sourceFilename));
+                dataset.setCount(totalLineCount);
+                return dataset;
+            }
+        });
+        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, `Invalid path`);
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            const fileKey = producer.settings.fileKey;
+            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                return allFileKeys.length > 0;
+            }
+            else {
+                const fileUrl = path_1.default.join(this._path, fileKey);
+                return fs.existsSync(fileUrl);
+            }
+        });
+        this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const stream = fs.createReadStream(fileUri);
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            let lineCounter = 0;
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
+                            if (line && line.length > 0)
+                                lines.push(line);
+                        }
+                        lineCounter++;
+                        if (lineCounter >= lineTo)
+                            break;
+                    }
+                    else {
+                        if (line && line.length > 0)
+                            lines.push(line);
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            reader.close();
+            stream.close();
+            return lines;
+        });
+        this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const excel = xlsx_1.default.readFile(fileUri);
+            let targetSheetName = sheetName;
+            if (!targetSheetName) {
+                (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
+                targetSheetName = excel.SheetNames[0];
+            }
+            else {
+                (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
+            }
+            const sheet = excel.Sheets[targetSheetName];
+            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
+            const lines = csv.split('\n');
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                return lines.slice(lineFrom, lineTo + 1);
+            else
+                return lines;
+        });
+        this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = fs.readFileSync(fileUri, 'utf-8');
+            const jsonData = XMLParser_1.default.xmlToJson(fileContent);
+            // Convert JSON data to string lines. This might need adjustment based on XML structure.
+            // Assuming jsonData is an array of objects, where each object is a record.
+            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
+            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
+                lines = lines.slice(lineFrom, lineTo + 1);
+            }
+            return lines;
+        });
+        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
+            const { fileKey, fileType, options } = request;
+            let lineFrom, lineTo, sheetName, hasHeaderRow;
+            if (options) {
+                lineFrom = options.lineFrom;
+                lineTo = options.lineTo;
+                sheetName = options.sheetName;
+                hasHeaderRow = options.hasHeaderRow;
+            }
+            const fileUrl = path_1.default.join(this._path, fileKey);
+            let lines = [];
+            switch (fileType) {
+                case 'CSV':
+                case 'JSON':
+                case 'JSONL':
+                case 'TXT':
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                        lines = yield this._readLines(fileUrl, lineFrom, lineTo);
+                    else
+                        lines = yield this._readLines(fileUrl);
+                    break;
+                case 'XLS':
+                case 'XLSX':
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                        lines = yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
+                    else
+                        lines = yield this._readExcelLines(fileUrl, sheetName);
+                    break;
+                case 'XML':
+                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
+                        lines = yield this._readXmlLines(fileUrl, lineFrom, lineTo);
+                    else
+                        lines = yield this._readXmlLines(fileUrl);
+                    break;
+            }
+            const firstLine = lines[0];
+            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
+                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                Logger_1.default.log(msg);
+                throw new Error(msg);
+            }
+            // If this is not the first file read in a pattern match AND the file type has an header,
+            // then I need to remove the header from the resulting lines or the header will be duplicated
+            if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {
+                lines = lines.slice(1);
+            }
+            return lines;
+        });
+        this.listFiles = (filekeyPattern) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            try {
+                // Get all files in the directory (recursively if needed)
+                const getAllFiles = (dirPath, basePath = '') => {
+                    const files = [];
+                    const items = fs.readdirSync(dirPath);
+                    for (const item of items) {
+                        const fullPath = path_1.default.join(dirPath, item);
+                        const relativePath = basePath ? path_1.default.join(basePath, item) : item;
+                        const stats = fs.statSync(fullPath);
+                        if (stats.isDirectory()) {
+                            // Recursively get files from subdirectories
+                            files.push(...getAllFiles(fullPath, relativePath));
+                        }
+                        else if (stats.isFile()) {
+                            files.push(relativePath);
+                        }
+                    }
+                    return files;
+                };
+                const allFiles = getAllFiles(this._path);
+                return Helper_1.default.matchPattern(filekeyPattern, allFiles);
+            }
+            catch (error) {
+                throw new Error(`Failed to list files in directory "${this._path}": ${error.message}`);
+            }
+        };
+        this.readFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            (0, Affirm_1.default)(fs.existsSync(filePath), `Source file does not exist: ${filePath}`);
+            return fs.readFileSync(filePath);
+        };
+        this.deleteFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            if (fs.existsSync(filePath)) {
+                fs.unlinkSync(filePath);
+            }
+        };
+        this.moveFile = (sourceFileKey, destinationPath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationPath, 'Invalid destination path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const sourceFilePath = path_1.default.join(this._path, sourceFileKey);
+            const destinationFilePath = path_1.default.join(destinationPath, destinationFileKey);
+            (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
+            // Ensure destination directory exists
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.renameSync(sourceFilePath, destinationFilePath);
+        };
+        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            // TODO: extra logic for encoded files (xml, xls, ...) to be decoded and prepared locally as a plain CSV
+            // then return the uri to this new temporary file
+            const { fileKey } = producer.settings;
+            if (fileKey.includes('%')) {
+                const allFileKeys = this.listFiles(fileKey);
+                const allFilePaths = allFileKeys.map(x => path_1.default.join(this._path, x));
+                return { files: allFilePaths.map(x => ({ fullUri: x })) };
+            }
+            else {
+                const sourceFilePath = path_1.default.join(this._path, fileKey);
+                return { files: [{ fullUri: sourceFilePath }] };
+            }
+        });
+    }
+}
+exports.default = LocalSourceDriver;