@forzalabs/remora 1.0.1 → 1.0.3

package/engines/scheduler/CronScheduler.js

@@ -47,8 +47,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const cron = __importStar(require("node-cron"));
 const Environment_1 = __importDefault(require("../Environment"));
-const UserManager_1 = __importDefault(require("../UserManager"));
 const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
+const settings_1 = require("../../settings");
 class CronScheduler {
     constructor() {
         this.scheduledJobs = new Map();
@@ -124,7 +124,7 @@ class CronScheduler {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
-                const user = UserManager_1.default.getRemoraWorkerUser();
+                const user = settings_1.REMORA_WORKER_USER;
                 const runner = { _id: user._id, name: user.name, type: 'actor' };
                 const result = yield ExecutorOrchestrator_1.default.launch({
                     consumer,

package/engines/scheduler/QueueManager.js

@@ -14,9 +14,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const Environment_1 = __importDefault(require("../Environment"));
-const UserManager_1 = __importDefault(require("../UserManager"));
 const SecretManager_1 = __importDefault(require("../SecretManager"));
 const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
+const settings_1 = require("../../settings");
 class QueueManager {
     constructor() {
         this.queueMappings = new Map();
@@ -198,7 +198,7 @@ class QueueManager {
                             }
                         }
                         console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
-                        const user = UserManager_1.default.getRemoraWorkerUser();
+                        const user = settings_1.REMORA_WORKER_USER;
                         const result = yield ExecutorOrchestrator_1.default.launch({
                             consumer: mapping.consumer,
                             details: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.1",
+    "version": "1.0.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/settings.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.REMORA_WORKER_USER = void 0;
+exports.REMORA_WORKER_USER = {
+    _id: '__remora_worker__',
+    auth: { oid: '', provider: 'internal' },
+    email: '',
+    name: 'Remora Worker',
+    roles: ['root'],
+    _signature: '',
+    lastLogin: new Date().toJSON()
+};

package/documentation/default_resources/schema.json

@@ -1,36 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "<example schema>",
-  "type": "object",
-  "version": 1,
-  "description": "<schema description>",
-  "properties": {
-    "<required id field>": {
-      "type": "<number | string | etc.>",
-      "description": "<field description>"
-    },
-    "<required name field>": {
-      "type": "<string>",
-      "description": "<field description>"
-    },
-    "<status field with enum>": {
-      "type": "<string>",
-      "description": "<field description>",
-      "enum": [
-        "<example status 1>",
-        "<example status 2>",
-        "<example status 3>"
-      ]
-    },
-    "<datetime field example>": {
-      "type": "<string>",
-      "format": "<date-time>",
-      "description": "<field description>"
-    }
-  },
-  "required": [
-    "<required id field>",
-    "<required name field>"
-  ],
-  "additionalProperties": false
-}

package/drivers/LocalDriver.js

@@ -1,542 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDestinationDriver = exports.LocalSourceDriver = void 0;
-const fs = __importStar(require("fs"));
-const path_1 = __importDefault(require("path"));
-const readline_1 = __importDefault(require("readline"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const Algo_1 = __importDefault(require("../core/Algo"));
-const xlsx_1 = __importDefault(require("xlsx"));
-const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser"));
-const XLSParser_1 = __importDefault(require("../engines/parsing/XLSParser"));
-const Helper_1 = __importDefault(require("../helper/Helper"));
-const ParseHelper_1 = __importDefault(require("../engines/parsing/ParseHelper"));
-const FileExporter_1 = __importDefault(require("../engines/file/FileExporter"));
-const Logger_1 = __importDefault(require("../helper/Logger"));
-const DriverHelper_1 = __importDefault(require("./DriverHelper"));
-const stream_1 = require("stream");
-class LocalSourceDriver {
-    constructor() {
-        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            const fileURL = source.authentication['path'];
-            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
-            const exist = fs.existsSync(fileURL);
-            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
-            this._path = source.authentication['path'];
-            return this;
-        });
-        this.readAll = (request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(request, `Invalid download request`);
-            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
-            (0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
-            const { fileKey } = request;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
-                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
-                const results = yield Promise.all(promises);
-                return results.flat();
-            }
-            else {
-                return yield this._get(request, '');
-            }
-        });
-        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(request, 'Invalid read options');
-            (0, Affirm_1.default)(request.fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(request.fileType, `Invalid file type`);
-            (0, Affirm_1.default)(request.options, `Invalid request options`);
-            Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
-            Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
-            const { fileKey } = request;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
-                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
-                const results = yield Promise.all(promises);
-                return results.flat();
-            }
-            else {
-                return yield this._get(request, '');
-            }
-        });
-        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            const file = dataset.getFile();
-            (0, Affirm_1.default)(file, 'Invalid dataset file');
-            (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const includeSourceFilename = file.includeSourceFilename === true;
-            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename, stream) {
-                const sourceFilePath = path_1.default.join(this._path, fileKey);
-                (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-                // Copy and validate header in a single stream pass
-                const readStream = fs.createReadStream(sourceFilePath);
-                let streamToUse = readStream;
-                if (['XLS', 'XLSX'].includes(file.fileType))
-                    streamToUse = stream;
-                else
-                    streamToUse = readStream;
-                return DriverHelper_1.default.appendToUnifiedFile({
-                    stream: streamToUse,
-                    fileKey,
-                    destinationPath: dataset.getPath(),
-                    append: appendMode,
-                    headerLine,
-                    fileType: file.fileType,
-                    hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter(),
-                    sourceFilename
-                });
-            });
-            const getTotalLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
-                let totalLineCount;
-                let streamXLS;
-                switch (fileType) {
-                    case 'XLS':
-                    case 'XLSX':
-                        streamXLS = (yield XLSParser_1.default.getStreamXls(path_1.default.join(this._path, fileKey), file.sheetName));
-                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename, streamXLS);
-                        break;
-                    default:
-                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), false, sourceFilename);
-                        break;
-                }
-                return totalLineCount;
-            });
-            const { fileKey } = file;
-            let totalLineCount = 0;
-            let sourceFilename;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                yield DriverHelper_1.default.setHeaderFromFile(allFileKeys[0], file, this._path, dataset);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
-                Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
-                totalLineCount = 0;
-                // Copy files sequentially to avoid file conflicts
-                for (let i = 0; i < allFileKeys.length; i++) {
-                    const currentFileKey = allFileKeys[i];
-                    // Pass the filename (just the basename) if includeSourceFilename is enabled
-                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
-                    totalLineCount += yield getTotalLineCount(currentFileKey, true, file.fileType, sourceFilename); // Append mode for subsequent files
-                }
-                dataset.setCount(totalLineCount);
-                return dataset;
-            }
-            else {
-                sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
-                yield DriverHelper_1.default.setHeaderFromFile(fileKey, file, this._path, dataset);
-                totalLineCount = (yield getTotalLineCount(fileKey, false, file.fileType, sourceFilename));
-                dataset.setCount(totalLineCount);
-                return dataset;
-            }
-        });
-        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            const fileKey = producer.settings.fileKey;
-            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                return allFileKeys.length > 0;
-            }
-            else {
-                const fileUrl = path_1.default.join(this._path, fileKey);
-                return fs.existsSync(fileUrl);
-            }
-        });
-        this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            const stream = fs.createReadStream(fileUri);
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            let lineCounter = 0;
-            try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
-                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
-                            if (line && line.length > 0)
-                                lines.push(line);
-                        }
-                        lineCounter++;
-                        if (lineCounter >= lineTo)
-                            break;
-                    }
-                    else {
-                        if (line && line.length > 0)
-                            lines.push(line);
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            reader.close();
-            stream.close();
-            return lines;
-        });
-        this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            const excel = xlsx_1.default.readFile(fileUri);
-            let targetSheetName = sheetName;
-            if (!targetSheetName) {
-                (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
-                targetSheetName = excel.SheetNames[0];
-            }
-            else {
-                (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
-            }
-            const sheet = excel.Sheets[targetSheetName];
-            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
-            const lines = csv.split('\n');
-            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                return lines.slice(lineFrom, lineTo + 1);
-            else
-                return lines;
-        });
-        this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            const fileContent = fs.readFileSync(fileUri, 'utf-8');
-            const jsonData = XMLParser_1.default.xmlToJson(fileContent);
-            // Convert JSON data to string lines. This might need adjustment based on XML structure.
-            // Assuming jsonData is an array of objects, where each object is a record.
-            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
-            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
-                lines = lines.slice(lineFrom, lineTo + 1);
-            }
-            return lines;
-        });
-        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
-            const { fileKey, fileType, options } = request;
-            let lineFrom, lineTo, sheetName, hasHeaderRow;
-            if (options) {
-                lineFrom = options.lineFrom;
-                lineTo = options.lineTo;
-                sheetName = options.sheetName;
-                hasHeaderRow = options.hasHeaderRow;
-            }
-            const fileUrl = path_1.default.join(this._path, fileKey);
-            let lines = [];
-            switch (fileType) {
-                case 'CSV':
-                case 'JSON':
-                case 'JSONL':
-                case 'TXT':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readLines(fileUrl, lineFrom, lineTo);
-                    else
-                        lines = yield this._readLines(fileUrl);
-                    break;
-                case 'XLS':
-                case 'XLSX':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
-                    else
-                        lines = yield this._readExcelLines(fileUrl, sheetName);
-                    break;
-                case 'XML':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readXmlLines(fileUrl, lineFrom, lineTo);
-                    else
-                        lines = yield this._readXmlLines(fileUrl);
-                    break;
-            }
-            const firstLine = lines[0];
-            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
-                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
-                Logger_1.default.log(msg);
-                throw new Error(msg);
-            }
-            // If this is not the first file read in a pattern match AND the file type has an header,
-            // then I need to remove the header from the resulting lines or the header will be duplicated
-            if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {
-                lines = lines.slice(1);
-            }
-            return lines;
-        });
-        this.listFiles = (filekeyPattern) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            try {
-                // Get all files in the directory (recursively if needed)
-                const getAllFiles = (dirPath, basePath = '') => {
-                    const files = [];
-                    const items = fs.readdirSync(dirPath);
-                    for (const item of items) {
-                        const fullPath = path_1.default.join(dirPath, item);
-                        const relativePath = basePath ? path_1.default.join(basePath, item) : item;
-                        const stats = fs.statSync(fullPath);
-                        if (stats.isDirectory()) {
-                            // Recursively get files from subdirectories
-                            files.push(...getAllFiles(fullPath, relativePath));
-                        }
-                        else if (stats.isFile()) {
-                            files.push(relativePath);
-                        }
-                    }
-                    return files;
-                };
-                const allFiles = getAllFiles(this._path);
-                return Helper_1.default.matchPattern(filekeyPattern, allFiles);
-            }
-            catch (error) {
-                throw new Error(`Failed to list files in directory "${this._path}": ${error.message}`);
-            }
-        };
-        this.readFile = (fileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            const filePath = path_1.default.join(this._path, fileKey);
-            (0, Affirm_1.default)(fs.existsSync(filePath), `Source file does not exist: ${filePath}`);
-            return fs.readFileSync(filePath);
-        };
-        this.deleteFile = (fileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            const filePath = path_1.default.join(this._path, fileKey);
-            if (fs.existsSync(filePath)) {
-                fs.unlinkSync(filePath);
-            }
-        };
-        this.moveFile = (sourceFileKey, destinationPath, destinationFileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
-            (0, Affirm_1.default)(destinationPath, 'Invalid destination path');
-            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
-            const sourceFilePath = path_1.default.join(this._path, sourceFileKey);
-            const destinationFilePath = path_1.default.join(destinationPath, destinationFileKey);
-            (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-            // Ensure destination directory exists
-            const destinationDir = path_1.default.dirname(destinationFilePath);
-            if (!fs.existsSync(destinationDir)) {
-                fs.mkdirSync(destinationDir, { recursive: true });
-            }
-            fs.renameSync(sourceFilePath, destinationFilePath);
-        };
-        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const { fileKey } = producer.settings;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                const allFilePaths = allFileKeys.map(x => path_1.default.join(this._path, x));
-                const readStreams = allFilePaths.map(x => fs.createReadStream(x));
-                let pass = new stream_1.PassThrough();
-                for (const [index, stream] of readStreams.entries())
-                    pass = stream.pipe(pass, { end: index === readStreams.length - 1 });
-                return pass;
-            }
-            else {
-                const sourceFilePath = path_1.default.join(this._path, fileKey);
-                const readStream = fs.createReadStream(sourceFilePath);
-                return readStream;
-            }
-        });
-    }
-}
-exports.LocalSourceDriver = LocalSourceDriver;
-class LocalDestinationDriver {
-    constructor() {
-        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(source, `Invalid source`);
-            const fileURL = source.authentication['path'];
-            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
-            const exist = fs.existsSync(fileURL);
-            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
-            this._path = source.authentication['path'];
-            return this;
-        });
-        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(options, 'Invalid upload options');
-            (0, Affirm_1.default)(options.name, 'File name is required');
-            (0, Affirm_1.default)(options.content != null, 'File content is required');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder))
-                    fs.mkdirSync(folder, { recursive: true });
-                const filePath = path_1.default.join(folder, options.name);
-                fs.writeFileSync(filePath, options.content);
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
-            }
-        });
-        this.uploadStream = (options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(options, `Invalid upload options`);
-            const { dataset, name, recordProjection } = options;
-            (0, Affirm_1.default)(dataset, 'No streaming dataset');
-            (0, Affirm_1.default)(name, 'No filename provided for upload stream');
-            (0, Affirm_1.default)(recordProjection, 'No recordProjection for upload stream');
-            const folder = this._path;
-            try {
-                if (!fs.existsSync(folder))
-                    fs.mkdirSync(folder, { recursive: true });
-                const filePath = path_1.default.join(folder, options.name);
-                fs.writeFileSync(filePath, '');
-                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
-                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
-                    for (const chunk of chunks)
-                        fs.appendFileSync(filePath, chunk);
-                }));
-                return { bucket: folder, key: filePath, res: true };
-            }
-            catch (error) {
-                // Clean up the partial file if it exists
-                const filePath = path_1.default.join(folder, options.name);
-                if (fs.existsSync(filePath)) {
-                    try {
-                        fs.unlinkSync(filePath);
-                    }
-                    catch (cleanupError) {
-                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
-                        throw cleanupError;
-                    }
-                }
-                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
-            }
-        });
-        this.saveFile = (fileKey, content) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(content, 'Invalid content');
-            const filePath = path_1.default.join(this._path, fileKey);
-            const directory = path_1.default.dirname(filePath);
-            // Create directory if it doesn't exist
-            if (!fs.existsSync(directory)) {
-                fs.mkdirSync(directory, { recursive: true });
-            }
-            fs.writeFileSync(filePath, content);
-            return Promise.resolve();
-        };
-        this.copyFromLocal = (sourceFilePath, destinationFileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(sourceFilePath, 'Invalid source file path');
-            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
-            const destinationFilePath = path_1.default.join(this._path, destinationFileKey);
-            const destinationDir = path_1.default.dirname(destinationFilePath);
-            // Ensure destination directory exists
-            if (!fs.existsSync(destinationDir)) {
-                fs.mkdirSync(destinationDir, { recursive: true });
-            }
-            fs.copyFileSync(sourceFilePath, destinationFilePath);
-        };
-        this.copyFromS3 = (s3Driver, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
-            const fileContent = yield s3Driver.downloadFile(sourceFileKey);
-            yield this.saveFile(destinationFileKey, fileContent);
-        });
-        this.ready = (destinationPath) => __awaiter(this, void 0, void 0, function* () {
-            return fs.createWriteStream(destinationPath);
-        });
-        this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
-            try {
-                const toFilePath = path_1.default.join(this._path, toName);
-                fs.renameSync(fromPath, toFilePath);
-                return { bucket: '', key: toFilePath, res: true };
-            }
-            catch (error) {
-                Logger_1.default.error(error);
-                return { bucket: '', key: '', res: false };
-            }
-        });
-        this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_2, _b, _c;
-            try {
-                const toFilePath = path_1.default.join(this._path, toName);
-                const decoder = new TextDecoder();
-                const reader = fs.createReadStream(fromPath);
-                const writer = fs.createWriteStream(toFilePath);
-                try {
-                    for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
-                        _c = reader_2_1.value;
-                        _d = false;
-                        const chunk = _c;
-                        const decoded = decoder.decode(chunk);
-                        const lines = decoded.split('\n');
-                        for (const line of lines) {
-                            writer.write(transform(line) + '\n');
-                        }
-                    }
-                }
-                catch (e_2_1) { e_2 = { error: e_2_1 }; }
-                finally {
-                    try {
-                        if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
-                    }
-                    finally { if (e_2) throw e_2.error; }
-                }
-                writer.close();
-                reader.close();
-                return { bucket: '', key: toFilePath, res: true };
-            }
-            catch (error) {
-                Logger_1.default.error(error);
-                return { bucket: '', key: '', res: false };
-            }
-        });
-    }
-}
-exports.LocalDestinationDriver = LocalDestinationDriver;