@forzalabs/remora 1.0.21 → 1.1.1

package/drivers/files/LocalSourceDriver.js

@@ -1,405 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs = __importStar(require("fs"));
-const path_1 = __importDefault(require("path"));
-const readline_1 = __importDefault(require("readline"));
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const xlsx_1 = __importDefault(require("xlsx"));
-const XMLParser_1 = __importDefault(require("../../engines/parsing/XMLParser")); // Added XMLParser import
-const XLSParser_1 = __importDefault(require("../../engines/parsing/XLSParser"));
-const Helper_1 = __importDefault(require("../../helper/Helper"));
-const ParseHelper_1 = __importDefault(require("../../engines/parsing/ParseHelper"));
-const Logger_1 = __importDefault(require("../../helper/Logger"));
-const DriverHelper_1 = __importDefault(require("../DriverHelper"));
-const ExecutorScope_1 = __importDefault(require("../../executors/ExecutorScope"));
-const ParseCompression_1 = __importDefault(require("../../engines/parsing/ParseCompression"));
-class LocalSourceDriver {
-    constructor() {
-        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            const fileURL = source.authentication['path'];
-            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
-            const exist = fs.existsSync(fileURL);
-            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
-            this._path = source.authentication['path'];
-            return this;
-        });
-        this.readAll = (request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(request, `Invalid download request`);
-            (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
-            (0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
-            const { fileKey } = request;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
-                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
-                const results = yield Promise.all(promises);
-                return results.flat();
-            }
-            else {
-                return yield this._get(request, '');
-            }
-        });
-        this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(request, 'Invalid read options');
-            (0, Affirm_1.default)(request.fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(request.fileType, `Invalid file type`);
-            (0, Affirm_1.default)(request.options, `Invalid request options`);
-            Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
-            Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
-            const { fileKey } = request;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
-                const firstPath = path_1.default.join(this._path, allFileKeys[0]);
-                const headerLine = (yield DriverHelper_1.default.quickReadFile(firstPath, 1))[0];
-                const promises = allFileKeys.map((x, i) => this._get(Object.assign(Object.assign({}, request), { fileKey: x }), headerLine, i));
-                const results = yield Promise.all(promises);
-                return results.flat();
-            }
-            else {
-                return yield this._get(request, '');
-            }
-        });
-        this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            const file = dataset.getFile();
-            (0, Affirm_1.default)(file, 'Invalid dataset file');
-            (0, Affirm_1.default)(file.fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(file.fileType, `Invalid file type`);
-            const includeSourceFilename = file.includeSourceFilename === true;
-            const copyLocally = (fileKey_1, headerLine_1, ...args_1) => __awaiter(this, [fileKey_1, headerLine_1, ...args_1], void 0, function* (fileKey, headerLine, appendMode = false, sourceFilename, stream) {
-                const sourceFilePath = path_1.default.join(this._path, fileKey);
-                (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-                // Copy and validate header in a single stream pass
-                const readStream = fs.createReadStream(sourceFilePath);
-                let streamToUse = readStream;
-                if (['XLS', 'XLSX'].includes(file.fileType))
-                    streamToUse = stream;
-                else
-                    streamToUse = readStream;
-                return DriverHelper_1.default.appendToUnifiedFile({
-                    stream: streamToUse,
-                    fileKey,
-                    destinationPath: dataset.getPath(),
-                    append: appendMode,
-                    headerLine,
-                    fileType: file.fileType,
-                    hasHeaderRow: file.hasHeaderRow,
-                    delimiter: dataset.getDelimiter(),
-                    sourceFilename
-                });
-            });
-            // this function copy the local file on the temporary file and retrive the number of line.
-            const handleFileAndGetLineCount = (fileKey, appendMode, fileType, sourceFilename) => __awaiter(this, void 0, void 0, function* () {
-                let totalLineCount;
-                let streamXLS;
-                switch (fileType) {
-                    case 'XLS':
-                    case 'XLSX':
-                        streamXLS = (yield XLSParser_1.default.getStreamXls(path_1.default.join(this._path, fileKey), file.sheetName));
-                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename, streamXLS);
-                        break;
-                    default:
-                        totalLineCount = yield copyLocally(fileKey, dataset.getFirstLine(), appendMode, sourceFilename);
-                        break;
-                }
-                return totalLineCount;
-            });
-            const { fileKey } = file;
-            let totalLineCount = 0;
-            let sourceFilename;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                yield DriverHelper_1.default.setHeaderFromFile(allFileKeys[0], file, this._path, dataset);
-                Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
-                Affirm_1.default.hasItems(allFileKeys, `The file key "${fileKey}" doesn't have any matches in path "${this._path}".`);
-                totalLineCount = 0;
-                // Copy files sequentially to avoid file conflicts
-                for (let i = 0; i < allFileKeys.length; i++) {
-                    const currentFileKey = allFileKeys[i];
-                    // Pass the filename (just the basename) if includeSourceFilename is enabled
-                    const sourceFilename = includeSourceFilename ? path_1.default.basename(currentFileKey) : undefined;
-                    totalLineCount += yield handleFileAndGetLineCount(currentFileKey, i > 0, file.fileType, sourceFilename); // Append mode for subsequent files
-                }
-                dataset.setCount(totalLineCount);
-                return dataset;
-            }
-            else {
-                sourceFilename = includeSourceFilename ? path_1.default.basename(fileKey) : undefined;
-                yield DriverHelper_1.default.setHeaderFromFile(fileKey, file, this._path, dataset);
-                totalLineCount = (yield handleFileAndGetLineCount(fileKey, false, file.fileType, sourceFilename));
-                dataset.setCount(totalLineCount);
-                return dataset;
-            }
-        });
-        this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._path, `Invalid path`);
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            const fileKey = producer.settings.fileKey;
-            (0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                return allFileKeys.length > 0;
-            }
-            else {
-                const fileUrl = path_1.default.join(this._path, fileKey);
-                return fs.existsSync(fileUrl);
-            }
-        });
-        this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            const stream = fs.createReadStream(fileUri);
-            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
-            const lines = [];
-            let lineCounter = 0;
-            try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
-                    _d = false;
-                    const line = _c;
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
-                        if (lineCounter >= lineFrom && lineCounter < lineTo) {
-                            if (line && line.length > 0)
-                                lines.push(line);
-                        }
-                        lineCounter++;
-                        if (lineCounter >= lineTo)
-                            break;
-                    }
-                    else {
-                        if (line && line.length > 0)
-                            lines.push(line);
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            reader.close();
-            stream.close();
-            return lines;
-        });
-        this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            const excel = xlsx_1.default.readFile(fileUri);
-            let targetSheetName = sheetName;
-            if (!targetSheetName) {
-                (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
-                targetSheetName = excel.SheetNames[0];
-            }
-            else {
-                (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
-            }
-            const sheet = excel.Sheets[targetSheetName];
-            const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
-            const lines = csv.split('\n');
-            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                return lines.slice(lineFrom, lineTo + 1);
-            else
-                return lines;
-        });
-        this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
-            const fileContent = fs.readFileSync(fileUri, 'utf-8');
-            const jsonData = XMLParser_1.default.xmlToJson(fileContent);
-            // Convert JSON data to string lines. This might need adjustment based on XML structure.
-            // Assuming jsonData is an array of objects, where each object is a record.
-            let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
-            if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
-                lines = lines.slice(lineFrom, lineTo + 1);
-            }
-            return lines;
-        });
-        this._get = (request, headerLine, index) => __awaiter(this, void 0, void 0, function* () {
-            const { fileKey, fileType, options } = request;
-            let lineFrom, lineTo, sheetName, hasHeaderRow;
-            if (options) {
-                lineFrom = options.lineFrom;
-                lineTo = options.lineTo;
-                sheetName = options.sheetName;
-                hasHeaderRow = options.hasHeaderRow;
-            }
-            const fileUrl = path_1.default.join(this._path, fileKey);
-            let lines = [];
-            switch (fileType) {
-                case 'CSV':
-                case 'JSON':
-                case 'JSONL':
-                case 'TXT':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readLines(fileUrl, lineFrom, lineTo);
-                    else
-                        lines = yield this._readLines(fileUrl);
-                    break;
-                case 'XLS':
-                case 'XLSX':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
-                    else
-                        lines = yield this._readExcelLines(fileUrl, sheetName);
-                    break;
-                case 'XML':
-                    if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
-                        lines = yield this._readXmlLines(fileUrl, lineFrom, lineTo);
-                    else
-                        lines = yield this._readXmlLines(fileUrl);
-                    break;
-            }
-            const firstLine = lines[0];
-            if (headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
-                const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
-                Logger_1.default.log(msg);
-                throw new Error(msg);
-            }
-            // If this is not the first file read in a pattern match AND the file type has an header,
-            // then I need to remove the header from the resulting lines or the header will be duplicated
-            if (index > 0 && ParseHelper_1.default.shouldHaveHeader(fileType, hasHeaderRow)) {
-                lines = lines.slice(1);
-            }
-            return lines;
-        });
-        this.listFiles = (filekeyPattern) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            try {
-                // Get all files in the directory (recursively if needed)
-                const getAllFiles = (dirPath, basePath = '') => {
-                    const files = [];
-                    const items = fs.readdirSync(dirPath);
-                    for (const item of items) {
-                        const fullPath = path_1.default.join(dirPath, item);
-                        const relativePath = basePath ? path_1.default.join(basePath, item) : item;
-                        const stats = fs.statSync(fullPath);
-                        if (stats.isDirectory()) {
-                            // Recursively get files from subdirectories
-                            files.push(...getAllFiles(fullPath, relativePath));
-                        }
-                        else if (stats.isFile()) {
-                            files.push(relativePath);
-                        }
-                    }
-                    return files;
-                };
-                const allFiles = getAllFiles(this._path);
-                return Helper_1.default.matchPattern(filekeyPattern, allFiles);
-            }
-            catch (error) {
-                throw new Error(`Failed to list files in directory "${this._path}": ${error.message}`);
-            }
-        };
-        this.readFile = (fileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            const filePath = path_1.default.join(this._path, fileKey);
-            (0, Affirm_1.default)(fs.existsSync(filePath), `Source file does not exist: ${filePath}`);
-            return fs.readFileSync(filePath);
-        };
-        this.deleteFile = (fileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            const filePath = path_1.default.join(this._path, fileKey);
-            if (fs.existsSync(filePath)) {
-                fs.unlinkSync(filePath);
-            }
-        };
-        this.moveFile = (sourceFileKey, destinationPath, destinationFileKey) => {
-            (0, Affirm_1.default)(this._path, 'Path not initialized');
-            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
-            (0, Affirm_1.default)(destinationPath, 'Invalid destination path');
-            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
-            const sourceFilePath = path_1.default.join(this._path, sourceFileKey);
-            const destinationFilePath = path_1.default.join(destinationPath, destinationFileKey);
-            (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
-            // Ensure destination directory exists
-            const destinationDir = path_1.default.dirname(destinationFilePath);
-            if (!fs.existsSync(destinationDir)) {
-                fs.mkdirSync(destinationDir, { recursive: true });
-            }
-            fs.renameSync(sourceFilePath, destinationFilePath);
-        };
-        this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(request, 'Invalid ready request');
-            const { producer, scope } = request;
-            // TODO: extra logic for encoded files (xml, xls, ...) to be decoded and prepared locally as a plain CSV
-            // then return the uri to this new temporary file
-            const { fileKey, compressionType } = producer.settings;
-            if (fileKey.includes('%')) {
-                const allFileKeys = this.listFiles(fileKey);
-                const allFilePaths = [];
-                for (const fileKey of allFileKeys) {
-                    const executionPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
-                    ExecutorScope_1.default.ensurePath(executionPath);
-                    const localPath = yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, executionPath);
-                    allFilePaths.push(localPath);
-                }
-                return { files: allFilePaths.map(x => ({ fullUri: x })) };
-            }
-            else {
-                const executionPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
-                ExecutorScope_1.default.ensurePath(executionPath);
-                const localPath = yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, executionPath);
-                return { files: [{ fullUri: localPath }] };
-            }
-        });
-    }
-}
-exports.default = LocalSourceDriver;

package/drivers/s3/S3DestinationDriver.js

@@ -1,197 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
-var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var g = generator.apply(thisArg, _arguments || []), i, q = [];
-    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
-    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
-    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
-    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
-    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
-    function fulfill(value) { resume("next", value); }
-    function reject(value) { resume("throw", value); }
-    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const client_s3_1 = require("@aws-sdk/client-s3");
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const SecretManager_1 = __importDefault(require("../../engines/SecretManager"));
-const path_1 = __importDefault(require("path"));
-const fs_1 = __importDefault(require("fs"));
-const readline_1 = __importDefault(require("readline"));
-class S3DestinationDriver {
-    constructor() {
-        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            this._bucketName = source.authentication['bucket'];
-            const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
-            const config = {
-                region: source.authentication['region'],
-                credentials: {
-                    accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
-                    secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
-                    sessionToken: sessionToken ? sessionToken : undefined
-                }
-            };
-            this._client = new client_s3_1.S3Client(config);
-            // TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
-            return this;
-        });
-        this.copyFromS3 = (sourceBucket, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
-            (0, Affirm_1.default)(sourceBucket, 'Invalid source bucket');
-            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
-            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
-            yield this._client.send(new client_s3_1.CopyObjectCommand({
-                CopySource: `${sourceBucket}/${sourceFileKey}`,
-                Bucket: this._bucketName,
-                Key: destinationFileKey
-            }));
-        });
-        this.saveFile = (fileKey, content) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
-            (0, Affirm_1.default)(fileKey, 'Invalid file key');
-            (0, Affirm_1.default)(content, 'Invalid content');
-            yield this._client.send(new client_s3_1.PutObjectCommand({
-                Bucket: this._bucketName,
-                Key: fileKey,
-                Body: content
-            }));
-        });
-        this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(fromPath, 'Invalid source path');
-            (0, Affirm_1.default)(toName, 'Invalid destination name');
-            (0, Affirm_1.default)(fs_1.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
-            const readStream = fs_1.default.createReadStream(fromPath);
-            return this._multipartUpload(toName, readStream);
-        });
-        this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(fromPath, 'Invalid source path');
-            (0, Affirm_1.default)(transform, 'Invalid transform function');
-            (0, Affirm_1.default)(toName, 'Invalid destination name');
-            (0, Affirm_1.default)(fs_1.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
-            const reader = fs_1.default.createReadStream(fromPath);
-            const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
-            const transformedLines = function () {
-                return __asyncGenerator(this, arguments, function* () {
-                    var _a, e_1, _b, _c;
-                    try {
-                        for (var _d = true, lineReader_1 = __asyncValues(lineReader), lineReader_1_1; lineReader_1_1 = yield __await(lineReader_1.next()), _a = lineReader_1_1.done, !_a; _d = true) {
-                            _c = lineReader_1_1.value;
-                            _d = false;
-                            const line = _c;
-                            yield yield __await(transform(line) + '\n');
-                        }
-                    }
-                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                    finally {
-                        try {
-                            if (!_d && !_a && (_b = lineReader_1.return)) yield __await(_b.call(lineReader_1));
-                        }
-                        finally { if (e_1) throw e_1.error; }
-                    }
-                });
-            };
-            return this._multipartUpload(toName, transformedLines());
-        });
-        this._multipartUpload = (toName, dataSource) => __awaiter(this, void 0, void 0, function* () {
-            var _a, dataSource_1, dataSource_1_1;
-            var _b, e_2, _c, _d;
-            let uploadId;
-            try {
-                // Create the multipart upload
-                const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: toName
-                }));
-                uploadId = createMultipartUploadRes.UploadId;
-                (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
-                const uploadedParts = [];
-                let partNumber = 1;
-                const MIN_PART_SIZE = 5 * 1024 * 1024; // 5MB
-                let accumulatedBuffer = Buffer.alloc(0);
-                const uploadPart = (buffer) => __awaiter(this, void 0, void 0, function* () {
-                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                        Bucket: this._bucketName,
-                        Key: toName,
-                        UploadId: uploadId,
-                        PartNumber: partNumber,
-                        Body: buffer
-                    }));
-                    uploadedParts.push({
-                        PartNumber: partNumber,
-                        ETag: uploadPartRes.ETag
-                    });
-                    partNumber++;
-                });
-                try {
-                    for (_a = true, dataSource_1 = __asyncValues(dataSource); dataSource_1_1 = yield dataSource_1.next(), _b = dataSource_1_1.done, !_b; _a = true) {
-                        _d = dataSource_1_1.value;
-                        _a = false;
-                        const chunk = _d;
-                        const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-                        accumulatedBuffer = Buffer.concat([accumulatedBuffer, chunkBuffer]);
-                        // If accumulated buffer is at least 5MB, upload it as a part
-                        while (accumulatedBuffer.length >= MIN_PART_SIZE) {
-                            const partBuffer = accumulatedBuffer.subarray(0, MIN_PART_SIZE);
-                            accumulatedBuffer = accumulatedBuffer.subarray(MIN_PART_SIZE);
-                            yield uploadPart(partBuffer);
-                        }
-                    }
-                }
-                catch (e_2_1) { e_2 = { error: e_2_1 }; }
-                finally {
-                    try {
-                        if (!_a && !_b && (_c = dataSource_1.return)) yield _c.call(dataSource_1);
-                    }
-                    finally { if (e_2) throw e_2.error; }
-                }
-                // Upload any remaining data as the final part (even if smaller than 5MB)
-                if (accumulatedBuffer.length > 0) {
-                    yield uploadPart(accumulatedBuffer);
-                }
-                // Complete the multipart upload
-                const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
-                    Bucket: this._bucketName,
-                    Key: toName,
-                    UploadId: uploadId,
-                    MultipartUpload: {
-                        Parts: uploadedParts
-                    }
-                }));
-                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${toName}": status code ${completeRes.$metadata.httpStatusCode}`);
-                return { res: true, key: path_1.default.join(this._bucketName, toName), bucket: this._bucketName };
-            }
-            catch (error) {
-                // If anything fails, make sure to abort the multipart upload
-                if (uploadId) {
-                    yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
-                        Bucket: this._bucketName,
-                        Key: toName,
-                        UploadId: uploadId
-                    }));
-                }
-                throw error;
-            }
-        });
-    }
-}
-exports.default = S3DestinationDriver;