@forzalabs/remora 0.0.18 → 0.0.20

package/Constants.js

@@ -1,8 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.18',
+    cliVersion: '0.0.20',
     lambdaVersion: 1,
-    port: 5069
+    port: 5069,
+    defaults: {
+        SQL_MAX_QUERY_ROWS: 10000,
+        STRING_MAX_CHARACTERS_LENGTH: 10000000
+    }
 };
 exports.default = CONSTANTS;

package/actions/compile.js

@@ -39,7 +39,7 @@ const compile = () => __awaiter(void 0, void 0, void 0, function* () {
         errors = [...errors, ...envErrors];
         if (errors.length === 0) {
             spinner.succeed(chalk_1.default.green('Project structure validated successfully'));
-            console.log(chalk_1.default.green.bold('\n✅ Compilation complete!'));
+            console.log(chalk_1.default.blueBright.bold('✅ Compilation complete!'));
         }
         else {
             spinner.fail(chalk_1.default.red('Validation failed with errors:'));

package/actions/run.js

@@ -19,9 +19,11 @@ const Environment_1 = __importDefault(require("../engines/Environment"));
 const UserManager_1 = __importDefault(require("../engines/UserManager"));
 const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEngine"));
 const compile_1 = require("./compile");
+const Helper_1 = __importDefault(require("../helper/Helper"));
 const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
+        console.log();
         const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
         const user = UserManager_1.default.getUser();
         const consumersToExecute = (consumerName && consumerName.length > 0)
@@ -32,26 +34,38 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
             const consumer = consumersToExecute[i];
             try {
                 const response = yield ConsumerEngine_1.default.execute(consumer, {}, user);
-                results.push({ consumer, response });
+                results.push({ success: true, consumer, response });
             }
             catch (error) {
-                console.error(chalk_1.default.red(`Failed to execute consumer ${consumer.name}:`, error instanceof Error ? error.message : String(error)));
+                const myErr = Helper_1.default.asError(error);
+                results.push({ success: false, consumer, error: myErr.message });
+                if (Helper_1.default.isDev())
+                    console.log(myErr.stack);
             }
         }
-        spinner.succeed('All consumers have been successfully executed');
-        results.forEach(result => {
-            if (result.response.fileUri) {
-                console.log(chalk_1.default.green(`• Consumer ${result.consumer.name} -> ${result.response.fileUri}`));
+        spinner.succeed('All consumers have been executed:');
+        results.forEach(({ response, consumer, success, error }) => {
+            if (success) {
+                if (response.fileUri)
+                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.fileUri}`));
+                else
+                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.data.slice(0, 5).join('\n')}`));
             }
             else {
-                console.log(chalk_1.default.green(`• Consumer ${result.consumer.name} -> ${result.response.data.slice(0, 5).join('\n')}`));
+                console.log(chalk_1.default.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
             }
         });
-        console.log(chalk_1.default.green('\n✅ Initialization complete!'));
+        if (results.some(x => !x.success))
+            console.log(chalk_1.default.blueBright('\nℹ️ Run completed with errors'));
+        else
+            console.log(chalk_1.default.green('\n✅ Run complete!'));
         process.exit(1);
     }
     catch (err) {
-        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during run:'), err instanceof Error ? err.message : String(err));
+        const myErr = Helper_1.default.asError(err);
+        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during run:'), myErr.message);
+        if (Helper_1.default.isDev())
+            console.log(myErr.stack);
         process.exit(1);
     }
 });

package/drivers/DriverFactory.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -12,7 +45,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const LocalDriver_1 = __importDefault(require("./LocalDriver"));
+const LocalDriver_1 = __importStar(require("./LocalDriver"));
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
 const S3Driver_1 = __importDefault(require("./S3Driver"));
 const S3SourceDriver_1 = __importDefault(require("./S3SourceDriver"));
@@ -47,6 +80,11 @@ class DriverFactoryClass {
                     yield driver.init(source);
                     return driver;
                 }
+                case 'local': {
+                    const driver = new LocalDriver_1.LocalDriverDestination();
+                    yield driver.init(source);
+                    return driver;
+                }
                 default: throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
             }
         });

package/drivers/LocalDriver.js

@@ -52,6 +52,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalDriverDestination = void 0;
 const fs = __importStar(require("fs"));
 const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
@@ -60,9 +61,9 @@ class LocalDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
             const fileURL = source.authentication['path'];
-            (0, Affirm_1.default)(fileURL, 'file path is not entered in configuration');
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
             const exist = fs.existsSync(fileURL);
-            (0, Affirm_1.default)(exist, 'The path entered is incorrect: ' + fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
             this._path = source.authentication['path'];
             return this;
         });
@@ -119,4 +120,71 @@ class LocalDriver {
         });
     }
 }
+class LocalDriverDestination {
+    constructor() {
+        this.init = (source) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(source, `Invalid source`);
+            const fileURL = source.authentication['path'];
+            (0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
+            const exist = fs.existsSync(fileURL);
+            (0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
+            this._path = source.authentication['path'];
+            return this;
+        });
+        this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(options, 'Invalid upload options');
+            (0, Affirm_1.default)(options.name, 'File name is required');
+            (0, Affirm_1.default)(options.content != null, 'File content is required');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder))
+                    fs.mkdirSync(folder, { recursive: true });
+                const filePath = path_1.default.join(folder, options.name);
+                fs.writeFileSync(filePath, options.content);
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
+            }
+        });
+        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(options, 'Invalid upload options');
+            (0, Affirm_1.default)(options.name, 'File name is required');
+            (0, Affirm_1.default)(options.contents && Array.isArray(options.contents), 'Contents must be an array');
+            (0, Affirm_1.default)(options.contents.length > 0, 'Contents array cannot be empty');
+            const folder = this._path;
+            try {
+                if (!fs.existsSync(folder)) {
+                    fs.mkdirSync(folder, { recursive: true });
+                }
+                const filePath = path_1.default.join(folder, options.name);
+                // Create or truncate the file first
+                fs.writeFileSync(filePath, '');
+                // Append each chunk
+                for (const chunk of options.contents) {
+                    (0, Affirm_1.default)(typeof chunk === 'string', 'Each chunk must be a string');
+                    fs.appendFileSync(filePath, chunk);
+                }
+                return { bucket: folder, key: filePath, res: true };
+            }
+            catch (error) {
+                // Clean up the partial file if it exists
+                const filePath = path_1.default.join(folder, options.name);
+                if (fs.existsSync(filePath)) {
+                    try {
+                        fs.unlinkSync(filePath);
+                    }
+                    catch (cleanupError) {
+                        console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
+                        throw cleanupError;
+                    }
+                }
+                throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
+            }
+        });
+    }
+}
+exports.LocalDriverDestination = LocalDriverDestination;
 exports.default = LocalDriver;

package/drivers/S3Driver.js

@@ -45,6 +45,58 @@ class S3Driver {
             (0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
             return { res: true, key: name, bucket: this._bucketName };
         });
+        this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(options, `Invalid upload options`);
+            (0, Affirm_1.default)(options.contents && options.contents.length > 0, 'No contents provided for multipart upload');
+            (0, Affirm_1.default)(options.name, 'No filename provided for multipart upload');
+            try {
+                // Create the multipart upload
+                const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
+                    Bucket: this._bucketName,
+                    Key: options.name
+                }));
+                const uploadId = createMultipartUploadRes.UploadId;
+                (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
+                // Upload each part
+                const uploadPromises = options.contents.map((content, index) => __awaiter(this, void 0, void 0, function* () {
+                    const partNumber = index + 1;
+                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
+                        Bucket: this._bucketName,
+                        Key: options.name,
+                        UploadId: uploadId,
+                        PartNumber: partNumber,
+                        Body: Buffer.from(content)
+                    }));
+                    return {
+                        PartNumber: partNumber,
+                        ETag: uploadPartRes.ETag
+                    };
+                }));
+                const uploadedParts = yield Promise.all(uploadPromises);
+                // Complete the multipart upload
+                const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
+                    Bucket: this._bucketName,
+                    Key: options.name,
+                    UploadId: uploadId,
+                    MultipartUpload: {
+                        Parts: uploadedParts
+                    }
+                }));
+                (0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
+                return { res: true, key: options.name, bucket: this._bucketName };
+            }
+            catch (error) {
+                // If anything fails, make sure to abort the multipart upload
+                if (error.UploadId) {
+                    yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
+                        Bucket: this._bucketName,
+                        Key: options.name,
+                        UploadId: error.UploadId
+                    }));
+                }
+                throw error;
+            }
+        });
     }
 }
 exports.default = S3Driver;

package/engines/Environment.js

@@ -8,6 +8,7 @@ const path_1 = __importDefault(require("path"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const SchemaValidator_1 = __importDefault(require("./schema/SchemaValidator"));
 const Validator_1 = __importDefault(require("./Validator"));
+const Constants_1 = __importDefault(require("../Constants"));
 class EnvironmentClass {
     constructor() {
         this._env = null;
@@ -71,9 +72,16 @@ class EnvironmentClass {
                 if (!SchemaValidator_1.default.validate('consumer-schema', consumer))
                     throw new Error(`Invalid consumer configuration: ${consumer.name}`);
             });
+            // Load the project settings
+            const envSettings = new Map(Object.entries(Object.assign({}, projectConfig.settings))
+                .map(([key, value]) => [key, String(value)]));
+            if (!envSettings.has('SQL_MAX_QUERY_ROWS'))
+                envSettings.set('SQL_MAX_QUERY_ROWS', Constants_1.default.defaults.SQL_MAX_QUERY_ROWS.toString());
+            if (!envSettings.has('STRING_MAX_CHARACTERS_LENGTH'))
+                envSettings.set('STRING_MAX_CHARACTERS_LENGTH', Constants_1.default.defaults.STRING_MAX_CHARACTERS_LENGTH.toString());
             // Initialize environment
             this.init({
-                settings: new Map(Object.entries(Object.assign({}, projectConfig.settings)).map(([key, value]) => [key, String(value)])),
+                settings: envSettings,
                 sources,
                 producers,
                 consumers,
@@ -93,7 +101,7 @@ class EnvironmentClass {
          */
         this.getFirstProducer = (producerName) => {
             (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            const prod = this._env.producers.find(x => x.name === producerName);
+            const prod = this._env.producers.find(x => x.name.toLocaleLowerCase() === producerName.toLowerCase());
             if (!prod) {
                 const consumer = this.getConsumer(producerName);
                 (0, Affirm_1.default)(consumer, `Invalid producer name`);
@@ -103,11 +111,11 @@ class EnvironmentClass {
         };
         this.getProducer = (producerName) => {
             (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            return this._env.producers.find(x => x.name === producerName);
+            return this._env.producers.find(x => x.name.toLowerCase() === producerName.toLowerCase());
         };
         this.getConsumer = (consumerName) => {
             (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
-            return this._env.consumers.find(x => x.name === consumerName);
+            return this._env.consumers.find(x => x.name.toLowerCase() === consumerName.toLowerCase());
         };
         this.getSchema = (schemaName) => {
             (0, Affirm_1.default)(schemaName, 'Invalid schema name');
@@ -123,7 +131,17 @@ class EnvironmentClass {
             try {
                 errors = [...errors, ...Validator_1.default.validateSources(this._env.sources)];
                 errors = [...errors, ...Validator_1.default.validateProducers(this._env.producers)];
+                this._env.producers.forEach(prod => {
+                    const ce = Validator_1.default.validateProducer(prod);
+                    if (ce.length > 0)
+                        errors.push(`Producer "${prod.name}" errors:\n${ce.map(x => `\t-${x}\n`)}`);
+                });
                 errors = [...errors, ...Validator_1.default.validateConsumers(this._env.consumers)];
+                this._env.consumers.forEach(cons => {
+                    const ce = Validator_1.default.validateConsumer(cons);
+                    if (ce.length > 0)
+                        errors.push(`Consumer "${cons.name}" errors:\n${ce.map(x => `\t-${x}\n`)}`);
+                });
             }
             catch (e) {
                 if (errors.length === 0)

package/engines/ProducerEngine.js

@@ -82,6 +82,7 @@ class ProducerEngineClass {
             }
         });
         this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
             (0, Affirm_1.default)(producer, 'Invalid producer');
             (0, Affirm_1.default)(options, 'Invalid options');
             if (options.readmode === 'lines')
@@ -95,7 +96,7 @@ class ProducerEngineClass {
                 lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
             else
                 lines = [(yield driver.download({ fileKey: producer.settings.fileKey }))];
-            switch (producer.settings.fileType.toUpperCase()) {
+            switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
                 case 'CSV': {
                     return { data: lines, dataType: 'lines-of-text' };
                 }

package/engines/Validator.js

@@ -65,7 +65,6 @@ class ValidatorClass {
                 const dupes = Algo_1.default.duplicatesObject(consumers, 'name');
                 if (dupes.length > 0)
                     errors.push(`Duplicate name(s) found in consumers: "${dupes.map(x => x.name).join(', ')}"`);
-                errors.push(...consumers.flatMap(x => this.validateConsumer(x)));
             }
             catch (e) {
                 if (errors.length === 0)
@@ -121,7 +120,7 @@ class ValidatorClass {
                     const groupingFields = fields.filter(x => x.grouping);
                     if (groupingFields.length > 1)
                         errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
-                    groupingFields.forEach(field => {
+                    groupingFields.forEach((field) => {
                         if (field.grouping)
                             errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
                     });

package/engines/execution/ExecutionPlanner.js

@@ -18,7 +18,7 @@ class ExecutionPlannerClas {
             }
         };
         this.plan = (consumer, options) => {
-            var _a;
+            var _a, _b;
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             const [source, producer] = ConsumerManager_1.default.getSource(consumer);
             const producerEngine = source.engine;
@@ -39,7 +39,7 @@ class ExecutionPlannerClas {
                         plan.push({ type: 'read-file-lines', producer: prod, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
                     else
                         plan.push({ type: 'read-file-whole', producer: prod });
-                    if (prod.settings.fileType.toUpperCase() === 'CSV') {
+                    if (((_b = prod.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'CSV') {
                         plan.push({ type: 'csv-to-json', producer: prod });
                     }
                     if (prod.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))

package/engines/file/FileContentBuilder.js

@@ -0,0 +1,34 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+class FileContentBuilderClass {
+    constructor() {
+        /**
+         * Converts an array of string to a single string separated with the separator.
+         * In the V8 engine there is a maximum length to a string so I can't just join it all.
+         * I use this to create chunks that are not too long.
+         */
+        this.compose = (lines, separator) => {
+            Affirm_1.default.hasValue(lines, 'Invalid lines');
+            Affirm_1.default.hasValue(lines, 'Invalid separator');
+            const maxStringLength = parseInt(Environment_1.default.get('STRING_MAX_CHARACTERS_LENGTH'));
+            const chunks = [];
+            let currentChunk = '';
+            for (const line of lines) {
+                currentChunk += (line + separator);
+                if (currentChunk.length >= maxStringLength) {
+                    chunks.push(currentChunk);
+                    currentChunk = '';
+                }
+            }
+            chunks.push(currentChunk);
+            return chunks;
+        };
+    }
+}
+const FileContentBuilder = new FileContentBuilderClass();
+exports.default = FileContentBuilder;

package/engines/file/FileExporter.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -50,14 +17,22 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
 const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Environment_1 = __importDefault(require("../Environment"));
-const fs = __importStar(require("fs"));
+const FileContentBuilder_1 = __importDefault(require("./FileContentBuilder"));
 class FileExporterClass {
     constructor() {
         this.export = (consumer, output, data) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             (0, Affirm_1.default)(output, `Invalid output`);
             (0, Affirm_1.default)(data, `Invalid export data`);
-            let exportData = null;
+            const preparedData = this._prepareData(consumer, output, data);
+            const writeRes = yield this._writeData(consumer, output, preparedData);
+            return writeRes;
+        });
+        this._prepareData = (consumer, output, data) => {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            (0, Affirm_1.default)(output, `Invalid output`);
+            (0, Affirm_1.default)(data, `Invalid export data`);
+            let exportDataChunks = null;
             let extension = null;
             // build the actual file in the requested format
             switch (output.format.toUpperCase()) {
@@ -71,8 +46,7 @@ class FileExporterClass {
                         const rowValues = columns.map(c => { var _a; return Algo_1.default.replaceAll((_a = c === null || c === void 0 ? void 0 : c.toString()) !== null && _a !== void 0 ? _a : '', '"', '""'); }).map(c => `"${c}"`).join(',');
                         lines.push(rowValues);
                     }
-                    const csv = lines.join('\n');
-                    exportData = csv;
+                    exportDataChunks = FileContentBuilder_1.default.compose(lines, '\n');
                     extension = 'csv';
                     break;
                 }
@@ -82,8 +56,7 @@ class FileExporterClass {
                         const row = data[i];
                         lines.push(JSON.stringify(row));
                     }
-                    const jsonl = lines.join('\n');
-                    exportData = jsonl;
+                    exportDataChunks = FileContentBuilder_1.default.compose(lines, '\n');
                     extension = 'jsonl';
                     break;
                 }
@@ -91,27 +64,37 @@ class FileExporterClass {
                     throw new Error(`Consumer output "${output.format}" not implemented yet`);
                 }
             }
+            return { textChunks: exportDataChunks, extension };
+        };
+        this._writeData = (consumer, output, preparedData) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            (0, Affirm_1.default)(output, `Invalid output`);
+            (0, Affirm_1.default)(preparedData, `Invalid prepared data`);
             // export it where it needs to go
             const source = Environment_1.default.getSource(output.exportDestination);
             (0, Affirm_1.default)(source, `Invalid consumer "${consumer.name}" export location source. Make sure that the export location is an available source.`);
+            const { extension, textChunks } = preparedData;
+            const name = this._composeFileName(consumer, extension);
             switch (source.engine) {
                 case 'local': {
-                    const folder = source.authentication.path;
-                    if (!fs.existsSync(folder))
-                        fs.mkdirSync(folder);
-                    const date = DSTE_1.default.now().toISOString().split('.')[0];
-                    const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}`;
-                    fs.writeFileSync(path, exportData);
-                    return path;
+                    const driver = yield DriverFactory_1.default.instantiateDestination(source);
+                    const res = yield driver.multipartUpload({ contents: textChunks, name });
+                    return res.key;
                 }
                 case 'aws-s3': {
                     const driver = yield DriverFactory_1.default.instantiateDestination(source);
-                    const date = DSTE_1.default.now().toISOString().split('.')[0];
-                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}` });
-                    return res.key;
+                    if (textChunks.length === 1) {
+                        const res = yield driver.uploadFile({ content: textChunks[0], name });
+                        return res.key;
+                    }
+                    else {
+                        const res = yield driver.multipartUpload({ contents: textChunks, name });
+                        return res.key;
+                    }
                 }
             }
         });
+        this._composeFileName = (consumer, extension) => `${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-')}.${extension}`;
     }
 }
 const FileExporter = new FileExporterClass();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.18",
+    "version": "0.0.20",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,