@forzalabs/remora 0.0.58-nasco.3 → 0.0.60-nasco.3

package/Constants.js

@@ -1,14 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.58-nasco',
+    cliVersion: '0.0.59-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {
         PRODUCER_TEMP_FOLDER: '.temp',
         SQL_MAX_QUERY_ROWS: 10000,
         STRING_MAX_CHARACTERS_LENGTH: 10000000,
-        MAX_ITEMS_IN_MEMORY: 200000
+        MAX_ITEMS_IN_MEMORY: 200000,
+        MIN_RUNTIME_HEAP_MB: 4000,
+        RECOMMENDED_RUNTIME_HEAP_MB: 8000
     }
 };
 exports.default = CONSTANTS;

package/drivers/LocalDriver.js

@@ -396,8 +396,9 @@ class LocalDestinationDriver {
                 const filePath = path_1.default.join(folder, options.name);
                 fs.writeFileSync(filePath, '');
                 yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
-                    const content = FileExporter_1.default.prepareBatch(batch, options);
-                    fs.appendFileSync(filePath, content);
+                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
+                    for (const chunk of chunks)
+                        fs.appendFileSync(filePath, chunk);
                 }));
                 return { bucket: folder, key: filePath, res: true };
             }

package/drivers/S3Driver.js

@@ -129,21 +129,24 @@ class S3DestinationDriver {
                 const uploadId = createMultipartUploadRes.UploadId;
                 (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
                 const uploadedParts = [];
-                yield dataset.streamBatches((batch, index) => __awaiter(this, void 0, void 0, function* () {
-                    const content = FileExporter_1.default.prepareBatch(batch, options);
-                    const partNumber = index + 1;
-                    const body = Buffer.from(content);
-                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                        Bucket: this._bucketName,
-                        Key: name,
-                        UploadId: uploadId,
-                        PartNumber: partNumber,
-                        Body: body
-                    }));
-                    uploadedParts.push({
-                        PartNumber: partNumber,
-                        ETag: uploadPartRes.ETag
-                    });
+                let partNumber = 1;
+                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
+                    for (const chunk of chunks) {
+                        const body = Buffer.from(chunk);
+                        const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
+                            Bucket: this._bucketName,
+                            Key: name,
+                            UploadId: uploadId,
+                            PartNumber: partNumber,
+                            Body: body
+                        }));
+                        uploadedParts.push({
+                            PartNumber: partNumber,
+                            ETag: uploadPartRes.ETag
+                        });
+                        partNumber++;
+                    }
                 }));
                 // Complete the multipart upload
                 const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({

package/engines/dataset/DatasetManager.js

@@ -41,7 +41,7 @@ class DatasetManagerClass {
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
-            var _a, _b, _c, _d, _e;
+            var _a, _b, _c, _d, _e, _f;
             (0, Affirm_1.default)(firstLine, `Invalid first line`);
             (0, Affirm_1.default)(dsFile, `Invalid dataset file`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
@@ -62,29 +62,42 @@ class DatasetManagerClass {
                     const columns = FileCompiler_1.default.compileProducer(producer, source);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
+                    if (discover) {
+                        return {
+                            delimiter: (_b = file.delimiter) !== null && _b !== void 0 ? _b : ',',
+                            dimensions: keys.map((x, i) => ({ hidden: false, index: i, key: x, name: x }))
+                        };
+                    }
                     const dimensions = [];
                     for (const pColumn of columns) {
-                        const columnKey = (_b = pColumn.aliasInProducer) !== null && _b !== void 0 ? _b : pColumn.nameInProducer;
+                        const columnKey = (_c = pColumn.aliasInProducer) !== null && _c !== void 0 ? _c : pColumn.nameInProducer;
                         const csvColumnIndex = keys.findIndex(x => x === columnKey);
                         (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
                         dimensions.push({ index: csvColumnIndex, key: columnKey, name: pColumn.nameInProducer, hidden: null });
                     }
-                    const delimiterChar = (_c = file.delimiter) !== null && _c !== void 0 ? _c : ',';
+                    const delimiterChar = (_d = file.delimiter) !== null && _d !== void 0 ? _d : ',';
                     return { dimensions, delimiter: delimiterChar };
                 }
                 case 'TXT': {
                     if (!file.hasHeaderRow) {
                         // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
-                        const delimiterChar = (_d = file.delimiter) !== null && _d !== void 0 ? _d : ',';
+                        const delimiterChar = (_e = file.delimiter) !== null && _e !== void 0 ? _e : ',';
                         const source = Environment_1.default.getSource(producer.source);
                         const columns = FileCompiler_1.default.compileProducer(producer, source);
+                        if (discover) {
+                            // Since I don't have an header, and I'm discovering, I just create placeholder dimensions based on the same number of columns of the txt
+                            return {
+                                delimiter: delimiterChar,
+                                dimensions: firstLine.split(delimiterChar).map((x, i) => ({ hidden: false, index: i, key: `Col ${i + 1}`, name: `Col ${i + 1}` }))
+                            };
+                        }
                         return {
                             dimensions: columns.map((x, i) => { var _a; return ({ key: (_a = x.aliasInProducer) !== null && _a !== void 0 ? _a : x.nameInProducer, name: x.nameInProducer, index: i, hidden: null }); }),
                             delimiter: delimiterChar
                         };
                     }
                     else {
-                        const delimiterChar = (_e = producer.settings.delimiter) !== null && _e !== void 0 ? _e : ',';
+                        const delimiterChar = (_f = producer.settings.delimiter) !== null && _f !== void 0 ? _f : ',';
                         const rawDimensions = ParseManager_1.default._extractHeader(firstLine, delimiterChar, producer, discover);
                         return {
                             dimensions: rawDimensions.map(x => ({ key: x.name, name: x.saveAs, index: x.index, hidden: null })),

package/engines/file/FileExporter.js

@@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const Constants_1 = __importDefault(require("../../Constants"));
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
@@ -42,15 +43,30 @@ class FileExporterClass {
         this.prepareBatch = (batch, options) => {
             switch (options.recordProjection.format) {
                 case 'JSON': {
-                    const content = batch.map(x => x.toJSON()).join('\n');
-                    return content;
+                    const jsonRecords = batch.map(x => x.toJSON());
+                    return this._splitIntoChunks(jsonRecords, '\n');
                 }
                 case 'CSV': {
-                    const content = batch.map(x => x.toCSV(options.recordProjection.delimiter)).join('\n');
-                    return content;
+                    const csvRecords = batch.map(x => x.toCSV(options.recordProjection.delimiter));
+                    return this._splitIntoChunks(csvRecords, '\n');
                 }
             }
         };
+        this._splitIntoChunks = (records, separator) => {
+            if (records.length === 0)
+                return [''];
+            const sampleRecord = records[0];
+            const sampleLength = sampleRecord.length + separator.length; // Include separator in calculation
+            const recordsPerChunk = Math.floor(Constants_1.default.defaults.STRING_MAX_CHARACTERS_LENGTH / sampleLength);
+            // Ensure at least 1 record per chunk
+            const chunkSize = Math.max(1, recordsPerChunk);
+            const chunks = [];
+            for (let i = 0; i < records.length; i += chunkSize) {
+                const chunk = records.slice(i, i + chunkSize);
+                chunks.push(chunk.join(separator));
+            }
+            return chunks;
+        };
         this._composeFileName = (consumer, extension) => `${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-')}.${extension}`;
     }
 }

package/helper/Runtime.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_v8_1 = __importDefault(require("node:v8"));
+class RuntimeClass {
+    constructor() {
+        this.getHeap = () => {
+            const { heap_size_limit, used_heap_size } = node_v8_1.default.getHeapStatistics();
+            return {
+                heapSizeMB: this._toMB(heap_size_limit),
+                usedHeapMB: this._toMB(used_heap_size)
+            };
+        };
+        this._toMB = (bytes) => Math.round(bytes / (1024 * 1024) * 100) / 100;
+    }
+}
+const Runtime = new RuntimeClass();
+exports.default = Runtime;

package/index.js

@@ -16,14 +16,23 @@ const create_producer_1 = require("./actions/create_producer");
 const create_consumer_1 = require("./actions/create_consumer");
 const Constants_1 = __importDefault(require("./Constants"));
 const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
+const Runtime_1 = __importDefault(require("./helper/Runtime"));
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
+// Validate the remora licence
 const remoraLicenceKey = process.env.REMORA_LICENCE_KEY;
 const check = LicenceManager_1.default.validate(remoraLicenceKey);
 if (!check.valid) {
     console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
     process.exit(1);
 }
+// Runtime check on heap size to warn user of insufficent runtime resources
+const { heapSizeMB } = Runtime_1.default.getHeap();
+if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
+    console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
+else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
+    console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
+// Initialize all commands
 program
     .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
     .description('CLI tool for setting up and managing Data-Remora');

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.58-nasco.3",
+    "version": "0.0.60-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,