@forzalabs/remora 0.0.57-nasco.3 → 0.0.59-nasco.3

package/Constants.js

@@ -1,14 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.57-nasco',
+    cliVersion: '0.0.59-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {
         PRODUCER_TEMP_FOLDER: '.temp',
         SQL_MAX_QUERY_ROWS: 10000,
         STRING_MAX_CHARACTERS_LENGTH: 10000000,
-        MAX_ITEMS_IN_MEMORY: 200000
+        MAX_ITEMS_IN_MEMORY: 200000,
+        MIN_RUNTIME_HEAP_MB: 4000,
+        RECOMMENDED_RUNTIME_HEAP_MB: 8000
     }
 };
 exports.default = CONSTANTS;

package/drivers/DriverHelper.js

@@ -28,29 +28,84 @@ const Affirm_1 = __importDefault(require("../core/Affirm"));
 const DriverHelper = {
     appendToUnifiedFile: (options) => __awaiter(void 0, void 0, void 0, function* () {
         (0, Affirm_1.default)(options, 'Invalid options');
-        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow } = options;
+        const { append, destinationPath, fileKey, headerLine, stream, fileType, hasHeaderRow, delimiter } = options;
+        const keys = (fileType === 'JSON' || fileType === 'JSONL') ? Object.keys(headerLine) : [];
         const shouldValidateHeader = fileType === 'CSV' || (fileType === 'TXT' && hasHeaderRow === true);
         let isFirstLine = true;
         let hasValidatedHeader = shouldValidateHeader ? false : true;
+        let leftoverData = '';
+        let globalIndex = 0;
         const headerValidationTransform = new stream_1.Transform({
             transform(chunk, encoding, callback) {
-                if (!hasValidatedHeader) {
-                    const chunkStr = chunk.toString();
-                    const lines = chunkStr.split('\n');
-                    if (isFirstLine && lines.length > 0) {
-                        const firstLine = lines[0];
-                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && firstLine.trim() !== headerLine.trim()) {
-                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${firstLine}\n\t-main: ${headerLine}`;
+                const chunkStr = leftoverData + chunk.toString();
+                const lines = chunkStr.split('\n');
+                // Keep the last line as leftover if it doesn't end with newline
+                leftoverData = lines.pop() || '';
+                const filteredLines = [];
+                for (let i = 0; i < lines.length; i++) {
+                    const line = lines[i];
+                    // Header validation for first line
+                    if (!hasValidatedHeader && isFirstLine && i === 0) {
+                        if (shouldValidateHeader && headerLine && headerLine.trim() !== '' && line.trim() !== headerLine.trim()) {
+                            const msg = `Error creating unified dataset: file "${fileKey}" has a different header line than the other files in this dataset\n\t-${fileKey}: ${line}\n\t-main: ${headerLine}`;
                             Logger_1.default.log(msg);
                             return callback(new Error(msg));
                         }
                         hasValidatedHeader = true;
                         isFirstLine = false;
                     }
+                    // Apply your filtering logic here
+                    if (shouldIncludeLine(line, globalIndex)) {
+                        filteredLines.push(processLine(line));
+                    }
+                    globalIndex++;
+                }
+                // Output filtered lines
+                if (filteredLines.length > 0) {
+                    const output = filteredLines.join('\n') + '\n';
+                    callback(null, Buffer.from(output));
+                }
+                else {
+                    callback(null, null); // No data to output
                 }
-                callback(null, chunk);
+            },
+            flush(callback) {
+                // Process any remaining data
+                if (leftoverData.trim()) {
+                    if (shouldIncludeLine(leftoverData, -1)) {
+                        callback(null, Buffer.from(processLine(leftoverData)));
+                    }
+                    else {
+                        callback(null, null);
+                    }
+                }
+                else {
+                    callback(null, null);
+                }
+                globalIndex++;
             }
         });
+        // Helper function to determine if a line should be included
+        const shouldIncludeLine = (line, lineIndex) => {
+            // For flat files (csv, txt) ignore the first line of the header (I already saved that line)
+            if (lineIndex === 0 && shouldValidateHeader)
+                return false;
+            // Skip empty lines
+            if (line.trim() === '')
+                return false;
+            return true;
+        };
+        const processLine = (line) => {
+            switch (fileType) {
+                case 'JSON':
+                case 'JSONL': {
+                    const parsed = JSON.parse(line);
+                    return keys.map(k => parsed[k]).join(delimiter);
+                }
+                default:
+                    return line;
+            }
+        };
         const writeOptions = append ? { flags: 'a' } : {};
         const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);
         yield (0, promises_1.pipeline)(stream, headerValidationTransform, writeStream);

package/drivers/LocalDriver.js

@@ -135,7 +135,8 @@ class LocalSourceDriver {
                     append: appendMode,
                     headerLine,
                     fileType: file.fileType,
-                    hasHeaderRow: file.hasHeaderRow
+                    hasHeaderRow: file.hasHeaderRow,
+                    delimiter: dataset.getDelimiter()
                 });
             });
             const { fileKey } = file;
@@ -144,6 +145,7 @@ class LocalSourceDriver {
                 Logger_1.default.log(`Matched ${allFileKeys.length} files, copying locally and creating unified dataset.`);
                 // Get header line from the first file
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
+                dataset.setFirstLine(headerLine);
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
                     yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
@@ -152,6 +154,8 @@ class LocalSourceDriver {
             }
             else {
                 // For single file, no header validation needed
+                const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
+                dataset.setFirstLine(headerLine);
                 yield copyLocally(fileKey, '', false);
                 return dataset;
             }
@@ -392,8 +396,9 @@ class LocalDestinationDriver {
                 const filePath = path_1.default.join(folder, options.name);
                 fs.writeFileSync(filePath, '');
                 yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
-                    const content = FileExporter_1.default.prepareBatch(batch, options);
-                    fs.appendFileSync(filePath, content);
+                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
+                    for (const chunk of chunks)
+                        fs.appendFileSync(filePath, chunk);
                 }));
                 return { bucket: folder, key: filePath, res: true };
             }

package/drivers/S3Driver.js

@@ -129,21 +129,24 @@ class S3DestinationDriver {
                 const uploadId = createMultipartUploadRes.UploadId;
                 (0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
                 const uploadedParts = [];
-                yield dataset.streamBatches((batch, index) => __awaiter(this, void 0, void 0, function* () {
-                    const content = FileExporter_1.default.prepareBatch(batch, options);
-                    const partNumber = index + 1;
-                    const body = Buffer.from(content);
-                    const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
-                        Bucket: this._bucketName,
-                        Key: name,
-                        UploadId: uploadId,
-                        PartNumber: partNumber,
-                        Body: body
-                    }));
-                    uploadedParts.push({
-                        PartNumber: partNumber,
-                        ETag: uploadPartRes.ETag
-                    });
+                let partNumber = 1;
+                yield dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
+                    const chunks = FileExporter_1.default.prepareBatch(batch, options);
+                    for (const chunk of chunks) {
+                        const body = Buffer.from(chunk);
+                        const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
+                            Bucket: this._bucketName,
+                            Key: name,
+                            UploadId: uploadId,
+                            PartNumber: partNumber,
+                            Body: body
+                        }));
+                        uploadedParts.push({
+                            PartNumber: partNumber,
+                            ETag: uploadPartRes.ETag
+                        });
+                        partNumber++;
+                    }
                 }));
                 // Complete the multipart upload
                 const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
@@ -243,7 +246,8 @@ class S3SourceDriver {
                     append: appendMode,
                     headerLine,
                     fileType: file.fileType,
-                    hasHeaderRow: file.hasHeaderRow
+                    hasHeaderRow: file.hasHeaderRow,
+                    delimiter: dataset.getDelimiter()
                 });
             });
             const { fileKey } = file;
@@ -259,6 +263,7 @@ class S3SourceDriver {
                 (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
                 const firstFileStream = firstFileResponse.Body;
                 const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                dataset.setFirstLine(headerLine);
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
                     yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
@@ -266,6 +271,16 @@ class S3SourceDriver {
                 return dataset;
             }
             else {
+                // Get header line from the first file
+                const firstFileCommand = new client_s3_1.GetObjectCommand({
+                    Bucket: this._bucketName,
+                    Key: fileKey
+                });
+                const firstFileResponse = yield this._client.send(firstFileCommand);
+                (0, Affirm_1.default)(firstFileResponse.Body, 'Failed to fetch first file from S3');
+                const firstFileStream = firstFileResponse.Body;
+                const headerLine = yield this.getFirstLineFromStream(firstFileStream);
+                dataset.setFirstLine(headerLine);
                 // For single file, no header validation needed
                 yield downloadLocally(fileKey, '');
                 return dataset;

package/engines/consumer/PostProcessor.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
+const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
 const DatasetRecord_1 = __importDefault(require("../dataset/DatasetRecord"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
@@ -25,6 +26,7 @@ class PostProcessorClass {
     constructor() {
         /**
          * Maps an array of objects and projects it to another array of objects but with different shape:
+         * - updates the dimensions of the dataset (drop, rename, reorder, hide)
          * - type casting
          * - default field values
          * - masking/hashing of data
@@ -33,9 +35,20 @@ class PostProcessorClass {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            let newDataset = yield dataset.wholeUpdateDimensions(fields);
-            newDataset = yield newDataset.map(record => {
+            const dimensionsUpdates = DatasetManager_1.default.computeDimensionsUpdates(dataset, consumer);
+            let updatedDimensions = null;
+            const newDataset = yield dataset.map(record => {
                 var _a, _b;
+                // First apply the updates to the dimensions of this record
+                if (dimensionsUpdates.length > 0) {
+                    for (const update of dimensionsUpdates) {
+                        record.wholeUpdateDimension(update);
+                    }
+                    record.sortDimensions();
+                }
+                if (!updatedDimensions)
+                    updatedDimensions = record._dimensions;
+                // Finally apply the rules and changes of the consumer fields to the record
                 for (const field of fields) {
                     const { key, alias } = field.cField;
                     const fieldKey = alias !== null && alias !== void 0 ? alias : key;
@@ -49,6 +62,7 @@ class PostProcessorClass {
                 }
                 return record;
             });
+            newDataset.setDimensinons(updatedDimensions);
             return newDataset;
         });
         /**

package/engines/dataset/Dataset.js

@@ -37,7 +37,6 @@ const Environment_1 = __importDefault(require("../Environment"));
 class Dataset {
     constructor(name, file, batchSize) {
         var _a;
-        this._pipeline = [];
         this.getPath = () => this._path;
         this.setPath = (path) => {
             this._path = path;
@@ -45,11 +44,11 @@ class Dataset {
         };
         this.getFile = () => this._file;
         this.getBatchSize = () => this._batchSize;
-        this.setBatchSize = (size) => {
-            this._batchSize = size;
-            this._recordPool.resize(size);
+        this.setFirstLine = (firstLine) => {
+            this._firstLine = firstLine;
             return this;
         };
+        this.getFirstLine = () => this._firstLine;
         this.getSize = () => this._size;
         this.getCycles = () => this._iterations;
         this.getDelimiter = () => this._delimiter;
@@ -633,25 +632,11 @@ class Dataset {
             this._delimiter = delimiter;
             this._dimensions = dimensions;
             switch (this._file.fileType) {
-                case 'TXT': {
-                    if (this._file.hasHeaderRow)
-                        yield this.filter((x, i) => i > 0 && !x.isEmpty());
-                    break;
-                }
-                case 'CSV': {
-                    yield this.filter((x, i) => i > 0 && !x.isEmpty());
-                    break;
-                }
+                case 'TXT':
+                case 'CSV':
                 case 'JSON':
-                case 'JSONL': {
-                    // Convert the JSON to the internal CSV format
-                    yield this.map(record => {
-                        const parsed = JSON.parse(record.getRaw());
-                        const preparedRow = this._dimensions.map(d => parsed[d.key]).join(this._delimiter);
-                        return new DatasetRecord_1.default(preparedRow, this._dimensions, this._delimiter);
-                    });
+                case 'JSONL':
                     break;
-                }
                 case 'XLS':
                 case 'XLSX': {
                     const excel = xlsx_1.default.readFile(this._path);
@@ -689,6 +674,10 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
+        this.setDimensinons = (dimensions) => {
+            this._dimensions = dimensions;
+            return this;
+        };
         /**
          * Update the record pool when dimensions change
          */
@@ -696,55 +685,6 @@ class Dataset {
             // Update all pooled records with current dimensions
             this._recordPool.updateDimensions(this._dimensions, this._delimiter);
         };
-        /**
-         * - remove dimension
-         * - rename a dimension
-         * - change hidden flag
-         * - move a dimension
-         */
-        this.wholeUpdateDimensions = (fields) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            let updates = [];
-            // Add all the updates
-            for (let i = 0; i < fields.length; i++) {
-                const { cField } = fields[i];
-                const currentMatch = structuredClone(this._dimensions.find(x => x.name === cField.key));
-                if (!currentMatch && !cField.fixed)
-                    throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying dataset "${this._name}" (${this._dimensions.map(x => x.name).join(', ')})`);
-                updates.push({
-                    currentDimension: currentMatch,
-                    newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
-                    newHidden: cField.hidden,
-                    newPosition: i,
-                    toDelete: false
-                });
-            }
-            // Add all the updates to remove dimensions
-            for (const dim of this._dimensions) {
-                if (!updates.find(x => { var _a; return ((_a = x.currentDimension) === null || _a === void 0 ? void 0 : _a.name) === dim.name; }))
-                    updates.push({ currentDimension: dim, toDelete: true });
-            }
-            // Now keep only the updates that actually change something
-            updates = updates.filter(x => x.toDelete
-                || !x.currentDimension
-                || (x.currentDimension && (x.currentDimension.name !== x.newName
-                    || (Algo_1.default.hasVal(x.newHidden) && x.newHidden !== x.currentDimension.hidden)
-                    || x.newPosition !== x.currentDimension.index)));
-            if (updates.length === 0)
-                return this;
-            let updatedDimensions = null;
-            const newDataset = yield this.map(record => {
-                for (const update of updates) {
-                    record.wholeUpdateDimension(update);
-                }
-                record._dimensions.sort((a, b) => a.index - b.index);
-                if (!updatedDimensions)
-                    updatedDimensions = record._dimensions;
-                return record;
-            });
-            this._dimensions = updatedDimensions;
-            return newDataset;
-        });
         this.print = (...args_1) => __awaiter(this, [...args_1], void 0, function* (count = 3, full = false) {
             console.log(`DS ${this._name} (${this._size} | ${this._iterations})`);
             console.log(this._dimensions.map(x => x.name).join(this._delimiter));
@@ -861,11 +801,11 @@ class Dataset {
         this._file = file;
         this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
+        this._firstLine = '';
         this._delimiter = ',';
         this._size = 0;
         this._iterations = 0;
         this._operations = [];
-        this._pipeline = [];
         // Initialize record pool for optimization
         this._recordPool = new DatasetRecordPool_1.default(this._batchSize);
         const datasetName = this._name

package/engines/dataset/DatasetManager.js

@@ -13,6 +13,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
@@ -34,7 +36,8 @@ class DatasetManagerClass {
         this.buildDimensions = (dataset_1, producer_1, ...args_1) => __awaiter(this, [dataset_1, producer_1, ...args_1], void 0, function* (dataset, producer, discover = false) {
             (0, Affirm_1.default)(dataset, `Invalid dataset`);
             (0, Affirm_1.default)(producer, `Invalid producer`);
-            const firstLine = (yield dataset.readLines(1))[0].getRaw();
+            const firstLine = dataset.getFirstLine();
+            (0, Affirm_1.default)(firstLine, `The first line of the dataset was not set.`);
             return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
         });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
@@ -97,6 +100,40 @@ class DatasetManagerClass {
                     break;
             }
         });
+        this.computeDimensionsUpdates = (dataset, consumer) => {
+            var _a;
+            (0, Affirm_1.default)(dataset, 'Invalid dataset');
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
+            const dimensions = dataset.getDimensions();
+            let updates = [];
+            // Add all the updates
+            for (let i = 0; i < fields.length; i++) {
+                const { cField } = fields[i];
+                const currentMatch = structuredClone(dimensions.find(x => x.name === cField.key));
+                if (!currentMatch && !cField.fixed)
+                    throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying dataset "${dataset['_name']}" (${dimensions.map(x => x.name).join(', ')})`);
+                updates.push({
+                    currentDimension: currentMatch,
+                    newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
+                    newHidden: cField.hidden,
+                    newPosition: i,
+                    toDelete: false
+                });
+            }
+            // Add all the updates to remove dimensions
+            for (const dim of dimensions) {
+                if (!updates.find(x => { var _a; return ((_a = x.currentDimension) === null || _a === void 0 ? void 0 : _a.name) === dim.name; }))
+                    updates.push({ currentDimension: dim, toDelete: true });
+            }
+            // Now keep only the updates that actually change something
+            updates = updates.filter(x => x.toDelete
+                || !x.currentDimension
+                || (x.currentDimension && (x.currentDimension.name !== x.newName
+                    || (Algo_1.default.hasVal(x.newHidden) && x.newHidden !== x.currentDimension.hidden)
+                    || x.newPosition !== x.currentDimension.index)));
+            return updates;
+        };
     }
 }
 const DatasetManager = new DatasetManagerClass();

package/engines/dataset/DatasetRecord.js

@@ -63,6 +63,12 @@ class DatasetRecord {
             }
             return this;
         };
+        this.sortDimensions = () => {
+            const isOutOfOrder = this._dimensions.some((dim, index) => dim.index !== index);
+            if (isOutOfOrder) {
+                this._dimensions.sort((a, b) => a.index - b.index);
+            }
+        };
         this.toJSON = () => {
             if (this._dimensions.some(x => x.hidden)) {
                 // remove the not wanted dimension

package/engines/dataset/TransformWorker.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/engines/dataset/example-parallel-transform.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/engines/dataset/test-parallel.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/engines/file/FileExporter.js

@@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const Constants_1 = __importDefault(require("../../Constants"));
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
@@ -42,15 +43,30 @@ class FileExporterClass {
         this.prepareBatch = (batch, options) => {
             switch (options.recordProjection.format) {
                 case 'JSON': {
-                    const content = batch.map(x => x.toJSON()).join('\n');
-                    return content;
+                    const jsonRecords = batch.map(x => x.toJSON());
+                    return this._splitIntoChunks(jsonRecords, '\n');
                 }
                 case 'CSV': {
-                    const content = batch.map(x => x.toCSV(options.recordProjection.delimiter)).join('\n');
-                    return content;
+                    const csvRecords = batch.map(x => x.toCSV(options.recordProjection.delimiter));
+                    return this._splitIntoChunks(csvRecords, '\n');
                 }
             }
         };
+        this._splitIntoChunks = (records, separator) => {
+            if (records.length === 0)
+                return [''];
+            const sampleRecord = records[0];
+            const sampleLength = sampleRecord.length + separator.length; // Include separator in calculation
+            const recordsPerChunk = Math.floor(Constants_1.default.defaults.STRING_MAX_CHARACTERS_LENGTH / sampleLength);
+            // Ensure at least 1 record per chunk
+            const chunkSize = Math.max(1, recordsPerChunk);
+            const chunks = [];
+            for (let i = 0; i < records.length; i += chunkSize) {
+                const chunk = records.slice(i, i + chunkSize);
+                chunks.push(chunk.join(separator));
+            }
+            return chunks;
+        };
         this._composeFileName = (consumer, extension) => `${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toISOString().split('.')[0], ':', '-')}.${extension}`;
     }
 }

package/helper/Runtime.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_v8_1 = __importDefault(require("node:v8"));
+class RuntimeClass {
+    constructor() {
+        this.getHeap = () => {
+            const { heap_size_limit, used_heap_size } = node_v8_1.default.getHeapStatistics();
+            return {
+                heapSizeMB: this._toMB(heap_size_limit),
+                usedHeapMB: this._toMB(used_heap_size)
+            };
+        };
+        this._toMB = (bytes) => Math.round(bytes / (1024 * 1024) * 100) / 100;
+    }
+}
+const Runtime = new RuntimeClass();
+exports.default = Runtime;

package/index.js

@@ -16,14 +16,23 @@ const create_producer_1 = require("./actions/create_producer");
 const create_consumer_1 = require("./actions/create_consumer");
 const Constants_1 = __importDefault(require("./Constants"));
 const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
+const Runtime_1 = __importDefault(require("./helper/Runtime"));
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
+// Validate the remora licence
 const remoraLicenceKey = process.env.REMORA_LICENCE_KEY;
 const check = LicenceManager_1.default.validate(remoraLicenceKey);
 if (!check.valid) {
     console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
     process.exit(1);
 }
+// Runtime check on heap size to warn user of insufficent runtime resources
+const { heapSizeMB } = Runtime_1.default.getHeap();
+if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
+    console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
+else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
+    console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
+// Initialize all commands
 program
     .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
     .description('CLI tool for setting up and managing Data-Remora');

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.57-nasco.3",
+    "version": "0.0.59-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,