@forzalabs/remora 0.0.54-nasco.3 → 0.0.56-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.54-nasco',
+    cliVersion: '0.0.56-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/engines/dataset/Dataset.js

@@ -26,14 +26,17 @@ const readline_1 = require("readline");
 const Constants_1 = __importDefault(require("../../Constants"));
 const DatasetManager_1 = __importDefault(require("./DatasetManager"));
 const DatasetRecord_1 = __importDefault(require("./DatasetRecord"));
+const DatasetRecordPool_1 = __importDefault(require("./DatasetRecordPool"));
 const xlsx_1 = __importDefault(require("xlsx"));
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const XMLParser_1 = __importDefault(require("../parsing/XMLParser"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const Environment_1 = __importDefault(require("../Environment"));
 class Dataset {
-    constructor(name, file, batchSize = Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY) {
+    constructor(name, file, batchSize) {
+        var _a;
         this._pipeline = [];
         this.getPath = () => this._path;
         this.setPath = (path) => {
@@ -44,6 +47,7 @@ class Dataset {
         this.getBatchSize = () => this._batchSize;
         this.setBatchSize = (size) => {
             this._batchSize = size;
+            this._recordPool.resize(size);
             return this;
         };
         this.getSize = () => this._size;
@@ -80,6 +84,7 @@ class Dataset {
                 const firstLine = typeof firstItem === 'object' ? JSON.stringify(firstItem) : String(firstItem);
                 const buildRes = yield DatasetManager_1.default.buildDimensionsFromFirstLine(firstLine, this._file, producer, discover);
                 this._dimensions = buildRes.dimensions;
+                this._updateRecordPoolDimensions();
                 // Clear existing file content
                 this.clear();
                 // Convert objects to DatasetRecord format and write to file
@@ -121,16 +126,17 @@ class Dataset {
             const readStream = (0, fs_2.createReadStream)(inputPath);
             const writeStream = (0, fs_2.createWriteStream)(outputPath);
             const rl = (0, readline_1.createInterface)({ input: readStream, crlfDelay: Infinity });
+            const dimensions = Algo_1.default.deepClone(this._dimensions);
             let batch = [];
             let lineCount = 0;
-            const dimensions = Algo_1.default.deepClone(this._dimensions);
             try {
                 for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
                     _c = rl_1_1.value;
                     _d = false;
                     const line = _c;
                     try {
-                        const record = new DatasetRecord_1.default(line, dimensions, this._delimiter);
+                        // Reuse record from pool and reinitialize it with new line data
+                        const record = this._recordPool.getNext(line, dimensions, this._delimiter);
                         batch.push(record);
                         lineCount++;
                         if (batch.length >= this._batchSize) {
@@ -139,6 +145,7 @@ class Dataset {
                                 writeStream.write(transformedRecord.stringify() + '\n');
                             }
                             batch = [];
+                            this._recordPool.reset(); // Reset pool index for next batch
                         }
                     }
                     catch (error) {
@@ -682,6 +689,13 @@ class Dataset {
             return this;
         });
         this.getDimensions = () => this._dimensions;
+        /**
+         * Update the record pool when dimensions change
+         */
+        this._updateRecordPoolDimensions = () => {
+            // Update all pooled records with current dimensions
+            this._recordPool.updateDimensions(this._dimensions, this._delimiter);
+        };
         /**
          * - remove dimension
          * - rename a dimension
@@ -845,13 +859,15 @@ class Dataset {
         };
         this._name = name;
         this._file = file;
-        this._batchSize = batchSize;
+        this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
         this._delimiter = ',';
         this._size = 0;
         this._iterations = 0;
         this._operations = [];
         this._pipeline = [];
+        // Initialize record pool for optimization
+        this._recordPool = new DatasetRecordPool_1.default(this._batchSize);
         const datasetName = this._name
             .replace(/[^a-zA-Z0-9_-]/g, '_')
             .replace(/_{2,}/g, '_')

package/engines/dataset/DatasetRecord.js

@@ -20,6 +20,17 @@ class DatasetRecord {
         this.getRaw = () => this._row;
         this.getValue = (dimension) => this._value[dimension];
         this.setValue = (dimension, value) => this._value[dimension] = value;
+        /**
+         * Reinitialize the record with new data instead of creating a new instance
+         * This is used for object pooling optimization
+         */
+        this.reinitialize = (row, dimensions, delimiter) => {
+            this._row = row;
+            this._dimensions = dimensions;
+            this._delimiter = delimiter;
+            this._value = {};
+            this.parse(row, delimiter, this._dimensions);
+        };
         this.wholeUpdateDimension = (update) => {
             var _a;
             if (update.toDelete) {
@@ -35,7 +46,9 @@ class DatasetRecord {
             }
             else {
                 // Change: name, hidden, position
-                const index = this._dimensions.findIndex(x => x.key === update.currentDimension.name);
+                let index = this._dimensions.findIndex(x => x.key === update.currentDimension.name);
+                if (index < 0)
+                    index = this._dimensions.findIndex(x => x.key === update.currentDimension.key);
                 const currentDim = this._dimensions[index];
                 const updatedDim = { name: update.newName, key: (_a = currentDim.key) !== null && _a !== void 0 ? _a : update.newName, hidden: update.newHidden, index: update.newPosition };
                 this._value[updatedDim.name] = this._value[currentDim.name];

package/engines/dataset/DatasetRecordPool.js

@@ -0,0 +1,77 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const DatasetRecord_1 = __importDefault(require("./DatasetRecord"));
+/**
+ * A pool of DatasetRecord objects to optimize memory allocation during batch processing
+ */
+class DatasetRecordPool {
+    constructor(poolSize) {
+        /**
+         * Initialize the pool with empty DatasetRecord objects
+         */
+        this._initializePool = () => {
+            this._pool = [];
+            for (let i = 0; i < this._poolSize; i++) {
+                this._pool.push(new DatasetRecord_1.default('', [], ','));
+            }
+            this._poolIndex = 0;
+        };
+        /**
+         * Get the next available record from the pool and reinitialize it with new data
+         * @param line The raw line data
+         * @param dimensions The dataset dimensions
+         * @param delimiter The delimiter to use
+         * @returns A reinitialized DatasetRecord from the pool
+         */
+        this.getNext = (line, dimensions, delimiter) => {
+            const record = this._pool[this._poolIndex];
+            record.reinitialize(line, dimensions, delimiter);
+            this._poolIndex = (this._poolIndex + 1) % this._poolSize;
+            return record;
+        };
+        /**
+         * Reset the pool index to start from the beginning
+         * This should be called when starting a new batch
+         */
+        this.reset = () => {
+            this._poolIndex = 0;
+        };
+        /**
+         * Update the pool size and reinitialize if necessary
+         * @param newSize The new pool size
+         */
+        this.resize = (newSize) => {
+            if (newSize !== this._poolSize) {
+                this._poolSize = newSize;
+                this._initializePool();
+            }
+        };
+        /**
+         * Update all pooled records with new dimensions and delimiter
+         * This should be called when dataset dimensions change
+         * @param dimensions The new dimensions
+         * @param delimiter The new delimiter
+         */
+        this.updateDimensions = (dimensions, delimiter) => {
+            for (const record of this._pool) {
+                record.reinitialize('', dimensions, delimiter);
+            }
+        };
+        /**
+         * Get the current pool size
+         */
+        this.getSize = () => this._poolSize;
+        /**
+         * Get the current pool index
+         */
+        this.getCurrentIndex = () => this._poolIndex;
+        this._poolSize = poolSize;
+        this._poolIndex = 0;
+        this._pool = [];
+        this._initializePool();
+    }
+}
+exports.default = DatasetRecordPool;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.54-nasco.3",
+    "version": "0.0.56-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,