@forzalabs/remora 0.0.63-nasco.3 → 0.1.2-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.63-nasco',
+    cliVersion: '0.1.2-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {
@@ -10,7 +10,9 @@ const CONSTANTS = {
         STRING_MAX_CHARACTERS_LENGTH: 10000000,
         MAX_ITEMS_IN_MEMORY: 200000,
         MIN_RUNTIME_HEAP_MB: 4000,
-        RECOMMENDED_RUNTIME_HEAP_MB: 8000
+        RECOMMENDED_RUNTIME_HEAP_MB: 8000,
+        INDICATIVE_THREAD_LINE_COUNT: 750000,
+        MAX_THREAD_COUNT: 8
     }
 };
 exports.default = CONSTANTS;

package/drivers/DriverHelper.js

@@ -38,6 +38,7 @@ const DriverHelper = {
         let hasValidatedHeader = shouldValidateHeader ? false : true;
         let leftoverData = '';
         let globalIndex = 0;
+        let lineCount = 0;
         const headerValidationTransform = new stream_1.Transform({
             transform(chunk, encoding, callback) {
                 const chunkStr = leftoverData + chunk.toString();
@@ -99,6 +100,7 @@ const DriverHelper = {
             return true;
         };
         const processLine = (line) => {
+            lineCount++;
             switch (fileType) {
                 case 'JSON':
                 case 'JSONL': {
@@ -118,6 +120,7 @@ const DriverHelper = {
         const writeOptions = append ? { flags: 'a' } : {};
         const writeStream = (0, fs_1.createWriteStream)(destinationPath, writeOptions);
         yield (0, promises_1.pipeline)(stream, headerValidationTransform, writeStream);
+        return lineCount;
     }),
     quickReadFile: (filePath, lineCount) => __awaiter(void 0, void 0, void 0, function* () {
         var _a, e_1, _b, _c;

package/drivers/LocalDriver.js

@@ -146,17 +146,20 @@ class LocalSourceDriver {
                 // Get header line from the first file
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, allFileKeys[0]), 1))[0];
                 dataset.setFirstLine(headerLine);
+                let totalLineCount = 0;
                 // Copy files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    totalLineCount += yield copyLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
+                dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
                 // For single file, no header validation needed
                 const headerLine = (yield DriverHelper_1.default.quickReadFile(path_1.default.join(this._path, fileKey), 1))[0];
                 dataset.setFirstLine(headerLine);
-                yield copyLocally(fileKey, headerLine, false);
+                const totalLineCount = yield copyLocally(fileKey, headerLine, false);
+                dataset.setCount(totalLineCount);
                 return dataset;
             }
         });

package/drivers/S3Driver.js

@@ -226,10 +226,12 @@ class S3SourceDriver {
                 const firstFileStream = firstFileResponse.Body;
                 const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 dataset.setFirstLine(headerLine);
+                let totalLineCount = 0;
                 // Download files sequentially to avoid file conflicts
                 for (let i = 0; i < allFileKeys.length; i++) {
-                    yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
+                    totalLineCount += yield downloadLocally(allFileKeys[i], headerLine, i > 0); // Append mode for subsequent files
                 }
+                dataset.setCount(totalLineCount);
                 return dataset;
             }
             else {
@@ -244,7 +246,8 @@ class S3SourceDriver {
                 const headerLine = yield this.getFirstLineFromStream(firstFileStream);
                 dataset.setFirstLine(headerLine);
                 // For single file, no header validation needed
-                yield downloadLocally(fileKey, headerLine);
+                const totalLineCount = yield downloadLocally(fileKey, headerLine);
+                dataset.setCount(totalLineCount);
                 return dataset;
             }
         });

package/engines/consumer/PostProcessor.js

@@ -31,7 +31,7 @@ class PostProcessorClass {
          * - default field values
          * - masking/hashing of data
          */
-        this.doProjection = (consumer, dataset) => __awaiter(this, void 0, void 0, function* () {
+        this.doProjection = (consumer, dataset, options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
@@ -61,7 +61,7 @@ class PostProcessorClass {
                         record.setValue(fieldKey, TypeCaster_1.default.cast(fieldValue, fieldType));
                 }
                 return record;
-            });
+            }, options);
             newDataset.setDimensinons(updatedDimensions);
             return newDataset;
         });

package/engines/dataset/Dataset.js

@@ -49,15 +49,24 @@ class Dataset {
             return this;
         };
         this.getFirstLine = () => this._firstLine;
-        this.getSize = () => this._size;
+        this.getCount = () => this._count;
+        this.setCount = (count) => {
+            this._count = count;
+            return this;
+        };
         this.getCycles = () => this._iterations;
         this.getDelimiter = () => this._delimiter;
+        this.setDelimiter = (delimiter) => {
+            this._delimiter = delimiter;
+            return this;
+        };
         this.getOperations = () => this._operations;
         this.load = (source) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(source, 'Invalid source');
             this._startOperation('load', { source: source.engine });
             const driver = yield DriverFactory_1.default.instantiateSource(source);
             yield driver.download(this);
+            this._size = this._computeSize();
             this._finishOperation('load');
             return this;
         });
@@ -101,7 +110,8 @@ class Dataset {
                 }
                 // Write all records to the dataset file
                 yield this.append(records);
-                this._size = data.length;
+                this._size = this._computeSize();
+                this._count = data.length;
                 this._finishOperation('load-from-memory');
                 return this;
             }
@@ -115,12 +125,14 @@ class Dataset {
          */
         this.transformStream = (transformer_1, ...args_1) => __awaiter(this, [transformer_1, ...args_1], void 0, function* (transformer, options = {}) {
             var _a, e_1, _b, _c;
+            var _d, _e, _f, _g;
             const inputPath = options.inputPath || this._path;
             const outputPath = options.outputPath || this._tempPath;
+            const fromLine = (_e = (_d = options.range) === null || _d === void 0 ? void 0 : _d.fromLine) !== null && _e !== void 0 ? _e : -1;
+            const toLine = (_g = (_f = options.range) === null || _f === void 0 ? void 0 : _f.toLine) !== null && _g !== void 0 ? _g : Infinity;
             this.ensureFile(outputPath);
-            if (!fs_1.default.existsSync(inputPath)) {
+            if (!fs_1.default.existsSync(inputPath))
                 throw new Error(`Input file does not exist: ${inputPath}`);
-            }
             this._startOperation('transform-stream');
             const readStream = (0, fs_2.createReadStream)(inputPath);
             const writeStream = (0, fs_2.createWriteStream)(outputPath);
@@ -128,12 +140,22 @@ class Dataset {
             const dimensions = Algo_1.default.deepClone(this._dimensions);
             let batch = [];
             let lineCount = 0;
+            let index = 0;
             try {
-                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                for (var _h = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _h = true) {
                     _c = rl_1_1.value;
-                    _d = false;
+                    _h = false;
                     const line = _c;
                     try {
+                        if (index < fromLine) {
+                            index++;
+                            continue;
+                        }
+                        else if (index >= toLine) {
+                            index++;
+                            break;
+                        }
+                        index++;
                         // Reuse record from pool and reinitialize it with new line data
                         const record = this._recordPool.getNext(line, dimensions, this._delimiter);
                         batch.push(record);
@@ -144,7 +166,7 @@ class Dataset {
                                 writeStream.write(transformedRecord.stringify() + '\n');
                             }
                             batch = [];
-                            this._recordPool.reset(); // Reset pool index for next batch
+                            this._recordPool.reset();
                         }
                     }
                     catch (error) {
@@ -155,7 +177,7 @@ class Dataset {
             catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                    if (!_h && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
                 }
                 finally { if (e_1) throw e_1.error; }
             }
@@ -176,14 +198,15 @@ class Dataset {
             if (outputPath === this._tempPath) {
                 fs_1.default.renameSync(this._tempPath, this._path);
             }
-            this._size = lineCount;
+            this._count = lineCount;
+            this._size = this._computeSize();
             this._iterations++;
             this._finishOperation('transform-stream');
         });
         /**
          * Filter items in the file using batched streaming
          */
-        this.filter = (predicate) => __awaiter(this, void 0, void 0, function* () {
+        this.filter = (predicate_1, ...args_1) => __awaiter(this, [predicate_1, ...args_1], void 0, function* (predicate, options = {}) {
             this._startOperation('filter');
             let globalIndex = 0;
             yield this.transformStream((batch) => __awaiter(this, void 0, void 0, function* () {
@@ -195,7 +218,7 @@ class Dataset {
                     globalIndex++;
                 }
                 return filteredBatch;
-            }));
+            }), options);
             this._finishOperation('filter');
             return this;
         });
@@ -617,7 +640,6 @@ class Dataset {
             }
             rl.close();
             readStream.close();
-            this._size = lineCount;
             this._finishOperation('read-lines');
             return results;
         });
@@ -628,7 +650,14 @@ class Dataset {
          */
         this.prepare = (producer) => __awaiter(this, void 0, void 0, function* () {
             this._startOperation('prepare');
-            const { delimiter, dimensions } = yield DatasetManager_1.default.buildDimensions(this, producer);
+            const dimsRes = yield DatasetManager_1.default.buildDimensions(this, producer);
+            yield this.prepareWithDimensions(dimsRes);
+            this._finishOperation('prepare');
+            return this;
+        });
+        this.prepareWithDimensions = (dimResult) => __awaiter(this, void 0, void 0, function* () {
+            this._startOperation('prepare-with-dimensions');
+            const { delimiter, dimensions } = dimResult;
             this._delimiter = delimiter;
             this._dimensions = dimensions;
             switch (this._file.fileType) {
@@ -670,7 +699,7 @@ class Dataset {
                     break;
                 }
             }
-            this._finishOperation('prepare');
+            this._finishOperation('prepare-with-dimensions');
             return this;
         });
         this.getDimensions = () => this._dimensions;
@@ -686,7 +715,7 @@ class Dataset {
             this._recordPool.updateDimensions(this._dimensions, this._delimiter);
         };
         this.print = (...args_1) => __awaiter(this, [...args_1], void 0, function* (count = 3, full = false) {
-            console.log(`DS ${this._name} (${this._size} | ${this._iterations})`);
+            console.log(`DS ${this.name} (${this._count} | ${this._iterations})`);
             console.log(this._dimensions.map(x => x.name).join(this._delimiter));
             const records = yield this.readLines(count);
             records.forEach(x => console.log(full ? x : x.stringify()));
@@ -695,7 +724,7 @@ class Dataset {
         this.printStats = () => {
             var _a, _b;
             const total = ((_b = (_a = this._operations) === null || _a === void 0 ? void 0 : _a.map(x => x.elapsedMs)) !== null && _b !== void 0 ? _b : []).reduce((sum, ms) => sum + ms, 0);
-            console.log(`DS[stats] ${this._name} (size: ${this._size} | cycles: ${this._iterations} | ms: ${Helper_1.default.formatDuration(total)})`);
+            console.log(`DS[stats] ${this.name} (size: ${this._count} | cycles: ${this._iterations} | ms: ${Helper_1.default.formatDuration(total)})`);
             console.log(`Operations: ${this._operations.length}`);
             console.log(JSON.stringify(this._operations, null, 4));
         };
@@ -760,7 +789,7 @@ class Dataset {
             const finishedOperation = this._findRunningOperation(name);
             if (finishedOperation) {
                 finishedOperation.status = 'completed';
-                finishedOperation.count = this._size;
+                finishedOperation.count = this._count;
                 finishedOperation.elapsedMs = performance.now() - finishedOperation.elapsedMs;
             }
             else {
@@ -797,18 +826,19 @@ class Dataset {
             };
             return searchInOperations(this._operations);
         };
-        this._name = name;
+        this._computeSize = () => fs_1.default.statSync(this._path).size / (1024 * 1024);
+        this.name = name;
         this._file = file;
         this._batchSize = (_a = batchSize !== null && batchSize !== void 0 ? batchSize : parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
         this._dimensions = [];
         this._firstLine = '';
         this._delimiter = ',';
-        this._size = 0;
+        this._count = 0;
         this._iterations = 0;
         this._operations = [];
         // Initialize record pool for optimization
         this._recordPool = new DatasetRecordPool_1.default(this._batchSize);
-        const datasetName = this._name
+        const datasetName = this.name
             .replace(/[^a-zA-Z0-9_-]/g, '_')
             .replace(/_{2,}/g, '_')
             .replace(/^_+|_+$/g, '')

package/engines/dataset/DatasetManager.js

@@ -19,6 +19,8 @@ const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const Dataset_1 = __importDefault(require("./Dataset"));
+const promises_1 = require("stream/promises");
+const fs_1 = require("fs");
 class DatasetManagerClass {
     constructor() {
         this.create = (producer) => {
@@ -29,7 +31,7 @@ class DatasetManagerClass {
                 fileType,
                 hasHeaderRow,
                 sheetName,
-                delimiter: delimiter
+                delimiter
             });
             return dataset;
         };
@@ -125,7 +127,7 @@ class DatasetManagerClass {
                 const { cField } = fields[i];
                 const currentMatch = structuredClone(dimensions.find(x => x.name === cField.key));
                 if (!currentMatch && !cField.fixed)
-                    throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying dataset "${dataset['_name']}" (${dimensions.map(x => x.name).join(', ')})`);
+                    throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying dataset "${dataset.name}" (${dimensions.map(x => x.name).join(', ')})`);
                 updates.push({
                     currentDimension: currentMatch,
                     newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
@@ -147,6 +149,24 @@ class DatasetManagerClass {
                     || x.newPosition !== x.currentDimension.index)));
             return updates;
         };
+        /**
+         * Each worker threads writes to his own dataset file to avoid concurrency and data loss,
+         * at the end of their work, I merge their results to a single file
+         */
+        this.mergeWorkersPaths = (threadPaths, dataset) => __awaiter(this, void 0, void 0, function* () {
+            dataset.clear();
+            const datasetPath = dataset.getPath();
+            for (let i = 0; i < threadPaths.length; i++) {
+                const path = threadPaths[i];
+                const readStream = (0, fs_1.createReadStream)(path);
+                // For the first file, create a new write stream
+                // For subsequent files, append to the existing file
+                const writeStream = (0, fs_1.createWriteStream)(datasetPath, { flags: i === 0 ? 'w' : 'a' });
+                yield (0, promises_1.pipeline)(readStream, writeStream);
+                (0, fs_1.unlinkSync)(path);
+            }
+            return dataset;
+        });
     }
 }
 const DatasetManager = new DatasetManagerClass();

package/engines/dataset/ParallelDataset.js

@@ -0,0 +1,158 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Constants_1 = __importDefault(require("../../Constants"));
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const workerpool_1 = __importDefault(require("workerpool"));
+const DatasetManager_1 = __importDefault(require("./DatasetManager"));
+const path_1 = __importDefault(require("path"));
+class ParallelDatasetClass {
+    constructor() {
+        this._getWorkerPath = () => {
+            // Get the current file's directory
+            const currentDir = __dirname;
+            // Check if we're already in the .build directory (production)
+            if (currentDir.includes('.build')) {
+                // We're in production (.build/engines/dataset), go to .build/workers
+                const buildDir = currentDir.split('.build')[0] + '.build';
+                return path_1.default.join(buildDir, 'workers');
+            }
+            else {
+                // We're in development, workers are in ./.build/workers
+                return path_1.default.resolve('./.build/workers');
+            }
+        };
+        this._scopeWork = (dataset) => {
+            var _a;
+            const datasetCount = dataset.getCount();
+            const batchSize = (_a = parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
+            const workerChunkSize = batchSize * Math.round(Constants_1.default.defaults.INDICATIVE_THREAD_LINE_COUNT / batchSize);
+            const workerCount = Math.min(Math.ceil(datasetCount / workerChunkSize), Constants_1.default.defaults.MAX_THREAD_COUNT);
+            const adjustedWorkerCount = Math.ceil(datasetCount / workerCount);
+            return { workerCount, adjustedWorkerCount };
+        };
+        this.filter = (dataset, filters) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(dataset, `Invalid dataset`);
+            (0, Affirm_1.default)(filters, `Invalid filters`);
+            // Distribute the work of the filter among the various workers, trying to have them match the batch size
+            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
+            dataset._startOperation('filter-parallel', { workerCount });
+            const threads = [];
+            for (let i = 0; i < workerCount; i++) {
+                const workerId = `worker_filter_${i}`;
+                const fromLine = adjustedWorkerCount * i;
+                const toLine = (i === workerCount - 1)
+                    ? Infinity
+                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
+                const workerData = {
+                    datasetDimensions: dataset.getDimensions(),
+                    datasetFile: dataset.getFile(),
+                    datasetName: dataset.name,
+                    datasetDelimiter: dataset.getDelimiter(),
+                    fromLine: fromLine,
+                    toLine: toLine,
+                    workerId: workerId,
+                    filterData: {
+                        rules: filters
+                    }
+                };
+                threads.push(this._filterPool.exec('filter', [workerData]));
+            }
+            const results = yield Promise.all(threads);
+            yield this._filterPool.terminate();
+            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: filter ${dataset.name}`);
+            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
+            dataset
+                .setDelimiter(results[0].datasetDelimiter)
+                .setDimensinons(results[0].datasetDimensions);
+            dataset._finishOperation('filter-parallel');
+            return dataset;
+        });
+        this.projection = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(dataset, `Invalid dataset`);
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
+            dataset._startOperation('projection-parallel', { workerCount });
+            const threads = [];
+            for (let i = 0; i < workerCount; i++) {
+                const workerId = `worker_projection_${i}`;
+                const fromLine = adjustedWorkerCount * i;
+                const toLine = (i === workerCount - 1)
+                    ? Infinity
+                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
+                const workerData = {
+                    datasetDimensions: dataset.getDimensions(),
+                    datasetFile: dataset.getFile(),
+                    datasetName: dataset.name,
+                    datasetDelimiter: dataset.getDelimiter(),
+                    fromLine: fromLine,
+                    toLine: toLine,
+                    workerId: workerId,
+                    projectionData: { consumerName: consumer.name }
+                };
+                threads.push(this._projectionPool.exec('projection', [workerData]));
+            }
+            const results = yield Promise.all(threads);
+            yield this._projectionPool.terminate();
+            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: projection ${dataset.name}`);
+            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
+            dataset
+                .setDelimiter(results[0].datasetDelimiter)
+                .setDimensinons(results[0].datasetDimensions);
+            dataset._finishOperation('projection-parallel');
+            return dataset;
+        });
+        this.transform = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(dataset, `Invalid dataset`);
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
+            dataset._startOperation('transform-parallel', { workerCount });
+            const threads = [];
+            for (let i = 0; i < workerCount; i++) {
+                const workerId = `worker_transform_${i}`;
+                const fromLine = adjustedWorkerCount * i;
+                const toLine = (i === workerCount - 1)
+                    ? Infinity
+                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
+                const workerData = {
+                    datasetDimensions: dataset.getDimensions(),
+                    datasetFile: dataset.getFile(),
+                    datasetName: dataset.name,
+                    datasetDelimiter: dataset.getDelimiter(),
+                    fromLine: fromLine,
+                    toLine: toLine,
+                    workerId: workerId,
+                    transformData: { consumerName: consumer.name }
+                };
+                threads.push(this._transformPool.exec('transform', [workerData]));
+            }
+            const results = yield Promise.all(threads);
+            yield this._transformPool.terminate();
+            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: projection ${dataset.name}`);
+            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
+            dataset
+                .setDelimiter(results[0].datasetDelimiter)
+                .setDimensinons(results[0].datasetDimensions);
+            dataset._finishOperation('transform-parallel');
+            return dataset;
+        });
+        const workerPath = this._getWorkerPath();
+        this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'));
+        this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'));
+        this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'));
+    }
+}
+const ParallelDataset = new ParallelDatasetClass();
+exports.default = ParallelDataset;

package/engines/execution/ExecutionEnvironment.js

@@ -21,12 +21,12 @@ const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
 const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
 const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
 const RequestExecutor_1 = __importDefault(require("./RequestExecutor"));
-const TransformationEngine_1 = __importDefault(require("../transform/TransformationEngine"));
 const JoinEngine_1 = __importDefault(require("../transform/JoinEngine"));
 const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
 const Environment_1 = __importDefault(require("../Environment"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
+const ParallelDataset_1 = __importDefault(require("../dataset/ParallelDataset"));
 class ExecutionEnvironment {
     constructor(consumer) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
@@ -95,7 +95,7 @@ class ExecutionEnvironment {
                         }
                         case 'post-process-json': {
                             const dataset = this._getIntermidiate(planStep);
-                            const newDataset = yield PostProcessor_1.default.doProjection(this._consumer, dataset);
+                            const newDataset = yield ParallelDataset_1.default.projection(dataset, this._consumer);
                             this._storeIntermidiate(planStep, newDataset);
                             break;
                         }
@@ -111,12 +111,11 @@ class ExecutionEnvironment {
                             break;
                         }
                         case 'apply-consumer-filters-on-JSON': {
-                            const rules = this._consumer.filters.map(x => x.rule);
-                            this._resultingDataset = yield RequestExecutor_1.default.applyFilters(this._resultingDataset, rules);
+                            this._resultingDataset = yield ParallelDataset_1.default.filter(this._resultingDataset, this._consumer.filters);
                             break;
                         }
                         case 'apply-transformations': {
-                            this._resultingDataset = yield TransformationEngine_1.default.apply(this._consumer, this._resultingDataset);
+                            this._resultingDataset = yield ParallelDataset_1.default.transform(this._resultingDataset, this._consumer);
                             break;
                         }
                         case 'join-producers-data': {
@@ -134,7 +133,7 @@ class ExecutionEnvironment {
                                 cycles: this._resultingDataset.getCycles(),
                                 elapsedMS: performance.now() - start,
                                 operations: structuredClone(this._resultingDataset.getOperations()),
-                                size: this._resultingDataset.getSize()
+                                size: this._resultingDataset.getCount()
                             };
                             break;
                         }
@@ -155,7 +154,7 @@ class ExecutionEnvironment {
             catch (error) {
                 const ds = (_c = this._resultingDataset) !== null && _c !== void 0 ? _c : (_d = this._producedData.at(-1)) === null || _d === void 0 ? void 0 : _d.dataset;
                 if (ds)
-                    Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getSize()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
+                    Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getCount()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
                 Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
                 throw error;
             }

package/engines/execution/RequestExecutor.js

@@ -27,10 +27,10 @@ class RequestExecutorClass {
                 dataset = yield this._applyOrdering(dataset, request.order);
             return dataset;
         });
-        this.applyFilters = (dataset, filters) => __awaiter(this, void 0, void 0, function* () {
+        this.applyFilters = (dataset_1, filters_1, ...args_1) => __awaiter(this, [dataset_1, filters_1, ...args_1], void 0, function* (dataset, filters, options = {}) {
             return yield dataset.filter(record => {
                 return filters.every(filter => this._evaluateFilter(record, filter));
-            });
+            }, options);
         });
         this._evaluateFilter = (record, filter) => {
             const evaluate = (baseRecord, baseFilter) => {

package/engines/transform/JoinEngine.js

@@ -142,7 +142,7 @@ class JoinEngineClass {
             for (const prodData of otherProducedData) {
                 const prodDimKey = getDimensionsKey(prodData.dataset);
                 if (mainDimKey !== prodDimKey)
-                    throw new Error(`On consumer "${consumer.name}", can't union the dataset "${prodData.dataset['_name']}" (producer: ${prodData.producerKey}) because the dimensions are different from the main dataset "${mainDataset['_name']}" (producer: ${producedData[0].producerKey}). "${mainDimKey}" != "${prodDimKey}"`);
+                    throw new Error(`On consumer "${consumer.name}", can't union the dataset "${prodData.dataset.name}" (producer: ${prodData.producerKey}) because the dimensions are different from the main dataset "${mainDataset.name}" (producer: ${producedData[0].producerKey}). "${mainDimKey}" != "${prodDimKey}"`);
                 yield prodData.dataset.streamBatches((batch) => __awaiter(this, void 0, void 0, function* () {
                     yield mainDataset.append(batch);
                 }));

package/engines/transform/TransformationEngine.js

@@ -17,7 +17,7 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
 const TypeCaster_1 = __importDefault(require("./TypeCaster"));
 class TransformationEngineClass {
     constructor() {
-        this.apply = (consumer, dataset) => __awaiter(this, void 0, void 0, function* () {
+        this.apply = (consumer, dataset, options) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(dataset, 'Invalid data');
             const fieldsToTransform = consumer.fields.filter(field => Algo_1.default.hasVal(field.transform));
@@ -53,7 +53,7 @@ class TransformationEngineClass {
                     }
                 }
                 return record;
-            });
+            }, options);
         });
         this.isFieldCombinationTransformation = (transformation) => {
             if (Array.isArray(transformation)) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.63-nasco.3",
+    "version": "0.1.2-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -9,7 +9,7 @@
     },
     "scripts": {
         "sync": "cd ../dev_ops && npm run sync",
-        "dev": "clear && npx tsx scripts/threaded.ts",
+        "dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
         "tsc-check": "npx tsc --noemit",
         "init": "npx tsx ./src/index.ts init",
         "version": "npx tsx ./src/index.ts -v",
@@ -21,6 +21,7 @@
         "create-producer": "npx tsx ./src/index.ts create-producer",
         "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
         "build": "npm i && npm run sync && tsc --outDir .build && npm run copy-static-file",
+        "fast-build": "tsc --outDir .build",
         "upload": "npm run build && cd .build && npm publish --tag nasco --access=public"
     },
     "keywords": [
@@ -55,6 +56,7 @@
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "seedrandom": "^3.0.5",
+        "workerpool": "^9.3.3",
         "xlsx": "^0.18.5",
         "zod": "^3.24.2"
     },

package/workers/FilterWorker.js

@@ -0,0 +1,62 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workerpool_1 = __importDefault(require("workerpool"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Dataset_1 = __importDefault(require("../engines/dataset/Dataset"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const RequestExecutor_1 = __importDefault(require("../engines/execution/RequestExecutor"));
+dotenv_1.default.configDotenv();
+const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
+    Environment_1.default.load('./');
+    try {
+        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, filterData: filter } = workerData;
+        Affirm_1.default.hasValue(fromLine, `Invalid from line`);
+        Affirm_1.default.hasValue(toLine, `Invalid to line`);
+        (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
+        (0, Affirm_1.default)(workerId, `Invalid worker id`);
+        (0, Affirm_1.default)(datasetFile, `Invalid dataset file`);
+        (0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
+        (0, Affirm_1.default)(filter, `Invalid filter data`);
+        (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
+        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        dataset
+            .setDimensinons(datasetDimensions)
+            .setDelimiter(datasetDelimiter);
+        const outputPath = dataset['_tempPath'] + workerId;
+        const rules = filter.rules.map(x => x.rule);
+        yield RequestExecutor_1.default.applyFilters(dataset, rules, { outputPath, range: { fromLine, toLine } });
+        const result = {
+            success: true,
+            datasetDelimiter: dataset.getDelimiter(),
+            datasetDimensions: dataset.getDimensions(),
+            datasetFile: dataset.getFile(),
+            datasetName: dataset.name,
+            datasetPath: outputPath
+        };
+        return result;
+    }
+    catch (error) {
+        console.error(error);
+        const result = {
+            success: false,
+            error
+        };
+        return result;
+    }
+});
+workerpool_1.default.worker({
+    filter: run
+});

package/workers/ProjectionWorker.js

@@ -0,0 +1,63 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workerpool_1 = __importDefault(require("workerpool"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Dataset_1 = __importDefault(require("../engines/dataset/Dataset"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const PostProcessor_1 = __importDefault(require("../engines/consumer/PostProcessor"));
+dotenv_1.default.configDotenv();
+const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
+    Environment_1.default.load('./');
+    try {
+        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, projectionData } = workerData;
+        Affirm_1.default.hasValue(fromLine, `Invalid from line`);
+        Affirm_1.default.hasValue(toLine, `Invalid to line`);
+        (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
+        (0, Affirm_1.default)(workerId, `Invalid worker id`);
+        (0, Affirm_1.default)(datasetFile, `Invalid dataset file`);
+        (0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
+        (0, Affirm_1.default)(projectionData, `Invalid projection data`);
+        (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
+        const consumer = Environment_1.default.getConsumer(projectionData.consumerName);
+        (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${projectionData.consumerName}" not found.`);
+        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        dataset
+            .setDimensinons(datasetDimensions)
+            .setDelimiter(datasetDelimiter);
+        const outputPath = dataset['_tempPath'] + workerId;
+        yield PostProcessor_1.default.doProjection(consumer, dataset, { outputPath, range: { fromLine, toLine } });
+        const result = {
+            success: true,
+            datasetDelimiter: dataset.getDelimiter(),
+            datasetDimensions: dataset.getDimensions(),
+            datasetFile: dataset.getFile(),
+            datasetName: dataset.name,
+            datasetPath: outputPath
+        };
+        return result;
+    }
+    catch (error) {
+        console.error(error);
+        const result = {
+            success: false,
+            error
+        };
+        return result;
+    }
+});
+workerpool_1.default.worker({
+    projection: run
+});

package/workers/TransformWorker.js

@@ -0,0 +1,63 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workerpool_1 = __importDefault(require("workerpool"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Dataset_1 = __importDefault(require("../engines/dataset/Dataset"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const TransformationEngine_1 = __importDefault(require("../engines/transform/TransformationEngine"));
+dotenv_1.default.configDotenv();
+const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
+    Environment_1.default.load('./');
+    try {
+        const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, transformData } = workerData;
+        Affirm_1.default.hasValue(fromLine, `Invalid from line`);
+        Affirm_1.default.hasValue(toLine, `Invalid to line`);
+        (0, Affirm_1.default)(datasetName, `Invalid dataset name`);
+        (0, Affirm_1.default)(workerId, `Invalid worker id`);
+        (0, Affirm_1.default)(datasetFile, `Invalid dataset file`);
+        (0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
+        (0, Affirm_1.default)(transformData, `Invalid transform data`);
+        (0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
+        const consumer = Environment_1.default.getConsumer(transformData.consumerName);
+        (0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${transformData.consumerName}" not found.`);
+        const dataset = new Dataset_1.default(datasetName, datasetFile);
+        dataset
+            .setDimensinons(datasetDimensions)
+            .setDelimiter(datasetDelimiter);
+        const outputPath = dataset['_tempPath'] + workerId;
+        yield TransformationEngine_1.default.apply(consumer, dataset, { outputPath, range: { fromLine, toLine } });
+        const result = {
+            success: true,
+            datasetDelimiter: dataset.getDelimiter(),
+            datasetDimensions: dataset.getDimensions(),
+            datasetFile: dataset.getFile(),
+            datasetName: dataset.name,
+            datasetPath: outputPath
+        };
+        return result;
+    }
+    catch (error) {
+        console.error(error);
+        const result = {
+            success: false,
+            error
+        };
+        return result;
+    }
+});
+workerpool_1.default.worker({
+    transform: run
+});

package/workers/TsWorker.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const workerpool_1 = __importDefault(require("workerpool"));
+const DatasetRecord_1 = __importDefault(require("../engines/dataset/DatasetRecord"));
+workerpool_1.default.worker({
+    ts: () => {
+        const tt = new DatasetRecord_1.default('bububub,bububbu', [{ hidden: false, index: 0, key: '11', name: '11' }], ',');
+        console.log(tt);
+        console.log('hello form typescript', tt.stringify());
+    }
+});

package/workers/definitions.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });