@forzalabs/remora 0.2.6 → 1.0.0

package/executors/ConsumerExecutor.js

@@ -0,0 +1,152 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const readline_1 = __importDefault(require("readline"));
+const promises_1 = __importDefault(require("fs/promises"));
+const TransformationEngine_1 = __importDefault(require("../engines/transform/TransformationEngine"));
+const RequestExecutor_1 = __importDefault(require("../engines/execution/RequestExecutor"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Constants_1 = __importDefault(require("../Constants"));
+class ConsumerExecutorClass {
+    constructor() {
+        this._getWorkPath = (consumer, executionId) => {
+            const execFolder = path_1.default.join(consumer.name, executionId);
+            const workPath = path_1.default.join('./remora', Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, execFolder, '.dataset');
+            return workPath;
+        };
+        this._clearWorkPath = (workPath) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                if (fs_1.default.existsSync(workPath)) {
+                    yield promises_1.default.unlink(workPath);
+                }
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            }
+            catch (error) {
+                // Ignore file deletion errors
+            }
+            try {
+                const dir = path_1.default.dirname(workPath);
+                if (fs_1.default.existsSync(dir)) {
+                    yield promises_1.default.rmdir(dir);
+                }
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            }
+            catch (error) {
+                // Ignore directory deletion errors
+            }
+        });
+        this._ensurePath = (pathUri) => {
+            // make sure that the workpath exists
+            const dir = path_1.default.dirname(pathUri);
+            if (!fs_1.default.existsSync(dir))
+                fs_1.default.mkdirSync(dir, { recursive: true });
+            if (!fs_1.default.existsSync(pathUri))
+                fs_1.default.writeFileSync(pathUri, '');
+        };
+        this.ready = (consumer, executionId) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const workPath = this._getWorkPath(consumer, executionId);
+            this._ensurePath(workPath);
+            return fs_1.default.createWriteStream(workPath);
+        };
+        this.processRecord = (options) => {
+            var _a;
+            const { consumer, fields, dimensions, producer, record, requestOptions } = options;
+            // map to consumer fields and apply consumer field logic
+            for (const field of fields) {
+                const { cField } = field;
+                const fieldKey = (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key;
+                // set the fixed default value for the field, or throw error if not present in the producer
+                const dimension = dimensions.find(x => x.name === cField.key);
+                if (!dimension) {
+                    if (cField.fixed && cField.default)
+                        record[fieldKey] = cField.default;
+                    else
+                        throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map(x => x.name).join(', ')})`);
+                }
+                // change the name of the dimension
+                if (cField.alias && cField.alias !== dimension.name) {
+                    record[cField.alias] = record[dimension.name];
+                    delete record[dimension.name];
+                }
+                // apply transformations
+                if (cField.transform)
+                    record[fieldKey] = TransformationEngine_1.default.applyTransformations(record[fieldKey], cField.transform, cField, record);
+            }
+            // remove un-wanted producer dimensions
+            for (const dimension of dimensions) {
+                const field = fields.find(x => x.cField.key === dimension.name);
+                if (!field)
+                    delete record[dimension.name];
+            }
+            // apply consumer filters
+            if (consumer.filters && consumer.filters.length > 0) {
+                const isKept = consumer.filters.every(x => RequestExecutor_1.default.evaluateFilter(record, x.rule));
+                if (!isKept)
+                    return null;
+            }
+            // apply request custom filters
+            if (requestOptions && requestOptions.filters) {
+                const isKept = requestOptions.filters.every(x => RequestExecutor_1.default.evaluateFilter(record, x));
+                if (!isKept)
+                    return null;
+            }
+            return record;
+        };
+        this.processDistinct = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            const workPath = this._getWorkPath(consumer, executionId);
+            const reader = fs_1.default.createReadStream(workPath);
+            const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
+            const tempWorkPath = workPath + '_tmp';
+            const writer = fs_1.default.createWriteStream(tempWorkPath);
+            let newLineCount = 0;
+            const seen = new Set();
+            try {
+                for (var _d = true, lineReader_1 = __asyncValues(lineReader), lineReader_1_1; lineReader_1_1 = yield lineReader_1.next(), _a = lineReader_1_1.done, !_a; _d = true) {
+                    _c = lineReader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (!seen.has(line)) {
+                        seen.add(line);
+                        writer.write(line + '\n');
+                        newLineCount++;
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = lineReader_1.return)) yield _b.call(lineReader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            writer.close();
+            reader.close();
+            fs_1.default.renameSync(tempWorkPath, workPath);
+            return newLineCount;
+        });
+    }
+}
+const ConsumerExecutor = new ConsumerExecutorClass();
+exports.default = ConsumerExecutor;

package/executors/Executor.js

@@ -0,0 +1,168 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const readline_1 = __importDefault(require("readline"));
+const ProducerExecutor_1 = __importDefault(require("./ProducerExecutor"));
+const ConsumerExecutor_1 = __importDefault(require("./ConsumerExecutor"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const OutputExecutor_1 = __importDefault(require("./OutputExecutor"));
+const ConsumerManager_1 = __importDefault(require("../engines/consumer/ConsumerManager"));
+const ExecutorPerformance_1 = __importDefault(require("./ExecutorPerformance"));
+class Executor {
+    constructor() {
+        this._REPORT_WORK_AFTER_LINES = 1000;
+        /**
+         * 1. check and ready the local file for processing
+         * 2. open read stream and write stream
+         * 3. process the file
+         * 4. cleanup and after execution actions
+         */
+        this.run = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            var _d;
+            (0, Affirm_1.default)(request, 'Invalid request');
+            const { consumer, producer, prodDimensions, workerId, chunk, options, reportWork } = request;
+            const counter = performance.now();
+            const result = {
+                executionId: workerId,
+                cycles: 1,
+                elapsedMS: -1,
+                inputCount: -1,
+                outputCount: -1,
+                resultUri: ConsumerExecutor_1.default._getWorkPath(consumer, workerId),
+                operations: {}
+            };
+            let totalOutputCount = 0, totalCycles = 1, perf = 0, lineIndex = 0;
+            const readStream = this.openReadStream(chunk);
+            const writeStream = this.openWriteStream(consumer, workerId);
+            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
+            const { isFirstChunk, start, end } = chunk;
+            const totalBytes = end - start;
+            let processedBytes = 0;
+            // Process all the line-independent operations of the consumer in a single pass
+            const lineStream = readline_1.default.createInterface({ input: readStream, crlfDelay: Infinity });
+            try {
+                for (var _e = true, lineStream_1 = __asyncValues(lineStream), lineStream_1_1; lineStream_1_1 = yield lineStream_1.next(), _a = lineStream_1_1.done, !_a; _e = true) {
+                    _c = lineStream_1_1.value;
+                    _e = false;
+                    const line = _c;
+                    if (lineIndex === 0 && isFirstChunk) {
+                        if (!this.shouldProcessFirstLine(producer)) {
+                            lineIndex++;
+                            continue;
+                        }
+                    }
+                    perf = performance.now();
+                    let record = ProducerExecutor_1.default.processLine({
+                        dimensions: prodDimensions,
+                        index: lineIndex,
+                        line,
+                        producer,
+                        tracker: this._performance
+                    });
+                    this._performance.measure('process-line', performance.now() - perf);
+                    if (!record) {
+                        lineIndex++;
+                        continue;
+                    }
+                    perf = performance.now();
+                    record = ConsumerExecutor_1.default.processRecord({
+                        record,
+                        index: lineIndex,
+                        consumer: consumer,
+                        fields,
+                        producer,
+                        dimensions: prodDimensions,
+                        requestOptions: options
+                    });
+                    this._performance.measure('process-record', performance.now() - perf);
+                    if (!record) {
+                        lineIndex++;
+                        continue;
+                    }
+                    perf = performance.now();
+                    const outputLine = OutputExecutor_1.default.outputRecord(record, consumer, fields);
+                    this._performance.measure('output-record', performance.now() - perf);
+                    writeStream.write(outputLine + '\n');
+                    totalOutputCount++;
+                    lineIndex++;
+                    // Report progress to the main thread
+                    if (reportWork && lineIndex % this._REPORT_WORK_AFTER_LINES === 0) {
+                        processedBytes = Math.min(readStream.bytesRead, totalBytes);
+                        reportWork({ processed: processedBytes, total: totalBytes, workerId: workerId });
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_e && !_a && (_b = lineStream_1.return)) yield _b.call(lineStream_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            // Process the operations that work on multiple lines
+            if (((_d = consumer.options) === null || _d === void 0 ? void 0 : _d.distinct) === true) {
+                perf = performance.now();
+                totalOutputCount = yield ConsumerExecutor_1.default.processDistinct(consumer, workerId);
+                this._performance.measure('process-distinct', performance.now() - perf);
+                totalCycles++;
+            }
+            result.elapsedMS = performance.now() - counter;
+            result.cycles = totalCycles;
+            result.inputCount = lineIndex;
+            result.outputCount = totalOutputCount;
+            result.resultUri = ConsumerExecutor_1.default._getWorkPath(consumer, workerId);
+            result.operations = this._performance.getOperations();
+            return result;
+        });
+        this.openReadStream = (chunk) => {
+            const { end, fileUri, start } = chunk;
+            return fs_1.default.createReadStream(fileUri, { start, end: end });
+        };
+        this.openWriteStream = (consumer, executionId) => {
+            return ConsumerExecutor_1.default.ready(consumer, executionId);
+        };
+        this.shouldProcessFirstLine = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const { settings: { fileType, hasHeaderRow } } = producer;
+            switch (fileType) {
+                case 'PARQUET':
+                case 'XML':
+                case 'XLS':
+                case 'XLSX':
+                case 'CSV':
+                    return false;
+                case 'TXT': {
+                    if (hasHeaderRow)
+                        return false;
+                    else
+                        return true;
+                }
+                case 'JSON':
+                case 'JSONL':
+                    return true;
+            }
+        };
+        this._performance = new ExecutorPerformance_1.default();
+    }
+}
+exports.default = Executor;

package/executors/ExecutorOrchestrator.js

@@ -0,0 +1,315 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const os_1 = __importDefault(require("os"));
+const fs_1 = __importDefault(require("fs"));
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const promises_2 = require("stream/promises");
+const workerpool_1 = __importDefault(require("workerpool"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const UsageManager_1 = __importDefault(require("../engines/usage/UsageManager"));
+const Helper_1 = __importDefault(require("../helper/Helper"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const ProducerExecutor_1 = __importDefault(require("./ProducerExecutor"));
+const Constants_1 = __importDefault(require("../Constants"));
+const DriverHelper_1 = __importDefault(require("../drivers/DriverHelper"));
+const ConsumerExecutor_1 = __importDefault(require("./ConsumerExecutor"));
+const OutputExecutor_1 = __importDefault(require("./OutputExecutor"));
+const ConsumerManager_1 = __importDefault(require("../engines/consumer/ConsumerManager"));
+const ExecutorPerformance_1 = __importDefault(require("./ExecutorPerformance"));
+const ExecutorProgress_1 = __importDefault(require("./ExecutorProgress"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const ConsumerOnFinishManager_1 = __importDefault(require("../engines/consumer/ConsumerOnFinishManager"));
+class ExecutorOrchestratorClass {
+    constructor() {
+        this.init = () => {
+            if (!this._executorPool) {
+                const options = {
+                    workerThreadOpts: {
+                        resourceLimits: {
+                            maxOldGenerationSizeMb: Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB
+                        }
+                    }
+                };
+                const workerPath = this._getWorkerPath();
+                this._executorPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ExecutorWorker.js'), options);
+            }
+        };
+        this.launch = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            (0, Affirm_1.default)(request, 'Invalid options');
+            const { consumer, details, logProgress, options } = request;
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(details, 'Invalid execution details');
+            const tracker = new ExecutorPerformance_1.default();
+            const _progress = new ExecutorProgress_1.default(logProgress);
+            const { usageId } = UsageManager_1.default.startUsage(consumer, details);
+            const workersId = [];
+            try {
+                const start = performance.now();
+                this.init();
+                const executorResults = [];
+                const sourceFilesByProducer = yield this.readySourceFiles(consumer);
+                for (const pair of sourceFilesByProducer) {
+                    const { prod, cProd, response } = pair;
+                    // Make sure that the data files are there, if missing and isOptional = true, then skip
+                    if (!fs_1.default.existsSync(response.files[0].fullUri)) {
+                        if (!cProd.isOptional)
+                            throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
+                        else if (cProd.isOptional === true)
+                            continue;
+                    }
+                    // Extract the dimensions for this producer just once
+                    const firstLine = (yield DriverHelper_1.default.quickReadFile(response.files[0].fullUri, 1))[0];
+                    const header = ProducerExecutor_1.default.processHeader(firstLine, prod);
+                    const prodDimensions = ProducerExecutor_1.default.reconcileHeader(header, prod);
+                    for (const file of response.files) {
+                        const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
+                        const workerThreads = [];
+                        for (const [index, chunk] of chunks.entries()) {
+                            // Spawn off thread
+                            const workerId = `${usageId}_${index}`;
+                            const workerData = {
+                                chunk,
+                                consumer,
+                                prodDimensions,
+                                producer: prod,
+                                workerId: workerId,
+                                options: options
+                            };
+                            _progress.register((index + 1).toString());
+                            workersId.push(workerId);
+                            workerThreads.push(this._executorPool.exec('executor', [workerData], {
+                                on: payload => this.onWorkAdvanced(payload, index, _progress)
+                            }));
+                        }
+                        executorResults.push(...yield Promise.all(workerThreads));
+                        // WARNING: will this not create problems when multiple are executed together at the same time since this is a singleton?!?
+                        yield this._executorPool.terminate();
+                    }
+                }
+                _progress.complete();
+                yield this.reconcileExecutorThreadsResults(consumer, usageId, executorResults, tracker);
+                // If there is more than one worker, then I need to redo the operations that are done on multiple lines (cause now the worker files have been merged together)
+                const postOperation = { totalOutputCount: null };
+                if (executorResults.length > 1) {
+                    if (((_a = consumer.options) === null || _a === void 0 ? void 0 : _a.distinct) === true) {
+                        const perf = performance.now();
+                        const unifiedOutputCount = yield ConsumerExecutor_1.default.processDistinct(consumer, usageId);
+                        tracker.measure('process-distinct:main', performance.now() - perf);
+                        postOperation.totalOutputCount = unifiedOutputCount;
+                    }
+                }
+                // Export to the destination
+                let perf = performance.now();
+                const exportRes = yield OutputExecutor_1.default.exportResult(consumer, usageId, ConsumerManager_1.default.getExpandedFields(consumer));
+                tracker.measure('export-result', performance.now() - perf);
+                // Perform on-success actions if any
+                if (consumer.outputs.some(x => x.onSuccess)) {
+                    perf = performance.now();
+                    yield ConsumerOnFinishManager_1.default.onConsumerSuccess(consumer, usageId);
+                    tracker.measure('on-success-actions', performance.now() - perf);
+                }
+                yield this.performCleanupOperations(consumer, usageId, executorResults.map(x => x.resultUri), tracker);
+                const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
+                finalResult.elapsedMS = performance.now() - start;
+                if (Algo_1.default.hasVal(postOperation.totalOutputCount))
+                    finalResult.outputCount = postOperation.totalOutputCount;
+                UsageManager_1.default.endUsage(usageId, finalResult.outputCount, finalResult);
+                return finalResult;
+            }
+            catch (error) {
+                yield ConsumerOnFinishManager_1.default.onConsumerError(consumer, usageId);
+                yield this.performCleanupOperations(consumer, usageId, workersId.map(x => ConsumerExecutor_1.default._getWorkPath(consumer, x)), tracker);
+                UsageManager_1.default.failUsage(usageId, Helper_1.default.asError(error).message);
+                throw error;
+            }
+        });
+        /**
+         * Calculates line-aligned chunk offsets for parallel file processing.
+         * Each chunk boundary is adjusted to the next newline to avoid breaking lines.
+         * Returns a single chunk for small files where parallelism overhead isn't worth it.
+         */
+        this.scopeWork = (fileUri, numChunks) => {
+            const fileSize = fs_1.default.statSync(fileUri).size;
+            if (fileSize === 0)
+                return [];
+            // Small files: single chunk, parallelism overhead not worth it
+            if (fileSize < Constants_1.default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
+                return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
+            }
+            // Calculate optimal chunk count based on file size and CPU cores
+            const cpus = numChunks !== null && numChunks !== void 0 ? numChunks : (os_1.default.cpus().length - 1);
+            const maxChunksBySize = Math.floor(fileSize / Constants_1.default.defaults.MIN_CHUNK_SIZE);
+            const effectiveChunks = Math.min(cpus, maxChunksBySize);
+            if (effectiveChunks <= 1)
+                return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
+            const targetChunkSize = Math.floor(fileSize / effectiveChunks);
+            const fd = fs_1.default.openSync(fileUri, 'r');
+            try {
+                const offsets = [];
+                let currentStart = 0;
+                for (let i = 0; i < cpus - 1; i++) {
+                    const targetEnd = currentStart + targetChunkSize;
+                    // Don't overshoot file size
+                    if (targetEnd >= fileSize) {
+                        break;
+                    }
+                    // Find next newline after target boundary
+                    const alignedEnd = this.findNextNewline(fd, targetEnd, fileSize);
+                    offsets.push({ start: currentStart, end: alignedEnd, isFirstChunk: i === 0, fileUri });
+                    currentStart = alignedEnd;
+                }
+                // Final chunk goes to end of file
+                if (currentStart < fileSize) {
+                    offsets.push({ start: currentStart, end: fileSize, isFirstChunk: offsets.length === 0, fileUri });
+                }
+                return offsets;
+            }
+            finally {
+                fs_1.default.closeSync(fd);
+            }
+        };
+        /**
+         * Efficiently finds the next newline character starting from a position.
+         * Uses small buffer reads for speed.
+         */
+        this.findNextNewline = (fd, position, fileSize) => {
+            const BUFFER_SIZE = 8192; // 8KB buffer for scanning
+            const buffer = Buffer.allocUnsafe(BUFFER_SIZE);
+            let currentPos = position;
+            while (currentPos < fileSize) {
+                const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
+                const bytesRead = fs_1.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
+                if (bytesRead === 0)
+                    break;
+                // Scan buffer for newline
+                for (let i = 0; i < bytesRead; i++) {
+                    if (buffer[i] === 0x0A) { // \n
+                        return currentPos + i + 1; // Position after the newline
+                    }
+                }
+                currentPos += bytesRead;
+            }
+            // No newline found, return file end
+            return fileSize;
+        };
+        this.readySourceFiles = (consumer) => __awaiter(this, void 0, void 0, function* () {
+            const results = [];
+            for (let i = 0; i < consumer.producers.length; i++) {
+                const cProd = consumer.producers[i];
+                const prod = Environment_1.default.getProducer(cProd.name);
+                results.push({ prod, cProd, response: yield ProducerExecutor_1.default.ready(prod) });
+            }
+            return results;
+        });
+        this._getWorkerPath = () => {
+            // Get the current file's directory
+            const currentDir = __dirname;
+            if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
+                return path_1.default.resolve('./.build/workers');
+            const forcedPath = process.env.REMORA_WORKERS_PATH;
+            if (forcedPath && forcedPath.length > 0)
+                return path_1.default.join(__dirname, forcedPath);
+            // Check if we're in a published npm package (no .build in path)
+            if (!currentDir.includes('.build')) {
+                // We're in the published package, workers are relative to package root
+                // __dirname is something like: /path/to/package/engines/dataset
+                // We need to go up to package root and then to workers
+                return path_1.default.join(__dirname, '../../workers');
+            }
+            else {
+                // We're in development, workers are in ./.build/workers
+                return path_1.default.resolve('./.build/workers');
+            }
+        };
+        this.reconcileExecutorThreadsResults = (consumer, executionId, executorResults, tracker) => __awaiter(this, void 0, void 0, function* () {
+            const workPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
+            ConsumerExecutor_1.default._ensurePath(workPath);
+            // Merge all the various files into a single one
+            if (executorResults.length > 1) {
+                const perf = performance.now();
+                const output = fs_1.default.createWriteStream(workPath);
+                output.setMaxListeners(executorResults.length + 1);
+                for (const workerResult of executorResults) {
+                    yield (0, promises_2.pipeline)(fs_1.default.createReadStream(workerResult.resultUri), output, { end: false });
+                }
+                output.end();
+                output.close();
+                tracker.measure('merge-workers', performance.now() - perf);
+            }
+            else if (executorResults.length === 1) {
+                // If there is only one worker, then just rename the worker .dataset to the general consumer one
+                yield promises_1.default.rename(executorResults[0].resultUri, workPath);
+            }
+        });
+        this.performCleanupOperations = (consumer, executionId, workersPath, tracker) => __awaiter(this, void 0, void 0, function* () {
+            const workPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
+            const start = performance.now();
+            yield Promise.all(workersPath.map(x => ConsumerExecutor_1.default._clearWorkPath(x)));
+            yield promises_1.default.rmdir(path_1.default.dirname(workPath));
+            tracker.measure('cleanup-operations', performance.now() - start);
+        });
+        this.computeFinalResult = (tracker, executorResults, executionId, resultUri) => {
+            const result = {
+                cycles: Algo_1.default.max(executorResults.map(x => x.cycles)),
+                elapsedMS: Algo_1.default.sum(executorResults.map(x => x.elapsedMS)),
+                inputCount: Algo_1.default.sum(executorResults.map(x => x.inputCount)),
+                outputCount: Algo_1.default.sum(executorResults.map(x => x.outputCount)),
+                workerCount: executorResults.length,
+                executionId,
+                resultUri,
+                operations: {}
+            };
+            for (const res of executorResults) {
+                for (const opKey of Object.keys(res.operations)) {
+                    const op = res.operations[opKey];
+                    let label = result.operations[opKey];
+                    if (!label) {
+                        result.operations[opKey] = { avg: -1, max: -1, min: -1, elapsedMS: [] };
+                        label = result.operations[opKey];
+                    }
+                    label.elapsedMS.push(op.elapsedMS);
+                }
+                // Calculate min, max, avg for all operations after collecting all data
+                for (const opKey of Object.keys(result.operations)) {
+                    const operation = result.operations[opKey];
+                    if (operation.elapsedMS.length > 0) {
+                        operation.min = Math.min(...operation.elapsedMS);
+                        operation.max = Math.max(...operation.elapsedMS);
+                        operation.avg = Algo_1.default.mean(operation.elapsedMS);
+                    }
+                }
+            }
+            // Add tracker operations to result
+            const trackerOperations = tracker.getOperations();
+            for (const opKey of Object.keys(trackerOperations)) {
+                const trackerOp = trackerOperations[opKey];
+                const value = trackerOp.elapsedMS;
+                if (!result.operations[opKey]) {
+                    result.operations[opKey] = { avg: value, max: value, min: value, elapsedMS: [] };
+                }
+                result.operations[opKey].elapsedMS.push(value);
+            }
+            return result;
+        };
+        this.onWorkAdvanced = (packet, index, progress) => {
+            const { processed, total } = packet;
+            progress.update((index + 1).toString(), processed / total);
+        };
+    }
+}
+const ExecutorOrchestrator = new ExecutorOrchestratorClass();
+exports.default = ExecutorOrchestrator;

package/executors/ExecutorPerformance.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+class ExecutorPerformance {
+    constructor() {
+        this.measure = (name, elapsedMS) => {
+            let tracker = this._operations[name];
+            if (!tracker) {
+                this._operations[name] = { elapsedMS: 0 };
+                tracker = this._operations[name];
+            }
+            tracker.elapsedMS += elapsedMS;
+        };
+        this.getOperations = () => this._operations;
+        this._operations = {};
+    }
+}
+exports.default = ExecutorPerformance;