@forzalabs/remora 1.0.11 → 1.0.14

package/engines/validation/Validator.js

@@ -119,11 +119,11 @@ class ValidatorClass {
                 // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
                 const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
                 if (uniqEngines.length !== 1)
-                    errors.push(`Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
+                    errors.push(`Sources with different engines are used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
                 // For now we also only support consumers that have producers ALL having the same exact source
                 const uniqNames = Algo_1.default.uniqBy(sources, 'name');
                 if (uniqNames.length !== 1)
-                    errors.push(`Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
+                    errors.push(`Producers with different sources are used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
                 if (consumer.filters && consumer.filters.length > 0) {
                     if (consumer.filters.some(x => x.sql && x.rule))
                         errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
@@ -188,6 +188,11 @@ class ValidatorClass {
                             errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
                     }
                 }
+                // Validate distinct
+                if (consumer.options) {
+                    if (Algo_1.default.hasVal(consumer.options.distinct) && Algo_1.default.hasVal(consumer.options.distinctOn))
+                        errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+                }
             }
             catch (e) {
                 if (errors.length === 0)

package/executors/ConsumerExecutor.js

@@ -25,9 +25,11 @@ const readline_1 = __importDefault(require("readline"));
 const promises_1 = __importDefault(require("fs/promises"));
 const TransformationEngine_1 = __importDefault(require("../engines/transform/TransformationEngine"));
 const RequestExecutor_1 = __importDefault(require("../engines/execution/RequestExecutor"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Constants_1 = __importDefault(require("../Constants"));
 const Algo_1 = __importDefault(require("../core/Algo"));
+const LineParser_1 = __importDefault(require("../engines/parsing/LineParser"));
+const OutputExecutor_1 = __importDefault(require("./OutputExecutor"));
+const ConsumerManager_1 = __importDefault(require("../engines/consumer/ConsumerManager"));
 class ConsumerExecutorClass {
     constructor() {
         this._getWorkPath = (consumer, executionId) => {
@@ -64,20 +66,14 @@ class ConsumerExecutorClass {
             if (!fs_1.default.existsSync(pathUri))
                 fs_1.default.writeFileSync(pathUri, '');
         };
-        this.ready = (consumer, executionId) => {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const workPath = this._getWorkPath(consumer, executionId);
-            this._ensurePath(workPath);
-            return fs_1.default.createWriteStream(workPath);
-        };
         this.processRecord = (options) => {
-            var _a;
+            var _a, _b;
             const { consumer, fields, dimensions, producer, record, requestOptions } = options;
-            // map to consumer fields and apply consumer field logic
+            // Map to consumer fields and apply consumer field logic
             for (const field of fields) {
                 const { cField } = field;
                 const fieldKey = (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key;
-                // set the fixed default value for the field, or throw error if not present in the producer
+                // Set the fixed default value for the field, or throw error if not present in the producer
                 const dimension = dimensions.find(x => x.name === cField.key);
                 if (!dimension) {
                     if (cField.fixed && Algo_1.default.hasVal(cField.default))
@@ -85,12 +81,16 @@ class ConsumerExecutorClass {
                     else
                         throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map(x => x.name).join(', ')})`);
                 }
-                // change the name of the dimension
+                // Change the name of the dimension
                 if (cField.alias && cField.alias !== dimension.name) {
                     record[cField.alias] = record[dimension.name];
                     delete record[dimension.name];
                 }
-                // apply transformations
+            }
+            // Transformations need to be applied after the mapping since they might refer to other fields with their new names
+            for (const field of fields) {
+                const { cField } = field;
+                const fieldKey = (_b = cField.alias) !== null && _b !== void 0 ? _b : cField.key;
                 if (cField.transform)
                     record[fieldKey] = TransformationEngine_1.default.applyTransformations(record[fieldKey], cField.transform, cField, record);
             }
@@ -114,12 +114,11 @@ class ConsumerExecutorClass {
             }
             return record;
         };
-        this.processDistinct = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
+        this.processDistinct = (datasetPath) => __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
-            const workPath = this._getWorkPath(consumer, executionId);
-            const reader = fs_1.default.createReadStream(workPath);
+            const reader = fs_1.default.createReadStream(datasetPath);
             const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
-            const tempWorkPath = workPath + '_tmp';
+            const tempWorkPath = datasetPath + '_tmp';
             const writer = fs_1.default.createWriteStream(tempWorkPath);
             let newLineCount = 0;
             const seen = new Set();
@@ -144,9 +143,120 @@ class ConsumerExecutorClass {
             }
             writer.close();
             reader.close();
-            fs_1.default.renameSync(tempWorkPath, workPath);
+            // Wait for the writer to finish before renaming
+            yield new Promise((resolve, reject) => {
+                writer.on('finish', resolve);
+                writer.on('error', reject);
+                writer.end();
+            });
+            fs_1.default.renameSync(tempWorkPath, datasetPath);
             return newLineCount;
         });
+        this.processDistinctOn = (consumer, datasetPath) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_2, _b, _c;
+            const reader = fs_1.default.createReadStream(datasetPath);
+            const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
+            const { distinctOn } = consumer.options;
+            const { keys, resolution } = distinctOn;
+            const { strategy, orderBy, direction = 'asc' } = resolution;
+            const internalRecordFormat = OutputExecutor_1.default._getInternalRecordFormat(consumer);
+            const internalFields = ConsumerManager_1.default.getExpandedFields(consumer);
+            // Map to store the winning record for each composite key
+            // Key: composite key string, Value: { record: parsed object, line: original JSON line }
+            const winners = new Map();
+            try {
+                for (var _d = true, lineReader_2 = __asyncValues(lineReader), lineReader_2_1; lineReader_2_1 = yield lineReader_2.next(), _a = lineReader_2_1.done, !_a; _d = true) {
+                    _c = lineReader_2_1.value;
+                    _d = false;
+                    const line = _c;
+                    const record = (internalRecordFormat === 'CSV' || internalRecordFormat === 'TXT')
+                        ? LineParser_1.default._internalParseCSV(line, internalFields)
+                        : LineParser_1.default._internalParseJSON(line);
+                    const compositeKey = keys.map(k => { var _a; return String((_a = record[k]) !== null && _a !== void 0 ? _a : ''); }).join('|');
+                    const existing = winners.get(compositeKey);
+                    if (!existing) {
+                        winners.set(compositeKey, { record, line });
+                        continue;
+                    }
+                    const shouldReplace = this._shouldReplaceRecord(existing.record, record, strategy, orderBy, direction);
+                    if (shouldReplace) {
+                        winners.set(compositeKey, { record, line });
+                    }
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = lineReader_2.return)) yield _b.call(lineReader_2);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            reader.close();
+            // Write the winning records to the temp file
+            const tempWorkPath = datasetPath + '_tmp';
+            const writer = fs_1.default.createWriteStream(tempWorkPath);
+            for (const { line } of winners.values()) {
+                writer.write(line + '\n');
+            }
+            // Wait for the writer to finish before renaming
+            yield new Promise((resolve, reject) => {
+                writer.on('finish', resolve);
+                writer.on('error', reject);
+                writer.end();
+            });
+            fs_1.default.renameSync(tempWorkPath, datasetPath);
+            return winners.size;
+        });
+        /**
+         * Determines if the new record should replace the existing record based on the resolution strategy
+         */
+        this._shouldReplaceRecord = (existing, newRecord, strategy, orderBy, direction) => {
+            switch (strategy) {
+                case 'first':
+                    return false;
+                case 'last':
+                    return true;
+                case 'min': {
+                    const existingVal = existing[orderBy];
+                    const newVal = newRecord[orderBy];
+                    const comparison = this._compareValues(newVal, existingVal);
+                    // For 'min', we want the smallest value
+                    // If direction is 'desc', we invert the logic (smallest becomes largest)
+                    return direction === 'asc' ? comparison < 0 : comparison > 0;
+                }
+                case 'max': {
+                    const existingVal = existing[orderBy];
+                    const newVal = newRecord[orderBy];
+                    const comparison = this._compareValues(newVal, existingVal);
+                    // For 'max', we want the largest value
+                    // If direction is 'desc', we invert the logic (largest becomes smallest)
+                    return direction === 'asc' ? comparison > 0 : comparison < 0;
+                }
+                default:
+                    return false;
+            }
+        };
+        /**
+         * Compares two values, handling numbers, strings, and dates
+         * Returns: negative if a < b, positive if a > b, 0 if equal
+         */
+        this._compareValues = (a, b) => {
+            // Handle null/undefined
+            if (a == null && b == null)
+                return 0;
+            if (a == null)
+                return -1;
+            if (b == null)
+                return 1;
+            // Try numeric comparison
+            const numA = Number(a);
+            const numB = Number(b);
+            if (!isNaN(numA) && !isNaN(numB)) {
+                return numA - numB;
+            }
+            // Fall back to string comparison
+            return String(a).localeCompare(String(b));
+        };
     }
 }
 const ConsumerExecutor = new ConsumerExecutorClass();

package/executors/Executor.js

@@ -27,6 +27,7 @@ const Affirm_1 = __importDefault(require("../core/Affirm"));
 const OutputExecutor_1 = __importDefault(require("./OutputExecutor"));
 const ConsumerManager_1 = __importDefault(require("../engines/consumer/ConsumerManager"));
 const ExecutorPerformance_1 = __importDefault(require("./ExecutorPerformance"));
+const ExecutorScope_1 = __importDefault(require("./ExecutorScope"));
 class Executor {
     constructor() {
         this._REPORT_WORK_AFTER_LINES = 1000;
@@ -38,9 +39,9 @@ class Executor {
          */
         this.run = (request) => __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
-            var _d;
+            var _d, _e;
             (0, Affirm_1.default)(request, 'Invalid request');
-            const { consumer, producer, prodDimensions, workerId, chunk, options, reportWork } = request;
+            const { consumer, producer, prodDimensions, workerId, chunk, options, scope, reportWork } = request;
             const counter = performance.now();
             const result = {
                 executionId: workerId,
@@ -48,12 +49,13 @@ class Executor {
                 elapsedMS: -1,
                 inputCount: -1,
                 outputCount: -1,
-                resultUri: ConsumerExecutor_1.default._getWorkPath(consumer, workerId),
+                resultUri: ExecutorScope_1.default.getWorkerPath(scope, workerId),
                 operations: {}
             };
+            ExecutorScope_1.default.ensurePath(result.resultUri);
             let totalOutputCount = 0, totalCycles = 1, perf = 0, lineIndex = 0;
             const readStream = this.openReadStream(chunk);
-            const writeStream = this.openWriteStream(consumer, workerId);
+            const writeStream = this.openWriteStream(scope, workerId);
             const fields = ConsumerManager_1.default.getExpandedFields(consumer);
             const { isFirstChunk, start, end } = chunk;
             const totalBytes = end - start;
@@ -61,9 +63,9 @@ class Executor {
             // Process all the line-independent operations of the consumer in a single pass
             const lineStream = readline_1.default.createInterface({ input: readStream, crlfDelay: Infinity });
             try {
-                for (var _e = true, lineStream_1 = __asyncValues(lineStream), lineStream_1_1; lineStream_1_1 = yield lineStream_1.next(), _a = lineStream_1_1.done, !_a; _e = true) {
+                for (var _f = true, lineStream_1 = __asyncValues(lineStream), lineStream_1_1; lineStream_1_1 = yield lineStream_1.next(), _a = lineStream_1_1.done, !_a; _f = true) {
                     _c = lineStream_1_1.value;
-                    _e = false;
+                    _f = false;
                     const line = _c;
                     if (lineIndex === 0 && isFirstChunk) {
                         if (!this.shouldProcessFirstLine(producer)) {
@@ -115,22 +117,27 @@ class Executor {
             catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_e && !_a && (_b = lineStream_1.return)) yield _b.call(lineStream_1);
+                    if (!_f && !_a && (_b = lineStream_1.return)) yield _b.call(lineStream_1);
                 }
                 finally { if (e_1) throw e_1.error; }
             }
             // Process the operations that work on multiple lines
             if (((_d = consumer.options) === null || _d === void 0 ? void 0 : _d.distinct) === true) {
                 perf = performance.now();
-                totalOutputCount = yield ConsumerExecutor_1.default.processDistinct(consumer, workerId);
+                totalOutputCount = yield ConsumerExecutor_1.default.processDistinct(result.resultUri);
                 this._performance.measure('process-distinct', performance.now() - perf);
                 totalCycles++;
             }
+            if ((_e = consumer.options) === null || _e === void 0 ? void 0 : _e.distinctOn) {
+                perf = performance.now();
+                totalOutputCount = yield ConsumerExecutor_1.default.processDistinctOn(consumer, result.resultUri);
+                this._performance.measure('process-distinct-on', performance.now() - perf);
+                totalCycles++;
+            }
             result.elapsedMS = performance.now() - counter;
             result.cycles = totalCycles;
             result.inputCount = lineIndex;
             result.outputCount = totalOutputCount;
-            result.resultUri = ConsumerExecutor_1.default._getWorkPath(consumer, workerId);
             result.operations = this._performance.getOperations();
             return result;
         });
@@ -138,8 +145,9 @@ class Executor {
             const { end, fileUri, start } = chunk;
             return fs_1.default.createReadStream(fileUri, { start, end: end });
         };
-        this.openWriteStream = (consumer, executionId) => {
-            return ConsumerExecutor_1.default.ready(consumer, executionId);
+        this.openWriteStream = (scope, workerId) => {
+            const workerPath = ExecutorScope_1.default.getWorkerPath(scope, workerId);
+            return fs_1.default.createWriteStream(workerPath);
         };
         this.shouldProcessFirstLine = (producer) => {
             (0, Affirm_1.default)(producer, 'Invalid producer');

package/executors/ExecutorOrchestrator.js

@@ -32,6 +32,7 @@ const ExecutorPerformance_1 = __importDefault(require("./ExecutorPerformance"));
 const ExecutorProgress_1 = __importDefault(require("./ExecutorProgress"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const ConsumerOnFinishManager_1 = __importDefault(require("../engines/consumer/ConsumerOnFinishManager"));
+const ExecutorScope_1 = __importDefault(require("./ExecutorScope"));
 class ExecutorOrchestratorClass {
     constructor() {
         this.init = () => {
@@ -48,7 +49,7 @@ class ExecutorOrchestratorClass {
             }
         };
         this.launch = (request) => __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             (0, Affirm_1.default)(request, 'Invalid options');
             const { consumer, details, logProgress, options } = request;
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
@@ -56,12 +57,14 @@ class ExecutorOrchestratorClass {
             const tracker = new ExecutorPerformance_1.default();
             const _progress = new ExecutorProgress_1.default(logProgress);
             const { usageId } = UsageManager_1.default.startUsage(consumer, details);
-            const workersId = [];
+            const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [] };
             try {
                 const start = performance.now();
                 this.init();
                 const executorResults = [];
-                const sourceFilesByProducer = yield this.readySourceFiles(consumer);
+                let counter = performance.now();
+                const sourceFilesByProducer = yield this.readySourceFiles(consumer, scope);
+                tracker.measure('ready-producers', performance.now() - counter);
                 let globalWorkerIndex = 0;
                 for (const pair of sourceFilesByProducer) {
                     const { prod, cProd, response } = pair;
@@ -86,15 +89,16 @@ class ExecutorOrchestratorClass {
                             const currentWorkerIndex = globalWorkerIndex;
                             globalWorkerIndex++;
                             const workerData = {
+                                producer: prod,
                                 chunk,
                                 consumer,
                                 prodDimensions,
-                                producer: prod,
-                                workerId: workerId,
-                                options: options
+                                workerId,
+                                scope,
+                                options
                             };
                             _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
-                            workersId.push(workerId);
+                            scope.workersId.push(workerId);
                             workerThreads.push(this._executorPool.exec('executor', [workerData], {
                                 on: payload => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
                             }));
@@ -105,28 +109,34 @@ class ExecutorOrchestratorClass {
                     }
                 }
                 _progress.complete();
-                yield this.reconcileExecutorThreadsResults(consumer, usageId, executorResults, tracker);
+                yield this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
                 // If there is more than one worker, then I need to redo the operations that are done on multiple lines (cause now the worker files have been merged together)
                 const postOperation = { totalOutputCount: null };
                 if (executorResults.length > 1) {
                     if (((_a = consumer.options) === null || _a === void 0 ? void 0 : _a.distinct) === true) {
-                        const perf = performance.now();
-                        const unifiedOutputCount = yield ConsumerExecutor_1.default.processDistinct(consumer, usageId);
-                        tracker.measure('process-distinct:main', performance.now() - perf);
+                        counter = performance.now();
+                        const unifiedOutputCount = yield ConsumerExecutor_1.default.processDistinct(ExecutorScope_1.default.getMainPath(scope));
+                        tracker.measure('process-distinct:main', performance.now() - counter);
+                        postOperation.totalOutputCount = unifiedOutputCount;
+                    }
+                    if (((_b = consumer.options) === null || _b === void 0 ? void 0 : _b.distinct) === true) {
+                        counter = performance.now();
+                        const unifiedOutputCount = yield ConsumerExecutor_1.default.processDistinctOn(consumer, ExecutorScope_1.default.getMainPath(scope));
+                        tracker.measure('process-distinct-on:main', performance.now() - counter);
                         postOperation.totalOutputCount = unifiedOutputCount;
                     }
                 }
                 // Export to the destination
-                let perf = performance.now();
-                const exportRes = yield OutputExecutor_1.default.exportResult(consumer, usageId, ConsumerManager_1.default.getExpandedFields(consumer));
-                tracker.measure('export-result', performance.now() - perf);
+                counter = performance.now();
+                const exportRes = yield OutputExecutor_1.default.exportResult(consumer, ConsumerManager_1.default.getExpandedFields(consumer), scope);
+                tracker.measure('export-result', performance.now() - counter);
                 // Perform on-success actions if any
                 if (consumer.outputs.some(x => x.onSuccess)) {
-                    perf = performance.now();
+                    counter = performance.now();
                     yield ConsumerOnFinishManager_1.default.onConsumerSuccess(consumer, usageId);
-                    tracker.measure('on-success-actions', performance.now() - perf);
+                    tracker.measure('on-success-actions', performance.now() - counter);
                 }
-                yield this.performCleanupOperations(consumer, usageId, executorResults.map(x => x.resultUri), tracker);
+                yield this.performCleanupOperations(scope, tracker);
                 const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
                 finalResult.elapsedMS = performance.now() - start;
                 if (Algo_1.default.hasVal(postOperation.totalOutputCount))
@@ -136,7 +146,7 @@ class ExecutorOrchestratorClass {
             }
             catch (error) {
                 yield ConsumerOnFinishManager_1.default.onConsumerError(consumer, usageId);
-                yield this.performCleanupOperations(consumer, usageId, workersId.map(x => ConsumerExecutor_1.default._getWorkPath(consumer, x)), tracker);
+                yield this.performCleanupOperations(scope, tracker);
                 UsageManager_1.default.failUsage(usageId, Helper_1.default.asError(error).message);
                 throw error;
             }
@@ -210,12 +220,12 @@ class ExecutorOrchestratorClass {
             // No newline found, return file end
             return fileSize;
         };
-        this.readySourceFiles = (consumer) => __awaiter(this, void 0, void 0, function* () {
+        this.readySourceFiles = (consumer, scope) => __awaiter(this, void 0, void 0, function* () {
             const results = [];
             for (let i = 0; i < consumer.producers.length; i++) {
                 const cProd = consumer.producers[i];
                 const prod = Environment_1.default.getProducer(cProd.name);
-                results.push({ prod, cProd, response: yield ProducerExecutor_1.default.ready(prod) });
+                results.push({ prod, cProd, response: yield ProducerExecutor_1.default.ready(prod, scope) });
             }
             return results;
         });
@@ -239,13 +249,13 @@ class ExecutorOrchestratorClass {
                 return path_1.default.resolve('./.build/workers');
             }
         };
-        this.reconcileExecutorThreadsResults = (consumer, executionId, executorResults, tracker) => __awaiter(this, void 0, void 0, function* () {
-            const workPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
-            ConsumerExecutor_1.default._ensurePath(workPath);
+        this.reconcileExecutorThreadsResults = (scope, executorResults, tracker) => __awaiter(this, void 0, void 0, function* () {
+            const mainPath = ExecutorScope_1.default.getMainPath(scope);
+            ConsumerExecutor_1.default._ensurePath(mainPath);
             // Merge all the various files into a single one
             if (executorResults.length > 1) {
                 const perf = performance.now();
-                const output = fs_1.default.createWriteStream(workPath);
+                const output = fs_1.default.createWriteStream(mainPath);
                 output.setMaxListeners(executorResults.length + 1);
                 for (const workerResult of executorResults) {
                     yield (0, promises_2.pipeline)(fs_1.default.createReadStream(workerResult.resultUri), output, { end: false });
@@ -256,14 +266,12 @@ class ExecutorOrchestratorClass {
             }
             else if (executorResults.length === 1) {
                 // If there is only one worker, then just rename the worker .dataset to the general consumer one
-                yield promises_1.default.rename(executorResults[0].resultUri, workPath);
+                yield promises_1.default.rename(executorResults[0].resultUri, mainPath);
             }
         });
-        this.performCleanupOperations = (consumer, executionId, workersPath, tracker) => __awaiter(this, void 0, void 0, function* () {
-            const workPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
+        this.performCleanupOperations = (scope, tracker) => __awaiter(this, void 0, void 0, function* () {
             const start = performance.now();
-            yield Promise.all(workersPath.map(x => ConsumerExecutor_1.default._clearWorkPath(x)));
-            yield ConsumerExecutor_1.default._clearWorkPath(workPath);
+            yield ExecutorScope_1.default.clearScope(scope);
             tracker.measure('cleanup-operations', performance.now() - start);
         });
         this.computeFinalResult = (tracker, executorResults, executionId, resultUri) => {

package/executors/ExecutorScope.js

@@ -0,0 +1,52 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const promises_1 = __importDefault(require("fs/promises"));
+const Constants_1 = __importDefault(require("../Constants"));
+class ExecutorScopeClass {
+    constructor() {
+        this.WORKERS_FOLDER = 'workers';
+        this.PRODUCERS_FOLDER = 'producers';
+        this.getWorkerPath = (scope, workerId) => {
+            return path_1.default.join(Constants_1.default.defaults.REMORA_PATH, Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, 
+            // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
+            scope.folder, this.WORKERS_FOLDER, `${workerId}.dataset`);
+        };
+        this.getProducerPath = (scope, producer, sourceFileKey) => {
+            return path_1.default.join(Constants_1.default.defaults.REMORA_PATH, Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, 
+            // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
+            scope.folder, this.PRODUCERS_FOLDER, producer.name, `${sourceFileKey}.dataset`);
+        };
+        this.getMainPath = (scope) => {
+            return path_1.default.join(Constants_1.default.defaults.REMORA_PATH, Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, scope.folder, 'main.dataset');
+        };
+        this.clearScope = (scope) => __awaiter(this, void 0, void 0, function* () {
+            const scopePath = path_1.default.join(Constants_1.default.defaults.REMORA_PATH, Constants_1.default.defaults.PRODUCER_TEMP_FOLDER, scope.folder);
+            if (fs_1.default.existsSync(scopePath)) {
+                yield promises_1.default.rm(scopePath, { recursive: true, force: true });
+            }
+        });
+        this.ensurePath = (fileUri) => {
+            const dir = path_1.default.dirname(fileUri);
+            if (!fs_1.default.existsSync(dir))
+                fs_1.default.mkdirSync(dir, { recursive: true });
+            if (!fs_1.default.existsSync(fileUri))
+                fs_1.default.writeFileSync(fileUri, '');
+        };
+    }
+}
+const ExecutorScope = new ExecutorScopeClass();
+exports.default = ExecutorScope;

package/executors/OutputExecutor.js

@@ -17,7 +17,7 @@ const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
 const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
 const CSVParser_1 = __importDefault(require("../engines/parsing/CSVParser"));
-const ConsumerExecutor_1 = __importDefault(require("./ConsumerExecutor"));
+const ExecutorScope_1 = __importDefault(require("./ExecutorScope"));
 class OutputExecutorClass {
     constructor() {
         this._getInternalRecordFormat = (consumer) => {
@@ -62,13 +62,13 @@ class OutputExecutorClass {
                 return JSON.stringify(record);
             }
         };
-        this.exportResult = (consumer, executionId, fields) => __awaiter(this, void 0, void 0, function* () {
+        this.exportResult = (consumer, fields, scope) => __awaiter(this, void 0, void 0, function* () {
             const internalFormat = this._getInternalRecordFormat(consumer);
             for (const output of consumer.outputs) {
                 const destination = Environment_1.default.getSource(output.exportDestination);
                 const driver = yield DriverFactory_1.default.instantiateDestination(destination);
-                const currentPath = ConsumerExecutor_1.default._getWorkPath(consumer, executionId);
-                const destinationName = this._composeFileName(consumer, output, this._getExtension(output), executionId);
+                const currentPath = ExecutorScope_1.default.getMainPath(scope);
+                const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
                 if (output.format === internalFormat) {
                     return yield driver.move(currentPath, destinationName);
                 }

package/executors/ProducerExecutor.js

@@ -22,12 +22,12 @@ const LineParser_1 = __importDefault(require("../engines/parsing/LineParser"));
 const CryptoEngine_1 = __importDefault(require("../engines/CryptoEngine"));
 class ProducerExecutorClass {
     constructor() {
-        this.ready = (producer) => __awaiter(this, void 0, void 0, function* () {
+        this.ready = (producer, scope) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const source = Environment_1.default.getSource(producer.source);
             (0, Affirm_1.default)(source, `Invalid source ${producer.source} on producer ${producer.name}`);
             const driver = yield DriverFactory_1.default.instantiateSource(source);
-            return yield driver.ready(producer);
+            return yield driver.ready({ producer, scope });
         });
         this.processHeader = (line, producer) => {
             const { settings: { fileType, hasHeaderRow, delimiter } } = producer;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.11",
+    "version": "1.0.14",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -21,7 +21,7 @@
         "debug": "npx tsx ./src/index.ts debug",
         "create-producer": "npx tsx ./src/index.ts create-producer",
         "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
-        "build": "npm i && npm run sync && tsc --outDir .build && npm run copy-static-file",
+        "build": "npm i && npm run sync && rm -rf .build && tsc --outDir .build && npm run copy-static-file",
         "fast-build": "tsc --outDir .build",
         "upload": "npm run build && cd .build && npm publish --access=public"
     },

package/workers/ExecutorWorker.js

@@ -21,11 +21,12 @@ dotenv_1.default.configDotenv();
 const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
     Environment_1.default.load('./');
     try {
-        const { workerId, chunk, consumer, producer, prodDimensions, options } = workerData;
+        const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;
         (0, Affirm_1.default)(workerId, `Invalid worker id`);
         (0, Affirm_1.default)(consumer, `Invalid consumer`);
         (0, Affirm_1.default)(producer, `Invalid producer`);
         (0, Affirm_1.default)(chunk, `Invalid chunk`);
+        (0, Affirm_1.default)(scope, `Invalid executor scope`);
         const executor = new Executor_1.default();
         const res = yield executor.run({
             consumer,
@@ -34,6 +35,7 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
             workerId,
             chunk,
             options,
+            scope,
             reportWork: packet => workerpool_1.default.workerEmit(packet)
         });
         return res;