@forzalabs/remora 1.0.1 → 1.0.2

package/engines/dataset/ParallelDataset.js

@@ -1,184 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Constants_1 = __importDefault(require("../../Constants"));
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Environment_1 = __importDefault(require("../Environment"));
-const workerpool_1 = __importDefault(require("workerpool"));
-const DatasetManager_1 = __importDefault(require("./DatasetManager"));
-const path_1 = __importDefault(require("path"));
-class ParallelDatasetClass {
-    constructor() {
-        this.init = () => {
-            /**
-             * I need the init to be called after all the setup has been completed because I need the .env to be loaded
-             */
-            if (!this._filterPool || !this._projectionPool || !this._transformPool) {
-                const options = {
-                    workerThreadOpts: {
-                        resourceLimits: {
-                            maxOldGenerationSizeMb: Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB
-                        }
-                    }
-                };
-                const workerPath = this._getWorkerPath();
-                this._filterPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'FilterWorker.js'), options);
-                this._projectionPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'ProjectionWorker.js'), options);
-                this._transformPool = workerpool_1.default.pool(path_1.default.join(workerPath, 'TransformWorker.js'), options);
-            }
-        };
-        this._getWorkerPath = () => {
-            // Get the current file's directory
-            const currentDir = __dirname;
-            if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
-                return path_1.default.resolve('./.build/workers');
-            const forcedPath = process.env.REMORA_WORKERS_PATH;
-            if (forcedPath && forcedPath.length > 0)
-                return path_1.default.join(__dirname, forcedPath);
-            // Check if we're in a published npm package (no .build in path)
-            if (!currentDir.includes('.build')) {
-                // We're in the published package, workers are relative to package root
-                // __dirname is something like: /path/to/package/engines/dataset
-                // We need to go up to package root and then to workers
-                return path_1.default.join(__dirname, '../../workers');
-            }
-            else {
-                // We're in development, workers are in ./.build/workers
-                return path_1.default.resolve('./.build/workers');
-            }
-        };
-        this._scopeWork = (dataset) => {
-            var _a;
-            const datasetCount = dataset.getCount();
-            const batchSize = (_a = parseInt(Environment_1.default.get('MAX_ITEMS_IN_MEMORY'))) !== null && _a !== void 0 ? _a : Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY;
-            const workerChunkSize = batchSize * Math.round(Constants_1.default.defaults.INDICATIVE_THREAD_LINE_COUNT / batchSize);
-            const workerCount = Math.max(Math.min(Math.ceil(datasetCount / workerChunkSize), Constants_1.default.defaults.MAX_THREAD_COUNT), 1);
-            const adjustedWorkerCount = Math.ceil(datasetCount / workerCount);
-            return { workerCount, adjustedWorkerCount };
-        };
-        this.filter = (dataset, filters) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            (0, Affirm_1.default)(filters, `Invalid filters`);
-            this.init();
-            // Distribute the work of the filter among the various workers, trying to have them match the batch size
-            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
-            dataset._startOperation('filter-parallel', { workerCount });
-            const threads = [];
-            for (let i = 0; i < workerCount; i++) {
-                const workerId = `worker_filter_${i}`;
-                const fromLine = adjustedWorkerCount * i;
-                const toLine = (i === workerCount - 1)
-                    ? Infinity
-                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
-                const workerData = {
-                    datasetDimensions: dataset.getDimensions(),
-                    datasetFile: dataset.getFile(),
-                    executionId: dataset.getExecutionId(),
-                    datasetName: dataset.name,
-                    datasetDelimiter: dataset.getDelimiter(),
-                    fromLine: fromLine,
-                    toLine: toLine,
-                    workerId: workerId,
-                    filterData: {
-                        rules: filters
-                    }
-                };
-                threads.push(this._filterPool.exec('filter', [workerData]));
-            }
-            const results = yield Promise.all(threads);
-            yield this._filterPool.terminate();
-            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: filter ${dataset.name}`);
-            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
-            dataset
-                .setDelimiter(results[0].datasetDelimiter)
-                .setDimensions(results[0].datasetDimensions);
-            dataset._finishOperation('filter-parallel');
-            return dataset;
-        });
-        this.projection = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            this.init();
-            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
-            dataset._startOperation('projection-parallel', { workerCount });
-            const threads = [];
-            for (let i = 0; i < workerCount; i++) {
-                const workerId = `worker_projection_${i}`;
-                const fromLine = adjustedWorkerCount * i;
-                const toLine = (i === workerCount - 1)
-                    ? Infinity
-                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
-                const workerData = {
-                    datasetDimensions: dataset.getDimensions(),
-                    datasetFile: dataset.getFile(),
-                    executionId: dataset.getExecutionId(),
-                    datasetName: dataset.name,
-                    datasetDelimiter: dataset.getDelimiter(),
-                    fromLine: fromLine,
-                    toLine: toLine,
-                    workerId: workerId,
-                    projectionData: { consumerName: consumer.name }
-                };
-                threads.push(this._projectionPool.exec('projection', [workerData]));
-            }
-            const results = yield Promise.all(threads);
-            yield this._projectionPool.terminate();
-            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: projection ${dataset.name}`);
-            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
-            dataset
-                .setDelimiter(results[0].datasetDelimiter)
-                .setDimensions(results[0].datasetDimensions);
-            dataset._finishOperation('projection-parallel');
-            return dataset;
-        });
-        this.transform = (dataset, consumer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            this.init();
-            const { adjustedWorkerCount, workerCount } = this._scopeWork(dataset);
-            dataset._startOperation('transform-parallel', { workerCount });
-            const threads = [];
-            for (let i = 0; i < workerCount; i++) {
-                const workerId = `worker_transform_${i}`;
-                const fromLine = adjustedWorkerCount * i;
-                const toLine = (i === workerCount - 1)
-                    ? Infinity
-                    : (adjustedWorkerCount * i) + adjustedWorkerCount;
-                const workerData = {
-                    datasetDimensions: dataset.getDimensions(),
-                    datasetFile: dataset.getFile(),
-                    executionId: dataset.getExecutionId(),
-                    datasetName: dataset.name,
-                    datasetDelimiter: dataset.getDelimiter(),
-                    fromLine: fromLine,
-                    toLine: toLine,
-                    workerId: workerId,
-                    transformData: { consumerName: consumer.name }
-                };
-                threads.push(this._transformPool.exec('transform', [workerData]));
-            }
-            const results = yield Promise.all(threads);
-            yield this._transformPool.terminate();
-            (0, Affirm_1.default)(results.every(x => x.success), `Error in processing the dataset on multiple threads: projection ${dataset.name}`);
-            yield DatasetManager_1.default.mergeWorkersPaths(results.map(x => x.datasetPath), dataset);
-            dataset
-                .setDelimiter(results[0].datasetDelimiter)
-                .setDimensions(results[0].datasetDimensions);
-            dataset._finishOperation('transform-parallel');
-            return dataset;
-        });
-    }
-}
-const ParallelDataset = new ParallelDatasetClass();
-exports.default = ParallelDataset;

package/engines/dataset/TransformWorker.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/engines/dataset/definitions.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/engines/dataset/example-parallel-transform.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/engines/dataset/test-parallel.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/engines/deployment/DeploymentPlanner.js

@@ -1,39 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
-const Environment_1 = __importDefault(require("../Environment"));
-class DeploymentPlannerClass {
-    constructor() {
-        this.planConsumer = (consumer) => {
-            const [source, producer] = ConsumerManager_1.default.getSource(consumer);
-            const plan = [];
-            for (let i = 0; i < consumer.outputs.length; i++) {
-                const output = consumer.outputs[i];
-                switch (output.format) {
-                    // csv, json, parquet outputs do not need to generate anything at deploy
-                    case 'API': {
-                        throw new Error(`Invalid consumer "${consumer.name}" format "${output.format}": not implemented yet.`);
-                    }
-                }
-            }
-            return plan;
-        };
-        this.planProducer = (producer) => {
-            const source = Environment_1.default.getSource(producer.source);
-            const plan = [];
-            switch (source.engine) {
-                case 'aws-redshift': {
-                    if (!producer.settings.direct)
-                        plan.push({ type: 'create-view', producer: producer });
-                    break;
-                }
-            }
-            return plan;
-        };
-    }
-}
-const DeploymentPlanner = new DeploymentPlannerClass();
-exports.default = DeploymentPlanner;

package/engines/execution/ExecutionEnvironment.js

@@ -1,209 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
-const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
-const FileExporter_1 = __importDefault(require("../file/FileExporter"));
-const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
-const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
-const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
-const RequestExecutor_1 = __importDefault(require("./RequestExecutor"));
-const JoinEngine_1 = __importDefault(require("../transform/JoinEngine"));
-const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
-const Environment_1 = __importDefault(require("../Environment"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const Logger_1 = __importDefault(require("../../helper/Logger"));
-const ParallelDataset_1 = __importDefault(require("../dataset/ParallelDataset"));
-const ConsumerOnFinishManager_1 = __importDefault(require("../consumer/ConsumerOnFinishManager"));
-class ExecutionEnvironment {
-    constructor(consumer, executionId) {
-        this.run = (options) => __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d;
-            (0, Affirm_1.default)(this._consumer, 'Invalid consumer');
-            const plan = ExecutionPlanner_1.default.plan(this._consumer, options);
-            (0, Affirm_1.default)(plan, `Invalid execution plan`);
-            (0, Affirm_1.default)(plan.length > 0, `Empty execution plan`);
-            Logger_1.default.log(`Starting execution of consumer with plan:\n${plan.map(x => `\t- ${x.type}`).join('\n')}`);
-            const start = performance.now();
-            const result = {
-                shape: ConsumerEngine_1.default.getOutputShape(this._consumer),
-                _stats: { cycles: -1, elapsedMS: -1, operations: [], size: -1 }
-            };
-            let currentStep = null;
-            try {
-                for (const planStep of plan) {
-                    currentStep = planStep;
-                    Logger_1.default.log(`Started step: ${planStep.type} "${(_b = (_a = planStep.producer) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : this._consumer.name}"`);
-                    switch (planStep.type) {
-                        case 'compile-consumer-to-SQL': {
-                            const sql = SQLCompiler_1.default.getConsumerReference(this._consumer);
-                            this._envData.consumerSQL = sql;
-                            this._envData.finalSQL = sql;
-                            break;
-                        }
-                        case 'compile-execution-request-to-SQL': {
-                            const sql = SQLBuilder_1.default.buildConsumerQuery(options);
-                            this._envData.executionRequestSQL = sql;
-                            this._envData.finalSQL = `WITH consumer AS (${this._envData.consumerSQL})\nSELECT * FROM consumer${this._envData.executionRequestSQL}`;
-                            break;
-                        }
-                        case 'execute-SQL': {
-                            (0, Affirm_1.default)(planStep.source, `Invalid source in execute-SQL step`);
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in execute-SQL step`);
-                            const driver = yield DriverFactory_1.default.instantiateSource(planStep.source);
-                            const queryData = (yield driver.query(this._envData.finalSQL)).rows;
-                            let dataset = DatasetManager_1.default.create(planStep.producer, { cProducer: planStep.cProducer, executionId: this._executionId });
-                            dataset = yield dataset.loadFromMemory(queryData, planStep.producer);
-                            this._storeIntermidiate(planStep, dataset);
-                            break;
-                        }
-                        case 'load-dataset': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
-                            const { producer, cProducer } = planStep;
-                            const source = Environment_1.default.getSource(producer.source);
-                            (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
-                            let dataset = DatasetManager_1.default.create(producer, { cProducer, executionId: this._executionId });
-                            dataset = yield dataset.load(source);
-                            this._storeIntermidiate(planStep, dataset);
-                            break;
-                        }
-                        case 'prepare-dataset': {
-                            const rawDataset = this._getIntermidiate(planStep);
-                            (0, Affirm_1.default)(rawDataset, `Invalid dataset in prepare-dataset step`);
-                            const { producer } = planStep;
-                            const dataset = yield rawDataset.prepare(producer);
-                            this._storeIntermidiate(planStep, dataset);
-                            break;
-                        }
-                        case 'nested-field-unpacking': {
-                            (0, Affirm_1.default)(planStep.producer, `Invalid producer in nested-field-unpacking step`);
-                            const unpackedData = yield PostProcessor_1.default.unpack(this._resultingDataset, planStep.producer);
-                            this._storeIntermidiate(planStep, unpackedData);
-                            break;
-                        }
-                        case 'post-process-json': {
-                            const dataset = this._getIntermidiate(planStep);
-                            const newDataset = yield ParallelDataset_1.default.projection(dataset, this._consumer);
-                            this._storeIntermidiate(planStep, newDataset);
-                            break;
-                        }
-                        case 'export-file': {
-                            (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
-                            (0, Affirm_1.default)(this._resultingDataset, 'Invalid resulting dataset in export-file step');
-                            const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._resultingDataset, this._executionId);
-                            result.fileUri = res;
-                            break;
-                        }
-                        case 'apply-execution-request-to-result': {
-                            this._resultingDataset = yield RequestExecutor_1.default.execute(this._resultingDataset, options);
-                            break;
-                        }
-                        case 'apply-consumer-filters-on-JSON': {
-                            this._resultingDataset = yield ParallelDataset_1.default.filter(this._resultingDataset, this._consumer.filters);
-                            break;
-                        }
-                        case 'apply-transformations': {
-                            this._resultingDataset = yield ParallelDataset_1.default.transform(this._resultingDataset, this._consumer);
-                            break;
-                        }
-                        case 'join-producers-data': {
-                            const dataset = yield JoinEngine_1.default.join(this._consumer, this._producedData);
-                            this._resultingDataset = dataset;
-                            break;
-                        }
-                        case 'apply-consumer-distinct': {
-                            this._resultingDataset = yield PostProcessor_1.default.distinct(this._resultingDataset);
-                            break;
-                        }
-                        case 'perform-on-success-actions': {
-                            yield ConsumerOnFinishManager_1.default.performOnSuccessActions(this._consumer, planStep.output);
-                            break;
-                        }
-                        case 'save-execution-stats': {
-                            (0, Affirm_1.default)(this._resultingDataset, `Invalid result dataset in save-execution-stats`);
-                            result._stats = {
-                                cycles: this._resultingDataset.getCycles(),
-                                elapsedMS: performance.now() - start,
-                                operations: structuredClone(this._resultingDataset.getOperations()),
-                                size: this._resultingDataset.getCount()
-                            };
-                            break;
-                        }
-                        case 'clean-datasets': {
-                            const datasets = [
-                                ...this._producedData.map(x => x.dataset),
-                                this._resultingDataset
-                            ].filter(Algo_1.default.hasVal);
-                            const promises = datasets.map(x => x.destroy());
-                            yield Promise.all(promises);
-                            break;
-                        }
-                        default: throw new Error(`Invalid execution plan step type "${planStep.type}"`);
-                    }
-                    Logger_1.default.log(`\tCompleted step: ${planStep.type}`);
-                }
-            }
-            catch (error) {
-                const ds = (_c = this._resultingDataset) !== null && _c !== void 0 ? _c : (_d = this._producedData.at(-1)) === null || _d === void 0 ? void 0 : _d.dataset;
-                if (ds)
-                    Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getCount()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
-                Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
-                try {
-                    yield ConsumerOnFinishManager_1.default.performOnErrorActions(this._consumer, currentStep.output);
-                }
-                catch (error) {
-                    Logger_1.default.log(`Error when trying to perform onError actions on failed consumer ${error}`);
-                }
-                // IMPORTANT: cleanup all the datasets to not leave any data around and to avoid memory leaks
-                const datasets = [
-                    ...this._producedData.map(x => x.dataset),
-                    this._resultingDataset
-                ].filter(Algo_1.default.hasVal);
-                const promises = datasets.map(x => x.destroy());
-                yield Promise.all(promises);
-                throw error;
-            }
-            Logger_1.default.log(`Completed execution of consumer:\n\tSize: ${result._stats.size}\n\tCycles: ${result._stats.cycles}\n\tTime: ${result._stats.elapsedMS}\n\tOperations: ${Logger_1.default.formatList(result._stats.operations)}`);
-            return result;
-        });
-        this._storeIntermidiate = (step, dataset) => {
-            var _a, _b;
-            (0, Affirm_1.default)(step, 'Invalid step');
-            const key = (_b = (_a = step.producer) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : '_default_';
-            let pData = this._producedData.find(x => x.producerKey === key);
-            if (!pData) {
-                pData = { producerKey: key, dataset: null };
-                this._producedData.push(pData);
-            }
-            pData.dataset = dataset;
-        };
-        this._getIntermidiate = (step) => {
-            var _a, _b;
-            (0, Affirm_1.default)(step, 'Invalid step');
-            const key = (_b = (_a = step.producer) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : '_default_';
-            const produced = this._producedData.find(x => x.producerKey === key);
-            (0, Affirm_1.default)(produced, `No produced dataset found for step "${step.type}" of producer "${key}".`);
-            return produced.dataset;
-        };
-        this._consumer = consumer;
-        this._envData = { consumerSQL: null, executionRequestSQL: null, finalSQL: null };
-        this._producedData = [];
-        this._resultingDataset = null;
-        // A short unique id to isolate temp dataset files & output names
-        this._executionId = executionId;
-    }
-}
-exports.default = ExecutionEnvironment;

package/engines/execution/ExecutionPlanner.js

@@ -1,131 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
-const Environment_1 = __importDefault(require("../Environment"));
-class ExecutionPlannerClass {
-    constructor() {
-        this.getEngineClass = (engine) => {
-            switch (engine) {
-                case 'aws-dynamodb': return 'no-sql';
-                case 'aws-redshift':
-                case 'postgres': return 'sql';
-                case 'delta-share':
-                case 'aws-s3': return 'file';
-                case 'local': return 'local';
-            }
-        };
-        this.plan = (consumer, options) => {
-            var _a, _b, _c;
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const producersPlan = this._planProducers(consumer, options);
-            const plan = [...producersPlan];
-            // At this point I have created the .dataset and can perform operations on it
-            // TODO: how to handle pagination of SQL results?
-            // Apply the transformations to the fields of the consumer
-            // TODO: transformations can also be applied directly to the producer... how???
-            if (consumer.fields.some(x => Algo_1.default.hasVal(x.transform)))
-                plan.push({ type: 'apply-transformations' });
-            const [source] = ConsumerManager_1.default.getSource(consumer);
-            const engineClass = this.getEngineClass(source.engine);
-            for (const output of consumer.outputs) {
-                switch (output.format.toUpperCase()) {
-                    case 'JSON': {
-                        if (engineClass === 'file' && Algo_1.default.hasVal(options))
-                            plan.push({ type: 'apply-execution-request-to-result' });
-                        // TODO: test if it is needed and if it doesn't break soething else
-                        if (engineClass === 'sql')
-                            plan.push({ type: 'post-process-json', output });
-                        if ((_a = consumer.options) === null || _a === void 0 ? void 0 : _a.distinct)
-                            plan.push({ type: 'apply-consumer-distinct' });
-                        plan.push({ type: 'export-file', output });
-                        break;
-                    }
-                    case 'CSV':
-                    case 'PARQUET': {
-                        if ((_b = consumer.options) === null || _b === void 0 ? void 0 : _b.distinct)
-                            plan.push({ type: 'apply-consumer-distinct' });
-                        plan.push({ type: 'export-file', output });
-                        break;
-                    }
-                    case 'API': {
-                        if (engineClass === 'file' && Algo_1.default.hasVal(options))
-                            plan.push({ type: 'apply-execution-request-to-result' });
-                        if ((_c = consumer.options) === null || _c === void 0 ? void 0 : _c.distinct)
-                            plan.push({ type: 'apply-consumer-distinct' });
-                        break;
-                    }
-                    case 'SQL': {
-                        // TODO: what should I do here?? do I need to do anything?
-                        break;
-                    }
-                    default:
-                        throw new Error(`Output format "${output.format}" not supported`);
-                }
-                if (output.onSuccess && output.onSuccess.length > 0)
-                    plan.push({ type: 'perform-on-success-actions', output });
-            }
-            plan.push({ type: 'clean-datasets' });
-            plan.push({ type: 'save-execution-stats' });
-            return plan;
-        };
-        this._planProducers = (consumer, options) => {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
-            (0, Affirm_1.default)(producers, `Invalid producers on consumer "${consumer.name}"`);
-            (0, Affirm_1.default)(producers.every(x => Algo_1.default.hasVal(x)), `One or more producers of consumer "${consumer.name}" not found.`);
-            const sources = producers.map(x => Environment_1.default.getSource(x.source));
-            (0, Affirm_1.default)(sources, `Invalid sources on consumer "${consumer.name}"`);
-            (0, Affirm_1.default)(sources.every(x => Algo_1.default.hasVal(x)), `One or more sources of consumer "${consumer.name}" not found.`);
-            const engineClasses = sources.map(x => this.getEngineClass(x.engine));
-            const uniqEngineClasses = Algo_1.default.uniq(engineClasses);
-            const plan = [];
-            if (uniqEngineClasses.length === 1 && uniqEngineClasses[0] === 'sql')
-                plan.push(...this._planProducer(producers[0], consumer.producers[0], options));
-            else
-                plan.push(...(producers.flatMap((x, i) => this._planProducer(x, consumer.producers[i], options))));
-            // I technically don't need this, but I keep it to merge all the datasets to a single one
-            // so the other steps of the plan can work with a single dataset variable
-            plan.push({ type: 'join-producers-data' });
-            if (consumer.filters && consumer.filters.length > 0)
-                plan.push({ type: 'apply-consumer-filters-on-JSON' });
-            return plan;
-        };
-        this._planProducer = (producer, cProducer, options) => {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `Source "${producer.source}" of producer "${producer.name}" not found.`);
-            const plan = [];
-            const producerEngine = source.engine;
-            switch (producerEngine) {
-                case 'postgres':
-                case 'aws-redshift': {
-                    plan.push({ type: 'compile-consumer-to-SQL', producer, cProducer });
-                    if (Algo_1.default.hasVal(options))
-                        plan.push({ type: 'compile-execution-request-to-SQL', producer, cProducer });
-                    plan.push({ type: 'execute-SQL', source: source, producer, cProducer });
-                    break;
-                }
-                case 'local':
-                case 'aws-s3':
-                case 'delta-share':
-                case 'http-api': {
-                    plan.push({ type: 'load-dataset', producer, cProducer });
-                    plan.push({ type: 'prepare-dataset', producer, cProducer });
-                    if (producer.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
-                        plan.push({ type: 'nested-field-unpacking', producer, cProducer });
-                    plan.push({ type: 'post-process-json', producer, cProducer });
-                    break;
-                }
-                default: throw new Error(`Engine "${producerEngine}" not supported`);
-            }
-            return plan;
-        };
-    }
-}
-const ExecutionPlanner = new ExecutionPlannerClass();
-exports.default = ExecutionPlanner;

package/engines/file/FileCompiler.js

@@ -1,29 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-class FileCompilerClass {
-    constructor() {
-        this.compileProducer = (producer, source) => {
-            var _a;
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            (0, Affirm_1.default)(source, `Invalid source`);
-            (0, Affirm_1.default)((_a = producer.settings.fileKey) !== null && _a !== void 0 ? _a : producer.settings.sqlTable, `Missing required file key in producer settings`);
-            (0, Affirm_1.default)(producer.settings.fileType, `Missing required file type in producer settings`);
-            (0, Affirm_1.default)(!producer.measures || producer.measures.length === 0, `Cannot use "measure" with a producer linked to a file (only dimensions are allowed).`);
-            const columns = producer.dimensions.map(x => ({
-                aliasInProducer: x.alias,
-                nameInProducer: x.name,
-                consumerAlias: null,
-                consumerKey: null,
-                owner: producer.name,
-                dimension: x
-            }));
-            return columns;
-        };
-    }
-}
-const FileCompiler = new FileCompilerClass();
-exports.default = FileCompiler;

package/engines/file/FileContentBuilder.js

@@ -1,34 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Environment_1 = __importDefault(require("../Environment"));
-class FileContentBuilderClass {
-    constructor() {
-        /**
-         * Converts an array of string to a single string separated with the separator.
-         * In the V8 engine there is a maximum length to a string so I can't just join it all.
-         * I use this to create chunks that are not too long.
-         */
-        this.compose = (lines, separator) => {
-            Affirm_1.default.hasValue(lines, 'Invalid lines');
-            Affirm_1.default.hasValue(lines, 'Invalid separator');
-            const maxStringLength = parseInt(Environment_1.default.get('STRING_MAX_CHARACTERS_LENGTH'));
-            const chunks = [];
-            let currentChunk = '';
-            for (const line of lines) {
-                currentChunk += (line + separator);
-                if (currentChunk.length >= maxStringLength) {
-                    chunks.push(currentChunk);
-                    currentChunk = '';
-                }
-            }
-            chunks.push(currentChunk);
-            return chunks;
-        };
-    }
-}
-const FileContentBuilder = new FileContentBuilderClass();
-exports.default = FileContentBuilder;