@forzalabs/remora 0.0.12

package/engines/consumer/ConsumerManager.js

@@ -0,0 +1,155 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const Environment_1 = __importDefault(require("../Environment"));
+class ConsumerManagerClass {
+    constructor() {
+        this.getConsumerFlatFields = (consumer) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            return this.getFlatFields(consumer.fields);
+        };
+        this.getFlatFields = (list) => {
+            let result = [...list];
+            for (let i = 0; i < list.length; i++) {
+                const field = list[i];
+                if (field.grouping && field.grouping.subFields && field.grouping.subFields.length > 0)
+                    result = [...result, ...this.getFlatFields(field.grouping.subFields)];
+            }
+            return result;
+        };
+        /**
+         * Returns the full list of fields that are used by a consumer, while keeping the nested structure of fields.
+         * If there are *, then replace them with the actual fields found in the underlying producer/consumer
+         */
+        this.getExpandedFields = (consumer) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const availableColumns = this.getAvailableColumns(consumer);
+            const convertedFields = this.convertFields(consumer.fields);
+            const expandedFields = convertedFields.flatMap(x => this.expandField(consumer, x, availableColumns));
+            return expandedFields;
+        };
+        this.convertFields = (fieldsToConvert) => {
+            (0, Affirm_1.default)(fieldsToConvert, 'Invalid fields');
+            const convertedFields = fieldsToConvert.map(x => ({
+                cField: x
+            }));
+            return convertedFields;
+        };
+        /**
+         * Return all the available columns (dimensions and measures) to the consumer given its producers
+         */
+        this.getAvailableColumns = (consumer) => {
+            const availableColumns = consumer.producers.flatMap(cProd => {
+                var _a, _b;
+                const producer = Environment_1.default.getProducer(cProd.name);
+                if (!producer) {
+                    const subConsumer = Environment_1.default.getConsumer(cProd.name);
+                    (0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
+                    return this.getAvailableColumns(subConsumer);
+                }
+                else {
+                    const dims = producer.dimensions.map(x => ({
+                        consumerAlias: null,
+                        consumerKey: null,
+                        nameInProducer: x.name,
+                        aliasInProducer: x.alias,
+                        dimension: x,
+                        owner: cProd.name
+                    }));
+                    const meas = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(x => ({
+                        consumerAlias: null,
+                        consumerKey: null,
+                        nameInProducer: x.name,
+                        aliasInProducer: x.name,
+                        measure: x,
+                        owner: cProd.name
+                    }))) !== null && _b !== void 0 ? _b : [];
+                    return [...dims, ...meas];
+                }
+            });
+            return availableColumns;
+        };
+        /**
+         * If the field is '*' then replace them with the actual fields found in the underlying producer/consumer
+         */
+        this.expandField = (consumer, field, availableColumns) => {
+            var _a;
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(field, 'Invalid consumer field');
+            const expandedFields = [];
+            if (field.cField.key === '*') {
+                const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
+                availableColumns.filter(x => x.owner === from).forEach(col => {
+                    expandedFields.push({
+                        cField: {
+                            key: col.nameInProducer,
+                            alias: col.nameInProducer,
+                            from: col.owner
+                        },
+                        dimension: col.dimension,
+                        measure: col.measure
+                    });
+                });
+            }
+            else if (field.cField.grouping) {
+                expandedFields.push({
+                    cField: {
+                        key: field.cField.key,
+                        alias: field.cField.alias,
+                        from: field.cField.from,
+                        grouping: {
+                            groupingKey: field.cField.grouping.groupingKey,
+                            subFields: field.cField.grouping.subFields.flatMap(x => this.expandField(consumer, { cField: x }, availableColumns)).map(x => x.cField)
+                        }
+                    },
+                    dimension: field.dimension,
+                    measure: field.measure
+                });
+            }
+            else {
+                const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
+                (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
+                expandedFields.push(field);
+            }
+            return expandedFields;
+        };
+        this.searchFieldInColumns = (field, columns, consumer) => {
+            (0, Affirm_1.default)(field, 'Invalid field');
+            (0, Affirm_1.default)(columns, 'Invalid columns');
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            if (field.from) {
+                return columns.find(x => x.owner === field.from && x.nameInProducer === field.key);
+            }
+            else if (consumer.producers.length === 1 && !field.from) {
+                return columns.find(x => x.nameInProducer === field.key);
+            }
+            else {
+                const matches = columns.filter(x => x.nameInProducer === field.key);
+                (0, Affirm_1.default)(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map(x => x.name).join(', ')})`);
+                (0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
+                return matches[0];
+            }
+        };
+        this.getSource = (consumer) => {
+            const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
+            (0, Affirm_1.default)(producers.length > 0, 'No producers found');
+            (0, Affirm_1.default)(producers.every(x => x), `Invalid producer found in consumer "${consumer.name}"`);
+            const sources = producers.map(x => Environment_1.default.getSource(x.source));
+            (0, Affirm_1.default)(sources.length > 0, 'No sources found');
+            (0, Affirm_1.default)(sources.every(x => x), `Invalid source found in consumer "${consumer.name}"`);
+            // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
+            const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
+            (0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
+            // For now we also only support consumers that have producers ALL having the same exact source
+            const uniqNames = Algo_1.default.uniqBy(sources, 'name');
+            (0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
+            return [sources[0], producers[0]];
+        };
+    }
+}
+const ConsumerManager = new ConsumerManagerClass();
+exports.default = ConsumerManager;

package/engines/consumer/PostProcessor.js

@@ -0,0 +1,143 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
+const Environment_1 = __importDefault(require("../Environment"));
+const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
+class PostProcessorClass {
+    constructor() {
+        /**
+         * Get an array of objects and projects it to another array of objects but with different shape
+         * Basically does a field mapping with some extra logic (like grouping and creating nested fields)
+         */
+        this.doProjection = (consumer, data) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(data, 'Invalid data');
+            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
+            const subProjection = (allFields, items) => {
+                (0, Affirm_1.default)(allFields, 'Invalid fields');
+                (0, Affirm_1.default)(Array.isArray(allFields), 'Invalid fields type, must be an array');
+                (0, Affirm_1.default)(items, 'Invalid items');
+                (0, Affirm_1.default)(Array.isArray(items), 'Invalid items type, must be an array');
+                if (allFields.some(x => x.cField.grouping)) {
+                    (0, Affirm_1.default)(allFields.filter(x => x.cField.grouping).length === 1, `Mutliple fields at the same level were found having "grouping" attributes.`);
+                    const groupingRule = allFields.find(x => x.cField.grouping).cField.grouping;
+                    const groups = Algo_1.default.groupBy(items, groupingRule.groupingKey);
+                    const projections = [];
+                    groups.forEach(gItems => {
+                        var _a;
+                        const projected = {};
+                        const first = gItems[0];
+                        for (const field of allFields) {
+                            const { key, alias, grouping } = field.cField;
+                            const fieldKey = alias !== null && alias !== void 0 ? alias : key;
+                            const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
+                            if (!field.cField.grouping) {
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey]);
+                            }
+                            else {
+                                const { subFields } = grouping;
+                                const convertedSubFields = ConsumerManager_1.default.convertFields(subFields);
+                                projected[fieldKey] = subProjection(convertedSubFields, gItems);
+                            }
+                        }
+                        projections.push(projected);
+                    });
+                    return projections;
+                }
+                else {
+                    return items.map(x => {
+                        var _a;
+                        const projected = {};
+                        for (const field of allFields) {
+                            const { key, alias } = field.cField;
+                            const fieldKey = alias !== null && alias !== void 0 ? alias : key;
+                            const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
+                            projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[fieldKey]);
+                        }
+                        return projected;
+                    });
+                }
+            };
+            return subProjection(fields, data);
+        };
+        /**
+         * Gets an array of objects (with potentially nested fields) and unpacks them to an array of objects with no nested fields
+         * If some nested keys are lists, then a logic similar to a SQL JOIN is used and rows are duplicated
+         */
+        this.unpack = (data, producer) => {
+            (0, Affirm_1.default)(data, 'Invalid data');
+            (0, Affirm_1.default)(Array.isArray(data), 'Invalid data type, must be an array');
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            console.log('BBBBBBBBB');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const columns = FileCompiler_1.default.compileProducer(producer, source);
+            (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
+            const unpackDimension = (item, dimension) => {
+                var _a;
+                const { nameInProducer, aliasInProducer } = dimension;
+                const maskType = (_a = dimension.dimension.mask) !== null && _a !== void 0 ? _a : undefined;
+                const keys = aliasInProducer.split('.');
+                let prevValue = item;
+                for (const key of keys) {
+                    if (key.includes('{')) {
+                        const cleanedKey = key.replace('{', '').replace('}', '');
+                        if (Array.isArray(prevValue))
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.map((x) => x[cleanedKey]);
+                        else
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[cleanedKey];
+                    }
+                    else if (key.includes('[')) {
+                        const cleanedKey = key.replace('[', '').replace(']', '');
+                        if (Array.isArray(prevValue))
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.flatMap((x) => x[cleanedKey]);
+                        else
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[cleanedKey];
+                    }
+                    else {
+                        if (Array.isArray(prevValue))
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.map((x) => x[key]);
+                        else
+                            prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[key];
+                    }
+                }
+                prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue);
+                const res = { [nameInProducer]: prevValue };
+                return res;
+            };
+            const splitArrayFields = (item) => {
+                const keysWithArrayValues = Object.keys(item).filter(key => Array.isArray(item[key]));
+                if (keysWithArrayValues.length === 0)
+                    return [item];
+                const key = keysWithArrayValues[0];
+                const values = item[key];
+                const remainingItem = Object.assign({}, item);
+                delete remainingItem[key];
+                const splitRemaining = splitArrayFields(remainingItem);
+                return values.flatMap((value) => {
+                    return splitRemaining.map((remaining) => {
+                        return Object.assign(Object.assign({}, remaining), { [key]: value });
+                    });
+                });
+            };
+            const unpackSingle = (item) => {
+                const unpackedItem = {};
+                for (const column of columns) {
+                    const value = unpackDimension(item, column);
+                    Object.assign(unpackedItem, value);
+                }
+                return splitArrayFields(unpackedItem);
+            };
+            const unpackedData = data.flatMap(x => unpackSingle(x));
+            return unpackedData;
+        };
+    }
+}
+const PostProcessor = new PostProcessorClass();
+exports.default = PostProcessor;

package/engines/deployment/DeploymentPlanner.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
+const Environment_1 = __importDefault(require("../Environment"));
+class DeploymentPlannerClass {
+    constructor() {
+        this.planConsumer = (consumer) => {
+            const [source, producer] = ConsumerManager_1.default.getSource(consumer);
+            const plan = [];
+            for (let i = 0; i < consumer.outputs.length; i++) {
+                const output = consumer.outputs[i];
+                switch (output.format) {
+                    // csv, json, parquet outputs do not need to generate anything at deploy
+                    case 'SQL': {
+                        if (output.accellerated && !output.direct)
+                            plan.push({ type: 'create-materialized-view', output: output });
+                        else if (!output.direct)
+                            plan.push({ type: 'create-view', output: output });
+                        break;
+                    }
+                    case 'API': {
+                        throw new Error(`Invalid consumer "${consumer.name}" format "${output.format}": not implemented yet.`);
+                    }
+                }
+            }
+            return plan;
+        };
+        this.planProducer = (producer) => {
+            const source = Environment_1.default.getSource(producer.source);
+            const plan = [];
+            switch (source.engine) {
+                case 'aws-redshift': {
+                    if (!producer.settings.direct)
+                        plan.push({ type: 'create-view', producer: producer });
+                    break;
+                }
+            }
+            return plan;
+        };
+    }
+}
+const DeploymentPlanner = new DeploymentPlannerClass();
+exports.default = DeploymentPlanner;

package/engines/execution/ExecutionEnvironment.js

@@ -0,0 +1,114 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const PostProcessor_1 = __importDefault(require("../consumer/PostProcessor"));
+const FileExporter_1 = __importDefault(require("../file/FileExporter"));
+const ParseManager_1 = __importDefault(require("../ParseManager"));
+const ProducerEngine_1 = __importDefault(require("../ProducerEngine"));
+const SQLBuilder_1 = __importDefault(require("../sql/SQLBuilder"));
+const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
+const ExecutionPlanner_1 = __importDefault(require("./ExecutionPlanner"));
+const RequestExecutor_1 = __importDefault(require("./RequestExecutor"));
+class ExecutionEnvironment {
+    constructor(consumer) {
+        this.run = (options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._consumer, 'Invalid consumer');
+            const plan = ExecutionPlanner_1.default.plan(this._consumer, options);
+            (0, Affirm_1.default)(plan, `Invalid execution plan`);
+            (0, Affirm_1.default)(plan.length > 0, `Empty execution plan`);
+            const result = { shape: ConsumerEngine_1.default.getOutputShape(this._consumer) };
+            for (const planStep of plan) {
+                switch (planStep.type) {
+                    case 'compile-consumer-to-SQL': {
+                        const sql = SQLCompiler_1.default.getConsumerReference(this._consumer);
+                        this._data.consumerSQL = sql;
+                        this._data.finalSQL = sql;
+                        break;
+                    }
+                    case 'compile-execution-request-to-SQL': {
+                        const sql = SQLBuilder_1.default.buildConsumerQuery(options);
+                        this._data.executionRequestSQL = sql;
+                        this._data.finalSQL = `WITH consumer AS (${this._data.consumerSQL})\nSELECT * FROM consumer${this._data.executionRequestSQL}`;
+                        break;
+                    }
+                    case 'execute-SQL': {
+                        (0, Affirm_1.default)(planStep.source, `Invalid source in execute-SQL step`);
+                        const driver = yield DriverFactory_1.default.instantiateSource(planStep.source);
+                        this._fetchedData = (yield driver.query(this._data.finalSQL)).rows;
+                        this._fetchedDataType = 'array-of-json';
+                        break;
+                    }
+                    case 'read-file-whole': {
+                        (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-whole step`);
+                        const fileData = yield ProducerEngine_1.default.readFile(planStep.producer, { readmode: 'all' });
+                        this._fetchedData = fileData.data;
+                        this._fetchedDataType = fileData.dataType;
+                        break;
+                    }
+                    case 'read-file-lines': {
+                        (0, Affirm_1.default)(planStep.lines, `Invalid lines in read-file-lines step`);
+                        (0, Affirm_1.default)(planStep.producer, `Invalid producer in read-file-lines step`);
+                        const { producer, lines: { from, to } } = planStep;
+                        const fileData = yield ProducerEngine_1.default.readFile(producer, { readmode: 'lines', lines: { from, to } });
+                        this._fetchedData = fileData.data;
+                        this._fetchedDataType = fileData.dataType;
+                        break;
+                    }
+                    case 'nested-field-unpacking': {
+                        (0, Affirm_1.default)(planStep.producer, `Invalid producer in nested-field-unpacking step`);
+                        this._fetchedData = PostProcessor_1.default.unpack(this._fetchedData, planStep.producer);
+                        break;
+                    }
+                    case 'post-process-json': {
+                        this._fetchedData = PostProcessor_1.default.doProjection(this._consumer, this._fetchedData);
+                        break;
+                    }
+                    case 'csv-to-json': {
+                        (0, Affirm_1.default)(this._fetchedData, 'Invalid data');
+                        (0, Affirm_1.default)(Array.isArray(this._fetchedData), 'Invalid data type, must be an array');
+                        (0, Affirm_1.default)(planStep.producer, `Invalid producer in csv-to-json step`);
+                        const csv = this._fetchedData[0];
+                        this._fetchedData = ParseManager_1.default.csvToJson(csv, planStep.producer);
+                        break;
+                    }
+                    case 'export-file': {
+                        (0, Affirm_1.default)(planStep.output, `Invalid output in export-file step`);
+                        const res = yield FileExporter_1.default.export(this._consumer, planStep.output, this._fetchedData);
+                        result.fileUri = res;
+                        break;
+                    }
+                    case 'apply-execution-request-to-result': {
+                        this._fetchedData = RequestExecutor_1.default.execute(this._fetchedData, options);
+                        break;
+                    }
+                    case 'apply-consumer-filters-on-JSON': {
+                        this._fetchedData = RequestExecutor_1.default._applyFilters(this._fetchedData, this._consumer.filters.map(x => x.rule));
+                        break;
+                    }
+                    default: throw new Error(`Invalid execution plan step type "${planStep.type}"`);
+                }
+            }
+            result.data = this._fetchedData;
+            return result;
+        });
+        this._consumer = consumer;
+        this._data = { consumerSQL: null, executionRequestSQL: null, finalSQL: null };
+        this._fetchedData = [];
+        this._fetchedDataType = null;
+    }
+}
+exports.default = ExecutionEnvironment;

package/engines/execution/ExecutionPlanner.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
+class ExecutionPlannerClas {
+    constructor() {
+        this.getEngineClass = (engine) => {
+            switch (engine) {
+                case 'aws-dynamodb': return 'no-sql';
+                case 'aws-redshift':
+                case 'postgres': return 'sql';
+                case 'aws-s3': return 'file';
+                case 'local': return 'local';
+            }
+        };
+        this.plan = (consumer, options) => {
+            var _a;
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const [source, producer] = ConsumerManager_1.default.getSource(consumer);
+            const producerEngine = source.engine;
+            const plan = [];
+            switch (producerEngine) {
+                case 'postgres':
+                case 'aws-redshift': {
+                    plan.push({ type: 'compile-consumer-to-SQL' });
+                    if (Algo_1.default.hasVal(options))
+                        plan.push({ type: 'compile-execution-request-to-SQL' });
+                    plan.push({ type: 'execute-SQL', source: source });
+                    break;
+                }
+                case 'local':
+                case 'aws-s3': {
+                    const prod = producer;
+                    if (Algo_1.default.hasVal(options) && (options.limit || options.offset))
+                        plan.push({ type: 'read-file-lines', producer: prod, lines: { from: (_a = options.offset) !== null && _a !== void 0 ? _a : 0, to: options.limit ? (options.offset + options.limit) : undefined } });
+                    else
+                        plan.push({ type: 'read-file-whole', producer: prod });
+                    if (prod.settings.fileType.toUpperCase() === 'CSV') {
+                        plan.push({ type: 'csv-to-json', producer: prod });
+                    }
+                    if (prod.dimensions.some(x => { var _a, _b; return ((_a = x.alias) === null || _a === void 0 ? void 0 : _a.includes('{')) || ((_b = x.alias) === null || _b === void 0 ? void 0 : _b.includes('[')); }))
+                        plan.push({ type: 'nested-field-unpacking', producer: prod });
+                    plan.push({ type: 'post-process-json' });
+                    if (consumer.filters && consumer.filters.length > 0)
+                        plan.push({ type: 'apply-consumer-filters-on-JSON' });
+                    break;
+                }
+                default: throw new Error(`Engine "${producerEngine}" not supported`);
+            }
+            // at this point I have the data loaded in memory
+            // TODO: can I handle streaming data? (e.g. a file that is too big to fit in memory)
+            // TODO: how to handle pagination of SQL results?
+            const engineClass = this.getEngineClass(producerEngine);
+            for (const output of consumer.outputs) {
+                switch (output.format) {
+                    case 'JSON': {
+                        if (engineClass === 'file' && Algo_1.default.hasVal(options))
+                            plan.push({ type: 'apply-execution-request-to-result' });
+                        // TODO: test if it is need ed and if it doesn't break soething else
+                        if (engineClass === 'sql')
+                            plan.push({ type: 'post-process-json' });
+                        plan.push({ type: 'export-file', output });
+                        break;
+                    }
+                    case 'CSV':
+                    case 'PARQUET': {
+                        plan.push({ type: 'export-file', output });
+                        break;
+                    }
+                    case 'API': {
+                        if (engineClass === 'file' && Algo_1.default.hasVal(options))
+                            plan.push({ type: 'apply-execution-request-to-result' });
+                        break;
+                    }
+                    case 'SQL': {
+                        // TODO: what should I do here?? do I need to do anything?
+                        break;
+                    }
+                    default:
+                        throw new Error(`Output format "${output.format}" not supported`);
+                }
+            }
+            return plan;
+        };
+    }
+}
+const ExecutionPlanner = new ExecutionPlannerClas();
+exports.default = ExecutionPlanner;

package/engines/execution/RequestExecutor.js

@@ -0,0 +1,100 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+class RequestExecutorClass {
+    constructor() {
+        /**
+         * Applies the filters, limit, offset and order on the in-memory-data
+         */
+        this.execute = (data, request) => {
+            (0, Affirm_1.default)(data, 'Invalid data');
+            (0, Affirm_1.default)(request, 'Invalid request');
+            (0, Affirm_1.default)(Array.isArray(data), `Invalid data type: should be an array`);
+            if (request.filters)
+                data = this._applyFilters(data, request.filters);
+            if (request.order)
+                data = this._applyOrdering(data, request.order);
+            return data;
+        };
+        this._applyFilters = (data, filters) => {
+            return data.filter((item) => {
+                return filters.every(filter => this._evaluateFilter(item, filter));
+            });
+        };
+        this._evaluateFilter = (item, filter) => {
+            const evaluate = (baseItem, baseFilter) => {
+                const { member, operator, values } = baseFilter;
+                const value = baseItem[member];
+                const singleValue = values[0];
+                switch (operator) {
+                    case 'equals':
+                        return value === singleValue;
+                    case 'notEquals':
+                        return value !== singleValue;
+                    case 'contains':
+                        return typeof value === 'string' && value.includes(singleValue);
+                    case 'notContains':
+                        return typeof value === 'string' && !value.includes(singleValue);
+                    case 'startsWith':
+                        return typeof value === 'string' && value.startsWith(singleValue);
+                    case 'endsWith':
+                        return typeof value === 'string' && value.endsWith(singleValue);
+                    case 'greaterThan':
+                        return typeof value === 'number' && value > Number(singleValue);
+                    case 'greaterThanOrEquals':
+                        return typeof value === 'number' && value >= Number(singleValue);
+                    case 'lessThan':
+                        return typeof value === 'number' && value < Number(singleValue);
+                    case 'lessThanOrEquals':
+                        return typeof value === 'number' && value <= Number(singleValue);
+                    case 'in':
+                        return values.includes(value);
+                    case 'notIn':
+                        return !values.includes(value);
+                    case 'between':
+                        return values.length === 2 && value >= values[0] && value <= values[1];
+                    case 'notBetween':
+                        return values.length === 2 && (value < values[0] || value > values[1]);
+                    case 'isNull':
+                        return value === null || value === undefined;
+                    case 'isNotNull':
+                        return value !== null && value !== undefined;
+                    case 'true':
+                        return value === true;
+                    case 'false':
+                        return value === false;
+                    case 'matches':
+                        return typeof value === 'string' && new RegExp(singleValue).test(value);
+                    case 'notMatches':
+                        return typeof value === 'string' && !new RegExp(singleValue).test(value);
+                    default:
+                        throw new Error(`Unsupported filter operator: ${operator}`);
+                }
+            };
+            const { and, or } = filter;
+            const baseResult = evaluate(item, filter);
+            if (and)
+                return baseResult && and.every(subFilter => this._evaluateFilter(item, subFilter));
+            if (or)
+                return baseResult || or.some(subFilter => this._evaluateFilter(item, subFilter));
+            else
+                return baseResult;
+        };
+        this._applyOrdering = (data, order) => {
+            return data.sort((a, b) => {
+                for (const [field, direction] of order) {
+                    if (a[field] < b[field])
+                        return direction === 'asc' ? -1 : 1;
+                    if (a[field] > b[field])
+                        return direction === 'asc' ? 1 : -1;
+                }
+                return 0;
+            });
+        };
+    }
+}
+const RequestExecutor = new RequestExecutorClass();
+exports.default = RequestExecutor;