@forzalabs/remora 0.2.5 → 1.0.0

package/engines/consumer/ConsumerEngine.js

@@ -1,28 +1,11 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-const Helper_1 = __importDefault(require("../../helper/Helper"));
-const DeploymentPlanner_1 = __importDefault(require("../deployment/DeploymentPlanner"));
 const Environment_1 = __importDefault(require("../Environment"));
-const ExecutionEnvironment_1 = __importDefault(require("../execution/ExecutionEnvironment"));
 const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
-const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
-const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
-const UsageManager_1 = __importDefault(require("../usage/UsageManager"));
 const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
 class ConsumerEngineClass {
     constructor() {
@@ -70,10 +53,6 @@ class ConsumerEngineClass {
                         selectedColumns.push(col);
                     });
                 }
-                else if (field.grouping) {
-                    // This field should be ignored since it is only created when building the output for supported formats (json)
-                    continue;
-                }
                 else {
                     const col = ConsumerManager_1.default.searchFieldInColumns(field, availableColumns, consumer);
                     (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
@@ -86,62 +65,6 @@ class ConsumerEngineClass {
             (0, Affirm_1.default)(columnsWithNoAlias.length === 0, `Consumer "${consumer.name}" compilation error: some selected fields don't have a correct alias or key (${columnsWithNoAlias.map(x => x.nameInProducer).join(', ')})`);
             return selectedColumns;
         };
-        this.deploy = (consumer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            const firstProd = Environment_1.default.getFirstProducer(consumer.producers[0].name);
-            (0, Affirm_1.default)(firstProd, `Missing producer in consumer "${consumer.name}"`);
-            // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
-            const allSources = consumer.producers.map(x => Environment_1.default.getSource(Environment_1.default.getProducer(x.name).source));
-            const uniqEngines = Algo_1.default.uniqBy(allSources, 'engine');
-            (0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in a single consumer (${uniqEngines.join(', ')})`);
-            // For now we also only support consumers that have producers ALL having the same exact source
-            const uniqNames = Algo_1.default.uniqBy(allSources, 'name');
-            (0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in a single consumer (${uniqNames.join(', ')})`);
-            const source = Environment_1.default.getSource(firstProd.source);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            const plan = DeploymentPlanner_1.default.planConsumer(consumer);
-            for (const planStep of plan) {
-                switch (planStep.type) {
-                    case 'create-materialized-view': {
-                        const sql = SQLCompiler_1.default.compileConsumer(consumer);
-                        (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
-                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
-                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
-                        // TODO When I want to update a materialize view there is no way except killing it and recreating it. The problem is that: 1) it is not said that it can be deleted since that materialize view could have some dependencies 2) we should find a way to update it without it going completely offline.
-                        const mvSQL = `
-                    DROP MATERIALIZED VIEW IF EXISTS "${internalSchema}"."${SQLUtils_1.default.acceleratedViewName(consumer.name)}";
-                    CREATE MATERIALIZED VIEW "${internalSchema}"."${SQLUtils_1.default.acceleratedViewName(consumer.name)}" AS ${sql}`;
-                        yield driver.execute(mvSQL);
-                        break;
-                    }
-                    case 'create-view': {
-                        const sql = SQLCompiler_1.default.compileConsumer(consumer);
-                        (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
-                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
-                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
-                        const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.sanitizeName(consumer.name)}" AS ${sql}`;
-                        yield driver.execute(vSQL);
-                        break;
-                    }
-                    default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
-                }
-            }
-        });
-        this.execute = (consumer, options, user, details) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            (0, Affirm_1.default)(options, `Invalid execute consume options`);
-            const { usageId } = UsageManager_1.default.startUsage(consumer, user, details);
-            try {
-                const execution = new ExecutionEnvironment_1.default(consumer, usageId);
-                const result = yield execution.run(options);
-                UsageManager_1.default.endUsage(usageId, result._stats.size);
-                return result;
-            }
-            catch (error) {
-                UsageManager_1.default.failUsage(usageId, Helper_1.default.asError(error).message);
-                throw error;
-            }
-        });
         this.getOutputShape = (consumer) => {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             const compiled = this.compile(consumer);

package/engines/consumer/ConsumerManager.js

@@ -6,21 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Environment_1 = __importDefault(require("../Environment"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 class ConsumerManagerClass {
     constructor() {
-        this.getConsumerFlatFields = (consumer) => {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            return this.getFlatFields(consumer.fields);
-        };
-        this.getFlatFields = (list) => {
-            let result = [...list];
-            for (let i = 0; i < list.length; i++) {
-                const field = list[i];
-                if (field.grouping && field.grouping.subFields && field.grouping.subFields.length > 0)
-                    result = [...result, ...this.getFlatFields(field.grouping.subFields)];
-            }
-            return result;
-        };
         /**
          * Returns the full list of fields that are used by a consumer, while keeping the nested structure of fields.
          * If there are *, then replace them with the actual fields found in the underlying producer/consumer
@@ -28,17 +16,16 @@ class ConsumerManagerClass {
         this.getExpandedFields = (consumer) => {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             const availableColumns = this.getAvailableColumns(consumer);
-            const convertedFields = this.convertFields(consumer.fields);
+            const convertedFields = consumer.fields.map(x => {
+                var _a;
+                return ({
+                    cField: x,
+                    finalKey: (_a = x.alias) !== null && _a !== void 0 ? _a : x.key
+                });
+            });
             const expandedFields = convertedFields.flatMap(x => this.expandField(consumer, x, availableColumns));
             return expandedFields;
         };
-        this.convertFields = (fieldsToConvert) => {
-            (0, Affirm_1.default)(fieldsToConvert, 'Invalid fields');
-            const convertedFields = fieldsToConvert.map(x => ({
-                cField: x
-            }));
-            return convertedFields;
-        };
         /**
          * Return all the available columns (dimensions and measures) to the consumer given its producers
          */
@@ -92,6 +79,7 @@ class ConsumerManagerClass {
                             alias: x.nameInProducer,
                             from: x.owner
                         },
+                        finalKey: x.nameInProducer,
                         dimension: x.dimension,
                         measure: x.measure
                     })));
@@ -105,27 +93,13 @@ class ConsumerManagerClass {
                                 alias: col.nameInProducer,
                                 from: col.owner
                             },
+                            finalKey: col.nameInProducer,
                             dimension: col.dimension,
                             measure: col.measure
                         });
                     });
                 }
             }
-            else if (field.cField.grouping) {
-                expandedFields.push({
-                    cField: {
-                        key: field.cField.key,
-                        alias: field.cField.alias,
-                        from: field.cField.from,
-                        grouping: {
-                            groupingKey: field.cField.grouping.groupingKey,
-                            subFields: field.cField.grouping.subFields.flatMap(x => this.expandField(consumer, { cField: x }, availableColumns)).map(x => x.cField)
-                        }
-                    },
-                    dimension: field.dimension,
-                    measure: field.measure
-                });
-            }
             else {
                 const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
                 (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
@@ -187,6 +161,57 @@ class ConsumerManagerClass {
             (0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
             return [sources[0], producers[0]];
         };
+        this.compile = (consumer) => {
+            var _a, _b;
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const availableColumns = this.getAvailableColumns(consumer);
+            const selectedColumns = [];
+            const flat = consumer.fields;
+            for (let i = 0; i < flat.length; i++) {
+                const field = flat[i];
+                // TODO: replace with the new funcitons in the consumermanager to reduce diplicate code
+                if (field.key === '*') {
+                    const from = (_a = field.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
+                    availableColumns.filter(x => x.owner === from).forEach(col => {
+                        col.consumerKey = col.nameInProducer;
+                        col.consumerAlias = col.nameInProducer;
+                        selectedColumns.push(col);
+                    });
+                }
+                else {
+                    const col = ConsumerManager.searchFieldInColumns(field, availableColumns, consumer);
+                    (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
+                    col.consumerKey = field.key;
+                    col.consumerAlias = (_b = field.alias) !== null && _b !== void 0 ? _b : field.key;
+                    selectedColumns.push(col);
+                }
+            }
+            const columnsWithNoAlias = selectedColumns.filter(x => !x.consumerAlias || !x.consumerKey);
+            (0, Affirm_1.default)(columnsWithNoAlias.length === 0, `Consumer "${consumer.name}" compilation error: some selected fields don't have a correct alias or key (${columnsWithNoAlias.map(x => x.nameInProducer).join(', ')})`);
+            return selectedColumns;
+        };
+        this.getOutputShape = (consumer) => {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const compiled = this.compile(consumer);
+            const outDimensions = compiled.map(x => {
+                var _a, _b, _c, _d, _e, _f, _g;
+                return ({
+                    name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
+                    type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
+                    classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
+                    description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
+                    mask: ProducerManager_1.default.getMask(x.dimension),
+                    pk: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.pk
+                });
+            });
+            return {
+                _version: consumer._version,
+                name: consumer.name,
+                description: consumer.description,
+                metadata: consumer.metadata,
+                dimensions: outDimensions
+            };
+        };
     }
 }
 const ConsumerManager = new ConsumerManagerClass();

package/engines/consumer/ConsumerOnFinishManager.js

@@ -17,6 +17,20 @@ const Environment_1 = __importDefault(require("../Environment"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 class ConsumerOnFinishManagerClass {
     constructor() {
+        this.onConsumerSuccess = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
+            void executionId;
+            for (const output of consumer.outputs) {
+                if (output.onSuccess)
+                    yield this.performOnSuccessActions(consumer, output);
+            }
+        });
+        this.onConsumerError = (consumer, executionId) => __awaiter(this, void 0, void 0, function* () {
+            void executionId;
+            for (const output of consumer.outputs) {
+                if (output.onError)
+                    yield this.performOnSuccessActions(consumer, output);
+            }
+        });
         this.performOnSuccessActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
             (0, Affirm_1.default)(output, 'Invalid output');

package/engines/consumer/PostProcessor.js

@@ -46,9 +46,8 @@ class PostProcessorClass {
                     }
                     record.sortDimensions();
                 }
-                if (!updatedDimensions) {
+                if (!updatedDimensions)
                     updatedDimensions = record._dimensions;
-                }
                 // Finally apply the rules and changes of the consumer fields to the record
                 for (const field of fields) {
                     const { key, alias } = field.cField;
@@ -71,11 +70,6 @@ class PostProcessorClass {
                     fakeRecord.wholeUpdateDimension(update);
                 updatedDimensions = fakeRecord._dimensions;
             }
-            // Validate that dimensions have sequential indexes with no gaps
-            const indexes = updatedDimensions.map(d => d.index).sort((a, b) => a - b);
-            for (let i = 0; i < indexes.length; i++) {
-                (0, Affirm_1.default)(indexes[i] === i, `Missing or duplicate dimension index: expected index ${i} but found ${indexes[i]}. See dimension updates applied on consumer "${consumer.name}".`);
-            }
             dataset.setDimensions(updatedDimensions);
             return dataset;
         });

package/engines/dataset/Dataset.js

@@ -27,9 +27,7 @@ const Constants_1 = __importDefault(require("../../Constants"));
 const DatasetManager_1 = __importDefault(require("./DatasetManager"));
 const DatasetRecord_1 = __importDefault(require("./DatasetRecord"));
 const DatasetRecordPool_1 = __importDefault(require("./DatasetRecordPool"));
-const xlsx_1 = __importDefault(require("xlsx"));
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const XMLParser_1 = __importDefault(require("../parsing/XMLParser"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
@@ -665,65 +663,6 @@ class Dataset {
             this._finishOperation('read-lines');
             return results;
         });
-        /**
-         * - computes dimensions + delimiter
-         * - cleans data (removes empty rows) and headers
-         * - parses json, xml, xls
-         */
-        this.prepare = (producer) => __awaiter(this, void 0, void 0, function* () {
-            this._startOperation('prepare');
-            const dimsRes = yield DatasetManager_1.default.buildDimensions(this, producer);
-            yield this.prepareWithDimensions(dimsRes);
-            this._finishOperation('prepare');
-            return this;
-        });
-        this.prepareWithDimensions = (dimResult) => __awaiter(this, void 0, void 0, function* () {
-            this._startOperation('prepare-with-dimensions');
-            const { delimiter, dimensions } = dimResult;
-            this._delimiter = delimiter;
-            this._dimensions = dimensions;
-            switch (this._file.fileType) {
-                case 'TXT':
-                case 'CSV':
-                case 'JSON':
-                case 'JSONL':
-                    break;
-                case 'XLS':
-                case 'XLSX': {
-                    const excel = xlsx_1.default.readFile(this._path);
-                    let targetSheetName = this._file.sheetName;
-                    if (!targetSheetName) {
-                        (0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
-                        targetSheetName = excel.SheetNames[0];
-                    }
-                    else {
-                        (0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
-                    }
-                    const sheet = excel.Sheets[targetSheetName];
-                    const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
-                    const lines = csv.split('\n');
-                    this.clear();
-                    const records = lines.map(x => new DatasetRecord_1.default(x, this._dimensions, this._delimiter));
-                    yield this.append(records);
-                    break;
-                }
-                case 'XML': {
-                    const fileContent = fs_1.default.readFileSync(this._path, 'utf-8');
-                    const jsonData = XMLParser_1.default.xmlToJson(fileContent);
-                    // Convert JSON data to string lines. This might need adjustment based on XML structure.
-                    // Assuming jsonData is an array of objects, where each object is a record.
-                    const lines = Array.isArray(jsonData)
-                        ? jsonData.map(item => JSON.stringify(item))
-                        : [JSON.stringify(jsonData)];
-                    this.clear();
-                    const records = lines.map(x => new DatasetRecord_1.default(x, this._dimensions, this._delimiter));
-                    yield this.append(records);
-                    break;
-                }
-            }
-            this._finishOperation('prepare-with-dimensions');
-            return this;
-        });
         this.getDimensions = () => this._dimensions;
         this.setDimensions = (dimensions) => {
             this._dimensions = dimensions;

package/engines/dataset/DatasetManager.js

@@ -13,16 +13,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const Algo_1 = __importDefault(require("../../core/Algo"));
-const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
-const Environment_1 = __importDefault(require("../Environment"));
-const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
 const ParseManager_1 = __importDefault(require("../parsing/ParseManager"));
 const Dataset_1 = __importDefault(require("./Dataset"));
-const promises_1 = require("stream/promises");
-const fs_1 = __importDefault(require("fs"));
 const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
 const Constants_1 = __importDefault(require("../../Constants"));
+const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
 class DatasetManagerClass {
     constructor() {
         /**
@@ -55,13 +50,6 @@ class DatasetManagerClass {
             });
             return dataset;
         };
-        this.buildDimensions = (dataset_1, producer_1, ...args_1) => __awaiter(this, [dataset_1, producer_1, ...args_1], void 0, function* (dataset, producer, discover = false) {
-            (0, Affirm_1.default)(dataset, `Invalid dataset`);
-            (0, Affirm_1.default)(producer, `Invalid producer`);
-            const firstLine = dataset.getFirstLine();
-            Affirm_1.default.hasValue(firstLine, `The first line of the dataset was not set.`);
-            return this.buildDimensionsFromFirstLine(firstLine, dataset.getFile(), producer, discover);
-        });
         this.buildDimensionsFromFirstLine = (firstLine_1, dsFile_1, producer_1, ...args_1) => __awaiter(this, [firstLine_1, dsFile_1, producer_1, ...args_1], void 0, function* (firstLine, dsFile, producer, discover = false) {
             var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
             Affirm_1.default.hasValue(firstLine, `Invalid first line`);
@@ -69,8 +57,6 @@ class DatasetManagerClass {
             (0, Affirm_1.default)(producer, `Invalid producer`);
             const file = dsFile;
             switch (file.fileType) {
-                case 'XLS':
-                case 'XLSX':
                 case 'CSV': {
                     const delimiterChar = (_a = file.delimiter) !== null && _a !== void 0 ? _a : ',';
                     const headerLine = firstLine;
@@ -88,12 +74,17 @@ class DatasetManagerClass {
                 }
                 case 'PARQUET':
                 case 'JSONL':
-                case 'XML':
                 case 'JSON': {
-                    const source = Environment_1.default.getSource(producer.source);
-                    const columns = FileCompiler_1.default.compileProducer(producer, source);
+                    const columns = ProducerManager_1.default.getColumns(producer);
                     const firstObject = JSON.parse(firstLine);
                     const keys = Object.keys(firstObject);
+                    // const columnsWithDot = columns.filter(x => x.aliasInProducer.includes('.'))
+                    // if (columnsWithDot.length > 0) {
+                    //     console.log(columns, keys, 'PAPAPAPP')
+                    //     for (const colWithDot of columnsWithDot) {
+                    //         console.log(colWithDot)
+                    //     }
+                    // }
                     // If includeSourceFilename is enabled, the driver has added $source_filename column
                     // We need to add it to the keys list so dimensions can reference it
                     const includeSourceFilename = file.includeSourceFilename === true;
@@ -146,8 +137,7 @@ class DatasetManagerClass {
                     if (!file.hasHeaderRow) {
                         // If the file is a TXT and there isn't an header row, then I add a fake one that maps directly to the producer
                         const delimiterChar = (_k = file.delimiter) !== null && _k !== void 0 ? _k : ',';
-                        const source = Environment_1.default.getSource(producer.source);
-                        const columns = FileCompiler_1.default.compileProducer(producer, source);
+                        const columns = ProducerManager_1.default.getColumns(producer);
                         const includeSourceFilename = file.includeSourceFilename === true;
                         if (discover) {
                             // Since I don't have an header, and I'm discovering, I just create placeholder dimensions based on the same number of columns of the txt
@@ -207,64 +197,14 @@ class DatasetManagerClass {
                         };
                     }
                 }
+                case 'XLS':
+                    break;
+                case 'XLSX':
+                    break;
+                case 'XML':
+                    break;
             }
         });
-        this.computeDimensionsUpdates = (dataset, consumer) => {
-            var _a;
-            (0, Affirm_1.default)(dataset, 'Invalid dataset');
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            const fields = ConsumerManager_1.default.getExpandedFields(consumer);
-            const dimensions = dataset.getDimensions();
-            let updates = [];
-            // Add all the updates
-            for (let i = 0; i < fields.length; i++) {
-                const { cField } = fields[i];
-                const currentMatch = structuredClone(dimensions.find(x => x.name === cField.key));
-                if (!currentMatch && !cField.fixed)
-                    throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying dataset "${dataset.name}" (${dimensions.map(x => x.name).join(', ')})`);
-                updates.push({
-                    currentDimension: currentMatch,
-                    newName: (_a = cField.alias) !== null && _a !== void 0 ? _a : cField.key,
-                    newHidden: cField.hidden,
-                    newPosition: i,
-                    toDelete: false
-                });
-            }
-            // Add all the updates to remove dimensions
-            for (const dim of dimensions) {
-                if (!updates.find(x => { var _a; return ((_a = x.currentDimension) === null || _a === void 0 ? void 0 : _a.name) === dim.name; }))
-                    updates.push({ currentDimension: dim, toDelete: true });
-            }
-            // Now keep only the updates that actually change something
-            updates = updates.filter(x => x.toDelete
-                || !x.currentDimension
-                || (x.currentDimension && (x.currentDimension.name !== x.newName
-                    || (Algo_1.default.hasVal(x.newHidden) && x.newHidden !== x.currentDimension.hidden)
-                    || x.newPosition !== x.currentDimension.index)));
-            return updates;
-        };
-        /**
-         * Each worker threads writes to his own dataset file to avoid concurrency and data loss,
-         * at the end of their work, I merge their results to a single file
-         */
-        this.mergeWorkersPaths = (threadPaths, dataset) => __awaiter(this, void 0, void 0, function* () {
-            dataset.clear();
-            const datasetPath = dataset.getPath();
-            for (let i = 0; i < threadPaths.length; i++) {
-                const path = threadPaths[i];
-                // If the thread skipped execution (maybe because no data needed to change), then the
-                // dataset file might not exist, in this case, just skip it
-                if (!fs_1.default.existsSync(path))
-                    continue;
-                const readStream = fs_1.default.createReadStream(path);
-                // For the first file, create a new write stream
-                // For subsequent files, append to the existing file
-                const writeStream = fs_1.default.createWriteStream(datasetPath, { flags: i === 0 ? 'w' : 'a' });
-                yield (0, promises_1.pipeline)(readStream, writeStream);
-                fs_1.default.unlinkSync(path);
-            }
-            return dataset;
-        });
     }
 }
 const DatasetManager = new DatasetManagerClass();

package/engines/dataset/DatasetRecord.js

@@ -11,7 +11,8 @@ class DatasetRecord {
         this.parse = (row, delimiter, dimensions) => {
             if (!this.isEmpty() && dimensions.length > 0) {
                 const parts = CSVParser_1.default.parseRow(row, delimiter);
-                for (const dim of dimensions) {
+                for (let i = 0; i < dimensions.length; i++) {
+                    const dim = dimensions[i];
                     // Use dim.index to get the correct column from the file, not the loop index
                     this._value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.type, dim.format);
                 }
@@ -52,8 +53,8 @@ class DatasetRecord {
                     index: update.newPosition,
                     key: update.newName,
                     name: update.newName,
-                    type: (_b = (_a = update.currentDimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string',
-                    hidden: update.newHidden
+                    hidden: update.newHidden,
+                    type: (_b = (_a = update.currentDimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string'
                 };
                 this._value[newDimension.name] = null;
                 this._dimensions = [...this._dimensions, newDimension];

package/engines/deployment/DeploymentPlanner.js

@@ -14,13 +14,6 @@ class DeploymentPlannerClass {
                 const output = consumer.outputs[i];
                 switch (output.format) {
                     // csv, json, parquet outputs do not need to generate anything at deploy
-                    case 'SQL': {
-                        if (output.accelerated && !output.direct)
-                            plan.push({ type: 'create-materialized-view', output: output });
-                        else if (!output.direct)
-                            plan.push({ type: 'create-view', output: output });
-                        break;
-                    }
                     case 'API': {
                         throw new Error(`Invalid consumer "${consumer.name}" format "${output.format}": not implemented yet.`);
                     }

package/engines/execution/ExecutionPlanner.js

@@ -7,7 +7,7 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
 const Environment_1 = __importDefault(require("../Environment"));
-class ExecutionPlannerClas {
+class ExecutionPlannerClass {
     constructor() {
         this.getEngineClass = (engine) => {
             switch (engine) {
@@ -127,5 +127,5 @@ class ExecutionPlannerClas {
         };
     }
 }
-const ExecutionPlanner = new ExecutionPlannerClas();
+const ExecutionPlanner = new ExecutionPlannerClass();
 exports.default = ExecutionPlanner;

package/engines/execution/RequestExecutor.js

@@ -1,41 +1,11 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
 class RequestExecutorClass {
     constructor() {
-        /**
-         * Applies the filters, limit, offset and order on the in-memory-data
-         */
-        this.execute = (dataset, request) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(dataset, 'Invalid data');
-            (0, Affirm_1.default)(request, 'Invalid request');
-            if (request.filters)
-                dataset = yield this.applyFilters(dataset, request.filters);
-            if (request.order)
-                dataset = yield this._applyOrdering(dataset, request.order);
-            return dataset;
-        });
-        this.applyFilters = (dataset_1, filters_1, ...args_1) => __awaiter(this, [dataset_1, filters_1, ...args_1], void 0, function* (dataset, filters, options = {}) {
-            return yield dataset.filter(record => {
-                return filters.every(filter => this._evaluateFilter(record, filter));
-            }, options);
-        });
-        this._evaluateFilter = (record, filter) => {
+        this.evaluateFilter = (record, filter) => {
             const evaluate = (baseRecord, baseFilter) => {
                 const { member, operator, values } = baseFilter;
-                const value = baseRecord.getValue(member);
+                const value = baseRecord[member];
                 const singleValue = values[0];
                 switch (operator) {
                     case 'equals':
@@ -85,23 +55,12 @@ class RequestExecutorClass {
             const { and, or } = filter;
             const baseResult = evaluate(record, filter);
             if (and)
-                return baseResult && and.every(subFilter => this._evaluateFilter(record, subFilter));
+                return baseResult && and.every(subFilter => this.evaluateFilter(record, subFilter));
             if (or)
-                return baseResult || or.some(subFilter => this._evaluateFilter(record, subFilter));
+                return baseResult || or.some(subFilter => this.evaluateFilter(record, subFilter));
             else
                 return baseResult;
         };
-        this._applyOrdering = (dataset, order) => __awaiter(this, void 0, void 0, function* () {
-            return yield dataset.sort((a, b) => {
-                for (const [field, direction] of order) {
-                    if (a[field] < b[field])
-                        return direction === 'asc' ? -1 : 1;
-                    if (a[field] > b[field])
-                        return direction === 'asc' ? 1 : -1;
-                }
-                return 0;
-            });
-        });
     }
 }
 const RequestExecutor = new RequestExecutorClass();