@forzalabs/remora 0.0.12

package/engines/file/FileCompiler.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+class FileCompilerClass {
+    constructor() {
+        this.compileProducer = (producer, source) => {
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            (0, Affirm_1.default)(source, `Invalid source`);
+            (0, Affirm_1.default)(producer.settings.fileKey, `Missing required file key in producer settings`);
+            (0, Affirm_1.default)(producer.settings.fileType, `Missing required file type in producer settings`);
+            (0, Affirm_1.default)(!producer.measures || producer.measures.length === 0, `Cannot use "measure" with a producer linked to a file (only dimensions are allowed).`);
+            const columns = producer.dimensions.map(x => ({
+                aliasInProducer: x.alias,
+                nameInProducer: x.name,
+                consumerAlias: null,
+                consumerKey: null,
+                owner: x.name,
+                dimension: x
+            }));
+            return columns;
+        };
+    }
+}
+const FileCompiler = new FileCompilerClass();
+exports.default = FileCompiler;

package/engines/file/FileExporter.js

@@ -0,0 +1,116 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
+const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
+const Environment_1 = __importDefault(require("../Environment"));
+const fs = __importStar(require("fs"));
+class FileExporterClass {
+    constructor() {
+        this.export = (consumer, output, data) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            (0, Affirm_1.default)(output, `Invalid output`);
+            (0, Affirm_1.default)(data, `Invalid export data`);
+            let exportData = null;
+            let extension = null;
+            // build the actual file in the requested format
+            switch (output.format) {
+                case 'CSV': {
+                    const lines = [];
+                    const keys = Object.keys(data[0]);
+                    lines.push(keys.map(x => `"${x}"`).join(','));
+                    for (let i = 0; i < data.length; i++) {
+                        const row = data[i];
+                        const columns = keys.map(key => row[key]);
+                        const rowValues = columns.map(c => Algo_1.default.replaceAll(c.toString(), '"', '""')).map(c => `"${c}"`).join(',');
+                        lines.push(rowValues);
+                    }
+                    const csv = lines.join('\n');
+                    exportData = csv;
+                    extension = 'csv';
+                    break;
+                }
+                case 'JSON': {
+                    const lines = [];
+                    for (let i = 0; i < data.length; i++) {
+                        const row = data[i];
+                        lines.push(JSON.stringify(row));
+                    }
+                    const jsonl = lines.join('\n');
+                    exportData = jsonl;
+                    extension = 'jsonl';
+                    break;
+                }
+                case 'PARQUET': {
+                    throw new Error(`Consumer output "${output.format}" not implemented yet`);
+                }
+            }
+            // export it where it needs to go
+            const source = Environment_1.default.getSource(output.exportDestination);
+            (0, Affirm_1.default)(source, `Invalid consumer "${consumer.name}" export location source. Make sure that the export location is an available source.`);
+            switch (source.engine) {
+                case 'local': {
+                    const folder = source.authentication.path;
+                    if (!fs.existsSync(folder))
+                        fs.mkdirSync(folder);
+                    const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toJSON(), ':', '-')}.${extension}`;
+                    fs.writeFileSync(path, exportData);
+                    return path;
+                }
+                case 'aws-s3': {
+                    const driver = yield DriverFactory_1.default.instantiateDestination(source);
+                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${DSTE_1.default.now().toJSON()}` });
+                    return res.key;
+                }
+            }
+        });
+    }
+}
+const FileExporter = new FileExporterClass();
+exports.default = FileExporter;

package/engines/schema/SchemaEngine.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const SchemaValidator_1 = __importDefault(require("./SchemaValidator"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+class SchemaEngineClass {
+    constructor() {
+        /**
+         * If the consumer has a ".schema" then checks that the output shape of the consumer complies with what the JSON schema says
+         */
+        this.enforceConsumerOutputSchema = (consumer) => {
+            var _a;
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            if (!consumer.schema)
+                return true;
+            const validator = SchemaValidator_1.default.getSchema(consumer.schema);
+            const schema = validator.schema;
+            // TODO: right now I just check that all the required properites are there...
+            // in the future this needs to be more fleshed (types, ...)
+            const properties = ConsumerEngine_1.default.compile(consumer);
+            const matches = (_a = schema.required) === null || _a === void 0 ? void 0 : _a.map(x => ({
+                exists: properties.find(k => k.consumerAlias === x) ? true : false,
+                property: x
+            }));
+            return matches.every(x => x.exists);
+        };
+    }
+}
+const SchemaEngine = new SchemaEngineClass();
+exports.default = SchemaEngine;

package/engines/schema/SchemaValidator.js

@@ -0,0 +1,67 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const ajv_1 = __importDefault(require("ajv"));
+const ajv_formats_1 = __importDefault(require("ajv-formats"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+class SchemaValidatorClass {
+    constructor() {
+        this.addSchema = (schema, schemaKey) => {
+            const isValid = this.ajv.validateSchema(schema);
+            (0, Affirm_1.default)(isValid, `Invalid JSON Schema: ${JSON.stringify(this.ajv.errors)}`);
+            this.ajv.addSchema(schema, schemaKey);
+        };
+        this.getSchema = (schemaKey) => {
+            return this.ajv.getSchema(schemaKey);
+        };
+        this.validate = (schemaKey, data) => {
+            var _a;
+            const validateFn = this.ajv.getSchema(schemaKey);
+            (0, Affirm_1.default)(validateFn, `Failed to get validator for schema '${schemaKey}'`);
+            const isValid = validateFn(data);
+            return { isValid, errors: (_a = validateFn.errors) !== null && _a !== void 0 ? _a : [], data };
+        };
+        this.validateAndReturn = (schemaKey, data) => {
+            const res = this.validate(schemaKey, data);
+            if (res.isValid)
+                return res.data;
+            else
+                return null;
+        };
+        this.unloadSchemas = () => {
+            const result = this.ajv.removeSchema();
+            (0, Affirm_1.default)(result, `Failed to remove schemas`);
+            this.loadInternalSchema();
+            return true;
+        };
+        this.loadInternalSchema = () => {
+            const internalSchemaDir = path_1.default.join(__dirname, '../../definitions/json_schemas');
+            const internalSchemas = [
+                'project-schema',
+                'source-schema',
+                'producer-schema',
+                'consumer-schema'
+            ];
+            internalSchemas.forEach(sn => {
+                const sp = path_1.default.join(internalSchemaDir, `${sn}.json`);
+                const parsed = JSON.parse(fs_1.default.readFileSync(sp, 'utf-8'));
+                this.addSchema(parsed, sn);
+            });
+            return true;
+        };
+        this.ajv = new ajv_1.default({
+            allErrors: true,
+            strict: true,
+            strictSchema: false,
+            validateFormats: true
+        });
+        (0, ajv_formats_1.default)(this.ajv);
+        this.loadInternalSchema();
+    }
+}
+const SchemaValidator = new SchemaValidatorClass();
+exports.default = SchemaValidator;

package/engines/sql/SQLBuilder.js

@@ -0,0 +1,96 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+class SQLBuilderClass {
+    constructor() {
+        this.getSQLOperator = (operator) => {
+            switch (operator) {
+                case 'equals': return '=';
+                case 'notEquals': return '!=';
+                case 'contains': return 'LIKE';
+                case 'notContains': return 'NOT LIKE';
+                case 'startsWith': return 'LIKE';
+                case 'endsWith': return 'LIKE';
+                case 'greaterThan': return '>';
+                case 'greaterThanOrEquals': return '>=';
+                case 'lessThan': return '<';
+                case 'lessThanOrEquals': return '<=';
+                case 'in': return 'IN';
+                case 'notIn': return 'NOT IN';
+                case 'between': return 'BETWEEN';
+                case 'notBetween': return 'NOT BETWEEN';
+                case 'isNull': return 'IS NULL';
+                case 'isNotNull': return 'IS NOT NULL';
+                case 'true': return '= TRUE';
+                case 'false': return '= FALSE';
+                case 'matches': return '~';
+                case 'notMatches': return '!~';
+                case 'sql': return ''; // Custom SQL should be handled separately
+                default: throw new Error(`Unsupported operator: ${operator}`);
+            }
+        };
+        this.buildFilterClause = (filter) => {
+            var _a, _b;
+            const operator = this.getSQLOperator(filter.operator);
+            let baseClause;
+            if (filter.operator === 'in' || filter.operator === 'notIn') {
+                baseClause = `${filter.member} ${operator} (${filter.values.map(value => `'${value}'`).join(', ')})`;
+            }
+            else if (filter.operator === 'between' || filter.operator === 'notBetween') {
+                baseClause = `${filter.member} ${operator} '${filter.values[0]}' AND '${filter.values[1]}'`;
+            }
+            else if (filter.operator === 'isNull' || filter.operator === 'isNotNull' || filter.operator === 'true' || filter.operator === 'false') {
+                baseClause = `${filter.member} ${operator}`;
+            }
+            else if (filter.operator === 'contains' || filter.operator === 'notContains') {
+                baseClause = `${filter.member} ${operator} '%${filter.values[0]}%'`;
+            }
+            else if (filter.operator === 'startsWith') {
+                baseClause = `${filter.member} ${operator} '${filter.values[0]}%'`;
+            }
+            else if (filter.operator === 'endsWith') {
+                baseClause = `${filter.member} ${operator} '%${filter.values[0]}'`;
+            }
+            else if (filter.operator === 'sql') {
+                baseClause = `(${filter.values[0]})`;
+            }
+            else {
+                baseClause = `${filter.member} ${operator} '${filter.values[0]}'`;
+            }
+            const orClauses = (_a = filter.or) === null || _a === void 0 ? void 0 : _a.map(this.buildFilterClause).join(' OR ');
+            const andClauses = (_b = filter.and) === null || _b === void 0 ? void 0 : _b.map(this.buildFilterClause).join(' AND ');
+            let combinedClause = baseClause;
+            if (orClauses) {
+                combinedClause = `(${combinedClause} OR ${orClauses})`;
+            }
+            if (andClauses) {
+                combinedClause = `(${combinedClause} AND ${andClauses})`;
+            }
+            return combinedClause;
+        };
+        this.buildConsumerQuery = (request) => {
+            (0, Affirm_1.default)(request, `Invalid build consumer request`);
+            let query = ``;
+            if (request.filters && request.filters.length > 0) {
+                const filterClauses = request.filters.map(this.buildFilterClause);
+                query += ` WHERE ${filterClauses.join(' AND ')}`;
+            }
+            if (request.order && request.order.length > 0) {
+                const orderClauses = request.order.map(([field, direction]) => `${field} ${direction}`);
+                query += ` ORDER BY ${orderClauses.join(', ')}`;
+            }
+            if (request.limit) {
+                query += ` LIMIT ${request.limit}`;
+            }
+            if (request.offset) {
+                query += ` OFFSET ${request.offset}`;
+            }
+            return query;
+        };
+    }
+}
+const SQLBuilder = new SQLBuilderClass();
+exports.default = SQLBuilder;

package/engines/sql/SQLCompiler.js

@@ -0,0 +1,140 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
+const Environment_1 = __importDefault(require("../Environment"));
+const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
+class SQLCompilerClass {
+    constructor() {
+        this.compileProducer = (producer, source) => {
+            var _a, _b;
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            (0, Affirm_1.default)(source, `Invalid source`);
+            if (Algo_1.default.hasVal(producer.settings.sql)) {
+                const libraryItem = Environment_1.default.getSqlInLibrary(producer.settings.sql);
+                (0, Affirm_1.default)(libraryItem, `Misconfiguration for producer "${producer.name}": SQL library item is missing. Required SQL item named "${producer.settings.sql}" was not found.`);
+                return libraryItem.sql;
+            }
+            (0, Affirm_1.default)(producer.settings.sqlTable, `Misconfiguration for producer "${producer.name}": SQL table name is missing. Required since no override to the SQL was used.`);
+            const dimensions = producer.dimensions.map(x => {
+                var _a;
+                const maskType = x.mask;
+                const columnReference = (_a = x.alias) !== null && _a !== void 0 ? _a : x.name;
+                const fieldReference = `"${producer.settings.sqlTable}"."${columnReference}"`;
+                if (x.mask && x.mask === 'hash')
+                    return CryptoEngine_1.default.hashQuery(maskType, fieldReference, x.name);
+                else
+                    return `${fieldReference} AS "${x.name}"`;
+            });
+            const measures = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(measure => {
+                const thisProducer = `"${producer.settings.sqlTable}"`;
+                const sanitized = measure.sql.replace('P', thisProducer).replace('PROD', thisProducer).replace('PRODUCER', thisProducer)
+                    .replace('${', '').replace('}', '');
+                return `${sanitized} AS "${measure.name}"`;
+            })) !== null && _b !== void 0 ? _b : [];
+            const columns = dimensions.concat(measures);
+            const sql = `SELECT ${columns.join(', ')} FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}"`;
+            if (measures.length > 0) {
+                const groupBys = `GROUP BY ${dimensions.map((_, i) => i + 1).join(', ')}`;
+                return `${sql} ${groupBys}`;
+            }
+            return sql;
+        };
+        this.deployProducer = (producer, source) => {
+            (0, Affirm_1.default)(producer, `Invalid producer`);
+            (0, Affirm_1.default)(source, `Invalid source`);
+            const sql = this.compileProducer(producer, source);
+            if (producer.settings.direct) {
+                return sql;
+            }
+            else {
+                const internalSchema = Environment_1.default.get('schema');
+                (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                return `CREATE OR REPLACE VIEW "${internalSchema}"."${producer.name}" AS ${sql}`;
+            }
+        };
+        /**
+         * Returns the SQL reference to this producer, used in FROM when constructing SQL statements of consumers (and others).
+         * This might easily just return a reference to a view or the underlying SQL of a producer if the view is not available.
+         */
+        this.getProducerReference = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}"`);
+            if (producer.settings.direct) {
+                return this.compileProducer(producer, source);
+            }
+            else {
+                const internalSchema = Environment_1.default.get('schema');
+                (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                return `SELECT * FROM "${internalSchema}"."${producer.name}"`;
+            }
+        };
+        this.getConsumerReference = (consumer) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            if (consumer.outputs.some(x => x.format === 'SQL' && x.accellerated))
+                return `SELECT * FROM "av_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
+            if (consumer.outputs.some(x => x.format === 'SQL' && !x.direct))
+                return `SELECT * FROM "v_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
+            return `SELECT * FROM (${this.compileConsumer(consumer)})`;
+        };
+        this.compileConsumer = (consumer) => {
+            var _a;
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            (0, Affirm_1.default)(consumer.producers.length > 0, `Consumer has no producers to draw data from ("${consumer.name}")`);
+            const subqueries = consumer.producers.map(cProd => {
+                const producer = Environment_1.default.getProducer(cProd.name);
+                if (!producer) {
+                    const consumer = Environment_1.default.getConsumer(cProd.name);
+                    (0, Affirm_1.default)(consumer, `No producer found for consumer "${consumer.name}" with name "${cProd.name}"`);
+                    return `"${cProd.name}" AS (${this.getConsumerReference(consumer)})`;
+                }
+                return `"${cProd.name}" AS (${this.getProducerReference(producer)})`;
+            });
+            const columns = ConsumerEngine_1.default.compile(consumer);
+            const sqlColumns = columns.map(x => { var _a, _b, _c; return `"${x.owner}"."${(_b = (_a = x.dimension) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : (_c = x.measure) === null || _c === void 0 ? void 0 : _c.name}" AS "${x.consumerAlias}"`; });
+            const columnQualifiers = sqlColumns.map(x => Algo_1.default.replaceAll(x.split('.')[1], '"', ''));
+            const dupes = Algo_1.default.duplicates(columnQualifiers);
+            (0, Affirm_1.default)(dupes.length === 0, `Wrong consumer configuration: duplicate columns where found for consumer "${consumer.name}" ("${dupes.join(', ')}")`);
+            const joins = consumer.producers.filter(x => x.joins && x.joins.length > 0).flatMap(cProd => {
+                return cProd.joins.map(x => {
+                    const otherProd = consumer.producers.find(k => k.name === x.otherName);
+                    (0, Affirm_1.default)(otherProd, `Invalid JOIN relantionship: the producer "${cProd.name}" is asking for a join with "${x.otherName}", but this one doesn't exists.`);
+                    const starts = Algo_1.default.locations(x.sql, '${');
+                    const ends = Algo_1.default.locations(x.sql, '}');
+                    (0, Affirm_1.default)(starts.length === ends.length, `Invalid JOIN SQL: number of condition parameters does not match on consumer "${consumer.name}" producer "${cProd.name}" ("${x.sql}")`);
+                    const params = starts.map((start, index) => x.sql.substring(start, ends[index] + 1));
+                    const thisProducer = `"${cProd.name}"`;
+                    const sqlParams = params.map(param => param.replace('P', thisProducer).replace('PROD', thisProducer).replace('PRODUCER', thisProducer)
+                        .replace('${', '').replace('}', ''));
+                    let sqlCondition = x.sql;
+                    params.forEach((p, i) => sqlCondition = sqlCondition.replace(p, sqlParams[i]));
+                    return `LEFT JOIN "${otherProd.name}" ON ${sqlCondition}`;
+                });
+            });
+            const filters = (_a = consumer.filters) === null || _a === void 0 ? void 0 : _a.map(x => {
+                const starts = Algo_1.default.locations(x.sql, '${');
+                const ends = Algo_1.default.locations(x.sql, '}');
+                (0, Affirm_1.default)(starts.length === ends.length, `Invalid filter SQL: number of condition parameters does not match on consumer "${consumer.name}" ("${x.sql}")`);
+                const params = starts.map((start, index) => x.sql.substring(start, ends[index] + 1));
+                const sqlParams = params.map(param => SQLUtils_1.default.findDimension(columns, param));
+                let sqlFilter = x.sql;
+                params.forEach((p, i) => sqlFilter = sqlFilter.replace(p, sqlParams[i].consumerAlias));
+                return sqlFilter;
+            });
+            let sql = `WITH ${subqueries.join(',\n')}\n\nSELECT ${sqlColumns.join(', ')} FROM "${consumer.producers[0].name}"`;
+            if (joins && joins.length > 0)
+                sql += ` ${joins.join(', ')}`;
+            if (filters && filters.length > 0)
+                sql += ` WHERE ${filters.join(', ')}`;
+            return sql;
+        };
+    }
+}
+const SQLCompiler = new SQLCompilerClass();
+exports.default = SQLCompiler;

package/engines/sql/SQLUtils.js

@@ -0,0 +1,22 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const SQLUtils = {
+    viewName: (name) => 'v_remora_' + SQLUtils.sanitizeName(name),
+    acceleratedViewName: (name) => 'av_remora_' + SQLUtils.sanitizeName(name),
+    findDimension: (compiledDimensions, dimensionQualifier) => {
+        const cleanedQualifier = Algo_1.default.replaceAll(dimensionQualifier.split('.')[1], '}', '');
+        const hit = compiledDimensions.find(x => x.consumerAlias === cleanedQualifier);
+        (0, Affirm_1.default)(hit, `Unable to find the column referenced by "${dimensionQualifier}" in the compiled list of columns (${compiledDimensions.map(x => `"${x.consumerAlias}"`).join(', ')})`);
+        return hit;
+    },
+    sanitizeName: (string) => {
+        (0, Affirm_1.default)(string, 'Invalid string to sanitize');
+        return string.replace(/[^a-z0-9]/gi, '_');
+    }
+};
+exports.default = SQLUtils;

package/engines/validation/Validator.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Algo_1 = __importDefault(require("../../core/Algo"));
+const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
+const Environment_1 = __importDefault(require("../Environment"));
+const ExecutionPlanner_1 = __importDefault(require("../execution/ExecutionPlanner"));
+class ValidatorClass {
+    constructor() {
+        this.validateSources = (sources) => {
+            (0, Affirm_1.default)(sources, 'Invalid sources');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(sources, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in sources: "${dupes.map(x => x.name).join(', ')}"`);
+                for (let i = 0; i < sources.length; i++) {
+                    const source = sources[i];
+                    if (source.engine === 'local' && !source.authentication.path)
+                        errors.push(`For source ${source.name}, the path has not been configured`);
+                }
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Sources. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateProducers = (producers) => {
+            (0, Affirm_1.default)(producers, 'Invalid producers');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(producers, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in producers: "${dupes.map(x => x.name).join(', ')}"`);
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Producers. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateProducer = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const errors = [];
+            try {
+                if (!producer.source || producer.source.length === 0)
+                    errors.push(`Missing parameter "source" in producer`);
+                if (producer.dimensions.some(x => x.name.includes('{') || x.name.includes('[')))
+                    errors.push(`Invalid dimension name found in producer "${producer.name}": can't use characters "{" or "[" in dimension names`);
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Producer. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateConsumers = (consumers) => {
+            (0, Affirm_1.default)(consumers, 'Invalid consumers');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(consumers, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in consumers: "${dupes.map(x => x.name).join(', ')}"`);
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Consumers. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateConsumer = (consumer) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const errors = [];
+            try {
+                // TODO: check that a consumer doens't consume himself
+                const allFieldsWithNoFrom = consumer.fields.filter(x => x.key === '*' && !x.from);
+                if (allFieldsWithNoFrom.length > 0 && consumer.producers.length > 1)
+                    errors.push(`Field with key "*" was used without specifying the "from" producer and multiple producers were found.`);
+                if (consumer.fields.some(x => x.key === '*' && x.grouping))
+                    errors.push(`Field with key "*" can't be used for "grouping". Either remove the grouping or change the key.`);
+                // Validation on producers
+                if (consumer.producers.length === 0)
+                    errors.push(`Consumer must have at least 1 producer.`);
+                const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
+                if (producers.length === 0)
+                    errors.push('No producers found');
+                if (producers.some(x => !x))
+                    errors.push(`Invalid producer found in consumer "${consumer.name}"`);
+                // Validation on sources
+                const sources = producers.map(x => Environment_1.default.getSource(x.source));
+                if (sources.length === 0)
+                    errors.push('No sources found');
+                if (sources.some(x => !x))
+                    errors.push(`Invalid source found in consumer "${consumer.name}"`);
+                // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
+                const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
+                if (uniqEngines.length !== 1)
+                    errors.push(`Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
+                // For now we also only support consumers that have producers ALL having the same exact source
+                const uniqNames = Algo_1.default.uniqBy(sources, 'name');
+                if (uniqNames.length !== 1)
+                    errors.push(`Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
+                if (consumer.filters && consumer.filters.length > 0) {
+                    if (consumer.filters.some(x => x.sql && x.rule))
+                        errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
+                    const [source] = ConsumerManager_1.default.getSource(consumer);
+                    const engineClass = ExecutionPlanner_1.default.getEngineClass(source.engine);
+                    if (engineClass === 'file' && consumer.filters.some(x => x.sql))
+                        errors.push(`Filters based on SQL are only valid for SQL based sources. (source: ${source.name})`);
+                    if (engineClass === 'sql' && consumer.filters.some(x => x.rule))
+                        errors.push(`Filters based on rules are only valid for non-SQL based sources. (source: ${source.name})`);
+                }
+                // Validation on fields
+                const validateGroupingLevels = (fields, level = 0) => {
+                    let errors = [];
+                    const groupingFields = fields.filter(x => x.grouping);
+                    if (groupingFields.length > 1)
+                        errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
+                    groupingFields.forEach(field => {
+                        if (field.grouping)
+                            errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
+                    });
+                    return errors;
+                };
+                errors.push(...validateGroupingLevels(consumer.fields));
+                // Validation outputs
+                const duplicatesOutputs = Algo_1.default.duplicatesObject(consumer.outputs, 'format');
+                if (duplicatesOutputs.length > 0) {
+                    const duplicatesTypes = Algo_1.default.uniq(duplicatesOutputs.map(x => x.format));
+                    errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(' and ')})`);
+                }
+                for (let i = 0; i < consumer.outputs.length; i++) {
+                    const output = consumer.outputs[i];
+                    if (output.format === 'SQL' && output.accellerated && output.direct)
+                        errors.push(`An output SQL cannot be both direct and accelerated. (output: ${output.format})`);
+                }
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Consumer. (error: ${e})`);
+            }
+            return errors;
+        };
+    }
+}
+const Validator = new ValidatorClass();
+exports.default = Validator;