@forzalabs/remora 0.0.12

package/engines/Environment.js

@@ -0,0 +1,139 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SchemaValidator_1 = __importDefault(require("./schema/SchemaValidator"));
+const Validator_1 = __importDefault(require("./Validator"));
+class EnvironmentClass {
+    constructor() {
+        this._env = null;
+        this.init = (env) => {
+            this._env = env;
+        };
+        this.load = (remoraPath) => {
+            const envPath = path_1.default.join(remoraPath, 'remora');
+            // Read and parse project.json
+            const projectPath = path_1.default.join(envPath, 'project.json');
+            if (!fs_1.default.existsSync(projectPath))
+                throw new Error('Project configuration file not found');
+            const projectConfig = JSON.parse(fs_1.default.readFileSync(projectPath, 'utf-8'));
+            // Validate project schema
+            const isValid = SchemaValidator_1.default.validate('project-schema', projectConfig);
+            if (!isValid)
+                throw new Error('Invalid project configuration');
+            const loadConfigurations = (basePath, paths) => {
+                const configs = [];
+                for (const configPath of paths) {
+                    const fullPath = path_1.default.join(basePath, configPath);
+                    try {
+                        if (fs_1.default.statSync(fullPath).isDirectory()) {
+                            const files = fs_1.default.readdirSync(fullPath).filter(file => file.endsWith('.json'));
+                            for (const file of files) {
+                                const config = JSON.parse(fs_1.default.readFileSync(path_1.default.join(fullPath, file), 'utf-8'));
+                                configs.push(config);
+                            }
+                        }
+                        else {
+                            const config = JSON.parse(fs_1.default.readFileSync(fullPath, 'utf-8'));
+                            configs.push(config);
+                        }
+                    }
+                    catch (error) {
+                        throw new Error(`Error loading configuration from ${configPath}: ${error.message}`);
+                    }
+                }
+                return configs;
+            };
+            // Unload all the schemes other than the default ones, because otherwise the RW could keep loaded schemes not saved in the remora folder
+            SchemaValidator_1.default.unloadSchemas();
+            // Load all schemas first since they're needed for validation
+            const schemas = loadConfigurations(envPath, projectConfig.schemas);
+            schemas.forEach(schema => {
+                SchemaValidator_1.default.addSchema(schema, schema.title);
+            });
+            // Load configurations with validation
+            const sources = loadConfigurations(envPath, projectConfig.sources);
+            sources.forEach(source => {
+                if (!SchemaValidator_1.default.validate('source-schema', source))
+                    throw new Error(`Invalid source configuration: ${source.name}`);
+            });
+            const producers = loadConfigurations(envPath, projectConfig.producers);
+            producers.forEach(producer => {
+                if (!SchemaValidator_1.default.validate('producer-schema', producer))
+                    throw new Error(`Invalid producer configuration: ${producer.name}`);
+            });
+            const consumers = loadConfigurations(envPath, projectConfig.consumers);
+            consumers.forEach(consumer => {
+                if (!SchemaValidator_1.default.validate('consumer-schema', consumer))
+                    throw new Error(`Invalid consumer configuration: ${consumer.name}`);
+            });
+            // Initialize environment
+            this.init({
+                settings: new Map(Object.entries({
+                    SQL_MAX_QUERY_ROWS: projectConfig.settings.SQL_MAX_QUERY_ROWS.toString()
+                })),
+                sources,
+                producers,
+                consumers,
+                schemas,
+                sqlLibrary: [] // TODO: Add SQL library loading if needed
+            });
+        };
+        this.get = (setting) => {
+            return this._env.settings.get(setting);
+        };
+        this.getSource = (sourceName) => {
+            (0, Affirm_1.default)(sourceName, 'Invalid source name');
+            return this._env.sources.find(x => x.name === sourceName);
+        };
+        /**
+         * A consumer can reference another consumer, in this case I need to keep digging to find the real producer
+         */
+        this.getFirstProducer = (producerName) => {
+            (0, Affirm_1.default)(producerName, `Invalid producer name`);
+            const prod = this._env.producers.find(x => x.name === producerName);
+            if (!prod) {
+                const consumer = this.getConsumer(producerName);
+                (0, Affirm_1.default)(consumer, `Invalid producer name`);
+                return this.getFirstProducer(consumer.producers[0].name);
+            }
+            return prod;
+        };
+        this.getProducer = (producerName) => {
+            (0, Affirm_1.default)(producerName, `Invalid producer name`);
+            return this._env.producers.find(x => x.name === producerName);
+        };
+        this.getConsumer = (consumerName) => {
+            (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
+            return this._env.consumers.find(x => x.name === consumerName);
+        };
+        this.getSchema = (schemaName) => {
+            (0, Affirm_1.default)(schemaName, 'Invalid schema name');
+            return this._env.schemas.find(x => x.title === schemaName);
+        };
+        this.getSqlInLibrary = (name) => {
+            (0, Affirm_1.default)(name, `Invalid sql library item name`);
+            return this._env.sqlLibrary.find(x => x.name === name);
+        };
+        this.validate = () => {
+            (0, Affirm_1.default)(this._env, 'Invalid environment');
+            let errors = [];
+            try {
+                errors = [...errors, ...Validator_1.default.validateSources(this._env.sources)];
+                errors = [...errors, ...Validator_1.default.validateProducers(this._env.producers)];
+                errors = [...errors, ...Validator_1.default.validateConsumers(this._env.consumers)];
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Environment. (error: ${e})`);
+            }
+            return errors;
+        };
+    }
+}
+const Environment = new EnvironmentClass();
+exports.default = Environment;

package/engines/ParseManager.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+class ParseManagerClass {
+    constructor() {
+        this.csvToJson = (csv, producer) => {
+            (0, Affirm_1.default)(csv, 'Invalid csv content');
+            Affirm_1.default.hasValue(csv.length, 'Invalid csv content length');
+            const fileRows = csv.split('\n');
+            return this.csvLinesToJson(fileRows, producer);
+        };
+        this.csvLinesToJson = (lines, producer) => {
+            var _a;
+            (0, Affirm_1.default)(lines, 'Invalid csv lines');
+            Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
+            const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
+            const fileRows = lines;
+            const columns = fileRows[0].split(delimiterChar).map(x => x.trim());
+            const rows = fileRows.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
+            const result = [];
+            for (let i = 0; i < rows.length; i++) {
+                const row = rows[i];
+                const rowObject = {};
+                for (let j = 0; j < columns.length; j++) {
+                    const column = columns[j];
+                    rowObject[column] = row[j];
+                }
+                result.push(rowObject);
+            }
+            return result;
+        };
+    }
+}
+const ParseManager = new ParseManagerClass();
+exports.default = ParseManager;

package/engines/ProducerEngine.js

@@ -0,0 +1,150 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
+const DeploymentPlanner_1 = __importDefault(require("./deployment/DeploymentPlanner"));
+const Environment_1 = __importDefault(require("./Environment"));
+const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
+const SQLCompiler_1 = __importDefault(require("./sql/SQLCompiler"));
+const SQLUtils_1 = __importDefault(require("./sql/SQLUtils"));
+const ParseManager_1 = __importDefault(require("./ParseManager"));
+class ProducerEngineClass {
+    constructor() {
+        this.compile = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            switch (source.engine) {
+                case 'aws-redshift':
+                case 'postgres': {
+                    const sql = SQLCompiler_1.default.compileProducer(producer, source);
+                    (0, Affirm_1.default)(sql, `Invalid SQL from compilation for producer "${producer.name}"`);
+                    return sql;
+                }
+                case 'aws-s3': {
+                    const columns = FileCompiler_1.default.compileProducer(producer, source);
+                    (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
+                    break;
+                }
+                default: throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not implemented yet`);
+            }
+        };
+        this.deploy = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            const plan = DeploymentPlanner_1.default.planProducer(producer);
+            for (const planStep of plan) {
+                switch (planStep.type) {
+                    case 'create-view': {
+                        const internalSchema = Environment_1.default.get('schema');
+                        (0, Affirm_1.default)(internalSchema, `Invalid schema set on the authentication for source "${source.name}"`);
+                        const sql = SQLCompiler_1.default.compileProducer(producer, source);
+                        const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.viewName(producer.name)}" AS ${sql}`;
+                        yield driver.execute(vSQL);
+                        break;
+                    }
+                    default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
+                }
+            }
+        });
+        this.execute = (producer) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            switch (source.engine) {
+                case 'aws-redshift': {
+                    const sql = SQLCompiler_1.default.compileProducer(producer, source);
+                    (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
+                    const res = yield driver.query(sql);
+                    return { data: res.rows, dataType: 'array-of-json' };
+                }
+                case 'aws-s3': {
+                    return this.readFile(producer, { readmode: 'all' });
+                }
+                default: throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not supported`);
+            }
+        });
+        this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            (0, Affirm_1.default)(options, 'Invalid options');
+            if (options.readmode === 'lines')
+                (0, Affirm_1.default)(options.lines, 'Invalid lines');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            const driver = yield DriverFactory_1.default.instantiateSource(source);
+            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+            let lines = [];
+            if (options.readmode === 'lines')
+                lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
+            else
+                lines = [(yield driver.download({ fileKey: producer.settings.fileKey }))];
+            switch (producer.settings.fileType.toUpperCase()) {
+                case 'CSV': {
+                    return { data: lines, dataType: 'lines-of-text' };
+                }
+                case 'JSONL':
+                case 'JSON': {
+                    const json = lines.map(x => JSON.parse(x));
+                    return { data: json, dataType: 'array-of-json' };
+                }
+                default:
+                    throw new Error(`Invalid file type "${producer.settings.fileType}" for engine type "${source.engine}" for producer "${producer.name}": not supported`);
+            }
+        });
+        this.readSampleData = (producer_1, ...args_1) => __awaiter(this, [producer_1, ...args_1], void 0, function* (producer, sampleSize = 10) {
+            var _a, _b, _c;
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            (0, Affirm_1.default)(sampleSize > 0, 'Sample size must be greater than 0');
+            const source = Environment_1.default.getSource(producer.source);
+            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
+            let sampleData = [];
+            switch (source.engine) {
+                case 'aws-redshift': {
+                    const sql = `SELECT * FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}" LIMIT ${sampleSize}`;
+                    (0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
+                    const driver = yield DriverFactory_1.default.instantiateSource(source);
+                    (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
+                    const res = yield driver.query(sql);
+                    sampleData = res.rows;
+                    break;
+                }
+                case 'local':
+                case 'aws-s3': {
+                    const fileData = yield this.readFile(producer, { readmode: 'lines', lines: { from: 0, to: sampleSize } });
+                    if (((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) === 'CSV') {
+                        sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer);
+                    }
+                    else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
+                        sampleData = fileData.data.map(line => JSON.parse(line));
+                    }
+                    else {
+                        sampleData = fileData.data;
+                    }
+                    break;
+                }
+                default:
+                    throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not supported`);
+            }
+            return sampleData;
+        });
+    }
+}
+const ProducerEngine = new ProducerEngineClass();
+exports.default = ProducerEngine;

package/engines/UsageManager.js

@@ -0,0 +1,61 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const DSTE_1 = __importDefault(require("../core/dste/DSTE"));
+const Helper_1 = __importDefault(require("../helper/Helper"));
+const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
+const Settings_1 = __importDefault(require("../helper/Settings"));
+class UsageManagerClass {
+    constructor() {
+        /**
+         * TODO: I need to group the usage stats into a bucket daily. When and how I do it is still a question...
+         */
+        this.getTodayBucketId = (consumer) => {
+            (0, Affirm_1.default)(consumer, `Invalid consumer`);
+            const now = DSTE_1.default.now();
+            return `${consumer.name}_${now.getUTCFullYear()}_${now.getUTCMonth()}_${now.getUTCDate()}`.toLowerCase();
+        };
+        this.startUsage = (consumer, user) => {
+            const newUsage = {
+                _id: Helper_1.default.uuid(),
+                consumer: consumer.name,
+                startedAt: DSTE_1.default.now(),
+                executedBy: { name: user.name, _id: user._id },
+                itemsCount: -1,
+                status: 'started',
+                _signature: ''
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, newUsage._id, newUsage);
+            return { usageId: newUsage._id, usage: updateRes };
+        };
+        this.endUsage = (usageId, itemsCount) => {
+            const update = {
+                itemsCount: itemsCount,
+                status: 'success',
+                finishedAt: DSTE_1.default.now()
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: null, usage: Promise.resolve(update) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
+            return { usageId: usageId, usage: updateRes };
+        };
+        this.failUsage = (usageId, error) => {
+            const update = {
+                status: 'failed',
+                error: error,
+                finishedAt: DSTE_1.default.now()
+            };
+            if (Helper_1.default.isDev())
+                return { usageId: null, usage: Promise.resolve(update) };
+            const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, usageId, update);
+            return { usageId: usageId, usage: updateRes };
+        };
+    }
+}
+const UsageManager = new UsageManagerClass();
+exports.default = UsageManager;

package/engines/UserManager.js

@@ -0,0 +1,43 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
+const Helper_1 = __importDefault(require("../helper/Helper"));
+const Settings_1 = __importDefault(require("../helper/Settings"));
+class UserManagerClass {
+    constructor() {
+        this.getUser = () => {
+            if (Helper_1.default.isDev())
+                return DEV_USER;
+            // TODO: figure out how to handle users
+        };
+        this.findOIDC = (oid) => __awaiter(this, void 0, void 0, function* () {
+            return yield DatabaseEngine_1.default.findOne(Settings_1.default.db.collections.users, { 'auth.oid': oid });
+        });
+        this.update = (user) => __awaiter(this, void 0, void 0, function* () {
+            return yield DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.users, user._id, user);
+        });
+    }
+}
+const UserManager = new UserManagerClass();
+exports.default = UserManager;
+const DEV_USER = {
+    _id: '__dev__',
+    auth: { oid: '', provider: 'azure' },
+    email: '',
+    name: 'Dev',
+    roles: ['admin'],
+    _signature: '',
+    lastLogin: new Date().toJSON()
+};

package/engines/Validator.js

@@ -0,0 +1,154 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Algo_1 = __importDefault(require("../core/Algo"));
+const ConsumerManager_1 = __importDefault(require("./consumer/ConsumerManager"));
+const Environment_1 = __importDefault(require("./Environment"));
+const ExecutionPlanner_1 = __importDefault(require("./execution/ExecutionPlanner"));
+class ValidatorClass {
+    constructor() {
+        this.validateSources = (sources) => {
+            (0, Affirm_1.default)(sources, 'Invalid sources');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(sources, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in sources: "${dupes.map(x => x.name).join(', ')}"`);
+                for (let i = 0; i < sources.length; i++) {
+                    const source = sources[i];
+                    if (source.engine === 'local' && !source.authentication.path)
+                        errors.push(`For source ${source.name}, the path has not been configured`);
+                }
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Sources. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateProducers = (producers) => {
+            (0, Affirm_1.default)(producers, 'Invalid producers');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(producers, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in producers: "${dupes.map(x => x.name).join(', ')}"`);
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Producers. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateProducer = (producer) => {
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const errors = [];
+            try {
+                if (!producer.source || producer.source.length === 0)
+                    errors.push(`Missing parameter "source" in producer`);
+                if (producer.dimensions.some(x => x.name.includes('{') || x.name.includes('[')))
+                    errors.push(`Invalid dimension name found in producer "${producer.name}": can't use characters "{" or "[" in dimension names`);
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Producer. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateConsumers = (consumers) => {
+            (0, Affirm_1.default)(consumers, 'Invalid consumers');
+            const errors = [];
+            try {
+                const dupes = Algo_1.default.duplicatesObject(consumers, 'name');
+                if (dupes.length > 0)
+                    errors.push(`Duplicate name(s) found in consumers: "${dupes.map(x => x.name).join(', ')}"`);
+                errors.push(...consumers.flatMap(x => this.validateConsumer(x)));
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Consumers. (error: ${e})`);
+            }
+            return errors;
+        };
+        this.validateConsumer = (consumer) => {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            const errors = [];
+            try {
+                // TODO: check that a consumer doesn't consume hitself
+                const allFieldsWithNoFrom = consumer.fields.filter(x => x.key === '*' && !x.from);
+                if (allFieldsWithNoFrom.length > 0 && consumer.producers.length > 1)
+                    errors.push(`Field with key "*" was used without specifying the "from" producer and multiple producers were found.`);
+                if (consumer.fields.some(x => x.key === '*' && x.grouping))
+                    errors.push(`Field with key "*" can't be used for "grouping". Either remove the grouping or change the key.`);
+                // Validation on producers
+                if (consumer.producers.length === 0)
+                    errors.push(`Consumer must have at least 1 producer.`);
+                const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
+                if (producers.length === 0)
+                    errors.push('No producers found');
+                if (producers.some(x => !x))
+                    errors.push(`Invalid producer found in consumer "${consumer.name}"`);
+                // Validation on sources
+                const sources = producers.map(x => Environment_1.default.getSource(x.source));
+                if (sources.length === 0)
+                    errors.push('No sources found');
+                if (sources.some(x => !x))
+                    errors.push(`Invalid source found in consumer "${consumer.name}"`);
+                // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
+                const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
+                if (uniqEngines.length !== 1)
+                    errors.push(`Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
+                // For now we also only support consumers that have producers ALL having the same exact source
+                const uniqNames = Algo_1.default.uniqBy(sources, 'name');
+                if (uniqNames.length !== 1)
+                    errors.push(`Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
+                if (consumer.filters && consumer.filters.length > 0) {
+                    if (consumer.filters.some(x => x.sql && x.rule))
+                        errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
+                    const [source] = ConsumerManager_1.default.getSource(consumer);
+                    const engineClass = ExecutionPlanner_1.default.getEngineClass(source.engine);
+                    if (engineClass === 'file' && consumer.filters.some(x => x.sql))
+                        errors.push(`Filters based on SQL are only valid for SQL based sources. (source: ${source.name})`);
+                    if (engineClass === 'sql' && consumer.filters.some(x => x.rule))
+                        errors.push(`Filters based on rules are only valid for non-SQL based sources. (source: ${source.name})`);
+                }
+                // Validation on fields
+                const validateGroupingLevels = (fields, level = 0) => {
+                    let errors = [];
+                    const groupingFields = fields.filter(x => x.grouping);
+                    if (groupingFields.length > 1)
+                        errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
+                    groupingFields.forEach(field => {
+                        if (field.grouping)
+                            errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
+                    });
+                    return errors;
+                };
+                errors.push(...validateGroupingLevels(consumer.fields));
+                // Validation outputs
+                const duplicatesOutputs = Algo_1.default.duplicatesObject(consumer.outputs, 'format');
+                if (duplicatesOutputs.length > 0) {
+                    const duplicatesTypes = Algo_1.default.uniq(duplicatesOutputs.map(x => x.format));
+                    errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(' and ')})`);
+                }
+                for (let i = 0; i < consumer.outputs.length; i++) {
+                    const output = consumer.outputs[i];
+                    if (output.format === 'SQL' && output.accellerated && output.direct)
+                        errors.push(`An output SQL cannot be both direct and accelerated (output: ${output.format})`);
+                    if ((output.format === 'CSV' || output.format === 'JSON' || output.format === 'PARQUET') && !output.exportDestination)
+                        errors.push(`A static file output must have an export destination set (${output.format})`);
+                }
+            }
+            catch (e) {
+                if (errors.length === 0)
+                    errors.push(`There was an error in the validation Consumer. (error: ${e})`);
+            }
+            return errors;
+        };
+    }
+}
+const Validator = new ValidatorClass();
+exports.default = Validator;

package/engines/ai/AutoMapperEngine.js

@@ -0,0 +1,37 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const LLM_1 = __importDefault(require("./LLM"));
+class AutoMapperEngineClass {
+    constructor() {
+        /**
+         * input: the first ten lines of the uploaded file
+         * outputs: the selected schemas
+         */
+        this.map = (input, outputs) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(input, 'Invalid input');
+            (0, Affirm_1.default)(outputs, 'Invalid outputs');
+            const llm = new LLM_1.default();
+            const producersRes = yield llm.inferProducers(input, outputs);
+            const consumersRes = yield llm.inferConsumers(producersRes.producers, outputs);
+            return {
+                consumers: consumersRes.consumers,
+                producers: producersRes.producers
+            };
+        });
+    }
+}
+const AutoMapperEngine = new AutoMapperEngineClass();
+exports.default = AutoMapperEngine;