@forzalabs/remora 1.0.22 → 1.1.1

package/engines/Environment.js

@@ -1,170 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const SchemaValidator_1 = __importDefault(require("./schema/SchemaValidator"));
-const Validator_1 = __importDefault(require("./validation/Validator"));
-const Constants_1 = __importDefault(require("../Constants"));
-const Logger_1 = __importDefault(require("../helper/Logger"));
-class EnvironmentClass {
-    constructor() {
-        this._env = null;
-        this.load = (remoraPath) => {
-            const envPath = path_1.default.join(remoraPath, 'remora');
-            // Read and parse project.json
-            const projectPath = path_1.default.join(envPath, 'project.json');
-            if (!fs_1.default.existsSync(projectPath))
-                throw new Error('Project configuration file not found');
-            const projectConfig = JSON.parse(fs_1.default.readFileSync(projectPath, 'utf-8'));
-            // Validate project schema
-            const isValid = SchemaValidator_1.default.validate('project-schema', projectConfig);
-            if (!isValid)
-                throw new Error('Invalid project configuration');
-            const loadConfigurations = (basePath, paths) => {
-                const configs = [];
-                for (const configPath of paths) {
-                    const fullPath = path_1.default.join(basePath, configPath);
-                    try {
-                        if (fs_1.default.statSync(fullPath).isDirectory()) {
-                            const files = fs_1.default.readdirSync(fullPath).filter(file => file.endsWith('.json'));
-                            for (const file of files) {
-                                const config = JSON.parse(fs_1.default.readFileSync(path_1.default.join(fullPath, file), 'utf-8'));
-                                configs.push(config);
-                            }
-                        }
-                        else {
-                            const config = JSON.parse(fs_1.default.readFileSync(fullPath, 'utf-8'));
-                            configs.push(config);
-                        }
-                    }
-                    catch (error) {
-                        throw new Error(`Error loading from ${path_1.default.resolve(remoraPath)} configuration from ${configPath}: ${error.message}`);
-                    }
-                }
-                return configs;
-            };
-            // Unload all the schemes other than the default ones, because otherwise the RW could keep loaded schemes not saved in the remora folder
-            SchemaValidator_1.default.unloadSchemas();
-            // Load all schemas first since they're needed for validation
-            const schemas = loadConfigurations(envPath, projectConfig.schemas);
-            schemas.forEach(schema => {
-                SchemaValidator_1.default.addSchema(schema, schema.title);
-            });
-            // Load configurations with validation
-            const sources = loadConfigurations(envPath, projectConfig.sources);
-            sources.forEach(source => {
-                if (!SchemaValidator_1.default.validate('source-schema', source))
-                    throw new Error(`Invalid source configuration: ${source.name}`);
-            });
-            const producers = loadConfigurations(envPath, projectConfig.producers);
-            producers.forEach(producer => {
-                if (!SchemaValidator_1.default.validate('producer-schema', producer))
-                    throw new Error(`Invalid producer configuration: ${producer.name}`);
-            });
-            const consumers = loadConfigurations(envPath, projectConfig.consumers);
-            consumers.forEach(consumer => {
-                if (!SchemaValidator_1.default.validate('consumer-schema', consumer))
-                    throw new Error(`Invalid consumer configuration: ${consumer.name}`);
-            });
-            // Load the project settings
-            const envSettings = new Map(Object.entries(Object.assign({}, projectConfig.settings))
-                .map(([key, value]) => [key, String(value)]));
-            if (!envSettings.has('SQL_MAX_QUERY_ROWS'))
-                envSettings.set('SQL_MAX_QUERY_ROWS', Constants_1.default.defaults.SQL_MAX_QUERY_ROWS.toString());
-            if (!envSettings.has('STRING_MAX_CHARACTERS_LENGTH'))
-                envSettings.set('STRING_MAX_CHARACTERS_LENGTH', Constants_1.default.defaults.STRING_MAX_CHARACTERS_LENGTH.toString());
-            if (!envSettings.has('MAX_ITEMS_IN_MEMORY'))
-                envSettings.set('MAX_ITEMS_IN_MEMORY', Constants_1.default.defaults.MAX_ITEMS_IN_MEMORY.toString());
-            const debugMode = envSettings.get('DEBUG_MODE');
-            try {
-                if (debugMode && debugMode.length > 0 && JSON.parse(debugMode) === true)
-                    Logger_1.default.setLevel('debug');
-            }
-            catch (_a) {
-                console.warn(`Invalid project setting "DEBUG_MODE" (${debugMode})`);
-            }
-            // Initialize environment
-            this.init({
-                settings: envSettings,
-                sources,
-                producers,
-                consumers,
-                schemas,
-                sqlLibrary: [] // TODO: Add SQL library loading if needed
-            });
-        };
-        this.init = (env) => {
-            this._env = env;
-        };
-        this.get = (setting) => {
-            return this._env.settings.get(setting);
-        };
-        this.getSource = (sourceName) => {
-            (0, Affirm_1.default)(sourceName, 'Invalid source name');
-            return this._env.sources.find(x => x.name === sourceName);
-        };
-        /**
-         * A consumer can reference another consumer, in this case I need to keep digging to find the real producer
-         */
-        this.getFirstProducer = (producerName) => {
-            (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            const prod = this._env.producers.find(x => x.name.toLocaleLowerCase() === producerName.toLowerCase());
-            if (!prod) {
-                const consumer = this.getConsumer(producerName);
-                (0, Affirm_1.default)(consumer, `Invalid producer name`);
-                return this.getFirstProducer(consumer.producers[0].name);
-            }
-            return prod;
-        };
-        this.getProducer = (producerName) => {
-            (0, Affirm_1.default)(producerName, `Invalid producer name`);
-            return this._env.producers.find(x => x.name.toLowerCase() === producerName.toLowerCase());
-        };
-        this.getConsumer = (consumerName) => {
-            (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
-            return this._env.consumers.find(x => x.name.toLowerCase() === consumerName.toLowerCase());
-        };
-        this.getAllConsumers = () => {
-            (0, Affirm_1.default)(this._env, 'Environment not initialized');
-            return this._env.consumers || [];
-        };
-        this.getSchema = (schemaName) => {
-            (0, Affirm_1.default)(schemaName, 'Invalid schema name');
-            return this._env.schemas.find(x => x.title === schemaName);
-        };
-        this.getSqlInLibrary = (name) => {
-            (0, Affirm_1.default)(name, `Invalid sql library item name`);
-            return this._env.sqlLibrary.find(x => x.name === name);
-        };
-        this.validate = () => {
-            (0, Affirm_1.default)(this._env, 'Invalid environment');
-            let errors = [];
-            try {
-                errors = [...errors, ...Validator_1.default.validateSources(this._env.sources)];
-                errors = [...errors, ...Validator_1.default.validateProducers(this._env.producers)];
-                this._env.producers.forEach(prod => {
-                    const ce = Validator_1.default.validateProducer(prod);
-                    if (ce.length > 0)
-                        errors.push(`Producer "${prod.name}" errors:\n${ce.map(x => `\t-${x}\n`)}`);
-                });
-                errors = [...errors, ...Validator_1.default.validateConsumers(this._env.consumers)];
-                this._env.consumers.forEach(cons => {
-                    const ce = Validator_1.default.validateConsumer(cons);
-                    if (ce.length > 0)
-                        errors.push(`Consumer "${cons.name}" errors:\n${ce.map(x => `\t-${x}\n`)}`);
-                });
-            }
-            catch (e) {
-                if (errors.length === 0)
-                    errors.push(`There was an error in the validation Environment. (error: ${e})`);
-            }
-            return errors;
-        };
-    }
-}
-const Environment = new EnvironmentClass();
-exports.default = Environment;

package/engines/ProcessENVManager.js

@@ -1,112 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs = __importStar(require("fs"));
-const process_1 = __importDefault(require("process"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
-const ProcessENV_1 = require("../definitions/ProcessENV");
-const Algo_1 = __importDefault(require("../core/Algo"));
-const Logger_1 = __importDefault(require("../helper/Logger"));
-class ProcessENVManagerClass {
-    constructor() {
-        this._USER_ACCEPTED_ENV_VARIABLE_BEGINNING = 'REMORA';
-        /**
-         * Load the environment variables in the manager.
-         * WARN: call this after the environment has been setup (after the process.env variables have been loaded).
-         */
-        this.init = () => __awaiter(this, void 0, void 0, function* () {
-            const appName = process_1.default.env.REMORA_APP;
-            (0, Affirm_1.default)(appName, `Missing REMORA_APP in the environment variables. Add the environment variable REMORA_APP=<cli | worker | backend | frontend> depending on the app you are running.`);
-            this._loadedVariables = {};
-            for (const variable of ProcessENV_1.REMORA_ENV_VARIABLES) {
-                if (variable.apps.some(x => x.name === appName)) {
-                    const { apps, name } = variable;
-                    const app = apps.find(x => x.name === appName);
-                    const value = this._loadEnvVarValue(name);
-                    if (app.required && !Algo_1.default.hasVal(value))
-                        throw new Error(`Missing required Remora environment variable: ${variable}`);
-                    this._loadedVariables[name] = value;
-                }
-            }
-            const userRemoraEnvVariables = Object.keys(process_1.default.env)
-                .filter(x => x.startsWith(this._USER_ACCEPTED_ENV_VARIABLE_BEGINNING))
-                .map(x => ({ name: x, value: process_1.default.env[x] }));
-            for (const envVar of userRemoraEnvVariables) {
-                const varName = envVar.name;
-                this._loadedVariables[varName] = this._loadEnvVarValue(varName);
-            }
-        });
-        this._loadEnvVarValue = (varName) => {
-            if (process_1.default.env[varName] && process_1.default.env[varName].includes('run/secrets/'))
-                return fs.readFileSync(process_1.default.env[varName], 'utf-8');
-            else
-                return process_1.default.env[varName];
-        };
-        /**
-         * Get the value of an environment variable and give an error/warn if not implemented.
-         * @param variable The name of the environment variable desired (usually all UPPER_CASE)
-         * @returns The value of environment variable
-         */
-        this.getEnvVariable = (variable) => {
-            (0, Affirm_1.default)(variable, `Cannot read an undefined env variable`);
-            if (!Algo_1.default.hasVal(this._loadedVariables)) {
-                Logger_1.default.log(`Trying to read an environment variable (${variable}) before initializing the env manager.`);
-                return null;
-            }
-            const result = this._loadedVariables[variable];
-            if (Algo_1.default.hasVal(result)) {
-                return result;
-            }
-            else {
-                console.warn(`Trying to read an environment variable that was not found: ${variable}`);
-                return undefined;
-            }
-        };
-    }
-}
-const ProcessENVManager = new ProcessENVManagerClass();
-exports.default = ProcessENVManager;

package/engines/RandomEngine.js

@@ -1,47 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const seedrandom_1 = __importDefault(require("seedrandom"));
-const Algo_1 = __importDefault(require("../core/Algo"));
-class RandomEngineClass {
-    constructor() {
-        this.rng = (min, max) => {
-            const rng = Math.random;
-            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
-                return Math.floor(rng() * (max - min + 1)) + min;
-            else
-                return rng();
-        };
-        this.sRng = (seed, min, max) => {
-            const rng = (0, seedrandom_1.default)(String(seed));
-            if (Algo_1.default.hasVal(min) || Algo_1.default.hasVal(max))
-                return Math.floor(rng() * (max - min + 1)) + min;
-            else
-                return rng();
-        };
-        this.rngDate = (min, max) => {
-            let myMin = min;
-            let myMax = max;
-            if (!Algo_1.default.hasVal(min) && !Algo_1.default.hasVal(max)) {
-                myMin = new Date('1900-01-01').getTime();
-                myMax = new Date().getTime();
-            }
-            const randomNumber = this.rng(myMin, myMax);
-            return new Date(randomNumber).toJSON();
-        };
-        this.sRngDate = (seed, min, max) => {
-            let myMin = min;
-            let myMax = max;
-            if (!Algo_1.default.hasVal(min) && !Algo_1.default.hasVal(max)) {
-                myMin = new Date('1900-01-01').getTime();
-                myMax = new Date().getTime();
-            }
-            const randomNumber = this.sRng(seed, myMin, myMax);
-            return new Date(randomNumber).toJSON();
-        };
-    }
-}
-const RandomEngine = new RandomEngineClass();
-exports.default = RandomEngine;

package/engines/SecretManager.js

@@ -1,23 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const ProcessENVManager_1 = __importDefault(require("./ProcessENVManager"));
-class SecretManagerClass {
-    constructor() {
-        /**
-         * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
-         * Starts with "{" and ends with "}".
-         * e.g. {AWS_ID}
-         */
-        this.replaceSecret = (value) => {
-            if (!value || value.length <= 2 || !value.startsWith('{') || !value.endsWith('}'))
-                return value;
-            const parsedValue = value.slice(1, value.length - 1);
-            return ProcessENVManager_1.default.getEnvVariable(parsedValue);
-        };
-    }
-}
-const SecretManager = new SecretManagerClass();
-exports.default = SecretManager;

package/engines/UserManager.js

@@ -1,66 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const DatabaseEngine_1 = __importDefault(require("../database/DatabaseEngine"));
-const Helper_1 = __importDefault(require("../helper/Helper"));
-const Settings_1 = __importDefault(require("../helper/Settings"));
-class UserManagerClass {
-    constructor() {
-        this.getUser = () => {
-            if (Helper_1.default.isDev())
-                return DEV_USER;
-            else
-                return MOCK_USER;
-            // TODO: figure out how to handle users
-        };
-        this.getRemoraWorkerUser = () => {
-            const remora = {
-                _id: '__remora_worker__',
-                auth: { oid: '', provider: 'internal' },
-                email: '',
-                name: 'Remora Worker',
-                roles: ['root'],
-                _signature: '',
-                lastLogin: new Date().toJSON()
-            };
-            return remora;
-        };
-        this.findOIDC = (oid) => __awaiter(this, void 0, void 0, function* () {
-            return yield DatabaseEngine_1.default.findOne(Settings_1.default.db.collections.users, { 'auth.oid': oid });
-        });
-        this.update = (user) => __awaiter(this, void 0, void 0, function* () {
-            return yield DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.users, user._id, user);
-        });
-    }
-}
-const UserManager = new UserManagerClass();
-exports.default = UserManager;
-const DEV_USER = {
-    _id: '__dev__',
-    auth: { oid: '', provider: 'azure' },
-    email: '',
-    name: 'Dev',
-    roles: ['admin'],
-    _signature: '',
-    lastLogin: new Date().toJSON()
-};
-const MOCK_USER = {
-    _id: '__mock__',
-    auth: { oid: '', provider: 'azure' },
-    email: 'mock.user@email.com',
-    name: 'Mock User',
-    roles: ['user'],
-    _signature: '',
-    lastLogin: new Date().toJSON()
-};

package/engines/ai/AutoMapperEngine.js

@@ -1,37 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const LLM_1 = __importDefault(require("./LLM"));
-class AutoMapperEngineClass {
-    constructor() {
-        /**
-         * input: the first ten lines of the uploaded file
-         * outputs: the selected schemas
-         */
-        this.map = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(input, 'Invalid input');
-            (0, Affirm_1.default)(outputs, 'Invalid outputs');
-            const llm = new LLM_1.default();
-            const producersRes = yield llm.inferProducers(input, outputs, fileName, sources);
-            const consumersRes = yield llm.inferConsumers(producersRes.producers, outputs);
-            return {
-                consumers: consumersRes.consumers,
-                producers: producersRes.producers
-            };
-        });
-    }
-}
-const AutoMapperEngine = new AutoMapperEngineClass();
-exports.default = AutoMapperEngine;