@forzalabs/remora 1.0.21 → 1.0.22

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '1.0.21',
+    cliVersion: '1.0.22',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,

package/actions/debug.js

@@ -21,7 +21,7 @@ const compile_1 = require("./compile");
 const debug = (options) => __awaiter(void 0, void 0, void 0, function* () {
     void options;
     try {
-        (0, compile_1.compile)();
+        yield (0, compile_1.compile)();
         console.log('\n');
         const spinner = (0, ora_1.default)({ text: '⏳ Checking producers connections...', color: 'blue' }).start();
         const producers = Environment_1.default._env.producers;

package/definitions/ProcessENV.js

@@ -1,2 +1,169 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.REMORA_ENV_VARIABLES = void 0;
+/**
+ * All the environment variables required for operativity of remora
+ */
+exports.REMORA_ENV_VARIABLES = [
+    {
+        name: 'REMORA_LICENCE_KEY',
+        apps: [
+            { name: 'cli', required: true },
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: 'The licence key to activate this remora licence. If you don\'t have it contact your tech support or reach out to Forza Labs.'
+    },
+    {
+        name: 'REMORA_APP',
+        apps: [
+            { name: 'cli', required: true },
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: 'The Remora app name (cli, worker, backend, frontend).'
+    },
+    {
+        name: 'REMORA_WORKER_HOST',
+        apps: [
+            { name: 'backend', required: true },
+            { name: 'cli', required: false }
+        ],
+        description: 'The host address for the deployed Remora worker instance.'
+    },
+    {
+        name: 'ADMIN_JWT_SECRET_FILE',
+        apps: [
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
+    },
+    {
+        name: 'JWT_SECRET_FILE',
+        apps: [
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: ''
+    },
+    {
+        name: 'ROOT_TEMP_PASSWORD_FILE',
+        apps: [
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: ''
+    },
+    {
+        name: 'REMORA_WORKERS_PATH',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: 'A valid path that overrides the default path for the multi-thread workers for the Remora engine.'
+    },
+    {
+        name: 'REMORA_KEY_SECRET_FILE',
+        apps: [
+            { name: 'worker', required: true },
+            { name: 'cli', required: false }
+        ],
+        description: 'The CLI '
+    },
+    {
+        name: 'ADMIN_JWT_SECRET_FILE',
+        apps: [
+            { name: 'worker', required: true }
+        ],
+        description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
+    },
+    {
+        name: 'ADMIN_JWT_SECRET_FILE',
+        apps: [
+            { name: 'worker', required: true }
+        ],
+        description: 'The secret used to verify the JWT from the admin user of the Remora WebApp. Used for all admin operations.'
+    },
+    {
+        name: 'MONGO_URI',
+        apps: [
+            { name: 'worker', required: true },
+            { name: 'backend', required: true }
+        ],
+        description: 'The databse uri (Mongo compatible) where to store Remora telemetry.'
+    },
+    {
+        name: 'REMORA_SALT',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: 'A secret salt value used when masking data.'
+    },
+    {
+        name: 'OPENAI_API_KEY',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'AWS_DEFAULT_REGION',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'AWS_ACCOUNT_ID',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'AWS_ACCESS_KEY_ID',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'AWS_SECRET_ACCESS_KEY',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'AWS_SESSION_TOKEN',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'DELTA_TOKEN',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false }
+        ],
+        description: ''
+    },
+    {
+        name: 'NODE_ENV',
+        apps: [
+            { name: 'worker', required: false },
+            { name: 'cli', required: false },
+            { name: 'backend', required: false }
+        ],
+        description: ''
+    }
+];

package/engines/Environment.js

@@ -13,9 +13,6 @@ const Logger_1 = __importDefault(require("../helper/Logger"));
 class EnvironmentClass {
     constructor() {
         this._env = null;
-        this.init = (env) => {
-            this._env = env;
-        };
         this.load = (remoraPath) => {
             const envPath = path_1.default.join(remoraPath, 'remora');
             // Read and parse project.json
@@ -100,6 +97,9 @@ class EnvironmentClass {
                 sqlLibrary: [] // TODO: Add SQL library loading if needed
             });
         };
+        this.init = (env) => {
+            this._env = env;
+        };
         this.get = (setting) => {
             return this._env.settings.get(setting);
         };

package/engines/ProcessENVManager.js

@@ -32,6 +32,15 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -39,42 +48,62 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const process_1 = __importDefault(require("process"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const ProcessENV_1 = require("../definitions/ProcessENV");
+const Algo_1 = __importDefault(require("../core/Algo"));
+const Logger_1 = __importDefault(require("../helper/Logger"));
 class ProcessENVManagerClass {
     constructor() {
+        this._USER_ACCEPTED_ENV_VARIABLE_BEGINNING = 'REMORA';
         /**
-         * All the variables foundamental for operativity of remora
-         */
-        this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES = ['REMORA_LICENCE_KEY'];
-        /**
-         * All the variables that are not foundamental for the operativity of remora
+         * Load the environment variables in the manager.
+         * WARN: call this after the environment has been setup (after the process.env variables have been loaded).
          */
-        this.ENVIRONMENT_VARIABLES = ['REMORA_KEY_SECRET_FILE', 'REMORA_WORKERS_PATH', 'ADMIN_JWT_SECRET_FILE', 'JWT_SECRET_FILE', 'ROOT_TEMP_PASSWORD_FILE', 'REMORA_WORKER_HOST', 'MONGO_URI', 'REMORA_SALT', 'OPENAI_API_KEY', 'AWS_DEFAULT_REGION', 'AWS_ACCOUNT_ID', 'AWS_ACCESS_KEY_ID', 'AWS_SESSION_TOKEN', 'NODE_ENV'];
+        this.init = () => __awaiter(this, void 0, void 0, function* () {
+            const appName = process_1.default.env.REMORA_APP;
+            (0, Affirm_1.default)(appName, `Missing REMORA_APP in the environment variables. Add the environment variable REMORA_APP=<cli | worker | backend | frontend> depending on the app you are running.`);
+            this._loadedVariables = {};
+            for (const variable of ProcessENV_1.REMORA_ENV_VARIABLES) {
+                if (variable.apps.some(x => x.name === appName)) {
+                    const { apps, name } = variable;
+                    const app = apps.find(x => x.name === appName);
+                    const value = this._loadEnvVarValue(name);
+                    if (app.required && !Algo_1.default.hasVal(value))
+                        throw new Error(`Missing required Remora environment variable: ${variable}`);
+                    this._loadedVariables[name] = value;
+                }
+            }
+            const userRemoraEnvVariables = Object.keys(process_1.default.env)
+                .filter(x => x.startsWith(this._USER_ACCEPTED_ENV_VARIABLE_BEGINNING))
+                .map(x => ({ name: x, value: process_1.default.env[x] }));
+            for (const envVar of userRemoraEnvVariables) {
+                const varName = envVar.name;
+                this._loadedVariables[varName] = this._loadEnvVarValue(varName);
+            }
+        });
+        this._loadEnvVarValue = (varName) => {
+            if (process_1.default.env[varName] && process_1.default.env[varName].includes('run/secrets/'))
+                return fs.readFileSync(process_1.default.env[varName], 'utf-8');
+            else
+                return process_1.default.env[varName];
+        };
         /**
-         * get the value of an environment variable and give an error/warn if not implemented.
-         * @param variable the string text of the environment varible desired
-         * @returns the value of environment variable
+         * Get the value of an environment variable and give an error/warn if not implemented.
+         * @param variable The name of the environment variable desired (usually all UPPER_CASE)
+         * @returns The value of environment variable
          */
         this.getEnvVariable = (variable) => {
-            (0, Affirm_1.default)(variable, `Cannot read an undefined variable`);
-            if (!this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES.includes(variable) && !this.ENVIRONMENT_VARIABLES.includes(variable)) {
-                console.warn(`Trying to read a variable that is not implemented: ${variable}`);
-            }
-            if (!process_1.default.env[variable] && this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES.includes(variable)) {
-                throw new Error(`Missing necessary property for ${variable}`);
-            }
-            else if (!process_1.default.env[variable]) {
-                // give a warn if it's not neccessary for the operativity of remora
-                console.warn(`Missing property for ${variable}`);
+            (0, Affirm_1.default)(variable, `Cannot read an undefined env variable`);
+            if (!Algo_1.default.hasVal(this._loadedVariables)) {
+                Logger_1.default.log(`Trying to read an environment variable (${variable}) before initializing the env manager.`);
                 return null;
             }
-            // checks if the property is a path of the docker secrets manager
-            if (process_1.default.env[variable]) {
-                if (process_1.default.env[variable].includes('run/secrets/') && process_1.default.env[variable]) {
-                    return fs.readFileSync(process_1.default.env[variable], 'utf-8');
-                }
-                else {
-                    return process_1.default.env[variable];
-                }
+            const result = this._loadedVariables[variable];
+            if (Algo_1.default.hasVal(result)) {
+                return result;
+            }
+            else {
+                console.warn(`Trying to read an environment variable that was not found: ${variable}`);
+                return undefined;
             }
         };
     }

package/index.js

@@ -22,97 +22,102 @@ const sample_1 = require("./actions/sample");
 const mock_1 = require("./actions/mock");
 const ProcessENVManager_1 = __importDefault(require("./engines/ProcessENVManager"));
 dotenv_1.default.configDotenv();
-const program = new commander_1.Command();
-// Validate the remora licence
-const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
-const check = LicenceManager_1.default.validate(remoraLicenceKey);
-if (!check.valid) {
-    console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
-    process.exit(1);
-}
-// Runtime check on heap size to warn user of insufficent runtime resources
-const { heapSizeMB } = Runtime_1.default.getHeap();
-if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
-    console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
-else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
-    console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
-// Initialize all commands
-program
-    .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
-    .description('CLI tool for setting up and managing Data-Remora');
-program
-    .command('version')
-    .description('Display the version of the CLI')
-    .action(() => {
-    const packageJson = require('../package.json');
-    console.log(`Data-Remora CLI version: ${packageJson.version}`);
-});
-program
-    .command('init')
-    .description('Initialize the application and set up configuration')
-    .action(init_1.init);
-program
-    .command('compile')
-    .description('Compile the config and check for errors')
-    .action(compile_1.compile);
-program
-    .command('debug')
-    .description('Check the connection with sources and producers')
-    .action(debug_1.debug);
-program
-    .command('deploy')
-    .description('Deploy the application to the specified environment')
-    .option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
-    .option('--skip-tests', 'Skip running tests before deployment')
-    .option('--build-only', 'Build without deploying')
-    .action(deploy_1.deploy);
-program
-    .command('run [name]')
-    .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
-    .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
-    .action((name, options) => (0, run_1.run)(name, options));
-program
-    .command('discover')
-    .description('Discover the data shape of a producer and automatically create the resource for it.')
-    .argument('<producer>', 'The producer to discover')
-    .action(discover_1.discover);
-program
-    .command('create-producer <name>')
-    .description('Create a new producer configuration with default settings')
-    .action(create_producer_1.create_producer);
-program
-    .command('automap')
-    .description('Automatically map a producer to consumers using specified schemas.')
-    .argument('<producer>', 'The producer to analyze')
-    .argument('<schemas...>', 'One or more schema names to map against')
-    .action(automap_1.automap);
-program
-    .command('create-consumer <name>')
-    .description('Create a new consumer configuration with default settings')
-    .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
-    .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
-program
-    .command('sample <name>')
-    .description('Sample data from a producer or consumer and display it in a formatted table')
-    .option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
-    .action((name, options) => {
-    const sampleSize = parseInt(options.size, 10);
-    if (isNaN(sampleSize) || sampleSize <= 0) {
-        console.error('Sample size must be a positive number');
-        process.exit(1);
-    }
-    (0, sample_1.sample)(name, sampleSize);
-});
-program
-    .command('mock <producer>')
-    .description('Generate mock data for a producer based on its dimensions')
-    .argument('<records>', 'Number of mock records to generate')
-    .action((producer, records) => {
-    const recordCount = parseInt(records, 10);
-    if (isNaN(recordCount) || recordCount <= 0) {
-        console.error('Record count must be a positive number');
+ProcessENVManager_1.default.init()
+    .then(() => {
+    // Validate the remora licence
+    const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
+    const check = LicenceManager_1.default.validate(remoraLicenceKey);
+    if (!check.valid) {
+        console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
         process.exit(1);
     }
-    (0, mock_1.mock)(producer, recordCount);
+    // Runtime check on heap size to warn user of insufficent runtime resources
+    const { heapSizeMB } = Runtime_1.default.getHeap();
+    if (heapSizeMB < Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB)
+        console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_1.default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
+    else if (heapSizeMB < Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB)
+        console.warn(`Remora is running with ${heapSizeMB} MB of runtime heap, which is below the recommended of ${Constants_1.default.defaults.RECOMMENDED_RUNTIME_HEAP_MB} MB.`);
+})
+    .then(() => {
+    // Initialize all commands
+    const program = new commander_1.Command();
+    program
+        .version(Constants_1.default.cliVersion + '', '-v, --version', 'Display the version of the CLI')
+        .description('CLI tool for setting up and managing Data-Remora');
+    program
+        .command('version')
+        .description('Display the version of the CLI')
+        .action(() => {
+        const packageJson = require('../package.json');
+        console.log(`Data-Remora CLI version: ${packageJson.version}`);
+    });
+    program
+        .command('init')
+        .description('Initialize the application and set up configuration')
+        .action(init_1.init);
+    program
+        .command('compile')
+        .description('Compile the config and check for errors')
+        .action(compile_1.compile);
+    program
+        .command('debug')
+        .description('Check the connection with sources and producers')
+        .action(debug_1.debug);
+    program
+        .command('deploy')
+        .description('Deploy the application to the specified environment')
+        .option('-e, --env <environment>', 'Target environment (staging, production)', 'staging')
+        .option('--skip-tests', 'Skip running tests before deployment')
+        .option('--build-only', 'Build without deploying')
+        .action(deploy_1.deploy);
+    program
+        .command('run [name]')
+        .description('Execute consumers. Optionally specify a single consumer name or use --project to run all consumers in a project.')
+        .option('-p, --project <name>', 'Run all consumers belonging to the specified project')
+        .action((name, options) => (0, run_1.run)(name, options));
+    program
+        .command('discover')
+        .description('Discover the data shape of a producer and automatically create the resource for it.')
+        .argument('<producer>', 'The producer to discover')
+        .action(discover_1.discover);
+    program
+        .command('create-producer <name>')
+        .description('Create a new producer configuration with default settings')
+        .action(create_producer_1.create_producer);
+    program
+        .command('automap')
+        .description('Automatically map a producer to consumers using specified schemas.')
+        .argument('<producer>', 'The producer to analyze')
+        .argument('<schemas...>', 'One or more schema names to map against')
+        .action(automap_1.automap);
+    program
+        .command('create-consumer <name>')
+        .description('Create a new consumer configuration with default settings')
+        .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
+        .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
+    program
+        .command('sample <name>')
+        .description('Sample data from a producer or consumer and display it in a formatted table')
+        .option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
+        .action((name, options) => {
+        const sampleSize = parseInt(options.size, 10);
+        if (isNaN(sampleSize) || sampleSize <= 0) {
+            console.error('Sample size must be a positive number');
+            process.exit(1);
+        }
+        (0, sample_1.sample)(name, sampleSize);
+    });
+    program
+        .command('mock <producer>')
+        .description('Generate mock data for a producer based on its dimensions')
+        .argument('<records>', 'Number of mock records to generate')
+        .action((producer, records) => {
+        const recordCount = parseInt(records, 10);
+        if (isNaN(recordCount) || recordCount <= 0) {
+            console.error('Record count must be a positive number');
+            process.exit(1);
+        }
+        (0, mock_1.mock)(producer, recordCount);
+    });
+    program.parse(process.argv);
 });
-program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.21",
+    "version": "1.0.22",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -17,8 +17,10 @@ const dotenv_1 = __importDefault(require("dotenv"));
 const Affirm_1 = __importDefault(require("../core/Affirm"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
 const Executor_1 = __importDefault(require("../executors/Executor"));
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 dotenv_1.default.configDotenv();
 const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
+    yield ProcessENVManager_1.default.init();
     Environment_1.default.load('./');
     try {
         const { workerId, chunk, consumer, producer, prodDimensions, scope, options } = workerData;