@forzalabs/remora 0.0.16 → 0.0.18

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.16',
+    cliVersion: '0.0.18',
     lambdaVersion: 1,
     port: 5069
 };

package/actions/create_consumer.js

@@ -0,0 +1,75 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_consumer = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const create_consumer = (name, producerName) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        if (!fs_1.default.existsSync('./remora/consumers')) {
+            throw new Error(chalk_1.default.red('Missing directory: ') + chalk_1.default.yellow('./remora/consumers'));
+        }
+        // Read the default consumer template
+        const defaultConsumerTemplate = fs_1.default.readFileSync(path_1.default.join(__dirname, '../documentation/default_resources/consumer.json'), 'utf-8');
+        const defaultConsumer = JSON.parse(defaultConsumerTemplate);
+        defaultConsumer.name = name;
+        // If a producer is specified, set up a one-to-one mapping
+        if (producerName) {
+            const producerPath = path_1.default.join('remora/producers', `${producerName}.json`);
+            if (!fs_1.default.existsSync(producerPath)) {
+                throw new Error(chalk_1.default.red('Producer not found: ') + chalk_1.default.yellow(producerPath));
+            }
+            const producerConfig = JSON.parse(fs_1.default.readFileSync(producerPath, 'utf-8'));
+            // Set up producer reference
+            defaultConsumer.producers = [{ name: producerName }];
+            // Set up fields to include all producer fields
+            defaultConsumer.fields = producerConfig.dimensions.map(dim => ({
+                key: dim.name,
+                from: producerName
+            }));
+            // If there are measures, include those too
+            if (producerConfig.measures && producerConfig.measures.length > 0) {
+                defaultConsumer.fields.push(...producerConfig.measures.map(measure => ({
+                    key: measure.name,
+                    from: producerName
+                })));
+            }
+            defaultConsumer.description = `Consumer for ${producerName} data`;
+            defaultConsumer.schema = undefined;
+            defaultConsumer.filters = undefined;
+            defaultConsumer.metadata = undefined;
+        }
+        const consumerPath = path_1.default.join('remora/consumers', `${name}.json`);
+        fs_1.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
+        console.log(chalk_1.default.green(`✅ Created consumer config at ${consumerPath}`));
+        if (!producerName) {
+            console.log(chalk_1.default.blue('Remember to:'));
+            console.log(chalk_1.default.blue('1. Set the correct producer name'));
+            console.log(chalk_1.default.blue('2. Configure the fields you want to include'));
+            console.log(chalk_1.default.blue('3. Set up the desired output formats'));
+        }
+        else {
+            console.log(chalk_1.default.blue('Consumer is configured to match all fields from producer:'), chalk_1.default.yellow(producerName));
+            console.log(chalk_1.default.blue('Remember to:'));
+            console.log(chalk_1.default.blue('1. Review the included fields'));
+            console.log(chalk_1.default.blue('2. Set up the desired output formats'));
+        }
+    }
+    catch (error) {
+        console.error(chalk_1.default.red('❌ Error creating consumer:'), error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
+exports.create_consumer = create_consumer;

package/actions/create_producer.js

@@ -0,0 +1,41 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_producer = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const create_producer = (name) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        if (!fs_1.default.existsSync('./remora/producers')) {
+            throw new Error(chalk_1.default.red('Missing directory: ') + chalk_1.default.yellow('./remora/producers'));
+        }
+        // Read the default producer template
+        const defaultProducerTemplate = fs_1.default.readFileSync(path_1.default.join(__dirname, '../documentation/default_resources/producer.json'), 'utf-8');
+        const defaultProducer = JSON.parse(defaultProducerTemplate);
+        defaultProducer.name = name;
+        const producerPath = path_1.default.join('remora/producers', `${name}.json`);
+        fs_1.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
+        console.log(chalk_1.default.green(`✅ Created producer config at ${producerPath}`));
+        console.log(chalk_1.default.blue('Remember to:'));
+        console.log(chalk_1.default.blue('1. Set the correct source name'));
+        console.log(chalk_1.default.blue('2. Configure the dimensions and measures (or use the "discover" command)'));
+        console.log(chalk_1.default.blue('3. Update the settings as needed'));
+    }
+    catch (error) {
+        console.error(chalk_1.default.red('❌ Error creating producer:'), error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
+exports.create_producer = create_producer;

package/actions/run.js

@@ -19,17 +19,24 @@ const Environment_1 = __importDefault(require("../engines/Environment"));
 const UserManager_1 = __importDefault(require("../engines/UserManager"));
 const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEngine"));
 const compile_1 = require("./compile");
-const run = (options, consumersName) => __awaiter(void 0, void 0, void 0, function* () {
+const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
-        const spinner = (0, ora_1.default)(chalk_1.default.blue('Run Consumers...\n')).start();
-        const consumersToExecute = (consumersName && consumersName.length > 0) ? [Environment_1.default.getConsumer(consumersName)] : Environment_1.default._env.consumers;
+        const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
         const user = UserManager_1.default.getUser();
+        const consumersToExecute = (consumerName && consumerName.length > 0)
+            ? [Environment_1.default.getConsumer(consumerName)]
+            : Environment_1.default._env.consumers;
         const results = [];
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];
-            const response = yield ConsumerEngine_1.default.execute(consumer, {}, user);
-            results.push({ consumer, response });
+            try {
+                const response = yield ConsumerEngine_1.default.execute(consumer, {}, user);
+                results.push({ consumer, response });
+            }
+            catch (error) {
+                console.error(chalk_1.default.red(`Failed to execute consumer ${consumer.name}:`, error instanceof Error ? error.message : String(error)));
+            }
         }
         spinner.succeed('All consumers have been successfully executed');
         results.forEach(result => {

package/engines/ProducerEngine.js

@@ -101,6 +101,8 @@ class ProducerEngineClass {
                 }
                 case 'JSONL':
                 case 'JSON': {
+                    if (lines.length === 1)
+                        lines = lines[0].split('\n');
                     const json = lines.map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }

package/engines/UserManager.js

@@ -20,6 +20,8 @@ class UserManagerClass {
         this.getUser = () => {
             if (Helper_1.default.isDev())
                 return DEV_USER;
+            else
+                return MOCK_USER;
             // TODO: figure out how to handle users
         };
         this.findOIDC = (oid) => __awaiter(this, void 0, void 0, function* () {
@@ -41,3 +43,12 @@ const DEV_USER = {
     _signature: '',
     lastLogin: new Date().toJSON()
 };
+const MOCK_USER = {
+    _id: '__mock__',
+    auth: { oid: '', provider: 'azure' },
+    email: '',
+    name: 'mock',
+    roles: ['user'],
+    _signature: '',
+    lastLogin: new Date().toJSON()
+};

package/engines/Validator.js

@@ -121,7 +121,7 @@ class ValidatorClass {
                     const groupingFields = fields.filter(x => x.grouping);
                     if (groupingFields.length > 1)
                         errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
-                    groupingFields.forEach((field) => {
+                    groupingFields.forEach(field => {
                         if (field.grouping)
                             errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
                     });
@@ -134,13 +134,16 @@ class ValidatorClass {
                     const duplicatesTypes = Algo_1.default.uniq(duplicatesOutputs.map(x => x.format));
                     errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(' and ')})`);
                 }
-                for (let i = 0; i < consumer.outputs.length; i++) {
-                    const output = consumer.outputs[i];
+                for (const output of consumer.outputs) {
                     const format = output.format.toUpperCase();
                     if (format === 'SQL' && output.accellerated && output.direct)
                         errors.push(`An output SQL cannot be both direct and accelerated (output: ${format})`);
-                    if ((format === 'CSV' || format === 'JSON' || format === 'PARQUET') && !output.exportDestination)
-                        errors.push(`A static file output must have an export destination set (${format})`);
+                    if ((format === 'CSV' || format === 'JSON' || format === 'PARQUET')) {
+                        if (!output.exportDestination)
+                            errors.push(`A static file output must have an export destination set (${format})`);
+                        else if (!Environment_1.default.getSource(output.exportDestination))
+                            errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
+                    }
                 }
             }
             catch (e) {

package/engines/consumer/PostProcessor.js

@@ -74,7 +74,6 @@ class PostProcessorClass {
             (0, Affirm_1.default)(data, 'Invalid data');
             (0, Affirm_1.default)(Array.isArray(data), 'Invalid data type, must be an array');
             (0, Affirm_1.default)(producer, 'Invalid producer');
-            console.log('BBBBBBBBB');
             const source = Environment_1.default.getSource(producer.source);
             (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
             const columns = FileCompiler_1.default.compileProducer(producer, source);

package/index.js

@@ -14,6 +14,8 @@ const run_1 = require("./actions/run");
 const Constants_1 = __importDefault(require("./Constants"));
 const discover_1 = require("./actions/discover");
 const automap_1 = require("./actions/automap");
+const create_producer_1 = require("./actions/create_producer");
+const create_consumer_1 = require("./actions/create_consumer");
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
 program
@@ -46,7 +48,7 @@ program
     .option('--build-only', 'Build without deploying')
     .action(deploy_1.deploy);
 program
-    .command('run [consumersName]')
+    .command('run [name]')
     .description('Execute consumers. Optionally specify a single consumers name to run.')
     .action(run_1.run);
 program
@@ -60,4 +62,13 @@ program
     .argument('<producer>', 'The producer to analyze')
     .argument('<schemas...>', 'One or more schema names to map against')
     .action(automap_1.automap);
+program
+    .command('create-producer <name>')
+    .description('Create a new producer configuration with default settings')
+    .action(create_producer_1.create_producer);
+program
+    .command('create-consumer <name>')
+    .description('Create a new consumer configuration with default settings')
+    .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
+    .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.16",
+    "version": "0.0.18",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,