@forzalabs/remora 0.0.15 → 0.0.17

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.15',
+    cliVersion: '0.0.17',
     lambdaVersion: 1,
     port: 5069
 };

package/actions/create_consumer.js

@@ -0,0 +1,75 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_consumer = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const create_consumer = (name, producerName) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        if (!fs_1.default.existsSync('./remora/consumers')) {
+            throw new Error(chalk_1.default.red('Missing directory: ') + chalk_1.default.yellow('./remora/consumers'));
+        }
+        // Read the default consumer template
+        const defaultConsumerTemplate = fs_1.default.readFileSync(path_1.default.join(__dirname, '../documentation/default_resources/consumer.json'), 'utf-8');
+        const defaultConsumer = JSON.parse(defaultConsumerTemplate);
+        defaultConsumer.name = name;
+        // If a producer is specified, set up a one-to-one mapping
+        if (producerName) {
+            const producerPath = path_1.default.join('remora/producers', `${producerName}.json`);
+            if (!fs_1.default.existsSync(producerPath)) {
+                throw new Error(chalk_1.default.red('Producer not found: ') + chalk_1.default.yellow(producerPath));
+            }
+            const producerConfig = JSON.parse(fs_1.default.readFileSync(producerPath, 'utf-8'));
+            // Set up producer reference
+            defaultConsumer.producers = [{ name: producerName }];
+            // Set up fields to include all producer fields
+            defaultConsumer.fields = producerConfig.dimensions.map(dim => ({
+                key: dim.name,
+                from: producerName
+            }));
+            // If there are measures, include those too
+            if (producerConfig.measures && producerConfig.measures.length > 0) {
+                defaultConsumer.fields.push(...producerConfig.measures.map(measure => ({
+                    key: measure.name,
+                    from: producerName
+                })));
+            }
+            defaultConsumer.description = `Consumer for ${producerName} data`;
+            defaultConsumer.schema = undefined;
+            defaultConsumer.filters = undefined;
+            defaultConsumer.metadata = undefined;
+        }
+        const consumerPath = path_1.default.join('remora/consumers', `${name}.json`);
+        fs_1.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
+        console.log(chalk_1.default.green(`✅ Created consumer config at ${consumerPath}`));
+        if (!producerName) {
+            console.log(chalk_1.default.blue('Remember to:'));
+            console.log(chalk_1.default.blue('1. Set the correct producer name'));
+            console.log(chalk_1.default.blue('2. Configure the fields you want to include'));
+            console.log(chalk_1.default.blue('3. Set up the desired output formats'));
+        }
+        else {
+            console.log(chalk_1.default.blue('Consumer is configured to match all fields from producer:'), chalk_1.default.yellow(producerName));
+            console.log(chalk_1.default.blue('Remember to:'));
+            console.log(chalk_1.default.blue('1. Review the included fields'));
+            console.log(chalk_1.default.blue('2. Set up the desired output formats'));
+        }
+    }
+    catch (error) {
+        console.error(chalk_1.default.red('❌ Error creating consumer:'), error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
+exports.create_consumer = create_consumer;

package/actions/create_producer.js

@@ -0,0 +1,41 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_producer = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const create_producer = (name) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        if (!fs_1.default.existsSync('./remora/producers')) {
+            throw new Error(chalk_1.default.red('Missing directory: ') + chalk_1.default.yellow('./remora/producers'));
+        }
+        // Read the default producer template
+        const defaultProducerTemplate = fs_1.default.readFileSync(path_1.default.join(__dirname, '../documentation/default_resources/producer.json'), 'utf-8');
+        const defaultProducer = JSON.parse(defaultProducerTemplate);
+        defaultProducer.name = name;
+        const producerPath = path_1.default.join('remora/producers', `${name}.json`);
+        fs_1.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
+        console.log(chalk_1.default.green(`✅ Created producer config at ${producerPath}`));
+        console.log(chalk_1.default.blue('Remember to:'));
+        console.log(chalk_1.default.blue('1. Set the correct source name'));
+        console.log(chalk_1.default.blue('2. Configure the dimensions and measures (or use the "discover" command)'));
+        console.log(chalk_1.default.blue('3. Update the settings as needed'));
+    }
+    catch (error) {
+        console.error(chalk_1.default.red('❌ Error creating producer:'), error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
+exports.create_producer = create_producer;

package/actions/run.js

@@ -19,12 +19,14 @@ const Environment_1 = __importDefault(require("../engines/Environment"));
 const UserManager_1 = __importDefault(require("../engines/UserManager"));
 const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEngine"));
 const compile_1 = require("./compile");
-const run = (options, consumersName) => __awaiter(void 0, void 0, void 0, function* () {
+const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
-        const spinner = (0, ora_1.default)(chalk_1.default.blue('Run Consumers...\n')).start();
-        const consumersToExecute = (consumersName && consumersName.length > 0) ? [Environment_1.default.getConsumer(consumersName)] : Environment_1.default._env.consumers;
+        const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
         const user = UserManager_1.default.getUser();
+        const consumersToExecute = (consumerName && consumerName.length > 0)
+            ? [Environment_1.default.getConsumer(consumerName)]
+            : Environment_1.default._env.consumers;
         const results = [];
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];

package/engines/ProducerEngine.js

@@ -101,6 +101,8 @@ class ProducerEngineClass {
                 }
                 case 'JSONL':
                 case 'JSON': {
+                    if (lines.length === 1)
+                        lines = lines[0].split('\n');
                     const json = lines.map(x => JSON.parse(x));
                     return { data: json, dataType: 'array-of-json' };
                 }
@@ -132,7 +134,11 @@ class ProducerEngineClass {
                         sampleData = ParseManager_1.default.csvLinesToJson(fileData.data, producer);
                     }
                     else if (((_b = producer.settings.fileType) === null || _b === void 0 ? void 0 : _b.toUpperCase()) === 'JSON' || ((_c = producer.settings.fileType) === null || _c === void 0 ? void 0 : _c.toUpperCase()) === 'JSONL') {
-                        sampleData = fileData.data.map(line => JSON.parse(line));
+                        // With JSON or JSONL the readFile function already parses the strings
+                        if (typeof fileData.data[0] === 'object')
+                            sampleData = fileData.data;
+                        else
+                            sampleData = fileData.data.map(line => JSON.parse(line));
                     }
                     else {
                         sampleData = fileData.data;

package/engines/consumer/PostProcessor.js

@@ -58,7 +58,7 @@ class PostProcessorClass {
                             const { key, alias } = field.cField;
                             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
                             const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
-                            projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[fieldKey]);
+                            projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[key]);
                         }
                         return projected;
                     });

package/engines/file/FileExporter.js

@@ -99,14 +99,15 @@ class FileExporterClass {
                     const folder = source.authentication.path;
                     if (!fs.existsSync(folder))
                         fs.mkdirSync(folder);
-                    const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(DSTE_1.default.now().toJSON(), ':', '-')}.${extension}`;
+                    const date = DSTE_1.default.now().toISOString().split('.')[0];
+                    const path = `${folder}/${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}`;
                     fs.writeFileSync(path, exportData);
                     return path;
                 }
                 case 'aws-s3': {
                     const driver = yield DriverFactory_1.default.instantiateDestination(source);
                     const date = DSTE_1.default.now().toISOString().split('.')[0];
-                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${date}.${extension}` });
+                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${Algo_1.default.replaceAll(date, ':', '-')}.${extension}` });
                     return res.key;
                 }
             }

package/index.js

@@ -14,6 +14,8 @@ const run_1 = require("./actions/run");
 const Constants_1 = __importDefault(require("./Constants"));
 const discover_1 = require("./actions/discover");
 const automap_1 = require("./actions/automap");
+const create_producer_1 = require("./actions/create_producer");
+const create_consumer_1 = require("./actions/create_consumer");
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
 program
@@ -46,7 +48,7 @@ program
     .option('--build-only', 'Build without deploying')
     .action(deploy_1.deploy);
 program
-    .command('run [consumersName]')
+    .command('run [name]')
     .description('Execute consumers. Optionally specify a single consumers name to run.')
     .action(run_1.run);
 program
@@ -60,4 +62,13 @@ program
     .argument('<producer>', 'The producer to analyze')
     .argument('<schemas...>', 'One or more schema names to map against')
     .action(automap_1.automap);
+program
+    .command('create-producer <name>')
+    .description('Create a new producer configuration with default settings')
+    .action(create_producer_1.create_producer);
+program
+    .command('create-consumer <name>')
+    .description('Create a new consumer configuration with default settings')
+    .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
+    .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.15",
+    "version": "0.0.17",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,