@forzalabs/remora 0.1.8-nasco.3 → 0.1.9

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.8-nasco',
+    cliVersion: '0.1.9',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/actions/automap.js

@@ -0,0 +1,77 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.automap = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const compile_1 = require("./compile");
+const AutoMapperEngine_1 = __importDefault(require("../engines/ai/AutoMapperEngine"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const ProducerEngine_1 = __importDefault(require("../engines/producer/ProducerEngine"));
+/**
+ * e.g. npm run automap -- myclaims Claim
+ */
+const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        (0, compile_1.compile)();
+        const spinner = (0, ora_1.default)(chalk_1.default.blue('Auto-mapping producer data...\n')).start();
+        // Get the producer
+        const producer = Environment_1.default.getProducer(producerName);
+        if (!producer) {
+            throw new Error(`Producer ${producerName} not found`);
+        }
+        const source = Environment_1.default.getSource(producer.source);
+        if (!source) {
+            throw new Error(`Source ${producer.source} not found`);
+        }
+        // Get the specified schemas
+        const schemas = [];
+        for (const schemaName of schemaNames) {
+            const schema = Environment_1.default.getSchema(schemaName);
+            if (!schema) {
+                throw new Error(`Schema ${schemaName} not found`);
+            }
+            schemas.push(schema);
+        }
+        // Read and convert sample data
+        const sampleData = yield ProducerEngine_1.default.readSampleData(producer);
+        // Convert sample data to strings for AutoMapperEngine
+        const sampleStrings = sampleData.map(item => JSON.stringify(item));
+        // Call the automapper
+        const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
+        // Create the producers based on the mapping
+        for (const producer of mapResult.producers) {
+            const producerPath = path_1.default.join('remora/producers', `${producer.name}.json`);
+            fs_1.default.writeFileSync(producerPath, JSON.stringify(producer, null, 4));
+            console.log(chalk_1.default.blue(`Created producer: ${producer.name}`));
+        }
+        // Create the consumers based on the mapping
+        for (const consumer of mapResult.consumers) {
+            const consumerPath = path_1.default.join('remora/consumers', `${consumer.name}.json`);
+            fs_1.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
+            console.log(chalk_1.default.blue(`Created consumer: ${consumer.name}`));
+        }
+        spinner.succeed('Producer has been successfully mapped');
+        console.log(chalk_1.default.green(`\n✅ Created ${mapResult.producers.length} producers!`));
+        console.log(chalk_1.default.green(`✅ Created ${mapResult.consumers.length} consumers!`));
+        process.exit(0);
+    }
+    catch (err) {
+        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during automapping:'), err instanceof Error ? err.message : String(err));
+        process.exit(1);
+    }
+});
+exports.automap = automap;

package/actions/run.js

@@ -16,16 +16,15 @@ exports.run = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const ora_1 = __importDefault(require("ora"));
 const Environment_1 = __importDefault(require("../engines/Environment"));
-const UserManager_1 = __importDefault(require("../engines/UserManager"));
 const ConsumerEngine_1 = __importDefault(require("../engines/consumer/ConsumerEngine"));
 const compile_1 = require("./compile");
 const Helper_1 = __importDefault(require("../helper/Helper"));
+const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
 const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
-        console.log();
+        console.log(); // needed for newline
         const spinner = (0, ora_1.default)(chalk_1.default.blue('Running consumer(s)...\n')).start();
-        const user = UserManager_1.default.getUser();
         const consumersToExecute = [];
         if (consumerName && consumerName.length > 0) {
             const cons = Environment_1.default.getConsumer(consumerName);
@@ -40,7 +39,13 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];
             try {
-                const response = yield ConsumerEngine_1.default.execute(consumer, {}, user);
+                const remoraLicenceKey = process.env.REMORA_LICENCE_KEY;
+                const check = LicenceManager_1.default.validate(remoraLicenceKey);
+                if (!check.valid) {
+                    console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
+                    process.exit(1);
+                }
+                const response = yield ConsumerEngine_1.default.execute(consumer, {}, { _id: check.customer, name: check.customer, type: 'licence' }, { invokedBy: 'CLI' });
                 results.push({ success: true, consumer, response });
             }
             catch (error) {

package/actions/sample.js

@@ -0,0 +1,176 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sample = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const Environment_1 = __importDefault(require("../engines/Environment"));
+const ProducerEngine_1 = __importDefault(require("../engines/producer/ProducerEngine"));
+const DatasetRecord_1 = __importDefault(require("../engines/dataset/DatasetRecord"));
+const compile_1 = require("./compile");
+const Helper_1 = __importDefault(require("../helper/Helper"));
+const sample = (resourceName_1, ...args_1) => __awaiter(void 0, [resourceName_1, ...args_1], void 0, function* (resourceName, sampleSize = 10) {
+    try {
+        (0, compile_1.compile)();
+        console.log(); // needed for newline
+        const spinner = (0, ora_1.default)(chalk_1.default.blue('Sampling dataset...')).start();
+        // Try to find the resource as a producer first, then as a consumer
+        const producer = Environment_1.default.getProducer(resourceName);
+        const consumer = Environment_1.default.getConsumer(resourceName);
+        if (!producer && !consumer) {
+            spinner.fail(chalk_1.default.red(`Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`));
+            process.exit(1);
+        }
+        let sampleData;
+        let resourceType;
+        if (producer) {
+            resourceType = 'Producer';
+            spinner.text = chalk_1.default.blue(`Sampling from producer "${resourceName}"...`);
+            sampleData = yield ProducerEngine_1.default.readSampleData(producer, sampleSize, false);
+        }
+        else {
+            resourceType = 'Consumer';
+            spinner.text = chalk_1.default.blue(`Sampling from consumer "${resourceName}"...`);
+            sampleData = yield sampleFromConsumer(consumer, sampleSize);
+        }
+        spinner.succeed(chalk_1.default.green(`Sample data retrieved from ${resourceType.toLowerCase()} "${resourceName}"`));
+        if (sampleData.length === 0) {
+            console.log(chalk_1.default.yellow('No data found in the dataset.'));
+            return;
+        }
+        // Display the sample data
+        console.log(chalk_1.default.cyan(`\n📊 Sample Data (showing ${sampleData.length} rows):`));
+        console.log(chalk_1.default.gray('─'.repeat(80)));
+        displayDataAsTable(sampleData);
+        console.log(chalk_1.default.gray('─'.repeat(80)));
+        console.log(chalk_1.default.green(`✅ Successfully sampled ${sampleData.length} rows from ${resourceType.toLowerCase()} "${resourceName}"`));
+    }
+    catch (err) {
+        const myErr = Helper_1.default.asError(err);
+        console.error(chalk_1.default.red.bold('\n❌ Error during sampling:'), myErr.message);
+        if (Helper_1.default.isDev())
+            console.log(myErr.stack);
+        process.exit(1);
+    }
+});
+exports.sample = sample;
+const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, void 0, function* () {
+    // For consumers, we need to get sample data from the first producer
+    // and then apply the consumer's field mappings to show what the output would look like
+    const firstProducerRef = consumer.producers[0];
+    if (!firstProducerRef) {
+        throw new Error(`Consumer "${consumer.name}" has no producers configured`);
+    }
+    const producer = Environment_1.default.getProducer(firstProducerRef.name);
+    if (!producer) {
+        const subConsumer = Environment_1.default.getConsumer(firstProducerRef.name);
+        if (!subConsumer) {
+            throw new Error(`Producer or consumer "${firstProducerRef.name}" not found for consumer "${consumer.name}"`);
+        }
+        // If it's a consumer that references another consumer, sample from that consumer
+        return yield sampleFromConsumer(subConsumer, sampleSize);
+    }
+    // Get raw sample data from the producer
+    const rawSampleData = yield ProducerEngine_1.default.readSampleData(producer, sampleSize, false);
+    // For consumers with wildcard fields ("*"), return all data as-is
+    const hasWildcard = consumer.fields.some(field => field.key === '*');
+    if (hasWildcard) {
+        return rawSampleData;
+    }
+    // For consumers with specific field mappings, show only the mapped fields
+    // This gives users a preview of what the consumer output would look like
+    const mappedData = rawSampleData.map(record => {
+        const mappedRecord = new DatasetRecord_1.default('', [], record._delimiter);
+        consumer.fields.forEach(field => {
+            if (field.key !== '*' && !field.grouping) {
+                const sourceValue = record.getValue(field.key);
+                const outputKey = field.alias || field.key;
+                mappedRecord.setValue(outputKey, sourceValue);
+            }
+        });
+        return mappedRecord;
+    });
+    return mappedData;
+});
+const displayDataAsTable = (data) => {
+    if (data.length === 0)
+        return;
+    // Get all unique field names from the sample data
+    const allFields = new Set();
+    data.forEach(record => {
+        Object.keys(record._value).forEach(key => allFields.add(key));
+    });
+    const fields = Array.from(allFields);
+    // Calculate column widths
+    const columnWidths = {};
+    // Start with header widths
+    fields.forEach(field => {
+        columnWidths[field] = field.length;
+    });
+    // Check data widths
+    data.forEach(record => {
+        fields.forEach(field => {
+            const value = record._value[field];
+            const displayValue = formatValue(value);
+            columnWidths[field] = Math.max(columnWidths[field], displayValue.length);
+        });
+    });
+    // Limit column width to prevent overly wide tables
+    const maxColumnWidth = 30;
+    fields.forEach(field => {
+        columnWidths[field] = Math.min(columnWidths[field], maxColumnWidth);
+    });
+    // Print header
+    const headerRow = fields.map(field => chalk_1.default.bold(field.padEnd(columnWidths[field]))).join(' │ ');
+    console.log('│ ' + headerRow + ' │');
+    // Print separator
+    const separator = fields.map(field => '─'.repeat(columnWidths[field])).join('─┼─');
+    console.log('├─' + separator + '─┤');
+    // Print data rows
+    data.forEach((record, index) => {
+        const dataRow = fields.map(field => {
+            const value = record._value[field];
+            const displayValue = formatValue(value);
+            const truncatedValue = displayValue.length > maxColumnWidth
+                ? displayValue.substring(0, maxColumnWidth - 3) + '...'
+                : displayValue;
+            return truncatedValue.padEnd(columnWidths[field]);
+        }).join(' │ ');
+        // Alternate row colors for better readability
+        if (index % 2 === 0) {
+            console.log('│ ' + chalk_1.default.white(dataRow) + ' │');
+        }
+        else {
+            console.log('│ ' + chalk_1.default.gray(dataRow) + ' │');
+        }
+    });
+};
+const formatValue = (value) => {
+    if (value === null || value === undefined) {
+        return chalk_1.default.dim('null');
+    }
+    if (typeof value === 'string') {
+        return value;
+    }
+    if (typeof value === 'number') {
+        return chalk_1.default.cyan(value.toString());
+    }
+    if (typeof value === 'boolean') {
+        return chalk_1.default.yellow(value.toString());
+    }
+    if (value instanceof Date) {
+        return chalk_1.default.magenta(value.toISOString());
+    }
+    return chalk_1.default.dim(JSON.stringify(value));
+};

package/engines/UserManager.js

@@ -24,6 +24,18 @@ class UserManagerClass {
                 return MOCK_USER;
             // TODO: figure out how to handle users
         };
+        this.getRemoraWorkerUser = () => {
+            const remora = {
+                _id: '__remora_worker__',
+                auth: { oid: '', provider: 'internal' },
+                email: '',
+                name: 'Remora Worker',
+                roles: ['root'],
+                _signature: '',
+                lastLogin: new Date().toJSON()
+            };
+            return remora;
+        };
         this.findOIDC = (oid) => __awaiter(this, void 0, void 0, function* () {
             return yield DatabaseEngine_1.default.findOne(Settings_1.default.db.collections.users, { 'auth.oid': oid });
         });

package/engines/ai/LLM.js

@@ -164,26 +164,6 @@ resulting consumer: """
 }
 """
 `;
-const baseQASystemPrompt = `
-# TASK
-You are an agent tasked with ensuring that the CONSUMER(S) created follow the guidelines given.
-You are going to receive a list of CONSUMERS and you need to return in the correct JSON format the same CONSUMERS with the needed updates to ensure that they follow all the rules.
-
-# CONSUMER DEFINITION
-A consumer takes the data from one or more producers and changes it's shape to transform it into the required output schema.
-## FIELDS
-- fields.from: used to distinct between the producers imported by the consumer. The value is the name of the producer.
-
-# RULES
-- If a field is not needed, do not add it e.g.
-- Only import a producer once
-- Awlays include this exact property as the first -> "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
-- Use "API" as the only valid output format.
-- The "from" must contain only the name of the producer
-
-# CONSUMERS
-{{consumers}}
-`;
 class LLM {
     constructor() {
         this.inferProducers = (input, outputs, fileName, sources) => __awaiter(this, void 0, void 0, function* () {
@@ -192,7 +172,7 @@ class LLM {
             systemPrompt = systemPrompt.replace('{{output data spec}}', outputs.map(x => `- ${JSON.stringify(x)}`).join('\n'));
             systemPrompt = systemPrompt.replace('{{file name}}', fileName);
             systemPrompt = systemPrompt.replace('{{sources}}', sources.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            const res = yield this._client.beta.chat.completions.parse({
+            const res = yield this._client.chat.completions.create({
                 model: 'gpt-4o',
                 messages: [
                     { role: 'system', content: systemPrompt }
@@ -219,7 +199,7 @@ class LLM {
                 }), 'environment')
             });
             const msg = res.choices[0].message;
-            return msg.parsed;
+            return JSON.parse(msg.content);
         });
         this.inferConsumers = (producers, outputs) => __awaiter(this, void 0, void 0, function* () {
             let systemPrompt = baseConsumersSystemPrompt;
@@ -261,9 +241,9 @@ class LLM {
                     }))
                 }), 'environment')
             };
-            const res = yield this._client.beta.chat.completions.parse(item);
+            const res = yield this._client.chat.completions.create(item);
             const msg = res.choices[0].message;
-            const finalDraft = msg.parsed;
+            const finalDraft = JSON.parse(msg.content);
             // Do some manual adjustments cause some things still don't work...
             if (finalDraft && finalDraft.consumers) {
                 for (const cons of finalDraft.consumers) {

package/engines/consumer/ConsumerEngine.js

@@ -127,10 +127,10 @@ class ConsumerEngineClass {
                 }
             }
         });
-        this.execute = (consumer, options, user) => __awaiter(this, void 0, void 0, function* () {
+        this.execute = (consumer, options, user, details) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(consumer, `Invalid consumer`);
             (0, Affirm_1.default)(options, `Invalid execute consume options`);
-            const { usageId } = UsageManager_1.default.startUsage(consumer, user);
+            const { usageId } = UsageManager_1.default.startUsage(consumer, user, details);
             try {
                 const execution = new ExecutionEnvironment_1.default(consumer, usageId);
                 const result = yield execution.run(options);

package/engines/dataset/Dataset.js

@@ -730,7 +730,7 @@ class Dataset {
             console.log(`DS ${this.name} (${this._count} | ${this._iterations})`);
             console.log(this._dimensions.map(x => x.name).join(this._delimiter));
             const records = yield this.readLines(count);
-            records.forEach(x => console.log(full ? x : x.stringify()));
+            records.forEach((x, i) => console.log(`[${i}]`, full ? x : x.stringify()));
             console.log('----------');
         });
         this.printStats = () => {

package/engines/dataset/DatasetRecord.js

@@ -4,19 +4,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const CSVParser_1 = __importDefault(require("../parsing/CSVParser"));
 const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
 class DatasetRecord {
     constructor(row, dimensions, delimiter) {
         this.parse = (row, delimiter, dimensions) => {
             if (!this.isEmpty() && dimensions.length > 0) {
-                const parts = row.split(delimiter);
+                const parts = CSVParser_1.default.parseRow(row, delimiter);
                 for (let i = 0; i < dimensions.length; i++) {
                     const dim = dimensions[i];
                     this._value[dim.name] = TypeCaster_1.default.cast(parts[i], dim.type, dim.format);
                 }
             }
         };
-        this.stringify = () => this._dimensions.map(x => this._value[x.name]).join(this._delimiter);
+        this.stringify = () => this._dimensions.map(x => `"${this._value[x.name]}"`).join(this._delimiter);
         this.isEmpty = () => { var _a; return ((_a = this._row) === null || _a === void 0 ? void 0 : _a.trim().length) === 0; };
         this.getRaw = () => this._row;
         this.getValue = (dimension) => this._value[dimension];

package/engines/parsing/CSVParser.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+class CSVParserClass {
+    constructor() {
+        this.parseRow = (row, delimiter) => {
+            (0, Affirm_1.default)(row, 'Invalid row');
+            (0, Affirm_1.default)(delimiter, 'Invalid delimiter');
+            const fields = [];
+            let currentField = '';
+            let inQuotes = false;
+            let i = 0;
+            while (i < row.length) {
+                const char = row[i];
+                const nextChar = row[i + 1];
+                if (char === '"') {
+                    if (!inQuotes) {
+                        // Starting a quoted field
+                        inQuotes = true;
+                    }
+                    else if (nextChar === '"') {
+                        // Escaped quote (double quotes within quoted field)
+                        currentField += '"';
+                        i++; // Skip the next quote
+                    }
+                    else {
+                        // Ending a quoted field
+                        inQuotes = false;
+                    }
+                }
+                else if (char === delimiter && !inQuotes) {
+                    // Field separator found outside quotes
+                    fields.push(currentField.trim());
+                    currentField = '';
+                }
+                else if (char === '\r' || char === '\n') {
+                    // Handle line endings - only break if not in quotes
+                    if (!inQuotes) {
+                        break;
+                    }
+                    currentField += char;
+                }
+                else {
+                    // Regular character
+                    currentField += char;
+                }
+                i++;
+            }
+            // Add the last field
+            fields.push(currentField.trim());
+            return fields;
+        };
+    }
+}
+const CSVParser = new CSVParserClass();
+exports.default = CSVParser;

package/engines/parsing/ParseManager.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+const CSVParser_1 = __importDefault(require("./CSVParser"));
 class ParseManagerClass {
     constructor() {
         this._extractHeader = (headerLine, delimiter, producer, discover) => {
@@ -15,7 +16,7 @@ class ParseManagerClass {
             (0, Affirm_1.default)(producer, 'Invalid producer');
             const source = Environment_1.default.getSource(producer.source);
             let columns = FileCompiler_1.default.compileProducer(producer, source);
-            const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            const headerColumns = CSVParser_1.default.parseRow(headerLine, delimiter).map(x => x.trim());
             // If I'm discovering the file, then it means that the dimensions are not set, so I use the ones that I get from the file directly
             if (discover)
                 columns = headerColumns.map(x => ({ nameInProducer: x }));

package/engines/scheduler/CronScheduler.js

@@ -124,9 +124,8 @@ class CronScheduler {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
-                const user = UserManager_1.default.getUser();
-                // Execute the consumer with default options
-                const result = yield ConsumerEngine_1.default.execute(consumer, {}, user);
+                const user = UserManager_1.default.getRemoraWorkerUser();
+                const result = yield ConsumerEngine_1.default.execute(consumer, {}, { _id: user._id, name: user.name, type: 'actor' }, { invokedBy: 'CRON' });
                 console.log(`CRON job completed successfully for consumer "${consumer.name}" output ${outputIndex}`);
                 // Log execution statistics
                 if (result && result._stats) {

package/engines/scheduler/QueueManager.js

@@ -198,9 +198,8 @@ class QueueManager {
                             }
                         }
                         console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
-                        const user = UserManager_1.default.getUser();
-                        // Execute the consumer with default options
-                        const result = yield ConsumerEngine_1.default.execute(mapping.consumer, {}, user);
+                        const user = UserManager_1.default.getRemoraWorkerUser();
+                        const result = yield ConsumerEngine_1.default.execute(mapping.consumer, {}, { _id: user._id, name: user.name, type: 'actor' }, { invokedBy: 'QUEUE' });
                         console.log(`Queue trigger completed successfully for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
                         // Log execution statistics
                         if (result && result._stats) {

package/engines/transform/TransformationEngine.js

@@ -15,6 +15,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const TypeCaster_1 = __importDefault(require("./TypeCaster"));
+const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
+const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
 class TransformationEngineClass {
     constructor() {
         this.apply = (consumer, dataset, options) => __awaiter(this, void 0, void 0, function* () {
@@ -237,6 +239,9 @@ class TransformationEngineClass {
                     return fieldValues.join(separator);
                 }
             }
+            if ('mask' in transformations) {
+                return this.applyMasking(value, transformations.mask, field);
+            }
             if ('conditional' in transformations) {
                 for (const clause of transformations.conditional.clauses) {
                     if (this.evaluateCondition(value, clause.if)) {
@@ -274,6 +279,19 @@ class TransformationEngineClass {
             }
             return false;
         };
+        this.applyMasking = (value, maskType, field) => {
+            if (!Algo_1.default.hasVal(value))
+                return value;
+            if (maskType === 'none')
+                return value;
+            const valueType = DeveloperEngine_1.default.inferDimensionType(value);
+            try {
+                return CryptoEngine_1.default.hashValue(maskType, String(value), valueType);
+            }
+            catch (error) {
+                throw new Error(`Failed to apply masking transformation '${maskType}' to field '${field.key}': ${error.message}`);
+            }
+        };
         this.applyDimensionsChanges = (transformations, field, dataset) => {
             if (Array.isArray(transformations)) {
                 for (const transform of transformations) {

package/engines/usage/UsageManager.js

@@ -18,14 +18,16 @@ class UsageManagerClass {
             const now = DSTE_1.default.now();
             return `${consumer.name}_${now.getUTCFullYear()}_${now.getUTCMonth()}_${now.getUTCDate()}`.toLowerCase();
         };
-        this.startUsage = (consumer, user) => {
+        this.startUsage = (consumer, user, details) => {
+            var _a;
             const newUsage = {
                 _id: Helper_1.default.uuid(),
                 consumer: consumer.name,
                 startedAt: DSTE_1.default.now(),
-                executedBy: { name: user.name, _id: user._id },
+                executedBy: user,
                 itemsCount: -1,
                 status: 'started',
+                invokedBy: (_a = details === null || details === void 0 ? void 0 : details.invokedBy) !== null && _a !== void 0 ? _a : 'UNKNOWN',
                 _signature: ''
             };
             if (Helper_1.default.isDev())

package/index.js

@@ -17,6 +17,8 @@ const create_consumer_1 = require("./actions/create_consumer");
 const Constants_1 = __importDefault(require("./Constants"));
 const LicenceManager_1 = __importDefault(require("./licencing/LicenceManager"));
 const Runtime_1 = __importDefault(require("./helper/Runtime"));
+const automap_1 = require("./actions/automap");
+const sample_1 = require("./actions/sample");
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
 // Validate the remora licence
@@ -75,9 +77,27 @@ program
     .command('create-producer <name>')
     .description('Create a new producer configuration with default settings')
     .action(create_producer_1.create_producer);
+program
+    .command('automap')
+    .description('Automatically map a producer to consumers using specified schemas.')
+    .argument('<producer>', 'The producer to analyze')
+    .argument('<schemas...>', 'One or more schema names to map against')
+    .action(automap_1.automap);
 program
     .command('create-consumer <name>')
     .description('Create a new consumer configuration with default settings')
     .option('-p, --producer <name>', 'Producer to create a one-to-one mapping from')
     .action((name, options) => (0, create_consumer_1.create_consumer)(name, options.producer));
+program
+    .command('sample <name>')
+    .description('Sample data from a producer or consumer and display it in a formatted table')
+    .option('-s, --size <number>', 'Number of sample rows to display (default: 10)', '10')
+    .action((name, options) => {
+    const sampleSize = parseInt(options.size, 10);
+    if (isNaN(sampleSize) || sampleSize <= 0) {
+        console.error('Sample size must be a positive number');
+        process.exit(1);
+    }
+    (0, sample_1.sample)(name, sampleSize);
+});
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.8-nasco.3",
+    "version": "0.1.9",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -22,7 +22,7 @@
         "copy-static-file": "npx tsx ./scripts/CopyStaticFile.js",
         "build": "npm i && npm run sync && tsc --outDir .build && npm run copy-static-file",
         "fast-build": "tsc --outDir .build",
-        "upload": "npm run build && cd .build && npm publish --tag nasco --access=public"
+        "upload": "npm run build && cd .build && npm publish --access=public"
     },
     "keywords": [
         "nextjs",
@@ -56,6 +56,7 @@
         "mongodb": "^6.15.0",
         "next": "^13.4.1",
         "node-cron": "^4.2.1",
+        "openai": "^6.0.0",
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",