@forzalabs/remora 0.2.6 → 1.0.1

package/engines/file/FileExporter.js

@@ -1,45 +1,13 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Constants_1 = __importDefault(require("../../Constants"));
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const DSTE_1 = __importDefault(require("../../core/dste/DSTE"));
-const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-const Environment_1 = __importDefault(require("../Environment"));
 class FileExporterClass {
     constructor() {
-        this.export = (consumer, output, dataset, executionId) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, `Invalid consumer`);
-            (0, Affirm_1.default)(output, `Invalid output`);
-            (0, Affirm_1.default)(dataset, `Invalid export dataset`);
-            const source = Environment_1.default.getSource(output.exportDestination);
-            (0, Affirm_1.default)(source, `Invalid consumer "${consumer.name}" export location source. Make sure that the export location is an available source.`);
-            const driver = yield DriverFactory_1.default.instantiateDestination(source);
-            const extension = output.format === 'CSV'
-                ? 'csv'
-                : output.format === 'JSON'
-                    ? 'jsonl'
-                    : 'txt';
-            const name = this._composeFileName(consumer, output, extension, executionId);
-            const uploadRes = yield driver.uploadStream({
-                dataset,
-                name,
-                recordProjection: { format: output.format, delimiter: ',' }
-            });
-            return uploadRes.key;
-        });
         this.prepareBatch = (batch, options) => {
             switch (options.recordProjection.format) {
                 case 'JSON': {
@@ -67,6 +35,13 @@ class FileExporterClass {
             }
             return chunks;
         };
+        this._extension = (output) => {
+            return output.format === 'CSV'
+                ? 'csv'
+                : output.format === 'JSON'
+                    ? 'jsonl'
+                    : 'txt';
+        };
         this._composeFileName = (consumer, output, extension, executionId) => {
             if (output.exportName && output.exportName.trim().length > 0) {
                 // Ensure no extension duplication

package/engines/parsing/CSVParser.js

@@ -10,47 +10,48 @@ class CSVParserClass {
             (0, Affirm_1.default)(row, 'Invalid row');
             (0, Affirm_1.default)(delimiter, 'Invalid delimiter');
             const fields = [];
-            let currentField = '';
+            const len = row.length;
+            let fieldStart = 0;
+            let fieldEnd = 0;
             let inQuotes = false;
+            let hasQuotes = false;
             let i = 0;
-            while (i < row.length) {
-                const char = row[i];
-                const nextChar = row[i + 1];
-                if (char === '"') {
+            while (i < len) {
+                const char = row.charCodeAt(i);
+                if (char === 34) { // '"'
                     if (!inQuotes) {
-                        // Starting a quoted field
                         inQuotes = true;
+                        hasQuotes = true;
+                        fieldStart = i + 1;
                     }
-                    else if (nextChar === '"') {
-                        // Escaped quote (double quotes within quoted field)
-                        currentField += '"';
-                        i++; // Skip the next quote
+                    else if (row.charCodeAt(i + 1) === 34) {
+                        i++; // Skip escaped quote, will handle in extraction
                     }
                     else {
-                        // Ending a quoted field
                         inQuotes = false;
+                        fieldEnd = i;
                     }
                 }
-                else if (char === delimiter && !inQuotes) {
-                    // Field separator found outside quotes
-                    fields.push(currentField.trim());
-                    currentField = '';
+                else if (char === delimiter.charCodeAt(0) && !inQuotes) {
+                    // Extract field
+                    const field = hasQuotes
+                        ? row.slice(fieldStart, fieldEnd).replaceAll('""', '"')
+                        : row.slice(fieldStart, i).trim();
+                    fields.push(field);
+                    fieldStart = i + 1;
+                    fieldEnd = 0;
+                    hasQuotes = false;
                 }
-                else if (char === '\r' || char === '\n') {
-                    // Handle line endings - only break if not in quotes
-                    if (!inQuotes) {
-                        break;
-                    }
-                    currentField += char;
-                }
-                else {
-                    // Regular character
-                    currentField += char;
+                else if ((char === 13 || char === 10) && !inQuotes) { // \r or \n
+                    break;
                 }
                 i++;
             }
             // Add the last field
-            fields.push(currentField.trim());
+            const field = hasQuotes
+                ? row.slice(fieldStart, fieldEnd).replaceAll('""', '"')
+                : row.slice(fieldStart, i).trim();
+            fields.push(field);
             return fields;
         };
     }

package/engines/parsing/LineParser.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
+const CSVParser_1 = __importDefault(require("./CSVParser"));
+class LineParserClass {
+    constructor() {
+        this.parse = (line, producer, dimensions, tracker) => {
+            var _a;
+            const { settings: { fileType, delimiter } } = producer;
+            switch (fileType) {
+                case 'PARQUET':
+                case 'TXT':
+                case 'XML':
+                case 'XLS':
+                case 'XLSX':
+                case 'CSV': {
+                    let counter = performance.now();
+                    const parts = CSVParser_1.default.parseRow(line, delimiter !== null && delimiter !== void 0 ? delimiter : ',');
+                    tracker.measure('process-line:parse-csv-row', performance.now() - counter);
+                    counter = performance.now();
+                    const value = {};
+                    for (const dim of dimensions) {
+                        value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.prodDimension.type);
+                    }
+                    tracker.measure('process-line:cast&build-record', performance.now() - counter);
+                    return value;
+                }
+                case 'JSON':
+                case 'JSONL': {
+                    let counter = performance.now();
+                    const parsed = JSON.parse(line);
+                    tracker.measure('process-line:parse-json', performance.now() - counter);
+                    counter = performance.now();
+                    const value = {};
+                    for (const dim of dimensions) {
+                        const key = (_a = dim.prodDimension.alias) !== null && _a !== void 0 ? _a : dim.prodDimension.name;
+                        value[dim.name] = TypeCaster_1.default.cast(parsed[key], dim.prodDimension.type);
+                    }
+                    tracker.measure('process-line:cast&build-record', performance.now() - counter);
+                    return value;
+                }
+                default:
+                    throw new Error(`File type ${fileType} not implemented yet.`);
+            }
+        };
+    }
+}
+const LineParser = new LineParserClass();
+exports.default = LineParser;

package/engines/parsing/XMLParser.js

@@ -47,8 +47,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fast_xml_parser_1 = require("fast-xml-parser");
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
-const fs = __importStar(require("fs"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
+const fs = __importStar(require("fs"));
 const DEFAULT_OPTIONS = {
     attributeNamePrefix: '@_',
     ignoreAttributes: false,

package/engines/producer/ProducerEngine.js

@@ -14,56 +14,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
-const DeploymentPlanner_1 = __importDefault(require("../deployment/DeploymentPlanner"));
 const Environment_1 = __importDefault(require("../Environment"));
-const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
-const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
-const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
 const DatasetManager_1 = __importDefault(require("../dataset/DatasetManager"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 class ProducerEngineClass {
     constructor() {
-        this.compile = (producer) => {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            switch (source.engine) {
-                case 'aws-redshift':
-                case 'postgres': {
-                    const sql = SQLCompiler_1.default.compileProducer(producer, source);
-                    (0, Affirm_1.default)(sql, `Invalid SQL from compilation for producer "${producer.name}"`);
-                    return sql;
-                }
-                case 'aws-s3':
-                case 'delta-share': {
-                    const columns = FileCompiler_1.default.compileProducer(producer, source);
-                    (0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
-                    break;
-                }
-                default: throw new Error(`Invalid engine type "${source.engine}" for producer "${producer.name}": not implemented yet`);
-            }
-        };
-        this.deploy = (producer) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(producer, 'Invalid producer');
-            const source = Environment_1.default.getSource(producer.source);
-            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
-            const driver = yield DriverFactory_1.default.instantiateSource(source);
-            (0, Affirm_1.default)(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
-            const plan = DeploymentPlanner_1.default.planProducer(producer);
-            for (const planStep of plan) {
-                switch (planStep.type) {
-                    case 'create-view': {
-                        const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
-                        (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
-                        const sql = SQLCompiler_1.default.compileProducer(producer, source);
-                        const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.viewName(producer.name)}" AS ${sql}`;
-                        yield driver.execute(vSQL);
-                        break;
-                    }
-                    default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
-                }
-            }
-        });
         this.readFile = (producer, options) => __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
             (0, Affirm_1.default)(producer, 'Invalid producer');

package/engines/scheduler/CronScheduler.js

@@ -47,8 +47,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const cron = __importStar(require("node-cron"));
 const Environment_1 = __importDefault(require("../Environment"));
-const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const UserManager_1 = __importDefault(require("../UserManager"));
+const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
 class CronScheduler {
     constructor() {
         this.scheduledJobs = new Map();
@@ -125,11 +125,19 @@ class CronScheduler {
             try {
                 console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
                 const user = UserManager_1.default.getRemoraWorkerUser();
-                const result = yield ConsumerEngine_1.default.execute(consumer, {}, { _id: user._id, name: user.name, type: 'actor' }, { invokedBy: 'CRON' });
+                const runner = { _id: user._id, name: user.name, type: 'actor' };
+                const result = yield ExecutorOrchestrator_1.default.launch({
+                    consumer,
+                    details: {
+                        invokedBy: 'CRON',
+                        user: runner
+                    },
+                    logProgress: false
+                });
                 console.log(`CRON job completed successfully for consumer "${consumer.name}" output ${outputIndex}`);
                 // Log execution statistics
-                if (result && result._stats) {
-                    console.log(`CRON job stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                if (result) {
+                    console.log(`CRON job stats: ${result.elapsedMS}ms, size: ${result.outputCount}, cycles: ${result.cycles}`);
                 }
             }
             catch (error) {

package/engines/scheduler/QueueManager.js

@@ -14,9 +14,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const Environment_1 = __importDefault(require("../Environment"));
-const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const UserManager_1 = __importDefault(require("../UserManager"));
 const SecretManager_1 = __importDefault(require("../SecretManager"));
+const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
 class QueueManager {
     constructor() {
         this.queueMappings = new Map();
@@ -199,11 +199,18 @@ class QueueManager {
                         }
                         console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
                         const user = UserManager_1.default.getRemoraWorkerUser();
-                        const result = yield ConsumerEngine_1.default.execute(mapping.consumer, {}, { _id: user._id, name: user.name, type: 'actor' }, { invokedBy: 'QUEUE' });
+                        const result = yield ExecutorOrchestrator_1.default.launch({
+                            consumer: mapping.consumer,
+                            details: {
+                                invokedBy: 'QUEUE',
+                                user: { _id: user._id, name: user.name, type: 'actor' }
+                            },
+                            logProgress: false
+                        });
                         console.log(`Queue trigger completed successfully for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
                         // Log execution statistics
-                        if (result && result._stats) {
-                            console.log(`Queue trigger stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                        if (result) {
+                            console.log(`Queue trigger stats: ${result.elapsedMS}ms, size: ${result.outputCount}, cycles: ${result.cycles}`);
                         }
                         messageProcessedByAnyConsumer = true;
                     }

package/engines/sql/SQLCompiler.js

@@ -78,10 +78,10 @@ class SQLCompilerClass {
         };
         this.getConsumerReference = (consumer) => {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            if (consumer.outputs.some(x => x.format === 'SQL' && x.accelerated))
-                return `SELECT * FROM "av_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
-            if (consumer.outputs.some(x => x.format === 'SQL' && !x.direct))
-                return `SELECT * FROM "v_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
+            // if (consumer.outputs.some(x => x.format === 'SQL' && x.accelerated))
+            //     return `SELECT * FROM "av_remora_${SQLUtils.sanitizeName(consumer.name)}"`
+            // if (consumer.outputs.some(x => x.format === 'SQL' && !x.direct))
+            //     return `SELECT * FROM "v_remora_${SQLUtils.sanitizeName(consumer.name)}"`
             return `SELECT * FROM (${this.compileConsumer(consumer)})`;
         };
         this.compileConsumer = (consumer) => {

package/engines/transform/JoinEngine.js

@@ -14,9 +14,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Environment_1 = __importDefault(require("../Environment"));
-const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
 const Dataset_1 = __importDefault(require("../dataset/Dataset"));
 const DatasetRecord_1 = __importDefault(require("../dataset/DatasetRecord"));
+const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
 class JoinEngineClass {
     constructor() {
         this.validateFieldInProducer = (fieldName, producerName) => {
@@ -94,8 +94,8 @@ class JoinEngineClass {
                 return this.findProducerData(consumer.producers[0].name, producedData);
             if (consumer.producers.some(x => x.union))
                 return yield this.union(consumer, producedData);
-            const consumerShape = ConsumerEngine_1.default.getOutputShape(consumer);
-            const consumerColumns = ConsumerEngine_1.default.compile(consumer);
+            const consumerShape = ConsumerManager_1.default.getOutputShape(consumer);
+            const consumerColumns = ConsumerManager_1.default.compile(consumer);
             // Create a new dataset for the joined result
             const resultDataset = new Dataset_1.default({
                 name: `joined_${consumer.name}`,

package/engines/transform/TransformationEngine.js

@@ -1,82 +1,14 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const TypeCaster_1 = __importDefault(require("./TypeCaster"));
 const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
 const DeveloperEngine_1 = __importDefault(require("../ai/DeveloperEngine"));
 class TransformationEngineClass {
     constructor() {
-        this.apply = (consumer, dataset, options) => __awaiter(this, void 0, void 0, function* () {
-            (0, Affirm_1.default)(consumer, 'Invalid consumer');
-            (0, Affirm_1.default)(dataset, 'Invalid data');
-            const fieldsToTransform = consumer.fields.filter(field => Algo_1.default.hasVal(field.transform));
-            Affirm_1.default.hasItems(fieldsToTransform, 'No fields with transformations');
-            yield dataset.map(record => {
-                var _a;
-                for (const field of fieldsToTransform) {
-                    if (!field.transform)
-                        continue;
-                    const fieldKey = (_a = field.alias) !== null && _a !== void 0 ? _a : field.key;
-                    const value = record.getValue(fieldKey);
-                    if (!Algo_1.default.hasVal(value) && Algo_1.default.hasVal(field.default))
-                        record.setValue(fieldKey, field.default);
-                    else if (!Algo_1.default.hasVal(value) && this.isFieldCombinationTransformation(field.transform))
-                        // For field combination transformations, we don't skip null values as they might combine with other fields
-                        continue;
-                    else if (!Algo_1.default.hasVal(value))
-                        continue;
-                    try {
-                        record.setValue(fieldKey, this.applyTransformations(value, field.transform, field, record));
-                    }
-                    catch (error) {
-                        switch (field.onError) {
-                            case 'set_default':
-                                record.setValue(fieldKey, field.default);
-                                break;
-                            case 'skip':
-                                break;
-                            case 'fail':
-                            default:
-                                if (field.transform)
-                                    throw new Error(`an error occured during the transformation ${field.transform} on filedKey: ${fieldKey} the error is the following: ${error}`);
-                                else
-                                    throw new Error(`an error occured at filedKey: ${fieldKey} the error is the following: ${error}`);
-                        }
-                    }
-                }
-                return record;
-            }, options);
-            /**
-             * Some transformations (for now only "cast") change the underlying type of the dataset dimension
-             * Here I update the dimension type of the dataset.
-             * TODO: I think that we may have a bug if you cast AND then do an operation on the number, since it reverts back to being a string in the same trnasformation chain, since the dimension type update is applied only at the end of all the transformations
-             */
-            for (const field of fieldsToTransform) {
-                if (!field.transform)
-                    continue;
-                this.applyDimensionsChanges(field.transform, field, dataset);
-            }
-            return dataset;
-        });
-        this.isFieldCombinationTransformation = (transformation) => {
-            if (Array.isArray(transformation)) {
-                return transformation.some(t => this.isFieldCombinationTransformation(t));
-            }
-            return 'combine_fields' in transformation;
-        };
         this.applyTransformations = (value, transformations, field, record) => {
             var _a;
             if (Array.isArray(transformations)) {
@@ -109,7 +41,7 @@ class TransformationEngineClass {
                 }
                 const { fields } = transformations.multiplyBy;
                 const fieldValues = fields.map(fieldName => {
-                    const fieldValue = record.getValue(fieldName);
+                    const fieldValue = record[fieldName];
                     return fieldValue !== null && fieldValue !== undefined ? TypeCaster_1.default.cast(fieldValue, 'number') : 1;
                 });
                 const product = fieldValues.reduce((accumulator, value) => accumulator * value, 1);
@@ -127,7 +59,7 @@ class TransformationEngineClass {
                 }
                 const { fields } = transformations.addBy;
                 const fieldValues = fields.map(fieldName => {
-                    const fieldValue = record.getValue(fieldName);
+                    const fieldValue = record[fieldName];
                     return fieldValue !== null && fieldValue !== undefined ? TypeCaster_1.default.cast(fieldValue, 'number') : 1;
                 });
                 const sum = fieldValues.reduce((accumulator, value) => accumulator + value);
@@ -249,7 +181,7 @@ class TransformationEngineClass {
                 const { fields, separator = '', template } = transformations.combine_fields;
                 // Get values from the specified fields
                 const fieldValues = fields.map(fieldName => {
-                    const fieldValue = record.getValue(fieldName);
+                    const fieldValue = record[fieldName];
                     return fieldValue !== null && fieldValue !== undefined ? String(fieldValue) : '';
                 });
                 // If template is provided, use it for formatting
@@ -337,24 +269,6 @@ class TransformationEngineClass {
                 throw new Error(`Failed to apply masking transformation '${maskType}' to field '${field.key}': ${error.message}`);
             }
         };
-        this.applyDimensionsChanges = (transformations, field, dataset) => {
-            if (Array.isArray(transformations)) {
-                for (const transform of transformations) {
-                    this.applyDimensionsChanges(transform, field, dataset);
-                }
-                return dataset;
-            }
-            // Single transformation
-            if ('cast' in transformations) {
-                const { cast, format } = transformations;
-                let oldDimension = dataset.getDimensions().find(x => x.name === field.key);
-                if (!oldDimension)
-                    oldDimension = dataset.getDimensions().find(x => x.key === field.key);
-                const newDimension = Object.assign(Object.assign({}, structuredClone(oldDimension)), { type: cast, format: format });
-                dataset.setSingleDimension(newDimension, oldDimension);
-            }
-            return dataset;
-        };
     }
 }
 const TransformationEngine = new TransformationEngineClass();

package/engines/usage/UsageManager.js

@@ -18,8 +18,8 @@ class UsageManagerClass {
             const now = DSTE_1.default.now();
             return `${consumer.name}_${now.getUTCFullYear()}_${now.getUTCMonth()}_${now.getUTCDate()}`.toLowerCase();
         };
-        this.startUsage = (consumer, user, details) => {
-            var _a;
+        this.startUsage = (consumer, details) => {
+            const { user, invokedBy } = details;
             const newUsage = {
                 _id: Helper_1.default.uuid(),
                 consumer: consumer.name,
@@ -27,19 +27,21 @@ class UsageManagerClass {
                 executedBy: user,
                 itemsCount: -1,
                 status: 'started',
-                invokedBy: (_a = details === null || details === void 0 ? void 0 : details.invokedBy) !== null && _a !== void 0 ? _a : 'UNKNOWN',
-                _signature: ''
+                invokedBy: invokedBy !== null && invokedBy !== void 0 ? invokedBy : 'UNKNOWN',
+                _signature: '',
+                stats: null
             };
             if (Helper_1.default.isDev())
                 return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
             const updateRes = DatabaseEngine_1.default.upsert(Settings_1.default.db.collections.usage, newUsage._id, newUsage);
             return { usageId: newUsage._id, usage: updateRes };
         };
-        this.endUsage = (usageId, itemsCount) => {
+        this.endUsage = (usageId, itemsCount, stats) => {
             const update = {
                 itemsCount: itemsCount,
                 status: 'success',
-                finishedAt: DSTE_1.default.now()
+                finishedAt: DSTE_1.default.now(),
+                stats
             };
             if (Helper_1.default.isDev())
                 return { usageId: null, usage: Promise.resolve(update) };

package/engines/validation/Validator.js

@@ -7,7 +7,6 @@ const Affirm_1 = __importDefault(require("../../core/Affirm"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
 const Environment_1 = __importDefault(require("../Environment"));
-const ExecutionPlanner_1 = __importDefault(require("../execution/ExecutionPlanner"));
 class ValidatorClass {
     constructor() {
         this.validateSources = (sources) => {
@@ -97,8 +96,6 @@ class ValidatorClass {
                 const allFieldsWithNoFrom = consumer.fields.filter(x => x.key === '*' && !x.from);
                 if (allFieldsWithNoFrom.length > 0 && consumer.producers.length > 1)
                     errors.push(`Field with key "*" was used without specifying the "from" producer and multiple producers were found.`);
-                if (consumer.fields.some(x => x.key === '*' && x.grouping))
-                    errors.push(`Field with key "*" can't be used for "grouping". Either remove the grouping or change the key.`);
                 // Validation on producers
                 if (consumer.producers.length === 0)
                     errors.push(`Consumer must have at least 1 producer.`);
@@ -131,13 +128,13 @@ class ValidatorClass {
                     if (consumer.filters.some(x => x.sql && x.rule))
                         errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
                     const [source] = ConsumerManager_1.default.getSource(consumer);
-                    const engineClass = ExecutionPlanner_1.default.getEngineClass(source.engine);
+                    const engineClass = this.getEngineClass(source.engine);
                     if (engineClass === 'file' && consumer.filters.some(x => x.sql))
                         errors.push(`Filters based on SQL are only valid for SQL based sources. (source: ${source.name})`);
                     if (engineClass === 'sql' && consumer.filters.some(x => x.rule))
                         errors.push(`Filters based on rules are only valid for non-SQL based sources. (source: ${source.name})`);
                     // Check that the members of the rules are present in the consumer
-                    const allFields = ConsumerManager_1.default.getConsumerFlatFields(consumer);
+                    const allFields = consumer.fields;
                     const ruleBasedFilters = consumer.filters.filter(x => x.rule);
                     const rulesWithMatchingFields = ruleBasedFilters.map(x => ({
                         match: allFields.find(k => { var _a; return ((_a = k.alias) !== null && _a !== void 0 ? _a : k.key) === x.rule.member; }),
@@ -148,18 +145,6 @@ class ValidatorClass {
                         errors.push(`Filter(s) on member(s) "${missingRules.map(x => x.rule.rule.member).join(', ')}" is invalid since the member specified is not present in the consumer. Check the member value or add the missing field to the consumer.`);
                     }
                 }
-                // Validation on fields
-                const validateGroupingLevels = (fields, level = 0) => {
-                    let errors = [];
-                    const groupingFields = fields.filter(x => x.grouping);
-                    if (groupingFields.length > 1)
-                        errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
-                    groupingFields.forEach(field => {
-                        if (field.grouping)
-                            errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
-                    });
-                    return errors;
-                };
                 const validateTransformations = (fields) => {
                     var _a;
                     const errors = [];
@@ -185,7 +170,6 @@ class ValidatorClass {
                     }
                     return errors;
                 };
-                errors.push(...validateGroupingLevels(consumer.fields));
                 errors.push(...validateTransformations(consumer.fields));
                 // Validation outputs
                 const duplicatesOutputs = Algo_1.default.duplicatesObject(consumer.outputs, 'format');
@@ -211,6 +195,16 @@ class ValidatorClass {
             }
             return errors;
         };
+        this.getEngineClass = (engine) => {
+            switch (engine) {
+                case 'aws-dynamodb': return 'no-sql';
+                case 'aws-redshift':
+                case 'postgres': return 'sql';
+                case 'delta-share':
+                case 'aws-s3': return 'file';
+                case 'local': return 'local';
+            }
+        };
     }
 }
 const Validator = new ValidatorClass();