@forzalabs/remora 0.1.5-nasco.3 → 0.1.7-nasco.3

package/engines/execution/ExecutionPlanner.js

@@ -13,6 +13,7 @@ class ExecutionPlannerClas {
             switch (engine) {
                 case 'aws-dynamodb': return 'no-sql';
                 case 'aws-redshift':
+                case 'delta-share':
                 case 'postgres': return 'sql';
                 case 'aws-s3': return 'file';
                 case 'local': return 'local';
@@ -65,6 +66,8 @@ class ExecutionPlannerClas {
                     default:
                         throw new Error(`Output format "${output.format}" not supported`);
                 }
+                if (output.onSuccess && output.onSuccess.length > 0)
+                    plan.push({ type: 'perform-on-success-actions', output });
             }
             plan.push({ type: 'clean-datasets' });
             plan.push({ type: 'save-execution-stats' });

package/engines/producer/ProducerEngine.js

@@ -95,6 +95,7 @@ class ProducerEngineClass {
                 case 'XLS':
                 case 'XLSX':
                     return { data: lines, dataset, dataType: 'lines-of-text' };
+                case 'PARQUET':
                 case 'JSONL':
                 case 'JSON': {
                     if (lines.length === 1) {

package/engines/scheduler/CronScheduler.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cron = __importStar(require("node-cron"));
+const Environment_1 = __importDefault(require("../Environment"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const UserManager_1 = __importDefault(require("../UserManager"));
+class CronScheduler {
+    constructor() {
+        this.scheduledJobs = new Map();
+        this.isInitialized = false;
+    }
+    /**
+     * Initialize the CRON scheduler by scanning all consumers and scheduling those with CRON triggers
+     */
+    initialize() {
+        if (this.isInitialized) {
+            console.log('CRON scheduler already initialized');
+            return;
+        }
+        console.log('Initializing CRON scheduler...');
+        try {
+            const consumers = Environment_1.default.getAllConsumers();
+            let cronJobCount = 0;
+            for (const consumer of consumers) {
+                if (this.hasCronTrigger(consumer)) {
+                    this.scheduleConsumer(consumer);
+                    cronJobCount++;
+                }
+            }
+            this.isInitialized = true;
+            console.log(`CRON scheduler initialized with ${cronJobCount} scheduled jobs`);
+        }
+        catch (error) {
+            console.error('Failed to initialize CRON scheduler:', error);
+            throw error;
+        }
+    }
+    /**
+     * Check if a consumer has any CRON triggers configured
+     */
+    hasCronTrigger(consumer) {
+        return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value; });
+    }
+    /**
+     * Schedule a consumer with CRON triggers
+     */
+    scheduleConsumer(consumer) {
+        consumer.outputs.forEach((output, index) => {
+            var _a;
+            if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value) {
+                const jobKey = `${consumer.name}_output_${index}`;
+                const cronExpression = output.trigger.value;
+                try {
+                    // Validate CRON expression
+                    if (!cron.validate(cronExpression)) {
+                        console.error(`Invalid CRON expression for consumer ${consumer.name}: ${cronExpression}`);
+                        return;
+                    }
+                    // Schedule the job
+                    const task = cron.schedule(cronExpression, () => __awaiter(this, void 0, void 0, function* () {
+                        yield this.executeConsumerOutput(consumer, output, index);
+                    }));
+                    // Don't start the task immediately, we'll start it manually
+                    task.stop();
+                    this.scheduledJobs.set(jobKey, task);
+                    task.start();
+                    console.log(`Scheduled CRON job for consumer "${consumer.name}" output ${index} with expression: ${cronExpression}`);
+                }
+                catch (error) {
+                    console.error(`Failed to schedule CRON job for consumer ${consumer.name}:`, error);
+                }
+            }
+        });
+    }
+    /**
+     * Execute a consumer output when triggered by CRON
+     */
+    executeConsumerOutput(consumer, output, outputIndex) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
+                const user = UserManager_1.default.getUser();
+                // Execute the consumer with default options
+                const result = yield ConsumerEngine_1.default.execute(consumer, {}, user);
+                console.log(`CRON job completed successfully for consumer "${consumer.name}" output ${outputIndex}`);
+                // Log execution statistics
+                if (result && result._stats) {
+                    console.log(`CRON job stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                }
+            }
+            catch (error) {
+                console.error(`CRON job failed for consumer "${consumer.name}" output ${outputIndex}:`, error);
+                // Optionally, you could implement error handling strategies here:
+                // - Send notifications
+                // - Log to a monitoring system
+                // - Retry logic
+                // - Disable the job after repeated failures
+            }
+        });
+    }
+    /**
+     * Add or update a CRON job for a specific consumer
+     */
+    updateConsumerSchedule(consumer) {
+        // First, remove any existing schedules for this consumer
+        this.removeConsumerSchedule(consumer.name);
+        // Then, add new schedules if they have CRON triggers
+        if (this.hasCronTrigger(consumer)) {
+            this.scheduleConsumer(consumer);
+        }
+    }
+    /**
+     * Remove all scheduled jobs for a consumer
+     */
+    removeConsumerSchedule(consumerName) {
+        const jobsToRemove = Array.from(this.scheduledJobs.keys()).filter(key => key.startsWith(`${consumerName}_output_`));
+        jobsToRemove.forEach(jobKey => {
+            const task = this.scheduledJobs.get(jobKey);
+            if (task) {
+                task.stop();
+                task.destroy();
+                this.scheduledJobs.delete(jobKey);
+                console.log(`Removed CRON job: ${jobKey}`);
+            }
+        });
+    }
+    /**
+     * Get information about all scheduled jobs
+     */
+    getScheduledJobs() {
+        return Array.from(this.scheduledJobs.entries()).map(([jobKey, task]) => ({
+            jobKey,
+            isRunning: task.getStatus() === 'scheduled'
+        }));
+    }
+    /**
+     * Stop all scheduled jobs
+     */
+    stopAllJobs() {
+        console.log('Stopping all CRON jobs...');
+        this.scheduledJobs.forEach((task, jobKey) => {
+            task.stop();
+            task.destroy();
+            console.log(`Stopped CRON job: ${jobKey}`);
+        });
+        this.scheduledJobs.clear();
+        this.isInitialized = false;
+        console.log('All CRON jobs stopped');
+    }
+    /**
+     * Restart the scheduler (useful for configuration reloads)
+     */
+    restart() {
+        console.log('Restarting CRON scheduler...');
+        this.stopAllJobs();
+        this.initialize();
+    }
+    /**
+     * Get the scheduler status
+     */
+    getStatus() {
+        return {
+            initialized: this.isInitialized,
+            jobCount: this.scheduledJobs.size,
+            jobs: this.getScheduledJobs()
+        };
+    }
+}
+// Export a singleton instance
+exports.default = new CronScheduler();

package/engines/scheduler/QueueManager.js

@@ -0,0 +1,307 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const Environment_1 = __importDefault(require("../Environment"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const UserManager_1 = __importDefault(require("../UserManager"));
+const SecretManager_1 = __importDefault(require("../SecretManager"));
+class QueueManager {
+    constructor() {
+        this.queueMappings = new Map();
+        this.pollingIntervals = new Map();
+        this.isInitialized = false;
+        this.POLLING_INTERVAL_MS = 5000; // Poll every 5 seconds
+        this.MAX_MESSAGES = 10; // Maximum messages to receive in one poll
+        // Initialize SQS client with default configuration
+        // Will be reconfigured when we know the specific queue details
+        this.sqsClient = new client_sqs_1.SQSClient({});
+    }
+    /**
+     * Initialize the Queue Manager by scanning all consumers and setting up queue listeners for those with QUEUE triggers
+     */
+    initialize() {
+        if (this.isInitialized) {
+            console.log('Queue Manager already initialized');
+            return;
+        }
+        console.log('Initializing Queue Manager...');
+        try {
+            const consumers = Environment_1.default.getAllConsumers();
+            let queueTriggerCount = 0;
+            for (const consumer of consumers) {
+                if (this.hasQueueTrigger(consumer)) {
+                    this.setupQueueListeners(consumer);
+                    queueTriggerCount++;
+                }
+            }
+            this.isInitialized = true;
+            console.log(`Queue Manager initialized with ${queueTriggerCount} queue triggers`);
+        }
+        catch (error) {
+            console.error('Failed to initialize Queue Manager:', error);
+            throw error;
+        }
+    }
+    /**
+     * Check if a consumer has any QUEUE triggers configured
+     */
+    hasQueueTrigger(consumer) {
+        return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value; });
+    }
+    /**
+     * Setup queue listeners for a consumer with QUEUE triggers
+     */
+    setupQueueListeners(consumer) {
+        consumer.outputs.forEach((output, index) => {
+            var _a;
+            if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value) {
+                try {
+                    const queueConfig = this.parseQueueConfig(output.trigger.value, output.trigger.metadata);
+                    const mapping = {
+                        consumer,
+                        outputIndex: index,
+                        queueUrl: queueConfig.queueUrl,
+                        messageType: queueConfig.messageType
+                    };
+                    // Add to mappings
+                    if (!this.queueMappings.has(queueConfig.queueUrl)) {
+                        this.queueMappings.set(queueConfig.queueUrl, []);
+                    }
+                    this.queueMappings.get(queueConfig.queueUrl).push(mapping);
+                    // Start polling for this queue if not already started
+                    if (!this.pollingIntervals.has(queueConfig.queueUrl)) {
+                        this.startQueuePolling(queueConfig.queueUrl, queueConfig.region, queueConfig.credentials);
+                    }
+                    console.log(`Setup queue listener for consumer "${consumer.name}" output ${index} on queue: ${queueConfig.queueUrl}`);
+                }
+                catch (error) {
+                    console.error(`Failed to setup queue listener for consumer ${consumer.name}:`, error);
+                }
+            }
+        });
+    }
+    /**
+     * Parse queue configuration from trigger value and metadata
+     */
+    parseQueueConfig(triggerValue, metadata) {
+        // triggerValue should be the queue URL or queue name
+        let queueUrl = triggerValue;
+        // If it's not a full URL, construct it
+        if (!queueUrl.startsWith('https://')) {
+            const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || process.env.AWS_DEFAULT_REGION || 'us-east-1';
+            const accountId = (metadata === null || metadata === void 0 ? void 0 : metadata.accountId) || process.env.AWS_ACCOUNT_ID;
+            if (!accountId) {
+                throw new Error('AWS Account ID is required for queue trigger. Set it in metadata.accountId or AWS_ACCOUNT_ID environment variable');
+            }
+            queueUrl = `https://sqs.${region}.amazonaws.com/${accountId}/${triggerValue}`;
+        }
+        // Extract region from URL if not provided in metadata
+        const urlParts = queueUrl.match(/https:\/\/sqs\.([^.]+)\.amazonaws\.com\//);
+        const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || (urlParts === null || urlParts === void 0 ? void 0 : urlParts[1]) || 'us-east-1';
+        // Get credentials from metadata or environment
+        let credentials;
+        const accessKeyId = (metadata === null || metadata === void 0 ? void 0 : metadata.accessKeyId) || process.env.AWS_ACCESS_KEY_ID;
+        const secretAccessKey = (metadata === null || metadata === void 0 ? void 0 : metadata.secretAccessKey) || process.env.AWS_SECRET_ACCESS_KEY;
+        const sessionToken = (metadata === null || metadata === void 0 ? void 0 : metadata.sessionToken) || process.env.AWS_SESSION_TOKEN;
+        if (accessKeyId && secretAccessKey) {
+            credentials = {
+                accessKeyId: SecretManager_1.default.replaceSecret(accessKeyId),
+                secretAccessKey: SecretManager_1.default.replaceSecret(secretAccessKey),
+                sessionToken: sessionToken ? SecretManager_1.default.replaceSecret(sessionToken) : undefined
+            };
+        }
+        return {
+            queueUrl,
+            messageType: metadata === null || metadata === void 0 ? void 0 : metadata.messageType,
+            region,
+            credentials
+        };
+    }
+    /**
+     * Start polling a specific queue for messages
+     */
+    startQueuePolling(queueUrl, region, credentials) {
+        // Create SQS client for this specific queue
+        const sqsClient = new client_sqs_1.SQSClient({
+            region,
+            credentials
+        });
+        const pollQueue = () => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const command = new client_sqs_1.ReceiveMessageCommand({
+                    QueueUrl: queueUrl,
+                    MaxNumberOfMessages: this.MAX_MESSAGES,
+                    WaitTimeSeconds: 20, // Long polling
+                    VisibilityTimeout: 300 // 5 minutes to process the message
+                });
+                const response = yield sqsClient.send(command);
+                if (response.Messages && response.Messages.length > 0) {
+                    console.log(`Received ${response.Messages.length} messages from queue: ${queueUrl}`);
+                    for (const message of response.Messages) {
+                        yield this.processMessage(queueUrl, message, sqsClient);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(`Error polling queue ${queueUrl}:`, error);
+            }
+        });
+        // Start continuous polling
+        const interval = setInterval(pollQueue, this.POLLING_INTERVAL_MS);
+        this.pollingIntervals.set(queueUrl, interval);
+        // Start immediately
+        pollQueue();
+        console.log(`Started polling queue: ${queueUrl}`);
+    }
+    /**
+     * Process a message from the queue
+     */
+    processMessage(queueUrl, message, sqsClient) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const mappings = this.queueMappings.get(queueUrl);
+                if (!mappings || mappings.length === 0) {
+                    console.log(`No consumer mappings found for queue: ${queueUrl}`);
+                    return;
+                }
+                // Parse message body
+                let messageData;
+                try {
+                    messageData = JSON.parse(message.Body || '{}');
+                }
+                catch (_a) {
+                    console.warn(`Failed to parse message body as JSON for queue ${queueUrl}. Using raw body.`);
+                    messageData = { body: message.Body };
+                }
+                let messageProcessedByAnyConsumer = false;
+                // Process message for each mapped consumer that matches the message criteria
+                for (const mapping of mappings) {
+                    try {
+                        // Check if message type matches (if specified)
+                        if (mapping.messageType) {
+                            const messageType = messageData.type || messageData.messageType || messageData.eventType;
+                            if (messageType !== mapping.messageType) {
+                                console.log(`Message type ${messageType} does not match expected ${mapping.messageType} for consumer ${mapping.consumer.name} - skipping`);
+                                continue;
+                            }
+                        }
+                        console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
+                        const user = UserManager_1.default.getUser();
+                        // Execute the consumer with default options
+                        const result = yield ConsumerEngine_1.default.execute(mapping.consumer, {}, user);
+                        console.log(`Queue trigger completed successfully for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
+                        // Log execution statistics
+                        if (result && result._stats) {
+                            console.log(`Queue trigger stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                        }
+                        messageProcessedByAnyConsumer = true;
+                    }
+                    catch (error) {
+                        console.error(`Queue trigger failed for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}:`, error);
+                        // Continue processing for other consumers even if one fails
+                    }
+                }
+                // Only delete message from queue if it was processed by at least one consumer
+                // This ensures messages intended for other consumers or systems remain in the queue
+                if (messageProcessedByAnyConsumer && message.ReceiptHandle) {
+                    yield sqsClient.send(new client_sqs_1.DeleteMessageCommand({
+                        QueueUrl: queueUrl,
+                        ReceiptHandle: message.ReceiptHandle
+                    }));
+                    console.log(`Deleted processed message ${message.MessageId} from queue`);
+                }
+                else if (!messageProcessedByAnyConsumer) {
+                    console.log(`Message ${message.MessageId} was not processed by any consumer - leaving in queue for other consumers or systems`);
+                }
+            }
+            catch (error) {
+                console.error(`Error processing message from queue ${queueUrl}:`, error);
+                // Message will remain in queue and be retried or go to DLQ based on queue configuration
+            }
+        });
+    }
+    /**
+     * Add or update queue listeners for a specific consumer
+     */
+    updateConsumerSchedule(consumer) {
+        // First, remove any existing listeners for this consumer
+        this.removeConsumerSchedule(consumer.name);
+        // Then, add new listeners if they have QUEUE triggers
+        if (this.hasQueueTrigger(consumer)) {
+            this.setupQueueListeners(consumer);
+        }
+    }
+    /**
+     * Remove all queue listeners for a consumer
+     */
+    removeConsumerSchedule(consumerName) {
+        // Remove mappings for this consumer
+        for (const [queueUrl, mappings] of this.queueMappings.entries()) {
+            const updatedMappings = mappings.filter(mapping => mapping.consumer.name !== consumerName);
+            if (updatedMappings.length === 0) {
+                // No more consumers listening to this queue, stop polling
+                const interval = this.pollingIntervals.get(queueUrl);
+                if (interval) {
+                    clearInterval(interval);
+                    this.pollingIntervals.delete(queueUrl);
+                    console.log(`Stopped polling queue: ${queueUrl}`);
+                }
+                this.queueMappings.delete(queueUrl);
+            }
+            else {
+                this.queueMappings.set(queueUrl, updatedMappings);
+            }
+        }
+    }
+    /**
+     * Stop all queue polling
+     */
+    stopAllQueues() {
+        console.log('Stopping all queue polling...');
+        this.pollingIntervals.forEach((interval, queueUrl) => {
+            clearInterval(interval);
+            console.log(`Stopped polling queue: ${queueUrl}`);
+        });
+        this.pollingIntervals.clear();
+        this.queueMappings.clear();
+        this.isInitialized = false;
+        console.log('All queue polling stopped');
+    }
+    /**
+     * Restart the queue manager (useful for configuration reloads)
+     */
+    restart() {
+        console.log('Restarting Queue Manager...');
+        this.stopAllQueues();
+        this.initialize();
+    }
+    /**
+     * Get the queue manager status
+     */
+    getStatus() {
+        const queues = Array.from(this.queueMappings.entries()).map(([queueUrl, mappings]) => ({
+            queueUrl,
+            consumerCount: mappings.length
+        }));
+        return {
+            initialized: this.isInitialized,
+            queueCount: this.queueMappings.size,
+            consumerCount: Array.from(this.queueMappings.values()).flat().length,
+            queues
+        };
+    }
+}
+// Export a singleton instance
+exports.default = new QueueManager();

package/engines/transform/TransformationEngine.js

@@ -283,11 +283,11 @@ class TransformationEngineClass {
             }
             // Single transformation
             if ('cast' in transformations) {
-                const { cast } = transformations;
+                const { cast, format } = transformations;
                 let oldDimension = dataset.getDimensions().find(x => x.name === field.key);
                 if (!oldDimension)
                     oldDimension = dataset.getDimensions().find(x => x.key === field.key);
-                const newDimension = Object.assign(Object.assign({}, structuredClone(oldDimension)), { type: cast });
+                const newDimension = Object.assign(Object.assign({}, structuredClone(oldDimension)), { type: cast, format: format });
                 dataset.setSingleDimension(newDimension, oldDimension);
             }
             return dataset;

package/engines/transform/TypeCaster.js

@@ -28,11 +28,19 @@ class TypeCasterClass {
             case 'datetime':
             case 'date': {
                 let dateValue = null;
-                if (format && typeof value === 'string')
-                    dateValue = dayjs_1.default.utc(value, format, true).toDate();
-                else
+                try {
+                    if (format && typeof value === 'string')
+                        dateValue = dayjs_1.default.utc(value, format, true).toDate();
+                    else
+                        dateValue = new Date(value);
+                    return dateValue.toISOString();
+                }
+                catch (error) {
                     dateValue = new Date(value);
-                return dateValue.toISOString();
+                    if (!isNaN(dateValue))
+                        return dateValue.toISOString();
+                    throw new Error(`Error casting "${value}" to date with format "${format}": ${error}`);
+                }
             }
             case 'number': {
                 if (typeof value === 'number')

package/engines/usage/UsageDataManager.js

@@ -106,5 +106,46 @@ class UsageDataManager {
             };
         });
     }
+    getFilteredUsageDetails(from, to, filters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const now = DSTE_1.default.now();
+            const fromDate = from || new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+            const toDate = to || now;
+            const collection = 'usage';
+            // Build match criteria
+            const matchCriteria = {
+                startedAt: { $gte: fromDate, $lte: toDate }
+            };
+            if (filters === null || filters === void 0 ? void 0 : filters.consumer) {
+                matchCriteria.consumer = filters.consumer;
+            }
+            if (filters === null || filters === void 0 ? void 0 : filters.user) {
+                matchCriteria['executedBy.name'] = filters.user;
+            }
+            if (filters === null || filters === void 0 ? void 0 : filters.status) {
+                matchCriteria.status = filters.status;
+            }
+            // Query with filters and sorting
+            const usageDetails = yield DatabaseEngine_1.default.query(collection, matchCriteria, {
+                sort: { startedAt: -1 },
+                limit: 1000 // Reasonable limit to prevent large result sets
+            });
+            return usageDetails;
+        });
+    }
+    getUsageById(id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const collection = 'usage';
+            try {
+                const usageDetail = yield DatabaseEngine_1.default.get(collection, id);
+                return usageDetail;
+            }
+            catch (error) {
+                // If document not found or invalid ID, return null
+                console.error('Error fetching usage by ID:', error);
+                return null;
+            }
+        });
+    }
 }
 exports.default = new UsageDataManager();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.5-nasco.3",
+    "version": "0.1.7-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -35,6 +35,7 @@
     "dependencies": {
         "@aws-sdk/client-redshift-data": "^3.699.0",
         "@aws-sdk/client-s3": "^3.701.0",
+        "@aws-sdk/client-sqs": "^3.886.0",
         "adm-zip": "^0.5.16",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
@@ -54,6 +55,7 @@
         "knex": "^2.4.2",
         "mongodb": "^6.15.0",
         "next": "^13.4.1",
+        "node-cron": "^4.2.1",
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",