@forzalabs/remora 0.1.6-nasco.3 → 0.1.7-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.6-nasco',
+    cliVersion: '0.1.7-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/database/DatabaseEngine.js

@@ -20,23 +20,28 @@ class DatabaseEngineClass {
         this.MAX_TRY_CONNECTION = 3;
         this.db = () => this._db;
         this.connect = () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // WARNING: this was changed during the deployment to ECS... I've reverted, but maybe it needs to be changed or looked into...
+            this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
+                ? 'mongodb://mongo:27017/remora'
+                : 'mongodb://localhost:27017/remora';
+            this._client = new mongodb_1.MongoClient(this._uri);
             const errors = [];
             for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
-                if (i < this.MAX_TRY_CONNECTION) {
-                    try {
-                        yield this._client.connect();
-                        this._db = this._client.db(Settings_1.default.db.name);
-                        console.log('Connected to MongoDB');
-                        break;
-                    }
-                    catch (error) {
-                        errors.push(i + '° connection to MongoDB throws this error:', error);
-                    }
+                try {
+                    console.log(`Attempting to connect to mongo: "${this._uri}"`);
+                    yield this._client.connect();
+                    this._db = this._client.db(Settings_1.default.db.name);
+                    this._connected = true;
+                    console.log('Connected to MongoDB');
+                    break;
                 }
-                else {
-                    console.error(`Despite several ${i} retries it was not possible to connect to mongoDb, these are the errors encountered:\n` + errors.join('\n'));
+                catch (error) {
+                    errors.push((i + 1) + ': connection to MongoDB throws this error:', error);
                 }
             }
+            if (!this._connected)
+                console.error(`Despite ${this.MAX_TRY_CONNECTION} retries it was not possible to connect to mongoDb, these are the errors encountered:\n` + errors.join('\n'));
         });
         this.disconnect = () => __awaiter(this, void 0, void 0, function* () {
             try {
@@ -121,8 +126,6 @@ class DatabaseEngineClass {
                 throw error;
             }
         });
-        const uri = Helper_1.default.isDev() ? 'mongodb://mongo:27017/remora' : 'mongodb://localhost:27017/remora';
-        this._client = new mongodb_1.MongoClient(uri);
     }
 }
 const DatabaseEngine = new DatabaseEngineClass();

package/definitions/json_schemas/consumer-schema.json

@@ -235,13 +235,47 @@
                                 "type": "string",
                                 "enum": [
                                     "CRON",
-                                    "API"
+                                    "API",
+                                    "QUEUE"
                                 ],
-                                "description": "The type of trigger schedule"
+                                "description": "The type of trigger schedule. CRON: time-based scheduling. API: HTTP endpoint trigger. QUEUE: SQS queue message trigger (supports shared queues with message type filtering)."
                             },
                             "value": {
                                 "type": "string",
-                                "description": "The value for the trigger (e.g., cron expression or API endpoint)"
+                                "description": "The value for the trigger. For CRON: cron expression (e.g., '0 0 * * *'). For API: endpoint path. For QUEUE: SQS queue URL or queue name (will construct full URL using metadata.region and metadata.accountId if needed)."
+                            },
+                            "metadata": {
+                                "type": "object",
+                                "description": "Additional metadata for the trigger (e.g., AWS credentials, message type filter)",
+                                "properties": {
+                                    "messageType": {
+                                        "type": "string",
+                                        "description": "Optional message type filter for QUEUE triggers. Only messages with matching 'type', 'messageType', or 'eventType' fields will be processed by this consumer. Messages without a matching type will be left in the queue for other consumers, enabling shared queue usage."
+                                    },
+                                    "region": {
+                                        "type": "string",
+                                        "description": "AWS region for the queue (for QUEUE triggers)"
+                                    },
+                                    "accountId": {
+                                        "type": "string",
+                                        "description": "AWS account ID for constructing queue URL (for QUEUE triggers)"
+                                    },
+                                    "accessKeyId": {
+                                        "type": "string",
+                                        "description": "AWS access key ID for queue authentication (for QUEUE triggers)"
+                                    },
+                                    "secretAccessKey": {
+                                        "type": "string",
+                                        "description": "AWS secret access key for queue authentication (for QUEUE triggers)"
+                                    },
+                                    "sessionToken": {
+                                        "type": "string",
+                                        "description": "AWS session token for temporary credentials (for QUEUE triggers)"
+                                    }
+                                },
+                                "additionalProperties": {
+                                    "type": "string"
+                                }
                             }
                         },
                         "required": [
@@ -249,6 +283,20 @@
                             "value"
                         ],
                         "additionalProperties": false
+                    },
+                    "onSuccess": {
+                        "type": "array",
+                        "description": "Actions to perform when the output operation completes successfully",
+                        "items": {
+                            "$ref": "#/definitions/consumerOutputOnFinish"
+                        }
+                    },
+                    "onError": {
+                        "type": "array",
+                        "description": "Actions to perform when the output operation fails",
+                        "items": {
+                            "$ref": "#/definitions/consumerOutputOnFinish"
+                        }
                     }
                 },
                 "required": [
@@ -792,6 +840,23 @@
                     "additionalProperties": false
                 }
             ]
+        },
+        "consumerOutputOnFinish": {
+            "type": "object",
+            "description": "Actions to perform when output operations complete",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": ["move-file"],
+                    "description": "The action to perform"
+                },
+                "moveToDestination": {
+                    "type": "string",
+                    "description": "If the action is 'move-file', this specifies the source destination in remora where the source file should be moved"
+                }
+            },
+            "required": ["action"],
+            "additionalProperties": false
         }
     },
     "examples": [

package/drivers/DeltaShareDriver.js

@@ -13,6 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 /**
  * Delta Share (Databricks Delta Sharing) Source Driver
@@ -31,7 +32,7 @@ class DeltaShareSourceDriver {
             const { authentication } = source;
             (0, Affirm_1.default)(authentication, 'Invalid authentication for delta-share source');
             this._shareUrl = authentication.host;
-            this._bearerToken = authentication.bearerToken || authentication.sessionToken || authentication.password;
+            this._bearerToken = SecretManager_1.default.replaceSecret(authentication.bearerToken || authentication.sessionToken || authentication.password);
             this._share = authentication.share;
             this._schema = authentication.schema;
             this._table = authentication.table;
@@ -58,7 +59,8 @@ class DeltaShareSourceDriver {
             (0, Affirm_1.default)(request, `Invalid download request`);
             (0, Affirm_1.default)(!request.fileKey.includes('%'), `On a delta-share the file key can not include "%"`);
             const deltaFiles = yield this._getAllFilesInTables(this._table);
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             const lines = [];
             for (const deltaFile of deltaFiles) {
                 const byteLength = (_b = (_a = deltaFile.file.deltaSingleAction.add) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : (_c = deltaFile.file.deltaSingleAction.remove) === null || _c === void 0 ? void 0 : _c.size;
@@ -75,7 +77,8 @@ class DeltaShareSourceDriver {
             (0, Affirm_1.default)(request.options.lineFrom !== undefined && request.options.lineTo !== undefined, 'Missing read range');
             const deltaFiles = yield this._getAllFilesInTables(this._table);
             const { options: { lineFrom, lineTo } } = request;
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             const lines = [];
             let index = 0;
             for (const deltaFile of deltaFiles) {
@@ -96,7 +99,8 @@ class DeltaShareSourceDriver {
             var _a, _b, _c;
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const deltaFiles = yield this._getAllFilesInTables(this._table);
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             // For each file, download it with the hyparquet package, read lines, then save locally to create the dataset
             let index = 0;
             let totalLineCount = 0;
@@ -142,7 +146,7 @@ class DeltaShareSourceDriver {
                     Authorization: `Bearer ${this._bearerToken}`
                 }
             });
-            (0, Affirm_1.default)(res.ok, `Error fetching version from the delta share: ${res.status} ${res.statusText}`);
+            (0, Affirm_1.default)(res.ok, `Error fetching version from the delta share: ${res.status} ${res.statusText} (${yield res.text()})`);
             const version = res.headers['delta-table-version'];
             return version;
         });

package/drivers/LocalDriver.js

@@ -320,6 +320,36 @@ class LocalSourceDriver {
                 throw new Error(`Failed to list files in directory "${this._path}": ${error.message}`);
             }
         };
+        this.readFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            (0, Affirm_1.default)(fs.existsSync(filePath), `Source file does not exist: ${filePath}`);
+            return fs.readFileSync(filePath);
+        };
+        this.deleteFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            if (fs.existsSync(filePath)) {
+                fs.unlinkSync(filePath);
+            }
+        };
+        this.moveFile = (sourceFileKey, destinationPath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationPath, 'Invalid destination path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const sourceFilePath = path_1.default.join(this._path, sourceFileKey);
+            const destinationFilePath = path_1.default.join(destinationPath, destinationFileKey);
+            (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
+            // Ensure destination directory exists
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.renameSync(sourceFilePath, destinationFilePath);
+        };
     }
 }
 exports.LocalSourceDriver = LocalSourceDriver;
@@ -385,6 +415,35 @@ class LocalDestinationDriver {
                 throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
             }
         });
+        this.saveFile = (fileKey, content) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            const filePath = path_1.default.join(this._path, fileKey);
+            const directory = path_1.default.dirname(filePath);
+            // Create directory if it doesn't exist
+            if (!fs.existsSync(directory)) {
+                fs.mkdirSync(directory, { recursive: true });
+            }
+            fs.writeFileSync(filePath, content);
+            return Promise.resolve();
+        };
+        this.copyFromLocal = (sourceFilePath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFilePath, 'Invalid source file path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const destinationFilePath = path_1.default.join(this._path, destinationFileKey);
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            // Ensure destination directory exists
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.copyFileSync(sourceFilePath, destinationFilePath);
+        };
+        this.copyFromS3 = (s3Driver, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = yield s3Driver.downloadFile(sourceFileKey);
+            yield this.saveFile(destinationFileKey, fileContent);
+        });
     }
 }
 exports.LocalDestinationDriver = LocalDestinationDriver;

package/drivers/S3Driver.js

@@ -134,6 +134,27 @@ class S3DestinationDriver {
                 throw error;
             }
         });
+        this.copyFromS3 = (sourceBucket, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(sourceBucket, 'Invalid source bucket');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            yield this._client.send(new client_s3_1.CopyObjectCommand({
+                CopySource: `${sourceBucket}/${sourceFileKey}`,
+                Bucket: this._bucketName,
+                Key: destinationFileKey
+            }));
+        });
+        this.saveFile = (fileKey, content) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            yield this._client.send(new client_s3_1.PutObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey,
+                Body: content
+            }));
+        });
     }
 }
 exports.S3DestinationDriver = S3DestinationDriver;
@@ -491,6 +512,36 @@ class S3SourceDriver {
             } while (continuationToken);
             return allFiles;
         });
+        this.downloadFile = (fileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const content = yield response.Body.transformToByteArray();
+            return Buffer.from(content);
+        });
+        this.deleteFile = (fileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            yield this._client.send(new client_s3_1.DeleteObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey
+            }));
+        });
+        this.copyFile = (sourceFileKey, destinationBucket, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationBucket, 'Invalid destination bucket');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            yield this._client.send(new client_s3_1.CopyObjectCommand({
+                CopySource: `${this._bucketName}/${sourceFileKey}`,
+                Bucket: destinationBucket,
+                Key: destinationFileKey
+            }));
+        });
     }
 }
 exports.S3SourceDriver = S3SourceDriver;

package/engines/Environment.js

@@ -128,6 +128,10 @@ class EnvironmentClass {
             (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
             return this._env.consumers.find(x => x.name.toLowerCase() === consumerName.toLowerCase());
         };
+        this.getAllConsumers = () => {
+            (0, Affirm_1.default)(this._env, 'Environment not initialized');
+            return this._env.consumers || [];
+        };
         this.getSchema = (schemaName) => {
             (0, Affirm_1.default)(schemaName, 'Invalid schema name');
             return this._env.schemas.find(x => x.title === schemaName);

package/engines/consumer/ConsumerOnFinishManager.js

@@ -0,0 +1,188 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
+class ConsumerOnFinishManagerClass {
+    constructor() {
+        this.performOnSuccessActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(output, 'Invalid output');
+            if (!output.onSuccess || output.onSuccess.length === 0)
+                return;
+            for (const onSuccess of output.onSuccess) {
+                switch (onSuccess.action) {
+                    case 'move-file': {
+                        yield this.moveSourceFiles(consumer, onSuccess.moveToDestination);
+                        break;
+                    }
+                    default:
+                        throw new Error(`On success action "${onSuccess.action}" is not implemented yet.`);
+                }
+            }
+        });
+        this.moveSourceFiles = (consumer, moveDestination) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(moveDestination, 'Move destination is required for move-file action');
+            const destinationSource = Environment_1.default.getSource(moveDestination);
+            (0, Affirm_1.default)(destinationSource, `Destination source "${moveDestination}" not found`);
+            // Get all unique source files from all producers used by this consumer
+            const sourceFilesToMove = [];
+            for (const consumerProducer of consumer.producers) {
+                const producer = Environment_1.default.getProducer(consumerProducer.name);
+                (0, Affirm_1.default)(producer, `Producer "${consumerProducer.name}" not found`);
+                const source = Environment_1.default.getSource(producer.source);
+                (0, Affirm_1.default)(source, `Source "${producer.source}" not found`);
+                // Only handle file-based sources that have fileKey
+                if (producer.settings.fileKey) {
+                    // Handle wildcard patterns (files with %)
+                    if (producer.settings.fileKey.includes('%')) {
+                        // Get all files matching the pattern
+                        const matchingFiles = yield this.getMatchingFiles(source, producer.settings.fileKey);
+                        for (const fileKey of matchingFiles) {
+                            sourceFilesToMove.push({
+                                sourceName: producer.source,
+                                filePath: this.getSourceFilePath(source, fileKey),
+                                fileKey
+                            });
+                        }
+                    }
+                    else {
+                        // Single file
+                        sourceFilesToMove.push({
+                            sourceName: producer.source,
+                            filePath: this.getSourceFilePath(source, producer.settings.fileKey),
+                            fileKey: producer.settings.fileKey
+                        });
+                    }
+                }
+            }
+            // Move all files to destination
+            yield this.moveFiles(sourceFilesToMove, destinationSource);
+        });
+        this.getMatchingFiles = (source, fileKeyPattern) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(source);
+            if (source.engine === 'aws-s3') {
+                // S3 driver has a public listFiles method that handles patterns
+                const s3Driver = sourceDriver;
+                return yield s3Driver.listFiles(fileKeyPattern);
+            }
+            else if (source.engine === 'local') {
+                // Local driver now has a public listFiles method consistent with S3
+                const localDriver = sourceDriver;
+                return localDriver.listFiles(fileKeyPattern);
+            }
+            throw new Error(`Unsupported source engine for file listing: ${source.engine}`);
+        });
+        this.getSourceFilePath = (source, fileKey) => {
+            if (source.engine === 'local') {
+                return `${source.authentication['path']}/${fileKey}`;
+            }
+            else if (source.engine === 'aws-s3') {
+                // For S3, we return the key as the path since S3 uses keys instead of file paths
+                return fileKey;
+            }
+            throw new Error(`Unsupported source engine for file move: ${source.engine}`);
+        };
+        this.moveFiles = (files, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            for (const file of files) {
+                const sourceSource = Environment_1.default.getSource(file.sourceName);
+                if (sourceSource.engine === 'local' && destinationSource.engine === 'local') {
+                    // Local to Local move
+                    yield this.moveLocalToLocal(file.filePath, destinationSource, file.fileKey);
+                }
+                else if (sourceSource.engine === 'local' && destinationSource.engine === 'aws-s3') {
+                    // Local to S3 move
+                    yield this.moveLocalToS3(file.filePath, destinationSource, file.fileKey);
+                }
+                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'local') {
+                    // S3 to Local move
+                    yield this.moveS3ToLocal(sourceSource, file.fileKey, destinationSource);
+                }
+                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'aws-s3') {
+                    // S3 to S3 move
+                    yield this.moveS3ToS3(sourceSource, file.fileKey, destinationSource);
+                }
+                else {
+                    throw new Error(`Unsupported move operation from ${sourceSource.engine} to ${destinationSource.engine}`);
+                }
+            }
+        });
+        this.moveLocalToLocal = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Read file from source
+            const fileContent = sourceDriver.readFile(fileKey);
+            // Save to destination
+            yield destinationDriver.saveFile(fileKey, fileContent);
+            // Delete from source
+            sourceDriver.deleteFile(fileKey);
+        });
+        this.findSourceForPath = (filePath) => {
+            // Extract directory from file path for source creation
+            const directory = filePath.substring(0, filePath.lastIndexOf('/'));
+            return {
+                name: 'temp-source',
+                engine: 'local',
+                authentication: { path: directory }
+            };
+        };
+        this.moveLocalToS3 = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Read file from local source
+            const fileContent = sourceDriver.readFile(fileKey);
+            // Upload to S3 destination
+            yield destinationDriver.saveFile(fileKey, fileContent);
+            // Remove source file after successful upload
+            sourceDriver.deleteFile(fileKey);
+        });
+        this.moveS3ToLocal = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Download from S3
+            const content = yield sourceDriver.downloadFile(fileKey);
+            // Save to local destination
+            yield destinationDriver.saveFile(fileKey, content);
+            // Delete from S3 source
+            yield sourceDriver.deleteFile(fileKey);
+        });
+        this.moveS3ToS3 = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Copy from source S3 to destination S3
+            yield destinationDriver.copyFromS3(sourceSource.authentication['bucket'], fileKey, fileKey);
+            // Delete from source S3
+            yield sourceDriver.deleteFile(fileKey);
+        });
+        this.performOnErrorActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(output, 'Invalid output');
+            if (!output.onError || output.onError.length === 0)
+                return;
+            for (const onError of output.onError) {
+                switch (onError.action) {
+                    case 'move-file': {
+                        yield this.moveSourceFiles(consumer, onError.moveToDestination);
+                        break;
+                    }
+                    default:
+                        throw new Error(`On success action "${onError.action}" is not implemented yet.`);
+                }
+            }
+        });
+    }
+}
+const ConsumerOnFinishManager = new ConsumerOnFinishManagerClass();
+exports.default = ConsumerOnFinishManager;

package/engines/dataset/ParallelDataset.js

@@ -43,6 +43,9 @@ class ParallelDatasetClass {
             const currentDir = __dirname;
             if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
                 return path_1.default.resolve('./.build/workers');
+            const forcedPath = process.env.REMORA_WORKERS_PATH;
+            if (forcedPath && forcedPath.length > 0)
+                return path_1.default.join(__dirname, forcedPath);
             // Check if we're in a published npm package (no .build in path)
             if (!currentDir.includes('.build')) {
                 // We're in the published package, workers are relative to package root

package/engines/execution/ExecutionEnvironment.js

@@ -27,6 +27,7 @@ const Environment_1 = __importDefault(require("../Environment"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 const ParallelDataset_1 = __importDefault(require("../dataset/ParallelDataset"));
+const ConsumerOnFinishManager_1 = __importDefault(require("../consumer/ConsumerOnFinishManager"));
 class ExecutionEnvironment {
     constructor(consumer, executionId) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
@@ -127,6 +128,10 @@ class ExecutionEnvironment {
                             this._resultingDataset = yield PostProcessor_1.default.distinct(this._resultingDataset);
                             break;
                         }
+                        case 'perform-on-success-actions': {
+                            yield ConsumerOnFinishManager_1.default.performOnSuccessActions(this._consumer, planStep.output);
+                            break;
+                        }
                         case 'save-execution-stats': {
                             (0, Affirm_1.default)(this._resultingDataset, `Invalid result dataset in save-execution-stats`);
                             result._stats = {
@@ -156,6 +161,12 @@ class ExecutionEnvironment {
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getCount()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
                 Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
+                try {
+                    yield ConsumerOnFinishManager_1.default.performOnErrorActions(this._consumer, currentStep.output);
+                }
+                catch (error) {
+                    Logger_1.default.log(`Error when trying to perform onError actions on failed consumer ${error}`);
+                }
                 // IMPORTANT: cleanup all the datasets to not leave any data around and to avoid memory leaks
                 const datasets = [
                     ...this._producedData.map(x => x.dataset),

package/engines/execution/ExecutionPlanner.js

@@ -13,6 +13,7 @@ class ExecutionPlannerClas {
             switch (engine) {
                 case 'aws-dynamodb': return 'no-sql';
                 case 'aws-redshift':
+                case 'delta-share':
                 case 'postgres': return 'sql';
                 case 'aws-s3': return 'file';
                 case 'local': return 'local';
@@ -65,6 +66,8 @@ class ExecutionPlannerClas {
                     default:
                         throw new Error(`Output format "${output.format}" not supported`);
                 }
+                if (output.onSuccess && output.onSuccess.length > 0)
+                    plan.push({ type: 'perform-on-success-actions', output });
             }
             plan.push({ type: 'clean-datasets' });
             plan.push({ type: 'save-execution-stats' });

package/engines/producer/ProducerEngine.js

@@ -95,6 +95,7 @@ class ProducerEngineClass {
                 case 'XLS':
                 case 'XLSX':
                     return { data: lines, dataset, dataType: 'lines-of-text' };
+                case 'PARQUET':
                 case 'JSONL':
                 case 'JSON': {
                     if (lines.length === 1) {

package/engines/scheduler/CronScheduler.js

@@ -0,0 +1,215 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cron = __importStar(require("node-cron"));
+const Environment_1 = __importDefault(require("../Environment"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const UserManager_1 = __importDefault(require("../UserManager"));
+class CronScheduler {
+    constructor() {
+        this.scheduledJobs = new Map();
+        this.isInitialized = false;
+    }
+    /**
+     * Initialize the CRON scheduler by scanning all consumers and scheduling those with CRON triggers
+     */
+    initialize() {
+        if (this.isInitialized) {
+            console.log('CRON scheduler already initialized');
+            return;
+        }
+        console.log('Initializing CRON scheduler...');
+        try {
+            const consumers = Environment_1.default.getAllConsumers();
+            let cronJobCount = 0;
+            for (const consumer of consumers) {
+                if (this.hasCronTrigger(consumer)) {
+                    this.scheduleConsumer(consumer);
+                    cronJobCount++;
+                }
+            }
+            this.isInitialized = true;
+            console.log(`CRON scheduler initialized with ${cronJobCount} scheduled jobs`);
+        }
+        catch (error) {
+            console.error('Failed to initialize CRON scheduler:', error);
+            throw error;
+        }
+    }
+    /**
+     * Check if a consumer has any CRON triggers configured
+     */
+    hasCronTrigger(consumer) {
+        return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value; });
+    }
+    /**
+     * Schedule a consumer with CRON triggers
+     */
+    scheduleConsumer(consumer) {
+        consumer.outputs.forEach((output, index) => {
+            var _a;
+            if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'CRON' && output.trigger.value) {
+                const jobKey = `${consumer.name}_output_${index}`;
+                const cronExpression = output.trigger.value;
+                try {
+                    // Validate CRON expression
+                    if (!cron.validate(cronExpression)) {
+                        console.error(`Invalid CRON expression for consumer ${consumer.name}: ${cronExpression}`);
+                        return;
+                    }
+                    // Schedule the job
+                    const task = cron.schedule(cronExpression, () => __awaiter(this, void 0, void 0, function* () {
+                        yield this.executeConsumerOutput(consumer, output, index);
+                    }));
+                    // Don't start the task immediately, we'll start it manually
+                    task.stop();
+                    this.scheduledJobs.set(jobKey, task);
+                    task.start();
+                    console.log(`Scheduled CRON job for consumer "${consumer.name}" output ${index} with expression: ${cronExpression}`);
+                }
+                catch (error) {
+                    console.error(`Failed to schedule CRON job for consumer ${consumer.name}:`, error);
+                }
+            }
+        });
+    }
+    /**
+     * Execute a consumer output when triggered by CRON
+     */
+    executeConsumerOutput(consumer, output, outputIndex) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                console.log(`Executing CRON job for consumer "${consumer.name}" output ${outputIndex}`);
+                const user = UserManager_1.default.getUser();
+                // Execute the consumer with default options
+                const result = yield ConsumerEngine_1.default.execute(consumer, {}, user);
+                console.log(`CRON job completed successfully for consumer "${consumer.name}" output ${outputIndex}`);
+                // Log execution statistics
+                if (result && result._stats) {
+                    console.log(`CRON job stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                }
+            }
+            catch (error) {
+                console.error(`CRON job failed for consumer "${consumer.name}" output ${outputIndex}:`, error);
+                // Optionally, you could implement error handling strategies here:
+                // - Send notifications
+                // - Log to a monitoring system
+                // - Retry logic
+                // - Disable the job after repeated failures
+            }
+        });
+    }
+    /**
+     * Add or update a CRON job for a specific consumer
+     */
+    updateConsumerSchedule(consumer) {
+        // First, remove any existing schedules for this consumer
+        this.removeConsumerSchedule(consumer.name);
+        // Then, add new schedules if they have CRON triggers
+        if (this.hasCronTrigger(consumer)) {
+            this.scheduleConsumer(consumer);
+        }
+    }
+    /**
+     * Remove all scheduled jobs for a consumer
+     */
+    removeConsumerSchedule(consumerName) {
+        const jobsToRemove = Array.from(this.scheduledJobs.keys()).filter(key => key.startsWith(`${consumerName}_output_`));
+        jobsToRemove.forEach(jobKey => {
+            const task = this.scheduledJobs.get(jobKey);
+            if (task) {
+                task.stop();
+                task.destroy();
+                this.scheduledJobs.delete(jobKey);
+                console.log(`Removed CRON job: ${jobKey}`);
+            }
+        });
+    }
+    /**
+     * Get information about all scheduled jobs
+     */
+    getScheduledJobs() {
+        return Array.from(this.scheduledJobs.entries()).map(([jobKey, task]) => ({
+            jobKey,
+            isRunning: task.getStatus() === 'scheduled'
+        }));
+    }
+    /**
+     * Stop all scheduled jobs
+     */
+    stopAllJobs() {
+        console.log('Stopping all CRON jobs...');
+        this.scheduledJobs.forEach((task, jobKey) => {
+            task.stop();
+            task.destroy();
+            console.log(`Stopped CRON job: ${jobKey}`);
+        });
+        this.scheduledJobs.clear();
+        this.isInitialized = false;
+        console.log('All CRON jobs stopped');
+    }
+    /**
+     * Restart the scheduler (useful for configuration reloads)
+     */
+    restart() {
+        console.log('Restarting CRON scheduler...');
+        this.stopAllJobs();
+        this.initialize();
+    }
+    /**
+     * Get the scheduler status
+     */
+    getStatus() {
+        return {
+            initialized: this.isInitialized,
+            jobCount: this.scheduledJobs.size,
+            jobs: this.getScheduledJobs()
+        };
+    }
+}
+// Export a singleton instance
+exports.default = new CronScheduler();

package/engines/scheduler/QueueManager.js

@@ -0,0 +1,307 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const client_sqs_1 = require("@aws-sdk/client-sqs");
+const Environment_1 = __importDefault(require("../Environment"));
+const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
+const UserManager_1 = __importDefault(require("../UserManager"));
+const SecretManager_1 = __importDefault(require("../SecretManager"));
+class QueueManager {
+    constructor() {
+        this.queueMappings = new Map();
+        this.pollingIntervals = new Map();
+        this.isInitialized = false;
+        this.POLLING_INTERVAL_MS = 5000; // Poll every 5 seconds
+        this.MAX_MESSAGES = 10; // Maximum messages to receive in one poll
+        // Initialize SQS client with default configuration
+        // Will be reconfigured when we know the specific queue details
+        this.sqsClient = new client_sqs_1.SQSClient({});
+    }
+    /**
+     * Initialize the Queue Manager by scanning all consumers and setting up queue listeners for those with QUEUE triggers
+     */
+    initialize() {
+        if (this.isInitialized) {
+            console.log('Queue Manager already initialized');
+            return;
+        }
+        console.log('Initializing Queue Manager...');
+        try {
+            const consumers = Environment_1.default.getAllConsumers();
+            let queueTriggerCount = 0;
+            for (const consumer of consumers) {
+                if (this.hasQueueTrigger(consumer)) {
+                    this.setupQueueListeners(consumer);
+                    queueTriggerCount++;
+                }
+            }
+            this.isInitialized = true;
+            console.log(`Queue Manager initialized with ${queueTriggerCount} queue triggers`);
+        }
+        catch (error) {
+            console.error('Failed to initialize Queue Manager:', error);
+            throw error;
+        }
+    }
+    /**
+     * Check if a consumer has any QUEUE triggers configured
+     */
+    hasQueueTrigger(consumer) {
+        return consumer.outputs.some(output => { var _a; return ((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value; });
+    }
+    /**
+     * Setup queue listeners for a consumer with QUEUE triggers
+     */
+    setupQueueListeners(consumer) {
+        consumer.outputs.forEach((output, index) => {
+            var _a;
+            if (((_a = output.trigger) === null || _a === void 0 ? void 0 : _a.type) === 'QUEUE' && output.trigger.value) {
+                try {
+                    const queueConfig = this.parseQueueConfig(output.trigger.value, output.trigger.metadata);
+                    const mapping = {
+                        consumer,
+                        outputIndex: index,
+                        queueUrl: queueConfig.queueUrl,
+                        messageType: queueConfig.messageType
+                    };
+                    // Add to mappings
+                    if (!this.queueMappings.has(queueConfig.queueUrl)) {
+                        this.queueMappings.set(queueConfig.queueUrl, []);
+                    }
+                    this.queueMappings.get(queueConfig.queueUrl).push(mapping);
+                    // Start polling for this queue if not already started
+                    if (!this.pollingIntervals.has(queueConfig.queueUrl)) {
+                        this.startQueuePolling(queueConfig.queueUrl, queueConfig.region, queueConfig.credentials);
+                    }
+                    console.log(`Setup queue listener for consumer "${consumer.name}" output ${index} on queue: ${queueConfig.queueUrl}`);
+                }
+                catch (error) {
+                    console.error(`Failed to setup queue listener for consumer ${consumer.name}:`, error);
+                }
+            }
+        });
+    }
+    /**
+     * Parse queue configuration from trigger value and metadata
+     */
+    parseQueueConfig(triggerValue, metadata) {
+        // triggerValue should be the queue URL or queue name
+        let queueUrl = triggerValue;
+        // If it's not a full URL, construct it
+        if (!queueUrl.startsWith('https://')) {
+            const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || process.env.AWS_DEFAULT_REGION || 'us-east-1';
+            const accountId = (metadata === null || metadata === void 0 ? void 0 : metadata.accountId) || process.env.AWS_ACCOUNT_ID;
+            if (!accountId) {
+                throw new Error('AWS Account ID is required for queue trigger. Set it in metadata.accountId or AWS_ACCOUNT_ID environment variable');
+            }
+            queueUrl = `https://sqs.${region}.amazonaws.com/${accountId}/${triggerValue}`;
+        }
+        // Extract region from URL if not provided in metadata
+        const urlParts = queueUrl.match(/https:\/\/sqs\.([^.]+)\.amazonaws\.com\//);
+        const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || (urlParts === null || urlParts === void 0 ? void 0 : urlParts[1]) || 'us-east-1';
+        // Get credentials from metadata or environment
+        let credentials;
+        const accessKeyId = (metadata === null || metadata === void 0 ? void 0 : metadata.accessKeyId) || process.env.AWS_ACCESS_KEY_ID;
+        const secretAccessKey = (metadata === null || metadata === void 0 ? void 0 : metadata.secretAccessKey) || process.env.AWS_SECRET_ACCESS_KEY;
+        const sessionToken = (metadata === null || metadata === void 0 ? void 0 : metadata.sessionToken) || process.env.AWS_SESSION_TOKEN;
+        if (accessKeyId && secretAccessKey) {
+            credentials = {
+                accessKeyId: SecretManager_1.default.replaceSecret(accessKeyId),
+                secretAccessKey: SecretManager_1.default.replaceSecret(secretAccessKey),
+                sessionToken: sessionToken ? SecretManager_1.default.replaceSecret(sessionToken) : undefined
+            };
+        }
+        return {
+            queueUrl,
+            messageType: metadata === null || metadata === void 0 ? void 0 : metadata.messageType,
+            region,
+            credentials
+        };
+    }
+    /**
+     * Start polling a specific queue for messages
+     */
+    startQueuePolling(queueUrl, region, credentials) {
+        // Create SQS client for this specific queue
+        const sqsClient = new client_sqs_1.SQSClient({
+            region,
+            credentials
+        });
+        const pollQueue = () => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const command = new client_sqs_1.ReceiveMessageCommand({
+                    QueueUrl: queueUrl,
+                    MaxNumberOfMessages: this.MAX_MESSAGES,
+                    WaitTimeSeconds: 20, // Long polling
+                    VisibilityTimeout: 300 // 5 minutes to process the message
+                });
+                const response = yield sqsClient.send(command);
+                if (response.Messages && response.Messages.length > 0) {
+                    console.log(`Received ${response.Messages.length} messages from queue: ${queueUrl}`);
+                    for (const message of response.Messages) {
+                        yield this.processMessage(queueUrl, message, sqsClient);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(`Error polling queue ${queueUrl}:`, error);
+            }
+        });
+        // Start continuous polling
+        const interval = setInterval(pollQueue, this.POLLING_INTERVAL_MS);
+        this.pollingIntervals.set(queueUrl, interval);
+        // Start immediately
+        pollQueue();
+        console.log(`Started polling queue: ${queueUrl}`);
+    }
+    /**
+     * Process a message from the queue
+     */
+    processMessage(queueUrl, message, sqsClient) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const mappings = this.queueMappings.get(queueUrl);
+                if (!mappings || mappings.length === 0) {
+                    console.log(`No consumer mappings found for queue: ${queueUrl}`);
+                    return;
+                }
+                // Parse message body
+                let messageData;
+                try {
+                    messageData = JSON.parse(message.Body || '{}');
+                }
+                catch (_a) {
+                    console.warn(`Failed to parse message body as JSON for queue ${queueUrl}. Using raw body.`);
+                    messageData = { body: message.Body };
+                }
+                let messageProcessedByAnyConsumer = false;
+                // Process message for each mapped consumer that matches the message criteria
+                for (const mapping of mappings) {
+                    try {
+                        // Check if message type matches (if specified)
+                        if (mapping.messageType) {
+                            const messageType = messageData.type || messageData.messageType || messageData.eventType;
+                            if (messageType !== mapping.messageType) {
+                                console.log(`Message type ${messageType} does not match expected ${mapping.messageType} for consumer ${mapping.consumer.name} - skipping`);
+                                continue;
+                            }
+                        }
+                        console.log(`Processing queue message for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
+                        const user = UserManager_1.default.getUser();
+                        // Execute the consumer with default options
+                        const result = yield ConsumerEngine_1.default.execute(mapping.consumer, {}, user);
+                        console.log(`Queue trigger completed successfully for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}`);
+                        // Log execution statistics
+                        if (result && result._stats) {
+                            console.log(`Queue trigger stats: ${result._stats.elapsedMS}ms, size: ${result._stats.size}, cycles: ${result._stats.cycles}`);
+                        }
+                        messageProcessedByAnyConsumer = true;
+                    }
+                    catch (error) {
+                        console.error(`Queue trigger failed for consumer "${mapping.consumer.name}" output ${mapping.outputIndex}:`, error);
+                        // Continue processing for other consumers even if one fails
+                    }
+                }
+                // Only delete message from queue if it was processed by at least one consumer
+                // This ensures messages intended for other consumers or systems remain in the queue
+                if (messageProcessedByAnyConsumer && message.ReceiptHandle) {
+                    yield sqsClient.send(new client_sqs_1.DeleteMessageCommand({
+                        QueueUrl: queueUrl,
+                        ReceiptHandle: message.ReceiptHandle
+                    }));
+                    console.log(`Deleted processed message ${message.MessageId} from queue`);
+                }
+                else if (!messageProcessedByAnyConsumer) {
+                    console.log(`Message ${message.MessageId} was not processed by any consumer - leaving in queue for other consumers or systems`);
+                }
+            }
+            catch (error) {
+                console.error(`Error processing message from queue ${queueUrl}:`, error);
+                // Message will remain in queue and be retried or go to DLQ based on queue configuration
+            }
+        });
+    }
+    /**
+     * Add or update queue listeners for a specific consumer
+     */
+    updateConsumerSchedule(consumer) {
+        // First, remove any existing listeners for this consumer
+        this.removeConsumerSchedule(consumer.name);
+        // Then, add new listeners if they have QUEUE triggers
+        if (this.hasQueueTrigger(consumer)) {
+            this.setupQueueListeners(consumer);
+        }
+    }
+    /**
+     * Remove all queue listeners for a consumer
+     */
+    removeConsumerSchedule(consumerName) {
+        // Remove mappings for this consumer
+        for (const [queueUrl, mappings] of this.queueMappings.entries()) {
+            const updatedMappings = mappings.filter(mapping => mapping.consumer.name !== consumerName);
+            if (updatedMappings.length === 0) {
+                // No more consumers listening to this queue, stop polling
+                const interval = this.pollingIntervals.get(queueUrl);
+                if (interval) {
+                    clearInterval(interval);
+                    this.pollingIntervals.delete(queueUrl);
+                    console.log(`Stopped polling queue: ${queueUrl}`);
+                }
+                this.queueMappings.delete(queueUrl);
+            }
+            else {
+                this.queueMappings.set(queueUrl, updatedMappings);
+            }
+        }
+    }
+    /**
+     * Stop all queue polling
+     */
+    stopAllQueues() {
+        console.log('Stopping all queue polling...');
+        this.pollingIntervals.forEach((interval, queueUrl) => {
+            clearInterval(interval);
+            console.log(`Stopped polling queue: ${queueUrl}`);
+        });
+        this.pollingIntervals.clear();
+        this.queueMappings.clear();
+        this.isInitialized = false;
+        console.log('All queue polling stopped');
+    }
+    /**
+     * Restart the queue manager (useful for configuration reloads)
+     */
+    restart() {
+        console.log('Restarting Queue Manager...');
+        this.stopAllQueues();
+        this.initialize();
+    }
+    /**
+     * Get the queue manager status
+     */
+    getStatus() {
+        const queues = Array.from(this.queueMappings.entries()).map(([queueUrl, mappings]) => ({
+            queueUrl,
+            consumerCount: mappings.length
+        }));
+        return {
+            initialized: this.isInitialized,
+            queueCount: this.queueMappings.size,
+            consumerCount: Array.from(this.queueMappings.values()).flat().length,
+            queues
+        };
+    }
+}
+// Export a singleton instance
+exports.default = new QueueManager();

package/engines/usage/UsageDataManager.js

@@ -106,5 +106,46 @@ class UsageDataManager {
             };
         });
     }
+    getFilteredUsageDetails(from, to, filters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const now = DSTE_1.default.now();
+            const fromDate = from || new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+            const toDate = to || now;
+            const collection = 'usage';
+            // Build match criteria
+            const matchCriteria = {
+                startedAt: { $gte: fromDate, $lte: toDate }
+            };
+            if (filters === null || filters === void 0 ? void 0 : filters.consumer) {
+                matchCriteria.consumer = filters.consumer;
+            }
+            if (filters === null || filters === void 0 ? void 0 : filters.user) {
+                matchCriteria['executedBy.name'] = filters.user;
+            }
+            if (filters === null || filters === void 0 ? void 0 : filters.status) {
+                matchCriteria.status = filters.status;
+            }
+            // Query with filters and sorting
+            const usageDetails = yield DatabaseEngine_1.default.query(collection, matchCriteria, {
+                sort: { startedAt: -1 },
+                limit: 1000 // Reasonable limit to prevent large result sets
+            });
+            return usageDetails;
+        });
+    }
+    getUsageById(id) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const collection = 'usage';
+            try {
+                const usageDetail = yield DatabaseEngine_1.default.get(collection, id);
+                return usageDetail;
+            }
+            catch (error) {
+                // If document not found or invalid ID, return null
+                console.error('Error fetching usage by ID:', error);
+                return null;
+            }
+        });
+    }
 }
 exports.default = new UsageDataManager();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.1.6-nasco.3",
+    "version": "0.1.7-nasco.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -35,6 +35,7 @@
     "dependencies": {
         "@aws-sdk/client-redshift-data": "^3.699.0",
         "@aws-sdk/client-s3": "^3.701.0",
+        "@aws-sdk/client-sqs": "^3.886.0",
         "adm-zip": "^0.5.16",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
@@ -54,6 +55,7 @@
         "knex": "^2.4.2",
         "mongodb": "^6.15.0",
         "next": "^13.4.1",
+        "node-cron": "^4.2.1",
         "ora": "^5.4.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",