@forzalabs/remora 0.1.5-nasco.3 → 0.1.7-nasco.3

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.1.5-nasco',
+    cliVersion: '0.1.7-nasco',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/database/DatabaseEngine.js

@@ -20,23 +20,28 @@ class DatabaseEngineClass {
         this.MAX_TRY_CONNECTION = 3;
         this.db = () => this._db;
         this.connect = () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // WARNING: this was changed during the deployment to ECS... I've reverted, but maybe it needs to be changed or looked into...
+            this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
+                ? 'mongodb://mongo:27017/remora'
+                : 'mongodb://localhost:27017/remora';
+            this._client = new mongodb_1.MongoClient(this._uri);
             const errors = [];
             for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
-                if (i < this.MAX_TRY_CONNECTION) {
-                    try {
-                        yield this._client.connect();
-                        this._db = this._client.db(Settings_1.default.db.name);
-                        console.log('Connected to MongoDB');
-                        break;
-                    }
-                    catch (error) {
-                        errors.push(i + '° connection to MongoDB throws this error:', error);
-                    }
+                try {
+                    console.log(`Attempting to connect to mongo: "${this._uri}"`);
+                    yield this._client.connect();
+                    this._db = this._client.db(Settings_1.default.db.name);
+                    this._connected = true;
+                    console.log('Connected to MongoDB');
+                    break;
                 }
-                else {
-                    console.error(`Despite several ${i} retries it was not possible to connect to mongoDb, these are the errors encountered:\n` + errors.join('\n'));
+                catch (error) {
+                    errors.push((i + 1) + ': connection to MongoDB throws this error:', error);
                 }
             }
+            if (!this._connected)
+                console.error(`Despite ${this.MAX_TRY_CONNECTION} retries it was not possible to connect to mongoDb, these are the errors encountered:\n` + errors.join('\n'));
         });
         this.disconnect = () => __awaiter(this, void 0, void 0, function* () {
             try {
@@ -121,8 +126,6 @@ class DatabaseEngineClass {
                 throw error;
             }
         });
-        const uri = Helper_1.default.isDev() ? 'mongodb://mongo:27017/remora' : 'mongodb://localhost:27017/remora';
-        this._client = new mongodb_1.MongoClient(uri);
     }
 }
 const DatabaseEngine = new DatabaseEngineClass();

package/definitions/json_schemas/consumer-schema.json

@@ -235,13 +235,47 @@
                                 "type": "string",
                                 "enum": [
                                     "CRON",
-                                    "API"
+                                    "API",
+                                    "QUEUE"
                                 ],
-                                "description": "The type of trigger schedule"
+                                "description": "The type of trigger schedule. CRON: time-based scheduling. API: HTTP endpoint trigger. QUEUE: SQS queue message trigger (supports shared queues with message type filtering)."
                             },
                             "value": {
                                 "type": "string",
-                                "description": "The value for the trigger (e.g., cron expression or API endpoint)"
+                                "description": "The value for the trigger. For CRON: cron expression (e.g., '0 0 * * *'). For API: endpoint path. For QUEUE: SQS queue URL or queue name (will construct full URL using metadata.region and metadata.accountId if needed)."
+                            },
+                            "metadata": {
+                                "type": "object",
+                                "description": "Additional metadata for the trigger (e.g., AWS credentials, message type filter)",
+                                "properties": {
+                                    "messageType": {
+                                        "type": "string",
+                                        "description": "Optional message type filter for QUEUE triggers. Only messages with matching 'type', 'messageType', or 'eventType' fields will be processed by this consumer. Messages without a matching type will be left in the queue for other consumers, enabling shared queue usage."
+                                    },
+                                    "region": {
+                                        "type": "string",
+                                        "description": "AWS region for the queue (for QUEUE triggers)"
+                                    },
+                                    "accountId": {
+                                        "type": "string",
+                                        "description": "AWS account ID for constructing queue URL (for QUEUE triggers)"
+                                    },
+                                    "accessKeyId": {
+                                        "type": "string",
+                                        "description": "AWS access key ID for queue authentication (for QUEUE triggers)"
+                                    },
+                                    "secretAccessKey": {
+                                        "type": "string",
+                                        "description": "AWS secret access key for queue authentication (for QUEUE triggers)"
+                                    },
+                                    "sessionToken": {
+                                        "type": "string",
+                                        "description": "AWS session token for temporary credentials (for QUEUE triggers)"
+                                    }
+                                },
+                                "additionalProperties": {
+                                    "type": "string"
+                                }
                             }
                         },
                         "required": [
@@ -249,6 +283,20 @@
                             "value"
                         ],
                         "additionalProperties": false
+                    },
+                    "onSuccess": {
+                        "type": "array",
+                        "description": "Actions to perform when the output operation completes successfully",
+                        "items": {
+                            "$ref": "#/definitions/consumerOutputOnFinish"
+                        }
+                    },
+                    "onError": {
+                        "type": "array",
+                        "description": "Actions to perform when the output operation fails",
+                        "items": {
+                            "$ref": "#/definitions/consumerOutputOnFinish"
+                        }
                     }
                 },
                 "required": [
@@ -792,6 +840,23 @@
                     "additionalProperties": false
                 }
             ]
+        },
+        "consumerOutputOnFinish": {
+            "type": "object",
+            "description": "Actions to perform when output operations complete",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": ["move-file"],
+                    "description": "The action to perform"
+                },
+                "moveToDestination": {
+                    "type": "string",
+                    "description": "If the action is 'move-file', this specifies the source destination in remora where the source file should be moved"
+                }
+            },
+            "required": ["action"],
+            "additionalProperties": false
         }
     },
     "examples": [

package/drivers/DeltaShareDriver.js

@@ -13,6 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 const DriverHelper_1 = __importDefault(require("./DriverHelper"));
 /**
  * Delta Share (Databricks Delta Sharing) Source Driver
@@ -31,7 +32,7 @@ class DeltaShareSourceDriver {
             const { authentication } = source;
             (0, Affirm_1.default)(authentication, 'Invalid authentication for delta-share source');
             this._shareUrl = authentication.host;
-            this._bearerToken = authentication.bearerToken || authentication.sessionToken || authentication.password;
+            this._bearerToken = SecretManager_1.default.replaceSecret(authentication.bearerToken || authentication.sessionToken || authentication.password);
             this._share = authentication.share;
             this._schema = authentication.schema;
             this._table = authentication.table;
@@ -58,7 +59,8 @@ class DeltaShareSourceDriver {
             (0, Affirm_1.default)(request, `Invalid download request`);
             (0, Affirm_1.default)(!request.fileKey.includes('%'), `On a delta-share the file key can not include "%"`);
             const deltaFiles = yield this._getAllFilesInTables(this._table);
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             const lines = [];
             for (const deltaFile of deltaFiles) {
                 const byteLength = (_b = (_a = deltaFile.file.deltaSingleAction.add) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : (_c = deltaFile.file.deltaSingleAction.remove) === null || _c === void 0 ? void 0 : _c.size;
@@ -75,7 +77,8 @@ class DeltaShareSourceDriver {
             (0, Affirm_1.default)(request.options.lineFrom !== undefined && request.options.lineTo !== undefined, 'Missing read range');
             const deltaFiles = yield this._getAllFilesInTables(this._table);
             const { options: { lineFrom, lineTo } } = request;
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             const lines = [];
             let index = 0;
             for (const deltaFile of deltaFiles) {
@@ -96,7 +99,8 @@ class DeltaShareSourceDriver {
             var _a, _b, _c;
             (0, Affirm_1.default)(dataset, 'Invalid dataset');
             const deltaFiles = yield this._getAllFilesInTables(this._table);
-            const { asyncBufferFromUrl, parquetReadObjects } = yield import('hyparquet');
+            const hyparquet = yield import('hyparquet');
+            const { asyncBufferFromUrl, parquetReadObjects } = hyparquet;
             // For each file, download it with the hyparquet package, read lines, then save locally to create the dataset
             let index = 0;
             let totalLineCount = 0;
@@ -142,7 +146,7 @@ class DeltaShareSourceDriver {
                     Authorization: `Bearer ${this._bearerToken}`
                 }
             });
-            (0, Affirm_1.default)(res.ok, `Error fetching version from the delta share: ${res.status} ${res.statusText}`);
+            (0, Affirm_1.default)(res.ok, `Error fetching version from the delta share: ${res.status} ${res.statusText} (${yield res.text()})`);
             const version = res.headers['delta-table-version'];
             return version;
         });

package/drivers/LocalDriver.js

@@ -320,6 +320,36 @@ class LocalSourceDriver {
                 throw new Error(`Failed to list files in directory "${this._path}": ${error.message}`);
             }
         };
+        this.readFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            (0, Affirm_1.default)(fs.existsSync(filePath), `Source file does not exist: ${filePath}`);
+            return fs.readFileSync(filePath);
+        };
+        this.deleteFile = (fileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const filePath = path_1.default.join(this._path, fileKey);
+            if (fs.existsSync(filePath)) {
+                fs.unlinkSync(filePath);
+            }
+        };
+        this.moveFile = (sourceFileKey, destinationPath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationPath, 'Invalid destination path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const sourceFilePath = path_1.default.join(this._path, sourceFileKey);
+            const destinationFilePath = path_1.default.join(destinationPath, destinationFileKey);
+            (0, Affirm_1.default)(fs.existsSync(sourceFilePath), `Source file does not exist: ${sourceFilePath}`);
+            // Ensure destination directory exists
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.renameSync(sourceFilePath, destinationFilePath);
+        };
     }
 }
 exports.LocalSourceDriver = LocalSourceDriver;
@@ -385,6 +415,35 @@ class LocalDestinationDriver {
                 throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
             }
         });
+        this.saveFile = (fileKey, content) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            const filePath = path_1.default.join(this._path, fileKey);
+            const directory = path_1.default.dirname(filePath);
+            // Create directory if it doesn't exist
+            if (!fs.existsSync(directory)) {
+                fs.mkdirSync(directory, { recursive: true });
+            }
+            fs.writeFileSync(filePath, content);
+            return Promise.resolve();
+        };
+        this.copyFromLocal = (sourceFilePath, destinationFileKey) => {
+            (0, Affirm_1.default)(this._path, 'Path not initialized');
+            (0, Affirm_1.default)(sourceFilePath, 'Invalid source file path');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            const destinationFilePath = path_1.default.join(this._path, destinationFileKey);
+            const destinationDir = path_1.default.dirname(destinationFilePath);
+            // Ensure destination directory exists
+            if (!fs.existsSync(destinationDir)) {
+                fs.mkdirSync(destinationDir, { recursive: true });
+            }
+            fs.copyFileSync(sourceFilePath, destinationFilePath);
+        };
+        this.copyFromS3 = (s3Driver, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            const fileContent = yield s3Driver.downloadFile(sourceFileKey);
+            yield this.saveFile(destinationFileKey, fileContent);
+        });
     }
 }
 exports.LocalDestinationDriver = LocalDestinationDriver;

package/drivers/S3Driver.js

@@ -134,6 +134,27 @@ class S3DestinationDriver {
                 throw error;
             }
         });
+        this.copyFromS3 = (sourceBucket, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(sourceBucket, 'Invalid source bucket');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            yield this._client.send(new client_s3_1.CopyObjectCommand({
+                CopySource: `${sourceBucket}/${sourceFileKey}`,
+                Bucket: this._bucketName,
+                Key: destinationFileKey
+            }));
+        });
+        this.saveFile = (fileKey, content) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            (0, Affirm_1.default)(content, 'Invalid content');
+            yield this._client.send(new client_s3_1.PutObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey,
+                Body: content
+            }));
+        });
     }
 }
 exports.S3DestinationDriver = S3DestinationDriver;
@@ -491,6 +512,36 @@ class S3SourceDriver {
             } while (continuationToken);
             return allFiles;
         });
+        this.downloadFile = (fileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            const response = yield this._client.send(new client_s3_1.GetObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey
+            }));
+            (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
+            const content = yield response.Body.transformToByteArray();
+            return Buffer.from(content);
+        });
+        this.deleteFile = (fileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(fileKey, 'Invalid file key');
+            yield this._client.send(new client_s3_1.DeleteObjectCommand({
+                Bucket: this._bucketName,
+                Key: fileKey
+            }));
+        });
+        this.copyFile = (sourceFileKey, destinationBucket, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "init()" first');
+            (0, Affirm_1.default)(sourceFileKey, 'Invalid source file key');
+            (0, Affirm_1.default)(destinationBucket, 'Invalid destination bucket');
+            (0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
+            yield this._client.send(new client_s3_1.CopyObjectCommand({
+                CopySource: `${this._bucketName}/${sourceFileKey}`,
+                Bucket: destinationBucket,
+                Key: destinationFileKey
+            }));
+        });
     }
 }
 exports.S3SourceDriver = S3SourceDriver;

package/engines/Environment.js

@@ -128,6 +128,10 @@ class EnvironmentClass {
             (0, Affirm_1.default)(consumerName, `Invalid consumer name`);
             return this._env.consumers.find(x => x.name.toLowerCase() === consumerName.toLowerCase());
         };
+        this.getAllConsumers = () => {
+            (0, Affirm_1.default)(this._env, 'Environment not initialized');
+            return this._env.consumers || [];
+        };
         this.getSchema = (schemaName) => {
             (0, Affirm_1.default)(schemaName, 'Invalid schema name');
             return this._env.schemas.find(x => x.title === schemaName);

package/engines/consumer/ConsumerOnFinishManager.js

@@ -0,0 +1,188 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Affirm_1 = __importDefault(require("../../core/Affirm"));
+const Environment_1 = __importDefault(require("../Environment"));
+const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
+class ConsumerOnFinishManagerClass {
+    constructor() {
+        this.performOnSuccessActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(output, 'Invalid output');
+            if (!output.onSuccess || output.onSuccess.length === 0)
+                return;
+            for (const onSuccess of output.onSuccess) {
+                switch (onSuccess.action) {
+                    case 'move-file': {
+                        yield this.moveSourceFiles(consumer, onSuccess.moveToDestination);
+                        break;
+                    }
+                    default:
+                        throw new Error(`On success action "${onSuccess.action}" is not implemented yet.`);
+                }
+            }
+        });
+        this.moveSourceFiles = (consumer, moveDestination) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(moveDestination, 'Move destination is required for move-file action');
+            const destinationSource = Environment_1.default.getSource(moveDestination);
+            (0, Affirm_1.default)(destinationSource, `Destination source "${moveDestination}" not found`);
+            // Get all unique source files from all producers used by this consumer
+            const sourceFilesToMove = [];
+            for (const consumerProducer of consumer.producers) {
+                const producer = Environment_1.default.getProducer(consumerProducer.name);
+                (0, Affirm_1.default)(producer, `Producer "${consumerProducer.name}" not found`);
+                const source = Environment_1.default.getSource(producer.source);
+                (0, Affirm_1.default)(source, `Source "${producer.source}" not found`);
+                // Only handle file-based sources that have fileKey
+                if (producer.settings.fileKey) {
+                    // Handle wildcard patterns (files with %)
+                    if (producer.settings.fileKey.includes('%')) {
+                        // Get all files matching the pattern
+                        const matchingFiles = yield this.getMatchingFiles(source, producer.settings.fileKey);
+                        for (const fileKey of matchingFiles) {
+                            sourceFilesToMove.push({
+                                sourceName: producer.source,
+                                filePath: this.getSourceFilePath(source, fileKey),
+                                fileKey
+                            });
+                        }
+                    }
+                    else {
+                        // Single file
+                        sourceFilesToMove.push({
+                            sourceName: producer.source,
+                            filePath: this.getSourceFilePath(source, producer.settings.fileKey),
+                            fileKey: producer.settings.fileKey
+                        });
+                    }
+                }
+            }
+            // Move all files to destination
+            yield this.moveFiles(sourceFilesToMove, destinationSource);
+        });
+        this.getMatchingFiles = (source, fileKeyPattern) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(source);
+            if (source.engine === 'aws-s3') {
+                // S3 driver has a public listFiles method that handles patterns
+                const s3Driver = sourceDriver;
+                return yield s3Driver.listFiles(fileKeyPattern);
+            }
+            else if (source.engine === 'local') {
+                // Local driver now has a public listFiles method consistent with S3
+                const localDriver = sourceDriver;
+                return localDriver.listFiles(fileKeyPattern);
+            }
+            throw new Error(`Unsupported source engine for file listing: ${source.engine}`);
+        });
+        this.getSourceFilePath = (source, fileKey) => {
+            if (source.engine === 'local') {
+                return `${source.authentication['path']}/${fileKey}`;
+            }
+            else if (source.engine === 'aws-s3') {
+                // For S3, we return the key as the path since S3 uses keys instead of file paths
+                return fileKey;
+            }
+            throw new Error(`Unsupported source engine for file move: ${source.engine}`);
+        };
+        this.moveFiles = (files, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            for (const file of files) {
+                const sourceSource = Environment_1.default.getSource(file.sourceName);
+                if (sourceSource.engine === 'local' && destinationSource.engine === 'local') {
+                    // Local to Local move
+                    yield this.moveLocalToLocal(file.filePath, destinationSource, file.fileKey);
+                }
+                else if (sourceSource.engine === 'local' && destinationSource.engine === 'aws-s3') {
+                    // Local to S3 move
+                    yield this.moveLocalToS3(file.filePath, destinationSource, file.fileKey);
+                }
+                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'local') {
+                    // S3 to Local move
+                    yield this.moveS3ToLocal(sourceSource, file.fileKey, destinationSource);
+                }
+                else if (sourceSource.engine === 'aws-s3' && destinationSource.engine === 'aws-s3') {
+                    // S3 to S3 move
+                    yield this.moveS3ToS3(sourceSource, file.fileKey, destinationSource);
+                }
+                else {
+                    throw new Error(`Unsupported move operation from ${sourceSource.engine} to ${destinationSource.engine}`);
+                }
+            }
+        });
+        this.moveLocalToLocal = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Read file from source
+            const fileContent = sourceDriver.readFile(fileKey);
+            // Save to destination
+            yield destinationDriver.saveFile(fileKey, fileContent);
+            // Delete from source
+            sourceDriver.deleteFile(fileKey);
+        });
+        this.findSourceForPath = (filePath) => {
+            // Extract directory from file path for source creation
+            const directory = filePath.substring(0, filePath.lastIndexOf('/'));
+            return {
+                name: 'temp-source',
+                engine: 'local',
+                authentication: { path: directory }
+            };
+        };
+        this.moveLocalToS3 = (sourceFilePath, destinationSource, fileKey) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(this.findSourceForPath(sourceFilePath));
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Read file from local source
+            const fileContent = sourceDriver.readFile(fileKey);
+            // Upload to S3 destination
+            yield destinationDriver.saveFile(fileKey, fileContent);
+            // Remove source file after successful upload
+            sourceDriver.deleteFile(fileKey);
+        });
+        this.moveS3ToLocal = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Download from S3
+            const content = yield sourceDriver.downloadFile(fileKey);
+            // Save to local destination
+            yield destinationDriver.saveFile(fileKey, content);
+            // Delete from S3 source
+            yield sourceDriver.deleteFile(fileKey);
+        });
+        this.moveS3ToS3 = (sourceSource, fileKey, destinationSource) => __awaiter(this, void 0, void 0, function* () {
+            const sourceDriver = yield DriverFactory_1.default.instantiateSource(sourceSource);
+            const destinationDriver = yield DriverFactory_1.default.instantiateDestination(destinationSource);
+            // Copy from source S3 to destination S3
+            yield destinationDriver.copyFromS3(sourceSource.authentication['bucket'], fileKey, fileKey);
+            // Delete from source S3
+            yield sourceDriver.deleteFile(fileKey);
+        });
+        this.performOnErrorActions = (consumer, output) => __awaiter(this, void 0, void 0, function* () {
+            (0, Affirm_1.default)(consumer, 'Invalid consumer');
+            (0, Affirm_1.default)(output, 'Invalid output');
+            if (!output.onError || output.onError.length === 0)
+                return;
+            for (const onError of output.onError) {
+                switch (onError.action) {
+                    case 'move-file': {
+                        yield this.moveSourceFiles(consumer, onError.moveToDestination);
+                        break;
+                    }
+                    default:
+                        throw new Error(`On success action "${onError.action}" is not implemented yet.`);
+                }
+            }
+        });
+    }
+}
+const ConsumerOnFinishManager = new ConsumerOnFinishManagerClass();
+exports.default = ConsumerOnFinishManager;

package/engines/dataset/Dataset.js

@@ -34,6 +34,7 @@ const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
 const Helper_1 = __importDefault(require("../../helper/Helper"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Environment_1 = __importDefault(require("../Environment"));
+const Logger_1 = __importDefault(require("../../helper/Logger"));
 class Dataset {
     constructor(name, file, batchSize, executionId) {
         var _a;
@@ -171,7 +172,8 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
+                        Logger_1.default.log(`Error parsing line ${line}\n${lineCount}: ${error}`);
+                        lineCount++;
                     }
                 }
             }
@@ -306,7 +308,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line during sort: ${error}`);
+                        Logger_1.default.log(`Error parsing line during sort: ${error}`);
                     }
                 }
             }
@@ -537,7 +539,7 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
+                        Logger_1.default.log(`Error parsing line ${line}\n${lineCount}: ${error}`);
                     }
                 }
             }
@@ -628,7 +630,8 @@ class Dataset {
                         }
                     }
                     catch (error) {
-                        console.warn(`Error parsing line ${line}\n${lineCount}: ${error}`);
+                        Logger_1.default.log(`Error parsing line ${line}\n${lineCount}: ${error}`);
+                        lineCount++;
                     }
                 }
             }

package/engines/dataset/DatasetRecord.js

@@ -12,7 +12,7 @@ class DatasetRecord {
                 const parts = row.split(delimiter);
                 for (let i = 0; i < dimensions.length; i++) {
                     const dim = dimensions[i];
-                    this._value[dim.name] = TypeCaster_1.default.cast(parts[i], dim.type);
+                    this._value[dim.name] = TypeCaster_1.default.cast(parts[i], dim.type, dim.format);
                 }
             }
         };

package/engines/dataset/ParallelDataset.js

@@ -43,6 +43,9 @@ class ParallelDatasetClass {
             const currentDir = __dirname;
             if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
                 return path_1.default.resolve('./.build/workers');
+            const forcedPath = process.env.REMORA_WORKERS_PATH;
+            if (forcedPath && forcedPath.length > 0)
+                return path_1.default.join(__dirname, forcedPath);
             // Check if we're in a published npm package (no .build in path)
             if (!currentDir.includes('.build')) {
                 // We're in the published package, workers are relative to package root

package/engines/execution/ExecutionEnvironment.js

@@ -27,6 +27,7 @@ const Environment_1 = __importDefault(require("../Environment"));
 const Algo_1 = __importDefault(require("../../core/Algo"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 const ParallelDataset_1 = __importDefault(require("../dataset/ParallelDataset"));
+const ConsumerOnFinishManager_1 = __importDefault(require("../consumer/ConsumerOnFinishManager"));
 class ExecutionEnvironment {
     constructor(consumer, executionId) {
         this.run = (options) => __awaiter(this, void 0, void 0, function* () {
@@ -127,6 +128,10 @@ class ExecutionEnvironment {
                             this._resultingDataset = yield PostProcessor_1.default.distinct(this._resultingDataset);
                             break;
                         }
+                        case 'perform-on-success-actions': {
+                            yield ConsumerOnFinishManager_1.default.performOnSuccessActions(this._consumer, planStep.output);
+                            break;
+                        }
                         case 'save-execution-stats': {
                             (0, Affirm_1.default)(this._resultingDataset, `Invalid result dataset in save-execution-stats`);
                             result._stats = {
@@ -156,6 +161,12 @@ class ExecutionEnvironment {
                 if (ds)
                     Logger_1.default.log(`Failed execution of consumer at step ${currentStep.type}:\n\tSize: ${ds.getCount()}\n\tCycles: ${ds.getCycles()}\n\tOperations: ${Logger_1.default.formatList(ds.getOperations())}`);
                 Logger_1.default.log(`\tFailed step: ${currentStep.type}->\n\t${error}`);
+                try {
+                    yield ConsumerOnFinishManager_1.default.performOnErrorActions(this._consumer, currentStep.output);
+                }
+                catch (error) {
+                    Logger_1.default.log(`Error when trying to perform onError actions on failed consumer ${error}`);
+                }
                 // IMPORTANT: cleanup all the datasets to not leave any data around and to avoid memory leaks
                 const datasets = [
                     ...this._producedData.map(x => x.dataset),