@forzalabs/remora 0.0.19 → 0.0.21

package/Constants.js

@@ -1,8 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.19',
+    cliVersion: '0.0.21',
     lambdaVersion: 1,
-    port: 5069
+    port: 5069,
+    defaults: {
+        SQL_MAX_QUERY_ROWS: 10000,
+        STRING_MAX_CHARACTERS_LENGTH: 10000000
+    }
 };
 exports.default = CONSTANTS;

package/actions/automap.js

@@ -33,6 +33,10 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
         if (!producer) {
             throw new Error(`Producer ${producerName} not found`);
         }
+        const source = Environment_1.default.getSource(producer.source);
+        if (!source) {
+            throw new Error(`Source ${producer.source} not found`);
+        }
         // Get the specified schemas
         const schemas = [];
         for (const schemaName of schemaNames) {
@@ -47,7 +51,7 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
         // Convert sample data to strings for AutoMapperEngine
         const sampleStrings = sampleData.map(item => JSON.stringify(item));
         // Call the automapper
-        const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas);
+        const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
         // Create the producers based on the mapping
         for (const producer of mapResult.producers) {
             const producerPath = path_1.default.join('remora/producers', `${producer.name}.json`);

package/actions/deploy.js

@@ -59,7 +59,6 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
                 const apiKey = process.env.REMORA_LICENCE_KEY;
                 if (!apiKey)
                     throw new Error('REMORA_LICENCE_KEY environment variable is not set');
-                console.log('workerAPI', workerAPI);
                 const response = yield fetch(workerAPI, {
                     method: 'POST',
                     headers: {

package/actions/run.js

@@ -37,7 +37,10 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
                 results.push({ success: true, consumer, response });
             }
             catch (error) {
-                results.push({ success: false, consumer, error: Helper_1.default.asError(error).message });
+                const myErr = Helper_1.default.asError(error);
+                results.push({ success: false, consumer, error: myErr.message });
+                if (Helper_1.default.isDev())
+                    console.log(myErr.stack);
             }
         }
         spinner.succeed('All consumers have been executed:');
@@ -46,7 +49,7 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
                 if (response.fileUri)
                     console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.fileUri}`));
                 else
-                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.data.slice(0, 5).join('\n')}`));
+                    console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> \n${response.data.slice(0, 5).map(x => JSON.stringify(x)).join('\n')}\n and ${response.data.length - 5} more...`));
             }
             else {
                 console.log(chalk_1.default.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
@@ -59,7 +62,10 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
         process.exit(1);
     }
     catch (err) {
-        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during run:'), err instanceof Error ? err.message : String(err));
+        const myErr = Helper_1.default.asError(err);
+        console.error(chalk_1.default.red.bold('\n❌ Unexpected error during run:'), myErr.message);
+        if (Helper_1.default.isDev())
+            console.log(myErr.stack);
         process.exit(1);
     }
 });

package/auth/JWTManager.js

@@ -46,7 +46,7 @@ class JWTManagerClass {
         this.verifyCLI = (token) => {
             (0, Affirm_1.default)(token, 'Invalid token');
             // ⚠️ ATTENTION!! The remoteSecretKey is saved in clear text because we are studying a way to not make it visible to the customer. ATTENTION!!
-            const REMORA_KEY_SECRET = this.remoraSecretKey; // process.env.REMORA_KEY_SECRET
+            const REMORA_KEY_SECRET = process.env.REMORA_KEY_SECRET;
             (0, Affirm_1.default)(REMORA_KEY_SECRET, 'Invalid CLI JWT SECRET');
             return jsonwebtoken_1.default.verify(token, REMORA_KEY_SECRET);
         };

package/database/DatabaseEngine.js

@@ -13,8 +13,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const mongodb_1 = require("mongodb");
-const Settings_1 = __importDefault(require("../helper/Settings"));
 const Helper_1 = __importDefault(require("../helper/Helper"));
+const Settings_1 = __importDefault(require("../helper/Settings"));
 class DatabaseEngineClass {
     constructor() {
         this.MAX_TRY_CONNECTION = 3;
@@ -47,10 +47,10 @@ class DatabaseEngineClass {
                 console.error('Error disconnecting from MongoDB:', error);
             }
         });
-        this.query = (collectionName, query) => __awaiter(this, void 0, void 0, function* () {
+        this.query = (collectionName, filter, options) => __awaiter(this, void 0, void 0, function* () {
             try {
                 const collection = this._db.collection(collectionName);
-                const result = yield collection.find(query).toArray();
+                const result = yield collection.find(filter, options).toArray();
                 return result;
             }
             catch (error) {
@@ -58,6 +58,16 @@ class DatabaseEngineClass {
                 throw error;
             }
         });
+        this.aggregate = (collectionName, aggregation) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const collection = this._db.collection(collectionName);
+                return yield collection.aggregate(aggregation).toArray();
+            }
+            catch (error) {
+                console.error('Error aggregate documents:', error);
+                throw error;
+            }
+        });
         this.get = (collectionName, id) => __awaiter(this, void 0, void 0, function* () {
             try {
                 const collection = this._db.collection(collectionName);

package/definitions/json_schemas/consumer-schema.json

@@ -127,6 +127,52 @@
                             "subFields"
                         ],
                         "additionalProperties": false
+                    },
+                    "transform": {
+                        "description": "Optional list of transformations to apply to the dataset before returning the data",
+                        "oneOf": [
+                            {
+                                "$ref": "#/definitions/singleTransformation"
+                            },
+                            {
+                                "type": "array",
+                                "items": {
+                                    "$ref": "#/definitions/singleTransformation"
+                                },
+                                "description": "A list of transformations to be applied in sequence"
+                            }
+                        ]
+                    },
+                    "validate": {
+                        "type": "object",
+                        "description": "Rules to check field value compliance and data quality",
+                        "properties": {
+                            "min": {
+                                "type": "number",
+                                "description": "Minimum value for numeric fields"
+                            },
+                            "max": {
+                                "type": "number",
+                                "description": "Maximum value for numeric fields"
+                            },
+                            "regex": {
+                                "type": "string",
+                                "description": "Regular expression pattern to validate string fields"
+                            },
+                            "required": {
+                                "type": "boolean",
+                                "description": "Whether the field is required"
+                            }
+                        },
+                        "additionalProperties": false
+                    },
+                    "onError": {
+                        "type": "string",
+                        "description": "Action to take if an error occurs during transformations or validation",
+                        "enum": ["set_default", "skip", "fail"]
+                    },
+                    "default": {
+                        "description": "Default value of the field if it is missing (or on error if specified)"
                     }
                 },
                 "required": [
@@ -325,12 +371,358 @@
                         "subFields"
                     ],
                     "additionalProperties": false
+                },
+                "transform": {
+                    "description": "Optional list of transformations to apply to the dataset before returning the data",
+                    "oneOf": [
+                        {
+                            "$ref": "#/definitions/singleTransformation"
+                        },
+                        {
+                            "type": "array",
+                            "items": {
+                                "$ref": "#/definitions/singleTransformation"
+                            },
+                            "description": "A list of transformations to be applied in sequence"
+                        }
+                    ]
+                },
+                "validate": {
+                    "type": "object",
+                    "description": "Rules to check field value compliance and data quality",
+                    "properties": {
+                        "min": {
+                            "type": "number",
+                            "description": "Minimum value for numeric fields"
+                        },
+                        "max": {
+                            "type": "number",
+                            "description": "Maximum value for numeric fields"
+                        },
+                        "regex": {
+                            "type": "string",
+                            "description": "Regular expression pattern to validate string fields"
+                        },
+                        "required": {
+                            "type": "boolean",
+                            "description": "Whether the field is required"
+                        }
+                    },
+                    "additionalProperties": false
+                },
+                "onError": {
+                    "type": "string",
+                    "description": "Action to take if an error occurs during transformations or validation",
+                    "enum": ["set_default", "skip", "fail"]
+                },
+                "default": {
+                    "description": "Default value of the field if it is missing (or on error if specified)"
                 }
             },
             "required": [
                 "key"
             ],
             "additionalProperties": false
+        },
+        "singleTransformation": {
+            "oneOf": [
+                {
+                    "type": "object",
+                    "properties": {
+                        "cast": {
+                            "type": "string",
+                            "description": "Cast the value to a specific type",
+                            "enum": ["string", "number", "date", "boolean"]
+                        }
+                    },
+                    "required": ["cast"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "multiply": {
+                            "type": "number",
+                            "description": "Multiply the numeric value by this factor"
+                        }
+                    },
+                    "required": ["multiply"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "add": {
+                            "type": "number",
+                            "description": "Add this number to the numeric value"
+                        }
+                    },
+                    "required": ["add"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "extract": {
+                            "type": "string",
+                            "enum": ["year", "month", "day", "hour", "minute"],
+                            "description": "Extract a component from a date value"
+                        }
+                    },
+                    "required": ["extract"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "concat": {
+                            "type": "object",
+                            "properties": {
+                                "separator": {
+                                    "type": "string",
+                                    "description": "The separator to use when joining array elements"
+                                }
+                            },
+                            "required": ["separator"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["concat"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "split": {
+                            "type": "object",
+                            "properties": {
+                                "separator": {
+                                    "type": "string",
+                                    "description": "The separator to use when splitting the string"
+                                },
+                                "index": {
+                                    "type": "number",
+                                    "description": "The index of the split part to keep"
+                                }
+                            },
+                            "required": ["separator", "index"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["split"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "regex_match": {
+                            "type": "object",
+                            "properties": {
+                                "pattern": {
+                                    "type": "string",
+                                    "description": "The regex pattern to test against"
+                                },
+                                "flags": {
+                                    "type": "string",
+                                    "description": "Regex flags (e.g., 'i' for case-insensitive)"
+                                }
+                            },
+                            "required": ["pattern"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["regex_match"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "regex_replace": {
+                            "type": "object",
+                            "properties": {
+                                "pattern": {
+                                    "type": "string",
+                                    "description": "The regex pattern to match"
+                                },
+                                "replacement": {
+                                    "type": "string",
+                                    "description": "The replacement string"
+                                },
+                                "flags": {
+                                    "type": "string",
+                                    "description": "Regex flags (e.g., 'g' for global)"
+                                }
+                            },
+                            "required": ["pattern", "replacement"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["regex_replace"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "regex_extract": {
+                            "type": "object",
+                            "properties": {
+                                "pattern": {
+                                    "type": "string",
+                                    "description": "The regex pattern to extract with"
+                                },
+                                "group": {
+                                    "type": "number",
+                                    "description": "The capture group index to extract (0 for full match)"
+                                },
+                                "flags": {
+                                    "type": "string",
+                                    "description": "Regex flags (e.g., 'i' for case-insensitive)"
+                                }
+                            },
+                            "required": ["pattern", "group"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["regex_extract"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "trim": {
+                            "type": "boolean",
+                            "description": "Trim whitespace from both ends of the string"
+                        }
+                    },
+                    "required": ["trim"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "to_lowercase": {
+                            "type": "boolean",
+                            "description": "Convert string to lowercase"
+                        }
+                    },
+                    "required": ["to_lowercase"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "to_uppercase": {
+                            "type": "boolean",
+                            "description": "Convert string to uppercase"
+                        }
+                    },
+                    "required": ["to_uppercase"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "capitalize": {
+                            "type": "boolean",
+                            "description": "Capitalize the first letter of the string"
+                        }
+                    },
+                    "required": ["capitalize"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "substring": {
+                            "type": "object",
+                            "properties": {
+                                "start": {
+                                    "type": "number",
+                                    "description": "Starting position of substring"
+                                },
+                                "end": {
+                                    "type": "number",
+                                    "description": "Optional ending position of substring"
+                                }
+                            },
+                            "required": ["start"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["substring"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "pad_start": {
+                            "type": "object",
+                            "properties": {
+                                "length": {
+                                    "type": "number",
+                                    "description": "Desired string length after padding"
+                                },
+                                "char": {
+                                    "type": "string",
+                                    "description": "Character to pad with",
+                                    "minLength": 1,
+                                    "maxLength": 1
+                                }
+                            },
+                            "required": ["length", "char"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["pad_start"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "pad_end": {
+                            "type": "object",
+                            "properties": {
+                                "length": {
+                                    "type": "number",
+                                    "description": "Desired string length after padding"
+                                },
+                                "char": {
+                                    "type": "string",
+                                    "description": "Character to pad with",
+                                    "minLength": 1,
+                                    "maxLength": 1
+                                }
+                            },
+                            "required": ["length", "char"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["pad_end"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "prepend": {
+                            "type": "string",
+                            "description": "String to add at the beginning"
+                        }
+                    },
+                    "required": ["prepend"],
+                    "additionalProperties": false
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "append": {
+                            "type": "string",
+                            "description": "String to add at the end"
+                        }
+                    },
+                    "required": ["append"],
+                    "additionalProperties": false
+                }
+            ]
         }
     },
     "examples": [

package/definitions/json_schemas/producer-schema.json

@@ -135,6 +135,10 @@
                         "CSV"
                     ],
                     "description": "The type of file to read"
+                },
+                "delimiter": {
+                    "type": "string",
+                    "description": "The column delimiter for the CSV file if different from the default (,)."
                 }
             },
             "additionalProperties": false

package/definitions/transform/Transformations.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/drivers/DriverFactory.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -12,7 +45,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const LocalDriver_1 = __importDefault(require("./LocalDriver"));
+const LocalDriver_1 = __importStar(require("./LocalDriver"));
 const RedshiftDriver_1 = __importDefault(require("./RedshiftDriver"));
 const S3Driver_1 = __importDefault(require("./S3Driver"));
 const S3SourceDriver_1 = __importDefault(require("./S3SourceDriver"));
@@ -47,6 +80,11 @@ class DriverFactoryClass {
                     yield driver.init(source);
                     return driver;
                 }
+                case 'local': {
+                    const driver = new LocalDriver_1.LocalDriverDestination();
+                    yield driver.init(source);
+                    return driver;
+                }
                 default: throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
             }
         });