@forzalabs/remora 1.0.18 → 1.0.20

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '1.0.18',
+    cliVersion: '1.0.20',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,

package/actions/deploy.js

@@ -18,6 +18,7 @@ const fs_1 = __importDefault(require("fs"));
 const adm_zip_1 = __importDefault(require("adm-zip"));
 const path_1 = __importDefault(require("path"));
 const Constants_1 = __importDefault(require("../Constants"));
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
     console.log(chalk_1.default.blue.bold(`🚀 Deploying to ${options.env}...`));
     try {
@@ -50,14 +51,14 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
             try {
                 // Read the zip file as a buffer
                 const zipBuffer = fs_1.default.readFileSync(tempZipPath);
-                const host = process.env.REMORA_WORKER_HOST;
+                const host = ProcessENVManager_1.default.getEnvVariable('REMORA_WORKER_HOST');
                 const version = Constants_1.default.workerVersion;
                 const workerAPI = `${host}/cli/v${version}/uploaddeployment`;
                 const formData = new FormData();
                 // @ts-ignore
                 const blob = new Blob([zipBuffer], { type: 'application/zip' });
                 formData.append('remora_config', blob, 'temp_deployment.zip'); // Updated to match the actual file name
-                const apiKey = process.env.REMORA_LICENCE_KEY;
+                const apiKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
                 if (!apiKey)
                     throw new Error('REMORA_LICENCE_KEY environment variable is not set');
                 const response = yield fetch(workerAPI, {

package/actions/run.js

@@ -19,6 +19,7 @@ const compile_1 = require("./compile");
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
 const ExecutorOrchestrator_1 = __importDefault(require("../executors/ExecutorOrchestrator"));
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         (0, compile_1.compile)();
@@ -49,7 +50,7 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
         for (let i = 0; i < consumersToExecute.length; i++) {
             const consumer = consumersToExecute[i];
             try {
-                const remoraLicenceKey = process.env.REMORA_LICENCE_KEY;
+                const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
                 const check = LicenceManager_1.default.validate(remoraLicenceKey);
                 if (!check.valid) {
                     console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);

package/database/DatabaseEngine.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const mongodb_1 = require("mongodb");
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const Settings_1 = __importDefault(require("../helper/Settings"));
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 class DatabaseEngineClass {
     constructor() {
         this.MAX_TRY_CONNECTION = 3;
@@ -23,7 +24,7 @@ class DatabaseEngineClass {
             // WARNING: this was changed during the deployment to ECS...
             // I've reverted it, but maybe it needs to be changed or looked into...
             var _a;
-            this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
+            this._uri = ((_a = ProcessENVManager_1.default.getEnvVariable('MONGO_URI')) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
                 ? 'mongodb://mongo:27017/remora'
                 : 'mongodb://localhost:27017/remora';
             this._client = new mongodb_1.MongoClient(this._uri);

package/definitions/ProcessENV.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/json_schemas/producer-schema.json

@@ -118,6 +118,15 @@
         },
         "settings": {
             "type": "object",
+            "compressionType": {
+                "type": "string",
+                "enum": [
+                    "GZ",
+                    "TAR",
+                    "ZIP"
+                    ],
+                "description": "The compression type used from the file to read"
+            },
             "description": "Settings for the producer",
             "properties": {
                 "sqlTable": {
@@ -200,6 +209,10 @@
                 {
                     "name": "email",
                     "type": "string",
+                    "format": {
+                        "type": "string",
+                        "description": "Format for datetype of the source ('YYYYMMDD', 'DD-MM-YYYY')"
+                    },
                     "classification": [
                         "PII",
                         "GDPR"

package/drivers/files/LocalSourceDriver.js

@@ -386,17 +386,17 @@ class LocalSourceDriver {
                 const allFileKeys = this.listFiles(fileKey);
                 const allFilePaths = [];
                 for (const fileKey of allFileKeys) {
-                    const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
-                    ExecutorScope_1.default.ensurePath(localPath);
-                    yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, localPath);
+                    const executionPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
+                    ExecutorScope_1.default.ensurePath(executionPath);
+                    const localPath = yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, executionPath);
                     allFilePaths.push(localPath);
                 }
                 return { files: allFilePaths.map(x => ({ fullUri: x })) };
             }
             else {
-                const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
-                ExecutorScope_1.default.ensurePath(localPath);
-                yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, localPath);
+                const executionPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
+                ExecutorScope_1.default.ensurePath(executionPath);
+                const localPath = yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, executionPath);
                 return { files: [{ fullUri: localPath }] };
             }
         });

package/engines/CryptoEngine.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const crypto_1 = __importDefault(require("crypto"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const RandomEngine_1 = __importDefault(require("./RandomEngine"));
+const ProcessENVManager_1 = __importDefault(require("./ProcessENVManager"));
 class CryptoEngineClass {
     constructor() {
         var _a;
@@ -67,7 +68,7 @@ class CryptoEngineClass {
                     throw new Error(`This type doesn't exist`);
             }
         };
-        this._salt = (_a = process.env.REMORA_SALT) !== null && _a !== void 0 ? _a : '';
+        this._salt = (_a = ProcessENVManager_1.default.getEnvVariable('REMORA_SALT')) !== null && _a !== void 0 ? _a : '';
     }
 }
 const CryptoEngine = new CryptoEngineClass();

package/engines/ProcessENVManager.js

@@ -0,0 +1,83 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const process_1 = __importDefault(require("process"));
+const Affirm_1 = __importDefault(require("../core/Affirm"));
+class ProcessENVManagerClass {
+    constructor() {
+        /**
+         * All the variables foundamental for operativity of remora
+         */
+        this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES = ['REMORA_LICENCE_KEY'];
+        /**
+         * All the variables that are not foundamental for the operativity of remora
+         */
+        this.ENVIRONMENT_VARIABLES = ['REMORA_KEY_SECRET_FILE', 'REMORA_WORKERS_PATH', 'ADMIN_JWT_SECRET_FILE', 'JWT_SECRET_FILE', 'ROOT_TEMP_PASSWORD_FILE', 'REMORA_WORKER_HOST', 'MONGO_URI', 'REMORA_SALT', 'OPENAI_API_KEY', 'AWS_DEFAULT_REGION', 'AWS_ACCOUNT_ID', 'AWS_ACCESS_KEY_ID', 'AWS_SESSION_TOKEN', 'NODE_ENV'];
+        /**
+         * get the value of an environment variable and give an error/warn if not implemented.
+         * @param variable the string text of the environment varible desired
+         * @returns the value of environment variable
+         */
+        this.getEnvVariable = (variable) => {
+            (0, Affirm_1.default)(variable, `Cannot read an undefined variable`);
+            if (!this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES.includes(variable) && !this.ENVIRONMENT_VARIABLES.includes(variable)) {
+                console.warn(`Trying to read a variable that is not implemented: ${variable}`);
+            }
+            if (!process_1.default.env[variable] && this.FOUNDAMENTAL_ENVIRONMENT_VARIABLES.includes(variable)) {
+                throw new Error(`Missing necessary property for ${variable}`);
+            }
+            else if (!process_1.default.env[variable]) {
+                // give a warn if it's not neccessary for the operativity of remora
+                console.warn(`Missing property for ${variable}`);
+                return null;
+            }
+            // checks if the property is a path of the docker secrets manager
+            if (process_1.default.env[variable]) {
+                if (process_1.default.env[variable].includes('run/secrets/') && process_1.default.env[variable]) {
+                    return fs.readFileSync(process_1.default.env[variable], 'utf-8');
+                }
+                else {
+                    return process_1.default.env[variable];
+                }
+            }
+        };
+    }
+}
+const ProcessENVManager = new ProcessENVManagerClass();
+exports.default = ProcessENVManager;

package/engines/SecretManager.js

@@ -1,5 +1,9 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+const ProcessENVManager_1 = __importDefault(require("./ProcessENVManager"));
 class SecretManagerClass {
     constructor() {
         /**
@@ -11,7 +15,7 @@ class SecretManagerClass {
             if (!value || value.length <= 2 || !value.startsWith('{') || !value.endsWith('}'))
                 return value;
             const parsedValue = value.slice(1, value.length - 1);
-            return process.env[parsedValue];
+            return ProcessENVManager_1.default.getEnvVariable(parsedValue);
         };
     }
 }

package/engines/ai/LLM.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const openai_1 = __importDefault(require("openai"));
 const zod_1 = require("openai/helpers/zod");
 const zod_2 = require("zod");
+const ProcessENVManager_1 = __importDefault(require("../ProcessENVManager"));
 const baseProducersSystemPrompt = `
 # TASK
 You are an agent tasked with creating the mapping between an INPUT DATA SPEC and one or more OUTPUT DATA SPEC.
@@ -247,7 +248,7 @@ class LLM {
             return finalDraft;
         });
         this._client = new openai_1.default({
-            apiKey: process.env.OPENAI_API_KEY
+            apiKey: ProcessENVManager_1.default.getEnvVariable('OPENAI_API_KEY')
         });
     }
 }

package/engines/parsing/LineParser.js

@@ -23,7 +23,7 @@ class LineParserClass {
                     counter = performance.now();
                     const value = {};
                     for (const dim of dimensions) {
-                        value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.prodDimension.type);
+                        value[dim.name] = TypeCaster_1.default.cast(parts[dim.index], dim.prodDimension.type, dim.prodDimension.format);
                     }
                     tracker.measure('process-line:cast&build-record', performance.now() - counter);
                     return value;

package/engines/parsing/ParseCompression.js

@@ -52,29 +52,42 @@ const path_1 = __importDefault(require("path"));
 const promises_1 = require("stream/promises");
 class ParseCompressionClass {
     constructor() {
-        this.decompressToFile = (compressionType, fileKey, standardPath, localPath) => __awaiter(this, void 0, void 0, function* () {
+        this.decompressToFile = (compressionType, fileKey, standardPath, executionPath) => __awaiter(this, void 0, void 0, function* () {
             // Ensure the directory for the file exists
-            const fileDir = path_1.default.dirname(localPath);
+            const fileDir = path_1.default.dirname(executionPath);
             if (!fs.existsSync(fileDir)) {
                 fs.mkdirSync(fileDir, { recursive: true });
             }
             switch (compressionType === null || compressionType === void 0 ? void 0 : compressionType.toUpperCase()) {
                 case 'ZIP':
-                case 'TAR':
-                    yield (0, decompress_1.default)(path_1.default.join(standardPath, fileKey), localPath);
-                    break;
-                case 'GZIP':
-                    yield this._gzipFile(fileKey, standardPath, localPath);
-                    break;
+                case 'TAR': {
+                    this._warningPossibleConvertionError(compressionType, fileKey);
+                    yield (0, decompress_1.default)(path_1.default.join(standardPath, fileKey), executionPath);
+                    return executionPath;
+                }
+                case 'GZ': {
+                    this._warningPossibleConvertionError(compressionType, fileKey);
+                    yield this._gzipFile(fileKey, standardPath, executionPath);
+                    return executionPath;
+                }
                 case null:
-                case undefined:
+                case undefined: {
+                    if (fileKey.includes('.zip') || fileKey.includes('.tar') || fileKey.includes('.gz'))
+                        console.log(`The file ${fileKey} seems to use a compressionType while there are no producer settings for compressionType `);
                     // No decompression needed for request
-                    break;
-                default:
+                    return path_1.default.join(standardPath, fileKey);
+                }
+                default: {
                     // throw an error if the request ask for an unsupported compression type
                     throw new Error(`this compression type it has not been implemented ${compressionType}`);
+                }
             }
         });
+        this._warningPossibleConvertionError = (compression, fileKey) => {
+            if (!fileKey.split('.').includes(compression.toLowerCase()) && compression) {
+                console.warn(`The compressionType ${compression.toUpperCase()} seems different to the file compression of ${fileKey}`);
+            }
+        };
         this._gzipFile = (fileKey, standardPath, localPath) => __awaiter(this, void 0, void 0, function* () {
             const fileContents = fs.createReadStream(path_1.default.join(standardPath, fileKey));
             const unzip = zlib_1.default.createGunzip();

package/engines/scheduler/QueueManager.js

@@ -14,6 +14,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const Environment_1 = __importDefault(require("../Environment"));
+const ProcessENVManager_1 = __importDefault(require("../ProcessENVManager"));
 const SecretManager_1 = __importDefault(require("../SecretManager"));
 const ExecutorOrchestrator_1 = __importDefault(require("../../executors/ExecutorOrchestrator"));
 const settings_1 = require("../../settings");
@@ -100,8 +101,8 @@ class QueueManager {
         let queueUrl = triggerValue;
         // If it's not a full URL, construct it
         if (!queueUrl.startsWith('https://')) {
-            const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || process.env.AWS_DEFAULT_REGION || 'us-east-1';
-            const accountId = (metadata === null || metadata === void 0 ? void 0 : metadata.accountId) || process.env.AWS_ACCOUNT_ID;
+            const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || ProcessENVManager_1.default.getEnvVariable('AWS_DEFAULT_REGION') || 'us-east-1';
+            const accountId = (metadata === null || metadata === void 0 ? void 0 : metadata.accountId) || ProcessENVManager_1.default.getEnvVariable('AWS_ACCOUNT_ID');
             if (!accountId) {
                 throw new Error('AWS Account ID is required for queue trigger. Set it in metadata.accountId or AWS_ACCOUNT_ID environment variable');
             }
@@ -112,9 +113,9 @@ class QueueManager {
         const region = (metadata === null || metadata === void 0 ? void 0 : metadata.region) || (urlParts === null || urlParts === void 0 ? void 0 : urlParts[1]) || 'us-east-1';
         // Get credentials from metadata or environment
         let credentials;
-        const accessKeyId = (metadata === null || metadata === void 0 ? void 0 : metadata.accessKeyId) || process.env.AWS_ACCESS_KEY_ID;
-        const secretAccessKey = (metadata === null || metadata === void 0 ? void 0 : metadata.secretAccessKey) || process.env.AWS_SECRET_ACCESS_KEY;
-        const sessionToken = (metadata === null || metadata === void 0 ? void 0 : metadata.sessionToken) || process.env.AWS_SESSION_TOKEN;
+        const accessKeyId = (metadata === null || metadata === void 0 ? void 0 : metadata.accessKeyId) || ProcessENVManager_1.default.getEnvVariable('AWS_ACCESS_KEY_ID');
+        const secretAccessKey = (metadata === null || metadata === void 0 ? void 0 : metadata.secretAccessKey) || ProcessENVManager_1.default.getEnvVariable('AWS_SECRET_ACCESS_KEY');
+        const sessionToken = (metadata === null || metadata === void 0 ? void 0 : metadata.sessionToken) || ProcessENVManager_1.default.getEnvVariable('AWS_SESSION_TOKEN');
         if (accessKeyId && secretAccessKey) {
             credentials = {
                 accessKeyId: SecretManager_1.default.replaceSecret(accessKeyId),

package/executors/ExecutorOrchestrator.js

@@ -33,6 +33,7 @@ const ExecutorProgress_1 = __importDefault(require("./ExecutorProgress"));
 const Algo_1 = __importDefault(require("../core/Algo"));
 const ConsumerOnFinishManager_1 = __importDefault(require("../engines/consumer/ConsumerOnFinishManager"));
 const ExecutorScope_1 = __importDefault(require("./ExecutorScope"));
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 class ExecutorOrchestratorClass {
     constructor() {
         this.init = () => {
@@ -75,6 +76,7 @@ class ExecutorOrchestratorClass {
                         else if (cProd.isOptional === true)
                             continue;
                     }
+                    console.log('Starting operations on ', response.files[0].fullUri);
                     // Extract the dimensions for this producer just once
                     const firstLine = (yield DriverHelper_1.default.quickReadFile(response.files[0].fullUri, 1))[0];
                     const header = ProducerExecutor_1.default.processHeader(firstLine, prod);
@@ -234,9 +236,9 @@ class ExecutorOrchestratorClass {
         this._getWorkerPath = () => {
             // Get the current file's directory
             const currentDir = __dirname;
-            if (process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development')
+            if (ProcessENVManager_1.default.getEnvVariable('NODE_ENV') === 'dev' || ProcessENVManager_1.default.getEnvVariable('NODE_ENV') === 'development')
                 return path_1.default.resolve('./.build/workers');
-            const forcedPath = process.env.REMORA_WORKERS_PATH;
+            const forcedPath = ProcessENVManager_1.default.getEnvVariable('REMORA_WORKERS_PATH');
             if (forcedPath && forcedPath.length > 0)
                 return path_1.default.join(__dirname, forcedPath);
             // Check if we're in a published npm package (no .build in path)

package/helper/Helper.js

@@ -32,12 +32,16 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const uuid_1 = require("uuid");
+const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
 const Helper = {
-    isDev: () => process.env.NODE_ENV !== 'production',
+    isDev: () => ProcessENVManager_1.default.getEnvVariable('NODE_ENV') !== 'production',
     uuid: () => (0, uuid_1.v4)(),
     asError: (error) => error instanceof Error ? error : new Error(error),
     readFile: (directory) => {

package/index.js

@@ -20,10 +20,11 @@ const Runtime_1 = __importDefault(require("./helper/Runtime"));
 const automap_1 = require("./actions/automap");
 const sample_1 = require("./actions/sample");
 const mock_1 = require("./actions/mock");
+const ProcessENVManager_1 = __importDefault(require("./engines/ProcessENVManager"));
 dotenv_1.default.configDotenv();
 const program = new commander_1.Command();
 // Validate the remora licence
-const remoraLicenceKey = process.env.REMORA_LICENCE_KEY;
+const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
 const check = LicenceManager_1.default.validate(remoraLicenceKey);
 if (!check.valid) {
     console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.18",
+    "version": "1.0.20",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,