@forzalabs/remora 1.0.16 → 1.0.18

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '1.0.14',
+    cliVersion: '1.0.18',
     backendVersion: 1,
     backendPort: 5088,
     workerVersion: 2,

package/actions/init.js

@@ -64,10 +64,10 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
         console.log(chalk_1.default.blue('• REMORA_WORKER_HOST') + chalk_1.default.gray(' → URL of your Remora Worker instance'));
         console.log(chalk_1.default.blue('• REMORA_LICENCE_KEY') + chalk_1.default.gray(' → Your valid Remora license key'));
         console.log(chalk_1.default.green('\n🔐 Production environment (store these in your Secret Manager):'));
-        console.log(chalk_1.default.yellow('• ROOT_TEMP_PASSWORD') + chalk_1.default.gray(' → Temporary root password for first-time setup'));
+        console.log(chalk_1.default.yellow('• ROOT_TEMP_PASSWORD_FILE') + chalk_1.default.gray(' → Temporary root password for first-time setup'));
         console.log(chalk_1.default.yellow('• INSTALLATION_ID') + chalk_1.default.gray(' → Unique ID for this installation instance'));
-        console.log(chalk_1.default.yellow('• ADMIN_JWT_SECRET') + chalk_1.default.gray(' → JWT secret for admin-level access'));
-        console.log(chalk_1.default.yellow('• JWT_SECRET') + chalk_1.default.gray(' → JWT secret for regular user authentication'));
+        console.log(chalk_1.default.yellow('• ADMIN_JWT_SECRET_FILE') + chalk_1.default.gray(' → JWT secret for admin-level access'));
+        console.log(chalk_1.default.yellow('• JWT_SECRET_FILE') + chalk_1.default.gray(' → JWT secret for regular user authentication'));
     }
     catch (error) {
         console.error(chalk_1.default.red('❌ Error during initialization:'), error instanceof Error ? error.message : String(error));

package/drivers/files/LocalSourceDriver.js

@@ -64,6 +64,8 @@ const Helper_1 = __importDefault(require("../../helper/Helper"));
 const ParseHelper_1 = __importDefault(require("../../engines/parsing/ParseHelper"));
 const Logger_1 = __importDefault(require("../../helper/Logger"));
 const DriverHelper_1 = __importDefault(require("../DriverHelper"));
+const ExecutorScope_1 = __importDefault(require("../../executors/ExecutorScope"));
+const ParseCompression_1 = __importDefault(require("../../engines/parsing/ParseCompression"));
 class LocalSourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
@@ -376,18 +378,26 @@ class LocalSourceDriver {
         };
         this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
             (0, Affirm_1.default)(request, 'Invalid ready request');
-            const { producer } = request;
+            const { producer, scope } = request;
             // TODO: extra logic for encoded files (xml, xls, ...) to be decoded and prepared locally as a plain CSV
             // then return the uri to this new temporary file
-            const { fileKey } = producer.settings;
+            const { fileKey, compressionType } = producer.settings;
             if (fileKey.includes('%')) {
                 const allFileKeys = this.listFiles(fileKey);
-                const allFilePaths = allFileKeys.map(x => path_1.default.join(this._path, x));
+                const allFilePaths = [];
+                for (const fileKey of allFileKeys) {
+                    const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
+                    ExecutorScope_1.default.ensurePath(localPath);
+                    yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, localPath);
+                    allFilePaths.push(localPath);
+                }
                 return { files: allFilePaths.map(x => ({ fullUri: x })) };
             }
             else {
-                const sourceFilePath = path_1.default.join(this._path, fileKey);
-                return { files: [{ fullUri: sourceFilePath }] };
+                const localPath = ExecutorScope_1.default.getProducerPath(scope, producer, fileKey);
+                ExecutorScope_1.default.ensurePath(localPath);
+                yield ParseCompression_1.default.decompressToFile(compressionType, fileKey, this._path, localPath);
+                return { files: [{ fullUri: localPath }] };
             }
         });
     }

package/engines/parsing/ParseCompression.js

@@ -0,0 +1,88 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const decompress_1 = __importDefault(require("decompress"));
+const zlib_1 = __importDefault(require("zlib"));
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("stream/promises");
+class ParseCompressionClass {
+    constructor() {
+        this.decompressToFile = (compressionType, fileKey, standardPath, localPath) => __awaiter(this, void 0, void 0, function* () {
+            // Ensure the directory for the file exists
+            const fileDir = path_1.default.dirname(localPath);
+            if (!fs.existsSync(fileDir)) {
+                fs.mkdirSync(fileDir, { recursive: true });
+            }
+            switch (compressionType === null || compressionType === void 0 ? void 0 : compressionType.toUpperCase()) {
+                case 'ZIP':
+                case 'TAR':
+                    yield (0, decompress_1.default)(path_1.default.join(standardPath, fileKey), localPath);
+                    break;
+                case 'GZIP':
+                    yield this._gzipFile(fileKey, standardPath, localPath);
+                    break;
+                case null:
+                case undefined:
+                    // No decompression needed for request
+                    break;
+                default:
+                    // throw an error if the request ask for an unsupported compression type
+                    throw new Error(`this compression type it has not been implemented ${compressionType}`);
+            }
+        });
+        this._gzipFile = (fileKey, standardPath, localPath) => __awaiter(this, void 0, void 0, function* () {
+            const fileContents = fs.createReadStream(path_1.default.join(standardPath, fileKey));
+            const unzip = zlib_1.default.createGunzip();
+            const writeStream = fs.createWriteStream(localPath);
+            yield (0, promises_1.pipeline)(fileContents, unzip, writeStream);
+        });
+        // TODO add a decompression method fo tarball (.tar.gz)
+    }
+}
+const ParseCompression = new ParseCompressionClass();
+exports.default = ParseCompression;

package/executors/ConsumerExecutor.js

@@ -155,7 +155,9 @@ class ConsumerExecutorClass {
                     reader.destroy();
                 });
             }
-            fs_1.default.renameSync(tempWorkPath, datasetPath);
+            // Delete original file first to avoid EPERM on Windows
+            yield promises_1.default.unlink(datasetPath);
+            yield promises_1.default.rename(tempWorkPath, datasetPath);
             return newLineCount;
         });
         this.processDistinctOn = (consumer, datasetPath) => __awaiter(this, void 0, void 0, function* () {
@@ -217,7 +219,9 @@ class ConsumerExecutorClass {
                     reader.destroy();
                 });
             }
-            fs_1.default.renameSync(tempWorkPath, datasetPath);
+            // Delete original file first to avoid EPERM on Windows
+            yield promises_1.default.unlink(datasetPath);
+            yield promises_1.default.rename(tempWorkPath, datasetPath);
             return winners.size;
         });
         /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.16",
+    "version": "1.0.18",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -46,6 +46,7 @@
         "commander": "^10.0.0",
         "cross-env": "^7.0.3",
         "dayjs": "^1.11.13",
+        "decompress": "^4.2.1",
         "dotenv": "^16.0.3",
         "fast-xml-parser": "^5.2.3",
         "fs-extra": "^11.1.0",