@forzalabs/remora 1.0.14 → 1.0.16

package/database/DatabaseEngine.js

@@ -15,7 +15,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const mongodb_1 = require("mongodb");
 const Helper_1 = __importDefault(require("../helper/Helper"));
 const Settings_1 = __importDefault(require("../helper/Settings"));
-const Affirm_1 = __importDefault(require("../core/Affirm"));
 class DatabaseEngineClass {
     constructor() {
         this.MAX_TRY_CONNECTION = 3;
@@ -23,15 +22,16 @@ class DatabaseEngineClass {
         this.connect = () => __awaiter(this, void 0, void 0, function* () {
             // WARNING: this was changed during the deployment to ECS...
             // I've reverted it, but maybe it needs to be changed or looked into...
-            (0, Affirm_1.default)(process.env.MONGO_USERNAME, 'No environment setting found for MONGO_USERNAME');
-            (0, Affirm_1.default)(process.env.MONGO_PASSWORD_FILE, 'No environment setting found for MONGO_PASSWORD');
-            const mongoUri = 'mongodb://' + process.env.MONGO_USERNAME + ':' + process.env.MONGO_PASSWORD + '@mongo:27017';
-            this._uri = Helper_1.default.isDev() ? 'mongodb://mongo:27017/remora' : mongoUri;
+            var _a;
+            this._uri = ((_a = process.env.MONGO_URI) !== null && _a !== void 0 ? _a : Helper_1.default.isDev())
+                ? 'mongodb://mongo:27017/remora'
+                : 'mongodb://localhost:27017/remora';
+            this._client = new mongodb_1.MongoClient(this._uri);
             const errors = [];
             this._client = new mongodb_1.MongoClient(this._uri);
             for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
                 try {
-                    console.log(`Attempting to connect to mongo`);
+                    console.log(`Attempting to connect to mongo "${this._uri}`);
                     yield this._client.connect();
                     this._db = this._client.db(Settings_1.default.db.name);
                     this._connected = true;

package/engines/Environment.js

@@ -45,7 +45,7 @@ class EnvironmentClass {
                         }
                     }
                     catch (error) {
-                        throw new Error(`Error loading configuration from ${configPath}: ${error.message}`);
+                        throw new Error(`Error loading from ${path_1.default.resolve(remoraPath)} configuration from ${configPath}: ${error.message}`);
                     }
                 }
                 return configs;

package/executors/ConsumerExecutor.js

@@ -141,14 +141,20 @@ class ConsumerExecutorClass {
                 }
                 finally { if (e_1) throw e_1.error; }
             }
-            writer.close();
-            reader.close();
+            lineReader.close();
             // Wait for the writer to finish before renaming
             yield new Promise((resolve, reject) => {
-                writer.on('finish', resolve);
+                writer.on('close', resolve);
                 writer.on('error', reject);
                 writer.end();
             });
+            // Ensure the reader is fully closed before renaming
+            if (!reader.destroyed) {
+                yield new Promise(resolve => {
+                    reader.once('close', resolve);
+                    reader.destroy();
+                });
+            }
             fs_1.default.renameSync(tempWorkPath, datasetPath);
             return newLineCount;
         });
@@ -191,7 +197,7 @@ class ConsumerExecutorClass {
                 }
                 finally { if (e_2) throw e_2.error; }
             }
-            reader.close();
+            lineReader.close();
             // Write the winning records to the temp file
             const tempWorkPath = datasetPath + '_tmp';
             const writer = fs_1.default.createWriteStream(tempWorkPath);
@@ -200,10 +206,17 @@ class ConsumerExecutorClass {
             }
             // Wait for the writer to finish before renaming
             yield new Promise((resolve, reject) => {
-                writer.on('finish', resolve);
+                writer.on('close', resolve);
                 writer.on('error', reject);
                 writer.end();
             });
+            // Ensure the reader is fully closed before renaming
+            if (!reader.destroyed) {
+                yield new Promise(resolve => {
+                    reader.once('close', resolve);
+                    reader.destroy();
+                });
+            }
             fs_1.default.renameSync(tempWorkPath, datasetPath);
             return winners.size;
         });

package/executors/ExecutorOrchestrator.js

@@ -109,6 +109,8 @@ class ExecutorOrchestratorClass {
                     }
                 }
                 _progress.complete();
+                if (executorResults.some(x => !Algo_1.default.hasVal(x)))
+                    throw new Error(`${executorResults.filter(x => !Algo_1.default.hasVal(x)).length} worker(s) failed to produce valid results`);
                 yield this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
                 // If there is more than one worker, then I need to redo the operations that are done on multiple lines (cause now the worker files have been merged together)
                 const postOperation = { totalOutputCount: null };
@@ -119,7 +121,7 @@ class ExecutorOrchestratorClass {
                         tracker.measure('process-distinct:main', performance.now() - counter);
                         postOperation.totalOutputCount = unifiedOutputCount;
                     }
-                    if (((_b = consumer.options) === null || _b === void 0 ? void 0 : _b.distinct) === true) {
+                    if ((_b = consumer.options) === null || _b === void 0 ? void 0 : _b.distinctOn) {
                         counter = performance.now();
                         const unifiedOutputCount = yield ConsumerExecutor_1.default.processDistinctOn(consumer, ExecutorScope_1.default.getMainPath(scope));
                         tracker.measure('process-distinct-on:main', performance.now() - counter);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.0.14",
+    "version": "1.0.16",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,