@forzalabs/remora 1.1.9 → 1.1.10

package/index.js

@@ -13259,6 +13259,13 @@ ${stack}` : base;
         stack: extra
       });
     };
+    this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        this._logger.on("finish", resolve);
+        this._logger.end();
+      });
+    };
   }
 };
 var FileLogService = new FileLogServiceClass();
@@ -13309,7 +13316,6 @@ var Logger = class {
       this._fileLoggingFolder = folder;
       this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
-      console.log(`Enabled file logger.`);
     };
     this.getConfig = () => ({
       level: this._level,
@@ -13332,6 +13338,7 @@ var Logger = class {
       console.info(message);
       FileLogService_default.write("INFO", String(message));
     };
+    this.flush = () => FileLogService_default.flush();
     this.error = (error) => {
       let message;
       let stack;
@@ -13473,7 +13480,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.9",
+  cliVersion: "1.1.10",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13842,9 +13849,10 @@ var ValidatorClass = class {
             const validAggregations = ["sum", "count", "avg", "min", "max"];
             if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
               errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const hasWildcardField = consumer.fields.some((x) => x.key === "*");
             const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
             const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
-            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            const missingFields = hasWildcardField ? [] : pivotFields.filter((f) => !allFieldKeys.includes(f));
             if (missingFields.length > 0)
               errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
           }
@@ -20859,6 +20867,7 @@ var ExecutorOrchestratorClass = class {
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        await Logger_default.flush();
         return finalResult;
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
@@ -20867,6 +20876,7 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
+        await Logger_default.flush();
         throw error;
       }
     };
@@ -20960,13 +20970,12 @@ var ExecutorOrchestratorClass = class {
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
-        const output = import_fs14.default.createWriteStream(mainPath);
-        output.setMaxListeners(executorResults.length + 1);
         for (const workerResult of executorResults) {
-          await (0, import_promises10.pipeline)(import_fs14.default.createReadStream(workerResult.resultUri), output, { end: false });
+          await (0, import_promises10.pipeline)(
+            import_fs14.default.createReadStream(workerResult.resultUri),
+            import_fs14.default.createWriteStream(mainPath, { flags: "a" })
+          );
         }
-        output.end();
-        output.close();
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
@@ -21123,15 +21132,18 @@ var run = async (consumerName, options) => {
       return sum + (result.response?.elapsedMS || 0);
     }, 0);
     const totalsInfo = import_chalk6.default.gray(` (${totalRows} rows, ${Helper_default.formatDuration(totalDuration)})`);
-    if (results.some((x) => !x.success))
+    const hasFailures = results.some((x) => !x.success);
+    if (hasFailures)
       console.log(import_chalk6.default.blueBright("\n\u2139\uFE0F Run completed with errors") + totalsInfo);
     else
       console.log(import_chalk6.default.green("\n\u2705 Run complete!") + totalsInfo);
-    process.exit(1);
+    await Logger_default.flush();
+    process.exit(hasFailures ? 1 : 0);
   } catch (err) {
     const myErr = Helper_default.asError(err);
     console.error(import_chalk6.default.red.bold("\n\u274C Unexpected error during run:"), myErr.message);
     Logger_default.error(myErr);
+    await Logger_default.flush();
     process.exit(1);
   }
 };
@@ -21452,8 +21464,10 @@ var mock = async (producerName, records) => {
 
 // src/index.ts
 import_dotenv.default.configDotenv();
-if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true")
+if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true") {
   Logger_default.enableFileLogging("./remora/logs");
+  console.log(`Enabled file logger.`);
+}
 if (!process.env.REMORA_WORKERS_PATH)
   process.env.REMORA_WORKERS_PATH = "./workers";
 var program = new import_commander.Command();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.9",
+    "version": "1.1.10",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13253,6 +13253,13 @@ ${stack}` : base;
         stack: extra
       });
     };
+    this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        this._logger.on("finish", resolve);
+        this._logger.end();
+      });
+    };
   }
 };
 var FileLogService = new FileLogServiceClass();
@@ -13303,7 +13310,6 @@ var Logger = class {
       this._fileLoggingFolder = folder;
       this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
-      console.log(`Enabled file logger.`);
     };
     this.getConfig = () => ({
       level: this._level,
@@ -13326,6 +13332,7 @@ var Logger = class {
       console.info(message);
       FileLogService_default.write("INFO", String(message));
     };
+    this.flush = () => FileLogService_default.flush();
     this.error = (error) => {
       let message;
       let stack;
@@ -13467,7 +13474,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.9",
+  cliVersion: "1.1.10",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13836,9 +13843,10 @@ var ValidatorClass = class {
             const validAggregations = ["sum", "count", "avg", "min", "max"];
             if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
               errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const hasWildcardField = consumer.fields.some((x) => x.key === "*");
             const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
             const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
-            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            const missingFields = hasWildcardField ? [] : pivotFields.filter((f) => !allFieldKeys.includes(f));
             if (missingFields.length > 0)
               errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
           }
@@ -20618,6 +20626,7 @@ var ExecutorOrchestratorClass = class {
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        await Logger_default.flush();
         return finalResult;
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
@@ -20626,6 +20635,7 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
+        await Logger_default.flush();
         throw error;
       }
     };
@@ -20719,13 +20729,12 @@ var ExecutorOrchestratorClass = class {
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
-        const output = import_fs13.default.createWriteStream(mainPath);
-        output.setMaxListeners(executorResults.length + 1);
         for (const workerResult of executorResults) {
-          await (0, import_promises10.pipeline)(import_fs13.default.createReadStream(workerResult.resultUri), output, { end: false });
+          await (0, import_promises10.pipeline)(
+            import_fs13.default.createReadStream(workerResult.resultUri),
+            import_fs13.default.createWriteStream(mainPath, { flags: "a" })
+          );
         }
-        output.end();
-        output.close();
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {