@forzalabs/remora 1.1.8 → 1.1.10

package/index.js

@@ -13259,6 +13259,13 @@ ${stack}` : base;
         stack: extra
       });
     };
+    this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        this._logger.on("finish", resolve);
+        this._logger.end();
+      });
+    };
   }
 };
 var FileLogService = new FileLogServiceClass();
@@ -13309,7 +13316,6 @@ var Logger = class {
       this._fileLoggingFolder = folder;
       this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
-      console.log(`Enabled file logger.`);
     };
     this.getConfig = () => ({
       level: this._level,
@@ -13332,9 +13338,22 @@ var Logger = class {
       console.info(message);
       FileLogService_default.write("INFO", String(message));
     };
+    this.flush = () => FileLogService_default.flush();
     this.error = (error) => {
-      console.error(error);
-      FileLogService_default.write("ERROR", error?.message ?? String(error), error?.stack);
+      let message;
+      let stack;
+      if (error instanceof Error) {
+        message = error.message;
+        stack = error.stack;
+      } else if (typeof error === "string") {
+        message = error;
+      } else {
+        message = String(error);
+      }
+      console.error(message);
+      FileLogService_default.write("ERROR", message, stack);
+      if (!FileLogService_default._enabled && this._level === "debug" && stack)
+        console.error(stack);
     };
     this.formatObject = (obj, depth = 0) => {
       if (obj === null || obj === void 0)
@@ -13461,7 +13480,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.8",
+  cliVersion: "1.1.10",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13802,6 +13821,16 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
+        const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
+        if (duplicateDimensionKeys.length > 0) {
+          const dupes = Algo_default.uniq(duplicateDimensionKeys).map((k) => {
+            const [from, key] = k.split("::");
+            return from === "_default_" ? `"${key}"` : `"${key}" (from: "${from}")`;
+          });
+          errors.push(`Consumer "${consumer.name}" has multiple fields reading from the same producer dimension: ${dupes.join(", ")}. Use "copyFrom" instead to reference the same dimension more than once.`);
+        }
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
@@ -13820,9 +13849,10 @@ var ValidatorClass = class {
             const validAggregations = ["sum", "count", "avg", "min", "max"];
             if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
               errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const hasWildcardField = consumer.fields.some((x) => x.key === "*");
             const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
             const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
-            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            const missingFields = hasWildcardField ? [] : pivotFields.filter((f) => !allFieldKeys.includes(f));
             if (missingFields.length > 0)
               errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
           }
@@ -19660,6 +19690,13 @@ var TransformationEngineClass = class {
         }
         return transformations.conditional.else !== void 0 ? transformations.conditional.else : value;
       }
+      if ("switch_case" in transformations) {
+        const { cases, default: defaultValue } = transformations.switch_case;
+        for (const c of cases) {
+          if (value === c.when) return c.then;
+        }
+        return defaultValue !== void 0 ? defaultValue : value;
+      }
       return value;
     };
     this.evaluateCondition = (value, condition) => {
@@ -20706,6 +20743,7 @@ var ExecutorOrchestratorClass = class {
           }
         };
         const workerPath = this._getWorkerPath();
+        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
         this._executorPool = import_workerpool.default.pool(import_path21.default.join(workerPath, "ExecutorWorker.js"), options);
       }
     };
@@ -20722,29 +20760,33 @@ var ExecutorOrchestratorClass = class {
         const start = performance.now();
         this.init();
         const executorResults = [];
-        Logger_default.log(`Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
           throw new Error(`No source data was found for producer ${sourceFilesByProducer[0].prod.name} of consumer ${consumer.name}. Make sure the configuration is correct.`);
-        Logger_default.log(`Consumer "${consumer.name}": ${sourceFilesByProducer.length} producer(s) ready, preparing workers`);
+        Logger_default.log(`[${usageId}] ${sourceFilesByProducer.length} producer(s) ready in ${Math.round(performance.now() - counter)}ms, preparing workers`);
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
           if (!import_fs14.default.existsSync(response.files[0].fullUri)) {
             if (!cProd.isOptional)
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true)
+            else if (cProd.isOptional === true) {
+              Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
+            }
           }
           const firstLine = (await DriverHelper_default.quickReadFile(response.files[0].fullUri, 1))[0];
           const header = ProducerExecutor_default.processHeader(firstLine, prod);
           const prodDimensions = ProducerExecutor_default.reconcileHeader(header, prod);
+          Logger_default.log(`[${usageId}] Producer "${prod.name}": ${prodDimensions.length} dimension(s) reconciled, ${response.files.length} file(s) to process`);
           const totalFiles = response.files.length;
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
             for (const chunk of chunks) {
               const workerId = `${usageId}_${globalWorkerIndex}`;
               const currentWorkerIndex = globalWorkerIndex;
@@ -20761,61 +20803,80 @@ var ExecutorOrchestratorClass = class {
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
+              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
               workerThreads.push(this._executorPool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
+            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
+            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
             await this._executorPool.terminate();
           }
         }
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
+        Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x) => x.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x) => x.outputCount))}`);
         await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+        Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
         const postOperation = { totalOutputCount: null };
         if (executorResults.length > 1) {
           if (consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
           }
           if (consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
         }
         if (consumer.options?.pivot) {
+          Logger_default.log(`[${usageId}] Running pivot operation`);
           counter = performance.now();
           const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
           tracker.measure("process-pivot:main", performance.now() - counter);
           postOperation.totalOutputCount = unifiedOutputCount;
+          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
         }
         counter = performance.now();
-        Logger_default.log(`Consumer "${consumer.name}": exporting results`);
+        Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
         tracker.measure("export-result", performance.now() - counter);
+        Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         if (consumer.outputs.some((x) => x.onSuccess)) {
+          Logger_default.log(`[${usageId}] Running on-success actions`);
           counter = performance.now();
           await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId);
           tracker.measure("on-success-actions", performance.now() - counter);
+          Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        Logger_default.log(`[${usageId}] Starting cleanup operations`);
         await this.performCleanupOperations(scope, tracker);
         const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
-        Logger_default.log(`Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        await Logger_default.flush();
         return finalResult;
       } catch (error) {
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
+        Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
+        await Logger_default.flush();
         throw error;
       }
     };
@@ -20882,7 +20943,10 @@ var ExecutorOrchestratorClass = class {
       for (let i = 0; i < consumer.producers.length; i++) {
         const cProd = consumer.producers[i];
         const prod = Environment_default.getProducer(cProd.name);
-        results.push({ prod, cProd, response: await ProducerExecutor_default.ready(prod, scope) });
+        Logger_default.log(`[${scope.id}] Readying producer "${prod.name}" (${i + 1}/${consumer.producers.length})`);
+        const response = await ProducerExecutor_default.ready(prod, scope);
+        Logger_default.log(`[${scope.id}] Producer "${prod.name}" ready: ${response.files.length} file(s)`);
+        results.push({ prod, cProd, response });
       }
       return results;
     };
@@ -20904,21 +20968,25 @@ var ExecutorOrchestratorClass = class {
       ConsumerExecutor_default._ensurePath(mainPath);
       const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
+        Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
-        const output = import_fs14.default.createWriteStream(mainPath);
-        output.setMaxListeners(executorResults.length + 1);
         for (const workerResult of executorResults) {
-          await (0, import_promises10.pipeline)(import_fs14.default.createReadStream(workerResult.resultUri), output, { end: false });
+          await (0, import_promises10.pipeline)(
+            import_fs14.default.createReadStream(workerResult.resultUri),
+            import_fs14.default.createWriteStream(mainPath, { flags: "a" })
+          );
         }
-        output.end();
-        output.close();
         tracker.measure("merge-workers", performance.now() - perf);
+        Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
+        Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       if (scope.limitFileSize) {
+        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
         await writer.splitBySize(scope, mainPath);
       }
+      return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();
@@ -21028,7 +21096,11 @@ var run = async (consumerName, options) => {
       } catch (error) {
         const myErr = Helper_default.asError(error);
         results.push({ success: false, consumer, error: myErr.message });
-        if (Helper_default.isDev()) console.log(myErr.stack);
+        const contextualMessage = `Consumer "${consumer.name}" failed: ${myErr.message}`;
+        const contextualError = new Error(contextualMessage);
+        if (myErr.stack)
+          contextualError.stack = myErr.stack;
+        Logger_default.error(contextualError);
       }
     }
     results.forEach(({ response, consumer, success, error }) => {
@@ -21042,9 +21114,16 @@ var run = async (consumerName, options) => {
         else
           console.log(import_chalk6.default.green(`\u2022 Consumer ${consumer.name} `) + performanceInfo);
       } else {
-        console.log(import_chalk6.default.red(`\u2022 Consumer ${consumer.name} -> Failed: ${error}`));
+        console.log(import_chalk6.default.red(`\u2022 Consumer ${consumer.name} -> Failed`));
       }
     });
+    const failedResults = results.filter((x) => !x.success);
+    if (failedResults.length > 0) {
+      console.log(import_chalk6.default.red("\nError details:"));
+      failedResults.forEach((result, index) => {
+        console.log(import_chalk6.default.red(`  ${index + 1}. ${result.consumer.name}: ${result.error}`));
+      });
+    }
     const successfulResults = results.filter((x) => x.success);
     const totalRows = successfulResults.reduce((sum, result) => {
       return sum + (result.response?.outputCount ?? 0);
@@ -21053,15 +21132,18 @@ var run = async (consumerName, options) => {
       return sum + (result.response?.elapsedMS || 0);
     }, 0);
     const totalsInfo = import_chalk6.default.gray(` (${totalRows} rows, ${Helper_default.formatDuration(totalDuration)})`);
-    if (results.some((x) => !x.success))
+    const hasFailures = results.some((x) => !x.success);
+    if (hasFailures)
       console.log(import_chalk6.default.blueBright("\n\u2139\uFE0F Run completed with errors") + totalsInfo);
     else
       console.log(import_chalk6.default.green("\n\u2705 Run complete!") + totalsInfo);
-    process.exit(1);
+    await Logger_default.flush();
+    process.exit(hasFailures ? 1 : 0);
   } catch (err) {
     const myErr = Helper_default.asError(err);
     console.error(import_chalk6.default.red.bold("\n\u274C Unexpected error during run:"), myErr.message);
-    if (Helper_default.isDev()) console.log(myErr.stack);
+    Logger_default.error(myErr);
+    await Logger_default.flush();
     process.exit(1);
   }
 };
@@ -21382,8 +21464,10 @@ var mock = async (producerName, records) => {
 
 // src/index.ts
 import_dotenv.default.configDotenv();
-if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true")
+if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true") {
   Logger_default.enableFileLogging("./remora/logs");
+  console.log(`Enabled file logger.`);
+}
 if (!process.env.REMORA_WORKERS_PATH)
   process.env.REMORA_WORKERS_PATH = "./workers";
 var program = new import_commander.Command();

package/json_schemas/consumer-schema.json

@@ -953,6 +953,58 @@
                     "required": ["conditional"],
                     "additionalProperties": false
                 },
+                {
+                    "type": "object",
+                    "description": "Map specific values to other values, similar to a switch/case statement. Matches are checked in order by strict equality. If no case matches, the default value is used (or the original value if no default is specified).",
+                    "properties": {
+                        "switch_case": {
+                            "type": "object",
+                            "properties": {
+                                "cases": {
+                                    "type": "array",
+                                    "description": "Array of when/then pairs evaluated in order. First matching case wins.",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "when": {
+                                                "description": "The value to match against (strict equality)",
+                                                "oneOf": [
+                                                    { "type": "string" },
+                                                    { "type": "number" },
+                                                    { "type": "boolean" }
+                                                ]
+                                            },
+                                            "then": {
+                                                "description": "The value to return if the case matches",
+                                                "oneOf": [
+                                                    { "type": "string" },
+                                                    { "type": "number" },
+                                                    { "type": "boolean" }
+                                                ]
+                                            }
+                                        },
+                                        "required": ["when", "then"],
+                                        "additionalProperties": false
+                                    },
+                                    "minItems": 1
+                                },
+                                "default": {
+                                    "description": "Default value if no case matches. If not specified, the original value is kept.",
+                                    "oneOf": [
+                                        { "type": "string" },
+                                        { "type": "number" },
+                                        { "type": "boolean" },
+                                        { "type": "null" }
+                                    ]
+                                }
+                            },
+                            "required": ["cases"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["switch_case"],
+                    "additionalProperties": false
+                },
                 {
                     "type": "object",
                     "properties": {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.8",
+    "version": "1.1.10",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13253,6 +13253,13 @@ ${stack}` : base;
         stack: extra
       });
     };
+    this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        this._logger.on("finish", resolve);
+        this._logger.end();
+      });
+    };
   }
 };
 var FileLogService = new FileLogServiceClass();
@@ -13303,7 +13310,6 @@ var Logger = class {
       this._fileLoggingFolder = folder;
       this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
-      console.log(`Enabled file logger.`);
     };
     this.getConfig = () => ({
       level: this._level,
@@ -13326,9 +13332,22 @@ var Logger = class {
       console.info(message);
       FileLogService_default.write("INFO", String(message));
     };
+    this.flush = () => FileLogService_default.flush();
     this.error = (error) => {
-      console.error(error);
-      FileLogService_default.write("ERROR", error?.message ?? String(error), error?.stack);
+      let message;
+      let stack;
+      if (error instanceof Error) {
+        message = error.message;
+        stack = error.stack;
+      } else if (typeof error === "string") {
+        message = error;
+      } else {
+        message = String(error);
+      }
+      console.error(message);
+      FileLogService_default.write("ERROR", message, stack);
+      if (!FileLogService_default._enabled && this._level === "debug" && stack)
+        console.error(stack);
     };
     this.formatObject = (obj, depth = 0) => {
       if (obj === null || obj === void 0)
@@ -13455,7 +13474,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.8",
+  cliVersion: "1.1.10",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13796,6 +13815,16 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
+        const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
+        if (duplicateDimensionKeys.length > 0) {
+          const dupes = Algo_default.uniq(duplicateDimensionKeys).map((k) => {
+            const [from, key] = k.split("::");
+            return from === "_default_" ? `"${key}"` : `"${key}" (from: "${from}")`;
+          });
+          errors.push(`Consumer "${consumer.name}" has multiple fields reading from the same producer dimension: ${dupes.join(", ")}. Use "copyFrom" instead to reference the same dimension more than once.`);
+        }
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
@@ -13814,9 +13843,10 @@ var ValidatorClass = class {
             const validAggregations = ["sum", "count", "avg", "min", "max"];
             if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
               errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const hasWildcardField = consumer.fields.some((x) => x.key === "*");
             const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
             const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
-            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            const missingFields = hasWildcardField ? [] : pivotFields.filter((f) => !allFieldKeys.includes(f));
             if (missingFields.length > 0)
               errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
           }
@@ -19002,6 +19032,13 @@ var TransformationEngineClass = class {
         }
         return transformations.conditional.else !== void 0 ? transformations.conditional.else : value;
       }
+      if ("switch_case" in transformations) {
+        const { cases, default: defaultValue } = transformations.switch_case;
+        for (const c of cases) {
+          if (value === c.when) return c.then;
+        }
+        return defaultValue !== void 0 ? defaultValue : value;
+      }
       return value;
     };
     this.evaluateCondition = (value, condition) => {
@@ -20465,6 +20502,7 @@ var ExecutorOrchestratorClass = class {
           }
         };
         const workerPath = this._getWorkerPath();
+        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
         this._executorPool = import_workerpool.default.pool(import_path18.default.join(workerPath, "ExecutorWorker.js"), options);
       }
     };
@@ -20481,29 +20519,33 @@ var ExecutorOrchestratorClass = class {
         const start = performance.now();
         this.init();
         const executorResults = [];
-        Logger_default.log(`Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
           throw new Error(`No source data was found for producer ${sourceFilesByProducer[0].prod.name} of consumer ${consumer.name}. Make sure the configuration is correct.`);
-        Logger_default.log(`Consumer "${consumer.name}": ${sourceFilesByProducer.length} producer(s) ready, preparing workers`);
+        Logger_default.log(`[${usageId}] ${sourceFilesByProducer.length} producer(s) ready in ${Math.round(performance.now() - counter)}ms, preparing workers`);
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
           if (!import_fs13.default.existsSync(response.files[0].fullUri)) {
             if (!cProd.isOptional)
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true)
+            else if (cProd.isOptional === true) {
+              Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
+            }
           }
           const firstLine = (await DriverHelper_default.quickReadFile(response.files[0].fullUri, 1))[0];
           const header = ProducerExecutor_default.processHeader(firstLine, prod);
           const prodDimensions = ProducerExecutor_default.reconcileHeader(header, prod);
+          Logger_default.log(`[${usageId}] Producer "${prod.name}": ${prodDimensions.length} dimension(s) reconciled, ${response.files.length} file(s) to process`);
           const totalFiles = response.files.length;
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
             for (const chunk of chunks) {
               const workerId = `${usageId}_${globalWorkerIndex}`;
               const currentWorkerIndex = globalWorkerIndex;
@@ -20520,61 +20562,80 @@ var ExecutorOrchestratorClass = class {
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
+              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
               workerThreads.push(this._executorPool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
+            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
+            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
             await this._executorPool.terminate();
           }
         }
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
+        Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x) => x.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x) => x.outputCount))}`);
         await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+        Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
         const postOperation = { totalOutputCount: null };
         if (executorResults.length > 1) {
           if (consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
           }
           if (consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
         }
         if (consumer.options?.pivot) {
+          Logger_default.log(`[${usageId}] Running pivot operation`);
           counter = performance.now();
           const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
           tracker.measure("process-pivot:main", performance.now() - counter);
           postOperation.totalOutputCount = unifiedOutputCount;
+          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
         }
         counter = performance.now();
-        Logger_default.log(`Consumer "${consumer.name}": exporting results`);
+        Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
         tracker.measure("export-result", performance.now() - counter);
+        Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         if (consumer.outputs.some((x) => x.onSuccess)) {
+          Logger_default.log(`[${usageId}] Running on-success actions`);
           counter = performance.now();
           await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId);
           tracker.measure("on-success-actions", performance.now() - counter);
+          Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        Logger_default.log(`[${usageId}] Starting cleanup operations`);
         await this.performCleanupOperations(scope, tracker);
         const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
-        Logger_default.log(`Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        await Logger_default.flush();
         return finalResult;
       } catch (error) {
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
+        Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
+        await Logger_default.flush();
         throw error;
       }
     };
@@ -20641,7 +20702,10 @@ var ExecutorOrchestratorClass = class {
       for (let i = 0; i < consumer.producers.length; i++) {
         const cProd = consumer.producers[i];
         const prod = Environment_default.getProducer(cProd.name);
-        results.push({ prod, cProd, response: await ProducerExecutor_default.ready(prod, scope) });
+        Logger_default.log(`[${scope.id}] Readying producer "${prod.name}" (${i + 1}/${consumer.producers.length})`);
+        const response = await ProducerExecutor_default.ready(prod, scope);
+        Logger_default.log(`[${scope.id}] Producer "${prod.name}" ready: ${response.files.length} file(s)`);
+        results.push({ prod, cProd, response });
       }
       return results;
     };
@@ -20663,21 +20727,25 @@ var ExecutorOrchestratorClass = class {
       ConsumerExecutor_default._ensurePath(mainPath);
       const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
+        Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
-        const output = import_fs13.default.createWriteStream(mainPath);
-        output.setMaxListeners(executorResults.length + 1);
         for (const workerResult of executorResults) {
-          await (0, import_promises10.pipeline)(import_fs13.default.createReadStream(workerResult.resultUri), output, { end: false });
+          await (0, import_promises10.pipeline)(
+            import_fs13.default.createReadStream(workerResult.resultUri),
+            import_fs13.default.createWriteStream(mainPath, { flags: "a" })
+          );
         }
-        output.end();
-        output.close();
         tracker.measure("merge-workers", performance.now() - perf);
+        Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
+        Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       if (scope.limitFileSize) {
+        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
         await writer.splitBySize(scope, mainPath);
       }
+      return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();