@forzalabs/remora 1.1.7 → 1.1.9

package/index.js

@@ -13333,8 +13333,20 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.error = (error) => {
-      console.error(error);
-      FileLogService_default.write("ERROR", error?.message ?? String(error), error?.stack);
+      let message;
+      let stack;
+      if (error instanceof Error) {
+        message = error.message;
+        stack = error.stack;
+      } else if (typeof error === "string") {
+        message = error;
+      } else {
+        message = String(error);
+      }
+      console.error(message);
+      FileLogService_default.write("ERROR", message, stack);
+      if (!FileLogService_default._enabled && this._level === "debug" && stack)
+        console.error(stack);
     };
     this.formatObject = (obj, depth = 0) => {
       if (obj === null || obj === void 0)
@@ -13461,7 +13473,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.7",
+  cliVersion: "1.1.9",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13802,9 +13814,40 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
+        const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
+        if (duplicateDimensionKeys.length > 0) {
+          const dupes = Algo_default.uniq(duplicateDimensionKeys).map((k) => {
+            const [from, key] = k.split("::");
+            return from === "_default_" ? `"${key}"` : `"${key}" (from: "${from}")`;
+          });
+          errors.push(`Consumer "${consumer.name}" has multiple fields reading from the same producer dimension: ${dupes.join(", ")}. Use "copyFrom" instead to reference the same dimension more than once.`);
+        }
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -18880,7 +18923,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18891,7 +18934,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18933,7 +18976,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18941,6 +18984,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -19625,6 +19682,13 @@ var TransformationEngineClass = class {
         }
         return transformations.conditional.else !== void 0 ? transformations.conditional.else : value;
       }
+      if ("switch_case" in transformations) {
+        const { cases, default: defaultValue } = transformations.switch_case;
+        for (const c of cases) {
+          if (value === c.when) return c.then;
+        }
+        return defaultValue !== void 0 ? defaultValue : value;
+      }
       return value;
     };
     this.evaluateCondition = (value, condition) => {
@@ -20124,6 +20188,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -20246,6 +20312,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs13.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs13.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20562,6 +20735,7 @@ var ExecutorOrchestratorClass = class {
           }
         };
         const workerPath = this._getWorkerPath();
+        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
         this._executorPool = import_workerpool.default.pool(import_path21.default.join(workerPath, "ExecutorWorker.js"), options);
       }
     };
@@ -20578,29 +20752,33 @@ var ExecutorOrchestratorClass = class {
         const start = performance.now();
         this.init();
         const executorResults = [];
-        Logger_default.log(`Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
           throw new Error(`No source data was found for producer ${sourceFilesByProducer[0].prod.name} of consumer ${consumer.name}. Make sure the configuration is correct.`);
-        Logger_default.log(`Consumer "${consumer.name}": ${sourceFilesByProducer.length} producer(s) ready, preparing workers`);
+        Logger_default.log(`[${usageId}] ${sourceFilesByProducer.length} producer(s) ready in ${Math.round(performance.now() - counter)}ms, preparing workers`);
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
           if (!import_fs14.default.existsSync(response.files[0].fullUri)) {
             if (!cProd.isOptional)
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true)
+            else if (cProd.isOptional === true) {
+              Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
+            }
           }
           const firstLine = (await DriverHelper_default.quickReadFile(response.files[0].fullUri, 1))[0];
           const header = ProducerExecutor_default.processHeader(firstLine, prod);
           const prodDimensions = ProducerExecutor_default.reconcileHeader(header, prod);
+          Logger_default.log(`[${usageId}] Producer "${prod.name}": ${prodDimensions.length} dimension(s) reconciled, ${response.files.length} file(s) to process`);
           const totalFiles = response.files.length;
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
             for (const chunk of chunks) {
               const workerId = `${usageId}_${globalWorkerIndex}`;
               const currentWorkerIndex = globalWorkerIndex;
@@ -20617,53 +20795,76 @@ var ExecutorOrchestratorClass = class {
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
+              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
               workerThreads.push(this._executorPool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
+            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
+            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
             await this._executorPool.terminate();
           }
         }
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
+        Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x) => x.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x) => x.outputCount))}`);
         await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+        Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
         const postOperation = { totalOutputCount: null };
         if (executorResults.length > 1) {
           if (consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
           }
           if (consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
         }
+        if (consumer.options?.pivot) {
+          Logger_default.log(`[${usageId}] Running pivot operation`);
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
-        Logger_default.log(`Consumer "${consumer.name}": exporting results`);
+        Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
         tracker.measure("export-result", performance.now() - counter);
+        Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         if (consumer.outputs.some((x) => x.onSuccess)) {
+          Logger_default.log(`[${usageId}] Running on-success actions`);
           counter = performance.now();
           await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId);
           tracker.measure("on-success-actions", performance.now() - counter);
+          Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        Logger_default.log(`[${usageId}] Starting cleanup operations`);
         await this.performCleanupOperations(scope, tracker);
         const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
-        Logger_default.log(`Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
         return finalResult;
       } catch (error) {
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
+        Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
         throw error;
@@ -20732,7 +20933,10 @@ var ExecutorOrchestratorClass = class {
       for (let i = 0; i < consumer.producers.length; i++) {
         const cProd = consumer.producers[i];
         const prod = Environment_default.getProducer(cProd.name);
-        results.push({ prod, cProd, response: await ProducerExecutor_default.ready(prod, scope) });
+        Logger_default.log(`[${scope.id}] Readying producer "${prod.name}" (${i + 1}/${consumer.producers.length})`);
+        const response = await ProducerExecutor_default.ready(prod, scope);
+        Logger_default.log(`[${scope.id}] Producer "${prod.name}" ready: ${response.files.length} file(s)`);
+        results.push({ prod, cProd, response });
       }
       return results;
     };
@@ -20754,6 +20958,7 @@ var ExecutorOrchestratorClass = class {
       ConsumerExecutor_default._ensurePath(mainPath);
       const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
+        Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
         const output = import_fs14.default.createWriteStream(mainPath);
         output.setMaxListeners(executorResults.length + 1);
@@ -20763,12 +20968,16 @@ var ExecutorOrchestratorClass = class {
         output.end();
         output.close();
         tracker.measure("merge-workers", performance.now() - perf);
+        Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
+        Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       if (scope.limitFileSize) {
+        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
         await writer.splitBySize(scope, mainPath);
       }
+      return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();
@@ -20878,7 +21087,11 @@ var run = async (consumerName, options) => {
       } catch (error) {
         const myErr = Helper_default.asError(error);
         results.push({ success: false, consumer, error: myErr.message });
-        if (Helper_default.isDev()) console.log(myErr.stack);
+        const contextualMessage = `Consumer "${consumer.name}" failed: ${myErr.message}`;
+        const contextualError = new Error(contextualMessage);
+        if (myErr.stack)
+          contextualError.stack = myErr.stack;
+        Logger_default.error(contextualError);
       }
     }
     results.forEach(({ response, consumer, success, error }) => {
@@ -20892,9 +21105,16 @@ var run = async (consumerName, options) => {
         else
           console.log(import_chalk6.default.green(`\u2022 Consumer ${consumer.name} `) + performanceInfo);
       } else {
-        console.log(import_chalk6.default.red(`\u2022 Consumer ${consumer.name} -> Failed: ${error}`));
+        console.log(import_chalk6.default.red(`\u2022 Consumer ${consumer.name} -> Failed`));
       }
     });
+    const failedResults = results.filter((x) => !x.success);
+    if (failedResults.length > 0) {
+      console.log(import_chalk6.default.red("\nError details:"));
+      failedResults.forEach((result, index) => {
+        console.log(import_chalk6.default.red(`  ${index + 1}. ${result.consumer.name}: ${result.error}`));
+      });
+    }
     const successfulResults = results.filter((x) => x.success);
     const totalRows = successfulResults.reduce((sum, result) => {
       return sum + (result.response?.outputCount ?? 0);
@@ -20911,7 +21131,7 @@ var run = async (consumerName, options) => {
   } catch (err) {
     const myErr = Helper_default.asError(err);
     console.error(import_chalk6.default.red.bold("\n\u274C Unexpected error during run:"), myErr.message);
-    if (Helper_default.isDev()) console.log(myErr.stack);
+    Logger_default.error(myErr);
     process.exit(1);
   }
 };

package/json_schemas/consumer-schema.json

@@ -166,6 +166,10 @@
                     "fixed": {
                         "type": "boolean",
                         "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                    },
+                    "copyFrom": {
+                        "type": "string",
+                        "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                     }
                 },
                 "required": [
@@ -408,6 +412,46 @@
                     },
                     "required": ["keys", "resolution"],
                     "additionalProperties": false
+                },
+                "pivot": {
+                    "type": "object",
+                    "description": "Performs a pivot operation that transforms row values into columns. Groups data by the specified row keys, takes distinct values from the pivot column and creates a new column for each, aggregating the value column with the specified function. Cannot be used together with 'distinct' or 'distinctOn'.",
+                    "properties": {
+                        "rowKeys": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "minItems": 1,
+                            "description": "The field(s) that identify a row in the pivoted output (the GROUP BY keys). Use the 'alias' if specified."
+                        },
+                        "pivotColumn": {
+                            "type": "string",
+                            "description": "The field whose distinct values become new columns in the output."
+                        },
+                        "valueColumn": {
+                            "type": "string",
+                            "description": "The field whose values are aggregated into each pivot cell."
+                        },
+                        "aggregation": {
+                            "type": "string",
+                            "enum": ["sum", "count", "avg", "min", "max"],
+                            "description": "The aggregation function to apply when combining values."
+                        },
+                        "pivotValues": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "description": "If provided, only these values from the pivot column will be used as output columns. This avoids a discovery pass over the data and makes the output shape statically known. If omitted, the distinct values are discovered automatically."
+                        },
+                        "columnPrefix": {
+                            "type": "string",
+                            "description": "Optional prefix for the generated pivot column names (e.g. 'revenue_' produces 'revenue_East', 'revenue_West')."
+                        }
+                    },
+                    "required": ["rowKeys", "pivotColumn", "valueColumn", "aggregation"],
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false
@@ -519,6 +563,10 @@
                 "fixed": {
                     "type": "boolean",
                     "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                },
+                "copyFrom": {
+                    "type": "string",
+                    "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                 }
             },
             "required": [
@@ -905,6 +953,58 @@
                     "required": ["conditional"],
                     "additionalProperties": false
                 },
+                {
+                    "type": "object",
+                    "description": "Map specific values to other values, similar to a switch/case statement. Matches are checked in order by strict equality. If no case matches, the default value is used (or the original value if no default is specified).",
+                    "properties": {
+                        "switch_case": {
+                            "type": "object",
+                            "properties": {
+                                "cases": {
+                                    "type": "array",
+                                    "description": "Array of when/then pairs evaluated in order. First matching case wins.",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "when": {
+                                                "description": "The value to match against (strict equality)",
+                                                "oneOf": [
+                                                    { "type": "string" },
+                                                    { "type": "number" },
+                                                    { "type": "boolean" }
+                                                ]
+                                            },
+                                            "then": {
+                                                "description": "The value to return if the case matches",
+                                                "oneOf": [
+                                                    { "type": "string" },
+                                                    { "type": "number" },
+                                                    { "type": "boolean" }
+                                                ]
+                                            }
+                                        },
+                                        "required": ["when", "then"],
+                                        "additionalProperties": false
+                                    },
+                                    "minItems": 1
+                                },
+                                "default": {
+                                    "description": "Default value if no case matches. If not specified, the original value is kept.",
+                                    "oneOf": [
+                                        { "type": "string" },
+                                        { "type": "number" },
+                                        { "type": "boolean" },
+                                        { "type": "null" }
+                                    ]
+                                }
+                            },
+                            "required": ["cases"],
+                            "additionalProperties": false
+                        }
+                    },
+                    "required": ["switch_case"],
+                    "additionalProperties": false
+                },
                 {
                     "type": "object",
                     "properties": {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.7",
+    "version": "1.1.9",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13327,8 +13327,20 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.error = (error) => {
-      console.error(error);
-      FileLogService_default.write("ERROR", error?.message ?? String(error), error?.stack);
+      let message;
+      let stack;
+      if (error instanceof Error) {
+        message = error.message;
+        stack = error.stack;
+      } else if (typeof error === "string") {
+        message = error;
+      } else {
+        message = String(error);
+      }
+      console.error(message);
+      FileLogService_default.write("ERROR", message, stack);
+      if (!FileLogService_default._enabled && this._level === "debug" && stack)
+        console.error(stack);
     };
     this.formatObject = (obj, depth = 0) => {
       if (obj === null || obj === void 0)
@@ -13455,7 +13467,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.7",
+  cliVersion: "1.1.9",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13796,9 +13808,40 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
+        const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
+        if (duplicateDimensionKeys.length > 0) {
+          const dupes = Algo_default.uniq(duplicateDimensionKeys).map((k) => {
+            const [from, key] = k.split("::");
+            return from === "_default_" ? `"${key}"` : `"${key}" (from: "${from}")`;
+          });
+          errors.push(`Consumer "${consumer.name}" has multiple fields reading from the same producer dimension: ${dupes.join(", ")}. Use "copyFrom" instead to reference the same dimension more than once.`);
+        }
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -18222,7 +18265,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18233,7 +18276,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18275,7 +18318,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18283,6 +18326,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -18967,6 +19024,13 @@ var TransformationEngineClass = class {
         }
         return transformations.conditional.else !== void 0 ? transformations.conditional.else : value;
       }
+      if ("switch_case" in transformations) {
+        const { cases, default: defaultValue } = transformations.switch_case;
+        for (const c of cases) {
+          if (value === c.when) return c.then;
+        }
+        return defaultValue !== void 0 ? defaultValue : value;
+      }
       return value;
     };
     this.evaluateCondition = (value, condition) => {
@@ -19728,6 +19792,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -19850,6 +19916,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs11.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs11.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20321,6 +20494,7 @@ var ExecutorOrchestratorClass = class {
           }
         };
         const workerPath = this._getWorkerPath();
+        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
         this._executorPool = import_workerpool.default.pool(import_path18.default.join(workerPath, "ExecutorWorker.js"), options);
       }
     };
@@ -20337,29 +20511,33 @@ var ExecutorOrchestratorClass = class {
         const start = performance.now();
         this.init();
         const executorResults = [];
-        Logger_default.log(`Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
         const sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
         tracker.measure("ready-producers", performance.now() - counter);
         if (sourceFilesByProducer.length === 1 && sourceFilesByProducer[0].response?.files.length === 0)
           throw new Error(`No source data was found for producer ${sourceFilesByProducer[0].prod.name} of consumer ${consumer.name}. Make sure the configuration is correct.`);
-        Logger_default.log(`Consumer "${consumer.name}": ${sourceFilesByProducer.length} producer(s) ready, preparing workers`);
+        Logger_default.log(`[${usageId}] ${sourceFilesByProducer.length} producer(s) ready in ${Math.round(performance.now() - counter)}ms, preparing workers`);
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
           if (!import_fs13.default.existsSync(response.files[0].fullUri)) {
             if (!cProd.isOptional)
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true)
+            else if (cProd.isOptional === true) {
+              Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
+            }
           }
           const firstLine = (await DriverHelper_default.quickReadFile(response.files[0].fullUri, 1))[0];
           const header = ProducerExecutor_default.processHeader(firstLine, prod);
           const prodDimensions = ProducerExecutor_default.reconcileHeader(header, prod);
+          Logger_default.log(`[${usageId}] Producer "${prod.name}": ${prodDimensions.length} dimension(s) reconciled, ${response.files.length} file(s) to process`);
           const totalFiles = response.files.length;
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
             for (const chunk of chunks) {
               const workerId = `${usageId}_${globalWorkerIndex}`;
               const currentWorkerIndex = globalWorkerIndex;
@@ -20376,53 +20554,76 @@ var ExecutorOrchestratorClass = class {
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
+              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
               workerThreads.push(this._executorPool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
+            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
+            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
             await this._executorPool.terminate();
           }
         }
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
+        Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x) => x.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x) => x.outputCount))}`);
         await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+        Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
         const postOperation = { totalOutputCount: null };
         if (executorResults.length > 1) {
           if (consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
           }
           if (consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
             const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
         }
+        if (consumer.options?.pivot) {
+          Logger_default.log(`[${usageId}] Running pivot operation`);
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
-        Logger_default.log(`Consumer "${consumer.name}": exporting results`);
+        Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
         tracker.measure("export-result", performance.now() - counter);
+        Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         if (consumer.outputs.some((x) => x.onSuccess)) {
+          Logger_default.log(`[${usageId}] Running on-success actions`);
           counter = performance.now();
           await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId);
           tracker.measure("on-success-actions", performance.now() - counter);
+          Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        Logger_default.log(`[${usageId}] Starting cleanup operations`);
         await this.performCleanupOperations(scope, tracker);
         const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
         UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
-        Logger_default.log(`Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} worker(s), ${Math.round(finalResult.elapsedMS)}ms`);
         return finalResult;
       } catch (error) {
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
+        Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
         UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
         throw error;
@@ -20491,7 +20692,10 @@ var ExecutorOrchestratorClass = class {
       for (let i = 0; i < consumer.producers.length; i++) {
         const cProd = consumer.producers[i];
         const prod = Environment_default.getProducer(cProd.name);
-        results.push({ prod, cProd, response: await ProducerExecutor_default.ready(prod, scope) });
+        Logger_default.log(`[${scope.id}] Readying producer "${prod.name}" (${i + 1}/${consumer.producers.length})`);
+        const response = await ProducerExecutor_default.ready(prod, scope);
+        Logger_default.log(`[${scope.id}] Producer "${prod.name}" ready: ${response.files.length} file(s)`);
+        results.push({ prod, cProd, response });
       }
       return results;
     };
@@ -20513,6 +20717,7 @@ var ExecutorOrchestratorClass = class {
       ConsumerExecutor_default._ensurePath(mainPath);
       const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
+        Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
         const output = import_fs13.default.createWriteStream(mainPath);
         output.setMaxListeners(executorResults.length + 1);
@@ -20522,12 +20727,16 @@ var ExecutorOrchestratorClass = class {
         output.end();
         output.close();
         tracker.measure("merge-workers", performance.now() - perf);
+        Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
+        Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       if (scope.limitFileSize) {
+        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
         await writer.splitBySize(scope, mainPath);
       }
+      return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();