@forzalabs/remora 1.1.6 → 1.1.8

package/index.js

@@ -13306,14 +13306,27 @@ var Logger = class {
   constructor() {
     this.setLevel = (level) => this._level = level;
     this.enableFileLogging = (folder, file) => {
+      this._fileLoggingFolder = folder;
+      this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
       console.log(`Enabled file logger.`);
     };
+    this.getConfig = () => ({
+      level: this._level,
+      fileLogging: FileLogService_default._enabled ? { folder: this._fileLoggingFolder, file: this._fileLoggingFile } : void 0
+    });
+    this.initFromConfig = (config) => {
+      this._level = config.level;
+      if (config.fileLogging)
+        this.enableFileLogging(config.fileLogging.folder, config.fileLogging.file);
+    };
     this.log = (message, level) => {
       const myLevel = level ?? this._level;
       if (myLevel !== "debug") return;
-      console.log(import_chalk.default.cyanBright("DEBUG"), message);
-      FileLogService_default.write("DEBUG", String(message));
+      if (FileLogService_default._enabled)
+        FileLogService_default.write("DEBUG", String(message));
+      else
+        console.log(import_chalk.default.cyanBright("DEBUG"), message);
     };
     this.info = (message) => {
       console.info(message);
@@ -13448,7 +13461,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.6",
+  cliVersion: "1.1.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13792,6 +13805,27 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -18646,85 +18680,73 @@ var DeveloperEngineClass = class {
         const record = {};
         for (const dimension of dimensions) {
           if (dimension.sourceFilename) continue;
-          record[dimension.name] = this.generateMockValue(dimension, i);
+          const key = dimension.alias || dimension.name;
+          record[key] = this.generateMockValue(dimension, i);
         }
         records.push(record);
       }
       return records;
     };
     this.generateMockValue = (dimension, index) => {
-      const { name, type } = dimension;
+      const { name, type, format: format2, pk } = dimension;
       const nameLower = name.toLowerCase();
-      if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"])) {
+      if (pk) {
+        if (type === "number") return index + 1;
         return `${index + 1}`;
       }
-      if (this.matchesPattern(nameLower, ["first_name", "firstname", "fname", "given_name"])) {
-        return this.pickRandom(["John", "Jane", "Michael", "Sarah", "David", "Emily", "Robert", "Lisa", "James", "Mary"]);
-      }
-      if (this.matchesPattern(nameLower, ["last_name", "lastname", "lname", "surname", "family_name"])) {
-        return this.pickRandom(["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Martinez", "Wilson"]);
-      }
-      if (this.matchesPattern(nameLower, ["name", "full_name", "fullname"])) {
-        const firstNames = ["John", "Jane", "Michael", "Sarah", "David", "Emily"];
-        const lastNames = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia"];
-        return `${this.pickRandom(firstNames)} ${this.pickRandom(lastNames)}`;
-      }
-      if (this.matchesPattern(nameLower, ["email", "mail"])) {
-        return `user${index + 1}@example.com`;
-      }
-      if (this.matchesPattern(nameLower, ["phone", "telephone", "mobile", "cell"])) {
-        return `555-${String(Math.floor(Math.random() * 900) + 100).padStart(3, "0")}-${String(Math.floor(Math.random() * 9e3) + 1e3).padStart(4, "0")}`;
-      }
-      if (this.matchesPattern(nameLower, ["address", "street", "addr"])) {
-        const streets = ["Main St", "Oak Ave", "Elm Dr", "Pine Rd", "Maple Ln", "Cedar Blvd"];
-        return `${Math.floor(Math.random() * 9999) + 1} ${this.pickRandom(streets)}`;
-      }
-      if (this.matchesPattern(nameLower, ["city", "town"])) {
-        return this.pickRandom(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego"]);
-      }
-      if (this.matchesPattern(nameLower, ["state", "province"])) {
-        return this.pickRandom(["CA", "TX", "FL", "NY", "PA", "IL", "OH", "GA", "NC", "MI"]);
-      }
-      if (this.matchesPattern(nameLower, ["zip", "postal", "zipcode"])) {
-        return String(Math.floor(Math.random() * 9e4) + 1e4);
-      }
-      if (this.matchesPattern(nameLower, ["country"])) {
-        return this.pickRandom(["USA", "Canada", "UK", "Germany", "France", "Australia"]);
-      }
-      if (this.matchesPattern(nameLower, ["age", "years"])) {
-        return Math.floor(Math.random() * 80) + 18;
-      }
-      if (this.matchesPattern(nameLower, ["sex", "gender"])) {
-        return this.pickRandom(["M", "F", "Male", "Female"]);
-      }
-      if (this.matchesPattern(nameLower, ["birth", "dob", "birthdate"])) {
-        const year = Math.floor(Math.random() * 60) + 1940;
-        const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-        const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-        return `${year}-${month}-${day}`;
-      }
-      if (this.matchesPattern(nameLower, ["date", "created", "updated", "timestamp"])) {
-        const year = Math.floor(Math.random() * 5) + 2020;
-        const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-        const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-        return `${year}-${month}-${day}`;
-      }
-      if (this.matchesPattern(nameLower, ["amount", "price", "cost", "total", "balance"])) {
-        return (Math.random() * 1e3).toFixed(2);
+      const contextual = this.generateContextualValue(nameLower, type, index);
+      if (contextual !== void 0) return contextual;
+      return this.generateValueByType(type, index, format2);
+    };
+    this.generateContextualValue = (nameLower, type, index) => {
+      if (type === "string") {
+        if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"]))
+          return `${index + 1}`;
+        if (this.matchesPattern(nameLower, ["first_name", "firstname", "fname", "given_name"]))
+          return this.pickRandom(["John", "Jane", "Michael", "Sarah", "David", "Emily", "Robert", "Lisa", "James", "Mary"]);
+        if (this.matchesPattern(nameLower, ["last_name", "lastname", "lname", "surname", "family_name"]))
+          return this.pickRandom(["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Martinez", "Wilson"]);
+        if (this.matchesPattern(nameLower, ["name", "full_name", "fullname"])) {
+          const firstNames = ["John", "Jane", "Michael", "Sarah", "David", "Emily"];
+          const lastNames = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia"];
+          return `${this.pickRandom(firstNames)} ${this.pickRandom(lastNames)}`;
+        }
+        if (this.matchesPattern(nameLower, ["email", "mail"]))
+          return `user${index + 1}@example.com`;
+        if (this.matchesPattern(nameLower, ["phone", "telephone", "mobile", "cell"]))
+          return `555-${String(Math.floor(Math.random() * 900) + 100).padStart(3, "0")}-${String(Math.floor(Math.random() * 9e3) + 1e3).padStart(4, "0")}`;
+        if (this.matchesPattern(nameLower, ["address", "street", "addr"])) {
+          const streets = ["Main St", "Oak Ave", "Elm Dr", "Pine Rd", "Maple Ln", "Cedar Blvd"];
+          return `${Math.floor(Math.random() * 9999) + 1} ${this.pickRandom(streets)}`;
+        }
+        if (this.matchesPattern(nameLower, ["city", "town"]))
+          return this.pickRandom(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego"]);
+        if (this.matchesPattern(nameLower, ["state", "province"]))
+          return this.pickRandom(["CA", "TX", "FL", "NY", "PA", "IL", "OH", "GA", "NC", "MI"]);
+        if (this.matchesPattern(nameLower, ["zip", "postal", "zipcode"]))
+          return String(Math.floor(Math.random() * 9e4) + 1e4);
+        if (this.matchesPattern(nameLower, ["country"]))
+          return this.pickRandom(["USA", "Canada", "UK", "Germany", "France", "Australia"]);
+        if (this.matchesPattern(nameLower, ["sex", "gender"]))
+          return this.pickRandom(["M", "F", "Male", "Female"]);
+        if (this.matchesPattern(nameLower, ["status"]))
+          return this.pickRandom(["active", "inactive", "pending", "completed", "cancelled"]);
+        if (this.matchesPattern(nameLower, ["type", "category"]))
+          return this.pickRandom(["TypeA", "TypeB", "TypeC", "TypeD"]);
+        if (this.matchesPattern(nameLower, ["description", "desc", "notes", "comment"]))
+          return `Sample description for record ${index + 1}`;
+      }
+      if (type === "number") {
+        if (this.matchesPattern(nameLower, ["age", "years"]))
+          return Math.floor(Math.random() * 80) + 18;
+        if (this.matchesPattern(nameLower, ["amount", "price", "cost", "total", "balance"]))
+          return parseFloat((Math.random() * 1e3).toFixed(2));
+        if (this.matchesPattern(nameLower, ["quantity", "count", "qty"]))
+          return Math.floor(Math.random() * 100) + 1;
+        if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"]))
+          return index + 1;
       }
-      if (this.matchesPattern(nameLower, ["quantity", "count", "qty"])) {
-        return Math.floor(Math.random() * 100) + 1;
-      }
-      if (this.matchesPattern(nameLower, ["status"])) {
-        return this.pickRandom(["active", "inactive", "pending", "completed", "cancelled"]);
-      }
-      if (this.matchesPattern(nameLower, ["type", "category"])) {
-        return this.pickRandom(["TypeA", "TypeB", "TypeC", "TypeD"]);
-      }
-      if (this.matchesPattern(nameLower, ["description", "desc", "notes", "comment"])) {
-        return `Sample description for record ${index + 1}`;
-      }
-      return this.generateValueByType(type, index);
+      return void 0;
     };
     this.matchesPattern = (fieldName, patterns) => {
       return patterns.some((pattern) => fieldName.includes(pattern));
@@ -18732,7 +18754,7 @@ var DeveloperEngineClass = class {
     this.pickRandom = (arr) => {
       return arr[Math.floor(Math.random() * arr.length)];
     };
-    this.generateValueByType = (type, index) => {
+    this.generateValueByType = (type, index, format2) => {
       switch (type) {
         case "string":
           return `value_${index + 1}`;
@@ -18742,9 +18764,14 @@ var DeveloperEngineClass = class {
           return Math.random() > 0.5;
         case "datetime": {
           const year = Math.floor(Math.random() * 5) + 2020;
-          const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-          const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-          return `${year}-${month}-${day}`;
+          const month = Math.floor(Math.random() * 12) + 1;
+          const day = Math.floor(Math.random() * 28) + 1;
+          const hour = Math.floor(Math.random() * 24);
+          const minute = Math.floor(Math.random() * 60);
+          const second = Math.floor(Math.random() * 60);
+          const date = (0, import_dayjs2.default)(new Date(year, month - 1, day, hour, minute, second));
+          if (format2) return date.format(format2);
+          return date.format("YYYY-MM-DD");
         }
         default:
           return `value_${index + 1}`;
@@ -18874,7 +18901,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18885,7 +18912,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18927,7 +18954,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18935,6 +18962,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -20118,6 +20159,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -20240,6 +20283,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs13.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs13.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20606,7 +20756,8 @@ var ExecutorOrchestratorClass = class {
                 prodDimensions,
                 workerId,
                 scope,
-                options
+                options,
+                loggerConfig: Logger_default.getConfig()
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
@@ -20637,6 +20788,12 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
           }
         }
+        if (consumer.options?.pivot) {
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+        }
         counter = performance.now();
         Logger_default.log(`Consumer "${consumer.name}": exporting results`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);

package/json_schemas/consumer-schema.json

@@ -166,6 +166,10 @@
                     "fixed": {
                         "type": "boolean",
                         "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                    },
+                    "copyFrom": {
+                        "type": "string",
+                        "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                     }
                 },
                 "required": [
@@ -408,6 +412,46 @@
                     },
                     "required": ["keys", "resolution"],
                     "additionalProperties": false
+                },
+                "pivot": {
+                    "type": "object",
+                    "description": "Performs a pivot operation that transforms row values into columns. Groups data by the specified row keys, takes distinct values from the pivot column and creates a new column for each, aggregating the value column with the specified function. Cannot be used together with 'distinct' or 'distinctOn'.",
+                    "properties": {
+                        "rowKeys": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "minItems": 1,
+                            "description": "The field(s) that identify a row in the pivoted output (the GROUP BY keys). Use the 'alias' if specified."
+                        },
+                        "pivotColumn": {
+                            "type": "string",
+                            "description": "The field whose distinct values become new columns in the output."
+                        },
+                        "valueColumn": {
+                            "type": "string",
+                            "description": "The field whose values are aggregated into each pivot cell."
+                        },
+                        "aggregation": {
+                            "type": "string",
+                            "enum": ["sum", "count", "avg", "min", "max"],
+                            "description": "The aggregation function to apply when combining values."
+                        },
+                        "pivotValues": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "description": "If provided, only these values from the pivot column will be used as output columns. This avoids a discovery pass over the data and makes the output shape statically known. If omitted, the distinct values are discovered automatically."
+                        },
+                        "columnPrefix": {
+                            "type": "string",
+                            "description": "Optional prefix for the generated pivot column names (e.g. 'revenue_' produces 'revenue_East', 'revenue_West')."
+                        }
+                    },
+                    "required": ["rowKeys", "pivotColumn", "valueColumn", "aggregation"],
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false
@@ -519,6 +563,10 @@
                 "fixed": {
                     "type": "boolean",
                     "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                },
+                "copyFrom": {
+                    "type": "string",
+                    "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                 }
             },
             "required": [

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.6",
+    "version": "1.1.8",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13300,14 +13300,27 @@ var Logger = class {
   constructor() {
     this.setLevel = (level) => this._level = level;
     this.enableFileLogging = (folder, file) => {
+      this._fileLoggingFolder = folder;
+      this._fileLoggingFile = file;
       FileLogService_default.enable(folder, file);
       console.log(`Enabled file logger.`);
     };
+    this.getConfig = () => ({
+      level: this._level,
+      fileLogging: FileLogService_default._enabled ? { folder: this._fileLoggingFolder, file: this._fileLoggingFile } : void 0
+    });
+    this.initFromConfig = (config) => {
+      this._level = config.level;
+      if (config.fileLogging)
+        this.enableFileLogging(config.fileLogging.folder, config.fileLogging.file);
+    };
     this.log = (message, level) => {
       const myLevel = level ?? this._level;
       if (myLevel !== "debug") return;
-      console.log(import_chalk.default.cyanBright("DEBUG"), message);
-      FileLogService_default.write("DEBUG", String(message));
+      if (FileLogService_default._enabled)
+        FileLogService_default.write("DEBUG", String(message));
+      else
+        console.log(import_chalk.default.cyanBright("DEBUG"), message);
     };
     this.info = (message) => {
       console.info(message);
@@ -13442,7 +13455,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.6",
+  cliVersion: "1.1.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13786,6 +13799,27 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -17988,85 +18022,73 @@ var DeveloperEngineClass = class {
         const record = {};
         for (const dimension of dimensions) {
           if (dimension.sourceFilename) continue;
-          record[dimension.name] = this.generateMockValue(dimension, i);
+          const key = dimension.alias || dimension.name;
+          record[key] = this.generateMockValue(dimension, i);
         }
         records.push(record);
       }
       return records;
     };
     this.generateMockValue = (dimension, index) => {
-      const { name, type } = dimension;
+      const { name, type, format: format2, pk } = dimension;
       const nameLower = name.toLowerCase();
-      if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"])) {
+      if (pk) {
+        if (type === "number") return index + 1;
         return `${index + 1}`;
       }
-      if (this.matchesPattern(nameLower, ["first_name", "firstname", "fname", "given_name"])) {
-        return this.pickRandom(["John", "Jane", "Michael", "Sarah", "David", "Emily", "Robert", "Lisa", "James", "Mary"]);
-      }
-      if (this.matchesPattern(nameLower, ["last_name", "lastname", "lname", "surname", "family_name"])) {
-        return this.pickRandom(["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Martinez", "Wilson"]);
-      }
-      if (this.matchesPattern(nameLower, ["name", "full_name", "fullname"])) {
-        const firstNames = ["John", "Jane", "Michael", "Sarah", "David", "Emily"];
-        const lastNames = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia"];
-        return `${this.pickRandom(firstNames)} ${this.pickRandom(lastNames)}`;
-      }
-      if (this.matchesPattern(nameLower, ["email", "mail"])) {
-        return `user${index + 1}@example.com`;
-      }
-      if (this.matchesPattern(nameLower, ["phone", "telephone", "mobile", "cell"])) {
-        return `555-${String(Math.floor(Math.random() * 900) + 100).padStart(3, "0")}-${String(Math.floor(Math.random() * 9e3) + 1e3).padStart(4, "0")}`;
-      }
-      if (this.matchesPattern(nameLower, ["address", "street", "addr"])) {
-        const streets = ["Main St", "Oak Ave", "Elm Dr", "Pine Rd", "Maple Ln", "Cedar Blvd"];
-        return `${Math.floor(Math.random() * 9999) + 1} ${this.pickRandom(streets)}`;
-      }
-      if (this.matchesPattern(nameLower, ["city", "town"])) {
-        return this.pickRandom(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego"]);
-      }
-      if (this.matchesPattern(nameLower, ["state", "province"])) {
-        return this.pickRandom(["CA", "TX", "FL", "NY", "PA", "IL", "OH", "GA", "NC", "MI"]);
-      }
-      if (this.matchesPattern(nameLower, ["zip", "postal", "zipcode"])) {
-        return String(Math.floor(Math.random() * 9e4) + 1e4);
-      }
-      if (this.matchesPattern(nameLower, ["country"])) {
-        return this.pickRandom(["USA", "Canada", "UK", "Germany", "France", "Australia"]);
-      }
-      if (this.matchesPattern(nameLower, ["age", "years"])) {
-        return Math.floor(Math.random() * 80) + 18;
-      }
-      if (this.matchesPattern(nameLower, ["sex", "gender"])) {
-        return this.pickRandom(["M", "F", "Male", "Female"]);
-      }
-      if (this.matchesPattern(nameLower, ["birth", "dob", "birthdate"])) {
-        const year = Math.floor(Math.random() * 60) + 1940;
-        const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-        const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-        return `${year}-${month}-${day}`;
-      }
-      if (this.matchesPattern(nameLower, ["date", "created", "updated", "timestamp"])) {
-        const year = Math.floor(Math.random() * 5) + 2020;
-        const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-        const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-        return `${year}-${month}-${day}`;
-      }
-      if (this.matchesPattern(nameLower, ["amount", "price", "cost", "total", "balance"])) {
-        return (Math.random() * 1e3).toFixed(2);
+      const contextual = this.generateContextualValue(nameLower, type, index);
+      if (contextual !== void 0) return contextual;
+      return this.generateValueByType(type, index, format2);
+    };
+    this.generateContextualValue = (nameLower, type, index) => {
+      if (type === "string") {
+        if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"]))
+          return `${index + 1}`;
+        if (this.matchesPattern(nameLower, ["first_name", "firstname", "fname", "given_name"]))
+          return this.pickRandom(["John", "Jane", "Michael", "Sarah", "David", "Emily", "Robert", "Lisa", "James", "Mary"]);
+        if (this.matchesPattern(nameLower, ["last_name", "lastname", "lname", "surname", "family_name"]))
+          return this.pickRandom(["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Martinez", "Wilson"]);
+        if (this.matchesPattern(nameLower, ["name", "full_name", "fullname"])) {
+          const firstNames = ["John", "Jane", "Michael", "Sarah", "David", "Emily"];
+          const lastNames = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia"];
+          return `${this.pickRandom(firstNames)} ${this.pickRandom(lastNames)}`;
+        }
+        if (this.matchesPattern(nameLower, ["email", "mail"]))
+          return `user${index + 1}@example.com`;
+        if (this.matchesPattern(nameLower, ["phone", "telephone", "mobile", "cell"]))
+          return `555-${String(Math.floor(Math.random() * 900) + 100).padStart(3, "0")}-${String(Math.floor(Math.random() * 9e3) + 1e3).padStart(4, "0")}`;
+        if (this.matchesPattern(nameLower, ["address", "street", "addr"])) {
+          const streets = ["Main St", "Oak Ave", "Elm Dr", "Pine Rd", "Maple Ln", "Cedar Blvd"];
+          return `${Math.floor(Math.random() * 9999) + 1} ${this.pickRandom(streets)}`;
+        }
+        if (this.matchesPattern(nameLower, ["city", "town"]))
+          return this.pickRandom(["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego"]);
+        if (this.matchesPattern(nameLower, ["state", "province"]))
+          return this.pickRandom(["CA", "TX", "FL", "NY", "PA", "IL", "OH", "GA", "NC", "MI"]);
+        if (this.matchesPattern(nameLower, ["zip", "postal", "zipcode"]))
+          return String(Math.floor(Math.random() * 9e4) + 1e4);
+        if (this.matchesPattern(nameLower, ["country"]))
+          return this.pickRandom(["USA", "Canada", "UK", "Germany", "France", "Australia"]);
+        if (this.matchesPattern(nameLower, ["sex", "gender"]))
+          return this.pickRandom(["M", "F", "Male", "Female"]);
+        if (this.matchesPattern(nameLower, ["status"]))
+          return this.pickRandom(["active", "inactive", "pending", "completed", "cancelled"]);
+        if (this.matchesPattern(nameLower, ["type", "category"]))
+          return this.pickRandom(["TypeA", "TypeB", "TypeC", "TypeD"]);
+        if (this.matchesPattern(nameLower, ["description", "desc", "notes", "comment"]))
+          return `Sample description for record ${index + 1}`;
+      }
+      if (type === "number") {
+        if (this.matchesPattern(nameLower, ["age", "years"]))
+          return Math.floor(Math.random() * 80) + 18;
+        if (this.matchesPattern(nameLower, ["amount", "price", "cost", "total", "balance"]))
+          return parseFloat((Math.random() * 1e3).toFixed(2));
+        if (this.matchesPattern(nameLower, ["quantity", "count", "qty"]))
+          return Math.floor(Math.random() * 100) + 1;
+        if (this.matchesPattern(nameLower, ["id", "identifier", "key", "pk"]))
+          return index + 1;
       }
-      if (this.matchesPattern(nameLower, ["quantity", "count", "qty"])) {
-        return Math.floor(Math.random() * 100) + 1;
-      }
-      if (this.matchesPattern(nameLower, ["status"])) {
-        return this.pickRandom(["active", "inactive", "pending", "completed", "cancelled"]);
-      }
-      if (this.matchesPattern(nameLower, ["type", "category"])) {
-        return this.pickRandom(["TypeA", "TypeB", "TypeC", "TypeD"]);
-      }
-      if (this.matchesPattern(nameLower, ["description", "desc", "notes", "comment"])) {
-        return `Sample description for record ${index + 1}`;
-      }
-      return this.generateValueByType(type, index);
+      return void 0;
     };
     this.matchesPattern = (fieldName, patterns) => {
       return patterns.some((pattern) => fieldName.includes(pattern));
@@ -18074,7 +18096,7 @@ var DeveloperEngineClass = class {
     this.pickRandom = (arr) => {
       return arr[Math.floor(Math.random() * arr.length)];
     };
-    this.generateValueByType = (type, index) => {
+    this.generateValueByType = (type, index, format2) => {
       switch (type) {
         case "string":
           return `value_${index + 1}`;
@@ -18084,9 +18106,14 @@ var DeveloperEngineClass = class {
           return Math.random() > 0.5;
         case "datetime": {
           const year = Math.floor(Math.random() * 5) + 2020;
-          const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, "0");
-          const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, "0");
-          return `${year}-${month}-${day}`;
+          const month = Math.floor(Math.random() * 12) + 1;
+          const day = Math.floor(Math.random() * 28) + 1;
+          const hour = Math.floor(Math.random() * 24);
+          const minute = Math.floor(Math.random() * 60);
+          const second = Math.floor(Math.random() * 60);
+          const date = (0, import_dayjs2.default)(new Date(year, month - 1, day, hour, minute, second));
+          if (format2) return date.format(format2);
+          return date.format("YYYY-MM-DD");
         }
         default:
           return `value_${index + 1}`;
@@ -18216,7 +18243,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18227,7 +18254,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18269,7 +18296,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18277,6 +18304,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -19722,6 +19763,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -19844,6 +19887,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs11.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs11.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20365,7 +20515,8 @@ var ExecutorOrchestratorClass = class {
                 prodDimensions,
                 workerId,
                 scope,
-                options
+                options,
+                loggerConfig: Logger_default.getConfig()
               };
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
@@ -20396,6 +20547,12 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
           }
         }
+        if (consumer.options?.pivot) {
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+        }
         counter = performance.now();
         Logger_default.log(`Consumer "${consumer.name}": exporting results`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -20580,6 +20737,8 @@ var ExecutorOrchestrator = new ExecutorOrchestratorClass();
 import_dotenv.default.configDotenv();
 var run = async (workerData) => {
   Environment_default.load("./");
+  if (workerData.loggerConfig)
+    Logger_default.initFromConfig(workerData.loggerConfig);
   try {
     const {
       workerId,