@forzalabs/remora 1.1.7 → 1.1.8

package/index.js

@@ -13461,7 +13461,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.7",
+  cliVersion: "1.1.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13805,6 +13805,27 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -18880,7 +18901,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18891,7 +18912,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18933,7 +18954,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18941,6 +18962,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -20124,6 +20159,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -20246,6 +20283,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs13.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs13.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20644,6 +20788,12 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
           }
         }
+        if (consumer.options?.pivot) {
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+        }
         counter = performance.now();
         Logger_default.log(`Consumer "${consumer.name}": exporting results`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);

package/json_schemas/consumer-schema.json

@@ -166,6 +166,10 @@
                     "fixed": {
                         "type": "boolean",
                         "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                    },
+                    "copyFrom": {
+                        "type": "string",
+                        "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                     }
                 },
                 "required": [
@@ -408,6 +412,46 @@
                     },
                     "required": ["keys", "resolution"],
                     "additionalProperties": false
+                },
+                "pivot": {
+                    "type": "object",
+                    "description": "Performs a pivot operation that transforms row values into columns. Groups data by the specified row keys, takes distinct values from the pivot column and creates a new column for each, aggregating the value column with the specified function. Cannot be used together with 'distinct' or 'distinctOn'.",
+                    "properties": {
+                        "rowKeys": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "minItems": 1,
+                            "description": "The field(s) that identify a row in the pivoted output (the GROUP BY keys). Use the 'alias' if specified."
+                        },
+                        "pivotColumn": {
+                            "type": "string",
+                            "description": "The field whose distinct values become new columns in the output."
+                        },
+                        "valueColumn": {
+                            "type": "string",
+                            "description": "The field whose values are aggregated into each pivot cell."
+                        },
+                        "aggregation": {
+                            "type": "string",
+                            "enum": ["sum", "count", "avg", "min", "max"],
+                            "description": "The aggregation function to apply when combining values."
+                        },
+                        "pivotValues": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "description": "If provided, only these values from the pivot column will be used as output columns. This avoids a discovery pass over the data and makes the output shape statically known. If omitted, the distinct values are discovered automatically."
+                        },
+                        "columnPrefix": {
+                            "type": "string",
+                            "description": "Optional prefix for the generated pivot column names (e.g. 'revenue_' produces 'revenue_East', 'revenue_West')."
+                        }
+                    },
+                    "required": ["rowKeys", "pivotColumn", "valueColumn", "aggregation"],
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false
@@ -519,6 +563,10 @@
                 "fixed": {
                     "type": "boolean",
                     "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
+                },
+                "copyFrom": {
+                    "type": "string",
+                    "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
                 }
             },
             "required": [

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.7",
+    "version": "1.1.8",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13455,7 +13455,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.7",
+  cliVersion: "1.1.8",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13799,6 +13799,27 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.pivot)) {
+            if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
+              errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
+            const { pivot } = consumer.options;
+            if (!pivot.rowKeys || pivot.rowKeys.length === 0)
+              errors.push(`Pivot option requires at least one "rowKeys" field (${consumer.name}).`);
+            if (!pivot.pivotColumn)
+              errors.push(`Pivot option requires a "pivotColumn" (${consumer.name}).`);
+            if (!pivot.valueColumn)
+              errors.push(`Pivot option requires a "valueColumn" (${consumer.name}).`);
+            if (!pivot.aggregation)
+              errors.push(`Pivot option requires an "aggregation" function (${consumer.name}).`);
+            const validAggregations = ["sum", "count", "avg", "min", "max"];
+            if (pivot.aggregation && !validAggregations.includes(pivot.aggregation))
+              errors.push(`Invalid pivot aggregation "${pivot.aggregation}" in consumer "${consumer.name}". Valid values: ${validAggregations.join(", ")}`);
+            const allFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            const pivotFields = [...pivot.rowKeys ?? [], pivot.pivotColumn, pivot.valueColumn].filter(Boolean);
+            const missingFields = pivotFields.filter((f) => !allFieldKeys.includes(f));
+            if (missingFields.length > 0)
+              errors.push(`Pivot references field(s) "${missingFields.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -18222,7 +18243,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed) {
+      } else if (!field.fixed && !field.copyFrom) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -18233,7 +18254,7 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default)) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
@@ -18275,7 +18296,7 @@ var ConsumerManagerClass = class {
     this.getOutputShape = (consumer) => {
       Affirm_default(consumer, `Invalid consumer`);
       const compiled = this.compile(consumer);
-      const outDimensions = compiled.map((x) => ({
+      let outDimensions = compiled.map((x) => ({
         name: x.consumerAlias ?? x.consumerKey,
         type: x.dimension?.type,
         classification: x.dimension?.classification,
@@ -18283,6 +18304,20 @@ var ConsumerManagerClass = class {
         mask: ProducerManager_default.getMask(x.dimension),
         pk: x.dimension?.pk
       }));
+      if (consumer.options?.pivot) {
+        const { rowKeys, pivotValues, columnPrefix = "", valueColumn } = consumer.options.pivot;
+        const rowDimensions = outDimensions.filter((x) => rowKeys.includes(x.name));
+        const valueType = outDimensions.find((x) => x.name === valueColumn)?.type ?? "number";
+        if (pivotValues && pivotValues.length > 0) {
+          const pivotDimensions = pivotValues.map((pv) => ({
+            name: columnPrefix + pv,
+            type: valueType
+          }));
+          outDimensions = [...rowDimensions, ...pivotDimensions];
+        } else {
+          outDimensions = rowDimensions;
+        }
+      }
       return {
         _version: consumer._version,
         name: consumer.name,
@@ -19728,6 +19763,8 @@ var ConsumerExecutorClass = class {
         if (!dimension) {
           if (cField.fixed && Algo_default.hasVal(cField.default))
             record[fieldKey] = cField.default;
+          else if (cField.copyFrom)
+            record[fieldKey] = record[cField.copyFrom];
           else
             throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
         }
@@ -19850,6 +19887,113 @@ var ConsumerExecutorClass = class {
       await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
+    this.processPivot = async (consumer, datasetPath) => {
+      const { pivot } = consumer.options;
+      const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let pivotValues = pivot.pivotValues;
+      if (!pivotValues) {
+        pivotValues = [];
+        const discoverySet = /* @__PURE__ */ new Set();
+        const discoverReader = import_fs11.default.createReadStream(datasetPath);
+        const discoverLineReader = import_readline7.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
+        for await (const line of discoverLineReader) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          const val = String(record[pivotColumn] ?? "");
+          if (!discoverySet.has(val)) {
+            discoverySet.add(val);
+            pivotValues.push(val);
+          }
+        }
+        discoverLineReader.close();
+        if (!discoverReader.destroyed) {
+          await new Promise((resolve) => {
+            discoverReader.once("close", resolve);
+            discoverReader.destroy();
+          });
+        }
+      }
+      const groups = /* @__PURE__ */ new Map();
+      const reader = import_fs11.default.createReadStream(datasetPath);
+      const lineReader = import_readline7.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        const record = this._parseLine(line, internalRecordFormat, internalFields);
+        const compositeKey = rowKeys.map((k) => String(record[k] ?? "")).join("|");
+        const pivotVal = String(record[pivotColumn] ?? "");
+        const numericVal = Number(record[valueColumn]) || 0;
+        if (!groups.has(compositeKey)) {
+          const rowRecord = {};
+          for (const k of rowKeys) rowRecord[k] = record[k];
+          groups.set(compositeKey, { rowRecord, cells: /* @__PURE__ */ new Map() });
+        }
+        const group = groups.get(compositeKey);
+        if (!group.cells.has(pivotVal)) {
+          group.cells.set(pivotVal, { sum: 0, count: 0, min: Infinity, max: -Infinity });
+        }
+        const cell = group.cells.get(pivotVal);
+        cell.sum += numericVal;
+        cell.count++;
+        cell.min = Math.min(cell.min, numericVal);
+        cell.max = Math.max(cell.max, numericVal);
+      }
+      lineReader.close();
+      const pivotedFields = [
+        ...rowKeys.map((k) => ({ cField: { key: k }, finalKey: k })),
+        ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
+      ];
+      const tempWorkPath = datasetPath + "_tmp";
+      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      let outputCount = 0;
+      for (const { rowRecord, cells } of groups.values()) {
+        const outputRecord = { ...rowRecord };
+        for (const pv of pivotValues) {
+          const colName = columnPrefix + pv;
+          const cell = cells.get(pv);
+          if (!cell) {
+            outputRecord[colName] = 0;
+            continue;
+          }
+          switch (aggregation) {
+            case "sum":
+              outputRecord[colName] = cell.sum;
+              break;
+            case "count":
+              outputRecord[colName] = cell.count;
+              break;
+            case "avg":
+              outputRecord[colName] = cell.count > 0 ? cell.sum / cell.count : 0;
+              break;
+            case "min":
+              outputRecord[colName] = cell.min === Infinity ? 0 : cell.min;
+              break;
+            case "max":
+              outputRecord[colName] = cell.max === -Infinity ? 0 : cell.max;
+              break;
+          }
+        }
+        const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
+        writer.write(line + "\n");
+        outputCount++;
+      }
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return outputCount;
+    };
+    this._parseLine = (line, format2, fields) => {
+      return format2 === "CSV" || format2 === "TXT" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+    };
     /**
      * Determines if the new record should replace the existing record based on the resolution strategy
      */
@@ -20403,6 +20547,12 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
           }
         }
+        if (consumer.options?.pivot) {
+          counter = performance.now();
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("process-pivot:main", performance.now() - counter);
+          postOperation.totalOutputCount = unifiedOutputCount;
+        }
         counter = performance.now();
         Logger_default.log(`Consumer "${consumer.name}": exporting results`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);