@forzalabs/remora 1.2.10 → 1.2.11

package/CHANGELOG.md

@@ -6,6 +6,13 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 ## Unreleased
 
+## V 1.2.11 - 2026-06-30
+
+### Added
+- Added `lineCount` as a consumer field property, allowing consumers to emit a consistent 1-based source line number even when files are processed across multiple workers
+- Added a dedicated `ConsumerExecutor` post-processing pass to normalize `lineCount` values after worker merge, with tracked execution timing similar to other dataset-level steps
+- Added `lineCount` test coverage to canary and CLI consumer configurations for easier verification of the feature
+
 ## V 1.2.2 - 2026-04-10
 
 ### Added

package/index.js

@@ -18751,7 +18751,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.10",
+  cliVersion: "1.2.11",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -19191,7 +19191,16 @@ var ValidatorClass = class {
         }
         const allAvailableFields = [...availableFieldsByProducer.values()].flat();
         for (const field of consumer.fields) {
-          if (field.key === "*" || field.fixed || field.copyFrom) continue;
+          const syntheticFieldModes = [field.fixed === true, !!field.copyFrom, field.lineCount === true].filter(Boolean);
+          if (syntheticFieldModes.length > 1)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" can only use one synthetic field mode among "fixed", "copyFrom", and "lineCount".`);
+          if (field.lineCount && field.key === "*")
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "lineCount" with the wildcard key "*".`);
+          if (field.lineCount && field.transform)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "transform" together with "lineCount".`);
+          if (field.lineCount && field.validate && field.validate.length > 0)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "validate" together with "lineCount".`);
+          if (field.key === "*" || field.fixed || field.copyFrom || field.lineCount) continue;
           if (field.from) {
             const producerFields = availableFieldsByProducer.get(field.from);
             if (producerFields && !producerFields.includes(field.key))
@@ -19212,6 +19221,8 @@ var ValidatorClass = class {
           const found = precedingFields.find((f) => (f.alias ?? f.key) === field.copyFrom);
           if (!found)
             errors.push(`Field "${field.alias ?? field.key}" uses copyFrom "${field.copyFrom}" but no field with that name/alias exists before it in consumer "${consumer.name}".`);
+          else if (found.lineCount)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use copyFrom on a field with "lineCount" because line counts are normalized after worker merge.`);
         }
         if (consumer.filters && consumer.filters.length > 0) {
           if (consumer.filters.some((x) => x.sql && x.rule))
@@ -19232,10 +19243,14 @@ var ValidatorClass = class {
             const missingRules = rulesWithMatchingFields.filter((x) => !x.match);
             errors.push(`Filter(s) on member(s) "${missingRules.map((x) => x.rule.rule.member).join(", ")}" is invalid since the member specified is not present in the consumer. Check the member value or add the missing field to the consumer.`);
           }
+          const lineCountRuleMembers = rulesWithMatchingFields.filter((x) => x.match?.lineCount);
+          if (lineCountRuleMembers.length > 0)
+            errors.push(`Consumer "${consumer.name}" cannot use rule-based filters on field(s) with "lineCount": ${lineCountRuleMembers.map((x) => `"${x.rule.rule.member}"`).join(", ")}.`);
         }
         const validateTransformations = (fields) => {
           const errors2 = [];
           const trxsFields = fields.filter((x) => x.transform);
+          const lineCountFieldNames = new Set(fields.filter((x) => x.lineCount).map((x) => x.alias ?? x.key));
           for (const field of trxsFields) {
             const trxToValidate = [];
             if (Array.isArray(field.transform))
@@ -19243,6 +19258,16 @@ var ValidatorClass = class {
             else
               trxToValidate.push(field.transform);
             for (const trans of trxToValidate) {
+              if ("multiplyBy" in trans) {
+                const invalidFields = trans.multiplyBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in multiplyBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
+              if ("addBy" in trans) {
+                const invalidFields = trans.addBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in addBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
               if ("combine_fields" in trans) {
                 const { combine_fields } = trans;
                 if (!combine_fields.fields || combine_fields.fields.length === 0)
@@ -19250,6 +19275,9 @@ var ValidatorClass = class {
                 const missingFieldsInConsumer = combine_fields.fields.map((x) => ({ field: x, found: fields.find((k) => (k.alias ?? k.key) === x) })).filter((x) => !x.found);
                 if (missingFieldsInConsumer.length > 0)
                   errors2.push(`The requested field(s) for a transformation is missing in the consumer -> missing field(s): "${missingFieldsInConsumer.map((x) => x.field).join(", ")}"; field transformation: "${field.alias ?? field.key}";`);
+                const invalidFields = combine_fields.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in combine_fields: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
               }
             }
           }
@@ -19272,7 +19300,7 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
-        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom && !x.lineCount);
         const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
         const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
         if (duplicateDimensionKeys.length > 0) {
@@ -23459,6 +23487,25 @@ var ConsumerManagerClass = class {
       const expandedFields = convertedFields.flatMap((x) => this.expandField(consumer, x, availableColumns));
       return expandedFields;
     };
+    this.getLineCountFields = (consumer, visibleOnly = false) => {
+      Affirm_default(consumer, "Invalid consumer");
+      return this.getLineCountFieldsFromExpanded(this.getExpandedFields(consumer), visibleOnly);
+    };
+    this.getLineCountFieldsFromExpanded = (fields, visibleOnly = false) => {
+      Affirm_default(fields, "Invalid consumer fields");
+      return fields.filter((field) => this.isLineCountField(field.cField) && (!visibleOnly || !field.cField.hidden));
+    };
+    this.hasLineCountFields = (consumer, visibleOnly = false) => {
+      return this.getLineCountFields(consumer, visibleOnly).length > 0;
+    };
+    this.isSyntheticField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.fixed === true || !!field.copyFrom || this.isLineCountField(field);
+    };
+    this.isLineCountField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.lineCount === true;
+    };
     this.getAvailableColumns = (consumer) => {
       return ResourcesUtils_default.getAvailableColumns(consumer);
     };
@@ -23511,7 +23558,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed && !field.copyFrom) {
+      } else if (!this.isSyntheticField(field)) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -23522,14 +23569,14 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || this.isSyntheticField(field)) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
             consumerAlias: field.alias ?? field.key,
             consumerKey: field.key,
             owner: field.from,
-            dimension: { name: field.key, type: typeof field.default }
+            dimension: { name: field.key, type: this.isLineCountField(field) ? "number" : typeof field.default }
           };
         }
       }
@@ -24841,16 +24888,23 @@ var ConsumerExecutorClass = class {
       for (const field of fields) {
         const { cField } = field;
         const fieldKey = cField.alias ?? cField.key;
+        if (cField.fixed && Algo_default.hasVal(cField.default)) {
+          record[fieldKey] = cField.default;
+          continue;
+        }
+        if (ConsumerManager_default.isLineCountField(cField)) {
+          record[fieldKey] = recordIndex + 1;
+          continue;
+        }
+        if (cField.copyFrom) {
+          record[fieldKey] = record[cField.copyFrom];
+          continue;
+        }
         let dimension;
         try {
           dimension = dimensions.find((x) => x.name === cField.key);
           if (!dimension) {
-            if (cField.fixed && Algo_default.hasVal(cField.default))
-              record[fieldKey] = cField.default;
-            else if (cField.copyFrom)
-              record[fieldKey] = record[cField.copyFrom];
-            else
-              throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
+            throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
           }
         } catch (error) {
           const err2 = new Error(`Resolving dimension for field "${fieldKey}" of producer "${producer.name}" failed (index: ${recordIndex}): ${error.message}`, { cause: error });
@@ -24963,6 +25017,64 @@ var ConsumerExecutorClass = class {
       }
       return record;
     };
+    this.processLineCount = async (consumer, datasetPath, executorResults) => {
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      const lineCountFields = ConsumerManager_default.getLineCountFieldsFromExpanded(internalFields, true);
+      if (lineCountFields.length === 0)
+        return 0;
+      const tempWorkPath = datasetPath + "_tmp";
+      const reader = import_fs12.default.createReadStream(datasetPath);
+      const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+      const writer = import_fs12.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
+      let workerIndex = 0;
+      let currentWorker = executorResults[workerIndex] ?? null;
+      let remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+      let processedLineCount = 0;
+      const advanceWorker = () => {
+        while (currentWorker && remainingLinesInWorker <= 0) {
+          workerIndex++;
+          currentWorker = executorResults[workerIndex] ?? null;
+          remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+        }
+      };
+      advanceWorker();
+      for await (const line of lineReader) {
+        advanceWorker();
+        Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
+        const rowOffset = currentWorker.rowOffset ?? 0;
+        if (rowOffset > 0) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          for (const field of lineCountFields) {
+            const currentValue = Number(record[field.finalKey]);
+            Affirm_default(!Number.isNaN(currentValue), `Invalid lineCount value found while processing ${datasetPath}`);
+            record[field.finalKey] = currentValue + rowOffset;
+          }
+          if (!writer.write(OutputExecutor_default.outputRecord(record, consumer, internalFields) + "\n"))
+            await waitForDrain();
+        } else if (!writer.write(line + "\n")) {
+          await waitForDrain();
+        }
+        remainingLinesInWorker--;
+        processedLineCount++;
+      }
+      lineReader.close();
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return processedLineCount;
+    };
     this.processDistinct = async (datasetPath) => {
       const reader = import_fs12.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
@@ -25554,6 +25666,7 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress2_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
+      const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
       let activePool = null;
       try {
         const start = performance.now();
@@ -25616,7 +25729,9 @@ var ExecutorOrchestratorClass = class {
                 }));
               }
               Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete  | ${OrchestratorHelper_default.formatMemoryUsage()}`);
-              executorResults.push(...await Promise.all(workerThreads));
+              const fileExecutorResults = await Promise.all(workerThreads);
+              this.assignChunkRowMetadata(chunks, fileExecutorResults);
+              executorResults.push(...fileExecutorResults);
               Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
             } finally {
               await activePool.terminate();
@@ -25651,14 +25766,21 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
-        }
-        if (consumer.options?.pivot) {
-          Logger_default.log(`[${usageId}] Running pivot operation`);
-          counter = performance.now();
-          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
-          tracker.measure("process-pivot:main", performance.now() - counter);
-          postOperation.totalOutputCount = unifiedOutputCount;
-          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          if (consumer.options?.pivot) {
+            Logger_default.log(`[${usageId}] Running pivot operation`);
+            counter = performance.now();
+            const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
+            tracker.measure("process-pivot:main", performance.now() - counter);
+            postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (ConsumerManager_default.hasLineCountFields(consumer, true)) {
+            Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
+            counter = performance.now();
+            const processedLineCount = await ConsumerExecutor_default.processLineCount(consumer, ExecutorScope_default.getMainPath(scope), executorResults);
+            tracker.measure("process-line-count:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${processedLineCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
         }
         if (consumer.validate && consumer.validate.length > 0) {
           Logger_default.log(`[${usageId}] Running dataset-level validations`);
@@ -25688,7 +25810,7 @@ var ExecutorOrchestratorClass = class {
         _progress.update({ phase: "Exporting result", progress: 0 });
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-        const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
+        const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, scope);
         tracker.measure("export-result", performance.now() - counter);
         Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         _progress.update({ phase: "Exporting result", progress: 1 });
@@ -25759,6 +25881,22 @@ var ExecutorOrchestratorClass = class {
         import_fs13.default.closeSync(fd);
       }
     };
+    this.assignChunkRowMetadata = (chunks, executorResults) => {
+      const validResults = executorResults.filter((result) => Algo_default.hasVal(result));
+      const resultsByChunkIndex = new Map(validResults.map((result) => [result.chunkIndex, result]));
+      let nextRowOffset = 0;
+      for (const chunk of chunks) {
+        const result = resultsByChunkIndex.get(chunk.index);
+        if (!result)
+          continue;
+        const rowCount = result.inputCount;
+        chunk.rowOffset = nextRowOffset;
+        chunk.rowCount = rowCount;
+        result.rowOffset = nextRowOffset;
+        result.rowCount = rowCount;
+        nextRowOffset += rowCount;
+      }
+    };
     /**
      * Efficiently finds the next newline character starting from a position.
      * Uses small buffer reads for speed.

package/json_schemas/consumer-schema.json

@@ -206,6 +206,10 @@
                         "type": "boolean",
                         "description": "If set, \"default\" must have a value. This field is not searched in the underlying dataset, but is a fixed value set by the \"default\" prop."
                     },
+                    "lineCount": {
+                        "type": "boolean",
+                        "description": "If set, this field is populated with the 1-based source line number of the current record. For multi-worker file processing the final value is normalized during merge so it remains consistent across chunks."
+                    },
                     "copyFrom": {
                         "type": "string",
                         "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.10",
+    "version": "1.2.11",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -18745,7 +18745,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.10",
+  cliVersion: "1.2.11",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -19184,7 +19184,16 @@ var ValidatorClass = class {
         }
         const allAvailableFields = [...availableFieldsByProducer.values()].flat();
         for (const field of consumer.fields) {
-          if (field.key === "*" || field.fixed || field.copyFrom) continue;
+          const syntheticFieldModes = [field.fixed === true, !!field.copyFrom, field.lineCount === true].filter(Boolean);
+          if (syntheticFieldModes.length > 1)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" can only use one synthetic field mode among "fixed", "copyFrom", and "lineCount".`);
+          if (field.lineCount && field.key === "*")
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "lineCount" with the wildcard key "*".`);
+          if (field.lineCount && field.transform)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "transform" together with "lineCount".`);
+          if (field.lineCount && field.validate && field.validate.length > 0)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "validate" together with "lineCount".`);
+          if (field.key === "*" || field.fixed || field.copyFrom || field.lineCount) continue;
           if (field.from) {
             const producerFields = availableFieldsByProducer.get(field.from);
             if (producerFields && !producerFields.includes(field.key))
@@ -19205,6 +19214,8 @@ var ValidatorClass = class {
           const found = precedingFields.find((f) => (f.alias ?? f.key) === field.copyFrom);
           if (!found)
             errors.push(`Field "${field.alias ?? field.key}" uses copyFrom "${field.copyFrom}" but no field with that name/alias exists before it in consumer "${consumer.name}".`);
+          else if (found.lineCount)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use copyFrom on a field with "lineCount" because line counts are normalized after worker merge.`);
         }
         if (consumer.filters && consumer.filters.length > 0) {
           if (consumer.filters.some((x) => x.sql && x.rule))
@@ -19225,10 +19236,14 @@ var ValidatorClass = class {
             const missingRules = rulesWithMatchingFields.filter((x) => !x.match);
             errors.push(`Filter(s) on member(s) "${missingRules.map((x) => x.rule.rule.member).join(", ")}" is invalid since the member specified is not present in the consumer. Check the member value or add the missing field to the consumer.`);
           }
+          const lineCountRuleMembers = rulesWithMatchingFields.filter((x) => x.match?.lineCount);
+          if (lineCountRuleMembers.length > 0)
+            errors.push(`Consumer "${consumer.name}" cannot use rule-based filters on field(s) with "lineCount": ${lineCountRuleMembers.map((x) => `"${x.rule.rule.member}"`).join(", ")}.`);
         }
         const validateTransformations = (fields) => {
           const errors2 = [];
           const trxsFields = fields.filter((x) => x.transform);
+          const lineCountFieldNames = new Set(fields.filter((x) => x.lineCount).map((x) => x.alias ?? x.key));
           for (const field of trxsFields) {
             const trxToValidate = [];
             if (Array.isArray(field.transform))
@@ -19236,6 +19251,16 @@ var ValidatorClass = class {
             else
               trxToValidate.push(field.transform);
             for (const trans of trxToValidate) {
+              if ("multiplyBy" in trans) {
+                const invalidFields = trans.multiplyBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in multiplyBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
+              if ("addBy" in trans) {
+                const invalidFields = trans.addBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in addBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
               if ("combine_fields" in trans) {
                 const { combine_fields } = trans;
                 if (!combine_fields.fields || combine_fields.fields.length === 0)
@@ -19243,6 +19268,9 @@ var ValidatorClass = class {
                 const missingFieldsInConsumer = combine_fields.fields.map((x) => ({ field: x, found: fields.find((k) => (k.alias ?? k.key) === x) })).filter((x) => !x.found);
                 if (missingFieldsInConsumer.length > 0)
                   errors2.push(`The requested field(s) for a transformation is missing in the consumer -> missing field(s): "${missingFieldsInConsumer.map((x) => x.field).join(", ")}"; field transformation: "${field.alias ?? field.key}";`);
+                const invalidFields = combine_fields.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in combine_fields: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
               }
             }
           }
@@ -19265,7 +19293,7 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
-        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom && !x.lineCount);
         const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
         const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
         if (duplicateDimensionKeys.length > 0) {
@@ -22783,6 +22811,25 @@ var ConsumerManagerClass = class {
       const expandedFields = convertedFields.flatMap((x) => this.expandField(consumer, x, availableColumns));
       return expandedFields;
     };
+    this.getLineCountFields = (consumer, visibleOnly = false) => {
+      Affirm_default(consumer, "Invalid consumer");
+      return this.getLineCountFieldsFromExpanded(this.getExpandedFields(consumer), visibleOnly);
+    };
+    this.getLineCountFieldsFromExpanded = (fields, visibleOnly = false) => {
+      Affirm_default(fields, "Invalid consumer fields");
+      return fields.filter((field) => this.isLineCountField(field.cField) && (!visibleOnly || !field.cField.hidden));
+    };
+    this.hasLineCountFields = (consumer, visibleOnly = false) => {
+      return this.getLineCountFields(consumer, visibleOnly).length > 0;
+    };
+    this.isSyntheticField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.fixed === true || !!field.copyFrom || this.isLineCountField(field);
+    };
+    this.isLineCountField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.lineCount === true;
+    };
     this.getAvailableColumns = (consumer) => {
       return ResourcesUtils_default.getAvailableColumns(consumer);
     };
@@ -22835,7 +22882,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed && !field.copyFrom) {
+      } else if (!this.isSyntheticField(field)) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -22846,14 +22893,14 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || this.isSyntheticField(field)) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
             consumerAlias: field.alias ?? field.key,
             consumerKey: field.key,
             owner: field.from,
-            dimension: { name: field.key, type: typeof field.default }
+            dimension: { name: field.key, type: this.isLineCountField(field) ? "number" : typeof field.default }
           };
         }
       }
@@ -24434,16 +24481,23 @@ var ConsumerExecutorClass = class {
       for (const field of fields) {
         const { cField } = field;
         const fieldKey = cField.alias ?? cField.key;
+        if (cField.fixed && Algo_default.hasVal(cField.default)) {
+          record[fieldKey] = cField.default;
+          continue;
+        }
+        if (ConsumerManager_default.isLineCountField(cField)) {
+          record[fieldKey] = recordIndex + 1;
+          continue;
+        }
+        if (cField.copyFrom) {
+          record[fieldKey] = record[cField.copyFrom];
+          continue;
+        }
         let dimension;
         try {
           dimension = dimensions.find((x) => x.name === cField.key);
           if (!dimension) {
-            if (cField.fixed && Algo_default.hasVal(cField.default))
-              record[fieldKey] = cField.default;
-            else if (cField.copyFrom)
-              record[fieldKey] = record[cField.copyFrom];
-            else
-              throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
+            throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
           }
         } catch (error) {
           const err2 = new Error(`Resolving dimension for field "${fieldKey}" of producer "${producer.name}" failed (index: ${recordIndex}): ${error.message}`, { cause: error });
@@ -24556,6 +24610,64 @@ var ConsumerExecutorClass = class {
       }
       return record;
     };
+    this.processLineCount = async (consumer, datasetPath, executorResults) => {
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      const lineCountFields = ConsumerManager_default.getLineCountFieldsFromExpanded(internalFields, true);
+      if (lineCountFields.length === 0)
+        return 0;
+      const tempWorkPath = datasetPath + "_tmp";
+      const reader = import_fs10.default.createReadStream(datasetPath);
+      const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+      const writer = import_fs10.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
+      let workerIndex = 0;
+      let currentWorker = executorResults[workerIndex] ?? null;
+      let remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+      let processedLineCount = 0;
+      const advanceWorker = () => {
+        while (currentWorker && remainingLinesInWorker <= 0) {
+          workerIndex++;
+          currentWorker = executorResults[workerIndex] ?? null;
+          remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+        }
+      };
+      advanceWorker();
+      for await (const line of lineReader) {
+        advanceWorker();
+        Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
+        const rowOffset = currentWorker.rowOffset ?? 0;
+        if (rowOffset > 0) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          for (const field of lineCountFields) {
+            const currentValue = Number(record[field.finalKey]);
+            Affirm_default(!Number.isNaN(currentValue), `Invalid lineCount value found while processing ${datasetPath}`);
+            record[field.finalKey] = currentValue + rowOffset;
+          }
+          if (!writer.write(OutputExecutor_default.outputRecord(record, consumer, internalFields) + "\n"))
+            await waitForDrain();
+        } else if (!writer.write(line + "\n")) {
+          await waitForDrain();
+        }
+        remainingLinesInWorker--;
+        processedLineCount++;
+      }
+      lineReader.close();
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return processedLineCount;
+    };
     this.processDistinct = async (datasetPath) => {
       const reader = import_fs10.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
@@ -24912,6 +25024,10 @@ var Executor = class {
         elapsedMS: -1,
         inputCount: -1,
         outputCount: -1,
+        chunkIndex: chunk.index,
+        fileUri: chunk.fileUri,
+        rowOffset: chunk.rowOffset ?? -1,
+        rowCount: -1,
         resultUri: ExecutorScope_default.getWorkerPath(scope, workerId),
         operations: {}
       };
@@ -25021,6 +25137,7 @@ var Executor = class {
       result.cycles = totalCycles;
       result.inputCount = lineIndex;
       result.outputCount = totalOutputCount;
+      result.rowCount = lineIndex;
       result.operations = this._performance.getOperations();
       Logger_default.log(`[${workerId}] Finished: ${lineIndex} input \u2192 ${totalOutputCount} output in ${Math.round(result.elapsedMS)}ms (${totalCycles} cycle(s))`);
       return result;
@@ -25321,6 +25438,7 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress2_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
+      const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
       let activePool = null;
       try {
         const start = performance.now();
@@ -25383,7 +25501,9 @@ var ExecutorOrchestratorClass = class {
                 }));
               }
               Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete  | ${OrchestratorHelper_default.formatMemoryUsage()}`);
-              executorResults.push(...await Promise.all(workerThreads));
+              const fileExecutorResults = await Promise.all(workerThreads);
+              this.assignChunkRowMetadata(chunks, fileExecutorResults);
+              executorResults.push(...fileExecutorResults);
               Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
             } finally {
               await activePool.terminate();
@@ -25418,14 +25538,21 @@ var ExecutorOrchestratorClass = class {
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
-        }
-        if (consumer.options?.pivot) {
-          Logger_default.log(`[${usageId}] Running pivot operation`);
-          counter = performance.now();
-          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
-          tracker.measure("process-pivot:main", performance.now() - counter);
-          postOperation.totalOutputCount = unifiedOutputCount;
-          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          if (consumer.options?.pivot) {
+            Logger_default.log(`[${usageId}] Running pivot operation`);
+            counter = performance.now();
+            const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
+            tracker.measure("process-pivot:main", performance.now() - counter);
+            postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (ConsumerManager_default.hasLineCountFields(consumer, true)) {
+            Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
+            counter = performance.now();
+            const processedLineCount = await ConsumerExecutor_default.processLineCount(consumer, ExecutorScope_default.getMainPath(scope), executorResults);
+            tracker.measure("process-line-count:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${processedLineCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
         }
         if (consumer.validate && consumer.validate.length > 0) {
           Logger_default.log(`[${usageId}] Running dataset-level validations`);
@@ -25455,7 +25582,7 @@ var ExecutorOrchestratorClass = class {
         _progress.update({ phase: "Exporting result", progress: 0 });
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-        const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
+        const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, scope);
         tracker.measure("export-result", performance.now() - counter);
         Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         _progress.update({ phase: "Exporting result", progress: 1 });
@@ -25526,6 +25653,22 @@ var ExecutorOrchestratorClass = class {
         import_fs12.default.closeSync(fd);
       }
     };
+    this.assignChunkRowMetadata = (chunks, executorResults) => {
+      const validResults = executorResults.filter((result) => Algo_default.hasVal(result));
+      const resultsByChunkIndex = new Map(validResults.map((result) => [result.chunkIndex, result]));
+      let nextRowOffset = 0;
+      for (const chunk of chunks) {
+        const result = resultsByChunkIndex.get(chunk.index);
+        if (!result)
+          continue;
+        const rowCount = result.inputCount;
+        chunk.rowOffset = nextRowOffset;
+        chunk.rowCount = rowCount;
+        result.rowOffset = nextRowOffset;
+        result.rowCount = rowCount;
+        nextRowOffset += rowCount;
+      }
+    };
     /**
      * Efficiently finds the next newline character starting from a position.
      * Uses small buffer reads for speed.