@forzalabs/remora 2.3.0 → 2.4.0

package/index.js

@@ -8490,7 +8490,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs39 = require("fs");
+    var fs40 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -8511,7 +8511,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs39.open(options2.file, "a+", "0644", (err2, fd) => {
+      fs40.open(options2.file, "a+", "0644", (err2, fd) => {
         if (err2) {
           if (!iter) {
             stream.emit("error", err2);
@@ -8523,10 +8523,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs39.close(fd, noop);
+            fs40.close(fd, noop);
             return;
           }
-          return fs39.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs40.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -8585,7 +8585,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs39 = require("fs");
+    var fs40 = require("fs");
     var path30 = require("path");
     var asyncSeries = require_series();
     var zlib3 = require("zlib");
@@ -8790,7 +8790,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs39.createReadStream(file, {
+        const stream = fs40.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err2) => {
@@ -8942,7 +8942,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path30.join(this.dirname, target);
-        fs39.stat(fullpath, (err2, stat) => {
+        fs40.stat(fullpath, (err2, stat) => {
           if (err2 && err2.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -9047,7 +9047,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path30.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs39.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs40.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -9126,7 +9126,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path30.join(this.dirname, filePath);
-        fs39.unlink(target, callback);
+        fs40.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -9149,17 +9149,17 @@ var require_file = __commonJS({
           tasks.push(function(i6, cb) {
             let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
             const tmppath = path30.join(this.dirname, fileName);
-            fs39.exists(tmppath, (exists) => {
+            fs40.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i6}${ext}${isZipped}`;
-              fs39.rename(tmppath, path30.join(this.dirname, fileName), cb);
+              fs40.rename(tmppath, path30.join(this.dirname, fileName), cb);
             });
           }.bind(this, x2));
         }
         asyncSeries(tasks, () => {
-          fs39.rename(
+          fs40.rename(
             path30.join(this.dirname, `${basename}${ext}${isZipped}`),
             path30.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -9175,22 +9175,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs39.access(src, fs39.F_OK, (err2) => {
+        fs40.access(src, fs40.F_OK, (err2) => {
           if (err2) {
             return callback();
           }
           var gzip = zlib3.createGzip();
-          var inp = fs39.createReadStream(src);
-          var out = fs39.createWriteStream(dest);
+          var inp = fs40.createReadStream(src);
+          var out = fs40.createWriteStream(dest);
           out.on("finish", () => {
-            fs39.unlink(src, callback);
+            fs40.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs39.existsSync(dirPath)) {
-          fs39.mkdirSync(dirPath, { recursive: true });
+        if (!fs40.existsSync(dirPath)) {
+          fs40.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11761,7 +11761,7 @@ var CONSTANTS, Constants_default;
 var init_Constants = __esm({
   "../../packages/constants/src/Constants.ts"() {
     CONSTANTS = {
-      cliVersion: "2.3.0",
+      cliVersion: "2.4.0",
       backendVersion: 1,
       backendPort: 5088,
       workerVersion: 2,
@@ -11890,7 +11890,25 @@ var init_Constants = __esm({
          * Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
          * export of it — `remora sample -s` and the preview's Data tab are both held to this.
          */
-        MAX_SAMPLE_ROWS: 100
+        MAX_SAMPLE_ROWS: 100,
+        /**
+         * Values held in memory while one parquet row group is encoded. A parquet writer buffers a
+         * whole row group before it can compress and flush it, so this — not the dataset size — is
+         * what bounds the export's peak memory. Budgeted in CELLS rather than rows because the two
+         * differ by three orders of magnitude across consumers: 100,000 rows of a 4-column consumer
+         * is 400k values, of a 1,300-field one it is 130 million.
+         */
+        PARQUET_ROW_GROUP_CELLS: 1e6,
+        /**
+         * Floor on the derived row count, so a consumer with hundreds of columns still writes row
+         * groups big enough to compress and to read back without a seek per handful of rows.
+         */
+        PARQUET_MIN_ROW_GROUP_ROWS: 1e3,
+        /**
+         * Ceiling on the derived row count, so a one-column consumer does not buffer a million rows
+         * to save a few row-group footers.
+         */
+        PARQUET_MAX_ROW_GROUP_ROWS: 1e5
       }
     };
     Constants_default = CONSTANTS;
@@ -11984,11 +12002,26 @@ var init_ExecutorScope = __esm({
     import_path2 = __toESM(require("path"), 1);
     import_fs2 = __toESM(require("fs"), 1);
     import_promises = __toESM(require("fs/promises"), 1);
+    init_src();
     init_src3();
     ExecutorScopeClass = class {
       constructor() {
         this.WORKERS_FOLDER = "workers";
         this.PRODUCERS_FOLDER = "producers";
+        /**
+         * The scope folders THIS process opened and has not cleared. The only list `deepClear` may
+         * delete from: two remora processes share one `.temp`, so a listing cannot tell a crashed run's
+         * leftovers from another process's live run.
+         */
+        this._ownedScopes = /* @__PURE__ */ new Set();
+        /** Registers a scope as this process's, so a crash on the way out clears it. */
+        this.open = (scope) => {
+          Affirm_default(scope?.folder, "Invalid executor scope");
+          this._ownedScopes.add(scope.folder);
+          return scope;
+        };
+        /** The scopes this process still holds. Empty once every run it started has cleaned up. */
+        this.getOwnedScopes = () => [...this._ownedScopes];
         this.getBasePath = () => import_path2.default.join(Constants_default.defaults.REMORA_PATH, Constants_default.defaults.PRODUCER_TEMP_FOLDER);
         this.getWorkerPath = (scope, workerId) => {
           return import_path2.default.join(
@@ -11999,6 +12032,13 @@ var init_ExecutorScope = __esm({
             `${workerId}.dataset`
           );
         };
+        /**
+         * Where a producer's source is staged before any worker reads it: a CSV extract WITH a header,
+         * written by whichever driver read the source.
+         *
+         * `.extract`, not `.dataset`: the two are different formats, and one extension meaning both is
+         * how a reader ends up applying the internal positional rules to a file that follows none.
+         */
         this.getProducerPath = (scope, producer, sourceFileKey) => {
           return import_path2.default.join(
             this.getBasePath(),
@@ -12006,7 +12046,7 @@ var init_ExecutorScope = __esm({
             scope.folder,
             this.PRODUCERS_FOLDER,
             producer.name,
-            `${sourceFileKey}.dataset`
+            `${sourceFileKey}.extract`
           );
         };
         this.getMainPath = (scope) => {
@@ -12042,16 +12082,32 @@ var init_ExecutorScope = __esm({
           if (import_fs2.default.existsSync(scopePath)) {
             await import_promises.default.rm(scopePath, { recursive: true, force: true });
           }
+          this._ownedScopes.delete(scope.folder);
         };
+        /**
+         * Clears what this process left open, on the way out. Synchronous: its one caller is the `exit`
+         * handler. A run that finished cleared its own scope, so a clean exit finds nothing to do.
+         */
         this.deepClear = () => {
           const basePath = this.getBasePath();
-          const openScopes = this.getOpenScopes();
-          for (const scopeFolder of openScopes) {
+          for (const scopeFolder of this._ownedScopes) {
             const scopePath = import_path2.default.join(basePath, scopeFolder);
             if (import_fs2.default.existsSync(scopePath)) {
               import_fs2.default.rmSync(scopePath, { recursive: true, force: true });
             }
           }
+          this._ownedScopes.clear();
+        };
+        /**
+         * Every scope folder on disk, whoever opened it. Answers "what is in .temp?", not "what is mine
+         * to delete" — nothing may be removed on the strength of it (see `getOwnedScopes`). The parallel
+         * canary samples it to watch concurrent runs come and go.
+         */
+        this.getOpenScopes = () => {
+          const basePath = this.getBasePath();
+          if (!import_fs2.default.existsSync(basePath))
+            return [];
+          return import_fs2.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
         };
         this.ensurePath = (fileUri) => {
           const dir = import_path2.default.dirname(fileUri);
@@ -12060,12 +12116,6 @@ var init_ExecutorScope = __esm({
           if (!import_fs2.default.existsSync(fileUri))
             import_fs2.default.writeFileSync(fileUri, "");
         };
-        this.getOpenScopes = () => {
-          const basePath = this.getBasePath();
-          if (!import_fs2.default.existsSync(basePath))
-            return [];
-          return import_fs2.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
-        };
       }
     };
     ExecutorScope = new ExecutorScopeClass();
@@ -12119,8 +12169,7 @@ var init_ProcessShutdownManager = __esm({
           if (this._cleaned)
             return;
           this._cleaned = true;
-          const openScopes = ExecutorScope_default.getOpenScopes();
-          const scopeCount = openScopes.length;
+          const scopeCount = ExecutorScope_default.getOwnedScopes().length;
           const shutdownState = this._shutdownState;
           const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
           if (scopeCount === 0) {
@@ -12532,7 +12581,7 @@ var init_ResourcesUtils = __esm({
               Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
               return this.getConsumerOutputColumns(subConsumer);
             } else {
-              const dims = producer.dimensions.map((x2) => ({
+              return producer.dimensions.map((x2) => ({
                 consumerAlias: null,
                 consumerKey: null,
                 nameInProducer: x2.name,
@@ -12540,15 +12589,6 @@ var init_ResourcesUtils = __esm({
                 dimension: x2,
                 owner: cProd.name
               }));
-              const meas = producer.measures?.map((x2) => ({
-                consumerAlias: null,
-                consumerKey: null,
-                nameInProducer: x2.name,
-                aliasInProducer: x2.name,
-                measure: x2,
-                owner: cProd.name
-              })) ?? [];
-              return [...dims, ...meas];
             }
           });
         };
@@ -12608,7 +12648,6 @@ var init_ResourcesUtils = __esm({
                   nameInProducer: col.nameInProducer,
                   aliasInProducer: col.nameInProducer,
                   dimension: col.dimension,
-                  measure: col.measure,
                   owner: consumer.name
                 });
               }
@@ -12622,7 +12661,6 @@ var init_ResourcesUtils = __esm({
                   nameInProducer: match2.name,
                   aliasInProducer: match2.name,
                   dimension: match2.column.dimension,
-                  measure: match2.column.measure,
                   owner: consumer.name
                 });
               }
@@ -12636,7 +12674,6 @@ var init_ResourcesUtils = __esm({
               nameInProducer: finalKey,
               aliasInProducer: finalKey,
               dimension: match?.dimension ?? { name: finalKey, type: "string" },
-              measure: match?.measure,
               owner: consumer.name
             });
           }
@@ -13088,11 +13125,8 @@ var init_Validator = __esm({
               return errors2;
             };
             errors.push(...validateTransformations(consumer.fields));
-            const duplicatesOutputs = Algo_default.duplicatesObject(consumer.outputs, "format");
-            if (duplicatesOutputs.length > 0) {
-              const duplicatesTypes = Algo_default.uniq(duplicatesOutputs.map((x2) => x2.format));
-              errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(" and ")})`);
-            }
+            if ((consumer.outputs ?? []).length > 1)
+              errors.push(`Consumer "${consumer.name}" declares ${consumer.outputs.length} outputs (${consumer.outputs.map((x2) => x2.format).join(", ")}). A consumer writes exactly one output \u2014 only the first was ever written \u2014 so split it into one consumer per output.`);
             for (const output of consumer.outputs) {
               const format4 = output.format.toUpperCase();
               if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
@@ -13190,47 +13224,41 @@ var init_Validator = __esm({
           const missingKeys = fieldKeys.includes("*") ? [] : keys.filter((key) => !fieldKeys.includes(key));
           if (missingKeys.length > 0)
             errors.push(`"split.by" references key(s) "${missingKeys.join(", ")}" that are not present in the consumer "${consumer.name}".`);
-          const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
-          if (positional.length === 0) return errors;
+          const format4 = consumer.outputs?.[0]?.format.toUpperCase();
+          if (!positionalFormats.includes(format4)) return errors;
           const hiddenKeys = keys.filter((key) => (consumer.fields ?? []).some((x2) => (x2.alias ?? x2.key) === key && x2.hidden));
           if (hiddenKeys.length > 0)
-            errors.push(`"split.by" of consumer "${consumer.name}" references hidden field(s) "${hiddenKeys.join(", ")}", which the ${positional.map((x2) => x2.format.toUpperCase()).join("/")} output does not write \u2014 a hidden field can't be split on in a positional format.`);
+            errors.push(`"split.by" of consumer "${consumer.name}" references hidden field(s) "${hiddenKeys.join(", ")}", which the ${format4} output does not write \u2014 a hidden field can't be split on in a positional format.`);
           return errors;
         };
         /**
-         * Checks `omitNull`, which drops a record's null fields on export. It only means anything where a
-         * record carries its own field names, and it is applied when the record is serialized — one record
-         * shape for every output of the consumer — so a positional sibling output would silently lose its
-         * column alignment. Both cases are rejected rather than ignored.
+         * Checks `omitNull`, which drops a record's null fields on export. It only means anything where
+         * a record carries its own field names: leaving a positional column out shifts every column
+         * after it, so a positional output declaring it is rejected rather than ignored.
          */
         this.validateOmitNull = (consumer) => {
           const errors = [];
           const positionalFormats = ["CSV", "TXT", "PARQUET"];
           for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
             const format4 = output.format.toUpperCase();
-            if (positionalFormats.includes(format4)) {
+            if (positionalFormats.includes(format4))
               errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", which only applies to JSON/API output: a positional column can't be omitted from a row.`);
-              continue;
-            }
-            const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
-            if (positional.length > 0)
-              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", but the consumer also writes ${positional.map((x2) => x2.format.toUpperCase()).join(" and ")}. Every output is serialized from one record, so omitting fields would break the positional one \u2014 split them into two consumers.`);
           }
           return errors;
         };
         /**
          * Checks `header`, which writes the field names as the first line of an exported file. A JSON
-         * record carries its own field names and an API response is not a file, so the flag means
-         * nothing there — rejected rather than ignored, so a config that asks for a header always gets
-         * one.
+         * record carries its own field names, a parquet file carries its own schema, and an API response
+         * is not a file, so the flag means nothing there — rejected rather than ignored, so a config that
+         * asks for a header always gets one.
          */
         this.validateHeader = (consumer) => {
           const errors = [];
-          const positionalFormats = ["CSV", "TXT", "PARQUET"];
+          const headerFormats = ["CSV", "TXT"];
           for (const output of (consumer.outputs ?? []).filter((x2) => Algo_default.hasVal(x2.header))) {
             const format4 = output.format.toUpperCase();
-            if (!positionalFormats.includes(format4))
-              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: a ${format4} record already carries its own field names.`);
+            if (!headerFormats.includes(format4))
+              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: a ${format4} ${format4 === "PARQUET" ? "file carries its own schema" : "record already carries its own field names"}.`);
           }
           return errors;
         };
@@ -33502,6 +33530,96 @@ var init_CSVParser = __esm({
   }
 });
 
+// ../../packages/parsing/src/DatasetCodec.ts
+var NULL_FIELD, NEEDS_ESCAPE, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
+var init_DatasetCodec = __esm({
+  "../../packages/parsing/src/DatasetCodec.ts"() {
+    init_src();
+    NULL_FIELD = "";
+    NEEDS_ESCAPE = /["\n\r]/;
+    DatasetCodecClass = class {
+      constructor() {
+        /**
+         * One value as an internal positional field: quoted, with `"`, `\n` and `\r` escaped - or the
+         * unquoted null marker when there is no value.
+         */
+        this.encodeField = (value) => {
+          if (value === null || value === void 0) return NULL_FIELD;
+          const text = typeof value === "string" ? value : String(value);
+          if (!NEEDS_ESCAPE.test(text)) return `"${text}"`;
+          return `"${text.replaceAll('"', '""').replaceAll("\r", "\\r").replaceAll("\n", "\\n")}"`;
+        };
+        /** One record as an internal positional line. The values are already in the schema's order. */
+        this.encodeRow = (values, delimiter = ",") => {
+          Affirm_default(values, "Invalid values");
+          Affirm_default(delimiter, "Invalid delimiter");
+          let row = "";
+          for (let index = 0; index < values.length; index++) {
+            if (index > 0)
+              row += delimiter;
+            row += this.encodeField(values[index]);
+          }
+          return row;
+        };
+        /**
+         * One internal positional line split back into its raw fields, with a null still a null. The
+         * array may be SHORTER than the schema's column list; a reader treats a missing entry as null.
+         */
+        this.decodeRow = (line, delimiter = ",") => {
+          Affirm_default(delimiter, "Invalid delimiter");
+          const fields = [];
+          const length = line?.length ?? 0;
+          const delimiterCode = delimiter.charCodeAt(0);
+          let fieldStart = 0;
+          let fieldEnd = 0;
+          let inQuotes = false;
+          let hasQuotes = false;
+          let index = 0;
+          while (index < length) {
+            const char = line.charCodeAt(index);
+            if (char === 34) {
+              if (!inQuotes) {
+                inQuotes = true;
+                hasQuotes = true;
+                fieldStart = index + 1;
+              } else if (line.charCodeAt(index + 1) === 34) {
+                index++;
+              } else {
+                inQuotes = false;
+                fieldEnd = index;
+              }
+            } else if (char === delimiterCode && !inQuotes) {
+              fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
+              fieldStart = index + 1;
+              fieldEnd = 0;
+              hasQuotes = false;
+            }
+            index++;
+          }
+          fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
+          return fields;
+        };
+        /**
+         * Re-encodes an internal line with another delimiter, which is what a `TXT` export is. Not via
+         * `CSVParser`: a plain parse reads both the null marker and `""` as the empty string.
+         */
+        this.redelimit = (line, delimiter) => {
+          Affirm_default(delimiter, "Invalid delimiter");
+          return this.encodeRow(this.decodeRow(line), delimiter);
+        };
+        /** One field's text, or null for the unquoted marker. Only a quoted token carries a value. */
+        this._field = (line, start, end, hasQuotes) => {
+          if (!hasQuotes) return null;
+          const raw = line.slice(start, end);
+          return raw.indexOf('"') === -1 ? raw : raw.replaceAll('""', '"');
+        };
+      }
+    };
+    DatasetCodec = new DatasetCodecClass();
+    DatasetCodec_default = DatasetCodec;
+  }
+});
+
 // ../../packages/parsing/src/FixedWidthParser.ts
 var FixedWidthParserClass, FixedWidthParser, FixedWidthParser_default;
 var init_FixedWidthParser = __esm({
@@ -33714,7 +33832,7 @@ var require_pend = __commonJS({
 // ../../node_modules/yauzl/fd-slicer.js
 var require_fd_slicer = __commonJS({
   "../../node_modules/yauzl/fd-slicer.js"(exports2) {
-    var fs39 = require("fs");
+    var fs40 = require("fs");
     var util = require("util");
     var stream = require("stream");
     var Readable10 = stream.Readable;
@@ -33734,7 +33852,7 @@ var require_fd_slicer = __commonJS({
     FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
       var self2 = this;
       self2.pend.go(function(cb) {
-        fs39.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
+        fs40.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
           cb();
           callback(err2, bytesRead, buffer2);
         });
@@ -33751,7 +33869,7 @@ var require_fd_slicer = __commonJS({
       self2.refCount -= 1;
       if (self2.refCount < 0) throw new Error("invalid unref");
       if (self2.refCount > 0) return;
-      fs39.close(self2.fd, onCloseDone);
+      fs40.close(self2.fd, onCloseDone);
       function onCloseDone(err2) {
         if (err2) {
           self2.emit("error", err2);
@@ -33783,7 +33901,7 @@ var require_fd_slicer = __commonJS({
       }
       self2.context.pend.go(function(cb) {
         var buffer = Buffer.allocUnsafe(toRead);
-        fs39.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
+        fs40.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
           if (err2) {
             self2.destroy(err2);
           } else if (bytesRead === 0) {
@@ -34143,7 +34261,7 @@ var require_crc32 = __commonJS({
 // ../../node_modules/yauzl/index.js
 var require_yauzl = __commonJS({
   "../../node_modules/yauzl/index.js"(exports2) {
-    var fs39 = require("fs");
+    var fs40 = require("fs");
     var zlib3 = require("zlib");
     var fd_slicer = require_fd_slicer();
     var util = require("util");
@@ -34212,10 +34330,10 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs39.open(path30, "r", function(err2, fd) {
+      fs40.open(path30, "r", function(err2, fd) {
         if (err2) return callback(err2);
         fromFd(fd, options2, function(err3, zipfile) {
-          if (err3) fs39.close(fd, defaultCallback);
+          if (err3) fs40.close(fd, defaultCallback);
           callback(err3, zipfile);
         });
       });
@@ -34232,7 +34350,7 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs39.fstat(fd, function(err2, stats) {
+      fs40.fstat(fd, function(err2, stats) {
         if (err2) return callback(err2);
         var reader = new fd_slicer.FdSlicer(fd);
         fromRandomAccessReader(reader, stats.size, options2, callback);
@@ -38311,6 +38429,7 @@ var init_XMLParser = __esm({
 var init_src6 = __esm({
   "../../packages/parsing/src/index.ts"() {
     init_CSVParser();
+    init_DatasetCodec();
     init_FixedWidthParser();
     init_ParseCompression();
     init_ParseHelper();
@@ -47818,7 +47937,7 @@ var init_ProducerManager = __esm({
 });
 
 // ../../packages/engines/src/transform/TypeCaster.ts
-var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TYPE_DEFAULTS, TypeCasterClass, TypeCaster, TypeCaster_default;
+var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TRUE_VALUES, TYPE_DEFAULTS, TypeCasterClass, TypeCaster, TypeCaster_default;
 var init_TypeCaster = __esm({
   "../../packages/engines/src/transform/TypeCaster.ts"() {
     init_src();
@@ -47839,6 +47958,7 @@ var init_TypeCaster = __esm({
       { token: "mm", width: 2 },
       { token: "ss", width: 2 }
     ];
+    TRUE_VALUES = /^(1|true|yes|y|t)$/i;
     TYPE_DEFAULTS = {
       string: "",
       number: 0,
@@ -47848,7 +47968,8 @@ var init_TypeCaster = __esm({
     TypeCasterClass = class {
       constructor() {
         /**
-         * Casts a value for a DECLARED dimension: the type cast, plus what a MISSING value means for it.
+         * Casts a value for a DECLARED field: the type cast, plus what a MISSING value means for it. The
+         * field is a producer dimension on the read path and a dataset column on the re-read path.
          *
          * Missing is null/undefined, and — for every type but `string` — the empty string, which is how
          * a NULL arrives from every source that stages through CSV (SQL, Delta Share, CSV files) and is
@@ -47880,6 +48001,7 @@ var init_TypeCaster = __esm({
         switch (type) {
           case "boolean": {
             if (typeof value === "boolean") return value;
+            if (typeof value === "string") return TRUE_VALUES.test(value.trim());
             return Boolean(value);
           }
           case "datetime":
@@ -48018,6 +48140,7 @@ var init_TypeCaster = __esm({
 var LineParserClass, LineParser, LineParser_default;
 var init_LineParser = __esm({
   "../../packages/engines/src/parsing/LineParser.ts"() {
+    init_src();
     init_TypeCaster();
     init_src6();
     LineParserClass = class {
@@ -48110,22 +48233,23 @@ var init_LineParser = __esm({
           return record;
         };
         /**
-         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
-         */
-        this._internalParseCSV = (line, fields, delimiter) => {
-          const parts = CSVParser_default.parseRow(line, delimiter ?? ",");
-          const record = {};
-          for (const [index, field] of fields.entries()) {
-            const fieldKey = field.finalKey;
-            record[fieldKey] = field.dimension ? TypeCaster_default.castDimension(parts[index], field.dimension) : TypeCaster_default.cast(parts[index], "string");
-          }
-          return record;
-        };
-        /**
-         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
+         * The parser of one staged `.dataset`, compiled once from the schema that file was written with.
+         * The schema is the ONLY description of the file - it has no header - and there is deliberately
+         * no overload taking a consumer's field list, which cannot know about a shape a pass changed.
          */
-        this._internalParseJSON = (line) => {
-          return JSON.parse(line);
+        this.datasetParser = (dsSchema) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default.hasItems(dsSchema.columns, "The dataset schema declares no column to read");
+          if (dsSchema.encoding === "JSON")
+            return (line) => JSON.parse(line);
+          const { columns } = dsSchema;
+          return (line) => {
+            const parts = DatasetCodec_default.decodeRow(line);
+            const record = {};
+            for (let index = 0; index < columns.length; index++)
+              record[columns[index].name] = TypeCaster_default.castDimension(parts[index], columns[index]);
+            return record;
+          };
         };
       }
     };
@@ -48242,7 +48366,7 @@ var init_ProducerEngine = __esm({
                 let effectiveProducer = producer;
                 let effectiveSource = source;
                 if (compressionType) {
-                  scope = { id: (0, import_crypto7.randomUUID)(), folder: `sample-${(0, import_crypto7.randomUUID)()}`, workersId: [] };
+                  scope = ExecutorScope_default.open({ id: (0, import_crypto7.randomUUID)(), folder: `sample-${(0, import_crypto7.randomUUID)()}`, workersId: [] });
                   const driver = await DriverFactory_default.instantiateSource(source);
                   const readyResult = await driver.ready({ producer, scope });
                   const firstFile = readyResult.files[0].fullUri;
@@ -48252,7 +48376,7 @@ var init_ProducerEngine = __esm({
                     ...producer,
                     settings: {
                       ...producer.settings,
-                      fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path18.default.basename(firstFile),
+                      fileKey: readyResult.files.length > 1 ? "%.extract" : import_path18.default.basename(firstFile),
                       compressionType: void 0
                     }
                   };
@@ -51506,8 +51630,7 @@ var init_ConsumerManager = __esm({
                   from: x2.owner
                 },
                 finalKey: x2.nameInProducer,
-                dimension: x2.dimension,
-                measure: x2.measure
+                dimension: x2.dimension
               })));
             } else {
               const from = field.cField.from ?? (consumer.producers.length === 1 ? consumer.producers[0].name : null);
@@ -51519,8 +51642,7 @@ var init_ConsumerManager = __esm({
                     from: col.owner
                   },
                   finalKey: col.nameInProducer,
-                  dimension: col.dimension,
-                  measure: col.measure
+                  dimension: col.dimension
                 });
               });
             }
@@ -51535,13 +51657,12 @@ var init_ConsumerManager = __esm({
                 from: field.cField.from ?? match.column.owner
               },
               finalKey: match.name,
-              dimension: match.column.dimension,
-              measure: match.column.measure
+              dimension: match.column.dimension
             })));
           } else {
             const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
             Affirm_default(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map((x2) => x2.name).join(", ")}")`);
-            expandedFields.push({ ...field, dimension: col.dimension, measure: col.measure });
+            expandedFields.push({ ...field, dimension: col.dimension });
           }
           return expandedFields;
         };
@@ -52603,10 +52724,9 @@ var init_FieldLineage = __esm({
         this._fromProducer = (producer, field) => {
           const node = LineageEngine_default.nodeId("producer", producer.name);
           const dimension = (producer.dimensions ?? []).find((x2) => x2.name === field.key);
-          const measure = (producer.measures ?? []).find((x2) => x2.name === field.key);
-          const applied = dimension?.alias ? [`source column "${dimension.alias}"`] : measure ? ["measure"] : [];
+          const applied = dimension?.alias ? [`source column "${dimension.alias}"`] : [];
           const step = { resource: producer.name, node, field: field.key, applied };
-          if (dimension || measure) return { path: [step] };
+          if (dimension) return { path: [step] };
           if (field.key === "*") return { path: [{ ...step, applied: ["every column of this producer"] }] };
           if (Algo_default.isGlob(field.key)) return { path: [{ ...step, applied: ["every matching column"] }] };
           return { path: [step], unresolved: `"${field.key}" is not a column of producer "${producer.name}"` };
@@ -52652,11 +52772,8 @@ var init_FieldLineage = __esm({
             ...field.hidden ? ["hidden from the export"] : []
           ];
         };
-        /** Every column a producer exposes, dimensions then measures, in declared order. */
-        this._producerColumns = (producer) => [
-          ...(producer.dimensions ?? []).map((x2) => x2.name),
-          ...(producer.measures ?? []).map((x2) => x2.name)
-        ];
+        /** Every column a producer exposes, in declared order. */
+        this._producerColumns = (producer) => (producer.dimensions ?? []).map((x2) => x2.name);
         /** Whether a resource exposes a column by that name. A `*` consumer exposes whatever it is asked. */
         this._exposes = (env2, name, column) => {
           const producer = (env2.producers ?? []).find((x2) => x2.name === name);
@@ -52988,7 +53105,6 @@ var init_ResourceView = __esm({
             ],
             tables: [
               this._dimensionsTable(producer),
-              ...producer.measures?.length ? [this._measuresTable(producer)] : [],
               this._usedByTable(env2, producer),
               this._keyValueTable("Settings", settings)
             ],
@@ -53025,16 +53141,6 @@ var init_ResourceView = __esm({
           ]);
           return this._table("Dimensions", columns, rows);
         };
-        this._measuresTable = (producer) => {
-          const rows = (producer.measures ?? []).map((measure) => [
-            { value: measure.name, note: this._note(measure.description) },
-            { value: measure.sql }
-          ]);
-          return this._table("Measures", [
-            { key: "name", label: "Name" },
-            { key: "sql", label: "SQL" }
-          ], rows);
-        };
         /**
          * Which consumers read each column, the downstream half of field-level lineage. It is the answer
          * to "can this column go", which is a question a wide layout raises constantly and which nothing
@@ -54994,6 +55100,145 @@ var init_RunOverlay = __esm({
   }
 });
 
+// ../../packages/engines/src/parsing/ParquetExporter.ts
+var fs29, PARQUET_TYPES, TRUE_VALUES2, ParquetExporterClass, ParquetExporter, ParquetExporter_default;
+var init_ParquetExporter = __esm({
+  "../../packages/engines/src/parsing/ParquetExporter.ts"() {
+    fs29 = __toESM(require("fs"), 1);
+    init_src();
+    init_src3();
+    init_src6();
+    init_src2();
+    PARQUET_TYPES = {
+      string: "STRING",
+      number: "DOUBLE",
+      boolean: "BOOLEAN",
+      datetime: "TIMESTAMP"
+    };
+    TRUE_VALUES2 = /^(1|true|yes|y|t)$/i;
+    ParquetExporterClass = class {
+      constructor() {
+        /**
+         * Writes `request.sourcePath` to `request.targetPath` as parquet, SNAPPY compressed, one typed
+         * column per visible consumer field.
+         *
+         * Peak memory is bounded by one row group rather than by the dataset: the writer pulls the rows
+         * a group at a time from the generator below, and nothing holds the file as a whole.
+         */
+        this.exportToParquet = async (request) => {
+          Affirm_default(request, "Missing required parameter request");
+          Affirm_default(request.sourcePath, "Missing required parameter request.sourcePath");
+          Affirm_default(request.targetPath, "Missing required parameter request.targetPath");
+          Affirm_default(request.dsSchema, "Missing required parameter request.dsSchema");
+          Affirm_default(request.consumerName, "Missing required parameter request.consumerName");
+          const columns = this._columns(request.dsSchema);
+          Affirm_default.hasItems(columns, `Consumer "${request.consumerName}" has no visible field to export as parquet`);
+          const { parquetWriteRows, fileWriter } = await import("hyparquet-writer");
+          const rowGroupSize = this._rowGroupSize(columns.length);
+          const startedMs = Date.now();
+          await parquetWriteRows({
+            writer: fileWriter(request.targetPath),
+            columns,
+            rows: this._readRecords(request, columns),
+            rowGroupSize
+          });
+          const bytes = fs29.statSync(request.targetPath).size;
+          Logger_default.info(`Wrote parquet "${request.targetPath}" for consumer "${request.consumerName}" (${columns.length} column(s), ${rowGroupSize} rows/group, ${bytes} bytes, ${Date.now() - startedMs}ms)`);
+        };
+        /**
+         * The exported schema, taken from the dataset's own: one column per column ON the line, in line
+         * order. Not from the consumer's fields, which cannot know about a shape a `pivot` changed.
+         */
+        this._columns = (dsSchema) => {
+          return dsSchema.columns.map((column) => ({
+            name: column.name,
+            type: PARQUET_TYPES[column.type] ?? "STRING",
+            nullable: true
+          }));
+        };
+        /**
+         * Rows per row group, derived from a budget of VALUES rather than a fixed row count: a row group
+         * is buffered whole before it can be compressed, so what bounds the export's memory is cells, and
+         * 100,000 rows means 400k values on a 4-column consumer and 130 million on a 1,300-field one.
+         * Clamped at both ends — see the PARQUET_ constants for why each bound is there.
+         */
+        this._rowGroupSize = (columnCount) => {
+          Affirm_default(columnCount > 0, "Cannot size a row group for zero columns");
+          const { PARQUET_ROW_GROUP_CELLS, PARQUET_MIN_ROW_GROUP_ROWS, PARQUET_MAX_ROW_GROUP_ROWS } = Constants_default.defaults;
+          const rows = Math.floor(PARQUET_ROW_GROUP_CELLS / columnCount);
+          return Math.min(Math.max(rows, PARQUET_MIN_ROW_GROUP_ROWS), PARQUET_MAX_ROW_GROUP_ROWS);
+        };
+        /** A positional line read onto the column names by position — the file holds the visible columns only. */
+        this._positionalValues = (line, columns) => {
+          const parts = DatasetCodec_default.decodeRow(line);
+          const values = {};
+          for (const [index, column] of columns.entries())
+            values[column.name] = parts[index];
+          return values;
+        };
+        /**
+         * One value as its column holds it. A missing value is a null in the column rather than the type's
+         * zero, so a reader can still tell "no value" from a real `0`, `false` or empty string.
+         *
+         * Missing is null/undefined, plus — for every type but `string` — the empty string, matching
+         * `TypeCaster.castDimension`: that is how a NULL arrives from every source that stages through
+         * text. An empty string IS a value for a string column and is kept.
+         */
+        this._toParquetValue = (value, column, consumerName) => {
+          if (value === null || value === void 0) return null;
+          if (value === "" && column.type !== "STRING") return null;
+          switch (column.type) {
+            case "BOOLEAN":
+              return typeof value === "boolean" ? value : TRUE_VALUES2.test(String(value).trim());
+            case "DOUBLE":
+              return typeof value === "number" ? value : Number(value);
+            case "TIMESTAMP":
+              return this._toDate(value, column.name, consumerName);
+            default:
+              return typeof value === "string" ? value : String(value);
+          }
+        };
+        /**
+         * A `datetime` column's value as a Date. An unparseable one FAILS the run, naming the field and
+         * the value: a consumer whose transformation reformats a date into "DDMMYYYY" under a `datetime`
+         * dimension is a configuration error, and a timestamp column silently full of 1970 hides it until
+         * someone queries the lake.
+         */
+        this._toDate = (value, name, consumerName) => {
+          const date2 = value instanceof Date ? value : new Date(value);
+          if (isNaN(date2.getTime()))
+            throw new Error(`Consumer "${consumerName}" cannot export field "${name}" as a parquet TIMESTAMP: "${String(value)}" is not a date. The field declares a "datetime" dimension, so either the value or that declaration is wrong.`);
+          return date2;
+        };
+      }
+      /**
+       * The dataset's records, one at a time, coerced to what each column holds. An async generator, so
+       * the writer pulls a row group's worth and no more — the file is never in memory as a whole.
+       *
+       * A parquet consumer stages JSON (see `DatasetSchema.encodingFor`), so the values arrive already
+       * typed. The positional branch covers a dataset staged the other way, reading the row WITHOUT
+       * casting since the cast is what loses a boolean (`Boolean('false')` is `true`).
+       */
+      async *_readRecords(request, columns) {
+        const readStream = fs29.createReadStream(request.sourcePath);
+        try {
+          for await (const line of readLines(readStream)) {
+            const values = request.dsSchema.encoding === "CSV" ? this._positionalValues(line, columns) : JSON.parse(line);
+            const row = {};
+            for (const column of columns)
+              row[column.name] = this._toParquetValue(values[column.name], column, request.consumerName);
+            yield row;
+          }
+        } finally {
+          readStream.destroy();
+        }
+      }
+    };
+    ParquetExporter = new ParquetExporterClass();
+    ParquetExporter_default = ParquetExporter;
+  }
+});
+
 // ../../packages/engines/src/producer/PushdownEngine.ts
 var MISSING_BY_TYPE, PushdownEngineClass, PushdownEngine, PushdownEngine_default;
 var init_PushdownEngine = __esm({
@@ -55826,6 +56071,7 @@ var init_src10 = __esm({
     init_RunOverlay();
     init_RunRegistry();
     init_LineParser();
+    init_ParquetExporter();
     init_DiscoverMerger();
     init_ProducerEngine();
     init_ProducerManager();
@@ -55841,6 +56087,108 @@ var init_src10 = __esm({
   }
 });
 
+// ../../packages/executors/src/DatasetSchema.ts
+var DatasetSchemaClass, DatasetSchema, DatasetSchema_default;
+var init_DatasetSchema = __esm({
+  "../../packages/executors/src/DatasetSchema.ts"() {
+    init_src();
+    init_src10();
+    DatasetSchemaClass = class {
+      constructor() {
+        /**
+         * How a consumer's records are staged on the way to the destination. A consumer declares exactly
+         * one output (`maxItems: 1` in consumer-schema.json), so `outputs[0]` is THE output.
+         */
+        this.encodingFor = (consumer) => {
+          Affirm_default(consumer, "Invalid consumer");
+          Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output`);
+          switch (consumer.outputs[0].format) {
+            case "JSON":
+            case "API":
+              return "JSON";
+            case "PARQUET":
+              return "JSON";
+            case "TXT":
+              return "CSV";
+            case "CSV":
+              return "CSV";
+            default:
+              throw new Error(`Export format ${consumer.outputs[0].format} not implemented yet.`);
+          }
+        };
+        /**
+         * The schema the run's workers write with: one column per VISIBLE consumer field, in field
+         * order. Hidden fields are not on the line at all.
+         */
+        this.compile = (consumer, fields) => {
+          Affirm_default(consumer, "Invalid consumer");
+          Affirm_default(fields, "Invalid consumer fields");
+          const columns = fields.filter((x2) => !x2.cField.hidden).map(this._column);
+          Affirm_default.hasItems(columns, `Consumer "${consumer.name}" has no visible field to write`);
+          return {
+            encoding: this.encodingFor(consumer),
+            columns,
+            rows: 0,
+            writtenBy: "worker"
+          };
+        };
+        /**
+         * The same shape, as the named stage left it. Returned by every pass that rewrites the dataset
+         * without changing its columns, so `rows` always describes the file as it is NOW.
+         */
+        this.rewritten = (dsSchema, writtenBy, rows) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default(rows >= 0, `Invalid row count ${rows} for a dataset written by "${writtenBy}"`);
+          return { ...dsSchema, writtenBy, rows };
+        };
+        /**
+         * The shape a `pivot` leaves behind: the row keys it grouped on, then one column per pivot value.
+         * A pivoted column has no producer dimension behind it - it is an aggregate this pass invented -
+         * so it declares `source: 'pivot'` and a `number` type.
+         */
+        this.forPivot = (dsSchema, rowKeys, pivotColumns, rows) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default.hasItems(rowKeys, "A pivot declares no row key");
+          const keyColumns = rowKeys.map((key) => {
+            const existing = dsSchema.columns.find((column) => column.name === key);
+            Affirm_default(existing, `The pivot row key "${key}" is not a column of the dataset being pivoted`);
+            return existing;
+          });
+          return {
+            encoding: dsSchema.encoding,
+            columns: [...keyColumns, ...pivotColumns.map((name) => ({ name, type: "number", source: "pivot" }))],
+            rows,
+            writtenBy: "pivot"
+          };
+        };
+        /** The column names on a line, in line order - the header of a positional export. */
+        this.names = (dsSchema) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          return dsSchema.columns.map((column) => column.name);
+        };
+        /**
+         * One column, typed from the dimension behind it. `type`, `format` and `nullable` are the only
+         * three properties a cast reads, so carrying those lets a reader cast a staged value back
+         * without going near the consumer config.
+         */
+        this._column = (field) => {
+          if (ConsumerManager_default.isLineCountField(field.cField))
+            return { name: field.finalKey, type: "number", source: "lineCount" };
+          const column = {
+            name: field.finalKey,
+            type: field.dimension?.type ?? "string"
+          };
+          if (field.dimension?.format) column.format = field.dimension.format;
+          if (field.dimension?.nullable === false) column.nullable = false;
+          return column;
+        };
+      }
+    };
+    DatasetSchema = new DatasetSchemaClass();
+    DatasetSchema_default = DatasetSchema;
+  }
+});
+
 // ../../packages/executors/src/OutputExecutor.ts
 var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
 var init_OutputExecutor = __esm({
@@ -55849,35 +56197,24 @@ var init_OutputExecutor = __esm({
     init_src();
     init_src8();
     init_src4();
+    init_src10();
     init_src6();
     init_src2();
+    init_DatasetSchema();
     OutputExecutorClass = class {
       constructor() {
-        this._getInternalRecordFormat = (consumer) => {
-          const output = consumer.outputs[0];
-          switch (output.format) {
-            case "API":
-              return "JSON";
-            case "TXT":
-              return "CSV";
-            default:
-              return output.format;
-          }
-        };
         /**
-         * The output decisions a consumer makes once — the internal format, whether nulls are omitted,
-         * which fields are visible — lifted off the per-record path. `outputRecord` used to re-derive
-         * all three for every record, and `toCSV` allocated a filtered field array on top of that.
+         * The write side of a staged dataset's schema: what to write, in what order, in which encoding.
+         * Derived from the schema, so the writer and every reader share one column list.
          */
-        this.compileOutput = (consumer, fields) => {
+        this.compileOutput = (consumer, dsSchema) => {
           Affirm_default(consumer, "Invalid consumer");
-          Affirm_default(fields, "Invalid consumer fields");
+          Affirm_default(dsSchema, "Invalid dataset schema");
           return {
-            format: this._getInternalRecordFormat(consumer),
+            format: dsSchema.encoding,
             delimiter: ",",
-            visibleKeys: fields.filter((x2) => !x2.cField.hidden).map((x2) => x2.finalKey),
-            hiddenKeys: fields.filter((x2) => x2.cField.hidden).map((x2) => x2.finalKey),
-            omitNull: consumer.outputs.some((x2) => x2.omitNull === true)
+            visibleKeys: DatasetSchema_default.names(dsSchema),
+            omitNull: consumer.outputs[0].omitNull === true
           };
         };
         this.outputRecord = (record, plan) => {
@@ -55890,26 +56227,38 @@ var init_OutputExecutor = __esm({
               throw new Error(`Export format ${plan.format} not implemented yet.`);
           }
         };
+        /**
+         * One positional line of the internal format. `DatasetCodec`, not `CSVParser`: a staged line has
+         * to carry a null that is not an empty string and must never hold a newline.
+         */
         this.toCSV = (record, plan) => {
           const { visibleKeys, delimiter } = plan;
           let row = "";
           for (let index = 0; index < visibleKeys.length; index++) {
             if (index > 0)
               row += delimiter;
-            row += CSVParser_default.escapeField(record[visibleKeys[index]]?.toString() ?? "");
+            row += DatasetCodec_default.encodeField(record[visibleKeys[index]]);
           }
           return row;
         };
+        /**
+         * One JSON object per line, keyed in the CONSUMER's field order rather than the record's own.
+         *
+         * A record arrives keyed in the order its producer's dimensions were read, so two producers of
+         * one consumer wrote the same record as two different lines — and `distinct` and
+         * `no_duplicates`, which compare the line, saw two rows. Building from `visibleKeys` gives every
+         * writer one order, and drops the hidden fields without deleting them out of the caller's record.
+         */
         this.toJSON = (record, plan) => {
-          for (const key of plan.hiddenKeys)
-            delete record[key];
-          if (plan.omitNull) {
-            for (const key of Object.keys(record)) {
-              if (record[key] === null || record[key] === void 0)
-                delete record[key];
-            }
+          const { visibleKeys, omitNull } = plan;
+          const output = {};
+          for (const key of visibleKeys) {
+            const value = record[key];
+            if (omitNull && (value === null || value === void 0))
+              continue;
+            output[key] = value === void 0 ? null : value;
           }
-          return JSON.stringify(record);
+          return JSON.stringify(output);
         };
         /**
          * The exported name of one split file: the destination name with the file's label before the
@@ -55922,66 +56271,80 @@ var init_OutputExecutor = __esm({
           return `${pathName.substring(0, dot)}_${label}${pathName.substring(dot)}`;
         };
         /**
-         * Writes the run's result to every configured destination, one call per file the split pass
+         * Writes the run's result to the consumer's destination, one call per file the split pass
          * produced. The files are passed in rather than read back out of the scope folder: the label a
          * file carries is what names it at the destination, and with no split configured that list is
          * the merged dataset alone, labelled "0" — the suffix an unsplit export has always had.
+         *
+         * One output per consumer (`maxItems: 1` in consumer-schema.json). This used to loop, but only
+         * ever shipped the first entry, and could not have done more: `move` unlinks what it ships.
          */
-        this.exportResult = async (consumer, fields, splitFiles) => {
+        this.exportResult = async (consumer, dsSchema, splitFiles) => {
           Affirm_default.hasItems(splitFiles, `Consumer "${consumer.name}" has no output file to export`);
-          const internalFormat = this._getInternalRecordFormat(consumer);
+          Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output to export to`);
+          Affirm_default(dsSchema, `Consumer "${consumer.name}" has no dataset schema to export against`);
+          const output = consumer.outputs[0];
+          const internalFormat = dsSchema.encoding;
+          const destination = Environment_default.getSource(output.exportDestination);
+          Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
+          const driver = await DriverFactory_default.instantiateDestination(destination);
+          const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
+          Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
+          const header = this._composeHeader(output, dsSchema);
           const results = [];
-          for (const output of consumer.outputs) {
-            const destination = Environment_default.getSource(output.exportDestination);
-            const driver = await DriverFactory_default.instantiateDestination(destination);
-            const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
-            Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
-            const header = this._composeHeader(output, fields);
-            for (const { path: startingPath, label } of splitFiles) {
-              const destinationPath = this.getCompletedPath(destinationName, label);
-              if (output.format === internalFormat) {
-                results.push(await driver.move(startingPath, destinationPath, header));
-              } else {
-                switch (output.format) {
-                  case "CSV":
-                    results.push(
-                      await driver.transformAndMove(startingPath, (line) => {
-                        const parsed = JSON.parse(line);
-                        return Object.keys(parsed).map((x2) => `"${parsed[x2]}"`).join(",");
-                      }, destinationPath, header)
-                    );
-                    break;
-                  case "TXT": {
-                    const delimiter = this._getDelimiter(output);
-                    results.push(
-                      await driver.transformAndMove(
-                        startingPath,
-                        (line) => CSVParser_default.stringifyRow(CSVParser_default.parseRow(line, ","), delimiter),
-                        destinationPath,
-                        header
-                      )
-                    );
-                    break;
-                  }
-                  case "API":
-                  case "JSON":
-                    results.push(
-                      await driver.transformAndMove(startingPath, (line) => {
-                        const parts = CSVParser_default.parseRow(line, ",");
-                        const value = {};
-                        for (const [index, field] of fields.entries())
-                          value[field.finalKey] = parts[index];
-                        return JSON.stringify(value);
-                      }, destinationPath)
-                    );
-                    break;
-                  default:
-                    throw new Error(`Export result to format ${output.format} not implemented yet.`);
-                }
+          for (const { path: startingPath, label } of splitFiles) {
+            const destinationPath = this.getCompletedPath(destinationName, label);
+            if (output.format === internalFormat || output.format === "API") {
+              results.push(await driver.move(startingPath, destinationPath, header));
+              continue;
+            }
+            switch (output.format) {
+              case "TXT": {
+                const delimiter = this._getDelimiter(output);
+                results.push(
+                  await driver.transformAndMove(
+                    startingPath,
+                    (line) => DatasetCodec_default.redelimit(line, delimiter),
+                    destinationPath,
+                    header
+                  )
+                );
+                break;
               }
+              case "PARQUET":
+                results.push(await this._exportParquet({
+                  sourcePath: startingPath,
+                  destinationPath,
+                  dsSchema,
+                  consumerName: consumer.name,
+                  driver
+                }));
+                break;
+              default:
+                throw new Error(`Export result to format ${output.format} not implemented yet.`);
             }
-            return this._unifingResult(results);
           }
+          return this._unifingResult(results);
+        };
+        /**
+         * Ships one split file as parquet. A parquet file is a columnar container whose footer can only
+         * be written once the last row group is encoded, so there is no per-line mapping to hand a
+         * driver: the file is built in the scope folder beside the dataset it is read from, then handed
+         * over as bytes with NO header — a parquet file carries its own schema, and `move` is
+         * byte-faithful in all three destination drivers.
+         *
+         * `move` unlinks what it shipped, so a successful export leaves no staged file behind. A failed
+         * one leaves it in the scope folder, which the run's cleanup removes with everything else.
+         */
+        this._exportParquet = async (plan) => {
+          const parquetPath = `${plan.sourcePath}.parquet`;
+          await ParquetExporter_default.exportToParquet({
+            sourcePath: plan.sourcePath,
+            targetPath: parquetPath,
+            dsSchema: plan.dsSchema,
+            consumerName: plan.consumerName
+          });
+          return plan.driver.move(parquetPath, plan.destinationPath);
         };
         /**
          * Folds the per-file upload results into the one the run reports.
@@ -56003,21 +56366,28 @@ var init_OutputExecutor = __esm({
          * does. Null for a JSON/API output, where a record already carries its own field names, and for
          * an output that turns the header off.
          *
-         * The names are the exported ones (`finalKey`, i.e. the alias where one is set), since a reader
-         * of the file has nothing else to match a column against.
+         * The names come from the dataset's own schema, so a `pivot` is described by the header.
          */
-        this._composeHeader = (output, fields) => {
+        this._composeHeader = (output, dsSchema) => {
           if (output.header === false) return null;
           if (output.format !== "CSV" && output.format !== "TXT") return null;
-          const names = fields.filter((x2) => !x2.cField.hidden).map((x2) => x2.finalKey);
-          return CSVParser_default.stringifyRow(names, this._getDelimiter(output));
+          return CSVParser_default.stringifyRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
         };
         /** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
         this._getDelimiter = (output) => {
           return output.format === "TXT" ? output.delimiter ?? "	" : ",";
         };
         this._getExtension = (output) => {
-          return output.format === "CSV" ? "csv" : output.format === "JSON" ? "jsonl" : "txt";
+          switch (output.format) {
+            case "CSV":
+              return "csv";
+            case "JSON":
+              return "jsonl";
+            case "PARQUET":
+              return "parquet";
+            default:
+              return "txt";
+          }
         };
         this._composeFileName = (consumer, output, extension, executionId) => {
           if (output.exportName && output.exportName.trim().length > 0) {
@@ -56049,7 +56419,7 @@ var init_ConsumerExecutor = __esm({
     init_src();
     init_src10();
     init_OutputExecutor();
-    init_src10();
+    init_DatasetSchema();
     init_src2();
     LIMIT_SCAN_BYTES = 64 * 1024;
     NEWLINE_BYTE = 10;
@@ -56173,13 +56543,20 @@ var init_ConsumerExecutor = __esm({
           }
           return output;
         };
-        this.processLineCount = async (consumer, datasetPath, executorResults) => {
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
-          const outputPlan = OutputExecutor_default.compileOutput(consumer, internalFields);
-          const lineCountFields = ConsumerManager_default.getLineCountFieldsFromExpanded(internalFields, true);
-          if (lineCountFields.length === 0)
-            return 0;
+        /**
+         * Rewrites every `lineCount` field of the merged dataset so it names the line of the SOURCE file
+         * the row came from, by adding each chunk's `rowOffset` to the counter its worker wrote.
+         *
+         * The shape is unchanged, so the schema comes back with a fresh `writtenBy` and the row count
+         * this pass actually saw - which also checks the file still holds what the workers reported.
+         */
+        this.processLineCount = async (consumer, dsSchema, datasetPath, executorResults) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          const lineCountColumns = dsSchema.columns.filter((column) => column.source === "lineCount");
+          if (lineCountColumns.length === 0)
+            return dsSchema;
+          const parseLine = LineParser_default.datasetParser(dsSchema);
+          const outputPlan = OutputExecutor_default.compileOutput(consumer, dsSchema);
           const tempWorkPath = datasetPath + "_tmp";
           const reader = import_fs24.default.createReadStream(datasetPath);
           const writer = import_fs24.default.createWriteStream(tempWorkPath);
@@ -56201,11 +56578,11 @@ var init_ConsumerExecutor = __esm({
             Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
             const rowOffset = currentWorker.rowOffset ?? 0;
             if (rowOffset > 0) {
-              const record = this._parseLine(line, internalRecordFormat, internalFields);
-              for (const field of lineCountFields) {
-                const currentValue = Number(record[field.finalKey]);
+              const record = parseLine(line);
+              for (const column of lineCountColumns) {
+                const currentValue = Number(record[column.name]);
                 Affirm_default(!Number.isNaN(currentValue), `Invalid lineCount value found while processing ${datasetPath}`);
-                record[field.finalKey] = currentValue + rowOffset;
+                record[column.name] = currentValue + rowOffset;
               }
               if (!writer.write(OutputExecutor_default.outputRecord(record, outputPlan) + "\n"))
                 await waitForDrain();
@@ -56228,11 +56605,11 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return processedLineCount;
+          return DatasetSchema_default.rewritten(dsSchema, "lineCount", processedLineCount);
         };
         /**
-         * Cuts the merged dataset down to the run's record limit, and reports the row count it left
-         * behind — or `null` when the dataset was already within the limit and nothing was touched.
+         * Cuts the merged dataset down to the run's record limit, and reports the shape it left behind.
+         * The schema comes back unchanged when the dataset was already within the limit.
          *
          * Every worker stops at the limit within its OWN chunk, which is the right thing for the read
          * (no thread reads further than the run could need) but leaves up to `limit` rows per chunk in
@@ -56242,7 +56619,8 @@ var init_ConsumerExecutor = __esm({
          * at then needs no arithmetic on line contents, so no line ending or multi-byte character can
          * put the cut in the middle of a row. Only the bytes up to the limit are ever read.
          */
-        this.processLimit = async (datasetPath, limit) => {
+        this.processLimit = async (dsSchema, datasetPath, limit) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           Affirm_default(datasetPath, "Invalid dataset path");
           Affirm_default(limit > 0, `Invalid record limit ${limit}`);
           const handle = await import_promises16.default.open(datasetPath, "r+");
@@ -56253,7 +56631,7 @@ var init_ConsumerExecutor = __esm({
             while (newlines < limit) {
               const { bytesRead } = await handle.read(buffer, 0, LIMIT_SCAN_BYTES, offset);
               if (bytesRead === 0)
-                return null;
+                return dsSchema;
               for (let index = 0; index < bytesRead; index++) {
                 offset++;
                 if (buffer[index] !== NEWLINE_BYTE) continue;
@@ -56263,14 +56641,15 @@ var init_ConsumerExecutor = __esm({
             }
             const { size } = await handle.stat();
             if (offset >= size)
-              return null;
+              return dsSchema;
             await handle.truncate(offset);
-            return limit;
+            return DatasetSchema_default.rewritten(dsSchema, "limit", limit);
           } finally {
             await handle.close();
           }
         };
-        this.processDistinct = async (datasetPath) => {
+        this.processDistinct = async (dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const reader = import_fs24.default.createReadStream(datasetPath);
           const tempWorkPath = datasetPath + "_tmp";
           const writer = import_fs24.default.createWriteStream(tempWorkPath);
@@ -56299,18 +56678,18 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return newLineCount;
+          return DatasetSchema_default.rewritten(dsSchema, "distinct", newLineCount);
         };
-        this.processDistinctOn = async (consumer, datasetPath) => {
+        this.processDistinctOn = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const reader = import_fs24.default.createReadStream(datasetPath);
           const { distinctOn } = consumer.options;
           const { keys, resolution } = distinctOn;
           const { strategy, orderBy, direction = "asc" } = resolution;
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           const winners = /* @__PURE__ */ new Map();
           for await (const line of readLines(reader)) {
-            const record = internalRecordFormat === "CSV" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
+            const record = parseLine(line);
             const compositeKey = keys.map((k7) => String(record[k7] ?? "")).join("|");
             const existing = winners.get(compositeKey);
             if (!existing) {
@@ -56342,20 +56721,27 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return winners.size;
+          return DatasetSchema_default.rewritten(dsSchema, "distinctOn", winners.size);
         };
-        this.processPivot = async (consumer, datasetPath) => {
+        /**
+         * Groups the dataset by `rowKeys` and turns one column's values into columns of their own.
+         *
+         * This is the one pass that changes the SHAPE of the file, which is why it returns the schema it
+         * wrote rather than a row count: the header, the split pass, the dataset validation and the
+         * parquet schema all read the result.
+         */
+        this.processPivot = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const { pivot } = consumer.options;
           const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           let pivotValues = pivot.pivotValues;
           if (!pivotValues) {
             pivotValues = [];
             const discoverySet = /* @__PURE__ */ new Set();
             const discoverReader = import_fs24.default.createReadStream(datasetPath);
             for await (const line of readLines(discoverReader)) {
-              const record = this._parseLine(line, internalRecordFormat, internalFields);
+              const record = parseLine(line);
               const val = String(record[pivotColumn] ?? "");
               if (!discoverySet.has(val)) {
                 discoverySet.add(val);
@@ -56372,7 +56758,7 @@ var init_ConsumerExecutor = __esm({
           const groups = /* @__PURE__ */ new Map();
           const reader = import_fs24.default.createReadStream(datasetPath);
           for await (const line of readLines(reader)) {
-            const record = this._parseLine(line, internalRecordFormat, internalFields);
+            const record = parseLine(line);
             const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
             const pivotVal = String(record[pivotColumn] ?? "");
             const numericVal = Number(record[valueColumn]) || 0;
@@ -56391,11 +56777,8 @@ var init_ConsumerExecutor = __esm({
             cell.min = Math.min(cell.min, numericVal);
             cell.max = Math.max(cell.max, numericVal);
           }
-          const pivotedFields = [
-            ...rowKeys.map((k7) => ({ cField: { key: k7 }, finalKey: k7 })),
-            ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
-          ];
-          const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedFields);
+          const pivotedDsSchema = DatasetSchema_default.forPivot(dsSchema, rowKeys, pivotValues.map((pv) => columnPrefix + pv), groups.size);
+          const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedDsSchema);
           const tempWorkPath = datasetPath + "_tmp";
           const writer = import_fs24.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
@@ -56445,10 +56828,8 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return outputCount;
-        };
-        this._parseLine = (line, format4, fields) => {
-          return format4 === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+          Affirm_default(outputCount === pivotedDsSchema.rows, `Pivot wrote ${outputCount} row(s) but declared ${pivotedDsSchema.rows}`);
+          return pivotedDsSchema;
         };
         /**
          * Determines if the new record should replace the existing record based on the resolution strategy
@@ -56475,11 +56856,11 @@ var init_ConsumerExecutor = __esm({
               return false;
           }
         };
-        this.processDatasetValidation = async (consumer, datasetPath) => {
+        this.processDatasetValidation = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const validations = consumer.validate;
           if (!validations || validations.length === 0) return [];
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           let rowCount = 0;
           const seenRows = /* @__PURE__ */ new Set();
           const fieldValueSets = /* @__PURE__ */ new Map();
@@ -56500,7 +56881,7 @@ var init_ConsumerExecutor = __esm({
                 seenRows.add(line);
             }
             if (uniqueFieldKeys.length > 0) {
-              const record = internalRecordFormat === "CSV" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
+              const record = parseLine(line);
               for (const fieldKey of uniqueFieldKeys) {
                 const valueSet = fieldValueSets.get(fieldKey);
                 const val = String(record[fieldKey] ?? "");
@@ -56637,14 +57018,13 @@ var init_ExecutorPerformance = __esm({
 });
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs30, ExecutorWriter, ExecutorWriter_default;
+var fs31, ExecutorWriter, ExecutorWriter_default;
 var init_ExecutorWriter = __esm({
   "../../packages/executors/src/ExecutorWriter.ts"() {
-    fs30 = __toESM(require("fs"));
+    fs31 = __toESM(require("fs"));
     init_src();
     init_src3();
     init_src10();
-    init_OutputExecutor();
     ExecutorWriter = class {
       constructor() {
         /**
@@ -56674,7 +57054,8 @@ var init_ExecutorWriter = __esm({
           Affirm_default(request, "Invalid split request");
           Affirm_default(request.sourcePath, "Missing dataset to split");
           Affirm_default(request.consumer, "Invalid consumer");
-          const { scope, sourcePath, consumer, fields } = request;
+          const { scope, sourcePath, consumer, dsSchema } = request;
+          Affirm_default(dsSchema, "Missing dataset schema to split against");
           const keys = scope.split?.by ?? [];
           const maxFileBytes = scope.split?.maxFileSizeGb > 0 ? scope.split.maxFileSizeGb * this.BYTES_PER_GB : null;
           if (keys.length === 0 && !maxFileBytes)
@@ -56689,21 +57070,21 @@ var init_ExecutorWriter = __esm({
             usedLabels: /* @__PURE__ */ new Set(),
             fileCount: 0
           };
-          await this._route(pass, fields);
+          await this._route(pass, dsSchema);
           if (pass.buckets.size === 0)
             return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
-          await fs30.promises.unlink(sourcePath);
+          await fs31.promises.unlink(sourcePath);
           return this._listFiles(pass);
         };
         /**
          * The pass itself: one line read, one line written, for every line of the dataset.
          */
-        this._route = async (pass, fields) => {
-          const internalFormat = OutputExecutor_default._getInternalRecordFormat(pass.consumer);
-          const readStream = fs30.createReadStream(pass.sourcePath);
+        this._route = async (pass, dsSchema) => {
+          const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
+          const readStream = fs31.createReadStream(pass.sourcePath);
           try {
             for await (const line of readLines(readStream)) {
-              const bucket = this._bucketFor(line, pass, internalFormat, fields);
+              const bucket = this._bucketFor(line, pass, parseLine);
               const lineBytes = Buffer.byteLength(line) + 1;
               if (this._needsRotation(bucket, pass, lineBytes))
                 await this._rotate(bucket, pass);
@@ -56723,8 +57104,8 @@ var init_ExecutorWriter = __esm({
          * own: a shared catch-all would silently merge two different things, and that is invisible until
          * it is wrong.
          */
-        this._bucketFor = (line, pass, internalFormat, fields) => {
-          const bucketKey = pass.keys.length === 0 ? "" : this._bucketKey(line, pass.keys, internalFormat, fields);
+        this._bucketFor = (line, pass, parseLine) => {
+          const bucketKey = parseLine ? this._bucketKey(line, pass.keys, parseLine) : "";
           const existing = pass.buckets.get(bucketKey);
           if (existing) return existing;
           const bucket = {
@@ -56738,8 +57119,8 @@ var init_ExecutorWriter = __esm({
           return bucket;
         };
         /** The composite value a line buckets on, read off the parsed record. */
-        this._bucketKey = (line, keys, internalFormat, fields) => {
-          const record = internalFormat === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+        this._bucketKey = (line, keys, parseLine) => {
+          const record = parseLine(line);
           return keys.map((key) => {
             const value = record[key];
             if (value === null || value === void 0) return "_null";
@@ -56804,7 +57185,7 @@ var init_ExecutorWriter = __esm({
           }
           if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
             await this._closeStream(pass.openStreams.shift());
-          bucket.stream = fs30.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
+          bucket.stream = fs31.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
           pass.openStreams.push(bucket);
           return bucket.stream;
         };
@@ -57093,7 +57474,6 @@ var init_LookupResolver = __esm({
     init_src4();
     init_src10();
     init_src2();
-    init_OutputExecutor();
     LookupResolverClass = class {
       constructor() {
         // Reuse dicts across sibling consumers within one process; invalidated when the config changes.
@@ -57116,28 +57496,35 @@ var init_LookupResolver = __esm({
               Affirm_default(!stack.includes(ref.consumer), `code_lookup cycle detected: ${[...stack, ref.consumer].join(" -> ")}`);
               const lookupConsumer = Environment_default.getConsumer(ref.consumer);
               Affirm_default(lookupConsumer, `code_lookup references unknown consumer "${ref.consumer}"`);
-              let capturePath = captureByConsumer.get(ref.consumer);
-              if (!capturePath) {
-                capturePath = this.getCapturePath(scope, ref.consumer);
+              let capture = captureByConsumer.get(ref.consumer);
+              if (!capture) {
+                const capturePath = this.getCapturePath(scope, ref.consumer);
                 ExecutorScope_default.ensurePath(capturePath);
-                await this.runLookupConsumer(lookupConsumer, capturePath, stack, request);
-                captureByConsumer.set(ref.consumer, capturePath);
+                capture = {
+                  path: capturePath,
+                  dsSchema: await this.runLookupConsumer(lookupConsumer, capturePath, stack, request)
+                };
+                captureByConsumer.set(ref.consumer, capture);
               }
-              this.cache.set(id, await this.buildDict(lookupConsumer, capturePath, ref));
+              this.cache.set(id, await this.buildDict(capture, ref));
             }
             ExecutorScope_default.ensurePath(outPath);
             await import_promises17.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
             manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path26.default.resolve(outPath) });
           }
-          for (const capturePath of captureByConsumer.values())
-            await import_promises17.default.rm(capturePath, { force: true });
+          for (const capture of captureByConsumer.values())
+            await import_promises17.default.rm(capture.path, { force: true });
           return manifest;
         };
-        // Runs the source consumer through the real pipeline, capturing its merged output before cleanup.
+        /**
+         * Runs the source consumer through the real pipeline, capturing its merged output before
+         * cleanup and returning the shape that run produced. The capture is a verbatim copy of the
+         * dataset, so its schema is the ONLY description of it.
+         */
         this.runLookupConsumer = async (lookupConsumer, capturePath, stack, request) => {
           const { default: ExecutorOrchestrator2 } = await Promise.resolve().then(() => (init_ExecutorOrchestrator(), ExecutorOrchestrator_exports));
           Logger_default.log(`[lookup] Materializing consumer "${lookupConsumer.name}" (path: ${stack.join(" -> ")})`);
-          await ExecutorOrchestrator2.launch({
+          const result = await ExecutorOrchestrator2.launch({
             consumer: lookupConsumer,
             details: request.details,
             captureMainDatasetTo: capturePath,
@@ -57145,22 +57532,23 @@ var init_LookupResolver = __esm({
             _lookupStack: stack,
             syntheticProducerInputs: request.syntheticProducerInputs
           });
+          Affirm_default(result.dsSchema, `code_lookup source consumer "${lookupConsumer.name}" reported no dataset schema`);
+          return result.dsSchema;
         };
         // Projects the captured dataset to a { code: value } dict, bounded by MAX_LOOKUP_ROWS.
-        this.buildDict = async (lookupConsumer, capturePath, ref) => {
-          const format4 = OutputExecutor_default._getInternalRecordFormat(lookupConsumer);
-          const visibleFields = ConsumerManager_default.getExpandedFields(lookupConsumer).filter((f7) => !f7.cField.hidden);
-          const finalKeys = visibleFields.map((f7) => f7.finalKey);
-          Affirm_default(finalKeys.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
-          Affirm_default(finalKeys.includes(ref.value), `code_lookup value column "${ref.value}" is not in the output of consumer "${ref.consumer}"`);
+        this.buildDict = async (capture, ref) => {
+          const columnNames = capture.dsSchema.columns.map((column) => column.name);
+          Affirm_default(columnNames.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
+          Affirm_default(columnNames.includes(ref.value), `code_lookup value column "${ref.value}" is not in the output of consumer "${ref.consumer}"`);
+          const parseLine = LineParser_default.datasetParser(capture.dsSchema);
           const dict = {};
           let rowsRead = 0;
           let warnedDup = false;
-          for await (const line of readLines(import_fs25.default.createReadStream(capturePath))) {
+          for await (const line of readLines(import_fs25.default.createReadStream(capture.path))) {
             if (!line.length) continue;
             rowsRead++;
             Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
-            const record = this.parseLine(line, format4, visibleFields);
+            const record = parseLine(line);
             const code = String(record[ref.key]);
             if (code in dict && !warnedDup) {
               Logger_default.warn(`code_lookup: duplicate key(s) in consumer "${ref.consumer}" for "${ref.key}"; last value wins`);
@@ -57171,10 +57559,6 @@ var init_LookupResolver = __esm({
           Logger_default.log(`[lookup] Built table "${ref.consumer}" (${ref.key}->${ref.value}): ${Object.keys(dict).length} entr(ies) from ${rowsRead} row(s)`);
           return dict;
         };
-        this.parseLine = (line, format4, visibleFields) => {
-          if (format4 === "CSV") return LineParser_default._internalParseCSV(line, visibleFields, ",");
-          return JSON.parse(line);
-        };
         this.getCapturePath = (scope, consumerName) => {
           const dir = import_path26.default.dirname(ExecutorScope_default.getLookupPath(scope, "x"));
           return import_path26.default.join(dir, `${consumerName}.capture`);
@@ -57208,7 +57592,6 @@ var init_ConsumerInputResolver = __esm({
     init_src();
     init_src3();
     init_src4();
-    init_src10();
     init_src2();
     ConsumerInputResolverClass = class {
       constructor() {
@@ -57233,23 +57616,29 @@ var init_ConsumerInputResolver = __esm({
               Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
               const capturePath = ExecutorScope_default.getConsumerInputPath(scope, name);
               ExecutorScope_default.ensurePath(capturePath);
-              await this.runUpstreamConsumer(upstream, capturePath, stack, request);
+              const dsSchema = await this.runUpstreamConsumer(upstream, capturePath, stack, request);
               const { size } = await import_promises18.default.stat(capturePath);
-              this.cache.set(name, { consumer: name, path: import_path27.default.resolve(capturePath), fileSize: size });
+              this.cache.set(name, { consumer: name, path: import_path27.default.resolve(capturePath), fileSize: size, dsSchema });
             }
             manifest.set(name, this.cache.get(name));
           }
           return manifest;
         };
-        // Builds the in-memory producer the orchestrator reads the JSONL capture through. Its dimensions
-        // are the upstream consumer's visible output fields, matching the keys written into the capture.
-        this.buildSyntheticProducer = (upstreamConsumer) => {
-          Affirm_default(upstreamConsumer, "Invalid upstream consumer");
-          const dimensions = ConsumerManager_default.getExpandedFields(upstreamConsumer).filter((field) => !field.cField.hidden).map((field) => ({ name: field.finalKey, type: field.dimension?.type ?? "string" }));
+        /**
+         * Builds the in-memory producer the orchestrator reads the JSONL capture through. Its dimensions
+         * are the columns the upstream run REPORTED writing, not the consumer's declared fields, which a
+         * `pivot` leaves describing a shape no longer in the file.
+         *
+         * No `format` is carried over: the capture holds values the upstream run already rendered, so
+         * re-parsing them with that dimension's input format would fail on its own output.
+         */
+        this.buildSyntheticProducer = (entry) => {
+          Affirm_default(entry, "Invalid consumer input entry");
+          Affirm_default(entry.dsSchema, `Consumer input "${entry.consumer}" carries no dataset schema`);
           return {
-            name: upstreamConsumer.name,
-            source: `__consumer_input__:${upstreamConsumer.name}`,
-            dimensions,
+            name: entry.consumer,
+            source: `__consumer_input__:${entry.consumer}`,
+            dimensions: entry.dsSchema.columns.map((column) => ({ name: column.name, type: column.type })),
             settings: { fileType: "JSONL" },
             _version: 1
           };
@@ -57264,11 +57653,12 @@ var init_ConsumerInputResolver = __esm({
           }
           return names;
         };
-        // Runs the upstream consumer through the real pipeline, capturing its full output as JSONL.
+        // Runs the upstream consumer through the real pipeline, capturing its output as JSONL and
+        // reporting the shape that run produced.
         this.runUpstreamConsumer = async (upstream, capturePath, stack, request) => {
           const { default: ExecutorOrchestrator2 } = await Promise.resolve().then(() => (init_ExecutorOrchestrator(), ExecutorOrchestrator_exports));
           Logger_default.log(`[consumer-input] Materializing consumer "${upstream.name}" (path: ${stack.join(" -> ")})`);
-          await ExecutorOrchestrator2.launch({
+          const result = await ExecutorOrchestrator2.launch({
             consumer: upstream,
             details: request.details,
             captureMainDatasetTo: capturePath,
@@ -57277,6 +57667,8 @@ var init_ConsumerInputResolver = __esm({
             _consumerInputStack: stack,
             syntheticProducerInputs: request.syntheticProducerInputs
           });
+          Affirm_default(result.dsSchema, `consumer-as-input source "${upstream.name}" reported no dataset schema`);
+          return result.dsSchema;
         };
         this.invalidateCacheIfStale = () => {
           const generation = Environment_default._configHash ?? null;
@@ -57318,6 +57710,7 @@ var init_ExecutorOrchestrator = __esm({
     init_src();
     init_src10();
     init_ExecutorWriter();
+    init_DatasetSchema();
     import_promises20 = require("stream/promises");
     init_src2();
     init_ExecutorProgress2();
@@ -57336,9 +57729,9 @@ var init_ExecutorOrchestrator = __esm({
          * spawn lazily, on first exec, and a thread that dies mid-run is replaced by another, so this is
          * a count of spawns and NOT a concurrency figure (see IWorkerCensus).
          *
-         * `workerData` carries the loaded environment, so it is cloned once per THREAD at spawn rather
-         * than read and validated from disk once per chunk. Only run-constant data belongs there; per-task
-         * data goes in the exec argument, which is cloned again for every chunk.
+         * `workerData` carries the loaded environment and the run's dataset schema, so both are cloned
+         * once per THREAD at spawn rather than per chunk. Only run-constant data belongs there; per-task
+         * data goes in the exec argument, cloned per chunk.
          */
         this.createPool = (maxWorkers, census, threadData) => {
           const options2 = {
@@ -57369,8 +57762,9 @@ var init_ExecutorOrchestrator = __esm({
           const tracker = new ExecutorPerformance_default();
           const _progress = new ExecutorProgress2_default(logProgress);
           const { usageId } = UsageManager_default.startUsage(consumer, details);
-          const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], split: OrchestratorHelper_default.resolveSplit(consumer) };
+          const scope = ExecutorScope_default.open({ id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], split: OrchestratorHelper_default.resolveSplit(consumer) });
           const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
+          let dsSchema = DatasetSchema_default.compile(consumer, expandedFields);
           const census = { created: 0, peak: 0 };
           try {
             const start = performance.now();
@@ -57405,6 +57799,7 @@ var init_ExecutorOrchestrator = __esm({
             executorResults.push(...await this.runWorkFiles({
               workFiles,
               consumer,
+              dsSchema,
               scope,
               options: options2,
               lookups,
@@ -57419,58 +57814,26 @@ var init_ExecutorOrchestrator = __esm({
             Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x2) => x2.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x2) => x2.outputCount))}`);
             _progress.update({ phase: "Reconciling work", progress: 0 });
             await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+            dsSchema = DatasetSchema_default.rewritten(dsSchema, "merge", Algo_default.sum(executorResults.map((x2) => x2.outputCount)));
             _progress.update({ phase: "Reconciling work", progress: 1 });
             Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
             _progress.update({ phase: "Post-processing", progress: 0 });
-            const postOperation = { totalOutputCount: null };
-            if (executorResults.length > 1) {
-              if (consumer.options?.distinct === true) {
-                Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-distinct:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (consumer.options?.distinctOn) {
-                Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-distinct-on:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (consumer.options?.pivot) {
-                Logger_default.log(`[${usageId}] Running pivot operation`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-pivot:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (ConsumerManager_default.hasLineCountFields(consumer, true)) {
-                Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
-                counter = performance.now();
-                const processedLineCount = await ConsumerExecutor_default.processLineCount(consumer, ExecutorScope_default.getMainPath(scope), executorResults);
-                tracker.measure("process-line-count:main", performance.now() - counter);
-                Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${processedLineCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-            }
+            dsSchema = await this.runPostPasses({ consumer, dsSchema, scope, executorResults, tracker, usageId });
             if (options2?.limit > 0) {
               counter = performance.now();
-              const limitedOutputCount = await ConsumerExecutor_default.processLimit(ExecutorScope_default.getMainPath(scope), options2.limit);
+              const limited = await ConsumerExecutor_default.processLimit(dsSchema, ExecutorScope_default.getMainPath(scope), options2.limit);
               tracker.measure("process-limit:main", performance.now() - counter);
-              if (Algo_default.hasVal(limitedOutputCount)) {
-                postOperation.totalOutputCount = limitedOutputCount;
-                Logger_default.log(`[${usageId}] Record limit applied: dataset cut to ${limitedOutputCount} row(s) in ${Math.round(performance.now() - counter)}ms`);
+              if (limited.writtenBy === "limit") {
+                Logger_default.log(`[${usageId}] Record limit applied: dataset cut to ${limited.rows} row(s) in ${Math.round(performance.now() - counter)}ms`);
               } else {
                 Logger_default.log(`[${usageId}] Record limit of ${options2.limit} not reached \u2014 dataset left whole`);
               }
+              dsSchema = limited;
             }
             if (consumer.validate && consumer.validate.length > 0) {
               Logger_default.log(`[${usageId}] Running dataset-level validations`);
               counter = performance.now();
-              const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default.getMainPath(scope));
+              const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, dsSchema, ExecutorScope_default.getMainPath(scope));
               tracker.measure("dataset-validation", performance.now() - counter);
               for (const result of validationResults) {
                 if (result.onFail === "fail") {
@@ -57484,7 +57847,7 @@ var init_ExecutorOrchestrator = __esm({
               Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
             }
             if (request.captureMainDatasetTo) {
-              await this.captureDataset(consumer, expandedFields, ExecutorScope_default.getMainPath(scope), request.captureMainDatasetTo, request.captureFormat);
+              await this.captureDataset(dsSchema, ExecutorScope_default.getMainPath(scope), request.captureMainDatasetTo, request.captureFormat);
               Logger_default.log(`[${usageId}] Captured dataset to ${request.captureMainDatasetTo}${request.captureFormat ? ` (as ${request.captureFormat})` : ""}`);
             }
             if (request.skipExport) {
@@ -57493,8 +57856,8 @@ var init_ExecutorOrchestrator = __esm({
               const materializeResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, null);
               materializeResult.elapsedMS = performance.now() - start;
               materializeResult.peakWorkers = census.peak;
-              if (Algo_default.hasVal(postOperation.totalOutputCount))
-                materializeResult.outputCount = postOperation.totalOutputCount;
+              materializeResult.outputCount = dsSchema.rows;
+              materializeResult.dsSchema = dsSchema;
               UsageManager_default.endUsage(usageId, materializeResult.outputCount, materializeResult);
               await Logger_default.flush();
               return materializeResult;
@@ -57504,7 +57867,7 @@ var init_ExecutorOrchestrator = __esm({
               scope,
               sourcePath: ExecutorScope_default.getMainPath(scope),
               consumer,
-              fields: expandedFields
+              dsSchema
             });
             tracker.measure("split-output", performance.now() - counter);
             if (scope.split)
@@ -57513,7 +57876,7 @@ var init_ExecutorOrchestrator = __esm({
             _progress.update({ phase: "Exporting result", progress: 0 });
             counter = performance.now();
             Logger_default.info(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-            const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, splitFiles);
+            const exportRes = await OutputExecutor_default.exportResult(consumer, dsSchema, splitFiles);
             tracker.measure("export-result", performance.now() - counter);
             Affirm_default(exportRes.res, `Consumer "${consumer.name}" failed to write its output to "${consumer.outputs.map((x2) => x2.exportDestination).join(", ")}" \u2014 see the logged driver error for the cause`);
             Logger_default.info(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
@@ -57530,8 +57893,8 @@ var init_ExecutorOrchestrator = __esm({
             const finalResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
             finalResult.elapsedMS = performance.now() - start;
             finalResult.peakWorkers = census.peak;
-            if (Algo_default.hasVal(postOperation.totalOutputCount))
-              finalResult.outputCount = postOperation.totalOutputCount;
+            finalResult.outputCount = dsSchema.rows;
+            finalResult.dsSchema = dsSchema;
             UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
             Logger_default.info(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} chunk(s) on ${census.peak} thread(s), ${Math.round(finalResult.elapsedMS)}ms`);
             await Logger_default.flush();
@@ -57609,7 +57972,10 @@ var init_ExecutorOrchestrator = __esm({
             `Invalid worker pool size ${poolSize} \u2014 it must come from getParallelWorkerCount(), never from the work list`
           );
           let phase = performance.now();
-          const pool = this.createPool(poolSize, census, { environment: Environment_default.createSnapshot() });
+          const pool = this.createPool(poolSize, census, {
+            environment: Environment_default.createSnapshot(),
+            dsSchema: request.dsSchema
+          });
           tracker.measure("pool-create", performance.now() - phase);
           const context = {
             consumer: request.consumer,
@@ -57832,24 +58198,23 @@ var init_ExecutorOrchestrator = __esm({
         };
         /**
          * Copies the merged main.dataset out of the run scope for a materialization run. With
-         * captureFormat 'JSONL' the output is normalized to newline-delimited JSON objects keyed by
-         * each visible field's finalKey — self-describing, so a synthetic producer can read it without a
+         * captureFormat 'JSONL' the output is normalized to newline-delimited JSON objects keyed by the
+         * dataset's own column names — self-describing, so a synthetic producer can read it without a
          * header row. main.dataset in JSON internal format is already JSONL (verbatim copy); CSV internal
          * format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
          */
-        this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
+        this.captureDataset = async (dsSchema, sourcePath, targetPath, captureFormat) => {
           await import_promises19.default.mkdir(import_path28.default.dirname(targetPath), { recursive: true });
-          const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
+          if (captureFormat !== "JSONL" || dsSchema.encoding !== "CSV") {
             await import_promises19.default.copyFile(sourcePath, targetPath);
             return;
           }
-          const visibleFields = fields.filter((field) => !field.cField.hidden);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           const output = import_fs27.default.createWriteStream(targetPath);
           try {
             for await (const line of readLines(import_fs27.default.createReadStream(sourcePath))) {
               if (!line.length) continue;
-              const record = LineParser_default._internalParseCSV(line, visibleFields, ",");
+              const record = parseLine(line);
               if (!output.write(JSON.stringify(record) + "\n"))
                 await new Promise((resolve) => output.once("drain", resolve));
             }
@@ -57862,7 +58227,7 @@ var init_ExecutorOrchestrator = __esm({
           }
         };
         /**
-         * Grabs the remote datasets and creates an internal .dataset to work on
+         * Grabs the remote datasets and stages each one as a local `.extract` to work on
          */
         this.readySourceFiles = async (args) => {
           const { consumer, scope, consumerInputs, syntheticInputs, options: options2, onProgress } = args;
@@ -57872,7 +58237,7 @@ var init_ExecutorOrchestrator = __esm({
             const reportProducer = (fraction) => onProgress?.((i6 + fraction) / consumer.producers.length);
             const consumerInput = consumerInputs.get(cProd.name);
             if (consumerInput) {
-              const prod2 = ConsumerInputResolver_default.buildSyntheticProducer(Environment_default.getConsumer(cProd.name));
+              const prod2 = ConsumerInputResolver_default.buildSyntheticProducer(consumerInput);
               const response2 = { files: [{ fullUri: consumerInput.path, fileSize: consumerInput.fileSize }] };
               Logger_default.log(`[${scope.id}] Producer "${cProd.name}" is a consumer input: ${consumerInput.fileSize} bytes (${i6 + 1}/${consumer.producers.length})`);
               results.push({ prod: prod2, cProd, response: response2 });
@@ -58021,6 +58386,49 @@ var init_ExecutorOrchestrator = __esm({
           }
           return mainPath;
         };
+        /**
+         * The passes that work across lines, run on the merged dataset. Returns the schema the last
+         * pass to write the file left behind — its `rows` is the run's row count.
+         *
+         * `lineCount` goes FIRST: it walks each chunk's `outputCount` in order to find the worker a line
+         * came from, which only holds while the file still has sum(outputCount) lines in worker order.
+         */
+        this.runPostPasses = async (request) => {
+          const { consumer, scope, executorResults, tracker, usageId } = request;
+          const datasetPath = ExecutorScope_default.getMainPath(scope);
+          const isMerged = executorResults.length > 1;
+          let dsSchema = request.dsSchema;
+          let counter = 0;
+          if (isMerged && ConsumerManager_default.hasLineCountFields(consumer, true)) {
+            Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processLineCount(consumer, dsSchema, datasetPath, executorResults);
+            tracker.measure("process-line-count:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (isMerged && consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processDistinct(dsSchema, datasetPath);
+            tracker.measure("process-distinct:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${dsSchema.rows} unique rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (isMerged && consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processDistinctOn(consumer, dsSchema, datasetPath);
+            tracker.measure("process-distinct-on:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (consumer.options?.pivot) {
+            Logger_default.log(`[${usageId}] Running pivot operation`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processPivot(consumer, dsSchema, datasetPath);
+            tracker.measure("process-pivot:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] Pivot complete: ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          return dsSchema;
+        };
         this.performCleanupOperations = async (scope, tracker) => {
           const start = performance.now();
           await ExecutorScope_default.clearScope(scope);
@@ -58518,6 +58926,7 @@ var LicenceManager_default = LicenceManager;
 // ../../packages/executors/src/index.ts
 init_src();
 init_ConsumerExecutor();
+init_DatasetSchema();
 
 // ../../packages/executors/src/Executor.ts
 init_ProducerExecutor();
@@ -58663,6 +59072,7 @@ init_src2();
 // ../../packages/executors/src/index.ts
 init_ExecutorOrchestrator();
 init_ExecutorPerformance();
+init_ExecutorWriter();
 init_OrchestratorHelper();
 
 // ../../packages/executors/src/workers/ExecutorWorkerRunner.ts
@@ -58670,6 +59080,8 @@ var import_workerpool2 = __toESM(require("workerpool"));
 var import_dotenv = __toESM(require("dotenv"));
 init_src();
 init_src4();
+init_src10();
+init_DatasetSchema();
 init_src2();
 import_dotenv.default.configDotenv();
 var MODULE_LOADED_AT = performance.timeOrigin + performance.now();
@@ -58744,8 +59156,7 @@ var SampleRunnerClass = class {
           captureMainDatasetTo: capturePath,
           captureFormat: "JSONL"
         });
-        const captured = await this._readCapture(capturePath, size);
-        const records = captured.map((record) => this._inFieldOrder(consumer, record));
+        const records = await this._readCapture(capturePath, size);
         Logger_default.log(`[sample] Consumer "${consumer.name}": ${records.length} row(s) from ${result.inputCount} input line(s)`);
         return {
           kind: "consumer",
@@ -58793,28 +59204,6 @@ var SampleRunnerClass = class {
         return this._inputProducers(nested, seen);
       });
     };
-    /**
-     * One row's keys in the order the consumer DECLARES its fields.
-     *
-     * A record comes out of a worker keyed in the order the producer's dimensions were read, which is
-     * the source's layout and not the consumer's — so a consumer declaring country, plan, is_active
-     * over a producer that stores is_active first would be shown its own columns in someone else's
-     * order. Invisible in an exported JSON object; the whole shape of the table here.
-     *
-     * Anything the record carries that the field list does not name is appended rather than dropped:
-     * a column that turns up unexpectedly is exactly the one worth seeing.
-     */
-    this._inFieldOrder = (consumer, record) => {
-      const declared = ConsumerManager_default.getExpandedFields(consumer).filter((field) => !field.cField.hidden).map((field) => field.finalKey);
-      const ordered = {};
-      for (const key of declared) {
-        if (key in record) ordered[key] = record[key];
-      }
-      for (const key of Object.keys(record)) {
-        if (!(key in ordered)) ordered[key] = record[key];
-      }
-      return ordered;
-    };
     /** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
     this._readCapture = async (capturePath, size) => {
       if (!import_fs28.default.existsSync(capturePath)) return [];
@@ -59359,7 +59748,7 @@ var create_producer = async (name) => {
     console.log(import_chalk12.default.green(`\u2705 Created producer config at ${producerPath}`));
     console.log(import_chalk12.default.blue("Remember to:"));
     console.log(import_chalk12.default.blue("1. Set the correct source name"));
-    console.log(import_chalk12.default.blue(`2. Configure the dimensions and measures, or run "remora discover ${name} --replace" to`));
+    console.log(import_chalk12.default.blue(`2. Configure the dimensions, or run "remora discover ${name} --replace" to`));
     console.log(import_chalk12.default.blue("   write them from the source over the placeholders this template ships with"));
     console.log(import_chalk12.default.blue("3. Update the settings as needed"));
   } catch (error) {
@@ -59397,13 +59786,6 @@ var create_consumer = async (name, producerName) => {
         description: dim.description,
         from: producerName
       }));
-      if (producerConfig.measures && producerConfig.measures.length > 0) {
-        defaultConsumer.fields.push(...producerConfig.measures.map((measure) => ({
-          key: measure.name,
-          description: measure.description,
-          from: producerName
-        })));
-      }
       defaultConsumer.description = producerConfig.description || `Consumer for ${producerName} data`;
       defaultConsumer.schema = void 0;
       defaultConsumer.filters = void 0;