@forzalabs/remora 2.3.0 → 2.5.0

package/workers/ExecutorWorker.js

@@ -8538,7 +8538,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs31 = require("fs");
+    var fs33 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -8559,7 +8559,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs31.open(options2.file, "a+", "0644", (err2, fd) => {
+      fs33.open(options2.file, "a+", "0644", (err2, fd) => {
         if (err2) {
           if (!iter) {
             stream.emit("error", err2);
@@ -8571,10 +8571,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs31.close(fd, noop);
+            fs33.close(fd, noop);
             return;
           }
-          return fs31.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs33.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -8633,7 +8633,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs31 = require("fs");
+    var fs33 = require("fs");
     var path22 = require("path");
     var asyncSeries = require_series();
     var zlib3 = require("zlib");
@@ -8838,7 +8838,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs31.createReadStream(file, {
+        const stream = fs33.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err2) => {
@@ -8990,7 +8990,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path22.join(this.dirname, target);
-        fs31.stat(fullpath, (err2, stat) => {
+        fs33.stat(fullpath, (err2, stat) => {
           if (err2 && err2.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -9095,7 +9095,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path22.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs31.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs33.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -9174,7 +9174,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path22.join(this.dirname, filePath);
-        fs31.unlink(target, callback);
+        fs33.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -9197,17 +9197,17 @@ var require_file = __commonJS({
           tasks.push(function(i6, cb) {
             let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
             const tmppath = path22.join(this.dirname, fileName);
-            fs31.exists(tmppath, (exists) => {
+            fs33.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i6}${ext}${isZipped}`;
-              fs31.rename(tmppath, path22.join(this.dirname, fileName), cb);
+              fs33.rename(tmppath, path22.join(this.dirname, fileName), cb);
             });
           }.bind(this, x2));
         }
         asyncSeries(tasks, () => {
-          fs31.rename(
+          fs33.rename(
             path22.join(this.dirname, `${basename}${ext}${isZipped}`),
             path22.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -9223,22 +9223,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs31.access(src, fs31.F_OK, (err2) => {
+        fs33.access(src, fs33.F_OK, (err2) => {
           if (err2) {
             return callback();
           }
           var gzip = zlib3.createGzip();
-          var inp = fs31.createReadStream(src);
-          var out = fs31.createWriteStream(dest);
+          var inp = fs33.createReadStream(src);
+          var out = fs33.createWriteStream(dest);
           out.on("finish", () => {
-            fs31.unlink(src, callback);
+            fs33.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs31.existsSync(dirPath)) {
-          fs31.mkdirSync(dirPath, { recursive: true });
+        if (!fs33.existsSync(dirPath)) {
+          fs33.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11809,7 +11809,7 @@ var CONSTANTS, Constants_default;
 var init_Constants = __esm({
   "../../packages/constants/src/Constants.ts"() {
     CONSTANTS = {
-      cliVersion: "2.3.0",
+      cliVersion: "2.5.0",
       backendVersion: 1,
       backendPort: 5088,
       workerVersion: 2,
@@ -11934,11 +11934,36 @@ var init_Constants = __esm({
          * mode when its next line arrives — its byte counter lives in the bucket, not the stream.
          */
         MAX_OPEN_SPLIT_STREAMS: 64,
+        /**
+         * Record layouts one FIXED output may write into a single file, which is also how many write
+         * streams its block-ordering pass holds open. A copybook of this size is already unusual —
+         * the NASCO POD feed writes five transaction types — and the bound stops a generated config
+         * from opening a descriptor per distinct value of the wrong field.
+         */
+        MAX_FIXED_LAYOUTS: 64,
         /**
          * Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
          * export of it — `remora sample -s` and the preview's Data tab are both held to this.
          */
-        MAX_SAMPLE_ROWS: 100
+        MAX_SAMPLE_ROWS: 100,
+        /**
+         * Values held in memory while one parquet row group is encoded. A parquet writer buffers a
+         * whole row group before it can compress and flush it, so this — not the dataset size — is
+         * what bounds the export's peak memory. Budgeted in CELLS rather than rows because the two
+         * differ by three orders of magnitude across consumers: 100,000 rows of a 4-column consumer
+         * is 400k values, of a 1,300-field one it is 130 million.
+         */
+        PARQUET_ROW_GROUP_CELLS: 1e6,
+        /**
+         * Floor on the derived row count, so a consumer with hundreds of columns still writes row
+         * groups big enough to compress and to read back without a seek per handful of rows.
+         */
+        PARQUET_MIN_ROW_GROUP_ROWS: 1e3,
+        /**
+         * Ceiling on the derived row count, so a one-column consumer does not buffer a million rows
+         * to save a few row-group footers.
+         */
+        PARQUET_MAX_ROW_GROUP_ROWS: 1e5
       }
     };
     Constants_default = CONSTANTS;
@@ -12032,11 +12057,26 @@ var init_ExecutorScope = __esm({
     import_path2 = __toESM(require("path"), 1);
     import_fs2 = __toESM(require("fs"), 1);
     import_promises = __toESM(require("fs/promises"), 1);
+    init_src();
     init_src3();
     ExecutorScopeClass = class {
       constructor() {
         this.WORKERS_FOLDER = "workers";
         this.PRODUCERS_FOLDER = "producers";
+        /**
+         * The scope folders THIS process opened and has not cleared. The only list `deepClear` may
+         * delete from: two remora processes share one `.temp`, so a listing cannot tell a crashed run's
+         * leftovers from another process's live run.
+         */
+        this._ownedScopes = /* @__PURE__ */ new Set();
+        /** Registers a scope as this process's, so a crash on the way out clears it. */
+        this.open = (scope) => {
+          Affirm_default(scope?.folder, "Invalid executor scope");
+          this._ownedScopes.add(scope.folder);
+          return scope;
+        };
+        /** The scopes this process still holds. Empty once every run it started has cleaned up. */
+        this.getOwnedScopes = () => [...this._ownedScopes];
         this.getBasePath = () => import_path2.default.join(Constants_default.defaults.REMORA_PATH, Constants_default.defaults.PRODUCER_TEMP_FOLDER);
         this.getWorkerPath = (scope, workerId) => {
           return import_path2.default.join(
@@ -12047,6 +12087,13 @@ var init_ExecutorScope = __esm({
             `${workerId}.dataset`
           );
         };
+        /**
+         * Where a producer's source is staged before any worker reads it: a CSV extract WITH a header,
+         * written by whichever driver read the source.
+         *
+         * `.extract`, not `.dataset`: the two are different formats, and one extension meaning both is
+         * how a reader ends up applying the internal positional rules to a file that follows none.
+         */
         this.getProducerPath = (scope, producer, sourceFileKey) => {
           return import_path2.default.join(
             this.getBasePath(),
@@ -12054,7 +12101,7 @@ var init_ExecutorScope = __esm({
             scope.folder,
             this.PRODUCERS_FOLDER,
             producer.name,
-            `${sourceFileKey}.dataset`
+            `${sourceFileKey}.extract`
           );
         };
         this.getMainPath = (scope) => {
@@ -12090,16 +12137,32 @@ var init_ExecutorScope = __esm({
           if (import_fs2.default.existsSync(scopePath)) {
             await import_promises.default.rm(scopePath, { recursive: true, force: true });
           }
+          this._ownedScopes.delete(scope.folder);
         };
+        /**
+         * Clears what this process left open, on the way out. Synchronous: its one caller is the `exit`
+         * handler. A run that finished cleared its own scope, so a clean exit finds nothing to do.
+         */
         this.deepClear = () => {
           const basePath = this.getBasePath();
-          const openScopes = this.getOpenScopes();
-          for (const scopeFolder of openScopes) {
+          for (const scopeFolder of this._ownedScopes) {
             const scopePath = import_path2.default.join(basePath, scopeFolder);
             if (import_fs2.default.existsSync(scopePath)) {
               import_fs2.default.rmSync(scopePath, { recursive: true, force: true });
             }
           }
+          this._ownedScopes.clear();
+        };
+        /**
+         * Every scope folder on disk, whoever opened it. Answers "what is in .temp?", not "what is mine
+         * to delete" — nothing may be removed on the strength of it (see `getOwnedScopes`). The parallel
+         * canary samples it to watch concurrent runs come and go.
+         */
+        this.getOpenScopes = () => {
+          const basePath = this.getBasePath();
+          if (!import_fs2.default.existsSync(basePath))
+            return [];
+          return import_fs2.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
         };
         this.ensurePath = (fileUri) => {
           const dir = import_path2.default.dirname(fileUri);
@@ -12108,12 +12171,6 @@ var init_ExecutorScope = __esm({
           if (!import_fs2.default.existsSync(fileUri))
             import_fs2.default.writeFileSync(fileUri, "");
         };
-        this.getOpenScopes = () => {
-          const basePath = this.getBasePath();
-          if (!import_fs2.default.existsSync(basePath))
-            return [];
-          return import_fs2.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
-        };
       }
     };
     ExecutorScope = new ExecutorScopeClass();
@@ -12167,8 +12224,7 @@ var init_ProcessShutdownManager = __esm({
           if (this._cleaned)
             return;
           this._cleaned = true;
-          const openScopes = ExecutorScope_default.getOpenScopes();
-          const scopeCount = openScopes.length;
+          const scopeCount = ExecutorScope_default.getOwnedScopes().length;
           const shutdownState = this._shutdownState;
           const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
           if (scopeCount === 0) {
@@ -12579,7 +12635,7 @@ var init_ResourcesUtils = __esm({
               Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
               return this.getConsumerOutputColumns(subConsumer);
             } else {
-              const dims = producer.dimensions.map((x2) => ({
+              return producer.dimensions.map((x2) => ({
                 consumerAlias: null,
                 consumerKey: null,
                 nameInProducer: x2.name,
@@ -12587,15 +12643,6 @@ var init_ResourcesUtils = __esm({
                 dimension: x2,
                 owner: cProd.name
               }));
-              const meas = producer.measures?.map((x2) => ({
-                consumerAlias: null,
-                consumerKey: null,
-                nameInProducer: x2.name,
-                aliasInProducer: x2.name,
-                measure: x2,
-                owner: cProd.name
-              })) ?? [];
-              return [...dims, ...meas];
             }
           });
         };
@@ -12655,7 +12702,6 @@ var init_ResourcesUtils = __esm({
                   nameInProducer: col.nameInProducer,
                   aliasInProducer: col.nameInProducer,
                   dimension: col.dimension,
-                  measure: col.measure,
                   owner: consumer.name
                 });
               }
@@ -12669,7 +12715,6 @@ var init_ResourcesUtils = __esm({
                   nameInProducer: match2.name,
                   aliasInProducer: match2.name,
                   dimension: match2.column.dimension,
-                  measure: match2.column.measure,
                   owner: consumer.name
                 });
               }
@@ -12683,7 +12728,6 @@ var init_ResourcesUtils = __esm({
               nameInProducer: finalKey,
               aliasInProducer: finalKey,
               dimension: match?.dimension ?? { name: finalKey, type: "string" },
-              measure: match?.measure,
               owner: consumer.name
             });
           }
@@ -13135,14 +13179,11 @@ var init_Validator = __esm({
               return errors2;
             };
             errors.push(...validateTransformations(consumer.fields));
-            const duplicatesOutputs = Algo_default.duplicatesObject(consumer.outputs, "format");
-            if (duplicatesOutputs.length > 0) {
-              const duplicatesTypes = Algo_default.uniq(duplicatesOutputs.map((x2) => x2.format));
-              errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(" and ")})`);
-            }
+            if ((consumer.outputs ?? []).length > 1)
+              errors.push(`Consumer "${consumer.name}" declares ${consumer.outputs.length} outputs (${consumer.outputs.map((x2) => x2.format).join(", ")}). A consumer writes exactly one output \u2014 only the first was ever written \u2014 so split it into one consumer per output.`);
             for (const output of consumer.outputs) {
               const format4 = output.format.toUpperCase();
-              if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
+              if (format4 === "CSV" || format4 === "FIXED" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
                 if (!output.exportDestination)
                   errors.push(`A static file output must have an export destination set (${format4})`);
                 else if (!Environment_default.getSource(output.exportDestination))
@@ -13152,6 +13193,7 @@ var init_Validator = __esm({
             errors.push(...this.validateOmitNull(consumer));
             errors.push(...this.validateHeader(consumer));
             errors.push(...this.validateSplit(consumer));
+            errors.push(...this.validateFixedOutput(consumer));
             const dimensionFields = consumer.fields.filter((x2) => x2.key !== "*" && !x2.fixed && !x2.copyFrom && !x2.lineCount);
             const dimensionKeys = dimensionFields.map((x2) => `${x2.from ?? "_default_"}::${x2.key}`);
             const duplicateDimensionKeys = dimensionKeys.filter((k7, i6) => dimensionKeys.indexOf(k7) !== i6);
@@ -13220,7 +13262,7 @@ var init_Validator = __esm({
         this.validateSplit = (consumer) => {
           const errors = [];
           const { split: split2 } = consumer;
-          const positionalFormats = ["CSV", "TXT", "PARQUET"];
+          const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
           if (!Algo_default.hasVal(split2)) return errors;
           if (Algo_default.hasVal(consumer.maximumFileSize))
             errors.push(`Consumer "${consumer.name}" sets both "maximumFileSize" and "split". They are the same cap: move the value to "split": { "maxFileSizeGb": ${consumer.maximumFileSize} } and remove "maximumFileSize".`);
@@ -13237,50 +13279,83 @@ var init_Validator = __esm({
           const missingKeys = fieldKeys.includes("*") ? [] : keys.filter((key) => !fieldKeys.includes(key));
           if (missingKeys.length > 0)
             errors.push(`"split.by" references key(s) "${missingKeys.join(", ")}" that are not present in the consumer "${consumer.name}".`);
-          const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
-          if (positional.length === 0) return errors;
+          const format4 = consumer.outputs?.[0]?.format.toUpperCase();
+          if (!positionalFormats.includes(format4)) return errors;
           const hiddenKeys = keys.filter((key) => (consumer.fields ?? []).some((x2) => (x2.alias ?? x2.key) === key && x2.hidden));
           if (hiddenKeys.length > 0)
-            errors.push(`"split.by" of consumer "${consumer.name}" references hidden field(s) "${hiddenKeys.join(", ")}", which the ${positional.map((x2) => x2.format.toUpperCase()).join("/")} output does not write \u2014 a hidden field can't be split on in a positional format.`);
+            errors.push(`"split.by" of consumer "${consumer.name}" references hidden field(s) "${hiddenKeys.join(", ")}", which the ${format4} output does not write \u2014 a hidden field can't be split on in a positional format.`);
           return errors;
         };
         /**
-         * Checks `omitNull`, which drops a record's null fields on export. It only means anything where a
-         * record carries its own field names, and it is applied when the record is serialized — one record
-         * shape for every output of the consumer — so a positional sibling output would silently lose its
-         * column alignment. Both cases are rejected rather than ignored.
+         * Checks `omitNull`, which drops a record's null fields on export. It only means anything where
+         * a record carries its own field names: leaving a positional column out shifts every column
+         * after it, so a positional output declaring it is rejected rather than ignored.
          */
         this.validateOmitNull = (consumer) => {
           const errors = [];
-          const positionalFormats = ["CSV", "TXT", "PARQUET"];
+          const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
           for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
             const format4 = output.format.toUpperCase();
-            if (positionalFormats.includes(format4)) {
+            if (positionalFormats.includes(format4))
               errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", which only applies to JSON/API output: a positional column can't be omitted from a row.`);
-              continue;
-            }
-            const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
-            if (positional.length > 0)
-              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", but the consumer also writes ${positional.map((x2) => x2.format.toUpperCase()).join(" and ")}. Every output is serialized from one record, so omitting fields would break the positional one \u2014 split them into two consumers.`);
           }
           return errors;
         };
         /**
          * Checks `header`, which writes the field names as the first line of an exported file. A JSON
-         * record carries its own field names and an API response is not a file, so the flag means
-         * nothing there — rejected rather than ignored, so a config that asks for a header always gets
-         * one.
+         * record carries its own field names, a parquet file carries its own schema, and an API response
+         * is not a file, so the flag means nothing there — rejected rather than ignored, so a config that
+         * asks for a header always gets one.
          */
         this.validateHeader = (consumer) => {
           const errors = [];
-          const positionalFormats = ["CSV", "TXT", "PARQUET"];
+          const headerFormats = ["CSV", "TXT"];
           for (const output of (consumer.outputs ?? []).filter((x2) => Algo_default.hasVal(x2.header))) {
             const format4 = output.format.toUpperCase();
-            if (!positionalFormats.includes(format4))
-              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: a ${format4} record already carries its own field names.`);
+            if (!headerFormats.includes(format4))
+              errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: ${this.headerlessReason(format4)}.`);
           }
           return errors;
         };
+        /** Why the format in hand has no header line, as the clause that finishes the sentence above. */
+        this.headerlessReason = (format4) => {
+          if (format4 === "FIXED") return "a positional line has no room for one, and a field name would not sit on the columns it describes";
+          if (format4 === "PARQUET") return `a ${format4} file carries its own schema`;
+          return `a ${format4} record already carries its own field names`;
+        };
+        /**
+         * Checks a consumer's FIXED output from its configuration alone: the properties only a
+         * positional file has, the ones it cannot have, and whether anything downstream tries to read
+         * one back. The layout itself — widths, columns and record lengths — is resolved against the
+         * dataset's columns, which the producers define, so it is checked by `compile` and again before
+         * a run starts rather than here.
+         *
+         * A property declared on the wrong format is rejected rather than ignored, the way a `position`
+         * on a non-FIXED producer is: silently dropping a rule the author wrote is how a file ships
+         * looking almost right.
+         */
+        this.validateFixedOutput = (consumer) => {
+          Affirm_default(consumer, "Invalid consumer");
+          const errors = [];
+          const output = consumer.outputs?.[0];
+          if (!output) return errors;
+          const format4 = output.format?.toUpperCase();
+          const positioned = (consumer.fields ?? []).filter((x2) => x2.position).map((x2) => `"${x2.alias ?? x2.key}"`);
+          if (format4 !== "FIXED") {
+            const fixedOnly = ["recordLength", "defaults", "lineEnding", "layouts"];
+            const declared = fixedOnly.filter((name) => Algo_default.hasVal(output[name]));
+            if (declared.length > 0)
+              errors.push(`The ${format4} output of consumer "${consumer.name}" declares ${declared.map((x2) => `"${x2}"`).join(", ")}, which only ${declared.length > 1 ? "apply" : "applies"} to a FIXED output, where ${declared.length > 1 ? "they are" : "it is"} ignored.`);
+            if (positioned.length > 0)
+              errors.push(`Consumer "${consumer.name}" declares "position" on field(s) ${positioned.join(", ")} but its output format is not "FIXED", where positions are ignored.`);
+            return errors;
+          }
+          if (Algo_default.hasVal(output.delimiter))
+            errors.push(`The FIXED output of consumer "${consumer.name}" declares a "delimiter". A positional line has none: its fields are found by the columns they occupy.`);
+          if ((consumer.fields ?? []).some((x2) => x2.key === "*"))
+            errors.push(`Consumer "${consumer.name}" selects fields with the wildcard key "*", which a FIXED output can't write: every field of a positional line declares its own "width", so the field list has to be explicit.`);
+          return errors;
+        };
         /**
          * Checks a field whose key selects columns by pattern ("addr*Street"). A pattern that matches
          * nothing is always a mistake — a renamed dimension, or a producer that never had those columns —
@@ -14861,7 +14936,7 @@ A consumer takes the data from one or more producers and changes it's shape to t
 # RULES
 - If a field is not needed, do not add it e.g.
 - Only import a producer once
-- Awlays include this exact property as the first -> $schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
+- Awlays include this exact property as the first -> "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
 - Use "API" as the only valid output format.
 - The "from" must contain only the name of the producer
 
@@ -33669,6 +33744,96 @@ var init_CSVParser = __esm({
   }
 });
 
+// ../../packages/parsing/src/DatasetCodec.ts
+var NULL_FIELD, NEEDS_ESCAPE, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
+var init_DatasetCodec = __esm({
+  "../../packages/parsing/src/DatasetCodec.ts"() {
+    init_src();
+    NULL_FIELD = "";
+    NEEDS_ESCAPE = /["\n\r]/;
+    DatasetCodecClass = class {
+      constructor() {
+        /**
+         * One value as an internal positional field: quoted, with `"`, `\n` and `\r` escaped - or the
+         * unquoted null marker when there is no value.
+         */
+        this.encodeField = (value) => {
+          if (value === null || value === void 0) return NULL_FIELD;
+          const text = typeof value === "string" ? value : String(value);
+          if (!NEEDS_ESCAPE.test(text)) return `"${text}"`;
+          return `"${text.replaceAll('"', '""').replaceAll("\r", "\\r").replaceAll("\n", "\\n")}"`;
+        };
+        /** One record as an internal positional line. The values are already in the schema's order. */
+        this.encodeRow = (values, delimiter = ",") => {
+          Affirm_default(values, "Invalid values");
+          Affirm_default(delimiter, "Invalid delimiter");
+          let row = "";
+          for (let index = 0; index < values.length; index++) {
+            if (index > 0)
+              row += delimiter;
+            row += this.encodeField(values[index]);
+          }
+          return row;
+        };
+        /**
+         * One internal positional line split back into its raw fields, with a null still a null. The
+         * array may be SHORTER than the schema's column list; a reader treats a missing entry as null.
+         */
+        this.decodeRow = (line, delimiter = ",") => {
+          Affirm_default(delimiter, "Invalid delimiter");
+          const fields = [];
+          const length = line?.length ?? 0;
+          const delimiterCode = delimiter.charCodeAt(0);
+          let fieldStart = 0;
+          let fieldEnd = 0;
+          let inQuotes = false;
+          let hasQuotes = false;
+          let index = 0;
+          while (index < length) {
+            const char = line.charCodeAt(index);
+            if (char === 34) {
+              if (!inQuotes) {
+                inQuotes = true;
+                hasQuotes = true;
+                fieldStart = index + 1;
+              } else if (line.charCodeAt(index + 1) === 34) {
+                index++;
+              } else {
+                inQuotes = false;
+                fieldEnd = index;
+              }
+            } else if (char === delimiterCode && !inQuotes) {
+              fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
+              fieldStart = index + 1;
+              fieldEnd = 0;
+              hasQuotes = false;
+            }
+            index++;
+          }
+          fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
+          return fields;
+        };
+        /**
+         * Re-encodes an internal line with another delimiter, which is what a `TXT` export is. Not via
+         * `CSVParser`: a plain parse reads both the null marker and `""` as the empty string.
+         */
+        this.redelimit = (line, delimiter) => {
+          Affirm_default(delimiter, "Invalid delimiter");
+          return this.encodeRow(this.decodeRow(line), delimiter);
+        };
+        /** One field's text, or null for the unquoted marker. Only a quoted token carries a value. */
+        this._field = (line, start, end, hasQuotes) => {
+          if (!hasQuotes) return null;
+          const raw = line.slice(start, end);
+          return raw.indexOf('"') === -1 ? raw : raw.replaceAll('""', '"');
+        };
+      }
+    };
+    DatasetCodec = new DatasetCodecClass();
+    DatasetCodec_default = DatasetCodec;
+  }
+});
+
 // ../../packages/parsing/src/FixedWidthParser.ts
 var FixedWidthParserClass, FixedWidthParser, FixedWidthParser_default;
 var init_FixedWidthParser = __esm({
@@ -33823,6 +33988,307 @@ var init_FixedWidthParser = __esm({
   }
 });
 
+// ../../packages/parsing/src/FixedWidthWriter.ts
+var BASE_FILL, TYPE_FILL, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
+var init_FixedWidthWriter = __esm({
+  "../../packages/parsing/src/FixedWidthWriter.ts"() {
+    init_src();
+    init_src3();
+    BASE_FILL = { align: "left", pad: " ", nullFill: " ", overflow: "fail" };
+    TYPE_FILL = {
+      string: BASE_FILL,
+      number: { ...BASE_FILL, align: "right", pad: "0" },
+      datetime: BASE_FILL,
+      boolean: BASE_FILL
+    };
+    FixedWidthWriterClass = class {
+      constructor() {
+        /**
+         * Resolves one record layout for writing: every column's fill rules decided, its columns
+         * computed, and the record length agreed with what the output declares.
+         *
+         * Called once per layout when an export starts, never per record. `columns` is the line as the
+         * dataset schema has it — line order, visible fields only — and `positions` carries each one's
+         * `position` block, keyed by the same name the column has.
+         *
+         * `context` is the phrase that names this layout in an error, e.g. `the FIXED output of consumer
+         * "x"`, so a message reads as one sentence wherever the layout came from.
+         */
+        this.compilePlan = (columns, positions, output, context = "the FIXED output") => {
+          Affirm_default(output, "Invalid consumer output");
+          const selected = columns?.map((column, index) => ({ column, index }));
+          return this.plan(selected, positions, output.defaults, output.recordLength, context);
+        };
+        /**
+         * A FIXED output resolved for INSPECTION rather than for writing: every problem with it, and the
+         * layouts it would write. What `compile` reports and prints.
+         *
+         * The rules are not restated here. The layouts are resolved by the same code the export runs,
+         * and what that throws is reported rather than raised — so a layout `compile` accepts is a
+         * layout the export can write. What review adds is that every field's own declaration is
+         * checked first, across the whole output: an author with six fields missing a width sees six
+         * lines from one `compile`, not one line from each of six.
+         *
+         * A field named by a layout that the dataset has no column for is left to the resolution below,
+         * which names it as the layout's error rather than as a field with no width.
+         */
+        this.review = (dsSchema, positions, output, consumerName) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default(output, "Invalid consumer output");
+          const context = `the FIXED output${consumerName ? ` of consumer "${consumerName}"` : ""}`;
+          const declared = this.declarationErrors(dsSchema, positions, output, context);
+          if (declared.length > 0) return { errors: declared, layouts: [] };
+          try {
+            if (!output.layouts)
+              return { errors: [], layouts: [{ value: null, plan: this.compilePlan(dsSchema.columns, positions, output, context) }] };
+            const layouts = this.compileLayouts(dsSchema, positions, output, consumerName);
+            return { errors: [], layouts: layouts.order.map((value) => ({ value, plan: layouts.plans.get(value) })) };
+          } catch (e7) {
+            return { errors: [this.reason(e7)], layouts: [] };
+          }
+        };
+        /** Every field's own declaration, over every field the output writes, before any offset is computed. */
+        this.declarationErrors = (dsSchema, positions, output, context) => {
+          const written = output.layouts ? [...new Set((output.layouts.order ?? []).flatMap((value) => output.layouts.values?.[value]?.fields ?? []))] : dsSchema.columns.map((column) => column.name);
+          const errors = [];
+          for (const name of written) {
+            const column = dsSchema.columns.find((x2) => x2.name === name);
+            if (column) errors.push(...this.fieldErrors(column, positions?.[name], output.defaults, context));
+          }
+          return errors;
+        };
+        /**
+         * What an affirm said, as a sentence a report can start a line with: without the prefix it
+         * carries for a log, and with its first letter upper-cased — the messages are written to be
+         * embedded ("... of the FIXED output of consumer x ..."), so some begin with that context
+         * rather than with a capital.
+         */
+        this.reason = (error) => {
+          const prefix2 = "Affirm failed: ";
+          const message = error instanceof Error ? error.message : String(error);
+          const sentence = message.startsWith(prefix2) ? message.slice(prefix2.length) : message;
+          return sentence.charAt(0).toUpperCase() + sentence.slice(1);
+        };
+        /**
+         * The record layouts of a multi-layout FIXED output, compiled in `order` so the export writes
+         * its blocks in the order the author declared.
+         *
+         * Each layout names a subset of the dataset's columns, in its own column order, so a field's
+         * place on the written line has nothing to do with its place on the staged one — which is what
+         * `IFixedWriteField.index` carries. The output's `defaults` stand under every layout's own.
+         */
+        this.compileLayouts = (dsSchema, positions, output, consumerName) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default(output?.layouts, "Invalid fixed-width layouts");
+          const { discriminator, order, values } = output.layouts;
+          const where = consumerName ? ` of consumer "${consumerName}"` : "";
+          Affirm_default(discriminator, `The FIXED layouts${where} name no discriminator field`);
+          Affirm_default.hasItems(order, `The FIXED layouts${where} declare no "order", which is the order their record blocks are written in`);
+          Affirm_default.noDupes(order, `The FIXED layouts${where} name a discriminator value twice in "order"`);
+          Affirm_default(
+            order.length <= Constants_default.defaults.MAX_FIXED_LAYOUTS,
+            `The FIXED layouts${where} declare ${order.length} record layouts, more than the ${Constants_default.defaults.MAX_FIXED_LAYOUTS} one output may write`
+          );
+          Affirm_default(
+            output.recordLength === void 0 || output.recordLength === null,
+            `The FIXED output${where} declares both "layouts" and a "recordLength". Each layout declares its own; the output has no single one.`
+          );
+          const index = dsSchema.columns.findIndex((column) => column.name === discriminator);
+          Affirm_default(index >= 0, `The FIXED discriminator "${discriminator}"${where} is not one of the consumer's visible fields`);
+          const plans = /* @__PURE__ */ new Map();
+          for (const value of order)
+            plans.set(value, this.layoutPlan(values?.[value], dsSchema, positions, output, value, consumerName));
+          return { discriminator, index, order, plans };
+        };
+        /** One entry of the `layouts` map as a plan: its named fields, found on the staged line, in its order. */
+        this.layoutPlan = (layout, dsSchema, positions, output, value, consumerName) => {
+          const of = consumerName ? ` of consumer "${consumerName}"` : "";
+          const where = `the FIXED layout "${value}"${of}`;
+          Affirm_default(layout, `The FIXED "order"${of} names the discriminator value "${value}", which has no layout under "values"`);
+          Affirm_default.hasItems(layout.fields, `${where} writes no field`);
+          Affirm_default.noDupes(layout.fields, `${where} writes a field twice`);
+          const selected = layout.fields.map((name) => {
+            const index = dsSchema.columns.findIndex((column) => column.name === name);
+            Affirm_default(index >= 0, `The field "${name}" of ${where} is not one of the consumer's visible fields`);
+            return { column: dsSchema.columns[index], index };
+          });
+          return this.plan(selected, positions, this.mergeDefaults(output.defaults, layout.defaults), layout.recordLength, where);
+        };
+        /**
+         * One record layout ready to write: every column's fill rules decided, its columns computed from
+         * the widths before it, and the record length agreed with what the layout declares.
+         */
+        this.plan = (selected, positions, defaults, recordLength, context) => {
+          Affirm_default.hasItems(selected, `${context} has no field to write`);
+          Affirm_default(positions, "Invalid fixed-width field positions");
+          const fields = [];
+          let start = 1;
+          for (const { column, index } of selected) {
+            const field = this.resolveField(column, positions[column.name], defaults, start, context);
+            fields.push({ ...field, index });
+            start = field.end + 1;
+          }
+          const composed = start - 1;
+          Affirm_default(
+            recordLength === void 0 || recordLength === null || recordLength === composed,
+            `${context} declares a recordLength of ${recordLength}, but its ${fields.length} field widths sum to ${composed}. A field, a filler or a width is missing.`
+          );
+          return { fields, recordLength: composed };
+        };
+        /**
+         * The layout's per-type fill rules over the output's, type by type — so an output states a rule
+         * once for the whole file and a layout only names where it differs.
+         */
+        this.mergeDefaults = (output, layout) => {
+          if (!layout) return output;
+          if (!output) return layout;
+          const merged = { ...output };
+          for (const type of Object.keys(layout))
+            merged[type] = this.override(output[type] ?? {}, layout[type]);
+          return merged;
+        };
+        /**
+         * The layout a record is written with, by the value of the discriminator field on its staged
+         * line. A value with no layout stops the run: it is a record the file has no block for, and
+         * writing it under another layout would put its fields at the wrong columns.
+         */
+        this.planFor = (values, layouts) => {
+          const value = values?.[layouts.index];
+          const plan = value === null || value === void 0 ? null : layouts.plans.get(String(value));
+          Affirm_default(plan, `No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`);
+          return plan;
+        };
+        /**
+         * Everything wrong with the way ONE field is declared, as messages rather than as a throw: no
+         * `position` at all, a width that is not a whole number of columns, a fill character that is not
+         * one character.
+         *
+         * Not the field's place on the line — that is only meaningful once every field before it stands
+         * on its own, so `resolveField` checks it where it computes it. This is the one statement of the
+         * rules: the export affirms them, and `review` collects them for every field of an output so
+         * `compile` reports six missing widths in one run rather than one per run.
+         */
+        this.fieldErrors = (column, position, defaults, context) => {
+          Affirm_default(column, "Invalid dataset column");
+          const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
+          if (!position)
+            return [`Missing "position" on ${where}: every visible field of a FIXED output declares the width it occupies`];
+          const errors = [];
+          if (!Number.isInteger(position.width) || position.width < 1)
+            errors.push(`"position.width" of ${where} must be a whole number of 1 or greater`);
+          const fill2 = this.fillOf(column.type, position, defaults);
+          if (fill2.pad.length !== 1)
+            errors.push(`"position.pad" of ${where} must be a single character, not "${fill2.pad}"`);
+          if (fill2.nullFill.length !== 1)
+            errors.push(`"position.nullFill" of ${where} must be a single character, not "${fill2.nullFill}"`);
+          return errors;
+        };
+        /**
+         * One column with its fill rules resolved and its place on the line computed. The field's own
+         * `position` wins over the output's `defaults` for its type, which wins over the type's own rule,
+         * so a wide layout states a rule once and names only its exceptions.
+         */
+        this.resolveField = (column, position, defaults, start, context) => {
+          const declaration = this.fieldErrors(column, position, defaults, context);
+          Affirm_default(declaration.length === 0, declaration.join(" "));
+          const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
+          const fill2 = this.fillOf(column.type, position, defaults);
+          const end = start + position.width - 1;
+          Affirm_default(
+            position.start === void 0 || position.start === null || position.start === start,
+            `"position.start" of ${where} says column ${position.start}, but the widths before it place the field at column ${start}. A field or a filler is missing, or two are out of order.`
+          );
+          return {
+            name: column.name,
+            width: position.width,
+            start,
+            end,
+            align: fill2.align,
+            pad: fill2.pad,
+            nullText: fill2.nullFill.repeat(position.width),
+            overflow: fill2.overflow
+          };
+        };
+        /** The three fill sources folded into one, nearest declaration first. */
+        this.fillOf = (type, position, defaults) => {
+          const base = TYPE_FILL[type] ?? TYPE_FILL.string;
+          return this.override(this.override(base, defaults?.[type]), position);
+        };
+        /** `fill` with whatever `override` actually declares. An absent rule leaves the one below it standing. */
+        this.override = (fill2, override) => {
+          if (!override) return fill2;
+          const merged = { ...fill2 };
+          if (override.align) merged.align = override.align;
+          if (override.pad) merged.pad = override.pad;
+          if (override.nullFill) merged.nullFill = override.nullFill;
+          if (override.overflow) merged.overflow = override.overflow;
+          return merged;
+        };
+        /**
+         * One record as its positional line, with no delimiter, no quoting and no terminator. The values
+         * are the record's fields as the STAGED line holds them, each plan field reading the one it was
+         * compiled against; a value the array does not reach is written as a null, which is what a
+         * staged line does when its last fields carry nothing.
+         */
+        this.composeLine = (values, plan) => {
+          Affirm_default(values, "Invalid record values");
+          Affirm_default(plan, "Invalid fixed-width write plan");
+          let line = "";
+          for (const field of plan.fields)
+            line += this.writeField(values[field.index], field);
+          Affirm_default(line.length === plan.recordLength, `A fixed-width record composed to ${line.length} characters, not the declared recordLength of ${plan.recordLength}`);
+          return line;
+        };
+        /**
+         * One value as one field: its text padded out to the field's width on the side `align` names.
+         *
+         * A null is the full width in `nullFill`, and an EMPTY STRING is a null — the staged encoding
+         * keeps the two apart and a positional line cannot. A value too long for its field stops the
+         * run; see `overflowed`.
+         */
+        this.writeField = (value, field) => {
+          if (value === null || value === void 0) return field.nullText;
+          const text = typeof value === "string" ? value : String(value);
+          if (text.length === 0) return field.nullText;
+          if (text.length > field.width) return this.overflowed(text, field);
+          return field.align === "right" ? text.padStart(field.width, field.pad) : text.padEnd(field.width, field.pad);
+        };
+        /**
+         * A value longer than its field. It fails the run by default: a truncated account number is a
+         * plausible-looking account number, and silent corruption in a feed nobody re-reads is the one
+         * failure mode a positional writer must not have.
+         *
+         * Where the layout genuinely specifies a cut, the end that survives follows `align`, which is
+         * what a COBOL MOVE does: a left-aligned value keeps its head, a right-aligned one keeps its
+         * low-order tail.
+         */
+        this.overflowed = (text, field) => {
+          Affirm_default(
+            field.overflow === "truncate",
+            `Value "${text}" is ${text.length} characters, longer than the ${field.width} of field "${field.name}" at columns ${field.start}-${field.end}. Widen the field, or set its "overflow" to "truncate" if the layout really specifies a cut.`
+          );
+          return field.align === "right" ? text.slice(text.length - field.width) : text.slice(0, field.width);
+        };
+        /**
+         * What a record carries after its last field, BEFORE the newline the destination writes. Every
+         * destination driver ends each transformed line with one `\n`, the last line included, so an
+         * `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
+         * CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
+         *
+         * A real positional feed picks one terminator and holds it: the NASCO POD files are CRLF on
+         * every line including the last, and `lf` is the default everywhere else.
+         */
+        this.lineSuffix = (output) => {
+          Affirm_default(output, "Invalid consumer output");
+          return output.lineEnding === "crlf" ? "\r" : "";
+        };
+      }
+    };
+    FixedWidthWriter = new FixedWidthWriterClass();
+    FixedWidthWriter_default = FixedWidthWriter;
+  }
+});
+
 // ../../node_modules/pend/index.js
 var require_pend = __commonJS({
   "../../node_modules/pend/index.js"(exports2, module2) {
@@ -33881,7 +34347,7 @@ var require_pend = __commonJS({
 // ../../node_modules/yauzl/fd-slicer.js
 var require_fd_slicer = __commonJS({
   "../../node_modules/yauzl/fd-slicer.js"(exports2) {
-    var fs31 = require("fs");
+    var fs33 = require("fs");
     var util = require("util");
     var stream = require("stream");
     var Readable10 = stream.Readable;
@@ -33901,7 +34367,7 @@ var require_fd_slicer = __commonJS({
     FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
       var self2 = this;
       self2.pend.go(function(cb) {
-        fs31.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
+        fs33.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
           cb();
           callback(err2, bytesRead, buffer2);
         });
@@ -33918,7 +34384,7 @@ var require_fd_slicer = __commonJS({
       self2.refCount -= 1;
       if (self2.refCount < 0) throw new Error("invalid unref");
       if (self2.refCount > 0) return;
-      fs31.close(self2.fd, onCloseDone);
+      fs33.close(self2.fd, onCloseDone);
       function onCloseDone(err2) {
         if (err2) {
           self2.emit("error", err2);
@@ -33950,7 +34416,7 @@ var require_fd_slicer = __commonJS({
       }
       self2.context.pend.go(function(cb) {
         var buffer = Buffer.allocUnsafe(toRead);
-        fs31.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
+        fs33.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
           if (err2) {
             self2.destroy(err2);
           } else if (bytesRead === 0) {
@@ -34310,7 +34776,7 @@ var require_crc32 = __commonJS({
 // ../../node_modules/yauzl/index.js
 var require_yauzl = __commonJS({
   "../../node_modules/yauzl/index.js"(exports2) {
-    var fs31 = require("fs");
+    var fs33 = require("fs");
     var zlib3 = require("zlib");
     var fd_slicer = require_fd_slicer();
     var util = require("util");
@@ -34379,10 +34845,10 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs31.open(path22, "r", function(err2, fd) {
+      fs33.open(path22, "r", function(err2, fd) {
         if (err2) return callback(err2);
         fromFd(fd, options2, function(err3, zipfile) {
-          if (err3) fs31.close(fd, defaultCallback);
+          if (err3) fs33.close(fd, defaultCallback);
           callback(err3, zipfile);
         });
       });
@@ -34399,7 +34865,7 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs31.fstat(fd, function(err2, stats) {
+      fs33.fstat(fd, function(err2, stats) {
         if (err2) return callback(err2);
         var reader = new fd_slicer.FdSlicer(fd);
         fromRandomAccessReader(reader, stats.size, options2, callback);
@@ -38478,7 +38944,9 @@ var init_XMLParser = __esm({
 var init_src5 = __esm({
   "../../packages/parsing/src/index.ts"() {
     init_CSVParser();
+    init_DatasetCodec();
     init_FixedWidthParser();
+    init_FixedWidthWriter();
     init_ParseCompression();
     init_ParseHelper();
     init_XLSParser();
@@ -46114,8 +46582,8 @@ var init_LocalSourceDriver = __esm({
             Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
             const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
             const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
-            const results = await Promise.all(promises2);
+            const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
+            const results = await Promise.all(promises3);
             return results.flat();
           } else {
             return await this._get(request, "");
@@ -46135,8 +46603,8 @@ var init_LocalSourceDriver = __esm({
             Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
             const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
             const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
-            const results = await Promise.all(promises2);
+            const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
+            const results = await Promise.all(promises3);
             return results.flat();
           } else {
             return await this._get(request, "");
@@ -46564,8 +47032,8 @@ var init_S3SourceDriver = __esm({
           if (fileKey.includes("%")) {
             const allFileKeys = await this.listFiles(fileKey);
             Affirm_default(allFileKeys.length < 50, `Pattern ${fileKey} of producer requested to S3 matches more than 50 files (${allFileKeys.length}), this is more than the S3 allowed limit. Please refine your pattern, remove some files or use a separate bucket.`);
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
-            const results = await Promise.all(promises2);
+            const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
+            const results = await Promise.all(promises3);
             return results.flat();
           } else {
             return await this._get(request);
@@ -46578,8 +47046,8 @@ var init_S3SourceDriver = __esm({
           const { fileKey } = request;
           if (fileKey.includes("%")) {
             const allFileKeys = await this.listFiles(fileKey);
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
-            const results = await Promise.all(promises2);
+            const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
+            const results = await Promise.all(promises3);
             return results.flat();
           } else {
             return await this._get(request);
@@ -47526,7 +47994,7 @@ var init_ProducerManager = __esm({
 });
 
 // ../../packages/engines/src/transform/TypeCaster.ts
-var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TYPE_DEFAULTS, TypeCasterClass, TypeCaster, TypeCaster_default;
+var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TRUE_VALUES, TYPE_DEFAULTS, TypeCasterClass, TypeCaster, TypeCaster_default;
 var init_TypeCaster = __esm({
   "../../packages/engines/src/transform/TypeCaster.ts"() {
     init_src();
@@ -47547,6 +48015,7 @@ var init_TypeCaster = __esm({
       { token: "mm", width: 2 },
       { token: "ss", width: 2 }
     ];
+    TRUE_VALUES = /^(1|true|yes|y|t)$/i;
     TYPE_DEFAULTS = {
       string: "",
       number: 0,
@@ -47556,7 +48025,8 @@ var init_TypeCaster = __esm({
     TypeCasterClass = class {
       constructor() {
         /**
-         * Casts a value for a DECLARED dimension: the type cast, plus what a MISSING value means for it.
+         * Casts a value for a DECLARED field: the type cast, plus what a MISSING value means for it. The
+         * field is a producer dimension on the read path and a dataset column on the re-read path.
          *
          * Missing is null/undefined, and — for every type but `string` — the empty string, which is how
          * a NULL arrives from every source that stages through CSV (SQL, Delta Share, CSV files) and is
@@ -47588,6 +48058,7 @@ var init_TypeCaster = __esm({
         switch (type) {
           case "boolean": {
             if (typeof value === "boolean") return value;
+            if (typeof value === "string") return TRUE_VALUES.test(value.trim());
             return Boolean(value);
           }
           case "datetime":
@@ -47726,6 +48197,7 @@ var init_TypeCaster = __esm({
 var LineParserClass, LineParser, LineParser_default;
 var init_LineParser = __esm({
   "../../packages/engines/src/parsing/LineParser.ts"() {
+    init_src();
     init_TypeCaster();
     init_src5();
     LineParserClass = class {
@@ -47818,22 +48290,23 @@ var init_LineParser = __esm({
           return record;
         };
         /**
-         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
+         * The parser of one staged `.dataset`, compiled once from the schema that file was written with.
+         * The schema is the ONLY description of the file - it has no header - and there is deliberately
+         * no overload taking a consumer's field list, which cannot know about a shape a pass changed.
          */
-        this._internalParseCSV = (line, fields, delimiter) => {
-          const parts = CSVParser_default.parseRow(line, delimiter ?? ",");
-          const record = {};
-          for (const [index, field] of fields.entries()) {
-            const fieldKey = field.finalKey;
-            record[fieldKey] = field.dimension ? TypeCaster_default.castDimension(parts[index], field.dimension) : TypeCaster_default.cast(parts[index], "string");
-          }
-          return record;
-        };
-        /**
-         * Used ONLY to parse internal records (inside the .dataset) since I know they are already prepared in the correct way
-         */
-        this._internalParseJSON = (line) => {
-          return JSON.parse(line);
+        this.datasetParser = (dsSchema) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default.hasItems(dsSchema.columns, "The dataset schema declares no column to read");
+          if (dsSchema.encoding === "JSON")
+            return (line) => JSON.parse(line);
+          const { columns } = dsSchema;
+          return (line) => {
+            const parts = DatasetCodec_default.decodeRow(line);
+            const record = {};
+            for (let index = 0; index < columns.length; index++)
+              record[columns[index].name] = TypeCaster_default.castDimension(parts[index], columns[index]);
+            return record;
+          };
         };
       }
     };
@@ -47950,7 +48423,7 @@ var init_ProducerEngine = __esm({
                 let effectiveProducer = producer;
                 let effectiveSource = source;
                 if (compressionType) {
-                  scope = { id: (0, import_crypto6.randomUUID)(), folder: `sample-${(0, import_crypto6.randomUUID)()}`, workersId: [] };
+                  scope = ExecutorScope_default.open({ id: (0, import_crypto6.randomUUID)(), folder: `sample-${(0, import_crypto6.randomUUID)()}`, workersId: [] });
                   const driver = await DriverFactory_default.instantiateSource(source);
                   const readyResult = await driver.ready({ producer, scope });
                   const firstFile = readyResult.files[0].fullUri;
@@ -47960,7 +48433,7 @@ var init_ProducerEngine = __esm({
                     ...producer,
                     settings: {
                       ...producer.settings,
-                      fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path14.default.basename(firstFile),
+                      fileKey: readyResult.files.length > 1 ? "%.extract" : import_path14.default.basename(firstFile),
                       compressionType: void 0
                     }
                   };
@@ -51213,8 +51686,7 @@ var init_ConsumerManager = __esm({
                   from: x2.owner
                 },
                 finalKey: x2.nameInProducer,
-                dimension: x2.dimension,
-                measure: x2.measure
+                dimension: x2.dimension
               })));
             } else {
               const from = field.cField.from ?? (consumer.producers.length === 1 ? consumer.producers[0].name : null);
@@ -51226,8 +51698,7 @@ var init_ConsumerManager = __esm({
                     from: col.owner
                   },
                   finalKey: col.nameInProducer,
-                  dimension: col.dimension,
-                  measure: col.measure
+                  dimension: col.dimension
                 });
               });
             }
@@ -51242,13 +51713,12 @@ var init_ConsumerManager = __esm({
                 from: field.cField.from ?? match.column.owner
               },
               finalKey: match.name,
-              dimension: match.column.dimension,
-              measure: match.column.measure
+              dimension: match.column.dimension
             })));
           } else {
             const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
             Affirm_default(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map((x2) => x2.name).join(", ")}")`);
-            expandedFields.push({ ...field, dimension: col.dimension, measure: col.measure });
+            expandedFields.push({ ...field, dimension: col.dimension });
           }
           return expandedFields;
         };
@@ -52310,10 +52780,9 @@ var init_FieldLineage = __esm({
         this._fromProducer = (producer, field) => {
           const node = LineageEngine_default.nodeId("producer", producer.name);
           const dimension = (producer.dimensions ?? []).find((x2) => x2.name === field.key);
-          const measure = (producer.measures ?? []).find((x2) => x2.name === field.key);
-          const applied = dimension?.alias ? [`source column "${dimension.alias}"`] : measure ? ["measure"] : [];
+          const applied = dimension?.alias ? [`source column "${dimension.alias}"`] : [];
           const step = { resource: producer.name, node, field: field.key, applied };
-          if (dimension || measure) return { path: [step] };
+          if (dimension) return { path: [step] };
           if (field.key === "*") return { path: [{ ...step, applied: ["every column of this producer"] }] };
           if (Algo_default.isGlob(field.key)) return { path: [{ ...step, applied: ["every matching column"] }] };
           return { path: [step], unresolved: `"${field.key}" is not a column of producer "${producer.name}"` };
@@ -52359,11 +52828,8 @@ var init_FieldLineage = __esm({
             ...field.hidden ? ["hidden from the export"] : []
           ];
         };
-        /** Every column a producer exposes, dimensions then measures, in declared order. */
-        this._producerColumns = (producer) => [
-          ...(producer.dimensions ?? []).map((x2) => x2.name),
-          ...(producer.measures ?? []).map((x2) => x2.name)
-        ];
+        /** Every column a producer exposes, in declared order. */
+        this._producerColumns = (producer) => (producer.dimensions ?? []).map((x2) => x2.name);
         /** Whether a resource exposes a column by that name. A `*` consumer exposes whatever it is asked. */
         this._exposes = (env2, name, column) => {
           const producer = (env2.producers ?? []).find((x2) => x2.name === name);
@@ -52694,7 +53160,6 @@ var init_ResourceView = __esm({
             ],
             tables: [
               this._dimensionsTable(producer),
-              ...producer.measures?.length ? [this._measuresTable(producer)] : [],
               this._usedByTable(env2, producer),
               this._keyValueTable("Settings", settings)
             ],
@@ -52731,16 +53196,6 @@ var init_ResourceView = __esm({
           ]);
           return this._table("Dimensions", columns, rows);
         };
-        this._measuresTable = (producer) => {
-          const rows = (producer.measures ?? []).map((measure) => [
-            { value: measure.name, note: this._note(measure.description) },
-            { value: measure.sql }
-          ]);
-          return this._table("Measures", [
-            { key: "name", label: "Name" },
-            { key: "sql", label: "SQL" }
-          ], rows);
-        };
         /**
          * Which consumers read each column, the downstream half of field-level lineage. It is the answer
          * to "can this column go", which is a question a wide layout raises constantly and which nothing
@@ -54986,6 +55441,145 @@ var init_RunOverlay = __esm({
   }
 });
 
+// ../../packages/engines/src/parsing/ParquetExporter.ts
+var fs24, PARQUET_TYPES, TRUE_VALUES2, ParquetExporterClass, ParquetExporter, ParquetExporter_default;
+var init_ParquetExporter = __esm({
+  "../../packages/engines/src/parsing/ParquetExporter.ts"() {
+    fs24 = __toESM(require("fs"), 1);
+    init_src();
+    init_src3();
+    init_src5();
+    init_src2();
+    PARQUET_TYPES = {
+      string: "STRING",
+      number: "DOUBLE",
+      boolean: "BOOLEAN",
+      datetime: "TIMESTAMP"
+    };
+    TRUE_VALUES2 = /^(1|true|yes|y|t)$/i;
+    ParquetExporterClass = class {
+      constructor() {
+        /**
+         * Writes `request.sourcePath` to `request.targetPath` as parquet, SNAPPY compressed, one typed
+         * column per visible consumer field.
+         *
+         * Peak memory is bounded by one row group rather than by the dataset: the writer pulls the rows
+         * a group at a time from the generator below, and nothing holds the file as a whole.
+         */
+        this.exportToParquet = async (request) => {
+          Affirm_default(request, "Missing required parameter request");
+          Affirm_default(request.sourcePath, "Missing required parameter request.sourcePath");
+          Affirm_default(request.targetPath, "Missing required parameter request.targetPath");
+          Affirm_default(request.dsSchema, "Missing required parameter request.dsSchema");
+          Affirm_default(request.consumerName, "Missing required parameter request.consumerName");
+          const columns = this._columns(request.dsSchema);
+          Affirm_default.hasItems(columns, `Consumer "${request.consumerName}" has no visible field to export as parquet`);
+          const { parquetWriteRows, fileWriter } = await import("hyparquet-writer");
+          const rowGroupSize = this._rowGroupSize(columns.length);
+          const startedMs = Date.now();
+          await parquetWriteRows({
+            writer: fileWriter(request.targetPath),
+            columns,
+            rows: this._readRecords(request, columns),
+            rowGroupSize
+          });
+          const bytes = fs24.statSync(request.targetPath).size;
+          Logger_default.info(`Wrote parquet "${request.targetPath}" for consumer "${request.consumerName}" (${columns.length} column(s), ${rowGroupSize} rows/group, ${bytes} bytes, ${Date.now() - startedMs}ms)`);
+        };
+        /**
+         * The exported schema, taken from the dataset's own: one column per column ON the line, in line
+         * order. Not from the consumer's fields, which cannot know about a shape a `pivot` changed.
+         */
+        this._columns = (dsSchema) => {
+          return dsSchema.columns.map((column) => ({
+            name: column.name,
+            type: PARQUET_TYPES[column.type] ?? "STRING",
+            nullable: true
+          }));
+        };
+        /**
+         * Rows per row group, derived from a budget of VALUES rather than a fixed row count: a row group
+         * is buffered whole before it can be compressed, so what bounds the export's memory is cells, and
+         * 100,000 rows means 400k values on a 4-column consumer and 130 million on a 1,300-field one.
+         * Clamped at both ends — see the PARQUET_ constants for why each bound is there.
+         */
+        this._rowGroupSize = (columnCount) => {
+          Affirm_default(columnCount > 0, "Cannot size a row group for zero columns");
+          const { PARQUET_ROW_GROUP_CELLS, PARQUET_MIN_ROW_GROUP_ROWS, PARQUET_MAX_ROW_GROUP_ROWS } = Constants_default.defaults;
+          const rows = Math.floor(PARQUET_ROW_GROUP_CELLS / columnCount);
+          return Math.min(Math.max(rows, PARQUET_MIN_ROW_GROUP_ROWS), PARQUET_MAX_ROW_GROUP_ROWS);
+        };
+        /** A positional line read onto the column names by position — the file holds the visible columns only. */
+        this._positionalValues = (line, columns) => {
+          const parts = DatasetCodec_default.decodeRow(line);
+          const values = {};
+          for (const [index, column] of columns.entries())
+            values[column.name] = parts[index];
+          return values;
+        };
+        /**
+         * One value as its column holds it. A missing value is a null in the column rather than the type's
+         * zero, so a reader can still tell "no value" from a real `0`, `false` or empty string.
+         *
+         * Missing is null/undefined, plus — for every type but `string` — the empty string, matching
+         * `TypeCaster.castDimension`: that is how a NULL arrives from every source that stages through
+         * text. An empty string IS a value for a string column and is kept.
+         */
+        this._toParquetValue = (value, column, consumerName) => {
+          if (value === null || value === void 0) return null;
+          if (value === "" && column.type !== "STRING") return null;
+          switch (column.type) {
+            case "BOOLEAN":
+              return typeof value === "boolean" ? value : TRUE_VALUES2.test(String(value).trim());
+            case "DOUBLE":
+              return typeof value === "number" ? value : Number(value);
+            case "TIMESTAMP":
+              return this._toDate(value, column.name, consumerName);
+            default:
+              return typeof value === "string" ? value : String(value);
+          }
+        };
+        /**
+         * A `datetime` column's value as a Date. An unparseable one FAILS the run, naming the field and
+         * the value: a consumer whose transformation reformats a date into "DDMMYYYY" under a `datetime`
+         * dimension is a configuration error, and a timestamp column silently full of 1970 hides it until
+         * someone queries the lake.
+         */
+        this._toDate = (value, name, consumerName) => {
+          const date2 = value instanceof Date ? value : new Date(value);
+          if (isNaN(date2.getTime()))
+            throw new Error(`Consumer "${consumerName}" cannot export field "${name}" as a parquet TIMESTAMP: "${String(value)}" is not a date. The field declares a "datetime" dimension, so either the value or that declaration is wrong.`);
+          return date2;
+        };
+      }
+      /**
+       * The dataset's records, one at a time, coerced to what each column holds. An async generator, so
+       * the writer pulls a row group's worth and no more — the file is never in memory as a whole.
+       *
+       * A parquet consumer stages JSON (see `DatasetSchema.encodingFor`), so the values arrive already
+       * typed. The positional branch covers a dataset staged the other way, reading the row WITHOUT
+       * casting since the cast is what loses a boolean (`Boolean('false')` is `true`).
+       */
+      async *_readRecords(request, columns) {
+        const readStream = fs24.createReadStream(request.sourcePath);
+        try {
+          for await (const line of readLines(readStream)) {
+            const values = request.dsSchema.encoding === "CSV" ? this._positionalValues(line, columns) : JSON.parse(line);
+            const row = {};
+            for (const column of columns)
+              row[column.name] = this._toParquetValue(values[column.name], column, request.consumerName);
+            yield row;
+          }
+        } finally {
+          readStream.destroy();
+        }
+      }
+    };
+    ParquetExporter = new ParquetExporterClass();
+    ParquetExporter_default = ParquetExporter;
+  }
+});
+
 // ../../packages/engines/src/producer/PushdownEngine.ts
 var MISSING_BY_TYPE, PushdownEngineClass, PushdownEngine, PushdownEngine_default;
 var init_PushdownEngine = __esm({
@@ -55818,6 +56412,7 @@ var init_src10 = __esm({
     init_RunOverlay();
     init_RunRegistry();
     init_LineParser();
+    init_ParquetExporter();
     init_DiscoverMerger();
     init_ProducerEngine();
     init_ProducerManager();
@@ -55833,6 +56428,216 @@ var init_src10 = __esm({
   }
 });
 
+// ../../packages/executors/src/DatasetSchema.ts
+var DatasetSchemaClass, DatasetSchema, DatasetSchema_default;
+var init_DatasetSchema = __esm({
+  "../../packages/executors/src/DatasetSchema.ts"() {
+    init_src();
+    init_src10();
+    DatasetSchemaClass = class {
+      constructor() {
+        /**
+         * How a consumer's records are staged on the way to the destination. A consumer declares exactly
+         * one output (`maxItems: 1` in consumer-schema.json), so `outputs[0]` is THE output.
+         */
+        this.encodingFor = (consumer) => {
+          Affirm_default(consumer, "Invalid consumer");
+          Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output`);
+          switch (consumer.outputs[0].format) {
+            case "JSON":
+            case "API":
+              return "JSON";
+            case "PARQUET":
+              return "JSON";
+            case "TXT":
+              return "CSV";
+            case "FIXED":
+              return "CSV";
+            case "CSV":
+              return "CSV";
+            default:
+              throw new Error(`Export format ${consumer.outputs[0].format} not implemented yet.`);
+          }
+        };
+        /**
+         * The schema the run's workers write with: one column per VISIBLE consumer field, in field
+         * order. Hidden fields are not on the line at all.
+         */
+        this.compile = (consumer, fields) => {
+          Affirm_default(consumer, "Invalid consumer");
+          Affirm_default(fields, "Invalid consumer fields");
+          const columns = fields.filter((x2) => !x2.cField.hidden).map(this._column);
+          Affirm_default.hasItems(columns, `Consumer "${consumer.name}" has no visible field to write`);
+          return {
+            encoding: this.encodingFor(consumer),
+            columns,
+            rows: 0,
+            writtenBy: "worker"
+          };
+        };
+        /**
+         * The same shape, as the named stage left it. Returned by every pass that rewrites the dataset
+         * without changing its columns, so `rows` always describes the file as it is NOW.
+         */
+        this.rewritten = (dsSchema, writtenBy, rows) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default(rows >= 0, `Invalid row count ${rows} for a dataset written by "${writtenBy}"`);
+          return { ...dsSchema, writtenBy, rows };
+        };
+        /**
+         * The shape a `pivot` leaves behind: the row keys it grouped on, then one column per pivot value.
+         * A pivoted column has no producer dimension behind it - it is an aggregate this pass invented -
+         * so it declares `source: 'pivot'` and a `number` type.
+         */
+        this.forPivot = (dsSchema, rowKeys, pivotColumns, rows) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          Affirm_default.hasItems(rowKeys, "A pivot declares no row key");
+          const keyColumns = rowKeys.map((key) => {
+            const existing = dsSchema.columns.find((column) => column.name === key);
+            Affirm_default(existing, `The pivot row key "${key}" is not a column of the dataset being pivoted`);
+            return existing;
+          });
+          return {
+            encoding: dsSchema.encoding,
+            columns: [...keyColumns, ...pivotColumns.map((name) => ({ name, type: "number", source: "pivot" }))],
+            rows,
+            writtenBy: "pivot"
+          };
+        };
+        /** The column names on a line, in line order - the header of a positional export. */
+        this.names = (dsSchema) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          return dsSchema.columns.map((column) => column.name);
+        };
+        /**
+         * One column, typed from the dimension behind it. `type`, `format` and `nullable` are the only
+         * three properties a cast reads, so carrying those lets a reader cast a staged value back
+         * without going near the consumer config.
+         */
+        this._column = (field) => {
+          if (ConsumerManager_default.isLineCountField(field.cField))
+            return { name: field.finalKey, type: "number", source: "lineCount" };
+          const column = {
+            name: field.finalKey,
+            type: field.dimension?.type ?? "string"
+          };
+          if (field.dimension?.format) column.format = field.dimension.format;
+          if (field.dimension?.nullable === false) column.nullable = false;
+          return column;
+        };
+      }
+    };
+    DatasetSchema = new DatasetSchemaClass();
+    DatasetSchema_default = DatasetSchema;
+  }
+});
+
+// ../../packages/executors/src/FixedBlockOrder.ts
+var fs25, FixedBlockOrderClass, FixedBlockOrder, FixedBlockOrder_default;
+var init_FixedBlockOrder = __esm({
+  "../../packages/executors/src/FixedBlockOrder.ts"() {
+    fs25 = __toESM(require("fs"));
+    init_src();
+    init_src5();
+    init_src2();
+    FixedBlockOrderClass = class {
+      constructor() {
+        /** The file one block's records are collected in. Named by the value's place in `order`, not by
+         * the value itself: a discriminator value is data and may hold anything a path may not. */
+        this.blockPath = (sourcePath, position) => `${sourcePath}.block_${position}`;
+        /**
+         * Routes the staged dataset into one file per layout and concatenates them in `layouts.order`,
+         * returning the path of the block-ordered dataset the export then transforms.
+         */
+        this.reorder = async (request) => {
+          Affirm_default(request, "Invalid block order request");
+          Affirm_default(request.sourcePath, "Missing dataset to order into blocks");
+          Affirm_default(request.layouts, "Invalid fixed-width layouts");
+          Affirm_default.hasItems(request.layouts.order, "The fixed-width layouts declare no block order");
+          const { sourcePath, layouts, consumerName } = request;
+          const orderedPath = `${sourcePath}.blocks`;
+          const counts = await this._route(request);
+          await fs25.promises.unlink(sourcePath);
+          await this._concat(sourcePath, layouts, orderedPath);
+          Logger_default.info(`Consumer "${consumerName}" writes its FIXED record blocks as ${layouts.order.map((value, position) => `${value} x${counts[position]}`).join(", ")}`);
+          return orderedPath;
+        };
+        /**
+         * The routing pass: every line read once, appended to the file of the block its discriminator
+         * names. One stream per layout, all open at once — `MAX_FIXED_LAYOUTS` is what bounds them, so
+         * there is no eviction to do and a block's file is only ever written by its own stream.
+         *
+         * Returns the records each block took, in `order`, which is what the log line reports.
+         */
+        this._route = async (request) => {
+          const { sourcePath, layouts } = request;
+          const streams = layouts.order.map((_3, position) => fs25.createWriteStream(this.blockPath(sourcePath, position)));
+          const positions = new Map(layouts.order.map((value, position) => [value, position]));
+          const counts = layouts.order.map(() => 0);
+          const readStream = fs25.createReadStream(sourcePath);
+          try {
+            for await (const line of readLines(readStream)) {
+              const position = this._positionOf(line, layouts, positions);
+              counts[position]++;
+              const stream = streams[position];
+              if (!stream.write(line + "\n"))
+                await new Promise((resolve) => stream.once("drain", resolve));
+            }
+          } finally {
+            for (const stream of streams)
+              await this._closeStream(stream);
+            readStream.destroy();
+          }
+          return counts;
+        };
+        /** The block a staged line belongs to. A record no layout covers stops the run; see `planFor`. */
+        this._positionOf = (line, layouts, positions) => {
+          const value = DatasetCodec_default.decodeRow(line)[layouts.index];
+          const position = value === null || value === void 0 ? void 0 : positions.get(String(value));
+          Affirm_default(
+            position !== void 0,
+            `No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`
+          );
+          return position;
+        };
+        /** The blocks appended into one dataset, in `order`, each file removed as it is consumed. */
+        this._concat = async (sourcePath, layouts, orderedPath) => {
+          const writer = fs25.createWriteStream(orderedPath);
+          try {
+            for (let position = 0; position < layouts.order.length; position++) {
+              const blockPath = this.blockPath(sourcePath, position);
+              await this._append(blockPath, writer);
+              await fs25.promises.unlink(blockPath);
+            }
+          } finally {
+            await this._closeStream(writer);
+          }
+        };
+        /** One block's bytes onto the end of the ordered dataset. Piped, so a block is never held in memory. */
+        this._append = (blockPath, writer) => {
+          return new Promise((resolve, reject) => {
+            const reader = fs25.createReadStream(blockPath);
+            reader.on("error", reject);
+            reader.on("end", resolve);
+            reader.pipe(writer, { end: false });
+          });
+        };
+        /** Flushes and releases a stream's descriptor. Awaited: the read that follows must not race the flush. */
+        this._closeStream = async (stream) => {
+          if (!stream) return;
+          stream.end();
+          await new Promise((resolve, reject) => {
+            stream.on("finish", resolve);
+            stream.on("error", reject);
+          });
+        };
+      }
+    };
+    FixedBlockOrder = new FixedBlockOrderClass();
+    FixedBlockOrder_default = FixedBlockOrder;
+  }
+});
+
 // ../../packages/executors/src/OutputExecutor.ts
 var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
 var init_OutputExecutor = __esm({
@@ -55841,35 +56646,25 @@ var init_OutputExecutor = __esm({
     init_src();
     init_src8();
     init_src4();
+    init_src10();
     init_src5();
     init_src2();
+    init_DatasetSchema();
+    init_FixedBlockOrder();
     OutputExecutorClass = class {
       constructor() {
-        this._getInternalRecordFormat = (consumer) => {
-          const output = consumer.outputs[0];
-          switch (output.format) {
-            case "API":
-              return "JSON";
-            case "TXT":
-              return "CSV";
-            default:
-              return output.format;
-          }
-        };
         /**
-         * The output decisions a consumer makes once — the internal format, whether nulls are omitted,
-         * which fields are visible — lifted off the per-record path. `outputRecord` used to re-derive
-         * all three for every record, and `toCSV` allocated a filtered field array on top of that.
+         * The write side of a staged dataset's schema: what to write, in what order, in which encoding.
+         * Derived from the schema, so the writer and every reader share one column list.
          */
-        this.compileOutput = (consumer, fields) => {
+        this.compileOutput = (consumer, dsSchema) => {
           Affirm_default(consumer, "Invalid consumer");
-          Affirm_default(fields, "Invalid consumer fields");
+          Affirm_default(dsSchema, "Invalid dataset schema");
           return {
-            format: this._getInternalRecordFormat(consumer),
+            format: dsSchema.encoding,
             delimiter: ",",
-            visibleKeys: fields.filter((x2) => !x2.cField.hidden).map((x2) => x2.finalKey),
-            hiddenKeys: fields.filter((x2) => x2.cField.hidden).map((x2) => x2.finalKey),
-            omitNull: consumer.outputs.some((x2) => x2.omitNull === true)
+            visibleKeys: DatasetSchema_default.names(dsSchema),
+            omitNull: consumer.outputs[0].omitNull === true
           };
         };
         this.outputRecord = (record, plan) => {
@@ -55882,26 +56677,38 @@ var init_OutputExecutor = __esm({
               throw new Error(`Export format ${plan.format} not implemented yet.`);
           }
         };
+        /**
+         * One positional line of the internal format. `DatasetCodec`, not `CSVParser`: a staged line has
+         * to carry a null that is not an empty string and must never hold a newline.
+         */
         this.toCSV = (record, plan) => {
           const { visibleKeys, delimiter } = plan;
           let row = "";
           for (let index = 0; index < visibleKeys.length; index++) {
             if (index > 0)
               row += delimiter;
-            row += CSVParser_default.escapeField(record[visibleKeys[index]]?.toString() ?? "");
+            row += DatasetCodec_default.encodeField(record[visibleKeys[index]]);
           }
           return row;
         };
+        /**
+         * One JSON object per line, keyed in the CONSUMER's field order rather than the record's own.
+         *
+         * A record arrives keyed in the order its producer's dimensions were read, so two producers of
+         * one consumer wrote the same record as two different lines — and `distinct` and
+         * `no_duplicates`, which compare the line, saw two rows. Building from `visibleKeys` gives every
+         * writer one order, and drops the hidden fields without deleting them out of the caller's record.
+         */
         this.toJSON = (record, plan) => {
-          for (const key of plan.hiddenKeys)
-            delete record[key];
-          if (plan.omitNull) {
-            for (const key of Object.keys(record)) {
-              if (record[key] === null || record[key] === void 0)
-                delete record[key];
-            }
+          const { visibleKeys, omitNull } = plan;
+          const output = {};
+          for (const key of visibleKeys) {
+            const value = record[key];
+            if (omitNull && (value === null || value === void 0))
+              continue;
+            output[key] = value === void 0 ? null : value;
           }
-          return JSON.stringify(record);
+          return JSON.stringify(output);
         };
         /**
          * The exported name of one split file: the destination name with the file's label before the
@@ -55914,66 +56721,141 @@ var init_OutputExecutor = __esm({
           return `${pathName.substring(0, dot)}_${label}${pathName.substring(dot)}`;
         };
         /**
-         * Writes the run's result to every configured destination, one call per file the split pass
+         * Writes the run's result to the consumer's destination, one call per file the split pass
          * produced. The files are passed in rather than read back out of the scope folder: the label a
          * file carries is what names it at the destination, and with no split configured that list is
          * the merged dataset alone, labelled "0" — the suffix an unsplit export has always had.
+         *
+         * One output per consumer (`maxItems: 1` in consumer-schema.json). This used to loop, but only
+         * ever shipped the first entry, and could not have done more: `move` unlinks what it ships.
          */
-        this.exportResult = async (consumer, fields, splitFiles) => {
+        this.exportResult = async (consumer, dsSchema, splitFiles) => {
           Affirm_default.hasItems(splitFiles, `Consumer "${consumer.name}" has no output file to export`);
-          const internalFormat = this._getInternalRecordFormat(consumer);
+          Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output to export to`);
+          Affirm_default(dsSchema, `Consumer "${consumer.name}" has no dataset schema to export against`);
+          const output = consumer.outputs[0];
+          const internalFormat = dsSchema.encoding;
+          const destination = Environment_default.getSource(output.exportDestination);
+          Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
+          const driver = await DriverFactory_default.instantiateDestination(destination);
+          const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
+          Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
+          const header = this._composeHeader(output, dsSchema);
+          const fixedLayouts = output.format === "FIXED" && output.layouts ? FixedWidthWriter_default.compileLayouts(dsSchema, this._fixedPositions(consumer), output, consumer.name) : null;
+          const fixedLine = output.format === "FIXED" ? this._fixedLineWriter(consumer, dsSchema, output, fixedLayouts) : null;
           const results = [];
-          for (const output of consumer.outputs) {
-            const destination = Environment_default.getSource(output.exportDestination);
-            const driver = await DriverFactory_default.instantiateDestination(destination);
-            const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
-            Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
-            const header = this._composeHeader(output, fields);
-            for (const { path: startingPath, label } of splitFiles) {
-              const destinationPath = this.getCompletedPath(destinationName, label);
-              if (output.format === internalFormat) {
-                results.push(await driver.move(startingPath, destinationPath, header));
-              } else {
-                switch (output.format) {
-                  case "CSV":
-                    results.push(
-                      await driver.transformAndMove(startingPath, (line) => {
-                        const parsed = JSON.parse(line);
-                        return Object.keys(parsed).map((x2) => `"${parsed[x2]}"`).join(",");
-                      }, destinationPath, header)
-                    );
-                    break;
-                  case "TXT": {
-                    const delimiter = this._getDelimiter(output);
-                    results.push(
-                      await driver.transformAndMove(
-                        startingPath,
-                        (line) => CSVParser_default.stringifyRow(CSVParser_default.parseRow(line, ","), delimiter),
-                        destinationPath,
-                        header
-                      )
-                    );
-                    break;
-                  }
-                  case "API":
-                  case "JSON":
-                    results.push(
-                      await driver.transformAndMove(startingPath, (line) => {
-                        const parts = CSVParser_default.parseRow(line, ",");
-                        const value = {};
-                        for (const [index, field] of fields.entries())
-                          value[field.finalKey] = parts[index];
-                        return JSON.stringify(value);
-                      }, destinationPath)
-                    );
-                    break;
-                  default:
-                    throw new Error(`Export result to format ${output.format} not implemented yet.`);
-                }
+          for (const { path: startingPath, label } of splitFiles) {
+            const destinationPath = this.getCompletedPath(destinationName, label);
+            if (output.format === internalFormat || output.format === "API") {
+              results.push(await driver.move(startingPath, destinationPath, header));
+              continue;
+            }
+            switch (output.format) {
+              case "TXT": {
+                const delimiter = this._getDelimiter(output);
+                results.push(
+                  await driver.transformAndMove(
+                    startingPath,
+                    (line) => DatasetCodec_default.redelimit(line, delimiter),
+                    destinationPath,
+                    header
+                  )
+                );
+                break;
+              }
+              case "FIXED": {
+                const fixedPath = fixedLayouts ? await FixedBlockOrder_default.reorder({ sourcePath: startingPath, layouts: fixedLayouts, consumerName: consumer.name }) : startingPath;
+                results.push(await driver.transformAndMove(fixedPath, fixedLine, destinationPath));
+                break;
               }
+              case "PARQUET":
+                results.push(await this._exportParquet({
+                  sourcePath: startingPath,
+                  destinationPath,
+                  dsSchema,
+                  consumerName: consumer.name,
+                  driver
+                }));
+                break;
+              default:
+                throw new Error(`Export result to format ${output.format} not implemented yet.`);
             }
-            return this._unifingResult(results);
           }
+          return this._unifingResult(results);
+        };
+        /**
+         * A consumer's FIXED output resolved without running it: the layouts the export would compile,
+         * and every problem with them. `compile` prints what it resolves and fails on what it reports,
+         * and a run resolves it before reading a producer — so a width that does not add up is caught
+         * where it was written rather than after the whole dataset has been built.
+         *
+         * Null for a consumer that writes anything else: there is no layout to resolve.
+         */
+        this.reviewFixedOutput = (consumer, dsSchema) => {
+          Affirm_default(consumer, "Invalid consumer");
+          const output = consumer.outputs?.[0];
+          if (output?.format !== "FIXED") return null;
+          const schema = dsSchema ?? DatasetSchema_default.compile(consumer, ConsumerManager_default.getExpandedFields(consumer));
+          return FixedWidthWriter_default.review(schema, this._fixedPositions(consumer), output, consumer.name);
+        };
+        /**
+         * The line function a FIXED export hands its destination driver: one staged record in, the
+         * positional line it becomes out. The layout is resolved here, once, and closed over — so the
+         * per-record cost is a decode of the staged line and a pad per field, with no config lookup.
+         *
+         * With `layouts` declared the record's own discriminator picks its plan, which is the only
+         * per-record branch on the path; the records are already grouped into their blocks by then.
+         *
+         * No header is passed alongside it: a positional file has no room for one, and the field names
+         * would not sit on the columns they describe.
+         */
+        this._fixedLineWriter = (consumer, dsSchema, output, layouts) => {
+          const suffix = FixedWidthWriter_default.lineSuffix(output);
+          if (layouts) {
+            for (const value of layouts.order)
+              Logger_default.info(`Consumer "${consumer.name}" writes its FIXED "${value}" record as ${layouts.plans.get(value).recordLength} characters over ${layouts.plans.get(value).fields.length} field(s)`);
+            return (line) => {
+              const values = DatasetCodec_default.decodeRow(line);
+              return FixedWidthWriter_default.composeLine(values, FixedWidthWriter_default.planFor(values, layouts)) + suffix;
+            };
+          }
+          const plan = FixedWidthWriter_default.compilePlan(dsSchema.columns, this._fixedPositions(consumer), output, `the FIXED output of consumer "${consumer.name}"`);
+          Logger_default.info(`Consumer "${consumer.name}" writes a FIXED record of ${plan.recordLength} characters over ${plan.fields.length} field(s)`);
+          return (line) => FixedWidthWriter_default.composeLine(DatasetCodec_default.decodeRow(line), plan) + suffix;
+        };
+        /**
+         * Every visible field's `position` block, keyed by the name its column carries on the staged
+         * line. Read from the EXPANDED fields rather than the consumer's own, so a glob key brings the
+         * `position` it was declared with to each of the columns it stands for, and a field the dataset
+         * has no `position` for is left absent for `compilePlan` to name.
+         */
+        this._fixedPositions = (consumer) => {
+          const positions = {};
+          for (const field of ConsumerManager_default.getExpandedFields(consumer)) {
+            if (field.cField.hidden) continue;
+            positions[field.finalKey] = field.cField.position;
+          }
+          return positions;
+        };
+        /**
+         * Ships one split file as parquet. A parquet file is a columnar container whose footer can only
+         * be written once the last row group is encoded, so there is no per-line mapping to hand a
+         * driver: the file is built in the scope folder beside the dataset it is read from, then handed
+         * over as bytes with NO header — a parquet file carries its own schema, and `move` is
+         * byte-faithful in all three destination drivers.
+         *
+         * `move` unlinks what it shipped, so a successful export leaves no staged file behind. A failed
+         * one leaves it in the scope folder, which the run's cleanup removes with everything else.
+         */
+        this._exportParquet = async (plan) => {
+          const parquetPath = `${plan.sourcePath}.parquet`;
+          await ParquetExporter_default.exportToParquet({
+            sourcePath: plan.sourcePath,
+            targetPath: parquetPath,
+            dsSchema: plan.dsSchema,
+            consumerName: plan.consumerName
+          });
+          return plan.driver.move(parquetPath, plan.destinationPath);
         };
         /**
          * Folds the per-file upload results into the one the run reports.
@@ -55992,24 +56874,32 @@ var init_OutputExecutor = __esm({
         /**
          * The header line of a positional export: the visible field names, in column order, quoted and
          * delimited exactly as the rows under them — so the header parses with the same reader the data
-         * does. Null for a JSON/API output, where a record already carries its own field names, and for
-         * an output that turns the header off.
+         * does. Null for a JSON/API output, where a record already carries its own field names, for a
+         * FIXED one, where a name does not fit the column it describes and the receiver reads by
+         * position anyway, and for an output that turns the header off.
          *
-         * The names are the exported ones (`finalKey`, i.e. the alias where one is set), since a reader
-         * of the file has nothing else to match a column against.
+         * The names come from the dataset's own schema, so a `pivot` is described by the header.
          */
-        this._composeHeader = (output, fields) => {
+        this._composeHeader = (output, dsSchema) => {
           if (output.header === false) return null;
           if (output.format !== "CSV" && output.format !== "TXT") return null;
-          const names = fields.filter((x2) => !x2.cField.hidden).map((x2) => x2.finalKey);
-          return CSVParser_default.stringifyRow(names, this._getDelimiter(output));
+          return CSVParser_default.stringifyRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
         };
         /** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
         this._getDelimiter = (output) => {
           return output.format === "TXT" ? output.delimiter ?? "	" : ",";
         };
         this._getExtension = (output) => {
-          return output.format === "CSV" ? "csv" : output.format === "JSON" ? "jsonl" : "txt";
+          switch (output.format) {
+            case "CSV":
+              return "csv";
+            case "JSON":
+              return "jsonl";
+            case "PARQUET":
+              return "parquet";
+            default:
+              return "txt";
+          }
         };
         this._composeFileName = (consumer, output, extension, executionId) => {
           if (output.exportName && output.exportName.trim().length > 0) {
@@ -56041,7 +56931,7 @@ var init_ConsumerExecutor = __esm({
     init_src();
     init_src10();
     init_OutputExecutor();
-    init_src10();
+    init_DatasetSchema();
     init_src2();
     LIMIT_SCAN_BYTES = 64 * 1024;
     NEWLINE_BYTE = 10;
@@ -56165,13 +57055,20 @@ var init_ConsumerExecutor = __esm({
           }
           return output;
         };
-        this.processLineCount = async (consumer, datasetPath, executorResults) => {
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
-          const outputPlan = OutputExecutor_default.compileOutput(consumer, internalFields);
-          const lineCountFields = ConsumerManager_default.getLineCountFieldsFromExpanded(internalFields, true);
-          if (lineCountFields.length === 0)
-            return 0;
+        /**
+         * Rewrites every `lineCount` field of the merged dataset so it names the line of the SOURCE file
+         * the row came from, by adding each chunk's `rowOffset` to the counter its worker wrote.
+         *
+         * The shape is unchanged, so the schema comes back with a fresh `writtenBy` and the row count
+         * this pass actually saw - which also checks the file still holds what the workers reported.
+         */
+        this.processLineCount = async (consumer, dsSchema, datasetPath, executorResults) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
+          const lineCountColumns = dsSchema.columns.filter((column) => column.source === "lineCount");
+          if (lineCountColumns.length === 0)
+            return dsSchema;
+          const parseLine = LineParser_default.datasetParser(dsSchema);
+          const outputPlan = OutputExecutor_default.compileOutput(consumer, dsSchema);
           const tempWorkPath = datasetPath + "_tmp";
           const reader = import_fs21.default.createReadStream(datasetPath);
           const writer = import_fs21.default.createWriteStream(tempWorkPath);
@@ -56193,11 +57090,11 @@ var init_ConsumerExecutor = __esm({
             Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
             const rowOffset = currentWorker.rowOffset ?? 0;
             if (rowOffset > 0) {
-              const record = this._parseLine(line, internalRecordFormat, internalFields);
-              for (const field of lineCountFields) {
-                const currentValue = Number(record[field.finalKey]);
+              const record = parseLine(line);
+              for (const column of lineCountColumns) {
+                const currentValue = Number(record[column.name]);
                 Affirm_default(!Number.isNaN(currentValue), `Invalid lineCount value found while processing ${datasetPath}`);
-                record[field.finalKey] = currentValue + rowOffset;
+                record[column.name] = currentValue + rowOffset;
               }
               if (!writer.write(OutputExecutor_default.outputRecord(record, outputPlan) + "\n"))
                 await waitForDrain();
@@ -56220,11 +57117,11 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return processedLineCount;
+          return DatasetSchema_default.rewritten(dsSchema, "lineCount", processedLineCount);
         };
         /**
-         * Cuts the merged dataset down to the run's record limit, and reports the row count it left
-         * behind — or `null` when the dataset was already within the limit and nothing was touched.
+         * Cuts the merged dataset down to the run's record limit, and reports the shape it left behind.
+         * The schema comes back unchanged when the dataset was already within the limit.
          *
          * Every worker stops at the limit within its OWN chunk, which is the right thing for the read
          * (no thread reads further than the run could need) but leaves up to `limit` rows per chunk in
@@ -56234,7 +57131,8 @@ var init_ConsumerExecutor = __esm({
          * at then needs no arithmetic on line contents, so no line ending or multi-byte character can
          * put the cut in the middle of a row. Only the bytes up to the limit are ever read.
          */
-        this.processLimit = async (datasetPath, limit) => {
+        this.processLimit = async (dsSchema, datasetPath, limit) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           Affirm_default(datasetPath, "Invalid dataset path");
           Affirm_default(limit > 0, `Invalid record limit ${limit}`);
           const handle = await import_promises16.default.open(datasetPath, "r+");
@@ -56245,7 +57143,7 @@ var init_ConsumerExecutor = __esm({
             while (newlines < limit) {
               const { bytesRead } = await handle.read(buffer, 0, LIMIT_SCAN_BYTES, offset);
               if (bytesRead === 0)
-                return null;
+                return dsSchema;
               for (let index = 0; index < bytesRead; index++) {
                 offset++;
                 if (buffer[index] !== NEWLINE_BYTE) continue;
@@ -56255,14 +57153,15 @@ var init_ConsumerExecutor = __esm({
             }
             const { size } = await handle.stat();
             if (offset >= size)
-              return null;
+              return dsSchema;
             await handle.truncate(offset);
-            return limit;
+            return DatasetSchema_default.rewritten(dsSchema, "limit", limit);
           } finally {
             await handle.close();
           }
         };
-        this.processDistinct = async (datasetPath) => {
+        this.processDistinct = async (dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const reader = import_fs21.default.createReadStream(datasetPath);
           const tempWorkPath = datasetPath + "_tmp";
           const writer = import_fs21.default.createWriteStream(tempWorkPath);
@@ -56291,18 +57190,18 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return newLineCount;
+          return DatasetSchema_default.rewritten(dsSchema, "distinct", newLineCount);
         };
-        this.processDistinctOn = async (consumer, datasetPath) => {
+        this.processDistinctOn = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const reader = import_fs21.default.createReadStream(datasetPath);
           const { distinctOn } = consumer.options;
           const { keys, resolution } = distinctOn;
           const { strategy, orderBy, direction = "asc" } = resolution;
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           const winners = /* @__PURE__ */ new Map();
           for await (const line of readLines(reader)) {
-            const record = internalRecordFormat === "CSV" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
+            const record = parseLine(line);
             const compositeKey = keys.map((k7) => String(record[k7] ?? "")).join("|");
             const existing = winners.get(compositeKey);
             if (!existing) {
@@ -56334,20 +57233,27 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return winners.size;
+          return DatasetSchema_default.rewritten(dsSchema, "distinctOn", winners.size);
         };
-        this.processPivot = async (consumer, datasetPath) => {
+        /**
+         * Groups the dataset by `rowKeys` and turns one column's values into columns of their own.
+         *
+         * This is the one pass that changes the SHAPE of the file, which is why it returns the schema it
+         * wrote rather than a row count: the header, the split pass, the dataset validation and the
+         * parquet schema all read the result.
+         */
+        this.processPivot = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const { pivot } = consumer.options;
           const { rowKeys, pivotColumn, valueColumn, aggregation, columnPrefix = "" } = pivot;
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           let pivotValues = pivot.pivotValues;
           if (!pivotValues) {
             pivotValues = [];
             const discoverySet = /* @__PURE__ */ new Set();
             const discoverReader = import_fs21.default.createReadStream(datasetPath);
             for await (const line of readLines(discoverReader)) {
-              const record = this._parseLine(line, internalRecordFormat, internalFields);
+              const record = parseLine(line);
               const val = String(record[pivotColumn] ?? "");
               if (!discoverySet.has(val)) {
                 discoverySet.add(val);
@@ -56364,7 +57270,7 @@ var init_ConsumerExecutor = __esm({
           const groups = /* @__PURE__ */ new Map();
           const reader = import_fs21.default.createReadStream(datasetPath);
           for await (const line of readLines(reader)) {
-            const record = this._parseLine(line, internalRecordFormat, internalFields);
+            const record = parseLine(line);
             const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
             const pivotVal = String(record[pivotColumn] ?? "");
             const numericVal = Number(record[valueColumn]) || 0;
@@ -56383,11 +57289,8 @@ var init_ConsumerExecutor = __esm({
             cell.min = Math.min(cell.min, numericVal);
             cell.max = Math.max(cell.max, numericVal);
           }
-          const pivotedFields = [
-            ...rowKeys.map((k7) => ({ cField: { key: k7 }, finalKey: k7 })),
-            ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
-          ];
-          const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedFields);
+          const pivotedDsSchema = DatasetSchema_default.forPivot(dsSchema, rowKeys, pivotValues.map((pv) => columnPrefix + pv), groups.size);
+          const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedDsSchema);
           const tempWorkPath = datasetPath + "_tmp";
           const writer = import_fs21.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
@@ -56437,10 +57340,8 @@ var init_ConsumerExecutor = __esm({
           }
           await import_promises16.default.unlink(datasetPath);
           await import_promises16.default.rename(tempWorkPath, datasetPath);
-          return outputCount;
-        };
-        this._parseLine = (line, format4, fields) => {
-          return format4 === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+          Affirm_default(outputCount === pivotedDsSchema.rows, `Pivot wrote ${outputCount} row(s) but declared ${pivotedDsSchema.rows}`);
+          return pivotedDsSchema;
         };
         /**
          * Determines if the new record should replace the existing record based on the resolution strategy
@@ -56467,11 +57368,11 @@ var init_ConsumerExecutor = __esm({
               return false;
           }
         };
-        this.processDatasetValidation = async (consumer, datasetPath) => {
+        this.processDatasetValidation = async (consumer, dsSchema, datasetPath) => {
+          Affirm_default(dsSchema, "Invalid dataset schema");
           const validations = consumer.validate;
           if (!validations || validations.length === 0) return [];
-          const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           let rowCount = 0;
           const seenRows = /* @__PURE__ */ new Set();
           const fieldValueSets = /* @__PURE__ */ new Map();
@@ -56492,7 +57393,7 @@ var init_ConsumerExecutor = __esm({
                 seenRows.add(line);
             }
             if (uniqueFieldKeys.length > 0) {
-              const record = internalRecordFormat === "CSV" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
+              const record = parseLine(line);
               for (const fieldKey of uniqueFieldKeys) {
                 const valueSet = fieldValueSets.get(fieldKey);
                 const val = String(record[fieldKey] ?? "");
@@ -56629,14 +57530,13 @@ var init_ExecutorPerformance = __esm({
 });
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs26, ExecutorWriter, ExecutorWriter_default;
+var fs28, ExecutorWriter, ExecutorWriter_default;
 var init_ExecutorWriter = __esm({
   "../../packages/executors/src/ExecutorWriter.ts"() {
-    fs26 = __toESM(require("fs"));
+    fs28 = __toESM(require("fs"));
     init_src();
     init_src3();
     init_src10();
-    init_OutputExecutor();
     ExecutorWriter = class {
       constructor() {
         /**
@@ -56666,7 +57566,8 @@ var init_ExecutorWriter = __esm({
           Affirm_default(request, "Invalid split request");
           Affirm_default(request.sourcePath, "Missing dataset to split");
           Affirm_default(request.consumer, "Invalid consumer");
-          const { scope, sourcePath, consumer, fields } = request;
+          const { scope, sourcePath, consumer, dsSchema } = request;
+          Affirm_default(dsSchema, "Missing dataset schema to split against");
           const keys = scope.split?.by ?? [];
           const maxFileBytes = scope.split?.maxFileSizeGb > 0 ? scope.split.maxFileSizeGb * this.BYTES_PER_GB : null;
           if (keys.length === 0 && !maxFileBytes)
@@ -56681,21 +57582,21 @@ var init_ExecutorWriter = __esm({
             usedLabels: /* @__PURE__ */ new Set(),
             fileCount: 0
           };
-          await this._route(pass, fields);
+          await this._route(pass, dsSchema);
           if (pass.buckets.size === 0)
             return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
-          await fs26.promises.unlink(sourcePath);
+          await fs28.promises.unlink(sourcePath);
           return this._listFiles(pass);
         };
         /**
          * The pass itself: one line read, one line written, for every line of the dataset.
          */
-        this._route = async (pass, fields) => {
-          const internalFormat = OutputExecutor_default._getInternalRecordFormat(pass.consumer);
-          const readStream = fs26.createReadStream(pass.sourcePath);
+        this._route = async (pass, dsSchema) => {
+          const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
+          const readStream = fs28.createReadStream(pass.sourcePath);
           try {
             for await (const line of readLines(readStream)) {
-              const bucket = this._bucketFor(line, pass, internalFormat, fields);
+              const bucket = this._bucketFor(line, pass, parseLine);
               const lineBytes = Buffer.byteLength(line) + 1;
               if (this._needsRotation(bucket, pass, lineBytes))
                 await this._rotate(bucket, pass);
@@ -56715,8 +57616,8 @@ var init_ExecutorWriter = __esm({
          * own: a shared catch-all would silently merge two different things, and that is invisible until
          * it is wrong.
          */
-        this._bucketFor = (line, pass, internalFormat, fields) => {
-          const bucketKey = pass.keys.length === 0 ? "" : this._bucketKey(line, pass.keys, internalFormat, fields);
+        this._bucketFor = (line, pass, parseLine) => {
+          const bucketKey = parseLine ? this._bucketKey(line, pass.keys, parseLine) : "";
           const existing = pass.buckets.get(bucketKey);
           if (existing) return existing;
           const bucket = {
@@ -56730,8 +57631,8 @@ var init_ExecutorWriter = __esm({
           return bucket;
         };
         /** The composite value a line buckets on, read off the parsed record. */
-        this._bucketKey = (line, keys, internalFormat, fields) => {
-          const record = internalFormat === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+        this._bucketKey = (line, keys, parseLine) => {
+          const record = parseLine(line);
           return keys.map((key) => {
             const value = record[key];
             if (value === null || value === void 0) return "_null";
@@ -56796,7 +57697,7 @@ var init_ExecutorWriter = __esm({
           }
           if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
             await this._closeStream(pass.openStreams.shift());
-          bucket.stream = fs26.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
+          bucket.stream = fs28.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
           pass.openStreams.push(bucket);
           return bucket.stream;
         };
@@ -57085,7 +57986,6 @@ var init_LookupResolver = __esm({
     init_src4();
     init_src10();
     init_src2();
-    init_OutputExecutor();
     LookupResolverClass = class {
       constructor() {
         // Reuse dicts across sibling consumers within one process; invalidated when the config changes.
@@ -57108,28 +58008,35 @@ var init_LookupResolver = __esm({
               Affirm_default(!stack.includes(ref.consumer), `code_lookup cycle detected: ${[...stack, ref.consumer].join(" -> ")}`);
               const lookupConsumer = Environment_default.getConsumer(ref.consumer);
               Affirm_default(lookupConsumer, `code_lookup references unknown consumer "${ref.consumer}"`);
-              let capturePath = captureByConsumer.get(ref.consumer);
-              if (!capturePath) {
-                capturePath = this.getCapturePath(scope, ref.consumer);
+              let capture = captureByConsumer.get(ref.consumer);
+              if (!capture) {
+                const capturePath = this.getCapturePath(scope, ref.consumer);
                 ExecutorScope_default.ensurePath(capturePath);
-                await this.runLookupConsumer(lookupConsumer, capturePath, stack, request);
-                captureByConsumer.set(ref.consumer, capturePath);
+                capture = {
+                  path: capturePath,
+                  dsSchema: await this.runLookupConsumer(lookupConsumer, capturePath, stack, request)
+                };
+                captureByConsumer.set(ref.consumer, capture);
               }
-              this.cache.set(id, await this.buildDict(lookupConsumer, capturePath, ref));
+              this.cache.set(id, await this.buildDict(capture, ref));
             }
             ExecutorScope_default.ensurePath(outPath);
             await import_promises17.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
             manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path22.default.resolve(outPath) });
           }
-          for (const capturePath of captureByConsumer.values())
-            await import_promises17.default.rm(capturePath, { force: true });
+          for (const capture of captureByConsumer.values())
+            await import_promises17.default.rm(capture.path, { force: true });
           return manifest;
         };
-        // Runs the source consumer through the real pipeline, capturing its merged output before cleanup.
+        /**
+         * Runs the source consumer through the real pipeline, capturing its merged output before
+         * cleanup and returning the shape that run produced. The capture is a verbatim copy of the
+         * dataset, so its schema is the ONLY description of it.
+         */
         this.runLookupConsumer = async (lookupConsumer, capturePath, stack, request) => {
           const { default: ExecutorOrchestrator2 } = await Promise.resolve().then(() => (init_ExecutorOrchestrator(), ExecutorOrchestrator_exports));
           Logger_default.log(`[lookup] Materializing consumer "${lookupConsumer.name}" (path: ${stack.join(" -> ")})`);
-          await ExecutorOrchestrator2.launch({
+          const result = await ExecutorOrchestrator2.launch({
             consumer: lookupConsumer,
             details: request.details,
             captureMainDatasetTo: capturePath,
@@ -57137,22 +58044,23 @@ var init_LookupResolver = __esm({
             _lookupStack: stack,
             syntheticProducerInputs: request.syntheticProducerInputs
           });
+          Affirm_default(result.dsSchema, `code_lookup source consumer "${lookupConsumer.name}" reported no dataset schema`);
+          return result.dsSchema;
         };
         // Projects the captured dataset to a { code: value } dict, bounded by MAX_LOOKUP_ROWS.
-        this.buildDict = async (lookupConsumer, capturePath, ref) => {
-          const format4 = OutputExecutor_default._getInternalRecordFormat(lookupConsumer);
-          const visibleFields = ConsumerManager_default.getExpandedFields(lookupConsumer).filter((f7) => !f7.cField.hidden);
-          const finalKeys = visibleFields.map((f7) => f7.finalKey);
-          Affirm_default(finalKeys.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
-          Affirm_default(finalKeys.includes(ref.value), `code_lookup value column "${ref.value}" is not in the output of consumer "${ref.consumer}"`);
+        this.buildDict = async (capture, ref) => {
+          const columnNames = capture.dsSchema.columns.map((column) => column.name);
+          Affirm_default(columnNames.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
+          Affirm_default(columnNames.includes(ref.value), `code_lookup value column "${ref.value}" is not in the output of consumer "${ref.consumer}"`);
+          const parseLine = LineParser_default.datasetParser(capture.dsSchema);
           const dict = {};
           let rowsRead = 0;
           let warnedDup = false;
-          for await (const line of readLines(import_fs23.default.createReadStream(capturePath))) {
+          for await (const line of readLines(import_fs23.default.createReadStream(capture.path))) {
             if (!line.length) continue;
             rowsRead++;
             Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
-            const record = this.parseLine(line, format4, visibleFields);
+            const record = parseLine(line);
             const code = String(record[ref.key]);
             if (code in dict && !warnedDup) {
               Logger_default.warn(`code_lookup: duplicate key(s) in consumer "${ref.consumer}" for "${ref.key}"; last value wins`);
@@ -57163,10 +58071,6 @@ var init_LookupResolver = __esm({
           Logger_default.log(`[lookup] Built table "${ref.consumer}" (${ref.key}->${ref.value}): ${Object.keys(dict).length} entr(ies) from ${rowsRead} row(s)`);
           return dict;
         };
-        this.parseLine = (line, format4, visibleFields) => {
-          if (format4 === "CSV") return LineParser_default._internalParseCSV(line, visibleFields, ",");
-          return JSON.parse(line);
-        };
         this.getCapturePath = (scope, consumerName) => {
           const dir = import_path22.default.dirname(ExecutorScope_default.getLookupPath(scope, "x"));
           return import_path22.default.join(dir, `${consumerName}.capture`);
@@ -57200,7 +58104,6 @@ var init_ConsumerInputResolver = __esm({
     init_src();
     init_src3();
     init_src4();
-    init_src10();
     init_src2();
     ConsumerInputResolverClass = class {
       constructor() {
@@ -57225,23 +58128,29 @@ var init_ConsumerInputResolver = __esm({
               Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
               const capturePath = ExecutorScope_default.getConsumerInputPath(scope, name);
               ExecutorScope_default.ensurePath(capturePath);
-              await this.runUpstreamConsumer(upstream, capturePath, stack, request);
+              const dsSchema = await this.runUpstreamConsumer(upstream, capturePath, stack, request);
               const { size } = await import_promises18.default.stat(capturePath);
-              this.cache.set(name, { consumer: name, path: import_path23.default.resolve(capturePath), fileSize: size });
+              this.cache.set(name, { consumer: name, path: import_path23.default.resolve(capturePath), fileSize: size, dsSchema });
             }
             manifest.set(name, this.cache.get(name));
           }
           return manifest;
         };
-        // Builds the in-memory producer the orchestrator reads the JSONL capture through. Its dimensions
-        // are the upstream consumer's visible output fields, matching the keys written into the capture.
-        this.buildSyntheticProducer = (upstreamConsumer) => {
-          Affirm_default(upstreamConsumer, "Invalid upstream consumer");
-          const dimensions = ConsumerManager_default.getExpandedFields(upstreamConsumer).filter((field) => !field.cField.hidden).map((field) => ({ name: field.finalKey, type: field.dimension?.type ?? "string" }));
+        /**
+         * Builds the in-memory producer the orchestrator reads the JSONL capture through. Its dimensions
+         * are the columns the upstream run REPORTED writing, not the consumer's declared fields, which a
+         * `pivot` leaves describing a shape no longer in the file.
+         *
+         * No `format` is carried over: the capture holds values the upstream run already rendered, so
+         * re-parsing them with that dimension's input format would fail on its own output.
+         */
+        this.buildSyntheticProducer = (entry) => {
+          Affirm_default(entry, "Invalid consumer input entry");
+          Affirm_default(entry.dsSchema, `Consumer input "${entry.consumer}" carries no dataset schema`);
           return {
-            name: upstreamConsumer.name,
-            source: `__consumer_input__:${upstreamConsumer.name}`,
-            dimensions,
+            name: entry.consumer,
+            source: `__consumer_input__:${entry.consumer}`,
+            dimensions: entry.dsSchema.columns.map((column) => ({ name: column.name, type: column.type })),
             settings: { fileType: "JSONL" },
             _version: 1
           };
@@ -57256,11 +58165,12 @@ var init_ConsumerInputResolver = __esm({
           }
           return names;
         };
-        // Runs the upstream consumer through the real pipeline, capturing its full output as JSONL.
+        // Runs the upstream consumer through the real pipeline, capturing its output as JSONL and
+        // reporting the shape that run produced.
         this.runUpstreamConsumer = async (upstream, capturePath, stack, request) => {
           const { default: ExecutorOrchestrator2 } = await Promise.resolve().then(() => (init_ExecutorOrchestrator(), ExecutorOrchestrator_exports));
           Logger_default.log(`[consumer-input] Materializing consumer "${upstream.name}" (path: ${stack.join(" -> ")})`);
-          await ExecutorOrchestrator2.launch({
+          const result = await ExecutorOrchestrator2.launch({
             consumer: upstream,
             details: request.details,
             captureMainDatasetTo: capturePath,
@@ -57269,6 +58179,8 @@ var init_ConsumerInputResolver = __esm({
             _consumerInputStack: stack,
             syntheticProducerInputs: request.syntheticProducerInputs
           });
+          Affirm_default(result.dsSchema, `consumer-as-input source "${upstream.name}" reported no dataset schema`);
+          return result.dsSchema;
         };
         this.invalidateCacheIfStale = () => {
           const generation = Environment_default._configHash ?? null;
@@ -57310,6 +58222,7 @@ var init_ExecutorOrchestrator = __esm({
     init_src();
     init_src10();
     init_ExecutorWriter();
+    init_DatasetSchema();
     import_promises20 = require("stream/promises");
     init_src2();
     init_ExecutorProgress2();
@@ -57328,9 +58241,9 @@ var init_ExecutorOrchestrator = __esm({
          * spawn lazily, on first exec, and a thread that dies mid-run is replaced by another, so this is
          * a count of spawns and NOT a concurrency figure (see IWorkerCensus).
          *
-         * `workerData` carries the loaded environment, so it is cloned once per THREAD at spawn rather
-         * than read and validated from disk once per chunk. Only run-constant data belongs there; per-task
-         * data goes in the exec argument, which is cloned again for every chunk.
+         * `workerData` carries the loaded environment and the run's dataset schema, so both are cloned
+         * once per THREAD at spawn rather than per chunk. Only run-constant data belongs there; per-task
+         * data goes in the exec argument, cloned per chunk.
          */
         this.createPool = (maxWorkers, census, threadData2) => {
           const options2 = {
@@ -57361,13 +58274,19 @@ var init_ExecutorOrchestrator = __esm({
           const tracker = new ExecutorPerformance_default();
           const _progress = new ExecutorProgress2_default(logProgress);
           const { usageId } = UsageManager_default.startUsage(consumer, details);
-          const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], split: OrchestratorHelper_default.resolveSplit(consumer) };
+          const scope = ExecutorScope_default.open({ id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], split: OrchestratorHelper_default.resolveSplit(consumer) });
           const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
+          let dsSchema = DatasetSchema_default.compile(consumer, expandedFields);
           const census = { created: 0, peak: 0 };
           try {
             const start = performance.now();
             const executorResults = [];
             Logger_default.info(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length}${OrchestratorHelper_default.formatRequestOptions(options2)}) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+            const fixedReview = request.skipExport ? null : OutputExecutor_default.reviewFixedOutput(consumer, dsSchema);
+            Affirm_default(
+              !fixedReview || fixedReview.errors.length === 0,
+              `Consumer "${consumer.name}" cannot write its FIXED output: ${fixedReview?.errors.join(" ")}`
+            );
             const lookups = await LookupResolver_default.materialize(consumer, expandedFields, scope, request);
             const consumerInputs = await ConsumerInputResolver_default.materialize(consumer, scope, request);
             let counter = performance.now();
@@ -57397,6 +58316,7 @@ var init_ExecutorOrchestrator = __esm({
             executorResults.push(...await this.runWorkFiles({
               workFiles,
               consumer,
+              dsSchema,
               scope,
               options: options2,
               lookups,
@@ -57411,58 +58331,26 @@ var init_ExecutorOrchestrator = __esm({
             Logger_default.log(`[${usageId}] All workers complete \u2014 ${executorResults.length} result(s), total input: ${Algo_default.sum(executorResults.map((x2) => x2.inputCount))}, total output: ${Algo_default.sum(executorResults.map((x2) => x2.outputCount))}`);
             _progress.update({ phase: "Reconciling work", progress: 0 });
             await this.reconcileExecutorThreadsResults(scope, executorResults, tracker);
+            dsSchema = DatasetSchema_default.rewritten(dsSchema, "merge", Algo_default.sum(executorResults.map((x2) => x2.outputCount)));
             _progress.update({ phase: "Reconciling work", progress: 1 });
             Logger_default.log(`[${usageId}] Reconciled ${executorResults.length} worker result(s)`);
             _progress.update({ phase: "Post-processing", progress: 0 });
-            const postOperation = { totalOutputCount: null };
-            if (executorResults.length > 1) {
-              if (consumer.options?.distinct === true) {
-                Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-distinct:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (consumer.options?.distinctOn) {
-                Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-distinct-on:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (consumer.options?.pivot) {
-                Logger_default.log(`[${usageId}] Running pivot operation`);
-                counter = performance.now();
-                const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
-                tracker.measure("process-pivot:main", performance.now() - counter);
-                postOperation.totalOutputCount = unifiedOutputCount;
-                Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-              if (ConsumerManager_default.hasLineCountFields(consumer, true)) {
-                Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
-                counter = performance.now();
-                const processedLineCount = await ConsumerExecutor_default.processLineCount(consumer, ExecutorScope_default.getMainPath(scope), executorResults);
-                tracker.measure("process-line-count:main", performance.now() - counter);
-                Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${processedLineCount} rows in ${Math.round(performance.now() - counter)}ms`);
-              }
-            }
+            dsSchema = await this.runPostPasses({ consumer, dsSchema, scope, executorResults, tracker, usageId });
             if (options2?.limit > 0) {
               counter = performance.now();
-              const limitedOutputCount = await ConsumerExecutor_default.processLimit(ExecutorScope_default.getMainPath(scope), options2.limit);
+              const limited = await ConsumerExecutor_default.processLimit(dsSchema, ExecutorScope_default.getMainPath(scope), options2.limit);
               tracker.measure("process-limit:main", performance.now() - counter);
-              if (Algo_default.hasVal(limitedOutputCount)) {
-                postOperation.totalOutputCount = limitedOutputCount;
-                Logger_default.log(`[${usageId}] Record limit applied: dataset cut to ${limitedOutputCount} row(s) in ${Math.round(performance.now() - counter)}ms`);
+              if (limited.writtenBy === "limit") {
+                Logger_default.log(`[${usageId}] Record limit applied: dataset cut to ${limited.rows} row(s) in ${Math.round(performance.now() - counter)}ms`);
               } else {
                 Logger_default.log(`[${usageId}] Record limit of ${options2.limit} not reached \u2014 dataset left whole`);
               }
+              dsSchema = limited;
             }
             if (consumer.validate && consumer.validate.length > 0) {
               Logger_default.log(`[${usageId}] Running dataset-level validations`);
               counter = performance.now();
-              const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default.getMainPath(scope));
+              const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, dsSchema, ExecutorScope_default.getMainPath(scope));
               tracker.measure("dataset-validation", performance.now() - counter);
               for (const result of validationResults) {
                 if (result.onFail === "fail") {
@@ -57476,7 +58364,7 @@ var init_ExecutorOrchestrator = __esm({
               Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
             }
             if (request.captureMainDatasetTo) {
-              await this.captureDataset(consumer, expandedFields, ExecutorScope_default.getMainPath(scope), request.captureMainDatasetTo, request.captureFormat);
+              await this.captureDataset(dsSchema, ExecutorScope_default.getMainPath(scope), request.captureMainDatasetTo, request.captureFormat);
               Logger_default.log(`[${usageId}] Captured dataset to ${request.captureMainDatasetTo}${request.captureFormat ? ` (as ${request.captureFormat})` : ""}`);
             }
             if (request.skipExport) {
@@ -57485,8 +58373,8 @@ var init_ExecutorOrchestrator = __esm({
               const materializeResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, null);
               materializeResult.elapsedMS = performance.now() - start;
               materializeResult.peakWorkers = census.peak;
-              if (Algo_default.hasVal(postOperation.totalOutputCount))
-                materializeResult.outputCount = postOperation.totalOutputCount;
+              materializeResult.outputCount = dsSchema.rows;
+              materializeResult.dsSchema = dsSchema;
               UsageManager_default.endUsage(usageId, materializeResult.outputCount, materializeResult);
               await Logger_default.flush();
               return materializeResult;
@@ -57496,7 +58384,7 @@ var init_ExecutorOrchestrator = __esm({
               scope,
               sourcePath: ExecutorScope_default.getMainPath(scope),
               consumer,
-              fields: expandedFields
+              dsSchema
             });
             tracker.measure("split-output", performance.now() - counter);
             if (scope.split)
@@ -57505,7 +58393,7 @@ var init_ExecutorOrchestrator = __esm({
             _progress.update({ phase: "Exporting result", progress: 0 });
             counter = performance.now();
             Logger_default.info(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-            const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, splitFiles);
+            const exportRes = await OutputExecutor_default.exportResult(consumer, dsSchema, splitFiles);
             tracker.measure("export-result", performance.now() - counter);
             Affirm_default(exportRes.res, `Consumer "${consumer.name}" failed to write its output to "${consumer.outputs.map((x2) => x2.exportDestination).join(", ")}" \u2014 see the logged driver error for the cause`);
             Logger_default.info(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
@@ -57522,8 +58410,8 @@ var init_ExecutorOrchestrator = __esm({
             const finalResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
             finalResult.elapsedMS = performance.now() - start;
             finalResult.peakWorkers = census.peak;
-            if (Algo_default.hasVal(postOperation.totalOutputCount))
-              finalResult.outputCount = postOperation.totalOutputCount;
+            finalResult.outputCount = dsSchema.rows;
+            finalResult.dsSchema = dsSchema;
             UsageManager_default.endUsage(usageId, finalResult.outputCount, finalResult);
             Logger_default.info(`[${usageId}] Consumer "${consumer.name}" completed: ${finalResult.outputCount} rows, ${finalResult.workerCount} chunk(s) on ${census.peak} thread(s), ${Math.round(finalResult.elapsedMS)}ms`);
             await Logger_default.flush();
@@ -57601,7 +58489,10 @@ var init_ExecutorOrchestrator = __esm({
             `Invalid worker pool size ${poolSize} \u2014 it must come from getParallelWorkerCount(), never from the work list`
           );
           let phase = performance.now();
-          const pool = this.createPool(poolSize, census, { environment: Environment_default.createSnapshot() });
+          const pool = this.createPool(poolSize, census, {
+            environment: Environment_default.createSnapshot(),
+            dsSchema: request.dsSchema
+          });
           tracker.measure("pool-create", performance.now() - phase);
           const context = {
             consumer: request.consumer,
@@ -57824,24 +58715,23 @@ var init_ExecutorOrchestrator = __esm({
         };
         /**
          * Copies the merged main.dataset out of the run scope for a materialization run. With
-         * captureFormat 'JSONL' the output is normalized to newline-delimited JSON objects keyed by
-         * each visible field's finalKey — self-describing, so a synthetic producer can read it without a
+         * captureFormat 'JSONL' the output is normalized to newline-delimited JSON objects keyed by the
+         * dataset's own column names — self-describing, so a synthetic producer can read it without a
          * header row. main.dataset in JSON internal format is already JSONL (verbatim copy); CSV internal
          * format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
          */
-        this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
+        this.captureDataset = async (dsSchema, sourcePath, targetPath, captureFormat) => {
           await import_promises19.default.mkdir(import_path24.default.dirname(targetPath), { recursive: true });
-          const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
-          if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
+          if (captureFormat !== "JSONL" || dsSchema.encoding !== "CSV") {
             await import_promises19.default.copyFile(sourcePath, targetPath);
             return;
           }
-          const visibleFields = fields.filter((field) => !field.cField.hidden);
+          const parseLine = LineParser_default.datasetParser(dsSchema);
           const output = import_fs25.default.createWriteStream(targetPath);
           try {
             for await (const line of readLines(import_fs25.default.createReadStream(sourcePath))) {
               if (!line.length) continue;
-              const record = LineParser_default._internalParseCSV(line, visibleFields, ",");
+              const record = parseLine(line);
               if (!output.write(JSON.stringify(record) + "\n"))
                 await new Promise((resolve) => output.once("drain", resolve));
             }
@@ -57854,7 +58744,7 @@ var init_ExecutorOrchestrator = __esm({
           }
         };
         /**
-         * Grabs the remote datasets and creates an internal .dataset to work on
+         * Grabs the remote datasets and stages each one as a local `.extract` to work on
          */
         this.readySourceFiles = async (args) => {
           const { consumer, scope, consumerInputs, syntheticInputs, options: options2, onProgress } = args;
@@ -57864,7 +58754,7 @@ var init_ExecutorOrchestrator = __esm({
             const reportProducer = (fraction) => onProgress?.((i6 + fraction) / consumer.producers.length);
             const consumerInput = consumerInputs.get(cProd.name);
             if (consumerInput) {
-              const prod2 = ConsumerInputResolver_default.buildSyntheticProducer(Environment_default.getConsumer(cProd.name));
+              const prod2 = ConsumerInputResolver_default.buildSyntheticProducer(consumerInput);
               const response2 = { files: [{ fullUri: consumerInput.path, fileSize: consumerInput.fileSize }] };
               Logger_default.log(`[${scope.id}] Producer "${cProd.name}" is a consumer input: ${consumerInput.fileSize} bytes (${i6 + 1}/${consumer.producers.length})`);
               results.push({ prod: prod2, cProd, response: response2 });
@@ -58013,6 +58903,49 @@ var init_ExecutorOrchestrator = __esm({
           }
           return mainPath;
         };
+        /**
+         * The passes that work across lines, run on the merged dataset. Returns the schema the last
+         * pass to write the file left behind — its `rows` is the run's row count.
+         *
+         * `lineCount` goes FIRST: it walks each chunk's `outputCount` in order to find the worker a line
+         * came from, which only holds while the file still has sum(outputCount) lines in worker order.
+         */
+        this.runPostPasses = async (request) => {
+          const { consumer, scope, executorResults, tracker, usageId } = request;
+          const datasetPath = ExecutorScope_default.getMainPath(scope);
+          const isMerged = executorResults.length > 1;
+          let dsSchema = request.dsSchema;
+          let counter = 0;
+          if (isMerged && ConsumerManager_default.hasLineCountFields(consumer, true)) {
+            Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processLineCount(consumer, dsSchema, datasetPath, executorResults);
+            tracker.measure("process-line-count:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (isMerged && consumer.options?.distinct === true) {
+            Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processDistinct(dsSchema, datasetPath);
+            tracker.measure("process-distinct:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] Distinct pass complete: ${dsSchema.rows} unique rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (isMerged && consumer.options?.distinctOn) {
+            Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processDistinctOn(consumer, dsSchema, datasetPath);
+            tracker.measure("process-distinct-on:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] DistinctOn pass complete: ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (consumer.options?.pivot) {
+            Logger_default.log(`[${usageId}] Running pivot operation`);
+            counter = performance.now();
+            dsSchema = await ConsumerExecutor_default.processPivot(consumer, dsSchema, datasetPath);
+            tracker.measure("process-pivot:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] Pivot complete: ${dsSchema.rows} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          return dsSchema;
+        };
         this.performCleanupOperations = async (scope, tracker) => {
           const start = performance.now();
           await ExecutorScope_default.clearScope(scope);
@@ -58034,6 +58967,7 @@ var init_ExecutorOrchestrator = __esm({
 // ../../packages/executors/src/index.ts
 init_src();
 init_ConsumerExecutor();
+init_DatasetSchema();
 
 // ../../packages/executors/src/Executor.ts
 var import_fs22 = __toESM(require("fs"));
@@ -58189,6 +59123,7 @@ var Executor = class {
     this.run = async (request) => {
       Affirm_default(request, "Invalid request");
       const { consumer, producer, prodDimensions, workerId, chunk, options: options2, scope, reportWork, lookups } = request;
+      Affirm_default(request.dsSchema, `Worker ${workerId} was given no dataset schema to write with`);
       const recordLimit = this.canBoundChunk(consumer) ? options2?.limit : null;
       this.hydrateLookups(lookups);
       const counter = performance.now();
@@ -58212,7 +59147,8 @@ var Executor = class {
       const writer = this.openWriteStream(scope, workerId);
       const fields = ConsumerManager_default.getExpandedFields(consumer);
       const plan = RecordPlan_default.compile({ consumer, producer, fields, dimensions: prodDimensions });
-      const outputPlan = OutputExecutor_default.compileOutput(consumer, fields);
+      let dsSchema = request.dsSchema;
+      const outputPlan = OutputExecutor_default.compileOutput(consumer, dsSchema);
       const linePlan = ProducerExecutor_default.compileLine(prodDimensions, chunk);
       const recordSelector = ProducerManager_default.resolveRecordSelector(producer);
       const { isFirstChunk, start, end } = chunk;
@@ -58296,14 +59232,16 @@ var Executor = class {
       if (consumer.options?.distinct === true) {
         Logger_default.log(`[${workerId}] Running distinct pass`);
         perf = performance.now();
-        totalOutputCount = await ConsumerExecutor_default.processDistinct(result.resultUri);
+        dsSchema = await ConsumerExecutor_default.processDistinct(dsSchema, result.resultUri);
+        totalOutputCount = dsSchema.rows;
         this._performance.measure("process-distinct", performance.now() - perf);
         totalCycles++;
       }
       if (consumer.options?.distinctOn) {
         Logger_default.log(`[${workerId}] Running distinctOn pass (${consumer.options.distinctOn})`);
         perf = performance.now();
-        totalOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, result.resultUri);
+        dsSchema = await ConsumerExecutor_default.processDistinctOn(consumer, dsSchema, result.resultUri);
+        totalOutputCount = dsSchema.rows;
         this._performance.measure("process-distinct-on", performance.now() - perf);
         totalCycles++;
       }
@@ -58353,6 +59291,7 @@ var Executor_default = Executor;
 // ../../packages/executors/src/index.ts
 init_ExecutorOrchestrator();
 init_ExecutorPerformance();
+init_ExecutorWriter();
 init_OrchestratorHelper();
 
 // ../../packages/executors/src/workers/ExecutorWorkerRunner.ts
@@ -58361,6 +59300,8 @@ var import_dotenv = __toESM(require("dotenv"));
 var import_worker_threads = require("worker_threads");
 init_src();
 init_src4();
+init_src10();
+init_DatasetSchema();
 init_src2();
 import_dotenv.default.configDotenv();
 var MODULE_LOADED_AT = performance.timeOrigin + performance.now();
@@ -58411,6 +59352,7 @@ var runExecutorTask = async (workerData) => {
       chunk,
       options: options2,
       scope,
+      dsSchema: resolveDsSchema(consumer),
       lookups,
       reportWork: (packet) => import_workerpool2.default.workerEmit(packet)
     });
@@ -58431,6 +59373,11 @@ var runExecutorTask = async (workerData) => {
     throw error;
   }
 };
+var resolveDsSchema = (consumer) => {
+  const sent = import_worker_threads.workerData?.dsSchema;
+  if (sent) return sent;
+  return DatasetSchema_default.compile(consumer, ConsumerManager_default.getExpandedFields(consumer));
+};
 var registerExecutorWorker = () => {
   import_workerpool2.default.worker({
     executor: runExecutorTask
@@ -58507,8 +59454,7 @@ var SampleRunnerClass = class {
           captureMainDatasetTo: capturePath,
           captureFormat: "JSONL"
         });
-        const captured = await this._readCapture(capturePath, size);
-        const records = captured.map((record) => this._inFieldOrder(consumer, record));
+        const records = await this._readCapture(capturePath, size);
         Logger_default.log(`[sample] Consumer "${consumer.name}": ${records.length} row(s) from ${result.inputCount} input line(s)`);
         return {
           kind: "consumer",
@@ -58556,28 +59502,6 @@ var SampleRunnerClass = class {
         return this._inputProducers(nested, seen);
       });
     };
-    /**
-     * One row's keys in the order the consumer DECLARES its fields.
-     *
-     * A record comes out of a worker keyed in the order the producer's dimensions were read, which is
-     * the source's layout and not the consumer's — so a consumer declaring country, plan, is_active
-     * over a producer that stores is_active first would be shown its own columns in someone else's
-     * order. Invisible in an exported JSON object; the whole shape of the table here.
-     *
-     * Anything the record carries that the field list does not name is appended rather than dropped:
-     * a column that turns up unexpectedly is exactly the one worth seeing.
-     */
-    this._inFieldOrder = (consumer, record) => {
-      const declared = ConsumerManager_default.getExpandedFields(consumer).filter((field) => !field.cField.hidden).map((field) => field.finalKey);
-      const ordered = {};
-      for (const key of declared) {
-        if (key in record) ordered[key] = record[key];
-      }
-      for (const key of Object.keys(record)) {
-        if (!(key in ordered)) ordered[key] = record[key];
-      }
-      return ordered;
-    };
     /** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
     this._readCapture = async (capturePath, size) => {
       if (!import_fs26.default.existsSync(capturePath)) return [];