@forzalabs/remora 2.1.1 → 2.2.0

package/workers/ExecutorWorker.js

@@ -72,6 +72,67 @@ var init_Affirm = __esm({
   }
 });
 
+// ../../packages/core/src/BatchWriter.ts
+var FLUSH_AFTER_CHARS, BatchWriter, BatchWriter_default;
+var init_BatchWriter = __esm({
+  "../../packages/core/src/BatchWriter.ts"() {
+    init_Affirm();
+    FLUSH_AFTER_CHARS = 1 << 18;
+    BatchWriter = class {
+      constructor(stream) {
+        /** How many times the stream's buffer filled and the caller had to wait for it. */
+        this.drains = 0;
+        this.buffer = "";
+        /** Queues one line and its newline. False means a batch went out and the stream wants a drain. */
+        this.write = (line) => {
+          this.buffer += line;
+          this.buffer += "\n";
+          if (this.buffer.length < FLUSH_AFTER_CHARS)
+            return true;
+          return this.flush();
+        };
+        this.waitForDrain = () => {
+          this.drains++;
+          return new Promise((resolve) => this.stream.once("drain", resolve));
+        };
+        /**
+         * Writes whatever is still buffered, then waits for the stream to be fully closed.
+         *
+         * Waits for `close`, not `finish`: `finish` only means `end()` has flushed its buffer to the OS,
+         * while `close` means the descriptor is released. Callers that rename or unlink the file right
+         * after — which most of the passes that write a dataset do — need the descriptor gone, or the
+         * rename fails with EPERM on Windows. `close` is strictly later than `finish`, so waiting for it
+         * is safe for the callers that do not care.
+         *
+         * The flush above may hand the stream more than its high-water mark, which `write` would report
+         * by returning false. There is nothing to wait for here: `end()` flushes what is buffered before
+         * it emits `finish`, so the batch lands either way.
+         */
+        this.close = async () => {
+          if (this.stream.destroyed)
+            return;
+          this.flush();
+          await new Promise((resolve, reject) => {
+            this.stream.once("close", resolve);
+            this.stream.once("error", reject);
+            this.stream.end();
+          });
+        };
+        this.flush = () => {
+          if (!this.buffer)
+            return true;
+          const batch = this.buffer;
+          this.buffer = "";
+          return this.stream.write(batch);
+        };
+        Affirm_default(stream, "Invalid write stream");
+        this.stream = stream;
+      }
+    };
+    BatchWriter_default = BatchWriter;
+  }
+});
+
 // ../../packages/core/src/Algo.ts
 var algo, Algo_default;
 var init_Algo = __esm({
@@ -233,6 +294,36 @@ var init_Algo = __esm({
   }
 });
 
+// ../../packages/core/src/LineReader.ts
+async function* readLines(input) {
+  Affirm_default(input, "Invalid input stream");
+  input.setEncoding("utf8");
+  let tail = "";
+  for await (const chunk of input) {
+    let text = tail + chunk;
+    let pendingReturn = "";
+    if (text.endsWith("\r")) {
+      pendingReturn = "\r";
+      text = text.slice(0, -1);
+    }
+    const parts = text.split(TERMINATOR);
+    tail = parts.pop() + pendingReturn;
+    for (const part of parts)
+      yield part;
+  }
+  if (tail.endsWith("\r"))
+    yield tail.slice(0, -1);
+  else if (tail.length > 0)
+    yield tail;
+}
+var TERMINATOR;
+var init_LineReader = __esm({
+  "../../packages/core/src/LineReader.ts"() {
+    init_Affirm();
+    TERMINATOR = /\r\n|\n|\r/;
+  }
+});
+
 // ../../packages/core/src/dste/DSTE.ts
 var DeterministicSimulationTestingEngine, DSTE, DSTE_default;
 var init_DSTE = __esm({
@@ -283,7 +374,9 @@ Tests summary:
 var init_src = __esm({
   "../../packages/core/src/index.ts"() {
     init_Affirm();
+    init_BatchWriter();
     init_Algo();
+    init_LineReader();
     init_DSTE();
   }
 });
@@ -4203,14 +4296,14 @@ var require_from = __commonJS({
       return (hint === "string" ? String : Number)(input);
     }
     var ERR_INVALID_ARG_TYPE = require_errors2().codes.ERR_INVALID_ARG_TYPE;
-    function from(Readable8, iterable, opts) {
+    function from(Readable10, iterable, opts) {
       var iterator;
       if (iterable && typeof iterable.next === "function") {
         iterator = iterable;
       } else if (iterable && iterable[Symbol.asyncIterator]) iterator = iterable[Symbol.asyncIterator]();
       else if (iterable && iterable[Symbol.iterator]) iterator = iterable[Symbol.iterator]();
       else throw new ERR_INVALID_ARG_TYPE("iterable", ["Iterable"], iterable);
-      var readable = new Readable8(_objectSpread({
+      var readable = new Readable10(_objectSpread({
         objectMode: true
       }, opts));
       var reading = false;
@@ -4250,9 +4343,9 @@ var require_from = __commonJS({
 var require_stream_readable = __commonJS({
   "../../node_modules/readable-stream/lib/_stream_readable.js"(exports2, module2) {
     "use strict";
-    module2.exports = Readable8;
+    module2.exports = Readable10;
     var Duplex;
-    Readable8.ReadableState = ReadableState;
+    Readable10.ReadableState = ReadableState;
     var EE = require("events").EventEmitter;
     var EElistenerCount = function EElistenerCount2(emitter, type) {
       return emitter.listeners(type).length;
@@ -4287,7 +4380,7 @@ var require_stream_readable = __commonJS({
     var StringDecoder;
     var createReadableStreamAsyncIterator;
     var from;
-    require_inherits()(Readable8, Stream);
+    require_inherits()(Readable10, Stream);
     var errorOrDestroy = destroyImpl.errorOrDestroy;
     var kProxyEvents = ["error", "close", "destroy", "pause", "resume"];
     function prependListener(emitter, event, fn) {
@@ -4331,9 +4424,9 @@ var require_stream_readable = __commonJS({
         this.encoding = options2.encoding;
       }
     }
-    function Readable8(options2) {
+    function Readable10(options2) {
       Duplex = Duplex || require_stream_duplex();
-      if (!(this instanceof Readable8)) return new Readable8(options2);
+      if (!(this instanceof Readable10)) return new Readable10(options2);
       var isDuplex = this instanceof Duplex;
       this._readableState = new ReadableState(options2, this, isDuplex);
       this.readable = true;
@@ -4343,7 +4436,7 @@ var require_stream_readable = __commonJS({
       }
       Stream.call(this);
     }
-    Object.defineProperty(Readable8.prototype, "destroyed", {
+    Object.defineProperty(Readable10.prototype, "destroyed", {
       // making it explicit this property is not enumerable
       // because otherwise some prototype manipulation in
       // userland will fail
@@ -4361,12 +4454,12 @@ var require_stream_readable = __commonJS({
         this._readableState.destroyed = value;
       }
     });
-    Readable8.prototype.destroy = destroyImpl.destroy;
-    Readable8.prototype._undestroy = destroyImpl.undestroy;
-    Readable8.prototype._destroy = function(err2, cb) {
+    Readable10.prototype.destroy = destroyImpl.destroy;
+    Readable10.prototype._undestroy = destroyImpl.undestroy;
+    Readable10.prototype._destroy = function(err2, cb) {
       cb(err2);
     };
-    Readable8.prototype.push = function(chunk, encoding) {
+    Readable10.prototype.push = function(chunk, encoding) {
       var state2 = this._readableState;
       var skipChunkCheck;
       if (!state2.objectMode) {
@@ -4383,7 +4476,7 @@ var require_stream_readable = __commonJS({
       }
       return readableAddChunk(this, chunk, encoding, false, skipChunkCheck);
     };
-    Readable8.prototype.unshift = function(chunk) {
+    Readable10.prototype.unshift = function(chunk) {
       return readableAddChunk(this, chunk, null, true, false);
     };
     function readableAddChunk(stream, chunk, encoding, addToFront, skipChunkCheck) {
@@ -4444,10 +4537,10 @@ var require_stream_readable = __commonJS({
       }
       return er2;
     }
-    Readable8.prototype.isPaused = function() {
+    Readable10.prototype.isPaused = function() {
       return this._readableState.flowing === false;
     };
-    Readable8.prototype.setEncoding = function(enc) {
+    Readable10.prototype.setEncoding = function(enc) {
       if (!StringDecoder) StringDecoder = require_string_decoder().StringDecoder;
       var decoder = new StringDecoder(enc);
       this._readableState.decoder = decoder;
@@ -4493,7 +4586,7 @@ var require_stream_readable = __commonJS({
       }
       return state2.length;
     }
-    Readable8.prototype.read = function(n4) {
+    Readable10.prototype.read = function(n4) {
       debug("read", n4);
       n4 = parseInt(n4, 10);
       var state2 = this._readableState;
@@ -4602,10 +4695,10 @@ var require_stream_readable = __commonJS({
       }
       state2.readingMore = false;
     }
-    Readable8.prototype._read = function(n4) {
+    Readable10.prototype._read = function(n4) {
       errorOrDestroy(this, new ERR_METHOD_NOT_IMPLEMENTED("_read()"));
     };
-    Readable8.prototype.pipe = function(dest, pipeOpts) {
+    Readable10.prototype.pipe = function(dest, pipeOpts) {
       var src = this;
       var state2 = this._readableState;
       switch (state2.pipesCount) {
@@ -4708,7 +4801,7 @@ var require_stream_readable = __commonJS({
         }
       };
     }
-    Readable8.prototype.unpipe = function(dest) {
+    Readable10.prototype.unpipe = function(dest) {
       var state2 = this._readableState;
       var unpipeInfo = {
         hasUnpiped: false
@@ -4742,7 +4835,7 @@ var require_stream_readable = __commonJS({
       dest.emit("unpipe", this, unpipeInfo);
       return this;
     };
-    Readable8.prototype.on = function(ev, fn) {
+    Readable10.prototype.on = function(ev, fn) {
       var res = Stream.prototype.on.call(this, ev, fn);
       var state2 = this._readableState;
       if (ev === "data") {
@@ -4763,15 +4856,15 @@ var require_stream_readable = __commonJS({
       }
       return res;
     };
-    Readable8.prototype.addListener = Readable8.prototype.on;
-    Readable8.prototype.removeListener = function(ev, fn) {
+    Readable10.prototype.addListener = Readable10.prototype.on;
+    Readable10.prototype.removeListener = function(ev, fn) {
       var res = Stream.prototype.removeListener.call(this, ev, fn);
       if (ev === "readable") {
         process.nextTick(updateReadableListening, this);
       }
       return res;
     };
-    Readable8.prototype.removeAllListeners = function(ev) {
+    Readable10.prototype.removeAllListeners = function(ev) {
       var res = Stream.prototype.removeAllListeners.apply(this, arguments);
       if (ev === "readable" || ev === void 0) {
         process.nextTick(updateReadableListening, this);
@@ -4791,7 +4884,7 @@ var require_stream_readable = __commonJS({
       debug("readable nexttick read 0");
       self2.read(0);
     }
-    Readable8.prototype.resume = function() {
+    Readable10.prototype.resume = function() {
       var state2 = this._readableState;
       if (!state2.flowing) {
         debug("resume");
@@ -4817,7 +4910,7 @@ var require_stream_readable = __commonJS({
       flow(stream);
       if (state2.flowing && !state2.reading) stream.read(0);
     }
-    Readable8.prototype.pause = function() {
+    Readable10.prototype.pause = function() {
       debug("call pause flowing=%j", this._readableState.flowing);
       if (this._readableState.flowing !== false) {
         debug("pause");
@@ -4832,7 +4925,7 @@ var require_stream_readable = __commonJS({
       debug("flow", state2.flowing);
       while (state2.flowing && stream.read() !== null) ;
     }
-    Readable8.prototype.wrap = function(stream) {
+    Readable10.prototype.wrap = function(stream) {
       var _this = this;
       var state2 = this._readableState;
       var paused = false;
@@ -4877,14 +4970,14 @@ var require_stream_readable = __commonJS({
       return this;
     };
     if (typeof Symbol === "function") {
-      Readable8.prototype[Symbol.asyncIterator] = function() {
+      Readable10.prototype[Symbol.asyncIterator] = function() {
         if (createReadableStreamAsyncIterator === void 0) {
           createReadableStreamAsyncIterator = require_async_iterator();
         }
         return createReadableStreamAsyncIterator(this);
       };
     }
-    Object.defineProperty(Readable8.prototype, "readableHighWaterMark", {
+    Object.defineProperty(Readable10.prototype, "readableHighWaterMark", {
       // making it explicit this property is not enumerable
       // because otherwise some prototype manipulation in
       // userland will fail
@@ -4893,7 +4986,7 @@ var require_stream_readable = __commonJS({
         return this._readableState.highWaterMark;
       }
     });
-    Object.defineProperty(Readable8.prototype, "readableBuffer", {
+    Object.defineProperty(Readable10.prototype, "readableBuffer", {
       // making it explicit this property is not enumerable
       // because otherwise some prototype manipulation in
       // userland will fail
@@ -4902,7 +4995,7 @@ var require_stream_readable = __commonJS({
         return this._readableState && this._readableState.buffer;
       }
     });
-    Object.defineProperty(Readable8.prototype, "readableFlowing", {
+    Object.defineProperty(Readable10.prototype, "readableFlowing", {
       // making it explicit this property is not enumerable
       // because otherwise some prototype manipulation in
       // userland will fail
@@ -4916,8 +5009,8 @@ var require_stream_readable = __commonJS({
         }
       }
     });
-    Readable8._fromList = fromList;
-    Object.defineProperty(Readable8.prototype, "readableLength", {
+    Readable10._fromList = fromList;
+    Object.defineProperty(Readable10.prototype, "readableLength", {
       // making it explicit this property is not enumerable
       // because otherwise some prototype manipulation in
       // userland will fail
@@ -4963,11 +5056,11 @@ var require_stream_readable = __commonJS({
       }
     }
     if (typeof Symbol === "function") {
-      Readable8.from = function(iterable, opts) {
+      Readable10.from = function(iterable, opts) {
         if (from === void 0) {
           from = require_from();
         }
-        return from(Readable8, iterable, opts);
+        return from(Readable10, iterable, opts);
       };
     }
     function indexOf(xs2, x2) {
@@ -4989,9 +5082,9 @@ var require_stream_duplex = __commonJS({
       return keys2;
     };
     module2.exports = Duplex;
-    var Readable8 = require_stream_readable();
+    var Readable10 = require_stream_readable();
     var Writable2 = require_stream_writable();
-    require_inherits()(Duplex, Readable8);
+    require_inherits()(Duplex, Readable10);
     {
       keys = objectKeys(Writable2.prototype);
       for (v3 = 0; v3 < keys.length; v3++) {
@@ -5004,7 +5097,7 @@ var require_stream_duplex = __commonJS({
     var v3;
     function Duplex(options2) {
       if (!(this instanceof Duplex)) return new Duplex(options2);
-      Readable8.call(this, options2);
+      Readable10.call(this, options2);
       Writable2.call(this, options2);
       this.allowHalfOpen = true;
       if (options2) {
@@ -6374,14 +6467,14 @@ var require_series = __commonJS({
 var require_stream_transform = __commonJS({
   "../../node_modules/readable-stream/lib/_stream_transform.js"(exports2, module2) {
     "use strict";
-    module2.exports = Transform4;
+    module2.exports = Transform5;
     var _require$codes = require_errors2().codes;
     var ERR_METHOD_NOT_IMPLEMENTED = _require$codes.ERR_METHOD_NOT_IMPLEMENTED;
     var ERR_MULTIPLE_CALLBACK = _require$codes.ERR_MULTIPLE_CALLBACK;
     var ERR_TRANSFORM_ALREADY_TRANSFORMING = _require$codes.ERR_TRANSFORM_ALREADY_TRANSFORMING;
     var ERR_TRANSFORM_WITH_LENGTH_0 = _require$codes.ERR_TRANSFORM_WITH_LENGTH_0;
     var Duplex = require_stream_duplex();
-    require_inherits()(Transform4, Duplex);
+    require_inherits()(Transform5, Duplex);
     function afterTransform(er2, data) {
       var ts2 = this._transformState;
       ts2.transforming = false;
@@ -6400,8 +6493,8 @@ var require_stream_transform = __commonJS({
         this._read(rs2.highWaterMark);
       }
     }
-    function Transform4(options2) {
-      if (!(this instanceof Transform4)) return new Transform4(options2);
+    function Transform5(options2) {
+      if (!(this instanceof Transform5)) return new Transform5(options2);
       Duplex.call(this, options2);
       this._transformState = {
         afterTransform: afterTransform.bind(this),
@@ -6429,14 +6522,14 @@ var require_stream_transform = __commonJS({
         done(this, null, null);
       }
     }
-    Transform4.prototype.push = function(chunk, encoding) {
+    Transform5.prototype.push = function(chunk, encoding) {
       this._transformState.needTransform = false;
       return Duplex.prototype.push.call(this, chunk, encoding);
     };
-    Transform4.prototype._transform = function(chunk, encoding, cb) {
+    Transform5.prototype._transform = function(chunk, encoding, cb) {
       cb(new ERR_METHOD_NOT_IMPLEMENTED("_transform()"));
     };
-    Transform4.prototype._write = function(chunk, encoding, cb) {
+    Transform5.prototype._write = function(chunk, encoding, cb) {
       var ts2 = this._transformState;
       ts2.writecb = cb;
       ts2.writechunk = chunk;
@@ -6446,7 +6539,7 @@ var require_stream_transform = __commonJS({
         if (ts2.needTransform || rs2.needReadable || rs2.length < rs2.highWaterMark) this._read(rs2.highWaterMark);
       }
     };
-    Transform4.prototype._read = function(n4) {
+    Transform5.prototype._read = function(n4) {
       var ts2 = this._transformState;
       if (ts2.writechunk !== null && !ts2.transforming) {
         ts2.transforming = true;
@@ -6455,7 +6548,7 @@ var require_stream_transform = __commonJS({
         ts2.needTransform = true;
       }
     };
-    Transform4.prototype._destroy = function(err2, cb) {
+    Transform5.prototype._destroy = function(err2, cb) {
       Duplex.prototype._destroy.call(this, err2, function(err22) {
         cb(err22);
       });
@@ -6476,11 +6569,11 @@ var require_stream_passthrough = __commonJS({
   "../../node_modules/readable-stream/lib/_stream_passthrough.js"(exports2, module2) {
     "use strict";
     module2.exports = PassThrough;
-    var Transform4 = require_stream_transform();
-    require_inherits()(PassThrough, Transform4);
+    var Transform5 = require_stream_transform();
+    require_inherits()(PassThrough, Transform5);
     function PassThrough(options2) {
       if (!(this instanceof PassThrough)) return new PassThrough(options2);
-      Transform4.call(this, options2);
+      Transform5.call(this, options2);
     }
     PassThrough.prototype._transform = function(chunk, encoding, cb) {
       cb(null, chunk);
@@ -6546,7 +6639,7 @@ var require_pipeline = __commonJS({
       if (typeof streams[streams.length - 1] !== "function") return noop;
       return streams.pop();
     }
-    function pipeline6() {
+    function pipeline7() {
       for (var _len = arguments.length, streams = new Array(_len), _key = 0; _key < _len; _key++) {
         streams[_key] = arguments[_key];
       }
@@ -6569,7 +6662,7 @@ var require_pipeline = __commonJS({
       });
       return streams.reduce(pipe);
     }
-    module2.exports = pipeline6;
+    module2.exports = pipeline7;
   }
 });
 
@@ -7807,15 +7900,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path21 = [graph[toModel].parent, toModel];
+      const path20 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path21.unshift(graph[cur].parent);
+        path20.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path21;
+      fn.conversion = path20;
       return fn;
     }
     function route(fromModel) {
@@ -8445,7 +8538,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs27 = require("fs");
+    var fs28 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -8466,7 +8559,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs27.open(options2.file, "a+", "0644", (err2, fd) => {
+      fs28.open(options2.file, "a+", "0644", (err2, fd) => {
         if (err2) {
           if (!iter) {
             stream.emit("error", err2);
@@ -8478,10 +8571,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs27.close(fd, noop);
+            fs28.close(fd, noop);
             return;
           }
-          return fs27.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs28.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -8540,8 +8633,8 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs27 = require("fs");
-    var path21 = require("path");
+    var fs28 = require("fs");
+    var path20 = require("path");
     var asyncSeries = require_series();
     var zlib3 = require("zlib");
     var { MESSAGE } = require_triple_beam();
@@ -8571,14 +8664,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options2.filename || options2.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options2.filename ? path21.basename(options2.filename) : "winston.log";
-          this.dirname = options2.dirname || path21.dirname(options2.filename);
+          this._basename = this.filename = options2.filename ? path20.basename(options2.filename) : "winston.log";
+          this.dirname = options2.dirname || path20.dirname(options2.filename);
           this.options = options2.options || { flags: "a" };
         } else if (options2.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options2.stream));
-          this.dirname = path21.dirname(this._dest.path);
+          this.dirname = path20.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -8741,11 +8834,11 @@ var require_file = __commonJS({
           options2 = {};
         }
         options2 = normalizeQuery(options2);
-        const file = path21.join(this.dirname, this.filename);
+        const file = path20.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs27.createReadStream(file, {
+        const stream = fs28.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err2) => {
@@ -8846,7 +8939,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options2 = {}) {
-        const file = path21.join(this.dirname, this.filename);
+        const file = path20.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -8896,8 +8989,8 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path21.join(this.dirname, target);
-        fs27.stat(fullpath, (err2, stat) => {
+        const fullpath = path20.join(this.dirname, target);
+        fs28.stat(fullpath, (err2, stat) => {
           if (err2 && err2.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -9000,9 +9093,9 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path21.join(this.dirname, this.filename);
+        const fullpath = path20.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs27.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs28.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -9025,16 +9118,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug("_incFile", this.filename);
-        const ext = path21.extname(this._basename);
-        const basename = path21.basename(this._basename, ext);
+        const ext = path20.extname(this._basename);
+        const basename = path20.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path21.join(this.dirname, `${basename}${num}${ext}`),
-                path21.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path20.join(this.dirname, `${basename}${num}${ext}`),
+                path20.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -9059,8 +9152,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path21.extname(this._basename);
-        const basename = path21.basename(this._basename, ext);
+        const ext = path20.extname(this._basename);
+        const basename = path20.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -9080,8 +9173,8 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path21.join(this.dirname, filePath);
-        fs27.unlink(target, callback);
+        const target = path20.join(this.dirname, filePath);
+        fs28.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -9103,20 +9196,20 @@ var require_file = __commonJS({
         for (let x2 = this.maxFiles - 1; x2 > 1; x2--) {
           tasks.push(function(i6, cb) {
             let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
-            const tmppath = path21.join(this.dirname, fileName);
-            fs27.exists(tmppath, (exists) => {
+            const tmppath = path20.join(this.dirname, fileName);
+            fs28.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i6}${ext}${isZipped}`;
-              fs27.rename(tmppath, path21.join(this.dirname, fileName), cb);
+              fs28.rename(tmppath, path20.join(this.dirname, fileName), cb);
             });
           }.bind(this, x2));
         }
         asyncSeries(tasks, () => {
-          fs27.rename(
-            path21.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path21.join(this.dirname, `${basename}1${ext}${isZipped}`),
+          fs28.rename(
+            path20.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path20.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -9130,22 +9223,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs27.access(src, fs27.F_OK, (err2) => {
+        fs28.access(src, fs28.F_OK, (err2) => {
           if (err2) {
             return callback();
           }
           var gzip = zlib3.createGzip();
-          var inp = fs27.createReadStream(src);
-          var out = fs27.createWriteStream(dest);
+          var inp = fs28.createReadStream(src);
+          var out = fs28.createWriteStream(dest);
           out.on("finish", () => {
-            fs27.unlink(src, callback);
+            fs28.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs27.existsSync(dirPath)) {
-          fs27.mkdirSync(dirPath, { recursive: true });
+        if (!fs28.existsSync(dirPath)) {
+          fs28.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -9229,9 +9322,9 @@ var require_http = __commonJS({
         };
         const auth = options2.params.auth || null;
         delete options2.params.auth;
-        const path21 = options2.params.path || null;
+        const path20 = options2.params.path || null;
         delete options2.params.path;
-        this._request(options2, auth, path21, (err2, res, body) => {
+        this._request(options2, auth, path20, (err2, res, body) => {
           if (res && res.statusCode !== 200) {
             err2 = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -9259,12 +9352,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options2
         };
-        const path21 = options2.params.path || null;
+        const path20 = options2.params.path || null;
         delete options2.params.path;
         const auth = options2.params.auth || null;
         delete options2.params.auth;
         let buff = "";
-        const req = this._request(options2, auth, path21);
+        const req = this._request(options2, auth, path20);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -9290,14 +9383,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options2, auth, path21, callback) {
+      _request(options2, auth, path20, callback) {
         options2 = options2 || {};
         auth = auth || this.auth;
-        path21 = path21 || this.path || "";
+        path20 = path20 || this.path || "";
         if (this.batch) {
-          this._doBatch(options2, callback, auth, path21);
+          this._doBatch(options2, callback, auth, path20);
         } else {
-          this._doRequest(options2, callback, auth, path21);
+          this._doRequest(options2, callback, auth, path20);
         }
       }
       /**
@@ -9307,18 +9400,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options2, callback, auth, path21) {
+      _doBatch(options2, callback, auth, path20) {
         this.batchOptions.push(options2);
         if (this.batchOptions.length === 1) {
           const me3 = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me3.batchTimeoutID = -1;
-            me3._doBatchRequest(me3.batchCallback, auth, path21);
+            me3._doBatchRequest(me3.batchCallback, auth, path20);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path21);
+          this._doBatchRequest(this.batchCallback, auth, path20);
         }
       }
       /**
@@ -9327,14 +9420,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path21) {
+      _doBatchRequest(callback, auth, path20) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path21);
+        this._doRequest(batchOptionsCopy, callback, auth, path20);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -9344,7 +9437,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options2, callback, auth, path21) {
+      _doRequest(options2, callback, auth, path20) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -9354,7 +9447,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path21.replace(/^\//, "")}`,
+          path: `/${path20.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -10272,7 +10365,7 @@ var require_profiler = __commonJS({
 var require_logger = __commonJS({
   "../../node_modules/winston/lib/winston/logger.js"(exports2, module2) {
     "use strict";
-    var { Stream, Transform: Transform4 } = require_readable();
+    var { Stream, Transform: Transform5 } = require_readable();
     var asyncForEach = require_forEach();
     var { LEVEL, SPLAT } = require_triple_beam();
     var isStream = require_is_stream();
@@ -10283,7 +10376,7 @@ var require_logger = __commonJS({
     var { warn } = require_common();
     var config = require_config2();
     var formatRegExp = /%[scdjifoO%]/g;
-    var Logger2 = class extends Transform4 {
+    var Logger2 = class extends Transform5 {
       /**
        * Constructor function for the Logger object responsible for persisting log
        * messages and metadata to one or more transports.
@@ -11682,7 +11775,7 @@ var CONSTANTS, Constants_default;
 var init_Constants = __esm({
   "../../packages/constants/src/Constants.ts"() {
     CONSTANTS = {
-      cliVersion: "2.1.1",
+      cliVersion: "2.2.0",
       backendVersion: 1,
       backendPort: 5088,
       workerVersion: 2,
@@ -11699,6 +11792,23 @@ var init_Constants = __esm({
        * for the whole project, and once per dimension when a worker compiles its line plan.
        */
       MASK_VALUES: ["hash", "mask", "crypt", "random", "seeded-random", "none"],
+      /**
+       * Where each kind of resource lives inside the `remora/` directory, as a path relative to it.
+       *
+       * Fixed, not configurable: project.json used to name these folders, which meant nothing outside
+       * the loader could find a resource from its path alone, and two projects could disagree about
+       * what a remora project looks like. `sources`, `producers` and `consumers` must exist; `schemas`
+       * and `codeSets` are read only when the folder is there.
+       */
+      RESOURCE_FOLDERS: {
+        sources: "/sources",
+        producers: "/producers",
+        consumers: "/consumers",
+        schemas: "/schemas",
+        codeSets: "/code_sets"
+      },
+      /** project.json keys that used to name the resource folders, kept only to warn on and ignore. */
+      LEGACY_PROJECT_PATH_KEYS: ["sources", "producers", "consumers", "schemas", "codeSets"],
       defaults: {
         REMORA_PATH: "./remora",
         PRODUCER_TEMP_FOLDER: ".temp",
@@ -11777,6 +11887,19 @@ var init_Constants = __esm({
          * than memory — a zip bomb stops with a named error instead of filling the scope folder.
          */
         MAX_ARCHIVE_ENTRIES: 1e4,
+        /**
+         * Files one consumer's output may be split into. Generous for a country, a region or a year,
+         * and low enough to catch "split by customer_id" before it exhausts the process's descriptor
+         * table. The run fails mid-pass with a named error rather than producing a directory nobody
+         * asked for.
+         */
+        MAX_SPLIT_FILES: 1e3,
+        /**
+         * Write streams the split pass may hold open at once, independent of how many files it ends
+         * up writing. Past this the least recently written bucket is closed and reopened in append
+         * mode when its next line arrives — its byte counter lives in the bucket, not the stream.
+         */
+        MAX_OPEN_SPLIT_STREAMS: 64,
         /**
          * Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
          * export of it — `remora sample -s` and the preview's Data tab are both held to this.
@@ -12804,6 +12927,7 @@ var init_Validator = __esm({
               }
             }
             errors.push(...this.validateOmitNull(consumer));
+            errors.push(...this.validateSplit(consumer));
             const dimensionFields = consumer.fields.filter((x2) => x2.key !== "*" && !x2.fixed && !x2.copyFrom && !x2.lineCount);
             const dimensionKeys = dimensionFields.map((x2) => `${x2.from ?? "_default_"}::${x2.key}`);
             const duplicateDimensionKeys = dimensionKeys.filter((k7, i6) => dimensionKeys.indexOf(k7) !== i6);
@@ -12860,6 +12984,42 @@ var init_Validator = __esm({
           }
           return errors;
         };
+        /**
+         * Checks `split`, which breaks a consumer's output into several files — by the value of one or
+         * more fields, by a size cap, or both.
+         *
+         * The alias-resolution and wildcard rules follow `distinctOn`: the keys name consumer fields as
+         * the output writes them, and a consumer selecting `*` has no field list to check them against
+         * until it reads its producers. A field it declares beside the `*` is still checked, since a
+         * hidden one is hidden whatever the wildcard expands to.
+         */
+        this.validateSplit = (consumer) => {
+          const errors = [];
+          const { split: split2 } = consumer;
+          const positionalFormats = ["CSV", "TXT", "PARQUET"];
+          if (!Algo_default.hasVal(split2)) return errors;
+          if (Algo_default.hasVal(consumer.maximumFileSize))
+            errors.push(`Consumer "${consumer.name}" sets both "maximumFileSize" and "split". They are the same cap: move the value to "split": { "maxFileSizeGb": ${consumer.maximumFileSize} } and remove "maximumFileSize".`);
+          const keys = split2.by ?? [];
+          if (keys.length === 0 && !Algo_default.hasVal(split2.maxFileSizeGb))
+            errors.push(`The "split" of consumer "${consumer.name}" sets neither "by" nor "maxFileSizeGb", so it would split nothing. Remove it, or say how the output should be split.`);
+          if (Algo_default.hasVal(split2.maxFileSizeGb) && !(split2.maxFileSizeGb > 0))
+            errors.push(`The "split.maxFileSizeGb" of consumer "${consumer.name}" must be greater than zero (got ${split2.maxFileSizeGb}).`);
+          const apiOutputs = (consumer.outputs ?? []).filter((x2) => x2.format.toUpperCase() === "API");
+          if (apiOutputs.length > 0)
+            errors.push(`Consumer "${consumer.name}" declares "split", but an API output is a response, not a file, and there is nothing to split it into.`);
+          if (keys.length === 0) return errors;
+          const fieldKeys = (consumer.fields ?? []).map((x2) => x2.alias ?? x2.key);
+          const missingKeys = fieldKeys.includes("*") ? [] : keys.filter((key) => !fieldKeys.includes(key));
+          if (missingKeys.length > 0)
+            errors.push(`"split.by" references key(s) "${missingKeys.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+          const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
+          if (positional.length === 0) return errors;
+          const hiddenKeys = keys.filter((key) => (consumer.fields ?? []).some((x2) => (x2.alias ?? x2.key) === key && x2.hidden));
+          if (hiddenKeys.length > 0)
+            errors.push(`"split.by" of consumer "${consumer.name}" references hidden field(s) "${hiddenKeys.join(", ")}", which the ${positional.map((x2) => x2.format.toUpperCase()).join("/")} output does not write \u2014 a hidden field can't be split on in a positional format.`);
+          return errors;
+        };
         /**
          * Checks `omitNull`, which drops a record's null fields on export. It only means anything where a
          * record carries its own field names, and it is applied when the record is serialized — one record
@@ -12956,7 +13116,7 @@ var init_Validator = __esm({
          */
         this.detectOnFinishRecursion = (consumer) => {
           const errors = [];
-          const path21 = [];
+          const path20 = [];
           const targetsOf = (cons) => {
             const names = [];
             for (const output of cons.outputs ?? []) {
@@ -12968,17 +13128,17 @@ var init_Validator = __esm({
             return names;
           };
           const walk = (name) => {
-            if (path21.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
-              errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path21, name].join(" -> ")}`);
+            if (path20.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
+              errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path20, name].join(" -> ")}`);
               return true;
             }
             const cons = Environment_default.getConsumer(name);
             if (!cons) return false;
-            path21.push(name);
+            path20.push(name);
             for (const target of targetsOf(cons)) {
               if (walk(target)) return true;
             }
-            path21.pop();
+            path20.pop();
             return false;
           };
           walk(consumer.name);
@@ -13563,7 +13723,7 @@ ${errors.map((x2) => `  - ${x2}`).join("\n")}`);
 });
 
 // ../../packages/common/src/Environment.ts
-var import_fs4, import_crypto, import_adm_zip, import_path4, import_client_s3, RESOURCE_SCHEMA_KEYS, INVALID_RESOURCE_MESSAGES, EnvironmentClass, Environment, Environment_default;
+var import_fs4, import_crypto, import_adm_zip, import_path4, import_client_s3, RESOURCE_SCHEMA_KEYS, INVALID_RESOURCE_MESSAGES, STAGING_DIRECTORY, EnvironmentClass, Environment, Environment_default;
 var init_Environment = __esm({
   "../../packages/common/src/Environment.ts"() {
     import_fs4 = __toESM(require("fs"), 1);
@@ -13592,6 +13752,7 @@ var init_Environment = __esm({
       codeSet: "Invalid code set configuration",
       schema: "Invalid schema configuration"
     };
+    STAGING_DIRECTORY = ".remora-incoming";
     EnvironmentClass = class {
       constructor() {
         this._env = null;
@@ -13671,31 +13832,72 @@ var init_Environment = __esm({
             Bucket: bucketName,
             Key: key
           });
+          let buffer;
           try {
             const response = await s3Client.send(command5);
-            return this._saveConfigFile(Buffer.from(await response.Body.transformToByteArray()), url);
+            buffer = Buffer.from(await response.Body.transformToByteArray());
           } catch (error) {
-            throw new Error(`Error at bucket ${bucketName}, while obtainign ${key}, caused ${error}`);
+            throw new Error(`Error at bucket ${bucketName}, while obtaining ${key}, caused ${error}`);
           }
+          return this._saveConfigFile(buffer, url);
         };
-        // save the configuration files on the local remora path
+        /**
+         * Extract a downloaded configuration archive over the local remora path.
+         *
+         * The archive is fully read and extracted to a staging directory BEFORE anything live is
+         * touched, because the destructive step is not reversible: a configuration that fails to
+         * extract half way through would otherwise leave the process running against an empty
+         * project directory until the next successful download.
+         */
         this._saveConfigFile = (buffer, url) => {
-          const zip = new import_adm_zip.default(buffer);
+          Affirm_default(buffer?.length, `Empty configuration archive downloaded from ${url}`);
+          const zip = this._readArchive(buffer, url);
           const extractPath = import_path4.default.join(this._localPath, "remora");
-          if (import_fs4.default.existsSync(extractPath)) {
-            for (const file of import_fs4.default.readdirSync(extractPath)) {
-              if (file === "temp") continue;
-              const filePath = import_path4.default.join(extractPath, file);
-              if (import_fs4.default.statSync(filePath).isDirectory())
-                import_fs4.default.rmSync(filePath, { recursive: true, force: true });
-              else
-                import_fs4.default.unlinkSync(filePath);
-            }
+          const stagingPath = import_path4.default.join(this._localPath, STAGING_DIRECTORY);
+          import_fs4.default.rmSync(stagingPath, { recursive: true, force: true });
+          try {
+            zip.extractAllTo(stagingPath, true);
+            this._clearConfigDirectory(extractPath);
+            for (const entry of import_fs4.default.readdirSync(stagingPath))
+              import_fs4.default.renameSync(import_path4.default.join(stagingPath, entry), import_path4.default.join(extractPath, entry));
+          } finally {
+            import_fs4.default.rmSync(stagingPath, { recursive: true, force: true });
           }
-          zip.extractAllTo(extractPath, true);
           Logger_default.log(`Remote configuration downloaded and extracted from ${url}`);
           return true;
         };
+        /**
+         * Read the archive's entry table up front. adm-zip parses entries lazily, so without this the
+         * first read of a malformed archive happens inside `extractAllTo` — after the live
+         * configuration has already been removed.
+         */
+        this._readArchive = (buffer, url) => {
+          try {
+            const zip = new import_adm_zip.default(buffer);
+            Affirm_default.hasItems(zip.getEntries(), `Configuration archive from ${url} contains no files`);
+            return zip;
+          } catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            throw new Error(
+              `Unreadable configuration archive from ${url}: ${reason}. Rebuild the archive from the project directory (\`remora package\`) instead of adding files to the existing one \u2014 appending leaves two entries under the same name, which is rejected because the two could hold different content.`
+            );
+          }
+        };
+        /** Remove everything the previous configuration left behind, keeping the run scratch directory. */
+        this._clearConfigDirectory = (extractPath) => {
+          if (!import_fs4.default.existsSync(extractPath)) {
+            import_fs4.default.mkdirSync(extractPath, { recursive: true });
+            return;
+          }
+          for (const file of import_fs4.default.readdirSync(extractPath)) {
+            if (file === "temp") continue;
+            const filePath = import_path4.default.join(extractPath, file);
+            if (import_fs4.default.statSync(filePath).isDirectory())
+              import_fs4.default.rmSync(filePath, { recursive: true, force: true });
+            else
+              import_fs4.default.unlinkSync(filePath);
+          }
+        };
         /**
          * Check if the configuration has changed and reload if so.
          * For remote URIs, re-downloads using ETag caching (HTTP 304 avoids redundant downloads).
@@ -13758,25 +13960,26 @@ var init_Environment = __esm({
           if (!projectConfig) return this._emptyInspection(problems);
           SchemaValidator_default.unloadSchemas();
           const read = (entries) => this._readConfigs(envPath, entries, remoraPath, problems);
-          const schemaPaths = (projectConfig.schemas ?? []).filter((x2) => import_fs4.default.existsSync(import_path4.default.join(envPath, x2)));
-          const schemaFiles = read(schemaPaths);
+          const optional = (folder) => import_fs4.default.existsSync(import_path4.default.join(envPath, folder)) ? [folder] : [];
+          const folders = Constants_default.RESOURCE_FOLDERS;
+          const schemaFiles = read(optional(folders.schemas));
           const schemas = schemaFiles.map((x2) => x2.config);
           schemas.forEach((schema) => this._registerSchema(schema, problems));
-          const sourceFiles = read(projectConfig.sources);
+          const sourceFiles = read([folders.sources]);
           const sources = sourceFiles.map((x2) => x2.config);
           sourceFiles.forEach((source) => this._recordSchemaProblem(source, "source", problems));
           this._checkApiQueueSource(projectConfig, sources, problems);
-          const producerFiles = read(projectConfig.producers);
+          const producerFiles = read([folders.producers]);
           const producers = producerFiles.map((x2) => x2.config);
           producerFiles.forEach(({ config: producer, file }) => {
             const schemaProblem = this._schemaProblem(producer, "producer", file);
             this._expandProducer(producer, problems);
             if (schemaProblem) problems.push(schemaProblem);
           });
-          const consumerFiles = read(projectConfig.consumers);
+          const consumerFiles = read([folders.consumers]);
           const consumers = consumerFiles.map((x2) => x2.config);
           consumerFiles.forEach((consumer) => this._recordSchemaProblem(consumer, "consumer", problems));
-          const codeSetFiles = read(projectConfig.codeSets ?? []);
+          const codeSetFiles = read(optional(folders.codeSets));
           const codeSets = codeSetFiles.map((x2) => x2.config);
           codeSetFiles.forEach((codeSet) => this._recordSchemaProblem(codeSet, "codeSet", problems));
           this._checkCodeSetNames(codeSets, problems);
@@ -13826,6 +14029,7 @@ var init_Environment = __esm({
           const { env: env2, projectName, projectSettings, problems } = this.inspect(remoraPath);
           const fatal = problems.find((x2) => x2.severity === "error");
           if (fatal) throw new Error(fatal.message);
+          problems.filter((x2) => x2.severity === "warning").forEach((x2) => Logger_default.warn(x2.message));
           this._projectName = projectName;
           this._projectSettings = projectSettings;
           this.init(env2);
@@ -13861,8 +14065,27 @@ var init_Environment = __esm({
               message: `Invalid project configuration - ${SchemaErrorReporter_default.describe(validation.errors, projectConfig)}`
             });
           }
+          this._checkLegacyPaths(projectConfig, projectPath, problems);
           return projectConfig;
         };
+        /**
+         * project.json used to name the folder each kind of resource was read from. It no longer does —
+         * the folders are fixed — so the old keys are reported and ignored.
+         *
+         * A WARNING, not an error: a project written against an older CLI still runs, and the author is
+         * told once, per load, what to delete. This is the one warning the loader raises.
+         */
+        this._checkLegacyPaths = (projectConfig, projectPath, problems) => {
+          const stale = Constants_default.LEGACY_PROJECT_PATH_KEYS.filter((key) => key in (projectConfig ?? {}));
+          if (stale.length === 0) return;
+          const folders = Object.values(Constants_default.RESOURCE_FOLDERS).join(", ");
+          problems.push({
+            severity: "warning",
+            scope: "project",
+            file: projectPath,
+            message: `Project configuration: ${stale.map((x2) => `"${x2}"`).join(", ")} no longer decide where resources live and can be removed - the folders are fixed (${folders})`
+          });
+        };
         /**
          * Reads every JSON file the entries point at.
          *
@@ -15028,14 +15251,14 @@ var init_readFile = __esm({
     import_promises3 = require("fs/promises");
     filePromises = {};
     fileIntercept = {};
-    readFile2 = (path21, options2) => {
-      if (fileIntercept[path21] !== void 0) {
-        return fileIntercept[path21];
+    readFile2 = (path20, options2) => {
+      if (fileIntercept[path20] !== void 0) {
+        return fileIntercept[path20];
       }
-      if (!filePromises[path21] || options2?.ignoreCache) {
-        filePromises[path21] = (0, import_promises3.readFile)(path21, "utf8");
+      if (!filePromises[path20] || options2?.ignoreCache) {
+        filePromises[path20] = (0, import_promises3.readFile)(path20, "utf8");
       }
-      return filePromises[path21];
+      return filePromises[path20];
     };
   }
 });
@@ -15147,8 +15370,8 @@ var init_externalDataInterceptor = __esm({
       getFileRecord() {
         return fileIntercept;
       },
-      interceptFile(path21, contents) {
-        fileIntercept[path21] = Promise.resolve(contents);
+      interceptFile(path20, contents) {
+        fileIntercept[path20] = Promise.resolve(contents);
       },
       getTokenRecord() {
         return tokenIntercept;
@@ -18228,8 +18451,8 @@ var init_createConfigValueProvider = __esm({
               return endpoint.url.href;
             }
             if ("hostname" in endpoint) {
-              const { protocol, hostname, port, path: path21 } = endpoint;
-              return `${protocol}//${hostname}${port ? ":" + port : ""}${path21}`;
+              const { protocol, hostname, port, path: path20 } = endpoint;
+              return `${protocol}//${hostname}${port ? ":" + port : ""}${path20}`;
             }
           }
           return endpoint;
@@ -18604,18 +18827,18 @@ var getAttrPathList;
 var init_getAttrPathList = __esm({
   "../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
     init_types2();
-    getAttrPathList = (path21) => {
-      const parts = path21.split(".");
+    getAttrPathList = (path20) => {
+      const parts = path20.split(".");
       const pathList = [];
       for (const part of parts) {
         const squareBracketIndex = part.indexOf("[");
         if (squareBracketIndex !== -1) {
           if (part.indexOf("]") !== part.length - 1) {
-            throw new EndpointError(`Path: '${path21}' does not end with ']'`);
+            throw new EndpointError(`Path: '${path20}' does not end with ']'`);
           }
           const arrayIndex = part.slice(squareBracketIndex + 1, -1);
           if (Number.isNaN(parseInt(arrayIndex))) {
-            throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path21}'`);
+            throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path20}'`);
           }
           if (squareBracketIndex !== 0) {
             pathList.push(part.slice(0, squareBracketIndex));
@@ -18636,9 +18859,9 @@ var init_getAttr = __esm({
   "../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttr.js"() {
     init_types2();
     init_getAttrPathList();
-    getAttr = (value, path21) => getAttrPathList(path21).reduce((acc, index) => {
+    getAttr = (value, path20) => getAttrPathList(path20).reduce((acc, index) => {
       if (typeof acc !== "object") {
-        throw new EndpointError(`Index '${index}' in '${path21}' not found in '${JSON.stringify(value)}'`);
+        throw new EndpointError(`Index '${index}' in '${path20}' not found in '${JSON.stringify(value)}'`);
       } else if (Array.isArray(acc)) {
         const i6 = parseInt(index);
         return acc[i6 < 0 ? acc.length + i6 : i6];
@@ -18699,8 +18922,8 @@ var init_parseURL = __esm({
             return value;
           }
           if (typeof value === "object" && "hostname" in value) {
-            const { hostname: hostname2, port, protocol: protocol2 = "", path: path21 = "", query = {} } = value;
-            const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path21}`);
+            const { hostname: hostname2, port, protocol: protocol2 = "", path: path20 = "", query = {} } = value;
+            const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path20}`);
             url.search = Object.entries(query).map(([k7, v3]) => `${k7}=${v3}`).join("&");
             return url;
           }
@@ -21044,11 +21267,11 @@ var init_HttpBindingProtocol = __esm({
           const opTraits = translateTraits(operationSchema.traits);
           if (opTraits.http) {
             request.method = opTraits.http[0];
-            const [path21, search] = opTraits.http[1].split("?");
+            const [path20, search] = opTraits.http[1].split("?");
             if (request.path == "/") {
-              request.path = path21;
+              request.path = path20;
             } else {
-              request.path += path21;
+              request.path += path20;
             }
             const traitSearchParams = new URLSearchParams(search ?? "");
             for (const [key, value] of traitSearchParams) {
@@ -24761,12 +24984,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
             const password = request.password ?? "";
             auth = `${username}:${password}`;
           }
-          let path21 = request.path;
+          let path20 = request.path;
           if (queryString) {
-            path21 += `?${queryString}`;
+            path20 += `?${queryString}`;
           }
           if (request.fragment) {
-            path21 += `#${request.fragment}`;
+            path20 += `#${request.fragment}`;
           }
           let hostname = request.hostname ?? "";
           if (hostname[0] === "[" && hostname.endsWith("]")) {
@@ -24778,7 +25001,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
             headers: request.headers,
             host: hostname,
             method: request.method,
-            path: path21,
+            path: path20,
             port: request.port,
             agent,
             auth
@@ -25687,10 +25910,10 @@ ${longDate}
 ${credentialScope}
 ${toHex(hashedRequest)}`;
       }
-      getCanonicalPath({ path: path21 }) {
+      getCanonicalPath({ path: path20 }) {
         if (this.uriEscapePath) {
           const normalizedPathSegments = [];
-          for (const pathSegment of path21.split("/")) {
+          for (const pathSegment of path20.split("/")) {
             if (pathSegment?.length === 0)
               continue;
             if (pathSegment === ".")
@@ -25701,11 +25924,11 @@ ${toHex(hashedRequest)}`;
               normalizedPathSegments.push(pathSegment);
             }
           }
-          const normalizedPath = `${path21?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path21?.endsWith("/") ? "/" : ""}`;
+          const normalizedPath = `${path20?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path20?.endsWith("/") ? "/" : ""}`;
           const doubleEncoded = escapeUri(normalizedPath);
           return doubleEncoded.replace(/%2F/g, "/");
         }
-        return path21;
+        return path20;
       }
       validateResolvedCredentials(credentials) {
         if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
@@ -33416,10 +33639,10 @@ var require_pend = __commonJS({
 // ../../node_modules/yauzl/fd-slicer.js
 var require_fd_slicer = __commonJS({
   "../../node_modules/yauzl/fd-slicer.js"(exports2) {
-    var fs27 = require("fs");
+    var fs28 = require("fs");
     var util = require("util");
     var stream = require("stream");
-    var Readable8 = stream.Readable;
+    var Readable10 = stream.Readable;
     var PassThrough = stream.PassThrough;
     var Pend = require_pend();
     var EventEmitter = require("events").EventEmitter;
@@ -33436,7 +33659,7 @@ var require_fd_slicer = __commonJS({
     FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
       var self2 = this;
       self2.pend.go(function(cb) {
-        fs27.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
+        fs28.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
           cb();
           callback(err2, bytesRead, buffer2);
         });
@@ -33453,7 +33676,7 @@ var require_fd_slicer = __commonJS({
       self2.refCount -= 1;
       if (self2.refCount < 0) throw new Error("invalid unref");
       if (self2.refCount > 0) return;
-      fs27.close(self2.fd, onCloseDone);
+      fs28.close(self2.fd, onCloseDone);
       function onCloseDone(err2) {
         if (err2) {
           self2.emit("error", err2);
@@ -33462,10 +33685,10 @@ var require_fd_slicer = __commonJS({
         }
       }
     };
-    util.inherits(ReadStream2, Readable8);
+    util.inherits(ReadStream2, Readable10);
     function ReadStream2(context, options2) {
       options2 = options2 || {};
-      Readable8.call(this, options2);
+      Readable10.call(this, options2);
       this.context = context;
       this.context.ref();
       this.start = options2.start || 0;
@@ -33485,7 +33708,7 @@ var require_fd_slicer = __commonJS({
       }
       self2.context.pend.go(function(cb) {
         var buffer = Buffer.allocUnsafe(toRead);
-        fs27.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
+        fs28.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
           if (err2) {
             self2.destroy(err2);
           } else if (bytesRead === 0) {
@@ -33845,12 +34068,12 @@ var require_crc32 = __commonJS({
 // ../../node_modules/yauzl/index.js
 var require_yauzl = __commonJS({
   "../../node_modules/yauzl/index.js"(exports2) {
-    var fs27 = require("fs");
+    var fs28 = require("fs");
     var zlib3 = require("zlib");
     var fd_slicer = require_fd_slicer();
     var util = require("util");
     var EventEmitter = require("events").EventEmitter;
-    var Transform4 = require("stream").Transform;
+    var Transform5 = require("stream").Transform;
     var PassThrough = require("stream").PassThrough;
     var Writable2 = require("stream").Writable;
     var crc322 = typeof zlib3.crc32 === "function" ? zlib3.crc32 : require_crc32();
@@ -33870,9 +34093,9 @@ var require_yauzl = __commonJS({
     exports2.Entry = Entry;
     exports2.LocalFileHeader = LocalFileHeader;
     exports2.RandomAccessReader = RandomAccessReader;
-    function openPromise(path21, options2) {
+    function openPromise(path20, options2) {
       return new Promise((resolve, reject) => {
-        open(path21, { ...options2, lazyEntries: true }, function(err2, zipfile) {
+        open(path20, { ...options2, lazyEntries: true }, function(err2, zipfile) {
           if (err2) return reject(err2);
           resolve(zipfile);
         });
@@ -33902,7 +34125,7 @@ var require_yauzl = __commonJS({
         });
       });
     }
-    function open(path21, options2, callback) {
+    function open(path20, options2, callback) {
       if (typeof options2 === "function") {
         callback = options2;
         options2 = null;
@@ -33914,10 +34137,10 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs27.open(path21, "r", function(err2, fd) {
+      fs28.open(path20, "r", function(err2, fd) {
         if (err2) return callback(err2);
         fromFd(fd, options2, function(err3, zipfile) {
-          if (err3) fs27.close(fd, defaultCallback);
+          if (err3) fs28.close(fd, defaultCallback);
           callback(err3, zipfile);
         });
       });
@@ -33934,7 +34157,7 @@ var require_yauzl = __commonJS({
       if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
       if (options2.strictFileNames == null) options2.strictFileNames = false;
       if (callback == null) callback = defaultCallback;
-      fs27.fstat(fd, function(err2, stats) {
+      fs28.fstat(fd, function(err2, stats) {
         if (err2) return callback(err2);
         var reader = new fd_slicer.FdSlicer(fd);
         fromRandomAccessReader(reader, stats.size, options2, callback);
@@ -34544,9 +34767,9 @@ var require_yauzl = __commonJS({
         callback();
       });
     }
-    util.inherits(AssertByteCountStream, Transform4);
+    util.inherits(AssertByteCountStream, Transform5);
     function AssertByteCountStream(byteCount) {
-      Transform4.call(this);
+      Transform5.call(this);
       this.actualByteCount = 0;
       this.expectedByteCount = byteCount;
     }
@@ -38161,8 +38384,8 @@ var init_HttpClient = __esm({
             throw new Error(err2.message ?? "An error occurred on GET.");
           }
         };
-        this.getWorkerUrl = (path21) => {
-          const cleanPath = path21.startsWith("/") ? path21.substring(1) : path21;
+        this.getWorkerUrl = (path20) => {
+          const cleanPath = path20.startsWith("/") ? path20.substring(1) : path20;
           return `${this.workerHost}/${cleanPath}`;
         };
         this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -44053,7 +44276,7 @@ var init_DeltaShareDriver = __esm({
          * disagree on their columns still produces a rectangular dataset.
          */
         this._writeDatasetFile = async (localPath, batches, delimiter) => {
-          const stream = fs9.createWriteStream(localPath);
+          const writer = new BatchWriter_default(fs9.createWriteStream(localPath));
           let header = null;
           let records = 0;
           try {
@@ -44061,25 +44284,19 @@ var init_DeltaShareDriver = __esm({
               for (const record of batch) {
                 if (!header) {
                   header = Object.keys(record);
-                  await this._writeLine(stream, CSVParser_default.stringifyRow(header, delimiter));
+                  if (!writer.write(CSVParser_default.stringifyRow(header, delimiter)))
+                    await writer.waitForDrain();
                 }
-                await this._writeLine(stream, CSVParser_default.stringifyRow(header.map((column) => this._serializeDatasetValue(record[column])), delimiter));
+                if (!writer.write(CSVParser_default.stringifyRow(header.map((column) => this._serializeDatasetValue(record[column])), delimiter)))
+                  await writer.waitForDrain();
                 records++;
               }
             }
           } finally {
-            await new Promise((resolve) => stream.end(resolve));
+            await writer.close();
           }
           return records;
         };
-        // A false from write() means the buffer is full; waiting for the drain just then keeps the queued
-        // rows bounded instead of letting them pile up faster than the disk takes them.
-        this._writeLine = async (stream, line) => {
-          if (stream.write(`${line}
-`))
-            return;
-          await new Promise((resolve) => stream.once("drain", resolve));
-        };
         /**
          * Every file the share reports for this table. `jsonPredicateHints` lets the server skip the
          * files that cannot hold a matching row and `limitHint` the ones past the rows the caller wants;
@@ -44258,18 +44475,20 @@ var init_SqlDatasetWriter = __esm({
           Affirm_default(localPath, "Invalid dataset path");
           Affirm_default.hasItems(columns, "Invalid dataset columns");
           Affirm_default(batches, "Invalid dataset batches");
-          const stream = import_fs9.default.createWriteStream(localPath);
+          const writer = new BatchWriter_default(import_fs9.default.createWriteStream(localPath));
           let rows = 0;
           try {
-            await this.writeLine(stream, CSVParser_default.stringifyRow(columns, delimiter));
+            if (!writer.write(CSVParser_default.stringifyRow(columns, delimiter)))
+              await writer.waitForDrain();
             for await (const batch of batches) {
               for (const row of batch) {
-                await this.writeLine(stream, CSVParser_default.stringifyRow(row.map((value) => this.serialize(value)), delimiter));
+                if (!writer.write(CSVParser_default.stringifyRow(row.map((value) => this.serialize(value)), delimiter)))
+                  await writer.waitForDrain();
                 rows++;
               }
             }
           } finally {
-            await new Promise((resolve) => stream.end(resolve));
+            await writer.close();
           }
           return rows;
         };
@@ -44291,14 +44510,6 @@ var init_SqlDatasetWriter = __esm({
             return value.toString("base64");
           return JSON.stringify(value);
         };
-        // A false from write() means the buffer is full; waiting for the drain just then keeps the queued
-        // rows bounded instead of letting them pile up faster than the disk takes them.
-        this.writeLine = async (stream, line) => {
-          if (stream.write(`${line}
-`))
-            return;
-          await new Promise((resolve) => stream.once("drain", resolve));
-        };
       }
     };
     SqlDatasetWriter = new SqlDatasetWriterClass();
@@ -44319,8 +44530,13 @@ var init_SqlQueryBuilder = __esm({
          *
          * Columns are projected in the producer's own order and the caller writes the header from
          * `columnKeys`, so the staged CSV never depends on how a source echoes column names back.
+         *
+         * `options` is what the run asked the source to do for it — a `WHERE` rendered by `SqlPredicate`
+         * and a row limit, both resolved by `SqlPushdown`. Neither changes what the run produces: the
+         * filters are applied per record and the limit enforced regardless, so a statement built without
+         * them reads more rows and returns the same result.
          */
-        this.buildProducerSelect = (producer, source, limit) => {
+        this.buildProducerSelect = (producer, source, options2) => {
           Affirm_default(producer, "Invalid producer");
           Affirm_default(source, "Invalid source");
           const { schema, table } = this.resolveTable(producer, source);
@@ -44328,8 +44544,30 @@ var init_SqlQueryBuilder = __esm({
           Affirm_default.hasItems(columns, `Producer "${producer.name}" declares no dimensions, so there is nothing to select from "${table}"`);
           const projection = columns.map((column) => this.quote(column, `column of producer "${producer.name}"`)).join(", ");
           const from = schema ? `${this.quote(schema, `schema of producer "${producer.name}"`)}.${this.quote(table, `table of producer "${producer.name}"`)}` : this.quote(table, `table of producer "${producer.name}"`);
-          const select = `SELECT ${projection} FROM ${from}`;
-          return limit > 0 ? `${select} LIMIT ${Math.floor(limit)}` : select;
+          const where = options2?.where ? ` WHERE ${options2.where}` : "";
+          const limit = options2?.limit > 0 ? ` LIMIT ${Math.floor(options2.limit)}` : "";
+          return `SELECT ${projection} FROM ${from}${where}${limit}`;
+        };
+        /**
+         * The statement that asks a SQL source what columns a producer's table actually HAS.
+         *
+         * This is what `discover` reads, and it deliberately does not go through `buildProducerSelect`:
+         * that statement is built FROM the producer's dimensions, so sampling it can only ever hand back
+         * the columns the producer already declared — a producer naming one column of a fourteen-column
+         * table discovered exactly that one and was told it had succeeded. The catalog is the only source
+         * of the answer, so the catalog is what is read.
+         *
+         * `information_schema.columns` rather than a dialect catalog (`svv_columns`, `pg_attribute`)
+         * because Redshift and Postgres both expose it with the same column names, which is the same
+         * reason the rest of this builder is dialect-free. One known limit: a Redshift external
+         * (Spectrum) or late-binding-view column is not in `information_schema` and so is not discovered.
+         */
+        this.buildColumnSelect = (producer, source) => {
+          Affirm_default(producer, "Invalid producer");
+          Affirm_default(source, "Invalid source");
+          const { schema, table } = this.resolveTable(producer, source);
+          const schemaMatch = schema ? `table_schema = ${this.nameLiteral(schema, `schema of producer "${producer.name}"`)}` : "table_schema = current_schema()";
+          return `SELECT column_name, data_type, is_nullable, ordinal_position FROM information_schema.columns WHERE ${schemaMatch} AND table_name = ${this.nameLiteral(table, `table of producer "${producer.name}"`)} ORDER BY ordinal_position`;
         };
         /**
          * The source column behind each dimension, in declaration order: `alias` when the dimension
@@ -44345,12 +44583,14 @@ var init_SqlQueryBuilder = __esm({
          * Where the producer's rows live. The table comes from `settings.sqlTable`, optionally qualified
          * as `schema.table`; an unqualified name falls back to the source's own `schema`, and with
          * neither the statement is left unqualified for the connection's `search_path` to resolve.
+         *
+         * A table name is the only thing a producer contributes: there is no hand-written statement to
+         * accept, so every statement sent to a source is one this builder emitted.
          */
         this.resolveTable = (producer, source) => {
           Affirm_default(producer, "Invalid producer");
           Affirm_default(source, "Invalid source");
-          const { sqlTable, sql, fileKey } = producer.settings;
-          Affirm_default(!sql, `Producer "${producer.name}" sets "sql", which is not supported on a SQL source yet \u2014 use "sqlTable"`);
+          const { sqlTable, fileKey } = producer.settings;
           Affirm_default(!fileKey || sqlTable, `Producer "${producer.name}" sets "fileKey", which a SQL source has no use for \u2014 use "sqlTable"`);
           Affirm_default(sqlTable, `Producer "${producer.name}" reads from SQL source "${source.name}" but declares no "sqlTable"`);
           const parts = sqlTable.split(".");
@@ -44359,6 +44599,12 @@ var init_SqlQueryBuilder = __esm({
             return { schema: parts[0], table: parts[1] };
           return { schema: source.authentication.schema || void 0, table: parts[0] };
         };
+        /**
+         * Whether a name is one `quote` would accept. For a caller that has to answer "can this be put in
+         * a statement at all" without being thrown at — see `SqlPredicate`, where an unusable column is a
+         * filter not pushed rather than a run that fails.
+         */
+        this.isIdentifier = (identifier) => !!identifier && IDENTIFIER_PATTERN2.test(identifier);
         /**
          * One identifier, quoted. The pattern check is the whole defence against a name reaching the
          * server as syntax rather than as a name — a producer's config is not a trusted statement — so
@@ -44369,6 +44615,23 @@ var init_SqlQueryBuilder = __esm({
           Affirm_default(IDENTIFIER_PATTERN2.test(identifier), `Invalid SQL identifier "${identifier}" for ${context}: expected letters, digits, "_" or "$", not starting with a digit`);
           return `"${identifier}"`;
         };
+        /**
+         * The same identifier as a string LITERAL, which is how the catalog stores it: in
+         * `information_schema` a table name is data, not syntax, so it is compared rather than quoted.
+         * It goes through the identical pattern check, which forbids a quote character outright, so
+         * there is nothing here to escape — a name this accepts cannot close the literal.
+         *
+         * The comparison is EXACT, which matches the projection: `buildProducerSelect` quotes every
+         * identifier, so `"Claim"` and `claim` are already two different tables to the server. Postgres
+         * folds an unquoted name to lower case when the table is created, and Redshift folds it always,
+         * so a `sqlTable` that does not match the stored spelling finds nothing — and the caller says so
+         * naming what it looked for, rather than guessing at the case.
+         */
+        this.nameLiteral = (identifier, context) => {
+          Affirm_default(identifier, `Missing SQL identifier for ${context}`);
+          Affirm_default(IDENTIFIER_PATTERN2.test(identifier), `Invalid SQL identifier "${identifier}" for ${context}: expected letters, digits, "_" or "$", not starting with a digit`);
+          return `'${identifier}'`;
+        };
       }
     };
     SqlQueryBuilder = new SqlQueryBuilderClass();
@@ -44376,6 +44639,288 @@ var init_SqlQueryBuilder = __esm({
   }
 });
 
+// ../../packages/drivers/src/sql/SqlPredicate.ts
+var ISO_DATE_SHAPE, NUMERIC_LITERAL, isControlCharacter, MAX_LITERAL_CHARS, SqlPredicateClass, SqlPredicate, SqlPredicate_default;
+var init_SqlPredicate = __esm({
+  "../../packages/drivers/src/sql/SqlPredicate.ts"() {
+    init_SqlQueryBuilder();
+    ISO_DATE_SHAPE = /^\d{4}-\d{2}-\d{2}/;
+    NUMERIC_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)$/;
+    isControlCharacter = (character) => {
+      const code = character.charCodeAt(0);
+      return code < 32 || code === 127;
+    };
+    MAX_LITERAL_CHARS = 1e3;
+    SqlPredicateClass = class {
+      constructor() {
+        /**
+         * The `WHERE` body for these filters, or `null` when none of them can be expressed.
+         *
+         * `columnTypes` is the type each column has in the SOURCE's own catalog, keyed by column name.
+         * A filter whose column is missing from it, or whose dimension type the catalog disagrees with,
+         * is not pushed: a producer is free to declare a `timestamptz` column as a `string`, and the
+         * statement that compares it as one either prunes the wrong rows or fails outright.
+         */
+        this.build = (filters, columnTypes) => {
+          if (!filters || filters.length === 0)
+            return null;
+          if (!columnTypes || columnTypes.size === 0)
+            return null;
+          return this.all(filters.map((filter) => this.toNode(filter, columnTypes)).filter((x2) => !!x2));
+        };
+        this.toNode = (filter, columnTypes) => {
+          const node = this.toComparison(filter, columnTypes);
+          if (!node)
+            return null;
+          if (filter.and?.length) {
+            const children = filter.and.map((x2) => this.toNode(x2, columnTypes)).filter((x2) => !!x2);
+            return this.all([node, ...children]);
+          }
+          if (filter.or?.length) {
+            const branches = filter.or.map((x2) => this.toNode(x2, columnTypes));
+            return branches.every((x2) => !!x2) ? this.any([node, ...branches]) : null;
+          }
+          return node;
+        };
+        /** The filter's own rule as SQL, ignoring its `and`/`or` children. */
+        this.toComparison = (filter, columnTypes) => {
+          if (columnTypes.get(filter.column) !== filter.type)
+            return null;
+          if (!SqlQueryBuilder_default.isIdentifier(filter.column))
+            return null;
+          const column = SqlQueryBuilder_default.quote(filter.column, `pushdown column "${filter.column}"`);
+          switch (filter.type) {
+            case "string":
+              return this.text(filter, column);
+            case "number":
+              return this.numeric(filter, column);
+            default:
+              return null;
+          }
+        };
+        /**
+         * A text column's rule. The LIKE patterns are built from a value with no backslash in it (see
+         * `isPlainText`), so the only pattern characters a value can carry are `%` and `_` — both of
+         * which match MORE than the literal they stand for, which is the safe direction.
+         */
+        this.text = (filter, column) => {
+          const values = filter.values ?? [];
+          if (values.length === 0 || !values.every(this.isPlainText))
+            return null;
+          switch (filter.operator) {
+            case "equals":
+              return values.length === 1 ? `${column} = ${this.textLiteral(values[0])}` : null;
+            case "in":
+              return `${column} IN (${values.map(this.textLiteral).join(", ")})`;
+            case "contains":
+              return `${column} LIKE ${this.textLiteral(`%${values[0]}%`)}`;
+            case "startsWith":
+              return `${column} LIKE ${this.textLiteral(`${values[0]}%`)}`;
+            case "endsWith":
+              return `${column} LIKE ${this.textLiteral(`%${values[0]}`)}`;
+            default:
+              return null;
+          }
+        };
+        /**
+         * A numeric column's rule. The literal goes over as the digits the filter was given rather than
+         * as the double they parse to, so `1.50` is compared as the server stores it.
+         *
+         * The one shape where the two sides can still disagree is a column holding more precision than a
+         * double: the server compares it exactly and remora compares it rounded, so two values a record
+         * reads as equal are two values to the server. A producer against such a column turns the lot
+         * off with `settings.disablePushdown`.
+         */
+        this.numeric = (filter, column) => {
+          const values = (filter.values ?? []).map(this.numberLiteral);
+          if (values.length === 0 || !values.every((x2) => x2 !== null))
+            return null;
+          switch (filter.operator) {
+            case "equals":
+              return values.length === 1 ? `${column} = ${values[0]}` : null;
+            case "in":
+              return `${column} IN (${values.join(", ")})`;
+            case "greaterThan":
+              return `${column} > ${values[0]}`;
+            case "greaterThanOrEquals":
+              return `${column} >= ${values[0]}`;
+            case "lessThan":
+              return `${column} < ${values[0]}`;
+            case "lessThanOrEquals":
+              return `${column} <= ${values[0]}`;
+            case "between":
+              return values.length === 2 ? `(${column} >= ${values[0]} AND ${column} <= ${values[1]})` : null;
+            default:
+              return null;
+          }
+        };
+        /**
+         * Whether this value can go into a statement as a string literal at all.
+         *
+         * A single quote is doubled, which is how SQL spells one and the only escape both engines read
+         * the same way whatever `standard_conforming_strings` is set to. A BACKSLASH is refused rather
+         * than escaped for that same reason — and because it is the default escape character of `LIKE`,
+         * where it would make the pattern match less than the value it came from.
+         */
+        this.isPlainText = (value) => {
+          const text = String(value ?? "");
+          if (text.includes("\\") || [...text].some(isControlCharacter))
+            return false;
+          if (ISO_DATE_SHAPE.test(text))
+            return false;
+          return text.length <= MAX_LITERAL_CHARS;
+        };
+        this.textLiteral = (value) => `'${String(value).replaceAll("'", "''")}'`;
+        /** The value as a numeric literal, or `null` when it does not name a finite number. */
+        this.numberLiteral = (value) => {
+          const text = String(value ?? "").trim();
+          if (!NUMERIC_LITERAL.test(text) || !Number.isFinite(Number(text)))
+            return null;
+          return text;
+        };
+        /** `null` when there is nothing to join, the node itself when there is one, a group otherwise. */
+        this.all = (nodes5) => this.group("AND", nodes5);
+        this.any = (nodes5) => this.group("OR", nodes5);
+        this.group = (operator, nodes5) => {
+          if (nodes5.length === 0)
+            return null;
+          return nodes5.length === 1 ? nodes5[0] : `(${nodes5.join(` ${operator} `)})`;
+        };
+      }
+    };
+    SqlPredicate = new SqlPredicateClass();
+    SqlPredicate_default = SqlPredicate;
+  }
+});
+
+// ../../packages/drivers/src/sql/SqlSchemaReader.ts
+var NUMERIC_TYPES, TEMPORAL_TYPES, SqlSchemaReaderClass, SqlSchemaReader, SqlSchemaReader_default;
+var init_SqlSchemaReader = __esm({
+  "../../packages/drivers/src/sql/SqlSchemaReader.ts"() {
+    init_src();
+    init_src2();
+    init_SqlQueryBuilder();
+    NUMERIC_TYPES = /* @__PURE__ */ new Set([
+      "smallint",
+      "integer",
+      "bigint",
+      "decimal",
+      "numeric",
+      "real",
+      "double precision",
+      // Redshift reports these spellings for the same types
+      "int",
+      "int2",
+      "int4",
+      "int8",
+      "float",
+      "float4",
+      "float8"
+    ]);
+    TEMPORAL_TYPES = /* @__PURE__ */ new Set(["date", "timestamp", "timestamptz", "timestamp without time zone", "timestamp with time zone"]);
+    SqlSchemaReaderClass = class {
+      constructor() {
+        /**
+         * Every column of the producer's table, in the table's own order.
+         *
+         * Goes through `driver.query`, which is already capped at `SQL_MAX_QUERY_ROWS` — a table with
+         * more columns than that cap is not a table remora can read anyway. A table that matches nothing
+         * fails here rather than producing a producer with no dimensions, because "this table has no
+         * columns" is never the truth: it is a name that did not resolve.
+         */
+        this.readColumns = async (driver, producer, source) => {
+          Affirm_default(driver, `Invalid source driver for producer "${producer?.name}"`);
+          Affirm_default(producer, "Invalid producer");
+          Affirm_default(source, "Invalid source");
+          const { schema, table } = SqlQueryBuilder_default.resolveTable(producer, source);
+          const select = SqlQueryBuilder_default.buildColumnSelect(producer, source);
+          Logger_default.log(`SQL schema read for producer "${producer.name}" with: ${select}`);
+          const result = await driver.query(select);
+          const rows = result?.rows ?? [];
+          Affirm_default.hasItems(rows, `Table "${schema ? `${schema}.` : ""}${table}" of producer "${producer.name}" has no columns in the catalog of source "${source.name}" \u2014 check the name exists and is spelled with the same case the table was created with, and that this user may read it`);
+          return rows.slice().sort((left, right) => Number(left.ordinal_position) - Number(right.ordinal_position)).map((row) => ({
+            name: String(row.column_name),
+            dataType: String(row.data_type ?? "").trim().toLowerCase(),
+            nullable: String(row.is_nullable ?? "YES").trim().toUpperCase() !== "NO"
+          }));
+        };
+        /**
+         * The dimension type a SQL column reads as. Anything not recognised is a `string`, which is both
+         * the safe answer and the true one for the types that reach a staged CSV as text anyway — `json`,
+         * `uuid`, an array, a Redshift `super`.
+         */
+        this.toDimensionType = (dataType) => {
+          const type = (dataType ?? "").trim().toLowerCase();
+          if (NUMERIC_TYPES.has(type)) return "number";
+          if (type === "boolean" || type === "bool") return "boolean";
+          if (TEMPORAL_TYPES.has(type)) return "datetime";
+          return "string";
+        };
+      }
+    };
+    SqlSchemaReader = new SqlSchemaReaderClass();
+    SqlSchemaReader_default = SqlSchemaReader;
+  }
+});
+
+// ../../packages/drivers/src/sql/SqlPushdown.ts
+var SqlPushdownClass, SqlPushdown, SqlPushdown_default;
+var init_SqlPushdown = __esm({
+  "../../packages/drivers/src/sql/SqlPushdown.ts"() {
+    init_src();
+    init_src2();
+    init_SqlPredicate();
+    init_SqlSchemaReader();
+    SqlPushdownClass = class {
+      constructor() {
+        /** What this producer's staging statement should carry. Empty when there is nothing to push. */
+        this.resolve = async (request) => {
+          Affirm_default(request, "Invalid SQL pushdown request");
+          const { driver, producer, source, plan } = request;
+          Affirm_default(driver, "Invalid source driver");
+          Affirm_default(producer, "Invalid producer");
+          Affirm_default(source, "Invalid source");
+          if (!plan || producer.settings?.disablePushdown)
+            return {};
+          return {
+            where: await this.buildWhere(driver, producer, source, plan),
+            limit: plan.limit > 0 ? plan.limit : void 0
+          };
+        };
+        this.buildWhere = async (driver, producer, source, plan) => {
+          const filters = plan.filters ?? [];
+          if (filters.length === 0)
+            return null;
+          const columnTypes = await this.readColumnTypes(driver, producer, source);
+          const where = SqlPredicate_default.build(filters, columnTypes);
+          if (!where && columnTypes)
+            Logger_default.log(`SQL pushdown: none of the ${filters.length} filter(s) of producer "${producer.name}" can be expressed as a WHERE clause`);
+          return where;
+        };
+        /**
+         * The type the SOURCE gives each of the table's columns, keyed by column name, or `null` when
+         * the catalog cannot be read.
+         *
+         * A table that is not in `information_schema` — a Redshift external (Spectrum) table, a
+         * late-binding view — is a table nothing is pushed to rather than a run that fails: the staging
+         * read itself does not go through the catalog and is unaffected.
+         */
+        this.readColumnTypes = async (driver, producer, source) => {
+          try {
+            const columns = await SqlSchemaReader_default.readColumns(driver, producer, source);
+            return new Map(columns.map((column) => [column.name, SqlSchemaReader_default.toDimensionType(column.dataType)]));
+          } catch (error) {
+            Logger_default.warn(`SQL pushdown: the columns of producer "${producer.name}" could not be read from the catalog of source "${source.name}", so its filters are applied per record instead: ${error}`);
+            return null;
+          }
+        };
+      }
+    };
+    SqlPushdown = new SqlPushdownClass();
+    SqlPushdown_default = SqlPushdown;
+  }
+});
+
 // ../../packages/drivers/src/RedshiftDriver.ts
 var import_fs10, import_client_redshift_data, RedshiftDriver, RedshiftDriver_default;
 var init_RedshiftDriver = __esm({
@@ -44389,6 +44934,7 @@ var init_RedshiftDriver = __esm({
     init_src2();
     init_AwsCredentials();
     init_SqlDatasetWriter();
+    init_SqlPushdown();
     init_SqlQueryBuilder();
     RedshiftDriver = class {
       constructor() {
@@ -44403,7 +44949,11 @@ var init_RedshiftDriver = __esm({
           Affirm_default(this._dbName, `Source "${source.name}" is missing authentication.database`);
           Affirm_default(this._workgroup || this._clusterId, `Source "${source.name}" is missing authentication.workgroup (Redshift Serverless) or authentication.clusterId (a provisioned cluster)`);
           const config = {
-            region: source.authentication["region"],
+            // Through replaceSecret like every other field: `region` is as much a `{ENV_VAR}`
+            // candidate as the database name, and the raw reference reached the SDK as a literal
+            // and failed as `region="{REDSHIFT_REGION}" is not a valid hostname component` — an
+            // error naming neither the source nor the variable that was not resolved.
+            region: SecretManager_default.replaceSecret(source.authentication["region"]),
             credentials: resolveAwsCredentials(source.authentication)
           };
           try {
@@ -44441,7 +44991,8 @@ var init_RedshiftDriver = __esm({
           Affirm_default(producer, "Invalid producer");
           Affirm_default(scope, "Invalid executor scope");
           const { table } = SqlQueryBuilder_default.resolveTable(producer, this._source);
-          const select = SqlQueryBuilder_default.buildProducerSelect(producer, this._source);
+          const pushdown = await SqlPushdown_default.resolve({ driver: this, producer, source: this._source, plan: request.pushdown });
+          const select = SqlQueryBuilder_default.buildProducerSelect(producer, this._source, pushdown);
           const columns = SqlQueryBuilder_default.columnKeys(producer);
           const localPath = ExecutorScope_default.getProducerPath(scope, producer, table);
           ExecutorScope_default.ensurePath(localPath);
@@ -44454,7 +45005,7 @@ var init_RedshiftDriver = __esm({
           const reported = statement.ResultRows >= 0 ? statement.ResultRows : null;
           Affirm_default(reported === null || reported === rows, `Redshift ready: staged ${rows} row(s) for producer "${producer.name}" but the statement reported ${reported}`);
           Logger_default.log(`Redshift ready: staged ${rows} row(s) (${fileSize} bytes) for producer "${producer.name}" from table "${table}"`);
-          return { files: [{ fullUri: localPath, fileSize }] };
+          return { files: [{ fullUri: localPath, fileSize }], filteredAtSource: !!pushdown.where };
         };
         this.execute = async (sql) => {
           Affirm_default(sql, `Invalid SQL`);
@@ -44669,6 +45220,7 @@ var init_PostgresDriver = __esm({
     init_src2();
     init_AwsCredentials();
     init_SqlDatasetWriter();
+    init_SqlPushdown();
     init_SqlQueryBuilder();
     PostgresDriver = class {
       constructor() {
@@ -44698,7 +45250,8 @@ var init_PostgresDriver = __esm({
           Affirm_default(producer, "Invalid producer");
           Affirm_default(scope, "Invalid executor scope");
           const { table } = SqlQueryBuilder_default.resolveTable(producer, this._source);
-          const select = SqlQueryBuilder_default.buildProducerSelect(producer, this._source);
+          const pushdown = await SqlPushdown_default.resolve({ driver: this, producer, source: this._source, plan: request.pushdown });
+          const select = SqlQueryBuilder_default.buildProducerSelect(producer, this._source, pushdown);
           const columns = SqlQueryBuilder_default.columnKeys(producer);
           const localPath = ExecutorScope_default.getProducerPath(scope, producer, table);
           ExecutorScope_default.ensurePath(localPath);
@@ -44714,7 +45267,7 @@ var init_PostgresDriver = __esm({
           onProgress?.(1);
           const fileSize = import_fs11.default.statSync(localPath).size;
           Logger_default.log(`Postgres ready: staged ${rows} row(s) (${fileSize} bytes) for producer "${producer.name}" from table "${table}"`);
-          return { files: [{ fullUri: localPath, fileSize }] };
+          return { files: [{ fullUri: localPath, fileSize }], filteredAtSource: !!pushdown.where };
         };
         /**
          * A bounded, ad-hoc read — the sample and discover paths, never a run. Capped at
@@ -44789,10 +45342,27 @@ var init_PostgresDriver = __esm({
           try {
             await client.connect();
           } catch (error) {
-            throw new Error(`Unable to connect to ${this.describeTarget()} for source "${this._source.name}": ${error.message}`);
+            throw new Error(`Unable to connect to ${this.describeTarget()} for source "${this._source.name}": ${this.describeConnectionError(error)}`);
           }
           return client;
         };
+        /**
+         * Why a connection failed, in words. A refused or timed-out connection is the commonest thing to
+         * go wrong with a SQL source and the one case where `error.message` is EMPTY: node resolves
+         * localhost to both ::1 and 127.0.0.1, and when every address fails `pg` rejects with an
+         * `AggregateError` whose message is blank and whose reasons are in `errors`. Reporting that
+         * verbatim produced "Unable to connect to postgres://host:5432/db for source "x": " — the
+         * message naming everything except the thing the operator needs. So the reason is taken from
+         * whichever of the three carries one, and the address list is deduplicated because "refused on
+         * ::1 and on 127.0.0.1" is one fact about one host, not two.
+         */
+        this.describeConnectionError = (error) => {
+          const failure = error;
+          if (failure?.message) return failure.message;
+          const reasons = [...new Set((failure?.errors ?? []).map((reason) => reason?.message).filter(Boolean))];
+          if (reasons.length > 0) return `${failure.code ? `${failure.code}: ` : ""}${reasons.join("; ")}`;
+          return failure?.code || "no reason reported by the driver";
+        };
         this.buildConfig = async () => {
           const host = SecretManager_default.replaceSecret(this._auth.host);
           const port = parseInt(SecretManager_default.replaceSecret(this._auth.port) || "5432");
@@ -45055,11 +45625,10 @@ var init_HttpApiDriver = __esm({
 });
 
 // ../../packages/drivers/src/DriverHelper.ts
-var import_stream2, import_readline, import_promises9, import_fs12, DriverHelper, DriverHelper_default;
+var import_stream2, import_promises9, import_fs12, DriverHelper, DriverHelper_default;
 var init_DriverHelper = __esm({
   "../../packages/drivers/src/DriverHelper.ts"() {
     import_stream2 = require("stream");
-    import_readline = require("readline");
     import_promises9 = require("stream/promises");
     import_fs12 = require("fs");
     init_src2();
@@ -45160,51 +45729,40 @@ var init_DriverHelper = __esm({
         Affirm_default(options2, "Invalid options");
         const { append, destinationPath, objects, delimiter } = options2;
         const writeOptions = append ? { flags: "a" } : {};
-        const writeStream = (0, import_fs12.createWriteStream)(destinationPath, writeOptions);
-        const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
+        const writer = new BatchWriter_default((0, import_fs12.createWriteStream)(destinationPath, writeOptions));
         let lineCount = 0;
         const keys = Object.keys(objects[0]);
         for (const obj of objects) {
-          const serialized = keys.map((k7) => obj[k7]).join(delimiter) + "\n";
-          if (!writeStream.write(serialized))
-            await waitForDrain();
+          if (!writer.write(keys.map((k7) => obj[k7]).join(delimiter)))
+            await writer.waitForDrain();
           lineCount++;
         }
-        writeStream.end();
-        await new Promise((resolve, reject) => {
-          writeStream.on("finish", resolve);
-          writeStream.on("error", reject);
-        });
+        await writer.close();
         return lineCount;
       },
       quickReadFile: async (filePath, lineCount) => {
         const fileStream = (0, import_fs12.createReadStream)(filePath);
-        const rl8 = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
         const lines = [];
         let counter = 0;
-        for await (const line of rl8) {
+        for await (const line of readLines(fileStream)) {
           lines.push(line);
           counter++;
           if (counter >= lineCount) {
             break;
           }
         }
-        rl8.close();
-        fileStream.close();
         return lines;
       },
       quickReadStream: async (stream, lineCount) => {
-        const rl8 = (0, import_readline.createInterface)({ input: stream, crlfDelay: Infinity });
         const lines = [];
         let counter = 0;
-        for await (const line of rl8) {
+        for await (const line of readLines(stream)) {
           lines.push(line);
           counter++;
           if (counter >= lineCount) {
             break;
           }
         }
-        rl8.close();
         return lines;
       }
     };
@@ -45213,12 +45771,11 @@ var init_DriverHelper = __esm({
 });
 
 // ../../packages/drivers/src/files/LocalSourceDriver.ts
-var fs13, import_path9, import_readline2, XLSX2, LocalSourceDriver;
+var fs13, import_path9, XLSX2, LocalSourceDriver;
 var init_LocalSourceDriver = __esm({
   "../../packages/drivers/src/files/LocalSourceDriver.ts"() {
     fs13 = __toESM(require("fs"), 1);
     import_path9 = __toESM(require("path"), 1);
-    import_readline2 = __toESM(require("readline"), 1);
     init_src();
     init_src();
     XLSX2 = __toESM(require("@e965/xlsx"), 1);
@@ -45293,10 +45850,9 @@ var init_LocalSourceDriver = __esm({
         };
         this._readLines = async (fileUri, lineFrom, lineTo) => {
           const stream = fs13.createReadStream(fileUri);
-          const reader = import_readline2.default.createInterface({ input: stream, crlfDelay: Infinity });
           const lines = [];
           let lineCounter = 0;
-          for await (const line of reader) {
+          for await (const line of readLines(stream)) {
             if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo)) {
               if (lineCounter >= lineFrom && lineCounter < lineTo) {
                 if (line && line.length > 0)
@@ -45309,8 +45865,6 @@ var init_LocalSourceDriver = __esm({
                 lines.push(line);
             }
           }
-          reader.close();
-          stream.close();
           return lines;
         };
         this._readExcelLines = async (fileUri, sheetName, lineFrom, lineTo, startRow, startColumn) => {
@@ -45456,12 +46010,12 @@ var init_LocalSourceDriver = __esm({
 });
 
 // ../../packages/drivers/src/files/LocalDestinationDriver.ts
-var fs14, import_promises10, import_readline3, import_path10, LocalDestinationDriver;
+var fs14, import_crypto4, import_promises10, import_path10, LocalDestinationDriver;
 var init_LocalDestinationDriver = __esm({
   "../../packages/drivers/src/files/LocalDestinationDriver.ts"() {
     fs14 = __toESM(require("fs"), 1);
+    import_crypto4 = require("crypto");
     import_promises10 = __toESM(require("fs/promises"), 1);
-    import_readline3 = __toESM(require("readline"), 1);
     import_path10 = __toESM(require("path"), 1);
     init_src();
     init_src2();
@@ -45492,37 +46046,57 @@ var init_LocalDestinationDriver = __esm({
           const fileContent = await s3Driver.downloadFile(sourceFileKey);
           await this.saveFile(destinationFileKey, fileContent);
         };
+        /**
+         * A private path to build the export at, beside the file it will become.
+         *
+         * Two runs of one consumer share an export name — that is what a configured `exportName` means —
+         * so they target the same path, and a run started by CRON while an API call is still exporting
+         * would otherwise have both streams writing into one file at their own offsets. The result is a
+         * file holding neither run: interleaved lines, and a torn line where they met.
+         *
+         * Building under a unique name and renaming into place makes the swap atomic, so concurrent runs
+         * degrade to a clean last-writer-wins — one run's output, whole — instead of corruption. A
+         * sibling of the destination on purpose: rename is only atomic within one filesystem.
+         */
+        this._stagingPath = (toFilePath) => `${toFilePath}.${(0, import_crypto4.randomUUID)()}.partial`;
+        /** Removes a staging file a failed export left behind; never masks the original error. */
+        this._discardStaging = async (stagingPath) => {
+          try {
+            await import_promises10.default.unlink(stagingPath);
+          } catch {
+          }
+        };
         this.move = async (fromPath, toName) => {
+          const toFilePath = import_path10.default.join(this._path, toName);
+          const stagingPath = this._stagingPath(toFilePath);
           try {
-            const toFilePath = import_path10.default.join(this._path, toName);
-            await import_promises10.default.copyFile(fromPath, toFilePath);
+            await import_promises10.default.copyFile(fromPath, stagingPath);
+            await import_promises10.default.rename(stagingPath, toFilePath);
             await import_promises10.default.unlink(fromPath);
             return { bucket: "", key: toFilePath, res: true };
           } catch (error) {
             Logger_default.error(error);
+            await this._discardStaging(stagingPath);
             return { bucket: "", key: "", res: false };
           }
         };
         this.transformAndMove = async (fromPath, transform, toName) => {
+          const toFilePath = import_path10.default.join(this._path, toName);
+          const stagingPath = this._stagingPath(toFilePath);
           try {
-            const toFilePath = import_path10.default.join(this._path, toName);
             const reader = fs14.createReadStream(fromPath);
-            const lineReader = import_readline3.default.createInterface({ input: reader, crlfDelay: Infinity });
-            const writer = fs14.createWriteStream(toFilePath);
-            const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
-            for await (const line of lineReader) {
-              if (!writer.write(transform(line) + "\n"))
-                await waitForDrain();
+            const writer = new BatchWriter_default(fs14.createWriteStream(stagingPath));
+            for await (const line of readLines(reader)) {
+              if (!writer.write(transform(line)))
+                await writer.waitForDrain();
             }
-            writer.end();
-            await new Promise((resolve, reject) => {
-              writer.on("finish", resolve);
-              writer.on("error", reject);
-            });
+            await writer.close();
             reader.close();
+            await import_promises10.default.rename(stagingPath, toFilePath);
             return { bucket: "", key: toFilePath, res: true };
           } catch (error) {
             Logger_default.error(error);
+            await this._discardStaging(stagingPath);
             return { bucket: "", key: "", res: false };
           }
         };
@@ -45531,89 +46105,24 @@ var init_LocalDestinationDriver = __esm({
   }
 });
 
-// ../../packages/drivers/src/s3/S3SourceDriver.ts
-var import_client_s32, import_stream3, import_promises11, import_readline4, import_path11, import_fs13, import_xlsx2, S3SourceDriver;
-var init_S3SourceDriver = __esm({
-  "../../packages/drivers/src/s3/S3SourceDriver.ts"() {
-    import_client_s32 = require("@aws-sdk/client-s3");
+// ../../packages/drivers/src/StreamReader.ts
+var import_xlsx2, StreamReaderClass, StreamReader, StreamReader_default;
+var init_StreamReader = __esm({
+  "../../packages/drivers/src/StreamReader.ts"() {
     init_src();
-    init_AwsCredentials();
-    import_stream3 = require("stream");
-    import_promises11 = require("stream/promises");
-    import_readline4 = __toESM(require("readline"), 1);
-    import_path11 = __toESM(require("path"), 1);
-    import_fs13 = __toESM(require("fs"), 1);
-    init_src();
-    import_xlsx2 = __toESM(require("@e965/xlsx"), 1);
     init_src5();
-    init_src6();
-    init_src5();
-    init_src4();
-    S3SourceDriver = class {
+    import_xlsx2 = __toESM(require("@e965/xlsx"), 1);
+    StreamReaderClass = class {
       constructor() {
-        this.init = async (source) => {
-          this._bucketName = source.authentication["bucket"];
-          const config = {
-            region: source.authentication["region"],
-            credentials: resolveAwsCredentials(source.authentication)
-          };
-          this._client = new import_client_s32.S3Client(config);
-          return this;
-        };
-        this.readAll = async (request) => {
-          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
-          Affirm_default(request, `Invalid download request`);
-          Affirm_default(request.fileKey, `Invalid file key for download request`);
-          const { fileKey } = request;
-          if (fileKey.includes("%")) {
-            const allFileKeys = await this.listFiles(fileKey);
-            Affirm_default(allFileKeys.length < 50, `Pattern ${fileKey} of producer requested to S3 matches more than 50 files (${allFileKeys.length}), this is more than the S3 allowed limit. Please refine your pattern, remove some files or use a separate bucket.`);
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
-            const results = await Promise.all(promises2);
-            return results.flat();
-          } else {
-            return await this._get(request);
-          }
-        };
-        this.readLinesInRange = async (request) => {
-          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
-          Affirm_default(request, "Invalid read request");
-          Affirm_default(request.options, "Invalid read request options");
-          const { fileKey } = request;
-          if (fileKey.includes("%")) {
-            const allFileKeys = await this.listFiles(fileKey);
-            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
-            const results = await Promise.all(promises2);
-            return results.flat();
-          } else {
-            return await this._get(request);
-          }
-        };
-        this.exist = async (producer) => {
-          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
-          Affirm_default(producer, "Invalid read producer");
-          const bucket = this._bucketName;
-          const fileKey = producer.settings.fileKey;
-          Affirm_default(fileKey, `Invalid file key for download request`);
-          if (fileKey.includes("%")) {
-            const allFileKeys = await this.listFiles(fileKey);
-            return allFileKeys.length > 0;
-          } else {
-            try {
-              await this._client.send(new import_client_s32.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
-              return true;
-            } catch (error) {
-              if (error.$metadata?.httpStatusCode === 404 || error.name === "NotFound")
-                return false;
-              throw error;
-            }
-          }
-        };
-        this._readLines = async (stream, lineFrom, lineTo) => {
-          const reader = import_readline4.default.createInterface({ input: stream, crlfDelay: Infinity });
+        /**
+         * Reads the stream as lines, optionally keeping only `[lineFrom, lineTo)`.
+         * Empty lines are dropped, and the read stops as soon as the range is satisfied.
+         */
+        this.readLines = async (stream, lineFrom, lineTo) => {
+          Affirm_default(stream, "Invalid stream");
           const lines = [];
           let lineCounter = 0;
-          for await (const line of reader) {
+          for await (const line of readLines(stream)) {
             if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo)) {
               if (lineCounter >= lineFrom && lineCounter < lineTo) {
                 if (line && line.length > 0)
@@ -45626,10 +46135,11 @@ var init_S3SourceDriver = __esm({
                 lines.push(line);
             }
           }
-          reader.close();
           return lines;
         };
-        this._readExcelLines = async (stream, sheetName, lineFrom, lineTo, startRow, startColumn) => {
+        /** Reads one sheet of a workbook as CSV lines. The whole workbook is buffered — XLSX has no streaming decode. */
+        this.readExcelLines = async (stream, sheetName, lineFrom, lineTo, startRow, startColumn) => {
+          Affirm_default(stream, "Invalid stream");
           Affirm_default(sheetName, `Invalid sheetname`);
           const chunks = [];
           for await (const chunk of stream) {
@@ -45652,7 +46162,9 @@ var init_S3SourceDriver = __esm({
           else
             return lines;
         };
-        this._readXmlLines = async (stream, lineFrom, lineTo) => {
+        /** Reads the stream as XML and flattens it to one JSON line per record. Buffered, like the Excel path. */
+        this.readXmlLines = async (stream, lineFrom, lineTo) => {
+          Affirm_default(stream, "Invalid stream");
           const chunks = [];
           for await (const chunk of stream) {
             chunks.push(chunk);
@@ -45665,24 +46177,19 @@ var init_S3SourceDriver = __esm({
           }
           return lines;
         };
-        this._get = async (request, index) => {
-          const { fileKey, fileType, options: options2 } = request;
-          const bucket = this._bucketName;
-          let lineFrom, lineTo, sheetName, hasHeaderRow, startRow, startColumn;
-          if (options2) {
-            lineFrom = options2.lineFrom;
-            lineTo = options2.lineTo;
-            sheetName = options2.sheetName;
-            hasHeaderRow = options2.hasHeaderRow;
-            startRow = options2.startRow;
-            startColumn = options2.startColumn;
-          }
-          const response = await this._client.send(new import_client_s32.GetObjectCommand({
-            Bucket: bucket,
-            Key: fileKey
-          }));
-          Affirm_default(response.Body, "Failed to fetch object from S3");
-          const stream = response.Body;
+        /**
+         * Reads a stream as the lines of `request.fileType`, honouring any line range in `request.options`.
+         *
+         * `index` is this file's position within a pattern match: anything but the first file drops its
+         * header row, or the header would repeat in the middle of the unified dataset.
+         */
+        this.read = async (stream, request, index) => {
+          Affirm_default(stream, "Invalid stream");
+          Affirm_default(request, "Invalid read request");
+          Affirm_default(request.fileType, "Invalid file type for read request");
+          const { fileType, options: options2 } = request;
+          const { lineFrom, lineTo, sheetName, hasHeaderRow, startRow, startColumn } = options2 ?? {};
+          const hasRange = Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo);
           let lines = [];
           switch (fileType) {
             case "CSV":
@@ -45690,30 +46197,110 @@ var init_S3SourceDriver = __esm({
             case "JSONL":
             case "TXT":
             case "FIXED":
-              if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo))
-                lines = await this._readLines(stream, lineFrom, lineTo);
-              else
-                lines = await this._readLines(stream);
+              lines = hasRange ? await this.readLines(stream, lineFrom, lineTo) : await this.readLines(stream);
               break;
             case "XLS":
             case "XLSX":
-              if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo))
-                lines = await this._readExcelLines(stream, sheetName, lineFrom, lineTo, startRow, startColumn);
-              else
-                lines = await this._readExcelLines(stream, sheetName, void 0, void 0, startRow, startColumn);
+              lines = hasRange ? await this.readExcelLines(stream, sheetName, lineFrom, lineTo, startRow, startColumn) : await this.readExcelLines(stream, sheetName, void 0, void 0, startRow, startColumn);
               break;
             case "XML":
-              if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo))
-                lines = await this._readXmlLines(stream, lineFrom, lineTo);
-              else
-                lines = await this._readXmlLines(stream);
+              lines = hasRange ? await this.readXmlLines(stream, lineFrom, lineTo) : await this.readXmlLines(stream);
               break;
           }
-          if (index > 0 && ParseHelper_default.shouldHaveHeader(fileType, hasHeaderRow)) {
+          if (index > 0 && ParseHelper_default.shouldHaveHeader(fileType, hasHeaderRow))
             lines = lines.slice(1);
-          }
           return lines;
         };
+      }
+    };
+    StreamReader = new StreamReaderClass();
+    StreamReader_default = StreamReader;
+  }
+});
+
+// ../../packages/drivers/src/s3/S3SourceDriver.ts
+var import_client_s32, import_stream3, import_promises11, import_path11, import_fs13, S3SourceDriver;
+var init_S3SourceDriver = __esm({
+  "../../packages/drivers/src/s3/S3SourceDriver.ts"() {
+    import_client_s32 = require("@aws-sdk/client-s3");
+    init_src();
+    init_AwsCredentials();
+    import_stream3 = require("stream");
+    import_promises11 = require("stream/promises");
+    import_path11 = __toESM(require("path"), 1);
+    import_fs13 = __toESM(require("fs"), 1);
+    init_src6();
+    init_StreamReader();
+    init_src4();
+    S3SourceDriver = class {
+      constructor() {
+        this.init = async (source) => {
+          this._bucketName = SecretManager_default.replaceSecret(source.authentication["bucket"]);
+          const config = {
+            region: SecretManager_default.replaceSecret(source.authentication["region"]),
+            credentials: resolveAwsCredentials(source.authentication)
+          };
+          this._client = new import_client_s32.S3Client(config);
+          return this;
+        };
+        this.readAll = async (request) => {
+          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
+          Affirm_default(request, `Invalid download request`);
+          Affirm_default(request.fileKey, `Invalid file key for download request`);
+          const { fileKey } = request;
+          if (fileKey.includes("%")) {
+            const allFileKeys = await this.listFiles(fileKey);
+            Affirm_default(allFileKeys.length < 50, `Pattern ${fileKey} of producer requested to S3 matches more than 50 files (${allFileKeys.length}), this is more than the S3 allowed limit. Please refine your pattern, remove some files or use a separate bucket.`);
+            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
+            const results = await Promise.all(promises2);
+            return results.flat();
+          } else {
+            return await this._get(request);
+          }
+        };
+        this.readLinesInRange = async (request) => {
+          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
+          Affirm_default(request, "Invalid read request");
+          Affirm_default(request.options, "Invalid read request options");
+          const { fileKey } = request;
+          if (fileKey.includes("%")) {
+            const allFileKeys = await this.listFiles(fileKey);
+            const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
+            const results = await Promise.all(promises2);
+            return results.flat();
+          } else {
+            return await this._get(request);
+          }
+        };
+        this.exist = async (producer) => {
+          Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
+          Affirm_default(producer, "Invalid read producer");
+          const bucket = this._bucketName;
+          const fileKey = producer.settings.fileKey;
+          Affirm_default(fileKey, `Invalid file key for download request`);
+          if (fileKey.includes("%")) {
+            const allFileKeys = await this.listFiles(fileKey);
+            return allFileKeys.length > 0;
+          } else {
+            try {
+              await this._client.send(new import_client_s32.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
+              return true;
+            } catch (error) {
+              if (error.$metadata?.httpStatusCode === 404 || error.name === "NotFound")
+                return false;
+              throw error;
+            }
+          }
+        };
+        this._get = async (request, index) => {
+          const { fileKey } = request;
+          const response = await this._client.send(new import_client_s32.GetObjectCommand({
+            Bucket: this._bucketName,
+            Key: fileKey
+          }));
+          Affirm_default(response.Body, "Failed to fetch object from S3");
+          return await StreamReader_default.read(response.Body, request, index);
+        };
         this._listFiles = async (fileKeyPattern, maxKeys, continuationToken) => {
           Affirm_default(this._client, 'S3 client not yet initialized, call "connect()" first');
           let prefix2 = "";
@@ -45843,22 +46430,22 @@ var init_S3SourceDriver = __esm({
 });
 
 // ../../packages/drivers/src/s3/S3DestinationDriver.ts
-var import_client_s33, import_path12, import_fs14, import_readline5, S3DestinationDriver;
+var import_client_s33, import_path12, import_fs14, S3DestinationDriver;
 var init_S3DestinationDriver = __esm({
   "../../packages/drivers/src/s3/S3DestinationDriver.ts"() {
     import_client_s33 = require("@aws-sdk/client-s3");
     init_src();
     init_src2();
+    init_src4();
     init_AwsCredentials();
     import_path12 = __toESM(require("path"), 1);
     import_fs14 = __toESM(require("fs"), 1);
-    import_readline5 = __toESM(require("readline"), 1);
     S3DestinationDriver = class {
       constructor() {
         this.init = async (source) => {
-          this._bucketName = source.authentication["bucket"];
+          this._bucketName = SecretManager_default.replaceSecret(source.authentication["bucket"]);
           const config = {
-            region: source.authentication["region"],
+            region: SecretManager_default.replaceSecret(source.authentication["region"]),
             credentials: resolveAwsCredentials(source.authentication)
           };
           this._client = new import_client_s33.S3Client(config);
@@ -45898,9 +46485,8 @@ var init_S3DestinationDriver = __esm({
           Affirm_default(toName, "Invalid destination name");
           Affirm_default(import_fs14.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
           const reader = import_fs14.default.createReadStream(fromPath);
-          const lineReader = import_readline5.default.createInterface({ input: reader, crlfDelay: Infinity });
           const transformedLines = async function* () {
-            for await (const line of lineReader) {
+            for await (const line of readLines(reader)) {
               yield transform(line) + "\n";
             }
           };
@@ -45983,6 +46569,382 @@ var init_S3DestinationDriver = __esm({
   }
 });
 
+// ../../packages/drivers/src/presigned/UrlMinter.ts
+var DEFAULT_TIMEOUT_MS, DEFAULT_URL_FIELD, DEFAULT_FILE_NAME_PARAM, MAX_ERROR_BODY_CHARS, UrlMinterClass, UrlMinter, UrlMinter_default;
+var init_UrlMinter = __esm({
+  "../../packages/drivers/src/presigned/UrlMinter.ts"() {
+    init_src();
+    init_src4();
+    init_src2();
+    init_OidcTokenProvider();
+    DEFAULT_TIMEOUT_MS = 3e4;
+    DEFAULT_URL_FIELD = "url";
+    DEFAULT_FILE_NAME_PARAM = "filename";
+    MAX_ERROR_BODY_CHARS = 500;
+    UrlMinterClass = class {
+      constructor() {
+        /**
+         * Mints a signed URL for one file name.
+         * Throws with the endpoint's own answer when it refuses — a mint that fails is a transfer that
+         * must not be attempted.
+         */
+        this.mint = async (auth, fileName) => {
+          Affirm_default(auth, "Missing authentication config for the presigned-url source");
+          Affirm_default(fileName, "Missing the file name to mint a signed URL for");
+          const endpoint = SecretManager_default.replaceSecret(auth.url);
+          Affirm_default(endpoint, "A presigned-url source requires its minting endpoint in authentication.url");
+          const payload = await this._request(auth, endpoint, fileName);
+          const urlField = auth.signedUrlField || DEFAULT_URL_FIELD;
+          const url = this._resolve(payload, urlField);
+          Affirm_default(url, `The minting endpoint ${this.safeUrl(endpoint)} returned no signed URL at "${urlField}". Set authentication.signedUrlField to where the URL is in its response.`);
+          Affirm_default(typeof url === "string", `The minting endpoint returned a ${typeof url} at "${urlField}", not a URL string`);
+          const headers = auth.signedHeadersField ? this._resolve(payload, auth.signedHeadersField) : void 0;
+          Affirm_default(!headers || typeof headers === "object", `The minting endpoint returned a ${typeof headers} at "${auth.signedHeadersField}", not an object of headers`);
+          Logger_default.log(`Minted a signed URL for "${fileName}" from ${this.safeUrl(endpoint)}: ${this.safeUrl(url)}`);
+          return { url, headers: headers ?? {} };
+        };
+        /**
+         * A URL with its query string removed, for logging. A signed URL carries its credentials there,
+         * so logging one whole would put a working upload token in the log pipeline — the same rule
+         * `StateEmitter` applies to every URL it emits.
+         */
+        this.safeUrl = (url) => {
+          try {
+            const parsed = new URL(url);
+            return `${parsed.origin}${parsed.pathname}`;
+          } catch {
+            return "(unparseable URL)";
+          }
+        };
+        this._request = async (auth, endpoint, fileName) => {
+          const method = auth.httpMethod || "POST";
+          const headers = await this._authHeaders(auth);
+          const fileNameParam = auth.fileNameParam || DEFAULT_FILE_NAME_PARAM;
+          const url = new URL(endpoint);
+          for (const [key, value] of Object.entries(auth.queryParams ?? {}))
+            url.searchParams.append(key, SecretManager_default.replaceSecret(value));
+          const options2 = {
+            method,
+            headers,
+            signal: AbortSignal.timeout(auth.timeout || DEFAULT_TIMEOUT_MS)
+          };
+          if (method === "POST" || method === "PUT" || method === "PATCH") {
+            headers["Content-Type"] = headers["Content-Type"] || "application/json";
+            options2.body = JSON.stringify({ [fileNameParam]: fileName });
+          } else {
+            url.searchParams.append(fileNameParam, fileName);
+          }
+          const response = await fetch(url.toString(), options2);
+          if (!response.ok) {
+            const body = await response.text().catch(() => "");
+            const detail = body ? `: ${body.slice(0, MAX_ERROR_BODY_CHARS)}` : "";
+            throw new Error(`The minting endpoint ${this.safeUrl(endpoint)} answered HTTP ${response.status} ${response.statusText} for "${fileName}"${detail}`);
+          }
+          return await response.json();
+        };
+        /**
+         * The credentials for the MINT call. Every method here already exists elsewhere in the codebase —
+         * `oidc` is the same client-credentials provider delta-share uses, which is what an API Gateway
+         * fronted by M2M authentication wants, and its token is cached and refreshed across runs.
+         */
+        this._authHeaders = async (auth) => {
+          const headers = {};
+          for (const [key, value] of Object.entries(auth.headers ?? {}))
+            headers[key] = SecretManager_default.replaceSecret(value);
+          switch (auth.method) {
+            case "oidc": {
+              headers["Authorization"] = `Bearer ${await OidcTokenProvider_default.getToken(this._oidcConfig(auth))}`;
+              break;
+            }
+            case "bearer-token": {
+              const token2 = SecretManager_default.replaceSecret(auth.bearerToken);
+              Affirm_default(token2, 'Authentication method "bearer-token" requires authentication.bearerToken');
+              headers["Authorization"] = `Bearer ${token2}`;
+              break;
+            }
+            case "api-key": {
+              const apiKey = SecretManager_default.replaceSecret(auth.apiKey);
+              Affirm_default(apiKey, 'Authentication method "api-key" requires authentication.apiKey');
+              headers[auth.apiKeyHeader || "X-API-Key"] = apiKey;
+              break;
+            }
+            case "username-password": {
+              const user = SecretManager_default.replaceSecret(auth.user);
+              const password = SecretManager_default.replaceSecret(auth.password);
+              Affirm_default(user, 'Authentication method "username-password" requires authentication.user');
+              Affirm_default(password, 'Authentication method "username-password" requires authentication.password');
+              headers["Authorization"] = `Basic ${Buffer.from(`${user}:${password}`).toString("base64")}`;
+              break;
+            }
+            case "none":
+              break;
+            default:
+              throw new Error(`Authentication method "${auth.method}" is not supported for a presigned-url source. Use oidc, bearer-token, api-key, username-password or none.`);
+          }
+          return headers;
+        };
+        this._oidcConfig = (auth) => {
+          const config = {
+            tokenEndpoint: SecretManager_default.replaceSecret(auth.tokenEndpoint),
+            clientId: SecretManager_default.replaceSecret(auth.clientId),
+            clientSecret: SecretManager_default.replaceSecret(auth.clientSecret),
+            scope: SecretManager_default.replaceSecret(auth.scope),
+            audience: SecretManager_default.replaceSecret(auth.audience),
+            clientAuthMethod: auth.clientAuthMethod
+          };
+          Affirm_default(config.tokenEndpoint, 'Authentication method "oidc" requires authentication.tokenEndpoint');
+          Affirm_default(config.clientId, 'Authentication method "oidc" requires authentication.clientId');
+          Affirm_default(config.clientSecret, 'Authentication method "oidc" requires authentication.clientSecret');
+          return config;
+        };
+        /** Walks a dot path (`data.uploadUrl`) into the response, answering undefined rather than throwing. */
+        this._resolve = (payload, path20) => {
+          let current = payload;
+          for (const segment of path20.split(".")) {
+            if (current === null || typeof current !== "object")
+              return void 0;
+            current = current[segment];
+          }
+          return current;
+        };
+      }
+    };
+    UrlMinter = new UrlMinterClass();
+    UrlMinter_default = UrlMinter;
+  }
+});
+
+// ../../packages/drivers/src/presigned/PresignedUrlSourceDriver.ts
+var import_stream4, import_promises12, import_fs15, MAX_ERROR_BODY_CHARS2, PresignedUrlSourceDriver;
+var init_PresignedUrlSourceDriver = __esm({
+  "../../packages/drivers/src/presigned/PresignedUrlSourceDriver.ts"() {
+    init_src();
+    init_src4();
+    init_src2();
+    import_stream4 = require("stream");
+    import_promises12 = require("stream/promises");
+    import_fs15 = __toESM(require("fs"), 1);
+    init_StreamReader();
+    init_UrlMinter();
+    MAX_ERROR_BODY_CHARS2 = 500;
+    PresignedUrlSourceDriver = class {
+      constructor() {
+        this.init = async (source) => {
+          Affirm_default(source, "Invalid source");
+          Affirm_default(source.authentication, `Missing authentication for source "${source.name}"`);
+          Affirm_default(source.authentication.url, `Source "${source.name}" is a presigned-url source and needs its minting endpoint in authentication.url`);
+          this._source = source;
+          this._auth = source.authentication;
+          Logger_default.log(`Presigned URL source "${source.name}" initialised: minting endpoint=${UrlMinter_default.safeUrl(SecretManager_default.replaceSecret(source.authentication.url))}`);
+          return this;
+        };
+        this.execute = async (_sql) => {
+          void _sql;
+          throw new Error("PresignedUrlSourceDriver.execute is not supported: a signed URL addresses a file, not a SQL engine");
+        };
+        this.query = async (_sql, _values) => {
+          void _sql;
+          void _values;
+          throw new Error("PresignedUrlSourceDriver.query is not supported: a signed URL addresses a file, not a SQL engine");
+        };
+        this.exist = async (producer) => {
+          Affirm_default(producer, "Invalid producer");
+          const fileKey = producer.settings?.fileKey;
+          Affirm_default(fileKey, `Producer "${producer.name}" has no settings.fileKey to mint a signed URL for`);
+          try {
+            const minted = await UrlMinter_default.mint(this._auth, fileKey);
+            const response = await fetch(minted.url, {
+              method: "GET",
+              headers: { ...minted.headers, range: "bytes=0-0" }
+            });
+            await response.body?.cancel();
+            return response.ok;
+          } catch (error) {
+            Logger_default.log(`Existence check for "${fileKey}" on source "${this._source.name}" answered no: ${error}`);
+            return false;
+          }
+        };
+        this.readAll = async (request) => {
+          Affirm_default(request, "Invalid download request");
+          Affirm_default(request.fileKey, "Invalid file key for download request");
+          Affirm_default(request.fileType, "Invalid file type for download request");
+          return await this._read(request);
+        };
+        this.readLinesInRange = async (request) => {
+          Affirm_default(request, "Invalid read request");
+          Affirm_default(request.fileKey, "Invalid file key for read request");
+          Affirm_default(request.fileType, "Invalid file type for read request");
+          Affirm_default(request.options, "Invalid read request options");
+          Affirm_default.hasValue(request.options.lineFrom, "Invalid read request option lineFrom");
+          Affirm_default.hasValue(request.options.lineTo, "Invalid read request option lineTo");
+          return await this._read(request);
+        };
+        /**
+         * Stages the producer's file locally, which is all a run needs from this source: everything after
+         * it reads the staged path and never learns where the bytes came from.
+         */
+        this.ready = async (request) => {
+          Affirm_default(request, "Invalid ready request");
+          Affirm_default(request.producer, "Invalid producer");
+          Affirm_default(request.scope, "Invalid executor scope");
+          const { producer, scope, onProgress } = request;
+          const fileKey = producer.settings?.fileKey;
+          Affirm_default(fileKey, `Producer "${producer.name}" has no settings.fileKey to mint a signed URL for`);
+          this._affirmSingleFile(fileKey);
+          const localPath = ExecutorScope_default.getProducerPath(scope, producer, fileKey);
+          ExecutorScope_default.ensurePath(localPath);
+          const minted = await UrlMinter_default.mint(this._auth, fileKey);
+          const response = await this._fetchSigned(minted, fileKey);
+          const totalBytes = Number(response.headers.get("content-length") ?? 0);
+          let stagedBytes = 0;
+          const meter = new import_stream4.Transform({
+            transform(chunk, _encoding, done) {
+              stagedBytes += chunk.length;
+              if (totalBytes > 0)
+                onProgress?.(Math.min(1, stagedBytes / totalBytes));
+              done(null, chunk);
+            }
+          });
+          await (0, import_promises12.pipeline)(import_stream4.Readable.fromWeb(response.body), meter, import_fs15.default.createWriteStream(localPath));
+          onProgress?.(1);
+          const fileSize = import_fs15.default.statSync(localPath).size;
+          Logger_default.info(`Staged "${fileKey}" from a signed URL to ${localPath} (${fileSize} bytes)`);
+          return { files: [{ fullUri: localPath, fileSize }] };
+        };
+        this._read = async (request) => {
+          this._affirmSingleFile(request.fileKey);
+          const minted = await UrlMinter_default.mint(this._auth, request.fileKey);
+          const response = await this._fetchSigned(minted, request.fileKey);
+          return await StreamReader_default.read(import_stream4.Readable.fromWeb(response.body), request);
+        };
+        /**
+         * A minting endpoint signs one named object, so there is nothing for a `%` pattern to expand
+         * against — no listing exists on this side. Refused here rather than silently staging the one
+         * file whose name happens to contain the pattern.
+         */
+        this._affirmSingleFile = (fileKey) => {
+          Affirm_default(
+            !fileKey.includes("%"),
+            `A presigned-url source cannot expand the file key pattern "${fileKey}": the minting endpoint signs one named file and this driver cannot list the others. Name the file exactly, or use a source that can list (aws-s3, local).`
+          );
+        };
+        this._fetchSigned = async (minted, fileKey) => {
+          const response = await fetch(minted.url, {
+            method: this._auth.signedUrlMethod || "GET",
+            headers: minted.headers
+          });
+          if (!response.ok) {
+            const body = await response.text().catch(() => "");
+            const detail = body ? `: ${body.slice(0, MAX_ERROR_BODY_CHARS2)}` : "";
+            throw new Error(`The signed URL for "${fileKey}" answered HTTP ${response.status} ${response.statusText}${detail}`);
+          }
+          Affirm_default(response.body, `The signed URL for "${fileKey}" returned no body`);
+          return response;
+        };
+      }
+    };
+  }
+});
+
+// ../../packages/drivers/src/presigned/PresignedUrlDestinationDriver.ts
+var import_stream5, import_crypto5, import_promises13, import_fs16, MAX_ERROR_BODY_CHARS3, PresignedUrlDestinationDriver;
+var init_PresignedUrlDestinationDriver = __esm({
+  "../../packages/drivers/src/presigned/PresignedUrlDestinationDriver.ts"() {
+    init_src();
+    init_src4();
+    init_src2();
+    import_stream5 = require("stream");
+    import_crypto5 = require("crypto");
+    import_promises13 = __toESM(require("fs/promises"), 1);
+    import_fs16 = __toESM(require("fs"), 1);
+    init_UrlMinter();
+    MAX_ERROR_BODY_CHARS3 = 500;
+    PresignedUrlDestinationDriver = class {
+      constructor() {
+        this.init = async (source) => {
+          Affirm_default(source, "Invalid source");
+          Affirm_default(source.authentication, `Missing authentication for source "${source.name}"`);
+          Affirm_default(source.authentication.url, `Source "${source.name}" is a presigned-url destination and needs its minting endpoint in authentication.url`);
+          this._source = source;
+          this._auth = source.authentication;
+          Logger_default.log(`Presigned URL destination "${source.name}" initialised: minting endpoint=${UrlMinter_default.safeUrl(SecretManager_default.replaceSecret(source.authentication.url))}`);
+          return this;
+        };
+        this.move = async (fromPath, toName) => {
+          Affirm_default(fromPath, "Invalid source path");
+          Affirm_default(toName, "Invalid destination name");
+          Affirm_default(import_fs16.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
+          try {
+            return await this._upload(fromPath, toName);
+          } catch (error) {
+            Logger_default.error(`Presigned upload of "${toName}" from source "${this._source.name}" failed: ${error}`);
+            return { res: false, key: "", bucket: "" };
+          }
+        };
+        this.transformAndMove = async (fromPath, transform, toName) => {
+          Affirm_default(fromPath, "Invalid source path");
+          Affirm_default(transform, "Invalid transform function");
+          Affirm_default(toName, "Invalid destination name");
+          Affirm_default(import_fs16.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
+          const stagingPath = `${fromPath}.${(0, import_crypto5.randomUUID)()}.partial`;
+          try {
+            await this._stageTransformed(fromPath, transform, stagingPath);
+            return await this._upload(stagingPath, toName);
+          } catch (error) {
+            Logger_default.error(`Presigned upload of "${toName}" from source "${this._source.name}" failed: ${error}`);
+            return { res: false, key: "", bucket: "" };
+          } finally {
+            await this._discardStaging(stagingPath);
+          }
+        };
+        this._upload = async (fromPath, toName) => {
+          const minted = await UrlMinter_default.mint(this._auth, toName);
+          const fileSize = import_fs16.default.statSync(fromPath).size;
+          const options2 = {
+            method: this._auth.signedUrlMethod || "PUT",
+            headers: { ...minted.headers, "content-length": String(fileSize) },
+            body: import_stream5.Readable.toWeb(import_fs16.default.createReadStream(fromPath)),
+            duplex: "half"
+          };
+          const response = await fetch(minted.url, options2);
+          if (!response.ok) {
+            const body = await response.text().catch(() => "");
+            const detail = body ? `: ${body.slice(0, MAX_ERROR_BODY_CHARS3)}` : "";
+            throw new Error(`The signed URL for "${toName}" answered HTTP ${response.status} ${response.statusText}${detail}`);
+          }
+          Logger_default.info(`Uploaded "${toName}" (${fileSize} bytes) to ${UrlMinter_default.safeUrl(minted.url)}`);
+          return { res: true, key: toName, bucket: this._host(minted.url) };
+        };
+        /** Writes the transformed lines to their own file, so the upload has a size to declare. */
+        this._stageTransformed = async (fromPath, transform, stagingPath) => {
+          const reader = import_fs16.default.createReadStream(fromPath);
+          const writer = new BatchWriter_default(import_fs16.default.createWriteStream(stagingPath));
+          for await (const line of readLines(reader)) {
+            if (!writer.write(transform(line)))
+              await writer.waitForDrain();
+          }
+          await writer.close();
+          reader.close();
+        };
+        /** Removes a staging file the export left behind; never masks the error that is being reported. */
+        this._discardStaging = async (stagingPath) => {
+          try {
+            await import_promises13.default.unlink(stagingPath);
+          } catch {
+          }
+        };
+        this._host = (url) => {
+          try {
+            return new URL(url).host;
+          } catch {
+            return "";
+          }
+        };
+      }
+    };
+  }
+});
+
 // ../../packages/drivers/src/DriverFactory.ts
 var STAGEABLE_ENGINES, SQL_ENGINES, DriverFactoryClass, DriverFactory, DriverFactory_default;
 var init_DriverFactory = __esm({
@@ -45995,7 +46957,9 @@ var init_DriverFactory = __esm({
     init_LocalDestinationDriver();
     init_S3SourceDriver();
     init_S3DestinationDriver();
-    STAGEABLE_ENGINES = ["local", "aws-s3", "delta-share", "aws-redshift", "postgres"];
+    init_PresignedUrlSourceDriver();
+    init_PresignedUrlDestinationDriver();
+    STAGEABLE_ENGINES = ["local", "aws-s3", "delta-share", "aws-redshift", "postgres", "presigned-url"];
     SQL_ENGINES = ["aws-redshift", "postgres"];
     DriverFactoryClass = class {
       constructor() {
@@ -46035,6 +46999,11 @@ var init_DriverFactory = __esm({
               await driver.init(source);
               return driver;
             }
+            case "presigned-url": {
+              const driver = new PresignedUrlSourceDriver();
+              await driver.init(source);
+              return driver;
+            }
             default:
               throw new Error(`Invalid driver type "${source.engine}". This driver is not implemented yet`);
           }
@@ -46046,6 +47015,11 @@ var init_DriverFactory = __esm({
               await driver.init(source);
               return driver;
             }
+            case "presigned-url": {
+              const driver = new PresignedUrlDestinationDriver();
+              await driver.init(source);
+              return driver;
+            }
             case "local": {
               const driver = new LocalDestinationDriver();
               await driver.init(source);
@@ -46076,9 +47050,16 @@ var init_src8 = __esm({
     init_LocalSourceDriver();
     init_S3DestinationDriver();
     init_S3SourceDriver();
+    init_PresignedUrlDestinationDriver();
+    init_PresignedUrlSourceDriver();
+    init_StreamReader();
+    init_UrlMinter();
     init_PostgresDriver();
     init_SqlDatasetWriter();
+    init_SqlPredicate();
+    init_SqlPushdown();
     init_SqlQueryBuilder();
+    init_SqlSchemaReader();
   }
 });
 
@@ -46216,7 +47197,7 @@ var init_ProducerManager = __esm({
 });
 
 // ../../packages/engines/src/transform/TypeCaster.ts
-var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TypeCasterClass, TypeCaster, TypeCaster_default;
+var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMAT_ONLY_TOKENS, DATE_PARSE_TOKENS, TYPE_DEFAULTS, TypeCasterClass, TypeCaster, TypeCaster_default;
 var init_TypeCaster = __esm({
   "../../packages/engines/src/transform/TypeCaster.ts"() {
     init_src();
@@ -46237,7 +47218,36 @@ var init_TypeCaster = __esm({
       { token: "mm", width: 2 },
       { token: "ss", width: 2 }
     ];
+    TYPE_DEFAULTS = {
+      string: "",
+      number: 0,
+      boolean: false,
+      datetime: null
+    };
     TypeCasterClass = class {
+      constructor() {
+        /**
+         * Casts a value for a DECLARED dimension: the type cast, plus what a MISSING value means for it.
+         *
+         * Missing is null/undefined, and — for every type but `string` — the empty string, which is how
+         * a NULL arrives from every source that stages through CSV (SQL, Delta Share, CSV files) and is
+         * indistinguishable from one there. An empty string IS a value for a `string` dimension, so it
+         * is left alone. A missing value reads as `null`, unless the dimension declares `nullable: false`
+         * and so asks for its type's own empty value instead. Without this, a NULL in a number column
+         * went through `Number('')` and arrived as a real `0` that joins, sums and filters like one the
+         * source actually held.
+         */
+        this.castDimension = (value, dimension) => {
+          if (dimension.type && this.isMissing(value, dimension.type))
+            return dimension.nullable === false ? TYPE_DEFAULTS[dimension.type] : null;
+          return this.cast(value, dimension.type, dimension.format);
+        };
+        /** null/undefined for any type, plus the empty string for every type but `string`. */
+        this.isMissing = (value, type) => {
+          if (!Algo_default.hasVal(value)) return true;
+          return value === "" && type !== "string";
+        };
+      }
       /**
        * Casts the value to the requested type (only if needed)
        * Optional format parameter currently supports:
@@ -46427,7 +47437,7 @@ var init_LineParser = __esm({
               counter = performance.now();
               const value = {};
               for (const dim of dimensions) {
-                value[dim.name] = TypeCaster_default.cast(parts[dim.index], dim.prodDimension.type, dim.prodDimension.format);
+                value[dim.name] = TypeCaster_default.castDimension(parts[dim.index], dim.prodDimension);
               }
               tracker.measure("process-line:cast&build-record", performance.now() - counter);
               return value;
@@ -46448,7 +47458,7 @@ var init_LineParser = __esm({
               const value = {};
               for (const dim of dimensions) {
                 const key = dim.prodDimension.alias ?? dim.prodDimension.name;
-                value[dim.name] = TypeCaster_default.cast(parsed[key], dim.prodDimension.type, dim.prodDimension.format);
+                value[dim.name] = TypeCaster_default.castDimension(parsed[key], dim.prodDimension);
               }
               tracker.measure("process-line:cast&build-record", performance.now() - counter);
               return value;
@@ -46470,8 +47480,11 @@ var init_LineParser = __esm({
             }
             if (!dim.range && !dim.placement) continue;
             const range2 = dim.placement ? FixedWidthParser_default.resolvePlacement(dim.placement, layout, occurrence) : dim.range;
-            const raw = range2 ? FixedWidthParser_default.readField(line, range2) : null;
-            record[dim.name] = TypeCaster_default.cast(raw, dim.prodDimension.type, dim.prodDimension.format);
+            if (!range2) {
+              record[dim.name] = null;
+              continue;
+            }
+            record[dim.name] = TypeCaster_default.castDimension(FixedWidthParser_default.readField(line, range2), dim.prodDimension);
           }
           return record;
         };
@@ -46483,7 +47496,7 @@ var init_LineParser = __esm({
           const record = {};
           for (const [index, field] of fields.entries()) {
             const fieldKey = field.finalKey;
-            record[fieldKey] = TypeCaster_default.cast(parts[index], field.dimension?.type ?? "string");
+            record[fieldKey] = field.dimension ? TypeCaster_default.castDimension(parts[index], field.dimension) : TypeCaster_default.cast(parts[index], "string");
           }
           return record;
         };
@@ -46501,11 +47514,11 @@ var init_LineParser = __esm({
 });
 
 // ../../packages/engines/src/producer/ProducerEngine.ts
-var import_path13, import_crypto4, noopTracker, ProducerEngineClass, ProducerEngine, ProducerEngine_default;
+var import_path13, import_crypto6, noopTracker, ProducerEngineClass, ProducerEngine, ProducerEngine_default;
 var init_ProducerEngine = __esm({
   "../../packages/engines/src/producer/ProducerEngine.ts"() {
     import_path13 = __toESM(require("path"), 1);
-    import_crypto4 = require("crypto");
+    import_crypto6 = require("crypto");
     init_src();
     init_src8();
     init_src4();
@@ -46591,7 +47604,7 @@ var init_ProducerEngine = __esm({
           switch (source.engine) {
             case "aws-redshift":
             case "postgres": {
-              const sql = `${SqlQueryBuilder_default.buildProducerSelect(producer, source, sampleSize)}`;
+              const sql = SqlQueryBuilder_default.buildProducerSelect(producer, source, { limit: sampleSize });
               Affirm_default(sql, `Invalid SQL from deployment compilation for producer "${producer.name}"`);
               const driver = await DriverFactory_default.instantiateSource(source);
               Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -46608,7 +47621,7 @@ var init_ProducerEngine = __esm({
                 let effectiveProducer = producer;
                 let effectiveSource = source;
                 if (compressionType) {
-                  scope = { id: (0, import_crypto4.randomUUID)(), folder: `sample-${(0, import_crypto4.randomUUID)()}`, workersId: [] };
+                  scope = { id: (0, import_crypto6.randomUUID)(), folder: `sample-${(0, import_crypto6.randomUUID)()}`, workersId: [] };
                   const driver = await DriverFactory_default.instantiateSource(source);
                   const readyResult = await driver.ready({ producer, scope });
                   const firstFile = readyResult.files[0].fullUri;
@@ -46660,7 +47673,7 @@ var init_ProducerEngine = __esm({
             const record = {};
             for (const dim of producer.dimensions) {
               const key = dim.alias ?? dim.name;
-              record[dim.name] = TypeCaster_default.cast(item[key], dim.type, dim.format);
+              record[dim.name] = TypeCaster_default.castDimension(item[key], dim);
             }
             return ProducerManager_default.applyFills(record, fills, sourceFilename);
           });
@@ -46767,53 +47780,178 @@ var init_ProducerEngine = __esm({
   }
 });
 
+// ../../packages/engines/src/producer/DiscoverMerger.ts
+var DiscoverMergerClass, DiscoverMerger, DiscoverMerger_default;
+var init_DiscoverMerger = __esm({
+  "../../packages/engines/src/producer/DiscoverMerger.ts"() {
+    init_src();
+    DiscoverMergerClass = class {
+      constructor() {
+        this.merge = (authored, discovered, mode = "merge") => {
+          Affirm_default(Array.isArray(authored), "Invalid authored dimensions");
+          Affirm_default.hasItems(discovered, "Discover produced no dimensions to merge");
+          Affirm_default(mode === "merge" || mode === "replace", `Unknown discover mode "${mode}" \u2014 expected "merge" or "replace".`);
+          const pairs = authored.map((dimension) => ({ dimension, found: this._lookup(discovered, dimension) }));
+          const consumed = new Set(pairs.map((pair) => pair.found?.name).filter(Boolean));
+          const added = discovered.filter((field) => !consumed.has(field.name));
+          const report = { added: added.map((field) => field.name), filled: [], orphaned: [], mismatched: [] };
+          for (const { dimension, found } of pairs)
+            this._reportMismatch(dimension, found, report);
+          if (mode === "replace") {
+            const unreadable = pairs.filter((pair) => !this._readsSource(pair.dimension)).map((pair) => pair.dimension);
+            report.orphaned = pairs.filter((pair) => !pair.found && this._readsSource(pair.dimension)).map((pair) => pair.dimension.name);
+            return { dimensions: [...discovered, ...unreadable], report };
+          }
+          const dimensions = pairs.map(({ dimension, found }) => {
+            if (found) return this._fill(dimension, found, report);
+            if (this._readsSource(dimension)) report.orphaned.push(dimension.name);
+            return dimension;
+          });
+          return { dimensions: [...dimensions, ...added], report };
+        };
+        /**
+         * What an authored dimension gains from the field discover matched it to: the properties it
+         * declared NOTHING for, and nothing else. In particular `type` is never changed — a producer is
+         * free to read a `timestamptz` as a string or a padded `char(n)` as one, and a re-discover that
+         * undid that would be the same un-curation this merge exists to stop. The disagreement is
+         * reported instead, so it is the author who decides.
+         */
+        this._fill = (authored, discovered, report) => {
+          const filled = { ...authored };
+          const properties = [];
+          if (!filled.type && discovered.type) {
+            filled.type = discovered.type;
+            properties.push("type");
+          }
+          if (filled.nullable === void 0 && discovered.nullable === false) {
+            filled.nullable = false;
+            properties.push("nullable");
+          }
+          if (properties.length > 0) report.filled.push({ name: authored.name, properties });
+          return filled;
+        };
+        this._reportMismatch = (authored, discovered, report) => {
+          if (!discovered?.type) return;
+          if (!authored.type || authored.type === discovered.type) return;
+          report.mismatched.push({ name: authored.name, declared: authored.type, discovered: discovered.type });
+        };
+        /**
+         * The discovered field an authored dimension reads, if any.
+         *
+         * A dimension names its source field with `alias`, falling back to its own `name`. Two
+         * fallbacks past that: the engines fold an unquoted SQL identifier to lower case, so a
+         * case-only difference is the same column; and an `alias` carrying a path (`address.city`)
+         * reaches INTO a field discover reports by its root name, so the root is what it matches —
+         * which also stops a decomposed object being re-added as a dimension of its own.
+         */
+        this._lookup = (discovered, dimension) => {
+          const key = dimension.alias ?? dimension.name;
+          if (!key) return void 0;
+          const byName = (name) => discovered.find((field) => field.name === name) ?? discovered.find((field) => field.name.toLowerCase() === name.toLowerCase());
+          return byName(key) ?? (key.includes(".") ? byName(key.split(".")[0]) : void 0);
+        };
+        /**
+         * Whether this dimension is supposed to correspond to a field of the source at all. A group
+         * produces no column of its own, and `sourceFilename` / `occurrenceOf` dimensions are filled by
+         * the read rather than read from the data — none of them missing from a discovery is news.
+         */
+        this._readsSource = (dimension) => !dimension.dimensions && !dimension.sourceFilename && !dimension.occurrenceOf;
+      }
+    };
+    DiscoverMerger = new DiscoverMergerClass();
+    DiscoverMerger_default = DiscoverMerger;
+  }
+});
+
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path14, import_promises12, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
+var import_path14, import_promises14, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
 var init_DeveloperEngine = __esm({
   "../../packages/engines/src/ai/DeveloperEngine.ts"() {
     init_src();
     init_ProducerEngine();
+    init_DiscoverMerger();
     init_src4();
     init_src8();
     import_path14 = __toESM(require("path"), 1);
-    import_promises12 = __toESM(require("fs/promises"), 1);
+    import_promises14 = __toESM(require("fs/promises"), 1);
     import_dayjs2 = __toESM(require("dayjs"), 1);
     import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
     import_dayjs2.default.extend(import_customParseFormat2.default);
     DeveloperEngineClass = class {
       constructor() {
-        this.discover = async (producer) => {
+        /**
+         * Learns a producer's fields from its source and writes them back into the producer's config.
+         *
+         * The write is a MERGE by default (see `DiscoverMerger`): what was hand-authored on a dimension
+         * survives a re-discover, and discover only supplies what the dimension declared nothing for.
+         * `replace` rewrites the dimension list from what was found, which is what bootstrapping a
+         * producer off the `create-producer` template needs.
+         */
+        this.discover = async (producer, mode = "merge") => {
           Affirm_default(producer, "Invalid producer");
           const source = Environment_default.getSource(producer.source);
           Affirm_default(source, `Cannot discover producer "${producer.name}": source "${producer.source}" was not found.`);
           await this.assertDiscoverable(producer, source);
-          const sampleData = await ProducerEngine_default.readSampleData(producer, 10, true);
-          Affirm_default(sampleData, "Discover process failed: no result found");
-          const typeDefinitions = this.extractFieldTypes(sampleData);
-          const mappedProducer = {
-            name: producer.name,
-            description: producer.description,
-            source: producer.source,
-            settings: { ...producer.settings },
-            dimensions: typeDefinitions.map((field) => ({
-              name: field.name,
-              type: this.mapFieldTypeToProducerType(field.type),
-              description: `Auto-mapped field: ${field.name}`,
-              classification: this.extractFieldClassification(field)?.isPHI ? ["PHI"] : void 0
-            })),
-            measures: [],
-            _version: producer._version ?? 1
-          };
-          mappedProducer["$schema"] = producer["$schema"];
+          const typeDefinitions = DriverFactory_default.isSql(source.engine) ? await this.discoverSqlFields(producer, source) : this.extractFieldTypes(await ProducerEngine_default.readSampleData(producer, 10, true));
+          Affirm_default.hasItems(typeDefinitions, `Discover found no fields for producer "${producer.name}" \u2014 there is nothing to write back.`);
           const producerPath = Environment_default.resolveResourceFilePath("producers", producer.name);
-          await import_promises12.default.mkdir(import_path14.default.dirname(producerPath), { recursive: true });
-          await import_promises12.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
-          return { producer: mappedProducer, fields: typeDefinitions };
+          const authored = await this.readAuthoredProducer(producerPath, producer);
+          const discovered = typeDefinitions.map(this.toDimension);
+          const { dimensions, report } = DiscoverMerger_default.merge(authored.dimensions ?? [], discovered, mode);
+          const mappedProducer = { ...authored, dimensions };
+          await import_promises14.default.mkdir(import_path14.default.dirname(producerPath), { recursive: true });
+          await import_promises14.default.writeFile(producerPath, `${JSON.stringify(mappedProducer, null, 4)}
+`, "utf-8");
+          return { producer: mappedProducer, fields: typeDefinitions, report };
+        };
+        /**
+         * The producer exactly as it sits on disk. Discover rewrites one property of a file somebody
+         * else wrote, so it starts from that file rather than from the loaded (expanded) resource.
+         */
+        this.readAuthoredProducer = async (producerPath, producer) => {
+          const raw = await import_promises14.default.readFile(producerPath, "utf-8").catch(() => null);
+          Affirm_default(raw, `Cannot discover producer "${producer.name}": its config file was not found at "${producerPath}".`);
+          const authored = JSON.parse(raw);
+          Affirm_default(authored?.name === producer.name, `Cannot discover producer "${producer.name}": "${producerPath}" declares producer "${authored?.name}".`);
+          return authored;
+        };
+        /** The dimension discover would write for a field it found, before any merge decides what of it is used. */
+        this.toDimension = (field) => ({
+          name: field.name,
+          type: field.dimensionType ?? this.mapFieldTypeToProducerType(field.type),
+          // Only written when the source STATED the column is NOT NULL. `nullable` defaults to
+          // true everywhere else, so leaving it off is the same declaration with less noise.
+          nullable: field.nullable === false ? false : void 0,
+          description: `Auto-mapped field: ${field.name}`,
+          classification: this.extractFieldClassification(field)?.isPHI ? ["PHI"] : void 0
+        });
+        /**
+         * The columns of a SQL producer's table, read from the source's own catalog.
+         *
+         * The catalog states the type, so nothing is inferred here: a `numeric` column is a number even
+         * when every row sampled from it was NULL, and `is_nullable` is a fact no number of sample rows
+         * could establish. Values are never read at all, which also means discovering a table costs one
+         * small statement rather than a scan of it.
+         */
+        this.discoverSqlFields = async (producer, source) => {
+          const driver = await DriverFactory_default.instantiateSource(source);
+          Affirm_default(driver, `Cannot discover producer "${producer.name}": no driver available for engine "${source.engine}".`);
+          const columns = await SqlSchemaReader_default.readColumns(driver, producer, source);
+          return columns.map((column) => ({
+            name: column.name,
+            type: column.dataType,
+            dimensionType: SqlSchemaReader_default.toDimensionType(column.dataType),
+            nullable: column.nullable
+          }));
         };
         // Discover-only precondition: sampling real data requires a configured, readable file.
         // Unlike the normal read path (which tolerates empty/absent files by design), discover
         // has nothing to infer from, so fail early with a message that says exactly what to fix.
         this.assertDiscoverable = async (producer, source) => {
+          if (DriverFactory_default.isSql(source.engine)) {
+            Affirm_default(producer.settings?.sqlTable, `Cannot discover producer "${producer.name}": no "sqlTable" is configured \u2014 set the table to read from source "${source.name}" first, as "table" or "schema.table".`);
+            return;
+          }
           const fileEngines = ["local", "aws-s3", "delta-share"];
           if (!fileEngines.includes(source.engine)) return;
           Affirm_default(producer.settings?.fileKey, `Cannot discover producer "${producer.name}": no "fileKey" is configured \u2014 set the file (or wildcard pattern) to read from source "${source.name}" first.`);
@@ -49344,10 +50482,10 @@ var init_MockDataFactory = __esm({
 });
 
 // ../../packages/engines/src/mock/SyntheticInputEngine.ts
-var import_promises13, import_os, import_path15, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine;
+var import_promises15, import_os, import_path15, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine;
 var init_SyntheticInputEngine = __esm({
   "../../packages/engines/src/mock/SyntheticInputEngine.ts"() {
-    import_promises13 = __toESM(require("fs/promises"), 1);
+    import_promises15 = __toESM(require("fs/promises"), 1);
     import_os = __toESM(require("os"), 1);
     import_path15 = __toESM(require("path"), 1);
     import_dayjs4 = __toESM(require("dayjs"), 1);
@@ -49373,7 +50511,7 @@ var init_SyntheticInputEngine = __esm({
           for (const cs2 of [...BUILTIN_CODE_SETS, ...codeSets]) codeSetMap.set(cs2.name.toLowerCase(), cs2);
           const producerMap = new Map(producers.map((p5) => [p5.name, p5]));
           const refTargets = this._collectReferences(producers, producerMap);
-          const dir = await import_promises13.default.mkdtemp(import_path15.default.join(import_os.default.tmpdir(), "remora-synth-"));
+          const dir = await import_promises15.default.mkdtemp(import_path15.default.join(import_os.default.tmpdir(), "remora-synth-"));
           const poolSize = Math.min(Math.max(opts.records, 1), POOL_CAP);
           const registry = new MockKeyRegistry();
           const specs = producers.flatMap((p5) => this._specs(p5, codeSetMap, producerMap, refTargets));
@@ -49390,7 +50528,7 @@ var init_SyntheticInputEngine = __esm({
           return { inputs, dir };
         };
         this.cleanup = async (bundle) => {
-          if (bundle?.dir) await import_promises13.default.rm(bundle.dir, { recursive: true, force: true });
+          if (bundle?.dir) await import_promises15.default.rm(bundle.dir, { recursive: true, force: true });
         };
         this._assertOpts = (opts) => {
           Affirm_default(opts, "Invalid options");
@@ -49440,8 +50578,8 @@ var init_SyntheticInputEngine = __esm({
             lines.push(JSON.stringify(row) + "\n");
           }
           const filePath = import_path15.default.join(dir, `${producer.name}.jsonl`);
-          await import_promises13.default.writeFile(filePath, lines.join(""), "utf-8");
-          const { size } = await import_promises13.default.stat(filePath);
+          await import_promises15.default.writeFile(filePath, lines.join(""), "utf-8");
+          const { size } = await import_promises15.default.stat(filePath);
           return { path: filePath, fileSize: size };
         };
         // Distinct structured code sets referenced by these specs -> their records (deduped by name).
@@ -49575,8 +50713,7 @@ var init_SyntheticInputEngine = __esm({
             return length;
           }
           Affirm_default(typeof value === "string", `Dimension "${dim.name}" synth "${prop}" must be a date string on a "datetime" dimension (got ${JSON.stringify(value)})`);
-          const formatted = dim.format ? import_dayjs4.default.utc(value, dim.format, true) : void 0;
-          const date2 = formatted?.isValid() ? formatted : import_dayjs4.default.utc(value);
+          const date2 = this._parseDate(value, dim.format);
           Affirm_default(date2.isValid(), `Dimension "${dim.name}" synth "${prop}" ("${value}") is not a valid date${dim.format ? ` for format "${dim.format}" or as an ISO date` : ""}`);
           return date2.valueOf();
         };
@@ -49588,11 +50725,43 @@ var init_SyntheticInputEngine = __esm({
           const synth = this._synth(dim);
           Affirm_default(!(synth.allowedValues && synth.codeSet), `Dimension "${dim.name}" cannot set both synth "allowedValues" and "codeSet"`);
           Affirm_default(!(synth.references && this._hasValueSource(dim)), `Dimension "${dim.name}" cannot combine synth "references" with "allowedValues"/"codeSet"`);
-          if (synth.allowedValues) return { enumValues: synth.allowedValues };
+          const type = dim.type ?? "string";
+          if (synth.allowedValues) {
+            Affirm_default(this._valuesFitType(synth.allowedValues, type, dim.format), `Dimension "${dim.name}" declares synth "allowedValues" that are not valid "${type}" values (${JSON.stringify(synth.allowedValues.slice(0, 3))}\u2026)`);
+            return { enumValues: synth.allowedValues };
+          }
           const ref = synth.codeSet ?? matchBuiltinBinding(name);
           if (!ref || synth.references) return {};
-          return this._resolveRef(ref, codeSetMap, dim.name);
+          const resolved = this._resolveRef(ref, codeSetMap, dim.name);
+          if (this._valuesFitType(this._setValues(resolved, codeSetMap), type, dim.format)) return resolved;
+          Affirm_default(!synth.codeSet, `Dimension "${dim.name}" is bound to code set "${this._refName(ref)}", whose values are not valid "${type}" values`);
+          return {};
         };
+        // Every value a resolved binding can emit: the flat set's own values, or the bound field's column
+        // across the structured set's records.
+        this._setValues = (resolved, codeSetMap) => resolved.enumValues ?? codeSetMap.get(resolved.codeSet.name.toLowerCase()).records.map((rec) => rec[resolved.codeSet.field]);
+        /**
+         * Whether a closed set of values survives the cast to the dimension's declared type. Enum and code
+         * set values are emitted verbatim and cast when the synthetic file is read back, so a set that
+         * can't produce that type fills the column with garbage — HEDIS codes cast to a `number` are all
+         * NaN, which serializes as null — rather than failing anywhere visible.
+         */
+        this._valuesFitType = (values, type, format4) => {
+          if (type === "string") return true;
+          if (type === "boolean") return values.every((value) => typeof value === "boolean");
+          if (type === "number") return values.every((value) => this._isNumeric(value));
+          return values.every((value) => this._parseDate(value, format4).isValid());
+        };
+        this._isNumeric = (value) => {
+          if (typeof value === "number") return Number.isFinite(value);
+          return typeof value === "string" && value.trim() !== "" && !isNaN(Number(value));
+        };
+        // Read the same way a bound is read: the dimension's own format first, a plain/ISO date otherwise.
+        this._parseDate = (value, format4) => {
+          const formatted = format4 && typeof value === "string" ? import_dayjs4.default.utc(value, format4, true) : void 0;
+          return formatted?.isValid() ? formatted : import_dayjs4.default.utc(value);
+        };
+        this._refName = (ref) => typeof ref === "string" ? ref : ref.name;
         this._resolveRef = (ref, codeSetMap, dimName) => {
           if (typeof ref === "string") {
             const set2 = codeSetMap.get(ref.toLowerCase());
@@ -50240,33 +51409,33 @@ var init_RequestValidator = __esm({
         this.resolveMembers = (consumer) => {
           return new Set(ConsumerManager_default.getExpandedFields(consumer).map((x2) => x2.finalKey));
         };
-        this.checkFilters = (value, context, path21, depth) => {
+        this.checkFilters = (value, context, path20, depth) => {
           const { MAX_REQUEST_FILTERS, MAX_REQUEST_FILTER_DEPTH } = Constants_default.defaults;
           if (!Array.isArray(value))
-            throw new Error(`Invalid "${path21}": expected an array of filters, got ${this.describe(value)}`);
+            throw new Error(`Invalid "${path20}": expected an array of filters, got ${this.describe(value)}`);
           if (value.length > MAX_REQUEST_FILTERS)
-            throw new Error(`Invalid "${path21}": ${value.length} filters is above the maximum of ${MAX_REQUEST_FILTERS}`);
+            throw new Error(`Invalid "${path20}": ${value.length} filters is above the maximum of ${MAX_REQUEST_FILTERS}`);
           if (depth > MAX_REQUEST_FILTER_DEPTH)
-            throw new Error(`Invalid filter at ${path21}: filters may not nest more than ${MAX_REQUEST_FILTER_DEPTH} deep`);
-          return value.map((x2, index) => this.checkFilter(x2, context, `${path21}[${index}]`, depth));
+            throw new Error(`Invalid filter at ${path20}: filters may not nest more than ${MAX_REQUEST_FILTER_DEPTH} deep`);
+          return value.map((x2, index) => this.checkFilter(x2, context, `${path20}[${index}]`, depth));
         };
-        this.checkFilter = (value, context, path21, depth) => {
+        this.checkFilter = (value, context, path20, depth) => {
           if (!value || typeof value !== "object" || Array.isArray(value))
-            throw new Error(`Invalid filter at ${path21}: expected an object, got ${this.describe(value)}`);
+            throw new Error(`Invalid filter at ${path20}: expected an object, got ${this.describe(value)}`);
           const filter = value;
           const unknownKey = Object.keys(filter).find((key) => !FILTER_KEYS.includes(key));
           if (unknownKey)
-            throw new Error(`Invalid filter at ${path21}: unknown property "${unknownKey}". Supported: ${FILTER_KEYS.join(", ")}`);
-          const operator = this.checkOperator(filter.operator, path21);
+            throw new Error(`Invalid filter at ${path20}: unknown property "${unknownKey}". Supported: ${FILTER_KEYS.join(", ")}`);
+          const operator = this.checkOperator(filter.operator, path20);
           const checked = {
-            member: this.checkMember(filter.member, context, path21),
+            member: this.checkMember(filter.member, context, path20),
             operator,
-            values: this.checkValues(filter.values, operator, path21)
+            values: this.checkValues(filter.values, operator, path20)
           };
           if (Algo_default.hasVal(filter.and))
-            checked.and = this.checkFilters(filter.and, context, `${path21}.and`, depth + 1);
+            checked.and = this.checkFilters(filter.and, context, `${path20}.and`, depth + 1);
           if (Algo_default.hasVal(filter.or))
-            checked.or = this.checkFilters(filter.or, context, `${path21}.or`, depth + 1);
+            checked.or = this.checkFilters(filter.or, context, `${path20}.or`, depth + 1);
           return checked;
         };
         /**
@@ -50274,18 +51443,18 @@ var init_RequestValidator = __esm({
          * own field names by the time filters run, so any other name reads `undefined` and quietly keeps
          * every row (or drops every row, for a negated operator).
          */
-        this.checkMember = (value, context, path21) => {
+        this.checkMember = (value, context, path20) => {
           if (typeof value !== "string" || value.length === 0)
-            throw new Error(`Invalid filter at ${path21}: "member" must be a non-empty string, got ${this.describe(value)}`);
+            throw new Error(`Invalid filter at ${path20}: "member" must be a non-empty string, got ${this.describe(value)}`);
           if (!context.members.has(value))
-            throw new Error(`Invalid filter at ${path21}: consumer "${context.consumer.name}" has no field named "${value}"`);
+            throw new Error(`Invalid filter at ${path20}: consumer "${context.consumer.name}" has no field named "${value}"`);
           return value;
         };
-        this.checkOperator = (value, path21) => {
+        this.checkOperator = (value, path20) => {
           if (typeof value !== "string" || !VALUES_BY_OPERATOR.has(value))
-            throw new Error(`Invalid filter at ${path21}: unknown operator ${this.describe(value)}. Supported: ${this.supportedOperators()}`);
+            throw new Error(`Invalid filter at ${path20}: unknown operator ${this.describe(value)}. Supported: ${this.supportedOperators()}`);
           if (VALUES_BY_OPERATOR.get(value) === "unsupported")
-            throw new Error(`Invalid filter at ${path21}: operator "${value}" is not supported by a run`);
+            throw new Error(`Invalid filter at ${path20}: operator "${value}" is not supported by a run`);
           return value;
         };
         /**
@@ -50293,24 +51462,24 @@ var init_RequestValidator = __esm({
          * own type and the filter's is read back into it, so a number or a boolean here is a client bug
          * that reads as a type mismatch (matching nothing) rather than as the comparison it meant.
          */
-        this.checkValues = (value, operator, path21) => {
+        this.checkValues = (value, operator, path20) => {
           const values = Algo_default.hasVal(value) ? value : [];
           if (!Array.isArray(values))
-            throw new Error(`Invalid filter at ${path21}: "values" must be an array of strings, got ${this.describe(value)}`);
+            throw new Error(`Invalid filter at ${path20}: "values" must be an array of strings, got ${this.describe(value)}`);
           const wrongIndex = values.findIndex((x2) => typeof x2 !== "string");
           if (wrongIndex >= 0)
-            throw new Error(`Invalid filter at ${path21}: "values[${wrongIndex}]" must be a string, got ${this.describe(values[wrongIndex])}`);
-          return this.checkArity(values, operator, path21);
+            throw new Error(`Invalid filter at ${path20}: "values[${wrongIndex}]" must be a string, got ${this.describe(values[wrongIndex])}`);
+          return this.checkArity(values, operator, path20);
         };
-        this.checkArity = (values, operator, path21) => {
+        this.checkArity = (values, operator, path20) => {
           const expected = VALUES_BY_OPERATOR.get(operator);
           const reason = `operator "${operator}"`;
           if (expected === "none" && values.length > 0)
-            throw new Error(`Invalid filter at ${path21}: ${reason} reads no values, ${values.length} given`);
+            throw new Error(`Invalid filter at ${path20}: ${reason} reads no values, ${values.length} given`);
           if (expected === "many" && values.length === 0)
-            throw new Error(`Invalid filter at ${path21}: ${reason} needs at least one value, none given`);
+            throw new Error(`Invalid filter at ${path20}: ${reason} needs at least one value, none given`);
           if (typeof expected === "number" && values.length !== expected)
-            throw new Error(`Invalid filter at ${path21}: ${reason} needs exactly ${expected} ${expected === 1 ? "value" : "values"}, ${values.length} given`);
+            throw new Error(`Invalid filter at ${path20}: ${reason} needs exactly ${expected} ${expected === 1 ? "value" : "values"}, ${values.length} given`);
           return values;
         };
         /** Every operator a request may ask for, in the order they are declared. */
@@ -50651,23 +51820,23 @@ var init_Diagnostics = __esm({
           return [.../* @__PURE__ */ new Set([...inputs, ...chained])];
         };
         this._walk = (start, next, onCycle) => {
-          const path21 = [];
+          const path20 = [];
           const onPath = /* @__PURE__ */ new Set();
           const stack = [{ name: start, done: false }];
           const visited = /* @__PURE__ */ new Set();
           while (stack.length > 0) {
             const frame = stack.pop();
             if (frame.done) {
-              onPath.delete(path21.pop());
+              onPath.delete(path20.pop());
               continue;
             }
             if (onPath.has(frame.name)) {
-              path21.slice(path21.indexOf(frame.name)).forEach((name) => onCycle.add(name));
+              path20.slice(path20.indexOf(frame.name)).forEach((name) => onCycle.add(name));
               continue;
             }
             if (visited.has(frame.name)) continue;
             visited.add(frame.name);
-            path21.push(frame.name);
+            path20.push(frame.name);
             onPath.add(frame.name);
             stack.push({ name: frame.name, done: true });
             const successors = next.get(frame.name) ?? [];
@@ -51189,7 +52358,7 @@ var init_ResourceView = __esm({
             description: producer.description,
             summary: [
               { label: "source", value: producer.source ?? null, link: producer.source ? LineageEngine_default.nodeId("source", producer.source) : void 0 },
-              { label: "reads", value: settings.sqlTable ?? settings.sql ?? settings.fileKey ?? null },
+              { label: "reads", value: settings.sqlTable ?? settings.fileKey ?? null },
               { label: "type", value: settings.fileType ?? "SQL" },
               { label: "dimensions", value: dimensions.length },
               { label: "primary keys", value: dimensions.filter((x2) => x2.pk).length }
@@ -51225,7 +52394,9 @@ var init_ResourceView = __esm({
             { value: this._text(dimension.type) },
             positioned ? { value: this._position(dimension.position) } : { value: this._text(dimension.alias), muted: !dimension.alias },
             { value: !!dimension.pk, muted: !dimension.pk },
-            { value: !!dimension.nullable, muted: !dimension.nullable },
+            // Absent means nullable, so an undeclared dimension reads true here rather than false;
+            // muted says the value was defaulted and not written down.
+            { value: dimension.nullable !== false, muted: dimension.nullable === void 0 },
             { value: this._text(dimension.mask), muted: !dimension.mask },
             { value: this._text(dimension.classification), muted: !dimension.classification }
           ]);
@@ -51271,7 +52442,7 @@ var init_ResourceView = __esm({
               { label: "fields", value: (consumer.fields ?? []).length },
               { label: "schema", value: consumer.schema ?? null, link: consumer.schema ? LineageEngine_default.nodeId("schema", consumer.schema) : void 0 },
               { label: "distinct", value: !!consumer.options?.distinct },
-              { label: "max file size (Gb)", value: consumer.maximumFileSize ?? null }
+              { label: "split", value: this._splitSummary(consumer) }
             ],
             tables: [
               this._consumerInputsTable(env2, consumer),
@@ -51283,6 +52454,19 @@ var init_ResourceView = __esm({
             raw: this._raw(consumer)
           };
         };
+        /**
+         * How the consumer splits its output, in one line: the key(s) it buckets on, the per-file cap,
+         * or both. `maximumFileSize` is the older spelling of the cap and reads the same here.
+         */
+        this._splitSummary = (consumer) => {
+          const keys = consumer.split?.by ?? [];
+          const maxFileSizeGb = consumer.split?.maxFileSizeGb ?? consumer.maximumFileSize;
+          const parts = [
+            keys.length > 0 ? `by ${keys.join(", ")}` : null,
+            maxFileSizeGb ? `max ${maxFileSizeGb} Gb per file` : null
+          ].filter((x2) => !!x2);
+          return parts.length > 0 ? parts.join(", ") : null;
+        };
         this._consumerInputsTable = (env2, consumer) => {
           const rows = (consumer.producers ?? []).map((input) => [
             { value: input.name, link: this._inputLink(env2, input.name), status: this._inputLink(env2, input.name) ? void 0 : "error" },
@@ -51345,8 +52529,8 @@ var init_ResourceView = __esm({
           if (!source || origin.path.length < 2) return { value: "declared here", muted: true };
           return { value: `${source.resource}.${source.field}`, link: source.node };
         };
-        this._pathText = (path21) => {
-          return path21.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
+        this._pathText = (path20) => {
+          return path20.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
         };
         this._consumerOutputsTable = (env2, consumer) => {
           const declared = new Set((env2.sources ?? []).map((x2) => x2.name));
@@ -52173,11 +53357,11 @@ var init_OpsHeartbeat = __esm({
 });
 
 // ../../packages/engines/src/usage/LocalUsageDB.ts
-var import_fs15, import_path16, COLUMNS, CLI_DB_FOLDER, CLI_DB_FILE, CLI_DB_PATH, MAX_USAGE_ROWS, MAX_LISTED_ROWS, EXECUTED_BY_SCHEMA, EXECUTOR_OPERATION_SCHEMA, STATS_SCHEMA, COLUMN_SCHEMAS, LocalUsageDBClass, LocalUsageDB, LocalUsageDB_default;
+var import_fs17, import_path16, COLUMNS, CLI_DB_FOLDER, CLI_DB_FILE, CLI_DB_PATH, MAX_USAGE_ROWS, MAX_LISTED_ROWS, EXECUTED_BY_SCHEMA, EXECUTOR_OPERATION_SCHEMA, STATS_SCHEMA, COLUMN_SCHEMAS, LocalUsageDBClass, LocalUsageDB, LocalUsageDB_default;
 var init_LocalUsageDB = __esm({
   "../../packages/engines/src/usage/LocalUsageDB.ts"() {
     init_src();
-    import_fs15 = require("fs");
+    import_fs17 = require("fs");
     import_path16 = require("path");
     COLUMNS = [
       "_id",
@@ -52368,21 +53552,21 @@ var init_LocalUsageDB = __esm({
           return usage;
         };
         this._getRows = () => {
-          const lines = (0, import_fs15.readFileSync)(CLI_DB_PATH, "utf8").split("\n").filter(Boolean);
+          const lines = (0, import_fs17.readFileSync)(CLI_DB_PATH, "utf8").split("\n").filter(Boolean);
           lines.shift();
           return lines;
         };
         this._writeRows = (rows) => {
           const trimmedRows = rows.slice(-MAX_USAGE_ROWS);
-          (0, import_fs15.writeFileSync)(CLI_DB_PATH, `${[COLUMNS.join(","), ...trimmedRows].join("\n")}
+          (0, import_fs17.writeFileSync)(CLI_DB_PATH, `${[COLUMNS.join(","), ...trimmedRows].join("\n")}
 `);
         };
         this.init = () => {
           try {
-            if (!(0, import_fs15.existsSync)(CLI_DB_FOLDER))
-              (0, import_fs15.mkdirSync)(CLI_DB_FOLDER, { recursive: true });
-            if (!(0, import_fs15.existsSync)(CLI_DB_PATH))
-              (0, import_fs15.writeFileSync)(CLI_DB_PATH, `${COLUMNS.join(",")}
+            if (!(0, import_fs17.existsSync)(CLI_DB_FOLDER))
+              (0, import_fs17.mkdirSync)(CLI_DB_FOLDER, { recursive: true });
+            if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
+              (0, import_fs17.writeFileSync)(CLI_DB_PATH, `${COLUMNS.join(",")}
 `);
             this._writeRows(this._getRows());
           } catch (error) {
@@ -52412,7 +53596,7 @@ var init_LocalUsageDB = __esm({
          */
         this.list = (limit = MAX_LISTED_ROWS, keep) => {
           try {
-            if (!(0, import_fs15.existsSync)(CLI_DB_PATH))
+            if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
               return [];
             const rows = this._getRows();
             const wanted = Math.max(1, limit);
@@ -52433,9 +53617,9 @@ var init_LocalUsageDB = __esm({
          */
         this.signature = () => {
           try {
-            if (!(0, import_fs15.existsSync)(CLI_DB_PATH))
+            if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
               return "";
-            const stats = (0, import_fs15.statSync)(CLI_DB_PATH);
+            const stats = (0, import_fs17.statSync)(CLI_DB_PATH);
             return `${stats.mtimeMs}:${stats.size}`;
           } catch (error) {
             this._handleError("signature", error);
@@ -52602,10 +53786,10 @@ var init_UsageManager = __esm({
 });
 
 // ../../packages/engines/src/ops/RunOverlay.ts
-var import_crypto5, MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay;
+var import_crypto7, MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay;
 var init_RunOverlay = __esm({
   "../../packages/engines/src/ops/RunOverlay.ts"() {
-    import_crypto5 = __toESM(require("crypto"), 1);
+    import_crypto7 = __toESM(require("crypto"), 1);
     init_src();
     init_LineageEngine();
     init_LocalUsageDB();
@@ -52659,7 +53843,7 @@ var init_RunOverlay = __esm({
          * finish action writes runs of its own, so a narrower token would miss a transition the graph
          * exists to show.
          */
-        this._token = (overlay) => import_crypto5.default.createHash("sha256").update(JSON.stringify(overlay)).digest("hex");
+        this._token = (overlay) => import_crypto7.default.createHash("sha256").update(JSON.stringify(overlay)).digest("hex");
         this._run = (stat) => {
           const startedAt = this._iso(stat.startedAt);
           return {
@@ -52941,12 +54125,12 @@ var init_LookupRegistry = __esm({
 });
 
 // ../../packages/engines/src/transform/LookupKey.ts
-var import_crypto6, DELIMITER, idFor, LookupKey, LookupKey_default;
+var import_crypto8, DELIMITER, idFor, LookupKey, LookupKey_default;
 var init_LookupKey = __esm({
   "../../packages/engines/src/transform/LookupKey.ts"() {
-    import_crypto6 = __toESM(require("crypto"), 1);
+    import_crypto8 = __toESM(require("crypto"), 1);
     DELIMITER = String.fromCharCode(0);
-    idFor = (ref) => import_crypto6.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
+    idFor = (ref) => import_crypto8.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
     LookupKey = { idFor };
     LookupKey_default = LookupKey;
   }
@@ -53581,6 +54765,7 @@ var init_src10 = __esm({
     init_RunOverlay();
     init_RunRegistry();
     init_LineParser();
+    init_DiscoverMerger();
     init_ProducerEngine();
     init_ProducerManager();
     init_PushdownEngine();
@@ -53596,16 +54781,14 @@ var init_src10 = __esm({
 });
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs19, import_path17, OutputExecutorClass, OutputExecutor, OutputExecutor_default;
+var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
 var init_OutputExecutor = __esm({
   "../../packages/executors/src/OutputExecutor.ts"() {
     init_src();
-    fs19 = __toESM(require("fs"));
     init_src();
     init_src8();
     init_src4();
     init_src5();
-    import_path17 = __toESM(require("path"));
     init_src2();
     OutputExecutorClass = class {
       constructor() {
@@ -53667,24 +54850,33 @@ var init_OutputExecutor = __esm({
           }
           return JSON.stringify(record);
         };
-        this.getCompletedPath = (pathName, fileNumber) => {
-          const pathAsArray = pathName.split(".");
-          const extension = pathAsArray[1];
-          return pathAsArray[0].concat(`_${fileNumber}.${extension}`);
+        /**
+         * The exported name of one split file: the destination name with the file's label before the
+         * extension, e.g. "sales.csv" + "italy_1" -> "sales_italy_1.csv". The extension is taken from
+         * the LAST dot, since a label (and an exportName) may carry one of its own.
+         */
+        this.getCompletedPath = (pathName, label) => {
+          const dot = pathName.lastIndexOf(".");
+          if (dot < 0) return `${pathName}_${label}`;
+          return `${pathName.substring(0, dot)}_${label}${pathName.substring(dot)}`;
         };
-        this.exportResult = async (consumer, fields, scope) => {
+        /**
+         * Writes the run's result to every configured destination, one call per file the split pass
+         * produced. The files are passed in rather than read back out of the scope folder: the label a
+         * file carries is what names it at the destination, and with no split configured that list is
+         * the merged dataset alone, labelled "0" — the suffix an unsplit export has always had.
+         */
+        this.exportResult = async (consumer, fields, splitFiles) => {
+          Affirm_default.hasItems(splitFiles, `Consumer "${consumer.name}" has no output file to export`);
           const internalFormat = this._getInternalRecordFormat(consumer);
           const results = [];
           for (const output of consumer.outputs) {
             const destination = Environment_default.getSource(output.exportDestination);
             const driver = await DriverFactory_default.instantiateDestination(destination);
-            const currentPath = import_path17.default.dirname(ExecutorScope_default.getMainPath(scope));
             const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
-            Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-            const filenameArray = fs19.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
-            for (const filename in filenameArray) {
-              const destinationPath = this.getCompletedPath(destinationName, filename);
-              const startingPath = import_path17.default.join(currentPath, filenameArray[filename]);
+            Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
+            for (const { path: startingPath, label } of splitFiles) {
+              const destinationPath = this.getCompletedPath(destinationName, label);
               if (output.format === internalFormat) {
                 results.push(await driver.move(startingPath, destinationPath));
               } else {
@@ -53728,9 +54920,18 @@ var init_OutputExecutor = __esm({
             return this._unifingResult(results);
           }
         };
+        /**
+         * Folds the per-file upload results into the one the run reports.
+         *
+         * `every`, not `some`: a destination write that failed is data that never arrived, and one
+         * successful file must not be able to report the run as exported. The drivers catch their own
+         * errors and answer `res: false`, so this flag is the only thing that carries a failed write
+         * back up — `launch` refuses the run on it.
+         */
         this._unifingResult = (resultList) => {
+          Affirm_default.hasItems(resultList, "Export produced no result to report");
           resultList[0].key = resultList.map((res) => res.key).join(", ");
-          resultList[0].res = resultList.map((res) => res.res).some((res) => res);
+          resultList[0].res = resultList.every((res) => res.res);
           return resultList[0];
         };
         this._getExtension = (output) => {
@@ -53753,14 +54954,13 @@ var init_OutputExecutor = __esm({
 });
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path18, import_fs16, import_readline6, import_promises14, import_crypto7, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
+var import_path17, import_fs18, import_promises16, import_crypto9, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
 var init_ConsumerExecutor = __esm({
   "../../packages/executors/src/ConsumerExecutor.ts"() {
-    import_path18 = __toESM(require("path"));
-    import_fs16 = __toESM(require("fs"));
-    import_readline6 = __toESM(require("readline"));
-    import_promises14 = __toESM(require("fs/promises"));
-    import_crypto7 = __toESM(require("crypto"));
+    import_path17 = __toESM(require("path"));
+    import_fs18 = __toESM(require("fs"));
+    import_promises16 = __toESM(require("fs/promises"));
+    import_crypto9 = __toESM(require("crypto"));
     init_src10();
     init_src10();
     init_src10();
@@ -53774,11 +54974,11 @@ var init_ConsumerExecutor = __esm({
     ConsumerExecutorClass = class {
       constructor() {
         this._ensurePath = (pathUri) => {
-          const dir = import_path18.default.dirname(pathUri);
-          if (!import_fs16.default.existsSync(dir))
-            import_fs16.default.mkdirSync(dir, { recursive: true });
-          if (!import_fs16.default.existsSync(pathUri))
-            import_fs16.default.writeFileSync(pathUri, "");
+          const dir = import_path17.default.dirname(pathUri);
+          if (!import_fs18.default.existsSync(dir))
+            import_fs18.default.mkdirSync(dir, { recursive: true });
+          if (!import_fs18.default.existsSync(pathUri))
+            import_fs18.default.writeFileSync(pathUri, "");
         };
         /**
          * Shapes one parsed producer record into one consumer record, following a plan compiled once per
@@ -53899,9 +55099,8 @@ var init_ConsumerExecutor = __esm({
           if (lineCountFields.length === 0)
             return 0;
           const tempWorkPath = datasetPath + "_tmp";
-          const reader = import_fs16.default.createReadStream(datasetPath);
-          const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
-          const writer = import_fs16.default.createWriteStream(tempWorkPath);
+          const reader = import_fs18.default.createReadStream(datasetPath);
+          const writer = import_fs18.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
           let workerIndex = 0;
           let currentWorker = executorResults[workerIndex] ?? null;
@@ -53915,7 +55114,7 @@ var init_ConsumerExecutor = __esm({
             }
           };
           advanceWorker();
-          for await (const line of lineReader) {
+          for await (const line of readLines(reader)) {
             advanceWorker();
             Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
             const rowOffset = currentWorker.rowOffset ?? 0;
@@ -53934,7 +55133,6 @@ var init_ConsumerExecutor = __esm({
             remainingLinesInWorker--;
             processedLineCount++;
           }
-          lineReader.close();
           await new Promise((resolve, reject) => {
             writer.on("close", resolve);
             writer.on("error", reject);
@@ -53946,8 +55144,8 @@ var init_ConsumerExecutor = __esm({
               reader.destroy();
             });
           }
-          await import_promises14.default.unlink(datasetPath);
-          await import_promises14.default.rename(tempWorkPath, datasetPath);
+          await import_promises16.default.unlink(datasetPath);
+          await import_promises16.default.rename(tempWorkPath, datasetPath);
           return processedLineCount;
         };
         /**
@@ -53965,7 +55163,7 @@ var init_ConsumerExecutor = __esm({
         this.processLimit = async (datasetPath, limit) => {
           Affirm_default(datasetPath, "Invalid dataset path");
           Affirm_default(limit > 0, `Invalid record limit ${limit}`);
-          const handle = await import_promises14.default.open(datasetPath, "r+");
+          const handle = await import_promises16.default.open(datasetPath, "r+");
           try {
             const buffer = Buffer.allocUnsafe(LIMIT_SCAN_BYTES);
             let newlines = 0;
@@ -53991,15 +55189,14 @@ var init_ConsumerExecutor = __esm({
           }
         };
         this.processDistinct = async (datasetPath) => {
-          const reader = import_fs16.default.createReadStream(datasetPath);
-          const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+          const reader = import_fs18.default.createReadStream(datasetPath);
           const tempWorkPath = datasetPath + "_tmp";
-          const writer = import_fs16.default.createWriteStream(tempWorkPath);
+          const writer = import_fs18.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
           let newLineCount = 0;
           const seen = /* @__PURE__ */ new Set();
-          for await (const line of lineReader) {
-            const hash = import_crypto7.default.createHash("sha1").update(line).digest("base64");
+          for await (const line of readLines(reader)) {
+            const hash = import_crypto9.default.createHash("sha1").update(line).digest("base64");
             if (!seen.has(hash)) {
               seen.add(hash);
               if (!writer.write(line + "\n"))
@@ -54007,7 +55204,6 @@ var init_ConsumerExecutor = __esm({
               newLineCount++;
             }
           }
-          lineReader.close();
           await new Promise((resolve, reject) => {
             writer.on("close", resolve);
             writer.on("error", reject);
@@ -54019,20 +55215,19 @@ var init_ConsumerExecutor = __esm({
               reader.destroy();
             });
           }
-          await import_promises14.default.unlink(datasetPath);
-          await import_promises14.default.rename(tempWorkPath, datasetPath);
+          await import_promises16.default.unlink(datasetPath);
+          await import_promises16.default.rename(tempWorkPath, datasetPath);
           return newLineCount;
         };
         this.processDistinctOn = async (consumer, datasetPath) => {
-          const reader = import_fs16.default.createReadStream(datasetPath);
-          const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+          const reader = import_fs18.default.createReadStream(datasetPath);
           const { distinctOn } = consumer.options;
           const { keys, resolution } = distinctOn;
           const { strategy, orderBy, direction = "asc" } = resolution;
           const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
           const internalFields = ConsumerManager_default.getExpandedFields(consumer);
           const winners = /* @__PURE__ */ new Map();
-          for await (const line of lineReader) {
+          for await (const line of readLines(reader)) {
             const record = internalRecordFormat === "CSV" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
             const compositeKey = keys.map((k7) => String(record[k7] ?? "")).join("|");
             const existing = winners.get(compositeKey);
@@ -54045,9 +55240,8 @@ var init_ConsumerExecutor = __esm({
               winners.set(compositeKey, { record, line });
             }
           }
-          lineReader.close();
           const tempWorkPath = datasetPath + "_tmp";
-          const writer = import_fs16.default.createWriteStream(tempWorkPath);
+          const writer = import_fs18.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
           for (const { line } of winners.values()) {
             if (!writer.write(line + "\n"))
@@ -54064,8 +55258,8 @@ var init_ConsumerExecutor = __esm({
               reader.destroy();
             });
           }
-          await import_promises14.default.unlink(datasetPath);
-          await import_promises14.default.rename(tempWorkPath, datasetPath);
+          await import_promises16.default.unlink(datasetPath);
+          await import_promises16.default.rename(tempWorkPath, datasetPath);
           return winners.size;
         };
         this.processPivot = async (consumer, datasetPath) => {
@@ -54077,9 +55271,8 @@ var init_ConsumerExecutor = __esm({
           if (!pivotValues) {
             pivotValues = [];
             const discoverySet = /* @__PURE__ */ new Set();
-            const discoverReader = import_fs16.default.createReadStream(datasetPath);
-            const discoverLineReader = import_readline6.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
-            for await (const line of discoverLineReader) {
+            const discoverReader = import_fs18.default.createReadStream(datasetPath);
+            for await (const line of readLines(discoverReader)) {
               const record = this._parseLine(line, internalRecordFormat, internalFields);
               const val = String(record[pivotColumn] ?? "");
               if (!discoverySet.has(val)) {
@@ -54087,7 +55280,6 @@ var init_ConsumerExecutor = __esm({
                 pivotValues.push(val);
               }
             }
-            discoverLineReader.close();
             if (!discoverReader.destroyed) {
               await new Promise((resolve) => {
                 discoverReader.once("close", resolve);
@@ -54096,9 +55288,8 @@ var init_ConsumerExecutor = __esm({
             }
           }
           const groups = /* @__PURE__ */ new Map();
-          const reader = import_fs16.default.createReadStream(datasetPath);
-          const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
-          for await (const line of lineReader) {
+          const reader = import_fs18.default.createReadStream(datasetPath);
+          for await (const line of readLines(reader)) {
             const record = this._parseLine(line, internalRecordFormat, internalFields);
             const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
             const pivotVal = String(record[pivotColumn] ?? "");
@@ -54118,14 +55309,13 @@ var init_ConsumerExecutor = __esm({
             cell.min = Math.min(cell.min, numericVal);
             cell.max = Math.max(cell.max, numericVal);
           }
-          lineReader.close();
           const pivotedFields = [
             ...rowKeys.map((k7) => ({ cField: { key: k7 }, finalKey: k7 })),
             ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
           ];
           const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedFields);
           const tempWorkPath = datasetPath + "_tmp";
-          const writer = import_fs16.default.createWriteStream(tempWorkPath);
+          const writer = import_fs18.default.createWriteStream(tempWorkPath);
           const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
           let outputCount = 0;
           for (const { rowRecord, cells } of groups.values()) {
@@ -54171,8 +55361,8 @@ var init_ConsumerExecutor = __esm({
               reader.destroy();
             });
           }
-          await import_promises14.default.unlink(datasetPath);
-          await import_promises14.default.rename(tempWorkPath, datasetPath);
+          await import_promises16.default.unlink(datasetPath);
+          await import_promises16.default.rename(tempWorkPath, datasetPath);
           return outputCount;
         };
         this._parseLine = (line, format4, fields) => {
@@ -54218,9 +55408,8 @@ var init_ConsumerExecutor = __esm({
           for (const fieldKey of uniqueFieldKeys) {
             fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
           }
-          const reader = import_fs16.default.createReadStream(datasetPath);
-          const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
-          for await (const line of lineReader) {
+          const reader = import_fs18.default.createReadStream(datasetPath);
+          for await (const line of readLines(reader)) {
             rowCount++;
             if (checkDuplicateRows) {
               if (seenRows.has(line))
@@ -54242,7 +55431,6 @@ var init_ConsumerExecutor = __esm({
               }
             }
           }
-          lineReader.close();
           return DataValidationEngine_default.evaluateDatasetValidations(validations, { rowCount, hasDuplicateRows, duplicateFields });
         };
         /**
@@ -54268,7 +55456,7 @@ var init_ConsumerExecutor = __esm({
 });
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path19, ProducerExecutorClass, ProducerExecutor, ProducerExecutor_default;
+var import_path18, ProducerExecutorClass, ProducerExecutor, ProducerExecutor_default;
 var init_ProducerExecutor = __esm({
   "../../packages/executors/src/ProducerExecutor.ts"() {
     init_src();
@@ -54276,7 +55464,7 @@ var init_ProducerExecutor = __esm({
     init_src4();
     init_src10();
     init_src10();
-    import_path19 = __toESM(require("path"));
+    import_path18 = __toESM(require("path"));
     init_src2();
     ProducerExecutorClass = class {
       constructor() {
@@ -54311,7 +55499,7 @@ var init_ProducerExecutor = __esm({
         this.compileLine = (dimensions, chunk) => {
           Affirm_default(dimensions, "Invalid producer dimensions");
           Affirm_default(chunk, "Invalid chunk");
-          return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename: import_path19.default.basename(chunk.fileUri) };
+          return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename: import_path18.default.basename(chunk.fileUri) };
         };
         /**
          * The record(s) one source line produces. Almost always exactly one — a producer with a group
@@ -54367,102 +55555,224 @@ var init_ExecutorPerformance = __esm({
 });
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs22, import_readline8, ExecutorWriter, ExecutorWriter_default;
+var fs23, ExecutorWriter, ExecutorWriter_default;
 var init_ExecutorWriter = __esm({
   "../../packages/executors/src/ExecutorWriter.ts"() {
-    fs22 = __toESM(require("fs"));
-    import_readline8 = __toESM(require("readline"));
+    fs23 = __toESM(require("fs"));
+    init_src();
+    init_src3();
+    init_src10();
+    init_OutputExecutor();
     ExecutorWriter = class {
       constructor() {
         /**
-         * Magic number used to retrive value in gigabytes
-         */
-        this.FAKE_GB = 1e3 * 1e3 * 1e3;
-        /**
-         * Retrive the NumberedPath for multifile purpose example: ./main1.dataset
-         */
-        this.getCompletedPath = (startingPath, count) => {
-          const finalPath = startingPath.substring(0, startingPath.length - 8).concat("_", count.toString(), ".dataset");
-          return finalPath;
-        };
-        this.splitBySize = async (scope, sourcePath) => {
-          const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-          const readStream = fs22.createReadStream(sourcePath);
-          const reader = import_readline8.default.createInterface({ input: readStream, crlfDelay: Infinity });
-          let writerIndex = 0;
-          let destPath = this.getCompletedPath(sourcePath, writerIndex);
-          let writeStream = fs22.createWriteStream(destPath, { flags: "a" });
-          const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
-          for await (const line of reader) {
-            if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
-              writeStream.end();
-              await new Promise((resolve, reject) => {
-                writeStream.on("finish", resolve);
-                writeStream.on("error", reject);
-              });
-              writerIndex++;
-              destPath = this.getCompletedPath(sourcePath, writerIndex);
-              writeStream = fs22.createWriteStream(destPath, { flags: "a" });
+         * Gigabytes as the configuration means them — 10^9, not 2^30. `maximumFileSize` has always been
+         * read this way and the two differ by 7%, so it stays.
+         */
+        this.BYTES_PER_GB = 1e3 * 1e3 * 1e3;
+        /** The label an unsplit output carries, kept at "0" so exported names do not change. */
+        this.UNSPLIT_LABEL = "0";
+        /** The path of one split file: `main.dataset` plus a label, e.g. `main_italy_1.dataset`. */
+        this.getCompletedPath = (startingPath, label) => {
+          return startingPath.substring(0, startingPath.length - ".dataset".length).concat("_", label, ".dataset");
+        };
+        /**
+         * Routes the merged dataset into one file per bucket, in a single pass.
+         *
+         * Splitting by key and splitting by size are the same operation — decide a line's bucket, append
+         * to that bucket's current file, start the next one when it is full — so one pass covers all
+         * three configurations: a size cap alone (one bucket that rotates), key(s) alone (a bucket each,
+         * never rotating), or both (a bucket each, each rotating on its own counter).
+         *
+         * Returns the files it produced, so the export names them from their label rather than
+         * rediscovering them in the scope folder. With nothing to split the merged dataset is handed
+         * back as-is and never read.
+         */
+        this.splitOutput = async (request) => {
+          Affirm_default(request, "Invalid split request");
+          Affirm_default(request.sourcePath, "Missing dataset to split");
+          Affirm_default(request.consumer, "Invalid consumer");
+          const { scope, sourcePath, consumer, fields } = request;
+          const keys = scope.split?.by ?? [];
+          const maxFileBytes = scope.split?.maxFileSizeGb > 0 ? scope.split.maxFileSizeGb * this.BYTES_PER_GB : null;
+          if (keys.length === 0 && !maxFileBytes)
+            return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
+          const pass = {
+            consumer,
+            sourcePath,
+            keys,
+            maxFileBytes,
+            buckets: /* @__PURE__ */ new Map(),
+            openStreams: [],
+            usedLabels: /* @__PURE__ */ new Set(),
+            fileCount: 0
+          };
+          await this._route(pass, fields);
+          if (pass.buckets.size === 0)
+            return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
+          await fs23.promises.unlink(sourcePath);
+          return this._listFiles(pass);
+        };
+        /**
+         * The pass itself: one line read, one line written, for every line of the dataset.
+         */
+        this._route = async (pass, fields) => {
+          const internalFormat = OutputExecutor_default._getInternalRecordFormat(pass.consumer);
+          const readStream = fs23.createReadStream(pass.sourcePath);
+          try {
+            for await (const line of readLines(readStream)) {
+              const bucket = this._bucketFor(line, pass, internalFormat, fields);
+              const lineBytes = Buffer.byteLength(line) + 1;
+              if (this._needsRotation(bucket, pass, lineBytes))
+                await this._rotate(bucket, pass);
+              const stream = await this._acquireStream(bucket, pass);
+              if (!stream.write(line + "\n"))
+                await new Promise((resolve) => stream.once("drain", resolve));
+              bucket.bytesWritten += lineBytes;
             }
-            if (!writeStream.write(line + "\n"))
-              await waitForDrain();
+          } finally {
+            while (pass.openStreams.length > 0)
+              await this._closeStream(pass.openStreams.shift());
+            readStream.destroy();
           }
-          writeStream.end();
+        };
+        /**
+         * The bucket a line belongs to, created on first sight of its key. Null and empty each get their
+         * own: a shared catch-all would silently merge two different things, and that is invisible until
+         * it is wrong.
+         */
+        this._bucketFor = (line, pass, internalFormat, fields) => {
+          const bucketKey = pass.keys.length === 0 ? "" : this._bucketKey(line, pass.keys, internalFormat, fields);
+          const existing = pass.buckets.get(bucketKey);
+          if (existing) return existing;
+          const bucket = {
+            label: pass.keys.length === 0 ? "" : this._uniqueLabel(bucketKey, pass.usedLabels),
+            stream: null,
+            bytesWritten: 0,
+            fileIndex: 0
+          };
+          pass.buckets.set(bucketKey, bucket);
+          this._countFile(pass);
+          return bucket;
+        };
+        /** The composite value a line buckets on, read off the parsed record. */
+        this._bucketKey = (line, keys, internalFormat, fields) => {
+          const record = internalFormat === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
+          return keys.map((key) => {
+            const value = record[key];
+            if (value === null || value === void 0) return "_null";
+            const text = String(value);
+            return text.length === 0 ? "_empty" : text;
+          }).join("|");
+        };
+        /**
+         * A key value turned into a file name segment. Anything that is not a letter, a digit, an
+         * underscore or a dash becomes an underscore — dots included, since the export composes names
+         * around the extension — and the result is capped so a long value cannot overrun the path limit.
+         *
+         * Two different values can sanitize to the same segment ("a/b" and "a b"), so a collision takes
+         * a numeric suffix: buckets are keyed by the raw value and must never merge into one file.
+         */
+        this._uniqueLabel = (bucketKey, usedLabels) => {
+          const base = bucketKey.replace(/[^\p{L}\p{N}_-]/gu, "_").substring(0, 64) || "_empty";
+          let label = base;
+          let collision = 1;
+          while (usedLabels.has(label)) {
+            label = `${base}_${collision}`;
+            collision++;
+          }
+          usedLabels.add(label);
+          return label;
+        };
+        /**
+         * Whether this line fills the bucket's current file. Asked BEFORE the line is written, so the
+         * cap is a cap and not a threshold the last line of every file steps over. A line larger than
+         * the cap on its own still goes out whole: rotating an empty file only produces an empty one.
+         */
+        this._needsRotation = (bucket, pass, lineBytes) => {
+          if (!pass.maxFileBytes || bucket.bytesWritten === 0) return false;
+          return bucket.bytesWritten + lineBytes > pass.maxFileBytes;
+        };
+        /** Closes the bucket's full file and points it at the next one. */
+        this._rotate = async (bucket, pass) => {
+          await this._releaseStream(bucket, pass.openStreams);
+          bucket.fileIndex++;
+          bucket.bytesWritten = 0;
+          this._countFile(pass);
+        };
+        /** Counts a file as it is opened, so the bound fails the run before the descriptors run out. */
+        this._countFile = (pass) => {
+          pass.fileCount++;
+          Affirm_default(
+            pass.fileCount <= Constants_default.defaults.MAX_SPLIT_FILES,
+            `Consumer "${pass.consumer.name}" splits its output into more than ${Constants_default.defaults.MAX_SPLIT_FILES} files${pass.keys.length ? ` \u2014 "${pass.keys.join(", ")}" has too many distinct values to split on` : ""}.`
+          );
+        };
+        /**
+         * The bucket's open stream, opening it if the LRU has evicted it. Append mode, always: an
+         * evicted bucket reopens onto the file it was already writing, and its byte counter lives in the
+         * bucket rather than the stream so the rotation point survives the eviction.
+         */
+        this._acquireStream = async (bucket, pass) => {
+          const position = pass.openStreams.indexOf(bucket);
+          if (position >= 0) {
+            pass.openStreams.splice(position, 1);
+            pass.openStreams.push(bucket);
+            return bucket.stream;
+          }
+          if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
+            await this._closeStream(pass.openStreams.shift());
+          bucket.stream = fs23.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
+          pass.openStreams.push(bucket);
+          return bucket.stream;
+        };
+        /**
+         * Closes a bucket's stream and takes it out of the open set, so the set only ever holds buckets
+         * whose stream is live — what lets `_acquireStream` read "in the set" as "already open".
+         */
+        this._releaseStream = async (bucket, openStreams) => {
+          const position = openStreams.indexOf(bucket);
+          if (position >= 0)
+            openStreams.splice(position, 1);
+          await this._closeStream(bucket);
+        };
+        /** Flushes and releases a bucket's descriptor. Awaited: a reopen must not race the flush. */
+        this._closeStream = async (bucket) => {
+          if (!bucket?.stream) return;
+          const stream = bucket.stream;
+          bucket.stream = null;
+          stream.end();
           await new Promise((resolve, reject) => {
-            writeStream.on("finish", resolve);
-            writeStream.on("error", reject);
+            stream.on("finish", resolve);
+            stream.on("error", reject);
           });
-          await fs22.promises.unlink(sourcePath);
-        };
-        /**
-         * Manage the Writestream for main.dataset
-         */
-        // write = async (reader: fs.ReadStream) => {
-        //     if (this._writer) {
-        //         if (this._writer.closed) {
-        //             this._writer = createWriteStream(this._mainPath)
-        //             console.log('new stream')
-        //         }
-        //     } else {
-        //         this._writer = createWriteStream(this._mainPath)
-        //     }
-        //     // Convert to array of lines
-        //     const lines = readline.createInterface({ input: reader, crlfDelay: Infinity })
-        //     for await (let line of lines) {
-        //         line = line.concat('\n')
-        //         if (this._writer) {
-        //             if (this._limitSizeByte) {
-        //                 if (this._bytesWrote + line.length >= this._limitSizeByte) {
-        //                     this._writer.end()
-        //                     this._writer.close()
-        //                     this._currentFileId++
-        //                     const newPathMain = this.getCompletedPath(this._mainPath)
-        //                     this._writer = createWriteStream(newPathMain)
-        //                     // reset the counter
-        //                     this._bytesWrote = line.length
-        //                     this._writer.write(line)
-        //                 } else {
-        //                     this._writer.write(line)
-        //                     // add to the counter the bytes wrote
-        //                     this._bytesWrote = this._bytesWrote + line.length
-        //                 }
-        //             } else {
-        //                 this._writer.write(line)
-        //             }
-        //         } else {
-        //             // add a new file and write
-        //             this._writer = createWriteStream(this._mainPath)
-        //             this._writer.write(line)
-        //         }
-        //     }
-        //     return
-        // }
-        /**
-         * Terminate the last WriteStream.
-         */
-        this.end = () => {
-          this._writer.end();
-          this._writer.close();
+        };
+        /**
+         * The label of one of a bucket's files: the sanitized key value, the rotation index, or both
+         * joined with an underscore — so a size split counts "0", "1", "2", a key split reads "italy",
+         * and the two together read "italy_0", "italy_1".
+         */
+        this._composeLabel = (bucket, pass, fileIndex) => {
+          const segments = [
+            pass.keys.length > 0 ? bucket.label : null,
+            pass.maxFileBytes ? String(fileIndex ?? bucket.fileIndex) : null
+          ];
+          return segments.filter((x2) => x2 !== null).join("_");
+        };
+        /**
+         * Every file the pass wrote, in the order the buckets were first seen and then by rotation.
+         * Derived from the counters rather than read back off the disk, so the export never depends on
+         * directory ordering.
+         */
+        this._listFiles = (pass) => {
+          const files = [];
+          for (const bucket of pass.buckets.values()) {
+            for (let fileIndex = 0; fileIndex <= bucket.fileIndex; fileIndex++) {
+              const label = this._composeLabel(bucket, pass, fileIndex);
+              files.push({ path: this.getCompletedPath(pass.sourcePath, label), label });
+            }
+          }
+          return files;
         };
       }
     };
@@ -54519,14 +55829,14 @@ var init_ExecutorProgress2 = __esm({
 });
 
 // ../../packages/executors/src/OrchestratorHelper.ts
-var import_os2, import_path20, OrchestratorHelper, OrchestratorHelper_default;
+var import_os2, import_path19, OrchestratorHelper, OrchestratorHelper_default;
 var init_OrchestratorHelper = __esm({
   "../../packages/executors/src/OrchestratorHelper.ts"() {
     init_src();
     init_src6();
     import_os2 = __toESM(require("os"));
     init_src4();
-    import_path20 = __toESM(require("path"));
+    import_path19 = __toESM(require("path"));
     init_src3();
     OrchestratorHelper = {
       getMemoryUsage: () => {
@@ -54618,14 +55928,14 @@ var init_OrchestratorHelper = __esm({
       getPhysicalWorkerPath: () => {
         const currentDir = __dirname;
         if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-          return import_path20.default.resolve("./.build/workers");
+          return import_path19.default.resolve("./.build/workers");
         const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
         if (forcedPath && forcedPath.length > 0)
-          return import_path20.default.join(__dirname, forcedPath);
+          return import_path19.default.join(__dirname, forcedPath);
         if (!currentDir.includes(".build")) {
-          return import_path20.default.join(__dirname, "../workers");
+          return import_path19.default.join(__dirname, "../workers");
         } else {
-          return import_path20.default.resolve("./.build/workers");
+          return import_path19.default.resolve("./.build/workers");
         }
       },
       /**
@@ -54660,6 +55970,29 @@ var init_OrchestratorHelper = __esm({
         const ceiling = Number(raw);
         Affirm_default(Number.isInteger(ceiling) && ceiling >= 1, `REMORA_MAX_WORKERS must be a whole number >= 1, got "${raw}"`);
         return ceiling;
+      },
+      /**
+       * The consumer's splitting, normalized for the run: `maximumFileSize` is the older spelling of
+       * `split.maxFileSizeGb` and is folded in here, so nothing downstream of the scope has to know
+       * there are two. Setting both is rejected at validation, not silently preferred.
+       *
+       * Answers undefined when the consumer asks for no splitting, which is what makes the split pass
+       * a no-op rather than a decision the orchestrator has to take.
+       */
+      resolveSplit: (consumer) => {
+        Affirm_default(consumer, "Invalid consumer");
+        const by = consumer.split?.by ?? [];
+        const maxFileSizeGb = consumer.split?.maxFileSizeGb ?? consumer.maximumFileSize;
+        if (by.length === 0 && !(maxFileSizeGb > 0))
+          return void 0;
+        return { by: by.length > 0 ? by : void 0, maxFileSizeGb: maxFileSizeGb > 0 ? maxFileSizeGb : void 0 };
+      },
+      /** How a run's splitting reads in the log: the keys, the cap, or both. */
+      formatSplit: (split2) => {
+        return [
+          split2.by?.length ? `by ${split2.by.join(", ")}` : null,
+          split2.maxFileSizeGb ? `max ${split2.maxFileSizeGb}Gb per file` : null
+        ].filter((x2) => !!x2).join(", ");
       }
     };
     OrchestratorHelper_default = OrchestratorHelper;
@@ -54667,13 +56000,12 @@ var init_OrchestratorHelper = __esm({
 });
 
 // ../../packages/executors/src/LookupResolver.ts
-var import_fs18, import_promises15, import_path21, import_readline9, LookupResolverClass, LookupResolver, LookupResolver_default;
+var import_fs20, import_promises17, import_path20, LookupResolverClass, LookupResolver, LookupResolver_default;
 var init_LookupResolver = __esm({
   "../../packages/executors/src/LookupResolver.ts"() {
-    import_fs18 = __toESM(require("fs"));
-    import_promises15 = __toESM(require("fs/promises"));
-    import_path21 = __toESM(require("path"));
-    import_readline9 = __toESM(require("readline"));
+    import_fs20 = __toESM(require("fs"));
+    import_promises17 = __toESM(require("fs/promises"));
+    import_path20 = __toESM(require("path"));
     init_src();
     init_src3();
     init_src4();
@@ -54712,11 +56044,11 @@ var init_LookupResolver = __esm({
               this.cache.set(id, await this.buildDict(lookupConsumer, capturePath, ref));
             }
             ExecutorScope_default.ensurePath(outPath);
-            await import_promises15.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
-            manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path21.default.resolve(outPath) });
+            await import_promises17.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
+            manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path20.default.resolve(outPath) });
           }
           for (const capturePath of captureByConsumer.values())
-            await import_promises15.default.rm(capturePath, { force: true });
+            await import_promises17.default.rm(capturePath, { force: true });
           return manifest;
         };
         // Runs the source consumer through the real pipeline, capturing its merged output before cleanup.
@@ -54742,8 +56074,7 @@ var init_LookupResolver = __esm({
           const dict = {};
           let rowsRead = 0;
           let warnedDup = false;
-          const lineStream = import_readline9.default.createInterface({ input: import_fs18.default.createReadStream(capturePath), crlfDelay: Infinity });
-          for await (const line of lineStream) {
+          for await (const line of readLines(import_fs20.default.createReadStream(capturePath))) {
             if (!line.length) continue;
             rowsRead++;
             Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
@@ -54763,8 +56094,8 @@ var init_LookupResolver = __esm({
           return JSON.parse(line);
         };
         this.getCapturePath = (scope, consumerName) => {
-          const dir = import_path21.default.dirname(ExecutorScope_default.getLookupPath(scope, "x"));
-          return import_path21.default.join(dir, `${consumerName}.capture`);
+          const dir = import_path20.default.dirname(ExecutorScope_default.getLookupPath(scope, "x"));
+          return import_path20.default.join(dir, `${consumerName}.capture`);
         };
         this.dedupe = (refs) => {
           const byId = /* @__PURE__ */ new Map();
@@ -54786,12 +56117,12 @@ var init_LookupResolver = __esm({
 });
 
 // ../../packages/executors/src/ConsumerInputResolver.ts
-var import_fs19, import_promises16, import_path22, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
+var import_fs21, import_promises18, import_path21, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
 var init_ConsumerInputResolver = __esm({
   "../../packages/executors/src/ConsumerInputResolver.ts"() {
-    import_fs19 = __toESM(require("fs"));
-    import_promises16 = __toESM(require("fs/promises"));
-    import_path22 = __toESM(require("path"));
+    import_fs21 = __toESM(require("fs"));
+    import_promises18 = __toESM(require("fs/promises"));
+    import_path21 = __toESM(require("path"));
     init_src();
     init_src3();
     init_src4();
@@ -54814,15 +56145,15 @@ var init_ConsumerInputResolver = __esm({
           Affirm_default(stack.length <= Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH, `consumer-as-input nesting exceeds MAX_CONSUMER_INPUT_DEPTH (${Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH}): ${stack.join(" -> ")}`);
           for (const name of upstreamNames) {
             const cached = this.cache.get(name);
-            if (!cached || !import_fs19.default.existsSync(cached.path)) {
+            if (!cached || !import_fs21.default.existsSync(cached.path)) {
               Affirm_default(!stack.includes(name), `consumer-as-input cycle detected: ${[...stack, name].join(" -> ")}`);
               const upstream = Environment_default.getConsumer(name);
               Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
               const capturePath = ExecutorScope_default.getConsumerInputPath(scope, name);
               ExecutorScope_default.ensurePath(capturePath);
               await this.runUpstreamConsumer(upstream, capturePath, stack, request);
-              const { size } = await import_promises16.default.stat(capturePath);
-              this.cache.set(name, { consumer: name, path: import_path22.default.resolve(capturePath), fileSize: size });
+              const { size } = await import_promises18.default.stat(capturePath);
+              this.cache.set(name, { consumer: name, path: import_path21.default.resolve(capturePath), fileSize: size });
             }
             manifest.set(name, this.cache.get(name));
           }
@@ -54884,13 +56215,12 @@ var ExecutorOrchestrator_exports = {};
 __export(ExecutorOrchestrator_exports, {
   default: () => ExecutorOrchestrator_default
 });
-var import_fs20, import_promises17, import_path23, import_readline10, import_workerpool, import_promises18, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
+var import_fs22, import_promises19, import_path22, import_workerpool, import_promises20, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
 var init_ExecutorOrchestrator = __esm({
   "../../packages/executors/src/ExecutorOrchestrator.ts"() {
-    import_fs20 = __toESM(require("fs"));
-    import_promises17 = __toESM(require("fs/promises"));
-    import_path23 = __toESM(require("path"));
-    import_readline10 = __toESM(require("readline"));
+    import_fs22 = __toESM(require("fs"));
+    import_promises19 = __toESM(require("fs/promises"));
+    import_path22 = __toESM(require("path"));
     import_workerpool = __toESM(require("workerpool"));
     init_src();
     init_src10();
@@ -54906,7 +56236,7 @@ var init_ExecutorOrchestrator = __esm({
     init_src();
     init_src10();
     init_ExecutorWriter();
-    import_promises18 = require("stream/promises");
+    import_promises20 = require("stream/promises");
     init_src2();
     init_ExecutorProgress2();
     init_src5();
@@ -54945,7 +56275,7 @@ var init_ExecutorOrchestrator = __esm({
           };
           const workerPath = OrchestratorHelper_default.getPhysicalWorkerPath();
           Logger_default.info(`Initializing worker pool from ${workerPath} (workers: ${maxWorkers}, heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
-          return import_workerpool.default.pool(import_path23.default.join(workerPath, "ExecutorWorker.js"), options2);
+          return import_workerpool.default.pool(import_path22.default.join(workerPath, "ExecutorWorker.js"), options2);
         };
         this.launch = async (request) => {
           Affirm_default(request, "Invalid options");
@@ -54957,7 +56287,7 @@ var init_ExecutorOrchestrator = __esm({
           const tracker = new ExecutorPerformance_default();
           const _progress = new ExecutorProgress2_default(logProgress);
           const { usageId } = UsageManager_default.startUsage(consumer, details);
-          const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
+          const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], split: OrchestratorHelper_default.resolveSplit(consumer) };
           const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
           const census = { created: 0, peak: 0 };
           try {
@@ -55087,20 +56417,23 @@ var init_ExecutorOrchestrator = __esm({
               await Logger_default.flush();
               return materializeResult;
             }
-            if (scope.limitFileSize) {
-              Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
-              counter = performance.now();
-              const writer = new ExecutorWriter_default();
-              await writer.splitBySize(scope, ExecutorScope_default.getMainPath(scope));
-              tracker.measure("split-by-size", performance.now() - counter);
-              Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
-            }
+            counter = performance.now();
+            const splitFiles = await new ExecutorWriter_default().splitOutput({
+              scope,
+              sourcePath: ExecutorScope_default.getMainPath(scope),
+              consumer,
+              fields: expandedFields
+            });
+            tracker.measure("split-output", performance.now() - counter);
+            if (scope.split)
+              Logger_default.log(`[${usageId}] Split output into ${splitFiles.length} file(s) (${OrchestratorHelper_default.formatSplit(scope.split)}) in ${Math.round(performance.now() - counter)}ms`);
             _progress.update({ phase: "Post-processing", progress: 1 });
             _progress.update({ phase: "Exporting result", progress: 0 });
             counter = performance.now();
             Logger_default.info(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-            const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, scope);
+            const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, splitFiles);
             tracker.measure("export-result", performance.now() - counter);
+            Affirm_default(exportRes.res, `Consumer "${consumer.name}" failed to write its output to "${consumer.outputs.map((x2) => x2.exportDestination).join(", ")}" \u2014 see the logged driver error for the cause`);
             Logger_default.info(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
             _progress.update({ phase: "Exporting result", progress: 1 });
             if (consumer.outputs.some((x2) => x2.onSuccess)) {
@@ -55146,7 +56479,7 @@ var init_ExecutorOrchestrator = __esm({
               Logger_default.info(`[${usageId}] Producer "${prod.name}": source returned no files, skipping`);
               continue;
             }
-            if (!import_fs20.default.existsSync(response.files[0].fullUri)) {
+            if (!import_fs22.default.existsSync(response.files[0].fullUri)) {
               if (!cProd.isOptional)
                 throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
               Logger_default.info(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
@@ -55308,7 +56641,7 @@ var init_ExecutorOrchestrator = __esm({
          * Returns a single chunk for small files where parallelism overhead isn't worth it.
          */
         this.scopeWork = (fileUri, numChunks) => {
-          const fileSize = import_fs20.default.statSync(fileUri).size;
+          const fileSize = import_fs22.default.statSync(fileUri).size;
           if (fileSize === 0) return [];
           if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
             return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
@@ -55318,7 +56651,7 @@ var init_ExecutorOrchestrator = __esm({
           const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
           if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
           const targetChunkSize = Math.floor(fileSize / effectiveChunks);
-          const fd = import_fs20.default.openSync(fileUri, "r");
+          const fd = import_fs22.default.openSync(fileUri, "r");
           try {
             const offsets = [];
             let currentStart = 0;
@@ -55336,7 +56669,7 @@ var init_ExecutorOrchestrator = __esm({
             }
             return offsets;
           } finally {
-            import_fs20.default.closeSync(fd);
+            import_fs22.default.closeSync(fd);
           }
         };
         /**
@@ -55404,7 +56737,7 @@ var init_ExecutorOrchestrator = __esm({
           let currentPos = position;
           while (currentPos < fileSize) {
             const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
-            const bytesRead = import_fs20.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
+            const bytesRead = import_fs22.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
             if (bytesRead === 0) break;
             for (let i6 = 0; i6 < bytesRead; i6++) {
               if (buffer[i6] === 10) {
@@ -55423,17 +56756,16 @@ var init_ExecutorOrchestrator = __esm({
          * format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
          */
         this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
-          await import_promises17.default.mkdir(import_path23.default.dirname(targetPath), { recursive: true });
+          await import_promises19.default.mkdir(import_path22.default.dirname(targetPath), { recursive: true });
           const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
           if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
-            await import_promises17.default.copyFile(sourcePath, targetPath);
+            await import_promises19.default.copyFile(sourcePath, targetPath);
             return;
           }
           const visibleFields = fields.filter((field) => !field.cField.hidden);
-          const input = import_readline10.default.createInterface({ input: import_fs20.default.createReadStream(sourcePath), crlfDelay: Infinity });
-          const output = import_fs20.default.createWriteStream(targetPath);
+          const output = import_fs22.default.createWriteStream(targetPath);
           try {
-            for await (const line of input) {
+            for await (const line of readLines(import_fs22.default.createReadStream(sourcePath))) {
               if (!line.length) continue;
               const record = LineParser_default._internalParseCSV(line, visibleFields, ",");
               if (!output.write(JSON.stringify(record) + "\n"))
@@ -55521,7 +56853,7 @@ var init_ExecutorOrchestrator = __esm({
             const decodedFiles = [];
             let decodedCount = 0;
             for (const file of response.files) {
-              const inferredType = fileType ?? import_path23.default.extname(file.fullUri).replace(".", "").toUpperCase();
+              const inferredType = fileType ?? import_path22.default.extname(file.fullUri).replace(".", "").toUpperCase();
               const decodedPath = `${file.fullUri}.decoded.csv`;
               if (inferredType === "XLS" || inferredType === "XLSX") {
                 const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
@@ -55529,22 +56861,22 @@ var init_ExecutorOrchestrator = __esm({
                   startRow: prod.settings.startRow,
                   startColumn: prod.settings.startColumn
                 });
-                await (0, import_promises18.pipeline)(
+                await (0, import_promises20.pipeline)(
                   xlsCsvStream,
-                  import_fs20.default.createWriteStream(decodedPath)
+                  import_fs22.default.createWriteStream(decodedPath)
                 );
-                const fileStats = await import_promises17.default.stat(decodedPath);
+                const fileStats = await import_promises19.default.stat(decodedPath);
                 decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
                 decodedCount++;
                 reportFile();
                 continue;
               }
               if (inferredType === "XML") {
-                const fileContent = await import_promises17.default.readFile(file.fullUri, "utf-8");
+                const fileContent = await import_promises19.default.readFile(file.fullUri, "utf-8");
                 const jsonData = XMLParser_default.xmlToJson(fileContent);
                 const records = normalizeXmlRows(jsonData);
                 if (records.length === 0) {
-                  await import_promises17.default.writeFile(decodedPath, "", "utf-8");
+                  await import_promises19.default.writeFile(decodedPath, "", "utf-8");
                 } else {
                   const columns = [];
                   for (const record of records) {
@@ -55560,9 +56892,9 @@ var init_ExecutorOrchestrator = __esm({
                     const row = columns.map((column) => csvSafeValue(record[column]));
                     lines.push(CSVParser_default.stringifyRow(row));
                   }
-                  await import_promises17.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+                  await import_promises19.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
                 }
-                const fileStats = await import_promises17.default.stat(decodedPath);
+                const fileStats = await import_promises19.default.stat(decodedPath);
                 decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
                 decodedCount++;
                 reportFile();
@@ -55590,20 +56922,20 @@ var init_ExecutorOrchestrator = __esm({
             Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
             const perf = performance.now();
             for (const workerResult of executorResults) {
-              await (0, import_promises18.pipeline)(
-                import_fs20.default.createReadStream(workerResult.resultUri),
-                import_fs20.default.createWriteStream(mainPath, { flags: "a" })
+              await (0, import_promises20.pipeline)(
+                import_fs22.default.createReadStream(workerResult.resultUri),
+                import_fs22.default.createWriteStream(mainPath, { flags: "a" })
               );
-              await import_promises17.default.unlink(workerResult.resultUri);
+              await import_promises19.default.unlink(workerResult.resultUri);
             }
             tracker.measure("merge-workers", performance.now() - perf);
             Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
           } else if (executorResults.length === 1) {
             Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
-            await import_promises17.default.rename(executorResults[0].resultUri, mainPath);
+            await import_promises19.default.rename(executorResults[0].resultUri, mainPath);
           } else {
             Logger_default.log(`[${scope.id}] No worker output \u2014 writing an empty dataset to ${mainPath}`);
-            await import_promises17.default.writeFile(mainPath, "");
+            await import_promises19.default.writeFile(mainPath, "");
           }
           return mainPath;
         };
@@ -55625,54 +56957,12 @@ var init_ExecutorOrchestrator = __esm({
   }
 });
 
-// ../../packages/executors/src/BatchWriter.ts
-init_src();
-var FLUSH_AFTER_CHARS = 1 << 18;
-var BatchWriter = class {
-  constructor(stream) {
-    /** How many times the stream's buffer filled and the caller had to wait for it. */
-    this.drains = 0;
-    this.buffer = "";
-    /** Queues one line and its newline. False means a batch went out and the stream wants a drain. */
-    this.write = (line) => {
-      this.buffer += line;
-      this.buffer += "\n";
-      if (this.buffer.length < FLUSH_AFTER_CHARS)
-        return true;
-      return this.flush();
-    };
-    this.waitForDrain = () => {
-      this.drains++;
-      return new Promise((resolve) => this.stream.once("drain", resolve));
-    };
-    /** Writes whatever is still buffered, then waits for the stream to finish. */
-    this.close = async () => {
-      this.flush();
-      await new Promise((resolve, reject) => {
-        this.stream.on("finish", resolve);
-        this.stream.on("error", reject);
-        this.stream.end();
-      });
-    };
-    this.flush = () => {
-      if (!this.buffer)
-        return true;
-      const batch = this.buffer;
-      this.buffer = "";
-      return this.stream.write(batch);
-    };
-    Affirm_default(stream, "Invalid write stream");
-    this.stream = stream;
-  }
-};
-var BatchWriter_default = BatchWriter;
-
 // ../../packages/executors/src/index.ts
+init_src();
 init_ConsumerExecutor();
 
 // ../../packages/executors/src/Executor.ts
-var import_fs17 = __toESM(require("fs"));
-var import_readline7 = __toESM(require("readline"));
+var import_fs19 = __toESM(require("fs"));
 init_ProducerExecutor();
 init_ConsumerExecutor();
 
@@ -55856,10 +57146,9 @@ var Executor = class {
       let processedBytes = 0;
       Logger_default.log(`[${workerId}] Opened streams \u2014 chunk ${start}-${end} (${Math.round(totalBytes / 1024)}KB), ${fields.length} field(s)`);
       Logger_default.log(`[${workerId}] Starting line-by-line processing`);
-      const lineStream = import_readline7.default.createInterface({ input: readStream, crlfDelay: Infinity });
       const isDebug = Logger_default._level === "debug";
       try {
-        for await (const line of lineStream) {
+        for await (const line of readLines(readStream)) {
           if (lineIndex === 0 && isFirstChunk) {
             if (!this.shouldProcessFirstLine(producer)) {
               lineIndex++;
@@ -55963,18 +57252,18 @@ var Executor = class {
       if (!lookups?.length) return;
       for (const entry of lookups) {
         if (LookupRegistry_default.has(entry.id)) continue;
-        const table = JSON.parse(import_fs17.default.readFileSync(entry.path, "utf-8"));
+        const table = JSON.parse(import_fs19.default.readFileSync(entry.path, "utf-8"));
         LookupRegistry_default.set(entry.id, table);
         Logger_default.log(`[lookup] Hydrated table "${entry.consumer}" (${entry.key}->${entry.value}) from ${entry.path}`);
       }
     };
     this.openReadStream = (chunk) => {
       const { end, fileUri, start } = chunk;
-      return import_fs17.default.createReadStream(fileUri, { start, end: end - 1 });
+      return import_fs19.default.createReadStream(fileUri, { start, end: end - 1 });
     };
     this.openWriteStream = (scope, workerId) => {
       const workerPath = ExecutorScope_default.getWorkerPath(scope, workerId);
-      return new BatchWriter_default(import_fs17.default.createWriteStream(workerPath));
+      return new BatchWriter_default(import_fs19.default.createWriteStream(workerPath));
     };
     // The first line of the first chunk is data unless the file type carries a header there.
     this.shouldProcessFirstLine = (producer) => {
@@ -56080,10 +57369,9 @@ init_OutputExecutor();
 init_ProducerExecutor();
 
 // ../../packages/executors/src/SampleRunner.ts
-var import_fs21 = __toESM(require("fs"));
-var import_promises19 = __toESM(require("fs/promises"));
-var import_path24 = __toESM(require("path"));
-var import_readline11 = __toESM(require("readline"));
+var import_fs23 = __toESM(require("fs"));
+var import_promises21 = __toESM(require("fs/promises"));
+var import_path23 = __toESM(require("path"));
 init_src();
 init_src3();
 init_src4();
@@ -56128,7 +57416,7 @@ var SampleRunnerClass = class {
      */
     this._runConsumer = async (consumer, size, details) => {
       this._assertStageable(consumer);
-      const capturePath = import_path24.default.resolve(import_path24.default.join(ExecutorScope_default.getBasePath(), `sample_${consumer.name}_${Helper_default.uuid()}.jsonl`));
+      const capturePath = import_path23.default.resolve(import_path23.default.join(ExecutorScope_default.getBasePath(), `sample_${consumer.name}_${Helper_default.uuid()}.jsonl`));
       ExecutorScope_default.ensurePath(capturePath);
       try {
         const result = await ExecutorOrchestrator_default.launch({
@@ -56155,7 +57443,7 @@ var SampleRunnerClass = class {
           note: `${records.length} row(s) from the real pipeline \u2014 filters, transformations, masking and validation all applied, exactly as in a run. Nothing was written to the destination.`
         };
       } finally {
-        await import_promises19.default.rm(capturePath, { force: true });
+        await import_promises21.default.rm(capturePath, { force: true });
       }
     };
     /**
@@ -56218,17 +57506,12 @@ var SampleRunnerClass = class {
     };
     /** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
     this._readCapture = async (capturePath, size) => {
-      if (!import_fs21.default.existsSync(capturePath)) return [];
+      if (!import_fs23.default.existsSync(capturePath)) return [];
       const records = [];
-      const lines = import_readline11.default.createInterface({ input: import_fs21.default.createReadStream(capturePath), crlfDelay: Infinity });
-      try {
-        for await (const line of lines) {
-          if (!line.length) continue;
-          records.push(JSON.parse(line));
-          if (records.length >= size) break;
-        }
-      } finally {
-        lines.close();
+      for await (const line of readLines(import_fs23.default.createReadStream(capturePath))) {
+        if (!line.length) continue;
+        records.push(JSON.parse(line));
+        if (records.length >= size) break;
       }
       return records;
     };