@forzalabs/remora 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1190,8 +1190,8 @@ var require_levels = __commonJS({
1190
1190
  var require_align = __commonJS({
1191
1191
  "../../node_modules/logform/align.js"(exports2, module2) {
1192
1192
  "use strict";
1193
- var format3 = require_format();
1194
- module2.exports = format3((info) => {
1193
+ var format4 = require_format();
1194
+ module2.exports = format4((info) => {
1195
1195
  info.message = ` ${info.message}`;
1196
1196
  return info;
1197
1197
  });
@@ -1202,9 +1202,9 @@ var require_align = __commonJS({
1202
1202
  var require_errors = __commonJS({
1203
1203
  "../../node_modules/logform/errors.js"(exports2, module2) {
1204
1204
  "use strict";
1205
- var format3 = require_format();
1205
+ var format4 = require_format();
1206
1206
  var { LEVEL, MESSAGE } = require_triple_beam();
1207
- module2.exports = format3((einfo, { stack, cause }) => {
1207
+ module2.exports = format4((einfo, { stack, cause }) => {
1208
1208
  if (einfo instanceof Error) {
1209
1209
  const info = Object.assign({}, einfo, {
1210
1210
  level: einfo.level,
@@ -1340,7 +1340,7 @@ var require_cli2 = __commonJS({
1340
1340
  var require_combine = __commonJS({
1341
1341
  "../../node_modules/logform/combine.js"(exports2, module2) {
1342
1342
  "use strict";
1343
- var format3 = require_format();
1343
+ var format4 = require_format();
1344
1344
  function cascade(formats) {
1345
1345
  if (!formats.every(isValidFormat)) {
1346
1346
  return;
@@ -1367,7 +1367,7 @@ var require_combine = __commonJS({
1367
1367
  return true;
1368
1368
  }
1369
1369
  module2.exports = (...formats) => {
1370
- const combinedFormat = format3(cascade(formats));
1370
+ const combinedFormat = format4(cascade(formats));
1371
1371
  const instance = combinedFormat();
1372
1372
  instance.Format = combinedFormat.Format;
1373
1373
  return instance;
@@ -1976,7 +1976,7 @@ ${originalIndentation}`;
1976
1976
  var require_json = __commonJS({
1977
1977
  "../../node_modules/logform/json.js"(exports2, module2) {
1978
1978
  "use strict";
1979
- var format3 = require_format();
1979
+ var format4 = require_format();
1980
1980
  var { MESSAGE } = require_triple_beam();
1981
1981
  var stringify = require_safe_stable_stringify();
1982
1982
  function replacer(key, value) {
@@ -1984,7 +1984,7 @@ var require_json = __commonJS({
1984
1984
  return value.toString();
1985
1985
  return value;
1986
1986
  }
1987
- module2.exports = format3((info, opts) => {
1987
+ module2.exports = format4((info, opts) => {
1988
1988
  const jsonStringify = stringify.configure(opts);
1989
1989
  info[MESSAGE] = jsonStringify(info, opts.replacer || replacer, opts.space);
1990
1990
  return info;
@@ -1996,8 +1996,8 @@ var require_json = __commonJS({
1996
1996
  var require_label = __commonJS({
1997
1997
  "../../node_modules/logform/label.js"(exports2, module2) {
1998
1998
  "use strict";
1999
- var format3 = require_format();
2000
- module2.exports = format3((info, opts) => {
1999
+ var format4 = require_format();
2000
+ module2.exports = format4((info, opts) => {
2001
2001
  if (opts.message) {
2002
2002
  info.message = `[${opts.label}] ${info.message}`;
2003
2003
  return info;
@@ -2012,10 +2012,10 @@ var require_label = __commonJS({
2012
2012
  var require_logstash = __commonJS({
2013
2013
  "../../node_modules/logform/logstash.js"(exports2, module2) {
2014
2014
  "use strict";
2015
- var format3 = require_format();
2015
+ var format4 = require_format();
2016
2016
  var { MESSAGE } = require_triple_beam();
2017
2017
  var jsonStringify = require_safe_stable_stringify();
2018
- module2.exports = format3((info) => {
2018
+ module2.exports = format4((info) => {
2019
2019
  const logstash = {};
2020
2020
  if (info.message) {
2021
2021
  logstash["@message"] = info.message;
@@ -2036,7 +2036,7 @@ var require_logstash = __commonJS({
2036
2036
  var require_metadata = __commonJS({
2037
2037
  "../../node_modules/logform/metadata.js"(exports2, module2) {
2038
2038
  "use strict";
2039
- var format3 = require_format();
2039
+ var format4 = require_format();
2040
2040
  function fillExcept(info, fillExceptKeys, metadataKey) {
2041
2041
  const savedKeys = fillExceptKeys.reduce((acc, key) => {
2042
2042
  acc[key] = info[key];
@@ -2061,7 +2061,7 @@ var require_metadata = __commonJS({
2061
2061
  }, {});
2062
2062
  return info;
2063
2063
  }
2064
- module2.exports = format3((info, opts = {}) => {
2064
+ module2.exports = format4((info, opts = {}) => {
2065
2065
  let metadataKey = "metadata";
2066
2066
  if (opts.key) {
2067
2067
  metadataKey = opts.key;
@@ -2205,9 +2205,9 @@ var require_ms = __commonJS({
2205
2205
  var require_ms2 = __commonJS({
2206
2206
  "../../node_modules/logform/ms.js"(exports2, module2) {
2207
2207
  "use strict";
2208
- var format3 = require_format();
2208
+ var format4 = require_format();
2209
2209
  var ms2 = require_ms();
2210
- module2.exports = format3((info) => {
2210
+ module2.exports = format4((info) => {
2211
2211
  const curr = +/* @__PURE__ */ new Date();
2212
2212
  exports2.diff = curr - (exports2.prevTime || curr);
2213
2213
  exports2.prevTime = curr;
@@ -2222,9 +2222,9 @@ var require_pretty_print = __commonJS({
2222
2222
  "../../node_modules/logform/pretty-print.js"(exports2, module2) {
2223
2223
  "use strict";
2224
2224
  var inspect = require("util").inspect;
2225
- var format3 = require_format();
2225
+ var format4 = require_format();
2226
2226
  var { LEVEL, MESSAGE, SPLAT } = require_triple_beam();
2227
- module2.exports = format3((info, opts = {}) => {
2227
+ module2.exports = format4((info, opts = {}) => {
2228
2228
  const stripped = Object.assign({}, info);
2229
2229
  delete stripped[LEVEL];
2230
2230
  delete stripped[MESSAGE];
@@ -2258,10 +2258,10 @@ var require_printf = __commonJS({
2258
2258
  var require_simple = __commonJS({
2259
2259
  "../../node_modules/logform/simple.js"(exports2, module2) {
2260
2260
  "use strict";
2261
- var format3 = require_format();
2261
+ var format4 = require_format();
2262
2262
  var { MESSAGE } = require_triple_beam();
2263
2263
  var jsonStringify = require_safe_stable_stringify();
2264
- module2.exports = format3((info) => {
2264
+ module2.exports = format4((info) => {
2265
2265
  const stringifiedRest = jsonStringify(Object.assign({}, info, {
2266
2266
  level: void 0,
2267
2267
  message: void 0,
@@ -2383,14 +2383,14 @@ function assign(origObj) {
2383
2383
  }
2384
2384
  return origObj;
2385
2385
  }
2386
- function parse(dateStr, format3, i18n) {
2386
+ function parse(dateStr, format4, i18n) {
2387
2387
  if (i18n === void 0) {
2388
2388
  i18n = {};
2389
2389
  }
2390
- if (typeof format3 !== "string") {
2390
+ if (typeof format4 !== "string") {
2391
2391
  throw new Error("Invalid format in fecha parse");
2392
2392
  }
2393
- format3 = globalMasks[format3] || format3;
2393
+ format4 = globalMasks[format4] || format4;
2394
2394
  if (dateStr.length > 1e3) {
2395
2395
  return null;
2396
2396
  }
@@ -2408,7 +2408,7 @@ function parse(dateStr, format3, i18n) {
2408
2408
  };
2409
2409
  var parseInfo = [];
2410
2410
  var literals = [];
2411
- var newFormat = format3.replace(literal, function($0, $1) {
2411
+ var newFormat = format4.replace(literal, function($0, $1) {
2412
2412
  literals.push(regexEscape($1));
2413
2413
  return "@@@";
2414
2414
  });
@@ -2770,8 +2770,8 @@ var require_timestamp = __commonJS({
2770
2770
  "../../node_modules/logform/timestamp.js"(exports2, module2) {
2771
2771
  "use strict";
2772
2772
  var fecha2 = (init_fecha(), __toCommonJS(fecha_exports));
2773
- var format3 = require_format();
2774
- module2.exports = format3((info, opts = {}) => {
2773
+ var format4 = require_format();
2774
+ module2.exports = format4((info, opts = {}) => {
2775
2775
  if (opts.format) {
2776
2776
  info.timestamp = typeof opts.format === "function" ? opts.format() : fecha2.format(/* @__PURE__ */ new Date(), opts.format);
2777
2777
  }
@@ -2791,9 +2791,9 @@ var require_uncolorize = __commonJS({
2791
2791
  "../../node_modules/logform/uncolorize.js"(exports2, module2) {
2792
2792
  "use strict";
2793
2793
  var colors = require_safe();
2794
- var format3 = require_format();
2794
+ var format4 = require_format();
2795
2795
  var { MESSAGE } = require_triple_beam();
2796
- module2.exports = format3((info, opts) => {
2796
+ module2.exports = format4((info, opts) => {
2797
2797
  if (opts.level !== false) {
2798
2798
  info.level = colors.strip(info.level);
2799
2799
  }
@@ -2812,10 +2812,10 @@ var require_uncolorize = __commonJS({
2812
2812
  var require_logform = __commonJS({
2813
2813
  "../../node_modules/logform/index.js"(exports2) {
2814
2814
  "use strict";
2815
- var format3 = exports2.format = require_format();
2815
+ var format4 = exports2.format = require_format();
2816
2816
  exports2.levels = require_levels();
2817
2817
  function exposeFormat(name, requireFormat) {
2818
- Object.defineProperty(format3, name, {
2818
+ Object.defineProperty(format4, name, {
2819
2819
  get() {
2820
2820
  return requireFormat();
2821
2821
  },
@@ -2880,17 +2880,17 @@ var require_logform = __commonJS({
2880
2880
  var require_common = __commonJS({
2881
2881
  "../../node_modules/winston/lib/winston/common.js"(exports2) {
2882
2882
  "use strict";
2883
- var { format: format3 } = require("util");
2883
+ var { format: format4 } = require("util");
2884
2884
  exports2.warn = {
2885
2885
  deprecated(prop) {
2886
2886
  return () => {
2887
- throw new Error(format3("{ %s } was removed in winston@3.0.0.", prop));
2887
+ throw new Error(format4("{ %s } was removed in winston@3.0.0.", prop));
2888
2888
  };
2889
2889
  },
2890
2890
  useFormat(prop) {
2891
2891
  return () => {
2892
2892
  throw new Error([
2893
- format3("{ %s } was removed in winston@3.0.0.", prop),
2893
+ format4("{ %s } was removed in winston@3.0.0.", prop),
2894
2894
  "Use a custom winston.format = winston.format(function) instead."
2895
2895
  ].join("\n"));
2896
2896
  };
@@ -4155,14 +4155,14 @@ var require_from = __commonJS({
4155
4155
  return (hint === "string" ? String : Number)(input);
4156
4156
  }
4157
4157
  var ERR_INVALID_ARG_TYPE = require_errors2().codes.ERR_INVALID_ARG_TYPE;
4158
- function from(Readable7, iterable, opts) {
4158
+ function from(Readable8, iterable, opts) {
4159
4159
  var iterator;
4160
4160
  if (iterable && typeof iterable.next === "function") {
4161
4161
  iterator = iterable;
4162
4162
  } else if (iterable && iterable[Symbol.asyncIterator]) iterator = iterable[Symbol.asyncIterator]();
4163
4163
  else if (iterable && iterable[Symbol.iterator]) iterator = iterable[Symbol.iterator]();
4164
4164
  else throw new ERR_INVALID_ARG_TYPE("iterable", ["Iterable"], iterable);
4165
- var readable = new Readable7(_objectSpread({
4165
+ var readable = new Readable8(_objectSpread({
4166
4166
  objectMode: true
4167
4167
  }, opts));
4168
4168
  var reading = false;
@@ -4202,9 +4202,9 @@ var require_from = __commonJS({
4202
4202
  var require_stream_readable = __commonJS({
4203
4203
  "../../node_modules/readable-stream/lib/_stream_readable.js"(exports2, module2) {
4204
4204
  "use strict";
4205
- module2.exports = Readable7;
4205
+ module2.exports = Readable8;
4206
4206
  var Duplex;
4207
- Readable7.ReadableState = ReadableState;
4207
+ Readable8.ReadableState = ReadableState;
4208
4208
  var EE = require("events").EventEmitter;
4209
4209
  var EElistenerCount = function EElistenerCount2(emitter, type) {
4210
4210
  return emitter.listeners(type).length;
@@ -4239,7 +4239,7 @@ var require_stream_readable = __commonJS({
4239
4239
  var StringDecoder;
4240
4240
  var createReadableStreamAsyncIterator;
4241
4241
  var from;
4242
- require_inherits()(Readable7, Stream);
4242
+ require_inherits()(Readable8, Stream);
4243
4243
  var errorOrDestroy = destroyImpl.errorOrDestroy;
4244
4244
  var kProxyEvents = ["error", "close", "destroy", "pause", "resume"];
4245
4245
  function prependListener(emitter, event, fn) {
@@ -4283,9 +4283,9 @@ var require_stream_readable = __commonJS({
4283
4283
  this.encoding = options.encoding;
4284
4284
  }
4285
4285
  }
4286
- function Readable7(options) {
4286
+ function Readable8(options) {
4287
4287
  Duplex = Duplex || require_stream_duplex();
4288
- if (!(this instanceof Readable7)) return new Readable7(options);
4288
+ if (!(this instanceof Readable8)) return new Readable8(options);
4289
4289
  var isDuplex = this instanceof Duplex;
4290
4290
  this._readableState = new ReadableState(options, this, isDuplex);
4291
4291
  this.readable = true;
@@ -4295,7 +4295,7 @@ var require_stream_readable = __commonJS({
4295
4295
  }
4296
4296
  Stream.call(this);
4297
4297
  }
4298
- Object.defineProperty(Readable7.prototype, "destroyed", {
4298
+ Object.defineProperty(Readable8.prototype, "destroyed", {
4299
4299
  // making it explicit this property is not enumerable
4300
4300
  // because otherwise some prototype manipulation in
4301
4301
  // userland will fail
@@ -4313,12 +4313,12 @@ var require_stream_readable = __commonJS({
4313
4313
  this._readableState.destroyed = value;
4314
4314
  }
4315
4315
  });
4316
- Readable7.prototype.destroy = destroyImpl.destroy;
4317
- Readable7.prototype._undestroy = destroyImpl.undestroy;
4318
- Readable7.prototype._destroy = function(err2, cb) {
4316
+ Readable8.prototype.destroy = destroyImpl.destroy;
4317
+ Readable8.prototype._undestroy = destroyImpl.undestroy;
4318
+ Readable8.prototype._destroy = function(err2, cb) {
4319
4319
  cb(err2);
4320
4320
  };
4321
- Readable7.prototype.push = function(chunk, encoding) {
4321
+ Readable8.prototype.push = function(chunk, encoding) {
4322
4322
  var state2 = this._readableState;
4323
4323
  var skipChunkCheck;
4324
4324
  if (!state2.objectMode) {
@@ -4335,7 +4335,7 @@ var require_stream_readable = __commonJS({
4335
4335
  }
4336
4336
  return readableAddChunk(this, chunk, encoding, false, skipChunkCheck);
4337
4337
  };
4338
- Readable7.prototype.unshift = function(chunk) {
4338
+ Readable8.prototype.unshift = function(chunk) {
4339
4339
  return readableAddChunk(this, chunk, null, true, false);
4340
4340
  };
4341
4341
  function readableAddChunk(stream, chunk, encoding, addToFront, skipChunkCheck) {
@@ -4396,10 +4396,10 @@ var require_stream_readable = __commonJS({
4396
4396
  }
4397
4397
  return er2;
4398
4398
  }
4399
- Readable7.prototype.isPaused = function() {
4399
+ Readable8.prototype.isPaused = function() {
4400
4400
  return this._readableState.flowing === false;
4401
4401
  };
4402
- Readable7.prototype.setEncoding = function(enc) {
4402
+ Readable8.prototype.setEncoding = function(enc) {
4403
4403
  if (!StringDecoder) StringDecoder = require_string_decoder().StringDecoder;
4404
4404
  var decoder = new StringDecoder(enc);
4405
4405
  this._readableState.decoder = decoder;
@@ -4445,7 +4445,7 @@ var require_stream_readable = __commonJS({
4445
4445
  }
4446
4446
  return state2.length;
4447
4447
  }
4448
- Readable7.prototype.read = function(n4) {
4448
+ Readable8.prototype.read = function(n4) {
4449
4449
  debug("read", n4);
4450
4450
  n4 = parseInt(n4, 10);
4451
4451
  var state2 = this._readableState;
@@ -4554,10 +4554,10 @@ var require_stream_readable = __commonJS({
4554
4554
  }
4555
4555
  state2.readingMore = false;
4556
4556
  }
4557
- Readable7.prototype._read = function(n4) {
4557
+ Readable8.prototype._read = function(n4) {
4558
4558
  errorOrDestroy(this, new ERR_METHOD_NOT_IMPLEMENTED("_read()"));
4559
4559
  };
4560
- Readable7.prototype.pipe = function(dest, pipeOpts) {
4560
+ Readable8.prototype.pipe = function(dest, pipeOpts) {
4561
4561
  var src = this;
4562
4562
  var state2 = this._readableState;
4563
4563
  switch (state2.pipesCount) {
@@ -4660,7 +4660,7 @@ var require_stream_readable = __commonJS({
4660
4660
  }
4661
4661
  };
4662
4662
  }
4663
- Readable7.prototype.unpipe = function(dest) {
4663
+ Readable8.prototype.unpipe = function(dest) {
4664
4664
  var state2 = this._readableState;
4665
4665
  var unpipeInfo = {
4666
4666
  hasUnpiped: false
@@ -4694,7 +4694,7 @@ var require_stream_readable = __commonJS({
4694
4694
  dest.emit("unpipe", this, unpipeInfo);
4695
4695
  return this;
4696
4696
  };
4697
- Readable7.prototype.on = function(ev, fn) {
4697
+ Readable8.prototype.on = function(ev, fn) {
4698
4698
  var res = Stream.prototype.on.call(this, ev, fn);
4699
4699
  var state2 = this._readableState;
4700
4700
  if (ev === "data") {
@@ -4715,15 +4715,15 @@ var require_stream_readable = __commonJS({
4715
4715
  }
4716
4716
  return res;
4717
4717
  };
4718
- Readable7.prototype.addListener = Readable7.prototype.on;
4719
- Readable7.prototype.removeListener = function(ev, fn) {
4718
+ Readable8.prototype.addListener = Readable8.prototype.on;
4719
+ Readable8.prototype.removeListener = function(ev, fn) {
4720
4720
  var res = Stream.prototype.removeListener.call(this, ev, fn);
4721
4721
  if (ev === "readable") {
4722
4722
  process.nextTick(updateReadableListening, this);
4723
4723
  }
4724
4724
  return res;
4725
4725
  };
4726
- Readable7.prototype.removeAllListeners = function(ev) {
4726
+ Readable8.prototype.removeAllListeners = function(ev) {
4727
4727
  var res = Stream.prototype.removeAllListeners.apply(this, arguments);
4728
4728
  if (ev === "readable" || ev === void 0) {
4729
4729
  process.nextTick(updateReadableListening, this);
@@ -4743,7 +4743,7 @@ var require_stream_readable = __commonJS({
4743
4743
  debug("readable nexttick read 0");
4744
4744
  self2.read(0);
4745
4745
  }
4746
- Readable7.prototype.resume = function() {
4746
+ Readable8.prototype.resume = function() {
4747
4747
  var state2 = this._readableState;
4748
4748
  if (!state2.flowing) {
4749
4749
  debug("resume");
@@ -4769,7 +4769,7 @@ var require_stream_readable = __commonJS({
4769
4769
  flow(stream);
4770
4770
  if (state2.flowing && !state2.reading) stream.read(0);
4771
4771
  }
4772
- Readable7.prototype.pause = function() {
4772
+ Readable8.prototype.pause = function() {
4773
4773
  debug("call pause flowing=%j", this._readableState.flowing);
4774
4774
  if (this._readableState.flowing !== false) {
4775
4775
  debug("pause");
@@ -4784,7 +4784,7 @@ var require_stream_readable = __commonJS({
4784
4784
  debug("flow", state2.flowing);
4785
4785
  while (state2.flowing && stream.read() !== null) ;
4786
4786
  }
4787
- Readable7.prototype.wrap = function(stream) {
4787
+ Readable8.prototype.wrap = function(stream) {
4788
4788
  var _this = this;
4789
4789
  var state2 = this._readableState;
4790
4790
  var paused = false;
@@ -4829,14 +4829,14 @@ var require_stream_readable = __commonJS({
4829
4829
  return this;
4830
4830
  };
4831
4831
  if (typeof Symbol === "function") {
4832
- Readable7.prototype[Symbol.asyncIterator] = function() {
4832
+ Readable8.prototype[Symbol.asyncIterator] = function() {
4833
4833
  if (createReadableStreamAsyncIterator === void 0) {
4834
4834
  createReadableStreamAsyncIterator = require_async_iterator();
4835
4835
  }
4836
4836
  return createReadableStreamAsyncIterator(this);
4837
4837
  };
4838
4838
  }
4839
- Object.defineProperty(Readable7.prototype, "readableHighWaterMark", {
4839
+ Object.defineProperty(Readable8.prototype, "readableHighWaterMark", {
4840
4840
  // making it explicit this property is not enumerable
4841
4841
  // because otherwise some prototype manipulation in
4842
4842
  // userland will fail
@@ -4845,7 +4845,7 @@ var require_stream_readable = __commonJS({
4845
4845
  return this._readableState.highWaterMark;
4846
4846
  }
4847
4847
  });
4848
- Object.defineProperty(Readable7.prototype, "readableBuffer", {
4848
+ Object.defineProperty(Readable8.prototype, "readableBuffer", {
4849
4849
  // making it explicit this property is not enumerable
4850
4850
  // because otherwise some prototype manipulation in
4851
4851
  // userland will fail
@@ -4854,7 +4854,7 @@ var require_stream_readable = __commonJS({
4854
4854
  return this._readableState && this._readableState.buffer;
4855
4855
  }
4856
4856
  });
4857
- Object.defineProperty(Readable7.prototype, "readableFlowing", {
4857
+ Object.defineProperty(Readable8.prototype, "readableFlowing", {
4858
4858
  // making it explicit this property is not enumerable
4859
4859
  // because otherwise some prototype manipulation in
4860
4860
  // userland will fail
@@ -4868,8 +4868,8 @@ var require_stream_readable = __commonJS({
4868
4868
  }
4869
4869
  }
4870
4870
  });
4871
- Readable7._fromList = fromList;
4872
- Object.defineProperty(Readable7.prototype, "readableLength", {
4871
+ Readable8._fromList = fromList;
4872
+ Object.defineProperty(Readable8.prototype, "readableLength", {
4873
4873
  // making it explicit this property is not enumerable
4874
4874
  // because otherwise some prototype manipulation in
4875
4875
  // userland will fail
@@ -4915,11 +4915,11 @@ var require_stream_readable = __commonJS({
4915
4915
  }
4916
4916
  }
4917
4917
  if (typeof Symbol === "function") {
4918
- Readable7.from = function(iterable, opts) {
4918
+ Readable8.from = function(iterable, opts) {
4919
4919
  if (from === void 0) {
4920
4920
  from = require_from();
4921
4921
  }
4922
- return from(Readable7, iterable, opts);
4922
+ return from(Readable8, iterable, opts);
4923
4923
  };
4924
4924
  }
4925
4925
  function indexOf(xs2, x2) {
@@ -4941,9 +4941,9 @@ var require_stream_duplex = __commonJS({
4941
4941
  return keys2;
4942
4942
  };
4943
4943
  module2.exports = Duplex;
4944
- var Readable7 = require_stream_readable();
4944
+ var Readable8 = require_stream_readable();
4945
4945
  var Writable2 = require_stream_writable();
4946
- require_inherits()(Duplex, Readable7);
4946
+ require_inherits()(Duplex, Readable8);
4947
4947
  {
4948
4948
  keys = objectKeys(Writable2.prototype);
4949
4949
  for (v3 = 0; v3 < keys.length; v3++) {
@@ -4956,7 +4956,7 @@ var require_stream_duplex = __commonJS({
4956
4956
  var v3;
4957
4957
  function Duplex(options) {
4958
4958
  if (!(this instanceof Duplex)) return new Duplex(options);
4959
- Readable7.call(this, options);
4959
+ Readable8.call(this, options);
4960
4960
  Writable2.call(this, options);
4961
4961
  this.allowHalfOpen = true;
4962
4962
  if (options) {
@@ -6498,7 +6498,7 @@ var require_pipeline = __commonJS({
6498
6498
  if (typeof streams[streams.length - 1] !== "function") return noop;
6499
6499
  return streams.pop();
6500
6500
  }
6501
- function pipeline5() {
6501
+ function pipeline6() {
6502
6502
  for (var _len = arguments.length, streams = new Array(_len), _key = 0; _key < _len; _key++) {
6503
6503
  streams[_key] = arguments[_key];
6504
6504
  }
@@ -6521,7 +6521,7 @@ var require_pipeline = __commonJS({
6521
6521
  });
6522
6522
  return streams.reduce(pipe);
6523
6523
  }
6524
- module2.exports = pipeline5;
6524
+ module2.exports = pipeline6;
6525
6525
  }
6526
6526
  });
6527
6527
 
@@ -7759,15 +7759,15 @@ var require_index_cjs = __commonJS({
7759
7759
  };
7760
7760
  }
7761
7761
  function wrapConversion(toModel, graph2) {
7762
- const path29 = [graph2[toModel].parent, toModel];
7762
+ const path30 = [graph2[toModel].parent, toModel];
7763
7763
  let fn = convert$1[graph2[toModel].parent][toModel];
7764
7764
  let cur = graph2[toModel].parent;
7765
7765
  while (graph2[cur].parent) {
7766
- path29.unshift(graph2[cur].parent);
7766
+ path30.unshift(graph2[cur].parent);
7767
7767
  fn = link(convert$1[graph2[cur].parent][cur], fn);
7768
7768
  cur = graph2[cur].parent;
7769
7769
  }
7770
- fn.conversion = path29;
7770
+ fn.conversion = path30;
7771
7771
  return fn;
7772
7772
  }
7773
7773
  function route(fromModel) {
@@ -8397,7 +8397,7 @@ var require_node2 = __commonJS({
8397
8397
  var require_tail_file = __commonJS({
8398
8398
  "../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
8399
8399
  "use strict";
8400
- var fs32 = require("fs");
8400
+ var fs33 = require("fs");
8401
8401
  var { StringDecoder } = require("string_decoder");
8402
8402
  var { Stream } = require_readable();
8403
8403
  function noop() {
@@ -8418,7 +8418,7 @@ var require_tail_file = __commonJS({
8418
8418
  stream.emit("end");
8419
8419
  stream.emit("close");
8420
8420
  };
8421
- fs32.open(options.file, "a+", "0644", (err2, fd) => {
8421
+ fs33.open(options.file, "a+", "0644", (err2, fd) => {
8422
8422
  if (err2) {
8423
8423
  if (!iter) {
8424
8424
  stream.emit("error", err2);
@@ -8430,10 +8430,10 @@ var require_tail_file = __commonJS({
8430
8430
  }
8431
8431
  (function read() {
8432
8432
  if (stream.destroyed) {
8433
- fs32.close(fd, noop);
8433
+ fs33.close(fd, noop);
8434
8434
  return;
8435
8435
  }
8436
- return fs32.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
8436
+ return fs33.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
8437
8437
  if (error) {
8438
8438
  if (!iter) {
8439
8439
  stream.emit("error", error);
@@ -8492,8 +8492,8 @@ var require_tail_file = __commonJS({
8492
8492
  var require_file = __commonJS({
8493
8493
  "../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
8494
8494
  "use strict";
8495
- var fs32 = require("fs");
8496
- var path29 = require("path");
8495
+ var fs33 = require("fs");
8496
+ var path30 = require("path");
8497
8497
  var asyncSeries = require_series();
8498
8498
  var zlib3 = require("zlib");
8499
8499
  var { MESSAGE } = require_triple_beam();
@@ -8523,14 +8523,14 @@ var require_file = __commonJS({
8523
8523
  this._onError = this._onError.bind(this);
8524
8524
  if (options.filename || options.dirname) {
8525
8525
  throwIf("filename or dirname", "stream");
8526
- this._basename = this.filename = options.filename ? path29.basename(options.filename) : "winston.log";
8527
- this.dirname = options.dirname || path29.dirname(options.filename);
8526
+ this._basename = this.filename = options.filename ? path30.basename(options.filename) : "winston.log";
8527
+ this.dirname = options.dirname || path30.dirname(options.filename);
8528
8528
  this.options = options.options || { flags: "a" };
8529
8529
  } else if (options.stream) {
8530
8530
  console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
8531
8531
  throwIf("stream", "filename", "maxsize");
8532
8532
  this._dest = this._stream.pipe(this._setupStream(options.stream));
8533
- this.dirname = path29.dirname(this._dest.path);
8533
+ this.dirname = path30.dirname(this._dest.path);
8534
8534
  } else {
8535
8535
  throw new Error("Cannot log to file without filename or stream.");
8536
8536
  }
@@ -8693,11 +8693,11 @@ var require_file = __commonJS({
8693
8693
  options = {};
8694
8694
  }
8695
8695
  options = normalizeQuery(options);
8696
- const file = path29.join(this.dirname, this.filename);
8696
+ const file = path30.join(this.dirname, this.filename);
8697
8697
  let buff = "";
8698
8698
  let results = [];
8699
8699
  let row = 0;
8700
- const stream = fs32.createReadStream(file, {
8700
+ const stream = fs33.createReadStream(file, {
8701
8701
  encoding: "utf8"
8702
8702
  });
8703
8703
  stream.on("error", (err2) => {
@@ -8798,7 +8798,7 @@ var require_file = __commonJS({
8798
8798
  * TODO: Refactor me.
8799
8799
  */
8800
8800
  stream(options = {}) {
8801
- const file = path29.join(this.dirname, this.filename);
8801
+ const file = path30.join(this.dirname, this.filename);
8802
8802
  const stream = new Stream();
8803
8803
  const tail = {
8804
8804
  file,
@@ -8848,8 +8848,8 @@ var require_file = __commonJS({
8848
8848
  */
8849
8849
  stat(callback) {
8850
8850
  const target = this._getFile();
8851
- const fullpath = path29.join(this.dirname, target);
8852
- fs32.stat(fullpath, (err2, stat) => {
8851
+ const fullpath = path30.join(this.dirname, target);
8852
+ fs33.stat(fullpath, (err2, stat) => {
8853
8853
  if (err2 && err2.code === "ENOENT") {
8854
8854
  debug("ENOENT\xA0ok", fullpath);
8855
8855
  this.filename = target;
@@ -8952,9 +8952,9 @@ var require_file = __commonJS({
8952
8952
  * @returns {WritableStream} Stream that writes to disk for the active file.
8953
8953
  */
8954
8954
  _createStream(source) {
8955
- const fullpath = path29.join(this.dirname, this.filename);
8955
+ const fullpath = path30.join(this.dirname, this.filename);
8956
8956
  debug("create stream start", fullpath, this.options);
8957
- const dest = fs32.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
8957
+ const dest = fs33.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
8958
8958
  debug("file open ok", fullpath);
8959
8959
  this.emit("open", fullpath);
8960
8960
  source.pipe(dest);
@@ -8977,16 +8977,16 @@ var require_file = __commonJS({
8977
8977
  */
8978
8978
  _incFile(callback) {
8979
8979
  debug("_incFile", this.filename);
8980
- const ext = path29.extname(this._basename);
8981
- const basename = path29.basename(this._basename, ext);
8980
+ const ext = path30.extname(this._basename);
8981
+ const basename = path30.basename(this._basename, ext);
8982
8982
  const tasks = [];
8983
8983
  if (this.zippedArchive) {
8984
8984
  tasks.push(
8985
8985
  function(cb) {
8986
8986
  const num = this._created > 0 && !this.tailable ? this._created : "";
8987
8987
  this._compressFile(
8988
- path29.join(this.dirname, `${basename}${num}${ext}`),
8989
- path29.join(this.dirname, `${basename}${num}${ext}.gz`),
8988
+ path30.join(this.dirname, `${basename}${num}${ext}`),
8989
+ path30.join(this.dirname, `${basename}${num}${ext}.gz`),
8990
8990
  cb
8991
8991
  );
8992
8992
  }.bind(this)
@@ -9011,8 +9011,8 @@ var require_file = __commonJS({
9011
9011
  * @private
9012
9012
  */
9013
9013
  _getFile() {
9014
- const ext = path29.extname(this._basename);
9015
- const basename = path29.basename(this._basename, ext);
9014
+ const ext = path30.extname(this._basename);
9015
+ const basename = path30.basename(this._basename, ext);
9016
9016
  const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
9017
9017
  return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
9018
9018
  }
@@ -9032,8 +9032,8 @@ var require_file = __commonJS({
9032
9032
  const isOldest = oldest !== 0 ? oldest : "";
9033
9033
  const isZipped = this.zippedArchive ? ".gz" : "";
9034
9034
  const filePath = `${basename}${isOldest}${ext}${isZipped}`;
9035
- const target = path29.join(this.dirname, filePath);
9036
- fs32.unlink(target, callback);
9035
+ const target = path30.join(this.dirname, filePath);
9036
+ fs33.unlink(target, callback);
9037
9037
  }
9038
9038
  /**
9039
9039
  * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -9055,20 +9055,20 @@ var require_file = __commonJS({
9055
9055
  for (let x2 = this.maxFiles - 1; x2 > 1; x2--) {
9056
9056
  tasks.push(function(i6, cb) {
9057
9057
  let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
9058
- const tmppath = path29.join(this.dirname, fileName);
9059
- fs32.exists(tmppath, (exists) => {
9058
+ const tmppath = path30.join(this.dirname, fileName);
9059
+ fs33.exists(tmppath, (exists) => {
9060
9060
  if (!exists) {
9061
9061
  return cb(null);
9062
9062
  }
9063
9063
  fileName = `${basename}${i6}${ext}${isZipped}`;
9064
- fs32.rename(tmppath, path29.join(this.dirname, fileName), cb);
9064
+ fs33.rename(tmppath, path30.join(this.dirname, fileName), cb);
9065
9065
  });
9066
9066
  }.bind(this, x2));
9067
9067
  }
9068
9068
  asyncSeries(tasks, () => {
9069
- fs32.rename(
9070
- path29.join(this.dirname, `${basename}${ext}${isZipped}`),
9071
- path29.join(this.dirname, `${basename}1${ext}${isZipped}`),
9069
+ fs33.rename(
9070
+ path30.join(this.dirname, `${basename}${ext}${isZipped}`),
9071
+ path30.join(this.dirname, `${basename}1${ext}${isZipped}`),
9072
9072
  callback
9073
9073
  );
9074
9074
  });
@@ -9082,22 +9082,22 @@ var require_file = __commonJS({
9082
9082
  * @private
9083
9083
  */
9084
9084
  _compressFile(src, dest, callback) {
9085
- fs32.access(src, fs32.F_OK, (err2) => {
9085
+ fs33.access(src, fs33.F_OK, (err2) => {
9086
9086
  if (err2) {
9087
9087
  return callback();
9088
9088
  }
9089
9089
  var gzip = zlib3.createGzip();
9090
- var inp = fs32.createReadStream(src);
9091
- var out = fs32.createWriteStream(dest);
9090
+ var inp = fs33.createReadStream(src);
9091
+ var out = fs33.createWriteStream(dest);
9092
9092
  out.on("finish", () => {
9093
- fs32.unlink(src, callback);
9093
+ fs33.unlink(src, callback);
9094
9094
  });
9095
9095
  inp.pipe(gzip).pipe(out);
9096
9096
  });
9097
9097
  }
9098
9098
  _createLogDirIfNotExist(dirPath) {
9099
- if (!fs32.existsSync(dirPath)) {
9100
- fs32.mkdirSync(dirPath, { recursive: true });
9099
+ if (!fs33.existsSync(dirPath)) {
9100
+ fs33.mkdirSync(dirPath, { recursive: true });
9101
9101
  }
9102
9102
  }
9103
9103
  };
@@ -9181,9 +9181,9 @@ var require_http = __commonJS({
9181
9181
  };
9182
9182
  const auth = options.params.auth || null;
9183
9183
  delete options.params.auth;
9184
- const path29 = options.params.path || null;
9184
+ const path30 = options.params.path || null;
9185
9185
  delete options.params.path;
9186
- this._request(options, auth, path29, (err2, res, body) => {
9186
+ this._request(options, auth, path30, (err2, res, body) => {
9187
9187
  if (res && res.statusCode !== 200) {
9188
9188
  err2 = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
9189
9189
  }
@@ -9211,12 +9211,12 @@ var require_http = __commonJS({
9211
9211
  method: "stream",
9212
9212
  params: options
9213
9213
  };
9214
- const path29 = options.params.path || null;
9214
+ const path30 = options.params.path || null;
9215
9215
  delete options.params.path;
9216
9216
  const auth = options.params.auth || null;
9217
9217
  delete options.params.auth;
9218
9218
  let buff = "";
9219
- const req = this._request(options, auth, path29);
9219
+ const req = this._request(options, auth, path30);
9220
9220
  stream.destroy = () => req.destroy();
9221
9221
  req.on("data", (data) => {
9222
9222
  data = (buff + data).split(/\n+/);
@@ -9242,14 +9242,14 @@ var require_http = __commonJS({
9242
9242
  * @param {string} path - request path
9243
9243
  * @param {function} callback - Continuation to respond to when complete.
9244
9244
  */
9245
- _request(options, auth, path29, callback) {
9245
+ _request(options, auth, path30, callback) {
9246
9246
  options = options || {};
9247
9247
  auth = auth || this.auth;
9248
- path29 = path29 || this.path || "";
9248
+ path30 = path30 || this.path || "";
9249
9249
  if (this.batch) {
9250
- this._doBatch(options, callback, auth, path29);
9250
+ this._doBatch(options, callback, auth, path30);
9251
9251
  } else {
9252
- this._doRequest(options, callback, auth, path29);
9252
+ this._doRequest(options, callback, auth, path30);
9253
9253
  }
9254
9254
  }
9255
9255
  /**
@@ -9259,18 +9259,18 @@ var require_http = __commonJS({
9259
9259
  * @param {Object?} auth - authentication options
9260
9260
  * @param {string} path - request path
9261
9261
  */
9262
- _doBatch(options, callback, auth, path29) {
9262
+ _doBatch(options, callback, auth, path30) {
9263
9263
  this.batchOptions.push(options);
9264
9264
  if (this.batchOptions.length === 1) {
9265
9265
  const me3 = this;
9266
9266
  this.batchCallback = callback;
9267
9267
  this.batchTimeoutID = setTimeout(function() {
9268
9268
  me3.batchTimeoutID = -1;
9269
- me3._doBatchRequest(me3.batchCallback, auth, path29);
9269
+ me3._doBatchRequest(me3.batchCallback, auth, path30);
9270
9270
  }, this.batchInterval);
9271
9271
  }
9272
9272
  if (this.batchOptions.length === this.batchCount) {
9273
- this._doBatchRequest(this.batchCallback, auth, path29);
9273
+ this._doBatchRequest(this.batchCallback, auth, path30);
9274
9274
  }
9275
9275
  }
9276
9276
  /**
@@ -9279,14 +9279,14 @@ var require_http = __commonJS({
9279
9279
  * @param {Object?} auth - authentication options
9280
9280
  * @param {string} path - request path
9281
9281
  */
9282
- _doBatchRequest(callback, auth, path29) {
9282
+ _doBatchRequest(callback, auth, path30) {
9283
9283
  if (this.batchTimeoutID > 0) {
9284
9284
  clearTimeout(this.batchTimeoutID);
9285
9285
  this.batchTimeoutID = -1;
9286
9286
  }
9287
9287
  const batchOptionsCopy = this.batchOptions.slice();
9288
9288
  this.batchOptions = [];
9289
- this._doRequest(batchOptionsCopy, callback, auth, path29);
9289
+ this._doRequest(batchOptionsCopy, callback, auth, path30);
9290
9290
  }
9291
9291
  /**
9292
9292
  * Make a request to a winstond server or any http server which can
@@ -9296,7 +9296,7 @@ var require_http = __commonJS({
9296
9296
  * @param {Object?} auth - authentication options
9297
9297
  * @param {string} path - request path
9298
9298
  */
9299
- _doRequest(options, callback, auth, path29) {
9299
+ _doRequest(options, callback, auth, path30) {
9300
9300
  const headers = Object.assign({}, this.headers);
9301
9301
  if (auth && auth.bearer) {
9302
9302
  headers.Authorization = `Bearer ${auth.bearer}`;
@@ -9306,7 +9306,7 @@ var require_http = __commonJS({
9306
9306
  method: "POST",
9307
9307
  host: this.host,
9308
9308
  port: this.port,
9309
- path: `/${path29.replace(/^\//, "")}`,
9309
+ path: `/${path30.replace(/^\//, "")}`,
9310
9310
  headers,
9311
9311
  auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
9312
9312
  agent: this.agent
@@ -10275,7 +10275,7 @@ var require_logger = __commonJS({
10275
10275
  */
10276
10276
  configure({
10277
10277
  silent,
10278
- format: format3,
10278
+ format: format4,
10279
10279
  defaultMeta,
10280
10280
  levels,
10281
10281
  level = "info",
@@ -10294,7 +10294,7 @@ var require_logger = __commonJS({
10294
10294
  this.clear();
10295
10295
  }
10296
10296
  this.silent = silent;
10297
- this.format = format3 || this.format || require_json()();
10297
+ this.format = format4 || this.format || require_json()();
10298
10298
  this.defaultMeta = defaultMeta || null;
10299
10299
  this.levels = levels || this.levels || config.npm.levels;
10300
10300
  this.level = level;
@@ -11116,18 +11116,32 @@ ${stack}` : base;
11116
11116
  });
11117
11117
 
11118
11118
  // ../../packages/logger/src/logger/Logger.ts
11119
- var import_chalk, MAX_CAUSE_DEPTH, Logger, logger, Logger_default;
11119
+ var import_util, import_chalk, MAX_CAUSE_DEPTH, LOG_RECORD_VERSION, MAX_FIELD_CHARS, ANSI_PATTERN, nativeConsole, Logger, logger, Logger_default;
11120
11120
  var init_Logger = __esm({
11121
11121
  "../../packages/logger/src/logger/Logger.ts"() {
11122
+ import_util = require("util");
11123
+ init_src();
11122
11124
  import_chalk = __toESM(require("chalk"), 1);
11123
11125
  init_FileLogService();
11124
11126
  MAX_CAUSE_DEPTH = 10;
11127
+ LOG_RECORD_VERSION = 1;
11128
+ MAX_FIELD_CHARS = 3e4;
11129
+ ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
11130
+ nativeConsole = {
11131
+ log: console.log.bind(console),
11132
+ info: console.info.bind(console),
11133
+ warn: console.warn.bind(console),
11134
+ error: console.error.bind(console)
11135
+ };
11125
11136
  Logger = class {
11126
11137
  constructor() {
11127
11138
  // Counts lines written to the console. In-place TTY renderers (the CLI progress bar) read it to
11128
11139
  // detect that something else has printed since their last frame, so they redraw on a new line
11129
11140
  // instead of moving the cursor up and overwriting a log message.
11130
11141
  this._consoleWrites = 0;
11142
+ // Whether console.* has been redirected into this logger. Once per process, and one-way: our
11143
+ // own writes use `nativeConsole`, so there is nothing to restore it for.
11144
+ this._consoleCaptured = false;
11131
11145
  this.setLevel = (level) => this._level = level;
11132
11146
  this.enableFileLogging = (folder, file) => {
11133
11147
  this._fileLoggingFolder = folder;
@@ -11161,9 +11175,28 @@ var init_Logger = __esm({
11161
11175
  this._level = "debug";
11162
11176
  if (fileLoggingFolder && process.env.NODE_ENV !== "development") {
11163
11177
  this.enableFileLogging(fileLoggingFolder);
11164
- console.error("Enabled file logger.");
11178
+ nativeConsole.error("Enabled file logger.");
11165
11179
  }
11166
11180
  };
11181
+ /**
11182
+ * Routes `console.*` through this logger, so code that never heard of it still writes one line
11183
+ * per record. Call once from a deployed process entrypoint; a no-op under the `text` format,
11184
+ * where a human is reading the console and Node's own formatting is the better rendering.
11185
+ *
11186
+ * Interception rather than a sweep of the call sites: the requirement is about the STREAM, not
11187
+ * about our code — an express handler, a driver and a dependency all write to the same stdout,
11188
+ * and a rule enforced at the call sites is a rule that the next `console.error` breaks.
11189
+ */
11190
+ this.captureConsole = () => {
11191
+ if (this._consoleCaptured) return;
11192
+ if (this.consoleFormat() !== "json") return;
11193
+ this._consoleCaptured = true;
11194
+ console.log = (...args) => this.info((0, import_util.format)(...args));
11195
+ console.info = console.log;
11196
+ console.debug = (...args) => this.log((0, import_util.format)(...args));
11197
+ console.warn = (...args) => this.warn((0, import_util.format)(...args));
11198
+ console.error = (...args) => args.length === 1 && args[0] instanceof Error ? this.error(args[0]) : this.error((0, import_util.format)(...args));
11199
+ };
11167
11200
  // The file sink replaces the console only for the CLI: there a human is watching the progress bar
11168
11201
  // and can read the log file afterwards. Anywhere else that file sits on an ephemeral container
11169
11202
  // layer nobody reads, and stdout is the only sink reaching the log driver (ECS -> CloudWatch), so
@@ -11174,18 +11207,15 @@ var init_Logger = __esm({
11174
11207
  if (myLevel !== "debug") return;
11175
11208
  FileLogService_default.write("DEBUG", String(message));
11176
11209
  if (this.isConsoleSuppressed()) return;
11177
- console.log(import_chalk.default.cyanBright("DEBUG"), message);
11178
- this._consoleWrites++;
11210
+ this.writeConsole("debug", message);
11179
11211
  };
11180
11212
  this.info = (message) => {
11181
11213
  FileLogService_default.write("INFO", String(message));
11182
11214
  if (this.isConsoleSuppressed()) return;
11183
- console.info(message);
11184
- this._consoleWrites++;
11215
+ this.writeConsole("info", message);
11185
11216
  };
11186
11217
  this.warn = (message) => {
11187
- console.warn(message);
11188
- this._consoleWrites++;
11218
+ this.writeConsole("warn", message);
11189
11219
  FileLogService_default.write("WARN", String(message));
11190
11220
  };
11191
11221
  this.flush = () => FileLogService_default.flush();
@@ -11203,11 +11233,88 @@ var init_Logger = __esm({
11203
11233
  message = String(error);
11204
11234
  }
11205
11235
  const suppressed = this.isConsoleSuppressed();
11206
- console.error(message);
11207
- this._consoleWrites++;
11236
+ this.writeConsole("error", message, suppressed ? void 0 : stack);
11208
11237
  FileLogService_default.write("ERROR", message, stack);
11209
- if (stack && !suppressed)
11210
- console.error(stack);
11238
+ };
11239
+ /**
11240
+ * How console records are rendered: `text` for a human at a terminal, `json` for a log pipeline.
11241
+ *
11242
+ * Deployed processes default to `json` because there ONE RECORD MUST BE ONE LINE: the awslogs
11243
+ * driver turns every newline on stdout into its own CloudWatch event, so a stack trace arrived as
11244
+ * thirty separate events, none of them carrying the message they belonged to, interleaved with
11245
+ * whatever the other threads were writing. `REMORA_LOG_FORMAT` overrides the default either way.
11246
+ * Same rule as StateEmitter, whose records share the stream.
11247
+ */
11248
+ this.consoleFormat = () => {
11249
+ const configured = process.env.REMORA_LOG_FORMAT?.toLowerCase();
11250
+ if (configured === "json" || configured === "text")
11251
+ return configured;
11252
+ return process.env.REMORA_RUNTIME_CONTEXT === "cli" ? "text" : "json";
11253
+ };
11254
+ /**
11255
+ * The one place anything of ours reaches the console.
11256
+ *
11257
+ * `stack` belongs to the record it explains: a second line in text mode, a field of the same JSON
11258
+ * object in json mode — never a record of its own, which is how it would lose its message.
11259
+ */
11260
+ this.writeConsole = (level, message, stack) => {
11261
+ this._consoleWrites++;
11262
+ if (this.consoleFormat() === "json")
11263
+ return this.writeJson(level, message, stack);
11264
+ this.writeText(level, message, stack);
11265
+ };
11266
+ this.writeText = (level, message, stack) => {
11267
+ if (level === "debug")
11268
+ nativeConsole.log(import_chalk.default.cyanBright("DEBUG"), message);
11269
+ else if (level === "info")
11270
+ nativeConsole.info(message);
11271
+ else if (level === "warn")
11272
+ nativeConsole.warn(message);
11273
+ else
11274
+ nativeConsole.error(message);
11275
+ if (stack)
11276
+ nativeConsole.error(stack);
11277
+ };
11278
+ /**
11279
+ * One JSON object, one line, one log event. Written straight to the stream rather than through
11280
+ * `console`, which reformats and re-wraps what it is given.
11281
+ */
11282
+ this.writeJson = (level, message, stack) => {
11283
+ const record = {
11284
+ remora: "log",
11285
+ v: LOG_RECORD_VERSION,
11286
+ ts: DSTE_default.now().toISOString(),
11287
+ level,
11288
+ message: this.asField(message),
11289
+ ...stack ? { stack: this.asField(stack) } : {}
11290
+ };
11291
+ const stream = level === "warn" || level === "error" ? process.stderr : process.stdout;
11292
+ stream.write(`${this.stringify(record)}
11293
+ `);
11294
+ };
11295
+ /** A loggable value as one clamped, colour-free string. */
11296
+ this.asField = (value) => {
11297
+ const plain = this.asText(value).replace(ANSI_PATTERN, "");
11298
+ if (plain.length <= MAX_FIELD_CHARS)
11299
+ return plain;
11300
+ return `${plain.slice(0, MAX_FIELD_CHARS)}\u2026[+${plain.length - MAX_FIELD_CHARS} chars]`;
11301
+ };
11302
+ this.asText = (value) => {
11303
+ if (typeof value === "string")
11304
+ return value;
11305
+ if (value instanceof Error)
11306
+ return value.stack ?? value.message;
11307
+ if (typeof value === "object" && value !== null)
11308
+ return this.stringify(value);
11309
+ return String(value);
11310
+ };
11311
+ /** JSON.stringify that cannot throw: a circular or unserializable value still has to be logged. */
11312
+ this.stringify = (value) => {
11313
+ try {
11314
+ return JSON.stringify(value);
11315
+ } catch {
11316
+ return String(value);
11317
+ }
11211
11318
  };
11212
11319
  // Node's Error.stack omits `cause` — only util.inspect renders it, and we log strings rather
11213
11320
  // than Error objects. Without unwrapping here the root reason of a wrapped error is lost on any
@@ -11527,7 +11634,7 @@ var CONSTANTS, Constants_default;
11527
11634
  var init_Constants = __esm({
11528
11635
  "../../packages/constants/src/Constants.ts"() {
11529
11636
  CONSTANTS = {
11530
- cliVersion: "2.0.2",
11637
+ cliVersion: "2.1.0",
11531
11638
  backendVersion: 1,
11532
11639
  backendPort: 5088,
11533
11640
  workerVersion: 2,
@@ -11584,7 +11691,12 @@ var init_Constants = __esm({
11584
11691
  * whole run, so the entire work list is submitted at once and needs an upper bound. A queued
11585
11692
  * task holds references, not copies, so this bounds the work list rather than memory.
11586
11693
  */
11587
- MAX_WORKER_QUEUE_SIZE: 1e5
11694
+ MAX_WORKER_QUEUE_SIZE: 1e5,
11695
+ /**
11696
+ * Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
11697
+ * export of it — `remora sample -s` and the preview's Data tab are both held to this.
11698
+ */
11699
+ MAX_SAMPLE_ROWS: 100
11588
11700
  }
11589
11701
  };
11590
11702
  Constants_default = CONSTANTS;
@@ -12572,10 +12684,10 @@ var init_Validator = __esm({
12572
12684
  errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(" and ")})`);
12573
12685
  }
12574
12686
  for (const output of consumer.outputs) {
12575
- const format3 = output.format.toUpperCase();
12576
- if (format3 === "CSV" || format3 === "JSON" || format3 === "PARQUET" || format3 === "TXT") {
12687
+ const format4 = output.format.toUpperCase();
12688
+ if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
12577
12689
  if (!output.exportDestination)
12578
- errors.push(`A static file output must have an export destination set (${format3})`);
12690
+ errors.push(`A static file output must have an export destination set (${format4})`);
12579
12691
  else if (!Environment_default.getSource(output.exportDestination))
12580
12692
  errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
12581
12693
  }
@@ -12647,14 +12759,14 @@ var init_Validator = __esm({
12647
12759
  const errors = [];
12648
12760
  const positionalFormats = ["CSV", "TXT", "PARQUET"];
12649
12761
  for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
12650
- const format3 = output.format.toUpperCase();
12651
- if (positionalFormats.includes(format3)) {
12652
- errors.push(`The ${format3} output of consumer "${consumer.name}" declares "omitNull", which only applies to JSON/API output: a positional column can't be omitted from a row.`);
12762
+ const format4 = output.format.toUpperCase();
12763
+ if (positionalFormats.includes(format4)) {
12764
+ errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", which only applies to JSON/API output: a positional column can't be omitted from a row.`);
12653
12765
  continue;
12654
12766
  }
12655
12767
  const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
12656
12768
  if (positional.length > 0)
12657
- errors.push(`The ${format3} output of consumer "${consumer.name}" declares "omitNull", but the consumer also writes ${positional.map((x2) => x2.format.toUpperCase()).join(" and ")}. Every output is serialized from one record, so omitting fields would break the positional one \u2014 split them into two consumers.`);
12769
+ errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", but the consumer also writes ${positional.map((x2) => x2.format.toUpperCase()).join(" and ")}. Every output is serialized from one record, so omitting fields would break the positional one \u2014 split them into two consumers.`);
12658
12770
  }
12659
12771
  return errors;
12660
12772
  };
@@ -12733,7 +12845,7 @@ var init_Validator = __esm({
12733
12845
  */
12734
12846
  this.detectOnFinishRecursion = (consumer) => {
12735
12847
  const errors = [];
12736
- const path29 = [];
12848
+ const path30 = [];
12737
12849
  const targetsOf = (cons) => {
12738
12850
  const names = [];
12739
12851
  for (const output of cons.outputs ?? []) {
@@ -12745,17 +12857,17 @@ var init_Validator = __esm({
12745
12857
  return names;
12746
12858
  };
12747
12859
  const walk = (name) => {
12748
- if (path29.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
12749
- errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path29, name].join(" -> ")}`);
12860
+ if (path30.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
12861
+ errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path30, name].join(" -> ")}`);
12750
12862
  return true;
12751
12863
  }
12752
12864
  const cons = Environment_default.getConsumer(name);
12753
12865
  if (!cons) return false;
12754
- path29.push(name);
12866
+ path30.push(name);
12755
12867
  for (const target of targetsOf(cons)) {
12756
12868
  if (walk(target)) return true;
12757
12869
  }
12758
- path29.pop();
12870
+ path30.pop();
12759
12871
  return false;
12760
12872
  };
12761
12873
  walk(consumer.name);
@@ -14241,8 +14353,8 @@ var init_HttpClient = __esm({
14241
14353
  throw new Error(err2.message ?? "An error occurred on GET.");
14242
14354
  }
14243
14355
  };
14244
- this.getWorkerUrl = (path29) => {
14245
- const cleanPath = path29.startsWith("/") ? path29.substring(1) : path29;
14356
+ this.getWorkerUrl = (path30) => {
14357
+ const cleanPath = path30.startsWith("/") ? path30.substring(1) : path30;
14246
14358
  return `${this.workerHost}/${cleanPath}`;
14247
14359
  };
14248
14360
  this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -14685,14 +14797,14 @@ var init_readFile = __esm({
14685
14797
  import_promises3 = require("fs/promises");
14686
14798
  filePromises = {};
14687
14799
  fileIntercept = {};
14688
- readFile2 = (path29, options) => {
14689
- if (fileIntercept[path29] !== void 0) {
14690
- return fileIntercept[path29];
14800
+ readFile2 = (path30, options) => {
14801
+ if (fileIntercept[path30] !== void 0) {
14802
+ return fileIntercept[path30];
14691
14803
  }
14692
- if (!filePromises[path29] || options?.ignoreCache) {
14693
- filePromises[path29] = (0, import_promises3.readFile)(path29, "utf8");
14804
+ if (!filePromises[path30] || options?.ignoreCache) {
14805
+ filePromises[path30] = (0, import_promises3.readFile)(path30, "utf8");
14694
14806
  }
14695
- return filePromises[path29];
14807
+ return filePromises[path30];
14696
14808
  };
14697
14809
  }
14698
14810
  });
@@ -14804,8 +14916,8 @@ var init_externalDataInterceptor = __esm({
14804
14916
  getFileRecord() {
14805
14917
  return fileIntercept;
14806
14918
  },
14807
- interceptFile(path29, contents) {
14808
- fileIntercept[path29] = Promise.resolve(contents);
14919
+ interceptFile(path30, contents) {
14920
+ fileIntercept[path30] = Promise.resolve(contents);
14809
14921
  },
14810
14922
  getTokenRecord() {
14811
14923
  return tokenIntercept;
@@ -17587,17 +17699,17 @@ var init_split_header = __esm({
17587
17699
  });
17588
17700
 
17589
17701
  // ../../node_modules/@smithy/core/dist-es/submodules/serde/value/NumericValue.js
17590
- var format2, NumericValue;
17702
+ var format3, NumericValue;
17591
17703
  var init_NumericValue = __esm({
17592
17704
  "../../node_modules/@smithy/core/dist-es/submodules/serde/value/NumericValue.js"() {
17593
- format2 = /^-?\d*(\.\d+)?$/;
17705
+ format3 = /^-?\d*(\.\d+)?$/;
17594
17706
  NumericValue = class _NumericValue {
17595
17707
  string;
17596
17708
  type;
17597
17709
  constructor(string, type) {
17598
17710
  this.string = string;
17599
17711
  this.type = type;
17600
- if (!format2.test(string)) {
17712
+ if (!format3.test(string)) {
17601
17713
  throw new Error(`@smithy/core/serde - NumericValue must only contain [0-9], at most one decimal point ".", and an optional negation prefix "-".`);
17602
17714
  }
17603
17715
  }
@@ -17609,7 +17721,7 @@ var init_NumericValue = __esm({
17609
17721
  return false;
17610
17722
  }
17611
17723
  const _nv = object;
17612
- return _NumericValue.prototype.isPrototypeOf(object) || _nv.type === "bigDecimal" && format2.test(_nv.string);
17724
+ return _NumericValue.prototype.isPrototypeOf(object) || _nv.type === "bigDecimal" && format3.test(_nv.string);
17613
17725
  }
17614
17726
  };
17615
17727
  }
@@ -17885,8 +17997,8 @@ var init_createConfigValueProvider = __esm({
17885
17997
  return endpoint.url.href;
17886
17998
  }
17887
17999
  if ("hostname" in endpoint) {
17888
- const { protocol, hostname, port, path: path29 } = endpoint;
17889
- return `${protocol}//${hostname}${port ? ":" + port : ""}${path29}`;
18000
+ const { protocol, hostname, port, path: path30 } = endpoint;
18001
+ return `${protocol}//${hostname}${port ? ":" + port : ""}${path30}`;
17890
18002
  }
17891
18003
  }
17892
18004
  return endpoint;
@@ -18261,18 +18373,18 @@ var getAttrPathList;
18261
18373
  var init_getAttrPathList = __esm({
18262
18374
  "../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
18263
18375
  init_types2();
18264
- getAttrPathList = (path29) => {
18265
- const parts = path29.split(".");
18376
+ getAttrPathList = (path30) => {
18377
+ const parts = path30.split(".");
18266
18378
  const pathList = [];
18267
18379
  for (const part of parts) {
18268
18380
  const squareBracketIndex = part.indexOf("[");
18269
18381
  if (squareBracketIndex !== -1) {
18270
18382
  if (part.indexOf("]") !== part.length - 1) {
18271
- throw new EndpointError(`Path: '${path29}' does not end with ']'`);
18383
+ throw new EndpointError(`Path: '${path30}' does not end with ']'`);
18272
18384
  }
18273
18385
  const arrayIndex = part.slice(squareBracketIndex + 1, -1);
18274
18386
  if (Number.isNaN(parseInt(arrayIndex))) {
18275
- throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path29}'`);
18387
+ throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path30}'`);
18276
18388
  }
18277
18389
  if (squareBracketIndex !== 0) {
18278
18390
  pathList.push(part.slice(0, squareBracketIndex));
@@ -18293,9 +18405,9 @@ var init_getAttr = __esm({
18293
18405
  "../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttr.js"() {
18294
18406
  init_types2();
18295
18407
  init_getAttrPathList();
18296
- getAttr = (value, path29) => getAttrPathList(path29).reduce((acc, index) => {
18408
+ getAttr = (value, path30) => getAttrPathList(path30).reduce((acc, index) => {
18297
18409
  if (typeof acc !== "object") {
18298
- throw new EndpointError(`Index '${index}' in '${path29}' not found in '${JSON.stringify(value)}'`);
18410
+ throw new EndpointError(`Index '${index}' in '${path30}' not found in '${JSON.stringify(value)}'`);
18299
18411
  } else if (Array.isArray(acc)) {
18300
18412
  const i6 = parseInt(index);
18301
18413
  return acc[i6 < 0 ? acc.length + i6 : i6];
@@ -18356,8 +18468,8 @@ var init_parseURL = __esm({
18356
18468
  return value;
18357
18469
  }
18358
18470
  if (typeof value === "object" && "hostname" in value) {
18359
- const { hostname: hostname2, port, protocol: protocol2 = "", path: path29 = "", query = {} } = value;
18360
- const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path29}`);
18471
+ const { hostname: hostname2, port, protocol: protocol2 = "", path: path30 = "", query = {} } = value;
18472
+ const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path30}`);
18361
18473
  url.search = Object.entries(query).map(([k7, v3]) => `${k7}=${v3}`).join("&");
18362
18474
  return url;
18363
18475
  }
@@ -20701,11 +20813,11 @@ var init_HttpBindingProtocol = __esm({
20701
20813
  const opTraits = translateTraits(operationSchema.traits);
20702
20814
  if (opTraits.http) {
20703
20815
  request.method = opTraits.http[0];
20704
- const [path29, search] = opTraits.http[1].split("?");
20816
+ const [path30, search] = opTraits.http[1].split("?");
20705
20817
  if (request.path == "/") {
20706
- request.path = path29;
20818
+ request.path = path30;
20707
20819
  } else {
20708
- request.path += path29;
20820
+ request.path += path30;
20709
20821
  }
20710
20822
  const traitSearchParams = new URLSearchParams(search ?? "");
20711
20823
  for (const [key, value] of traitSearchParams) {
@@ -21074,8 +21186,8 @@ var init_FromStringShapeDeserializer = __esm({
21074
21186
  return (this.serdeContext?.base64Decoder ?? fromBase64)(data);
21075
21187
  }
21076
21188
  if (ns2.isTimestampSchema()) {
21077
- const format3 = determineTimestampFormat(ns2, this.settings);
21078
- switch (format3) {
21189
+ const format4 = determineTimestampFormat(ns2, this.settings);
21190
+ switch (format4) {
21079
21191
  case 5:
21080
21192
  return _parseRfc3339DateTimeWithOffset(data);
21081
21193
  case 6:
@@ -21197,8 +21309,8 @@ var init_ToStringShapeSerializer = __esm({
21197
21309
  if (!(value instanceof Date)) {
21198
21310
  throw new Error(`@smithy/core/protocols - received non-Date value ${value} when schema expected Date in ${ns2.getName(true)}`);
21199
21311
  }
21200
- const format3 = determineTimestampFormat(ns2, this.settings);
21201
- switch (format3) {
21312
+ const format4 = determineTimestampFormat(ns2, this.settings);
21313
+ switch (format4) {
21202
21314
  case 5:
21203
21315
  this.stringBuffer = value.toISOString().replace(".000Z", "Z");
21204
21316
  break;
@@ -24418,12 +24530,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
24418
24530
  const password = request.password ?? "";
24419
24531
  auth = `${username}:${password}`;
24420
24532
  }
24421
- let path29 = request.path;
24533
+ let path30 = request.path;
24422
24534
  if (queryString) {
24423
- path29 += `?${queryString}`;
24535
+ path30 += `?${queryString}`;
24424
24536
  }
24425
24537
  if (request.fragment) {
24426
- path29 += `#${request.fragment}`;
24538
+ path30 += `#${request.fragment}`;
24427
24539
  }
24428
24540
  let hostname = request.hostname ?? "";
24429
24541
  if (hostname[0] === "[" && hostname.endsWith("]")) {
@@ -24435,7 +24547,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
24435
24547
  headers: request.headers,
24436
24548
  host: hostname,
24437
24549
  method: request.method,
24438
- path: path29,
24550
+ path: path30,
24439
24551
  port: request.port,
24440
24552
  agent,
24441
24553
  auth
@@ -25344,10 +25456,10 @@ ${longDate}
25344
25456
  ${credentialScope}
25345
25457
  ${toHex(hashedRequest)}`;
25346
25458
  }
25347
- getCanonicalPath({ path: path29 }) {
25459
+ getCanonicalPath({ path: path30 }) {
25348
25460
  if (this.uriEscapePath) {
25349
25461
  const normalizedPathSegments = [];
25350
- for (const pathSegment of path29.split("/")) {
25462
+ for (const pathSegment of path30.split("/")) {
25351
25463
  if (pathSegment?.length === 0)
25352
25464
  continue;
25353
25465
  if (pathSegment === ".")
@@ -25358,11 +25470,11 @@ ${toHex(hashedRequest)}`;
25358
25470
  normalizedPathSegments.push(pathSegment);
25359
25471
  }
25360
25472
  }
25361
- const normalizedPath = `${path29?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path29?.endsWith("/") ? "/" : ""}`;
25473
+ const normalizedPath = `${path30?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path30?.endsWith("/") ? "/" : ""}`;
25362
25474
  const doubleEncoded = escapeUri(normalizedPath);
25363
25475
  return doubleEncoded.replace(/%2F/g, "/");
25364
25476
  }
25365
- return path29;
25477
+ return path30;
25366
25478
  }
25367
25479
  validateResolvedCredentials(credentials) {
25368
25480
  if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
@@ -26432,8 +26544,8 @@ var init_JsonShapeDeserializer = __esm({
26432
26544
  return value;
26433
26545
  }
26434
26546
  if (ns2.isTimestampSchema() && value != null) {
26435
- const format3 = determineTimestampFormat(ns2, this.settings);
26436
- switch (format3) {
26547
+ const format4 = determineTimestampFormat(ns2, this.settings);
26548
+ switch (format4) {
26437
26549
  case 5:
26438
26550
  return parseRfc3339DateTimeWithOffset(value);
26439
26551
  case 6:
@@ -26657,8 +26769,8 @@ var init_JsonShapeSerializer = __esm({
26657
26769
  return (this.serdeContext?.base64Encoder ?? toBase64)(value);
26658
26770
  }
26659
26771
  if (value instanceof Date && (ns2.isTimestampSchema() || ns2.isDocumentSchema())) {
26660
- const format3 = determineTimestampFormat(ns2, this.settings);
26661
- switch (format3) {
26772
+ const format4 = determineTimestampFormat(ns2, this.settings);
26773
+ switch (format4) {
26662
26774
  case 5:
26663
26775
  return value.toISOString().replace(".000Z", "Z");
26664
26776
  case 6:
@@ -27286,8 +27398,8 @@ var init_QueryShapeSerializer = __esm({
27286
27398
  } else if (ns2.isTimestampSchema()) {
27287
27399
  if (value instanceof Date) {
27288
27400
  this.writeKey(prefix2);
27289
- const format3 = determineTimestampFormat(ns2, this.settings);
27290
- switch (format3) {
27401
+ const format4 = determineTimestampFormat(ns2, this.settings);
27402
+ switch (format4) {
27291
27403
  case 5:
27292
27404
  this.writeValue(value.toISOString().replace(".000Z", "Z"));
27293
27405
  break;
@@ -41722,11 +41834,13 @@ var init_src7 = __esm({
41722
41834
  });
41723
41835
 
41724
41836
  // ../../packages/drivers/src/DeltaShareDriver.ts
41725
- var fs10, import_promises7, MAX_LIMIT_HINT, DeltaShareSourceDriver, DeltaShareDriver_default;
41837
+ var fs10, import_promises7, import_stream, import_promises8, MAX_LIMIT_HINT, PARQUET_BATCH_ROWS, DeltaShareSourceDriver, DeltaShareDriver_default;
41726
41838
  var init_DeltaShareDriver = __esm({
41727
41839
  "../../packages/drivers/src/DeltaShareDriver.ts"() {
41728
41840
  fs10 = __toESM(require("fs"), 1);
41729
41841
  import_promises7 = __toESM(require("fs/promises"), 1);
41842
+ import_stream = require("stream");
41843
+ import_promises8 = require("stream/promises");
41730
41844
  init_src();
41731
41845
  init_src4();
41732
41846
  init_src2();
@@ -41734,6 +41848,7 @@ var init_DeltaShareDriver = __esm({
41734
41848
  init_DeltaSharePredicate();
41735
41849
  init_OidcTokenProvider();
41736
41850
  MAX_LIMIT_HINT = 2147483647;
41851
+ PARQUET_BATCH_ROWS = 5e4;
41737
41852
  DeltaShareSourceDriver = class {
41738
41853
  constructor() {
41739
41854
  this._query = "{prefix}/shares/{share}/schemas/{schema}/tables/{table}/query";
@@ -41743,6 +41858,9 @@ var init_DeltaShareDriver = __esm({
41743
41858
  // Column Delta types per table, for the predicate's literals. A table's schema does not change
41744
41859
  // under a run, and only a read with something to push ever asks for it.
41745
41860
  this._columnTypes = /* @__PURE__ */ new Map();
41861
+ // Rows held as objects at once. A run leaves this at PARQUET_BATCH_ROWS; the tests lower it so a
41862
+ // fixture small enough to read whole still splits its row groups the way a real part file does.
41863
+ this._batchRows = PARQUET_BATCH_ROWS;
41746
41864
  this.init = async (source) => {
41747
41865
  Affirm_default(source, "Invalid source");
41748
41866
  const { authentication } = source;
@@ -41803,12 +41921,11 @@ var init_DeltaShareDriver = __esm({
41803
41921
  Affirm_default(request, "Invalid download request");
41804
41922
  const table = this._resolveTable(request.fileKey);
41805
41923
  const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
41806
- const hyparquet = await import("hyparquet");
41807
41924
  const lines = [];
41808
- for (const deltaFile of deltaFiles) {
41809
- const parquetRecords = await this._readParquetObjects(deltaFile);
41810
- lines.push(...parquetRecords.map((x2) => JSON.stringify(hyparquet.toJson(x2))));
41811
- }
41925
+ for (const deltaFile of deltaFiles)
41926
+ for await (const batch of this._readRemoteParquetBatches(deltaFile))
41927
+ for (const record of batch)
41928
+ lines.push(JSON.stringify(record));
41812
41929
  Logger_default.log(`Delta Share readAll: ${lines.length} record(s) from ${deltaFiles.length} file(s) of table "${table}"`);
41813
41930
  return lines;
41814
41931
  };
@@ -41818,22 +41935,41 @@ var init_DeltaShareDriver = __esm({
41818
41935
  Affirm_default(request.options.lineFrom !== void 0 && request.options.lineTo !== void 0, "Missing read range");
41819
41936
  const table = this._resolveTable(request.fileKey);
41820
41937
  const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
41821
- const hyparquet = await import("hyparquet");
41822
41938
  const { options: { lineFrom, lineTo } } = request;
41939
+ const lines = await this._collectLinesInRange(deltaFiles, { from: lineFrom, to: lineTo });
41940
+ Logger_default.log(`Delta Share readLinesInRange [${lineFrom}, ${lineTo}) on table "${table}": ${lines.length} record(s)`);
41941
+ return lines;
41942
+ };
41943
+ /**
41944
+ * The records of the given range, serialized. Reading stops the moment the range is filled: a
41945
+ * part file holds hundreds of thousands of rows and a preview asks for a handful of them, so
41946
+ * decoding past `to` is both slow and the memory the batching above exists to avoid.
41947
+ */
41948
+ this._collectLinesInRange = async (deltaFiles, range2) => {
41949
+ if (range2.to <= range2.from)
41950
+ return [];
41823
41951
  const lines = [];
41824
41952
  let index = 0;
41825
41953
  for (const deltaFile of deltaFiles) {
41826
- const parquetRecords = await this._readParquetObjects(deltaFile);
41827
- for (const record of parquetRecords) {
41828
- if (index >= lineFrom && index < lineTo)
41829
- lines.push(JSON.stringify(hyparquet.toJson(record)));
41830
- index++;
41831
- if (index >= lineTo) break;
41954
+ for await (const batch of this._readRemoteParquetBatches(deltaFile)) {
41955
+ index = this._pushRecordsInRange(lines, batch, index, range2);
41956
+ if (index >= range2.to)
41957
+ return lines;
41832
41958
  }
41833
41959
  }
41834
- Logger_default.log(`Delta Share readLinesInRange [${lineFrom}, ${lineTo}) on table "${table}": ${lines.length} record(s)`);
41835
41960
  return lines;
41836
41961
  };
41962
+ // The index the batch ends on, so the next batch carries on counting where this one stopped.
41963
+ this._pushRecordsInRange = (lines, batch, index, range2) => {
41964
+ for (const record of batch) {
41965
+ if (index >= range2.from && index < range2.to)
41966
+ lines.push(JSON.stringify(record));
41967
+ index++;
41968
+ if (index >= range2.to)
41969
+ break;
41970
+ }
41971
+ return index;
41972
+ };
41837
41973
  this.exist = async (_producer) => {
41838
41974
  void _producer;
41839
41975
  let nextPageToken;
@@ -41977,20 +42113,48 @@ var init_DeltaShareDriver = __esm({
41977
42113
  return "(unparseable url)";
41978
42114
  }
41979
42115
  };
41980
- this._readParquetObjects = async (deltaFile) => {
42116
+ // Split out so the batch loop above stays one level of nesting, and so both failures name the file
42117
+ this._readParquetMetadata = async (file, label) => {
41981
42118
  const hyparquet = await import("hyparquet");
41982
- const { compressors: compressors2 } = await Promise.resolve().then(() => (init_src7(), src_exports));
41983
- const byteLength = deltaFile.file.deltaSingleAction.add?.size ?? deltaFile.file.deltaSingleAction.remove?.size;
41984
- const startedMs = Date.now();
41985
- let records;
41986
42119
  try {
41987
- const file = await hyparquet.asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
41988
- records = await hyparquet.parquetReadObjects({ file, compressors: compressors2 });
42120
+ return await hyparquet.parquetMetadataAsync(file);
41989
42121
  } catch (error) {
41990
- throw new Error(`Delta Share failed reading parquet from ${this._redactUrl(deltaFile.file.url)} (${byteLength ?? "?"} bytes)`, { cause: error });
42122
+ throw new Error(`Delta Share failed reading the parquet footer of ${label}`, { cause: error });
41991
42123
  }
41992
- Logger_default.log(`Delta Share read ${records.length} record(s) from ${this._redactUrl(deltaFile.file.url)} (${byteLength ?? "?"} bytes) in ${Date.now() - startedMs}ms`);
41993
- return records;
42124
+ };
42125
+ this._readParquetBatch = async (options, label) => {
42126
+ const hyparquet = await import("hyparquet");
42127
+ try {
42128
+ return await hyparquet.parquetReadObjects(options);
42129
+ } catch (error) {
42130
+ throw new Error(`Delta Share failed reading parquet rows [${options.rowStart}, ${options.rowEnd}) of ${label}`, { cause: error });
42131
+ }
42132
+ };
42133
+ // Reads straight from the pre-signed URL. Only for the bounded readers (preview, sample): every
42134
+ // batch refetches the row group, which is why `ready` stages the file locally instead.
42135
+ this._readRemoteParquetBatches = (deltaFile) => {
42136
+ const byteLength = deltaFile.file.deltaSingleAction.add?.size ?? deltaFile.file.deltaSingleAction.remove?.size;
42137
+ const label = `${this._redactUrl(deltaFile.file.url)} (${byteLength ?? "?"} bytes)`;
42138
+ const open = async () => {
42139
+ const hyparquet = await import("hyparquet");
42140
+ return await hyparquet.asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
42141
+ };
42142
+ return this._streamParquetBatchesFrom(open, label);
42143
+ };
42144
+ /**
42145
+ * Copies a part file to local disk, streamed, so nothing but the socket buffer is ever in memory.
42146
+ * `ready` re-reads the file once per batch and re-reading it over HTTP would refetch the same
42147
+ * bytes every time.
42148
+ */
42149
+ this._downloadParquet = async (deltaFile, localPath) => {
42150
+ const startedMs = Date.now();
42151
+ const res = await this._fetch(deltaFile.file.url, { method: "GET" }, `parquet download of ${this._redactUrl(deltaFile.file.url)}`);
42152
+ if (!res.ok)
42153
+ throw new Error(`Error downloading parquet from the delta share: ${res.status} ${res.statusText} (${this._redactUrl(deltaFile.file.url)})`);
42154
+ Affirm_default(res.body, `Empty response body downloading parquet from ${this._redactUrl(deltaFile.file.url)}`);
42155
+ await (0, import_promises8.pipeline)(import_stream.Readable.fromWeb(res.body), fs10.createWriteStream(localPath));
42156
+ const { size } = await import_promises7.default.stat(localPath);
42157
+ Logger_default.log(`Delta Share staged ${this._redactUrl(deltaFile.file.url)} (${size} bytes) in ${Date.now() - startedMs}ms`);
41994
42158
  };
41995
42159
  this._serializeDatasetValue = (value) => {
41996
42160
  if (value === null || value === void 0)
@@ -42004,17 +42168,41 @@ var init_DeltaShareDriver = __esm({
42004
42168
  this._getDeltaFileKey = (table, index) => {
42005
42169
  return `${this._resolveSchema()}/${table}/part-${index}.parquet`;
42006
42170
  };
42007
- this._writeDatasetFile = async (localPath, records, delimiter) => {
42008
- if (records.length === 0)
42009
- return false;
42010
- const header = Object.keys(records[0]);
42011
- const lines = [
42012
- CSVParser_default.stringifyRow(header, delimiter),
42013
- ...records.map((record) => CSVParser_default.stringifyRow(header.map((column) => this._serializeDatasetValue(record[column])), delimiter))
42014
- ];
42015
- await import_promises7.default.writeFile(localPath, `${lines.join("\n")}
42016
- `);
42017
- return true;
42171
+ /**
42172
+ * Writes the batches out as one CSV dataset file and reports how many records landed. Rows are
42173
+ * written as they arrive rather than joined into one string first: a file's worth of lines, and
42174
+ * the single string joining them, were each as large as the records themselves.
42175
+ *
42176
+ * The header is taken from the first record and reused for every later one, so a file whose rows
42177
+ * disagree on their columns still produces a rectangular dataset.
42178
+ */
42179
+ this._writeDatasetFile = async (localPath, batches, delimiter) => {
42180
+ const stream = fs10.createWriteStream(localPath);
42181
+ let header = null;
42182
+ let records = 0;
42183
+ try {
42184
+ for await (const batch of batches) {
42185
+ for (const record of batch) {
42186
+ if (!header) {
42187
+ header = Object.keys(record);
42188
+ await this._writeLine(stream, CSVParser_default.stringifyRow(header, delimiter));
42189
+ }
42190
+ await this._writeLine(stream, CSVParser_default.stringifyRow(header.map((column) => this._serializeDatasetValue(record[column])), delimiter));
42191
+ records++;
42192
+ }
42193
+ }
42194
+ } finally {
42195
+ await new Promise((resolve) => stream.end(resolve));
42196
+ }
42197
+ return records;
42198
+ };
42199
+ // A false from write() means the buffer is full; waiting for the drain just then keeps the queued
42200
+ // rows bounded instead of letting them pile up faster than the disk takes them.
42201
+ this._writeLine = async (stream, line) => {
42202
+ if (stream.write(`${line}
42203
+ `))
42204
+ return;
42205
+ await new Promise((resolve) => stream.once("drain", resolve));
42018
42206
  };
42019
42207
  /**
42020
42208
  * Every file the share reports for this table. `jsonPredicateHints` lets the server skip the
@@ -42078,30 +42266,22 @@ var init_DeltaShareDriver = __esm({
42078
42266
  const table = this._resolveTable(producer.settings.fileKey);
42079
42267
  const hints = await this._buildHints(producer, table, request.pushdown);
42080
42268
  const deltaFiles = await this._getAllFilesInTable(table, producer.settings.disableHistory, hints);
42081
- const hyparquet = await import("hyparquet");
42082
42269
  const delimiter = producer.settings.delimiter ?? ",";
42083
42270
  const files = [];
42084
42271
  let records = 0;
42085
42272
  let skipped = 0;
42086
42273
  for (const [index, deltaFile] of deltaFiles.entries()) {
42087
42274
  onProgress?.(index / deltaFiles.length);
42088
- const parquetRecords = await this._readParquetObjects(deltaFile);
42089
- if (parquetRecords.length === 0) {
42090
- skipped++;
42091
- Logger_default.log(`Delta Share ready: file ${index + 1}/${deltaFiles.length} of table "${table}" holds no records, skipping`);
42092
- continue;
42093
- }
42094
42275
  const sourceFileKey = this._getDeltaFileKey(table, index);
42095
42276
  const localPath = ExecutorScope_default.getProducerPath(scope, producer, sourceFileKey);
42096
42277
  ExecutorScope_default.ensurePath(localPath);
42097
- const jsonRecords = parquetRecords.map((record) => hyparquet.toJson(record));
42098
- const written = await this._writeDatasetFile(localPath, jsonRecords, delimiter);
42099
- if (written) {
42278
+ const written = await this._stageDeltaFile(deltaFile, localPath, delimiter);
42279
+ if (written > 0) {
42100
42280
  files.push({ fullUri: localPath, fileSize: fs10.statSync(localPath).size });
42101
- records += jsonRecords.length;
42281
+ records += written;
42102
42282
  } else {
42103
42283
  skipped++;
42104
- Logger_default.log(`Delta Share ready: nothing written for ${sourceFileKey}, discarding the staged file`);
42284
+ Logger_default.log(`Delta Share ready: file ${index + 1}/${deltaFiles.length} of table "${table}" (${sourceFileKey}) holds no records, discarding the staged file`);
42105
42285
  await import_promises7.default.unlink(localPath).catch(() => {
42106
42286
  });
42107
42287
  }
@@ -42112,6 +42292,57 @@ var init_DeltaShareDriver = __esm({
42112
42292
  Logger_default.log(`Delta Share ready: staged ${records} record(s) into ${files.length} dataset file(s) (${bytes} bytes) for producer "${producer.name}" from table "${table}"${skippedNote}`);
42113
42293
  return { files, filteredAtSource: !!hints?.jsonPredicateHints };
42114
42294
  };
42295
+ /**
42296
+ * One part file turned into one CSV dataset file, downloaded then converted a batch at a time so
42297
+ * that neither step's memory grows with the file. The download is dropped either way — it is the
42298
+ * dataset file the run reads from here on.
42299
+ */
42300
+ this._stageDeltaFile = async (deltaFile, localPath, delimiter) => {
42301
+ const downloadPath = `${localPath}.parquet`;
42302
+ await this._downloadParquet(deltaFile, downloadPath);
42303
+ try {
42304
+ const open = async () => {
42305
+ const hyparquet = await import("hyparquet");
42306
+ return await hyparquet.asyncBufferFromFile(downloadPath);
42307
+ };
42308
+ return await this._writeDatasetFile(localPath, this._streamParquetBatchesFrom(open, this._redactUrl(deltaFile.file.url)), delimiter);
42309
+ } finally {
42310
+ await import_promises7.default.unlink(downloadPath).catch(() => {
42311
+ });
42312
+ }
42313
+ };
42314
+ }
42315
+ /**
42316
+ * The rows of a part file, `_batchRows` at a time and already JSON-safe.
42317
+ *
42318
+ * hyparquet decodes a whole row group's columns per read either way, but only the requested rows
42319
+ * are turned into objects — and the objects are what dominates — so a row group larger than a
42320
+ * batch is read in several passes. Every pass re-reads the group's bytes, which is why `ready`
42321
+ * stages the file on disk first and only the small readers below stream it from its URL.
42322
+ */
42323
+ async *_streamParquetBatches(file, label) {
42324
+ const hyparquet = await import("hyparquet");
42325
+ const { compressors: compressors2 } = await Promise.resolve().then(() => (init_src7(), src_exports));
42326
+ const startedMs = Date.now();
42327
+ const metadata = await this._readParquetMetadata(file, label);
42328
+ let records = 0;
42329
+ let groupStart = 0;
42330
+ for (const rowGroup of metadata.row_groups) {
42331
+ const groupRows = Number(rowGroup.num_rows);
42332
+ const groupEnd = groupStart + groupRows;
42333
+ for (let rowStart = groupStart; rowStart < groupEnd; rowStart += this._batchRows) {
42334
+ const rowEnd = Math.min(rowStart + this._batchRows, groupEnd);
42335
+ const batch = await this._readParquetBatch({ file, metadata, compressors: compressors2, rowStart, rowEnd }, label);
42336
+ records += batch.length;
42337
+ yield batch.map((record) => hyparquet.toJson(record));
42338
+ }
42339
+ groupStart = groupEnd;
42340
+ }
42341
+ Logger_default.log(`Delta Share read ${records} record(s) from ${label} in ${Date.now() - startedMs}ms`);
42342
+ }
42343
+ // The buffer is opened inside the generator so nothing is fetched until the first batch is pulled
42344
+ async *_streamParquetBatchesFrom(open, label) {
42345
+ yield* this._streamParquetBatches(await open(), label);
42115
42346
  }
42116
42347
  };
42117
42348
  DeltaShareDriver_default = DeltaShareSourceDriver;
@@ -42445,12 +42676,12 @@ var init_HttpApiDriver = __esm({
42445
42676
  });
42446
42677
 
42447
42678
  // ../../packages/drivers/src/DriverHelper.ts
42448
- var import_stream, import_readline, import_promises8, import_fs9, DriverHelper, DriverHelper_default;
42679
+ var import_stream2, import_readline, import_promises9, import_fs9, DriverHelper, DriverHelper_default;
42449
42680
  var init_DriverHelper = __esm({
42450
42681
  "../../packages/drivers/src/DriverHelper.ts"() {
42451
- import_stream = require("stream");
42682
+ import_stream2 = require("stream");
42452
42683
  import_readline = require("readline");
42453
- import_promises8 = require("stream/promises");
42684
+ import_promises9 = require("stream/promises");
42454
42685
  import_fs9 = require("fs");
42455
42686
  init_src2();
42456
42687
  init_src();
@@ -42467,7 +42698,7 @@ var init_DriverHelper = __esm({
42467
42698
  let leftoverData = "";
42468
42699
  let globalIndex = 0;
42469
42700
  let lineCount = 0;
42470
- const headerValidationTransform = new import_stream.Transform({
42701
+ const headerValidationTransform = new import_stream2.Transform({
42471
42702
  transform(chunk, encoding, callback) {
42472
42703
  const chunkStr = leftoverData + chunk.toString();
42473
42704
  const lines = chunkStr.split("\n");
@@ -42543,7 +42774,7 @@ var init_DriverHelper = __esm({
42543
42774
  };
42544
42775
  const writeOptions = append ? { flags: "a" } : {};
42545
42776
  const writeStream = (0, import_fs9.createWriteStream)(destinationPath, writeOptions);
42546
- await (0, import_promises8.pipeline)(stream, headerValidationTransform, writeStream);
42777
+ await (0, import_promises9.pipeline)(stream, headerValidationTransform, writeStream);
42547
42778
  return lineCount;
42548
42779
  },
42549
42780
  appendObjectsToUnifiedFile: async (options) => {
@@ -42569,32 +42800,32 @@ var init_DriverHelper = __esm({
42569
42800
  },
42570
42801
  quickReadFile: async (filePath, lineCount) => {
42571
42802
  const fileStream = (0, import_fs9.createReadStream)(filePath);
42572
- const rl6 = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
42803
+ const rl7 = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
42573
42804
  const lines = [];
42574
42805
  let counter = 0;
42575
- for await (const line of rl6) {
42806
+ for await (const line of rl7) {
42576
42807
  lines.push(line);
42577
42808
  counter++;
42578
42809
  if (counter >= lineCount) {
42579
42810
  break;
42580
42811
  }
42581
42812
  }
42582
- rl6.close();
42813
+ rl7.close();
42583
42814
  fileStream.close();
42584
42815
  return lines;
42585
42816
  },
42586
42817
  quickReadStream: async (stream, lineCount) => {
42587
- const rl6 = (0, import_readline.createInterface)({ input: stream, crlfDelay: Infinity });
42818
+ const rl7 = (0, import_readline.createInterface)({ input: stream, crlfDelay: Infinity });
42588
42819
  const lines = [];
42589
42820
  let counter = 0;
42590
- for await (const line of rl6) {
42821
+ for await (const line of rl7) {
42591
42822
  lines.push(line);
42592
42823
  counter++;
42593
42824
  if (counter >= lineCount) {
42594
42825
  break;
42595
42826
  }
42596
42827
  }
42597
- rl6.close();
42828
+ rl7.close();
42598
42829
  return lines;
42599
42830
  }
42600
42831
  };
@@ -42853,11 +43084,11 @@ var init_LocalSourceDriver = __esm({
42853
43084
  });
42854
43085
 
42855
43086
  // ../../packages/drivers/src/files/LocalDestinationDriver.ts
42856
- var fs12, import_promises9, import_readline3, import_path10, LocalDestinationDriver;
43087
+ var fs12, import_promises10, import_readline3, import_path10, LocalDestinationDriver;
42857
43088
  var init_LocalDestinationDriver = __esm({
42858
43089
  "../../packages/drivers/src/files/LocalDestinationDriver.ts"() {
42859
43090
  fs12 = __toESM(require("fs"), 1);
42860
- import_promises9 = __toESM(require("fs/promises"), 1);
43091
+ import_promises10 = __toESM(require("fs/promises"), 1);
42861
43092
  import_readline3 = __toESM(require("readline"), 1);
42862
43093
  import_path10 = __toESM(require("path"), 1);
42863
43094
  init_src();
@@ -42892,8 +43123,8 @@ var init_LocalDestinationDriver = __esm({
42892
43123
  this.move = async (fromPath, toName) => {
42893
43124
  try {
42894
43125
  const toFilePath = import_path10.default.join(this._path, toName);
42895
- await import_promises9.default.copyFile(fromPath, toFilePath);
42896
- await import_promises9.default.unlink(fromPath);
43126
+ await import_promises10.default.copyFile(fromPath, toFilePath);
43127
+ await import_promises10.default.unlink(fromPath);
42897
43128
  return { bucket: "", key: toFilePath, res: true };
42898
43129
  } catch (error) {
42899
43130
  Logger_default.error(error);
@@ -42929,14 +43160,14 @@ var init_LocalDestinationDriver = __esm({
42929
43160
  });
42930
43161
 
42931
43162
  // ../../packages/drivers/src/s3/S3SourceDriver.ts
42932
- var import_client_s32, import_stream2, import_promises10, import_readline4, import_path11, import_fs10, import_xlsx2, S3SourceDriver;
43163
+ var import_client_s32, import_stream3, import_promises11, import_readline4, import_path11, import_fs10, import_xlsx2, S3SourceDriver;
42933
43164
  var init_S3SourceDriver = __esm({
42934
43165
  "../../packages/drivers/src/s3/S3SourceDriver.ts"() {
42935
43166
  import_client_s32 = require("@aws-sdk/client-s3");
42936
43167
  init_src();
42937
43168
  init_AwsCredentials();
42938
- import_stream2 = require("stream");
42939
- import_promises10 = require("stream/promises");
43169
+ import_stream3 = require("stream");
43170
+ import_promises11 = require("stream/promises");
42940
43171
  import_readline4 = __toESM(require("readline"), 1);
42941
43172
  import_path11 = __toESM(require("path"), 1);
42942
43173
  import_fs10 = __toESM(require("fs"), 1);
@@ -43204,7 +43435,7 @@ var init_S3SourceDriver = __esm({
43204
43435
  }
43205
43436
  const totalBytes = response.ContentLength ?? 0;
43206
43437
  let stagedBytes = 0;
43207
- const meter = new import_stream2.Transform({
43438
+ const meter = new import_stream3.Transform({
43208
43439
  transform(chunk, _encoding, done) {
43209
43440
  stagedBytes += chunk.length;
43210
43441
  if (totalBytes > 0)
@@ -43213,7 +43444,7 @@ var init_S3SourceDriver = __esm({
43213
43444
  }
43214
43445
  });
43215
43446
  const writeStream = import_fs10.default.createWriteStream(localPath);
43216
- await (0, import_promises10.pipeline)(response.Body, meter, writeStream);
43447
+ await (0, import_promises11.pipeline)(response.Body, meter, writeStream);
43217
43448
  report3(1);
43218
43449
  };
43219
43450
  if (fileKey.includes("%")) {
@@ -43368,7 +43599,7 @@ var init_S3DestinationDriver = __esm({
43368
43599
  });
43369
43600
 
43370
43601
  // ../../packages/drivers/src/DriverFactory.ts
43371
- var DriverFactoryClass, DriverFactory, DriverFactory_default;
43602
+ var STAGEABLE_ENGINES, DriverFactoryClass, DriverFactory, DriverFactory_default;
43372
43603
  var init_DriverFactory = __esm({
43373
43604
  "../../packages/drivers/src/DriverFactory.ts"() {
43374
43605
  init_RedshiftDriver();
@@ -43378,8 +43609,11 @@ var init_DriverFactory = __esm({
43378
43609
  init_LocalDestinationDriver();
43379
43610
  init_S3SourceDriver();
43380
43611
  init_S3DestinationDriver();
43612
+ STAGEABLE_ENGINES = ["local", "aws-s3", "delta-share"];
43381
43613
  DriverFactoryClass = class {
43382
43614
  constructor() {
43615
+ /** Whether a run can stage a producer from this engine — see `STAGEABLE_ENGINES`. */
43616
+ this.canStage = (engine) => STAGEABLE_ENGINES.includes(engine);
43383
43617
  this.instantiateSource = async (source) => {
43384
43618
  switch (source.engine) {
43385
43619
  case "aws-redshift": {
@@ -44132,6 +44366,7 @@ var init_ProducerManager = __esm({
44132
44366
  init_src4();
44133
44367
  init_src3();
44134
44368
  init_src6();
44369
+ init_CryptoEngine();
44135
44370
  ProducerManagerClass = class {
44136
44371
  constructor() {
44137
44372
  this.getColumns = (producer) => {
@@ -44186,6 +44421,46 @@ var init_ProducerManager = __esm({
44186
44421
  return true;
44187
44422
  return FixedWidthParser_default.matchesSelector(line, resolved.selector, resolved.range);
44188
44423
  };
44424
+ /**
44425
+ * The post-parse work a producer implies for every record it produces: which keys carry the
44426
+ * source file name, and which carry a mask.
44427
+ *
44428
+ * Resolved once per chunk, never per line — `getMask` resolves a secret and validates the value,
44429
+ * and neither depends on the record. Empty for almost every producer, which is the case worth
44430
+ * keeping fast.
44431
+ */
44432
+ this.compileFills = (dimensions) => {
44433
+ Affirm_default(dimensions, "Invalid producer dimensions");
44434
+ const fills = [];
44435
+ for (const dimension of dimensions) {
44436
+ const { prodDimension } = dimension;
44437
+ const mask = this.getMask(prodDimension);
44438
+ const sourceFilename = prodDimension.sourceFilename === true;
44439
+ if (!sourceFilename && !Algo_default.hasVal(mask))
44440
+ continue;
44441
+ fills.push({ key: dimension.name, mask, sourceFilename, type: prodDimension.type });
44442
+ }
44443
+ return fills;
44444
+ };
44445
+ /**
44446
+ * Applies the compiled fills to one parsed record, in place.
44447
+ *
44448
+ * The one place a producer dimension's mask is ever applied, on purpose: a run reaches it through
44449
+ * `ProducerExecutor.processLine` and a sample through `ProducerEngine`, and a masked dimension
44450
+ * that came back in the clear from one of the two would be a `hash` the project declared and the
44451
+ * screen ignored — a disclosure, not a display bug.
44452
+ */
44453
+ this.applyFills = (record, fills, sourceFilename) => {
44454
+ if (!fills?.length)
44455
+ return record;
44456
+ for (const fill2 of fills) {
44457
+ if (fill2.sourceFilename)
44458
+ record[fill2.key] = sourceFilename;
44459
+ if (Algo_default.hasVal(fill2.mask))
44460
+ record[fill2.key] = CryptoEngine_default.hashValue(fill2.mask, record[fill2.key]?.toString(), fill2.type);
44461
+ }
44462
+ return record;
44463
+ };
44189
44464
  }
44190
44465
  };
44191
44466
  ProducerManager = new ProducerManagerClass();
@@ -44222,7 +44497,7 @@ var init_TypeCaster = __esm({
44222
44497
  * - Parsing dates (type 'date'/'datetime') from string with tokens: yyyy, mm, dd
44223
44498
  * - Formatting dates when casting to string with same tokens
44224
44499
  */
44225
- cast(value, type, format3) {
44500
+ cast(value, type, format4) {
44226
44501
  if (!Algo_default.hasVal(value)) return value;
44227
44502
  switch (type) {
44228
44503
  case "boolean": {
@@ -44233,8 +44508,8 @@ var init_TypeCaster = __esm({
44233
44508
  case "date": {
44234
44509
  let dateValue = null;
44235
44510
  try {
44236
- if (format3 && typeof value === "string")
44237
- dateValue = this.parseDate(value, format3, true).toDate();
44511
+ if (format4 && typeof value === "string")
44512
+ dateValue = this.parseDate(value, format4, true).toDate();
44238
44513
  else
44239
44514
  dateValue = new Date(value);
44240
44515
  return dateValue.toISOString();
@@ -44242,7 +44517,7 @@ var init_TypeCaster = __esm({
44242
44517
  dateValue = new Date(value);
44243
44518
  if (!isNaN(dateValue))
44244
44519
  return dateValue.toISOString();
44245
- throw new Error(`Error casting "${value}" to date with format "${format3}": ${error}`);
44520
+ throw new Error(`Error casting "${value}" to date with format "${format4}": ${error}`);
44246
44521
  }
44247
44522
  }
44248
44523
  case "number": {
@@ -44269,13 +44544,13 @@ var init_TypeCaster = __esm({
44269
44544
  * advancedFormat plugin, and then unpadded for `DDD`, while a positional ordinal date is always
44270
44545
  * 3 digits wide; the token is therefore rendered here rather than handed to dayjs.
44271
44546
  */
44272
- formatDate(value, format3) {
44547
+ formatDate(value, format4) {
44273
44548
  const date2 = import_dayjs.default.utc(value);
44274
- if (!DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format3)))
44275
- return date2.format(format3);
44549
+ if (!DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format4)))
44550
+ return date2.format(format4);
44276
44551
  const startOfYear = import_dayjs.default.utc(`${String(date2.year()).padStart(4, "0")}-01-01`, "YYYY-MM-DD", true);
44277
44552
  const dayOfYear = date2.diff(startOfYear, "day") + 1;
44278
- return date2.format(format3.replace(/D{3,4}/g, `[${String(dayOfYear).padStart(3, "0")}]`));
44553
+ return date2.format(format4.replace(/D{3,4}/g, `[${String(dayOfYear).padStart(3, "0")}]`));
44279
44554
  }
44280
44555
  /**
44281
44556
  * Parses a date string with a dayjs format, as UTC. Single entry point for every date parse, so
@@ -44284,13 +44559,13 @@ var init_TypeCaster = __esm({
44284
44559
  * ever FORMAT with are rejected outright — non-strict parsing would otherwise read `2007305` as
44285
44560
  * 2007-01-05 rather than 2007-11-01 and report no error at all.
44286
44561
  */
44287
- parseDate(value, format3, strict = true) {
44288
- if (DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format3)))
44289
- return this._parseDayOfYear(value, format3);
44290
- this._assertParsableFormat(format3);
44291
- const parsed = import_dayjs.default.utc(value, format3, strict);
44562
+ parseDate(value, format4, strict = true) {
44563
+ if (DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format4)))
44564
+ return this._parseDayOfYear(value, format4);
44565
+ this._assertParsableFormat(format4);
44566
+ const parsed = import_dayjs.default.utc(value, format4, strict);
44292
44567
  if (!parsed.isValid())
44293
- throw new Error(`Cannot parse "${value}" with input format "${format3}"`);
44568
+ throw new Error(`Cannot parse "${value}" with input format "${format4}"`);
44294
44569
  return parsed;
44295
44570
  }
44296
44571
  /**
@@ -44299,61 +44574,61 @@ var init_TypeCaster = __esm({
44299
44574
  * walked token by token against the format — every token of such a format has a fixed width, so
44300
44575
  * a plain cursor is enough — and the date is then built from the year plus the ordinal offset.
44301
44576
  */
44302
- _parseDayOfYear(value, format3) {
44577
+ _parseDayOfYear(value, format4) {
44303
44578
  const parts = {};
44304
44579
  let cursor = 0;
44305
44580
  let index = 0;
44306
- while (index < format3.length) {
44307
- if (format3[index] === "[") {
44308
- const close = format3.indexOf("]", index);
44581
+ while (index < format4.length) {
44582
+ if (format4[index] === "[") {
44583
+ const close = format4.indexOf("]", index);
44309
44584
  if (close < 0)
44310
- throw new Error(`Unterminated "[" literal in date format "${format3}"`);
44311
- const literal2 = format3.slice(index + 1, close);
44585
+ throw new Error(`Unterminated "[" literal in date format "${format4}"`);
44586
+ const literal2 = format4.slice(index + 1, close);
44312
44587
  if (value.slice(cursor, cursor + literal2.length) !== literal2)
44313
- throw new Error(`Cannot parse "${value}" with input format "${format3}": expected "${literal2}" at character ${cursor + 1}`);
44588
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": expected "${literal2}" at character ${cursor + 1}`);
44314
44589
  cursor += literal2.length;
44315
44590
  index = close + 1;
44316
44591
  continue;
44317
44592
  }
44318
- const match = DATE_PARSE_TOKENS.find((x2) => format3.startsWith(x2.token, index));
44593
+ const match = DATE_PARSE_TOKENS.find((x2) => format4.startsWith(x2.token, index));
44319
44594
  if (!match) {
44320
- if (/[A-Za-z]/.test(format3[index]))
44321
- throw new Error(`Date format "${format3}" mixes a day-of-year token with "${format3[index]}", which can't be parsed alongside it. Use YYYY/YY, DDD/DDDD, HH, mm, ss and SSS only.`);
44322
- if (value[cursor] !== format3[index])
44323
- throw new Error(`Cannot parse "${value}" with input format "${format3}": expected "${format3[index]}" at character ${cursor + 1}`);
44595
+ if (/[A-Za-z]/.test(format4[index]))
44596
+ throw new Error(`Date format "${format4}" mixes a day-of-year token with "${format4[index]}", which can't be parsed alongside it. Use YYYY/YY, DDD/DDDD, HH, mm, ss and SSS only.`);
44597
+ if (value[cursor] !== format4[index])
44598
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": expected "${format4[index]}" at character ${cursor + 1}`);
44324
44599
  cursor++;
44325
44600
  index++;
44326
44601
  continue;
44327
44602
  }
44328
44603
  const digits = value.slice(cursor, cursor + match.width);
44329
44604
  if (digits.length !== match.width || !/^\d+$/.test(digits))
44330
- throw new Error(`Cannot parse "${value}" with input format "${format3}": expected ${match.width} digit(s) for "${match.token}" at character ${cursor + 1}`);
44605
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": expected ${match.width} digit(s) for "${match.token}" at character ${cursor + 1}`);
44331
44606
  parts[match.token] = Number(digits);
44332
44607
  cursor += match.width;
44333
44608
  index += match.token.length;
44334
44609
  }
44335
44610
  if (cursor !== value.length)
44336
- throw new Error(`Cannot parse "${value}" with input format "${format3}": ${value.length - cursor} unexpected trailing character(s)`);
44611
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": ${value.length - cursor} unexpected trailing character(s)`);
44337
44612
  const year2 = Algo_default.hasVal(parts["YYYY"]) ? parts["YYYY"] : parts["YY"];
44338
44613
  const dayOfYear = Algo_default.hasVal(parts["DDDD"]) ? parts["DDDD"] : parts["DDD"];
44339
44614
  if (!Algo_default.hasVal(year2))
44340
- throw new Error(`Date format "${format3}" declares a day of year but no year, so the date is ambiguous.`);
44615
+ throw new Error(`Date format "${format4}" declares a day of year but no year, so the date is ambiguous.`);
44341
44616
  const startOfYear = import_dayjs.default.utc(`${String(year2).padStart(4, "0")}-01-01`, "YYYY-MM-DD", true);
44342
44617
  if (!startOfYear.isValid())
44343
- throw new Error(`Cannot parse "${value}" with input format "${format3}": "${year2}" is not a valid year`);
44618
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": "${year2}" is not a valid year`);
44344
44619
  const parsed = startOfYear.add(dayOfYear - 1, "day").add(parts["HH"] ?? 0, "hour").add(parts["mm"] ?? 0, "minute").add(parts["ss"] ?? 0, "second").add(parts["SSS"] ?? 0, "millisecond");
44345
44620
  if (dayOfYear < 1 || parsed.year() !== startOfYear.year())
44346
- throw new Error(`Cannot parse "${value}" with input format "${format3}": day of year ${dayOfYear} is outside year ${year2} (which has ${startOfYear.add(1, "year").diff(startOfYear, "day")} days)`);
44621
+ throw new Error(`Cannot parse "${value}" with input format "${format4}": day of year ${dayOfYear} is outside year ${year2} (which has ${startOfYear.add(1, "year").diff(startOfYear, "day")} days)`);
44347
44622
  return parsed;
44348
44623
  }
44349
- _assertParsableFormat(format3) {
44350
- const stripped = this._stripLiterals(format3);
44624
+ _assertParsableFormat(format4) {
44625
+ const stripped = this._stripLiterals(format4);
44351
44626
  const offending = FORMAT_ONLY_TOKENS.find((token2) => stripped.includes(token2));
44352
44627
  if (offending)
44353
- throw new Error(`Date format "${format3}" uses the token "${offending}", which dayjs can only write, never read: parsing would silently produce the wrong date. Use a parseable token, or bracket it as a literal ("[${offending}]") if it is part of the text.`);
44628
+ throw new Error(`Date format "${format4}" uses the token "${offending}", which dayjs can only write, never read: parsing would silently produce the wrong date. Use a parseable token, or bracket it as a literal ("[${offending}]") if it is part of the text.`);
44354
44629
  }
44355
- _stripLiterals(format3) {
44356
- return String(format3 ?? "").replace(/\[[^\]]*\]/g, "");
44630
+ _stripLiterals(format4) {
44631
+ return String(format4 ?? "").replace(/\[[^\]]*\]/g, "");
44357
44632
  }
44358
44633
  };
44359
44634
  TypeCaster = new TypeCasterClass();
@@ -44600,7 +44875,8 @@ var init_ProducerEngine = __esm({
44600
44875
  }
44601
44876
  };
44602
44877
  }
44603
- const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
44878
+ const headerLines = ParseHelper_default.hasHeaderLine(effectiveProducer.settings.fileType, effectiveProducer.settings.hasHeaderRow) ? 1 : 0;
44879
+ const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize + headerLines } }, effectiveSource);
44604
44880
  rawData = fileData.data;
44605
44881
  } finally {
44606
44882
  if (scope)
@@ -44619,13 +44895,15 @@ var init_ProducerEngine = __esm({
44619
44895
  if (!rawData || rawData.length === 0) return [];
44620
44896
  const firstItem = rawData[0];
44621
44897
  const isLineData = typeof firstItem === "string";
44898
+ const fills = discover2 ? [] : ProducerManager_default.compileFills(this._fillDimensions(producer));
44899
+ const sourceFilename = import_path18.default.basename(producer.settings.fileKey ?? producer.settings.sqlTable ?? "");
44622
44900
  if (isLineData) {
44623
44901
  const lines = rawData;
44624
44902
  const dimensions = this.deriveDimensions(lines[0], producer, discover2);
44625
44903
  const { fileType, hasHeaderRow } = producer.settings;
44626
44904
  const dataLines = ParseHelper_default.hasHeaderLine(fileType, hasHeaderRow) ? lines.slice(1) : lines;
44627
44905
  const recordSelector = ProducerManager_default.resolveRecordSelector(producer);
44628
- return dataLines.filter((line) => line.trim().length > 0).filter((line) => ProducerManager_default.shouldReadLine(line, recordSelector)).flatMap((line) => LineParser_default.parseAll(line, producer, dimensions, noopTracker));
44906
+ return dataLines.filter((line) => line.trim().length > 0).filter((line) => ProducerManager_default.shouldReadLine(line, recordSelector)).flatMap((line) => LineParser_default.parseAll(line, producer, dimensions, noopTracker)).map((record) => ProducerManager_default.applyFills(record, fills, sourceFilename));
44629
44907
  }
44630
44908
  if (discover2) {
44631
44909
  return rawData.map((item) => ({ ...item }));
@@ -44636,9 +44914,15 @@ var init_ProducerEngine = __esm({
44636
44914
  const key = dim.alias ?? dim.name;
44637
44915
  record[dim.name] = TypeCaster_default.cast(item[key], dim.type, dim.format);
44638
44916
  }
44639
- return record;
44917
+ return ProducerManager_default.applyFills(record, fills, sourceFilename);
44640
44918
  });
44641
44919
  };
44920
+ /**
44921
+ * The producer's declared dimensions in the shape `compileFills` reads them in. Declaration
44922
+ * order, because a fill is addressed by key and never by position — unlike the executor
44923
+ * dimensions of a run, whose index is reconciled against the file's own header.
44924
+ */
44925
+ this._fillDimensions = (producer) => producer.dimensions.map((prodDimension, index) => ({ index, name: prodDimension.name, prodDimension }));
44642
44926
  /**
44643
44927
  * Builds the executor dimensions of a producer. There are two shapes of source: a FIXED file has
44644
44928
  * no header, so its layout comes from the positions declared on the dimensions; every other file
@@ -44736,7 +45020,7 @@ var init_ProducerEngine = __esm({
44736
45020
  });
44737
45021
 
44738
45022
  // ../../packages/engines/src/ai/DeveloperEngine.ts
44739
- var import_path19, import_promises11, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
45023
+ var import_path19, import_promises12, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
44740
45024
  var init_DeveloperEngine = __esm({
44741
45025
  "../../packages/engines/src/ai/DeveloperEngine.ts"() {
44742
45026
  init_src();
@@ -44744,7 +45028,7 @@ var init_DeveloperEngine = __esm({
44744
45028
  init_src4();
44745
45029
  init_src8();
44746
45030
  import_path19 = __toESM(require("path"), 1);
44747
- import_promises11 = __toESM(require("fs/promises"), 1);
45031
+ import_promises12 = __toESM(require("fs/promises"), 1);
44748
45032
  import_dayjs2 = __toESM(require("dayjs"), 1);
44749
45033
  import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
44750
45034
  import_dayjs2.default.extend(import_customParseFormat2.default);
@@ -44774,8 +45058,8 @@ var init_DeveloperEngine = __esm({
44774
45058
  };
44775
45059
  mappedProducer["$schema"] = producer["$schema"];
44776
45060
  const producerPath = Environment_default.resolveResourceFilePath("producers", producer.name);
44777
- await import_promises11.default.mkdir(import_path19.default.dirname(producerPath), { recursive: true });
44778
- await import_promises11.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
45061
+ await import_promises12.default.mkdir(import_path19.default.dirname(producerPath), { recursive: true });
45062
+ await import_promises12.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
44779
45063
  return { producer: mappedProducer, fields: typeDefinitions };
44780
45064
  };
44781
45065
  // Discover-only precondition: sampling real data requires a configured, readable file.
@@ -47106,9 +47390,10 @@ var init_MockDataFactory = __esm({
47106
47390
  if (type === "number") {
47107
47391
  const first = bounds?.min ?? 1;
47108
47392
  const last = first + poolSize - 1;
47393
+ const ceiling = bounds?.max;
47109
47394
  Affirm_default(
47110
- bounds?.max === void 0 || bounds.max >= last,
47111
- `Key "${norm}" has a synth range of ${first}-${bounds.max}, which holds fewer than the ${poolSize} distinct values it has to generate; widen it or generate fewer records`
47395
+ ceiling === void 0 || ceiling >= last,
47396
+ `Key "${norm}" has a synth range of ${first}-${ceiling}, which holds fewer than the ${poolSize} distinct values it has to generate; widen it or generate fewer records`
47112
47397
  );
47113
47398
  for (let i6 = 0; i6 < poolSize; i6++) pool.push(first + i6);
47114
47399
  return pool;
@@ -47311,10 +47596,10 @@ var init_MockDataFactory = __esm({
47311
47596
  });
47312
47597
 
47313
47598
  // ../../packages/engines/src/mock/SyntheticInputEngine.ts
47314
- var import_promises12, import_os, import_path20, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine, SyntheticInputEngine_default;
47599
+ var import_promises13, import_os, import_path20, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine, SyntheticInputEngine_default;
47315
47600
  var init_SyntheticInputEngine = __esm({
47316
47601
  "../../packages/engines/src/mock/SyntheticInputEngine.ts"() {
47317
- import_promises12 = __toESM(require("fs/promises"), 1);
47602
+ import_promises13 = __toESM(require("fs/promises"), 1);
47318
47603
  import_os = __toESM(require("os"), 1);
47319
47604
  import_path20 = __toESM(require("path"), 1);
47320
47605
  import_dayjs4 = __toESM(require("dayjs"), 1);
@@ -47340,7 +47625,7 @@ var init_SyntheticInputEngine = __esm({
47340
47625
  for (const cs2 of [...BUILTIN_CODE_SETS, ...codeSets]) codeSetMap.set(cs2.name.toLowerCase(), cs2);
47341
47626
  const producerMap = new Map(producers.map((p5) => [p5.name, p5]));
47342
47627
  const refTargets = this._collectReferences(producers, producerMap);
47343
- const dir = await import_promises12.default.mkdtemp(import_path20.default.join(import_os.default.tmpdir(), "remora-synth-"));
47628
+ const dir = await import_promises13.default.mkdtemp(import_path20.default.join(import_os.default.tmpdir(), "remora-synth-"));
47344
47629
  const poolSize = Math.min(Math.max(opts.records, 1), POOL_CAP);
47345
47630
  const registry = new MockKeyRegistry();
47346
47631
  const specs = producers.flatMap((p5) => this._specs(p5, codeSetMap, producerMap, refTargets));
@@ -47357,7 +47642,7 @@ var init_SyntheticInputEngine = __esm({
47357
47642
  return { inputs, dir };
47358
47643
  };
47359
47644
  this.cleanup = async (bundle) => {
47360
- if (bundle?.dir) await import_promises12.default.rm(bundle.dir, { recursive: true, force: true });
47645
+ if (bundle?.dir) await import_promises13.default.rm(bundle.dir, { recursive: true, force: true });
47361
47646
  };
47362
47647
  this._assertOpts = (opts) => {
47363
47648
  Affirm_default(opts, "Invalid options");
@@ -47407,8 +47692,8 @@ var init_SyntheticInputEngine = __esm({
47407
47692
  lines.push(JSON.stringify(row) + "\n");
47408
47693
  }
47409
47694
  const filePath = import_path20.default.join(dir, `${producer.name}.jsonl`);
47410
- await import_promises12.default.writeFile(filePath, lines.join(""), "utf-8");
47411
- const { size } = await import_promises12.default.stat(filePath);
47695
+ await import_promises13.default.writeFile(filePath, lines.join(""), "utf-8");
47696
+ const { size } = await import_promises13.default.stat(filePath);
47412
47697
  return { path: filePath, fileSize: size };
47413
47698
  };
47414
47699
  // Distinct structured code sets referenced by these specs -> their records (deduped by name).
@@ -47519,7 +47804,7 @@ var init_SyntheticInputEngine = __esm({
47519
47804
  // True when a date bound names a day rather than an instant — neither the value nor the dimension's
47520
47805
  // format carries a time. Both have to agree: the value alone can't tell `"20240131"` from a compact
47521
47806
  // `YYYYMMDDHHmmss` instant, and the format alone can't tell that an ISO value overrode it.
47522
- this._isDayGranular = (value, format3) => !/[T ]\d{1,2}:/.test(value) && !/[HhmsSAa]/.test(format3 ?? "");
47807
+ this._isDayGranular = (value, format4) => !/[T ]\d{1,2}:/.test(value) && !/[HhmsSAa]/.test(format4 ?? "");
47523
47808
  /**
47524
47809
  * One bound in the units the generator works in: the number itself, a count of characters for a
47525
47810
  * `string`, or epoch milliseconds for a `datetime`. A date bound is read with the dimension's own
@@ -48449,23 +48734,23 @@ var init_Diagnostics = __esm({
48449
48734
  return [.../* @__PURE__ */ new Set([...inputs, ...chained])];
48450
48735
  };
48451
48736
  this._walk = (start, next, onCycle) => {
48452
- const path29 = [];
48737
+ const path30 = [];
48453
48738
  const onPath = /* @__PURE__ */ new Set();
48454
48739
  const stack = [{ name: start, done: false }];
48455
48740
  const visited = /* @__PURE__ */ new Set();
48456
48741
  while (stack.length > 0) {
48457
48742
  const frame = stack.pop();
48458
48743
  if (frame.done) {
48459
- onPath.delete(path29.pop());
48744
+ onPath.delete(path30.pop());
48460
48745
  continue;
48461
48746
  }
48462
48747
  if (onPath.has(frame.name)) {
48463
- path29.slice(path29.indexOf(frame.name)).forEach((name) => onCycle.add(name));
48748
+ path30.slice(path30.indexOf(frame.name)).forEach((name) => onCycle.add(name));
48464
48749
  continue;
48465
48750
  }
48466
48751
  if (visited.has(frame.name)) continue;
48467
48752
  visited.add(frame.name);
48468
- path29.push(frame.name);
48753
+ path30.push(frame.name);
48469
48754
  onPath.add(frame.name);
48470
48755
  stack.push({ name: frame.name, done: true });
48471
48756
  const successors = next.get(frame.name) ?? [];
@@ -48809,7 +49094,7 @@ var init_LineageRenderer = __esm({
48809
49094
  });
48810
49095
 
48811
49096
  // ../../packages/engines/src/lineage/ResourceView.ts
48812
- var VISIBLE_AUTH_KEYS, MAX_TABLE_ROWS, MAX_RAW_BYTES, MAX_CELL_CHARS, ResourceViewClass, ResourceView, ResourceView_default;
49097
+ var VISIBLE_AUTH_KEYS, MAX_TABLE_ROWS, MAX_RAW_BYTES, MAX_CELL_CHARS, MAX_NOTE_CHARS, ResourceViewClass, ResourceView, ResourceView_default;
48813
49098
  var init_ResourceView = __esm({
48814
49099
  "../../packages/engines/src/lineage/ResourceView.ts"() {
48815
49100
  init_src();
@@ -48848,6 +49133,7 @@ var init_ResourceView = __esm({
48848
49133
  MAX_TABLE_ROWS = 500;
48849
49134
  MAX_RAW_BYTES = 256 * 1024;
48850
49135
  MAX_CELL_CHARS = 200;
49136
+ MAX_NOTE_CHARS = 240;
48851
49137
  ResourceViewClass = class {
48852
49138
  constructor() {
48853
49139
  /**
@@ -49019,7 +49305,7 @@ var init_ResourceView = __esm({
49019
49305
  { key: "classification", label: "Classification" }
49020
49306
  ];
49021
49307
  const rows = dimensions.map((dimension) => [
49022
- { value: dimension.name },
49308
+ { value: dimension.name, note: this._note(dimension.description) },
49023
49309
  { value: this._text(dimension.type) },
49024
49310
  positioned ? { value: this._position(dimension.position) } : { value: this._text(dimension.alias), muted: !dimension.alias },
49025
49311
  { value: !!dimension.pk, muted: !dimension.pk },
@@ -49031,7 +49317,7 @@ var init_ResourceView = __esm({
49031
49317
  };
49032
49318
  this._measuresTable = (producer) => {
49033
49319
  const rows = (producer.measures ?? []).map((measure) => [
49034
- { value: measure.name },
49320
+ { value: measure.name, note: this._note(measure.description) },
49035
49321
  { value: measure.sql }
49036
49322
  ]);
49037
49323
  return this._table("Measures", [
@@ -49097,7 +49383,7 @@ var init_ResourceView = __esm({
49097
49383
  };
49098
49384
  this._consumerFieldsTable = (env2, consumer) => {
49099
49385
  const rows = (consumer.fields ?? []).map((field) => [
49100
- { value: field.key },
49386
+ { value: field.key, note: this._note(field.description) },
49101
49387
  { value: this._text(field.alias), muted: !field.alias },
49102
49388
  { value: this._text(field.from), link: field.from ? this._inputLink(env2, field.from) : void 0, muted: !field.from },
49103
49389
  { value: this._text(field.transform ? Object.keys(field.transform) : null), muted: !field.transform },
@@ -49143,8 +49429,8 @@ var init_ResourceView = __esm({
49143
49429
  if (!source || origin.path.length < 2) return { value: "declared here", muted: true };
49144
49430
  return { value: `${source.resource}.${source.field}`, link: source.node };
49145
49431
  };
49146
- this._pathText = (path29) => {
49147
- return path29.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
49432
+ this._pathText = (path30) => {
49433
+ return path30.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
49148
49434
  };
49149
49435
  this._consumerOutputsTable = (env2, consumer) => {
49150
49436
  const declared = new Set((env2.sources ?? []).map((x2) => x2.name));
@@ -49186,7 +49472,7 @@ var init_ResourceView = __esm({
49186
49472
  const required = new Set(schema.required ?? []);
49187
49473
  const properties = Object.entries(schema.properties ?? {});
49188
49474
  const rows = properties.map(([property, definition]) => [
49189
- { value: property },
49475
+ { value: property, note: this._note(this._schemaKeyword(definition, "description")) },
49190
49476
  { value: this._schemaType(definition) },
49191
49477
  { value: required.has(property), muted: !required.has(property) },
49192
49478
  { value: this._text(this._schemaKeyword(definition, "format")), muted: !this._schemaKeyword(definition, "format") }
@@ -49292,6 +49578,15 @@ var init_ResourceView = __esm({
49292
49578
  const minutes = Math.floor(elapsedMS / 6e4);
49293
49579
  return `${minutes}m ${Math.round(elapsedMS % 6e4 / 1e3)}s`;
49294
49580
  };
49581
+ /**
49582
+ * A description as the secondary line of a cell. Blank prose is the same as none, so a field
49583
+ * documented with a stray space does not cost the row a line.
49584
+ */
49585
+ this._note = (description) => {
49586
+ const text = (description ?? "").trim();
49587
+ if (!text) return void 0;
49588
+ return text.length > MAX_NOTE_CHARS ? `${text.slice(0, MAX_NOTE_CHARS)}\u2026` : text;
49589
+ };
49295
49590
  /** Anything that is not already a scalar becomes readable text; absent becomes `null`. */
49296
49591
  this._text = (value) => {
49297
49592
  if (value === void 0 || value === null) return null;
@@ -49608,7 +49903,7 @@ var init_OpsHeartbeat = __esm({
49608
49903
  "../../packages/engines/src/ops/OpsHeartbeat.ts"() {
49609
49904
  init_src2();
49610
49905
  init_OpsService();
49611
- DEFAULT_INTERVAL_MS = 6e4;
49906
+ DEFAULT_INTERVAL_MS = 3e5;
49612
49907
  MIN_INTERVAL_MS = 5e3;
49613
49908
  MAX_INTERVAL_MS = 36e5;
49614
49909
  OpsHeartbeatClass = class {
@@ -49648,11 +49943,24 @@ var init_OpsHeartbeat = __esm({
49648
49943
  Logger_default.error(error);
49649
49944
  }
49650
49945
  };
49946
+ /**
49947
+ * Resolve the configured interval, in milliseconds. A value that is unusable or out of range is
49948
+ * reported rather than silently replaced: an operator who set an interval and got a different
49949
+ * one has no other way to find out.
49950
+ */
49651
49951
  this.resolveInterval = () => {
49652
- const configured = Number(process.env.REMORA_STATE_INTERVAL_MS);
49653
- if (!Number.isFinite(configured) || configured <= 0)
49952
+ const raw = process.env.REMORA_STATE_INTERVAL_MS;
49953
+ if (!raw)
49954
+ return DEFAULT_INTERVAL_MS;
49955
+ const configured = Number(raw);
49956
+ if (!Number.isFinite(configured) || configured <= 0) {
49957
+ Logger_default.warn(`REMORA_STATE_INTERVAL_MS is not a positive number of milliseconds ("${raw}"); using ${DEFAULT_INTERVAL_MS}ms.`);
49654
49958
  return DEFAULT_INTERVAL_MS;
49655
- return Math.min(Math.max(configured, MIN_INTERVAL_MS), MAX_INTERVAL_MS);
49959
+ }
49960
+ const clamped = Math.min(Math.max(configured, MIN_INTERVAL_MS), MAX_INTERVAL_MS);
49961
+ if (clamped !== configured)
49962
+ Logger_default.warn(`REMORA_STATE_INTERVAL_MS of ${configured}ms is outside ${MIN_INTERVAL_MS}ms..${MAX_INTERVAL_MS}ms; using ${clamped}ms.`);
49963
+ return clamped;
49656
49964
  };
49657
49965
  }
49658
49966
  };
@@ -49886,17 +50194,30 @@ var init_LocalUsageDB = __esm({
49886
50194
  }
49887
50195
  };
49888
50196
  /**
49889
- * The recorded runs, newest first. Only the tail of the file is decoded: a reader wants the recent
50197
+ * The recorded runs, newest first — the newest `limit` of the rows `keep` accepts.
50198
+ *
50199
+ * Decoded from the end and stopped as soon as that many are held: a reader wants the recent
49890
50200
  * executions, and decoding ten thousand rows to show ten of them is work nobody asked for.
49891
50201
  *
50202
+ * `keep` is applied BEFORE the limit, which is the only way "the newest N runs of this kind" can
50203
+ * mean it: filtering afterwards lets the rows a caller does not want displace the ones it does,
50204
+ * so one sample recorded here would silently push a real run out of a history someone is reading.
50205
+ *
49892
50206
  * Never throws. A missing file reads as no history, and so does a half-written one — this store is
49893
50207
  * appended to while it is being read, so failing on that would make it unreadable during a run.
49894
50208
  */
49895
- this.list = (limit = MAX_LISTED_ROWS) => {
50209
+ this.list = (limit = MAX_LISTED_ROWS, keep) => {
49896
50210
  try {
49897
50211
  if (!(0, import_fs16.existsSync)(CLI_DB_PATH))
49898
50212
  return [];
49899
- return this._getRows().slice(-Math.max(1, limit)).map((line) => this._fromLine(line)).reverse();
50213
+ const rows = this._getRows();
50214
+ const wanted = Math.max(1, limit);
50215
+ const found = [];
50216
+ for (let index = rows.length - 1; index >= 0 && found.length < wanted; index--) {
50217
+ const usage = this._fromLine(rows[index]);
50218
+ if (!keep || keep(usage)) found.push(usage);
50219
+ }
50220
+ return found;
49900
50221
  } catch (error) {
49901
50222
  this._handleError("list", error);
49902
50223
  return [];
@@ -50077,9 +50398,10 @@ var init_UsageManager = __esm({
50077
50398
  });
50078
50399
 
50079
50400
  // ../../packages/engines/src/ops/RunOverlay.ts
50080
- var MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay, RunOverlay_default;
50401
+ var import_crypto5, MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay, RunOverlay_default;
50081
50402
  var init_RunOverlay = __esm({
50082
50403
  "../../packages/engines/src/ops/RunOverlay.ts"() {
50404
+ import_crypto5 = __toESM(require("crypto"), 1);
50083
50405
  init_src();
50084
50406
  init_LineageEngine();
50085
50407
  init_LocalUsageDB();
@@ -50088,6 +50410,8 @@ var init_RunOverlay = __esm({
50088
50410
  MAX_OPERATIONS = 40;
50089
50411
  RunOverlayClass = class {
50090
50412
  constructor() {
50413
+ /** The last store token read, and the overlay token it resolved to. */
50414
+ this._signed = { store: null, overlay: "" };
50091
50415
  /**
50092
50416
  * The recent executions and their aggregates. ONE read of the store answers both, so a page
50093
50417
  * showing a graph, a run list and a timing breakdown costs one file read rather than three.
@@ -50098,20 +50422,40 @@ var init_RunOverlay = __esm({
50098
50422
  this.read = (limit = MAX_RUNS) => {
50099
50423
  Affirm_default(limit > 0, "Run overlay limit must be greater than zero");
50100
50424
  if (!UsageManager_default.recordsLocally()) return { runs: [], metrics: {} };
50101
- const stats = LocalUsageDB_default.list(limit).filter((stat) => stat?.consumer);
50425
+ const stats = LocalUsageDB_default.list(limit, (stat) => !!stat?.consumer && stat.invokedBy !== "SAMPLE");
50102
50426
  return { runs: stats.map((stat) => this._run(stat)), metrics: this._metrics(stats) };
50103
50427
  };
50104
50428
  /**
50105
- * A token that changes when the run store does. Empty when there is nothing to read.
50429
+ * A token that changes when the OVERLAY does. Empty when there is nothing to read.
50106
50430
  *
50107
50431
  * A run finishing changes no configuration file, so the config hash cannot notice it — this is
50108
- * the second half of what the preview polls, and it is cheap for the same reason the hash is
50109
- * affordable: it reads metadata, not records.
50432
+ * the second half of what the preview polls.
50433
+ *
50434
+ * It signs the painted runs rather than the store, because the two do not move together: a
50435
+ * sample is recorded like any other execution and then filtered out of `read`, so the store's
50436
+ * own token changes while the picture does not. Signing the store there announced a project
50437
+ * change for every sample — and a page told its resource changed drops the sample it is showing,
50438
+ * which is exactly the rows someone had just asked for.
50439
+ *
50440
+ * Still cheap to poll: the store's mtime/size gates the read, so the rows are decoded only after
50441
+ * something has actually been written, never once a second.
50110
50442
  */
50111
50443
  this.signature = () => {
50112
50444
  if (!UsageManager_default.recordsLocally()) return "";
50113
- return LocalUsageDB_default.signature();
50445
+ const store = LocalUsageDB_default.signature();
50446
+ if (store === this._signed.store) return this._signed.overlay;
50447
+ const overlay = this._token(this.read());
50448
+ this._signed = { store, overlay };
50449
+ return overlay;
50114
50450
  };
50451
+ /**
50452
+ * A token over the whole overlay, runs and metrics alike.
50453
+ *
50454
+ * Everything it carries, not the newest row and a count: `endUsage` updates a row in place and a
50455
+ * finish action writes runs of its own, so a narrower token would miss a transition the graph
50456
+ * exists to show.
50457
+ */
50458
+ this._token = (overlay) => import_crypto5.default.createHash("sha256").update(JSON.stringify(overlay)).digest("hex");
50115
50459
  this._run = (stat) => {
50116
50460
  const startedAt = this._iso(stat.startedAt);
50117
50461
  return {
@@ -50358,64 +50702,6 @@ var init_PushdownEngine = __esm({
50358
50702
  }
50359
50703
  });
50360
50704
 
50361
- // ../../packages/engines/src/producer/SampleEngine.ts
50362
- var MAX_CHAIN_DEPTH2, SampleEngineClass, SampleEngine, SampleEngine_default;
50363
- var init_SampleEngine = __esm({
50364
- "../../packages/engines/src/producer/SampleEngine.ts"() {
50365
- init_src();
50366
- init_src4();
50367
- init_ProducerEngine();
50368
- MAX_CHAIN_DEPTH2 = 20;
50369
- SampleEngineClass = class {
50370
- constructor() {
50371
- /**
50372
- * @param resourceName a producer or consumer name, as declared.
50373
- * @throws when the name is neither, or when the source cannot be read.
50374
- */
50375
- this.read = async (resourceName, size = 10) => {
50376
- Affirm_default(resourceName, "Missing resource name to sample");
50377
- Affirm_default(size > 0, "Sample size must be greater than 0");
50378
- const producer = Environment_default.getProducer(resourceName);
50379
- if (producer)
50380
- return { kind: "producer", name: resourceName, records: await ProducerEngine_default.readSampleData(producer, size, false) };
50381
- const consumer = Environment_default.getConsumer(resourceName);
50382
- Affirm_default(consumer, `Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`);
50383
- return { kind: "consumer", name: resourceName, records: await this._readConsumer(consumer, size, []) };
50384
- };
50385
- /**
50386
- * A consumer has no data of its own until it runs, so what it shows is its FIRST input, projected
50387
- * through its field list — near enough to answer "what will come out of this" without a run.
50388
- */
50389
- this._readConsumer = async (consumer, size, visited) => {
50390
- Affirm_default(!visited.includes(consumer.name), `Consumer "${consumer.name}" reads itself through ${visited.join(" \u2192 ")}`);
50391
- Affirm_default(visited.length < MAX_CHAIN_DEPTH2, `Consumer chain from "${visited[0]}" is more than ${MAX_CHAIN_DEPTH2} deep`);
50392
- const input = consumer.producers[0];
50393
- Affirm_default(input, `Consumer "${consumer.name}" has no producers configured`);
50394
- const producer = Environment_default.getProducer(input.name);
50395
- if (!producer) {
50396
- const upstream = Environment_default.getConsumer(input.name);
50397
- Affirm_default(upstream, `Producer or consumer "${input.name}" not found for consumer "${consumer.name}"`);
50398
- return this._readConsumer(upstream, size, [...visited, consumer.name]);
50399
- }
50400
- const records = await ProducerEngine_default.readSampleData(producer, size, false);
50401
- if (consumer.fields.some((field) => field.key === "*")) return records;
50402
- return records.map((record) => this._project(consumer, record));
50403
- };
50404
- this._project = (consumer, record) => {
50405
- const projected = {};
50406
- for (const field of consumer.fields) {
50407
- if (field.key === "*") continue;
50408
- projected[field.alias || field.key] = record[field.key];
50409
- }
50410
- return projected;
50411
- };
50412
- }
50413
- };
50414
- SampleEngine = new SampleEngineClass();
50415
- SampleEngine_default = SampleEngine;
50416
- }
50417
- });
50418
-
50419
50705
  // ../../packages/engines/src/transform/LookupRegistry.ts
50420
50706
  var LookupRegistryClass, LookupRegistry, LookupRegistry_default;
50421
50707
  var init_LookupRegistry = __esm({
@@ -50452,12 +50738,12 @@ var init_LookupRegistry = __esm({
50452
50738
  });
50453
50739
 
50454
50740
  // ../../packages/engines/src/transform/LookupKey.ts
50455
- var import_crypto5, DELIMITER, idFor, LookupKey, LookupKey_default;
50741
+ var import_crypto6, DELIMITER, idFor, LookupKey, LookupKey_default;
50456
50742
  var init_LookupKey = __esm({
50457
50743
  "../../packages/engines/src/transform/LookupKey.ts"() {
50458
- import_crypto5 = __toESM(require("crypto"), 1);
50744
+ import_crypto6 = __toESM(require("crypto"), 1);
50459
50745
  DELIMITER = String.fromCharCode(0);
50460
- idFor = (ref) => import_crypto5.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
50746
+ idFor = (ref) => import_crypto6.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
50461
50747
  LookupKey = { idFor };
50462
50748
  LookupKey_default = LookupKey;
50463
50749
  }
@@ -50484,8 +50770,8 @@ var init_TransformationEngine = __esm({
50484
50770
  return result;
50485
50771
  }
50486
50772
  if ("cast" in transformations) {
50487
- const { cast, format: format3 } = transformations;
50488
- const casted = TypeCaster_default.cast(value, cast, format3);
50773
+ const { cast, format: format4 } = transformations;
50774
+ const casted = TypeCaster_default.cast(value, cast, format4);
50489
50775
  if (cast === "number" && isNaN(casted))
50490
50776
  throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
50491
50777
  if (cast === "datetime" && casted instanceof Date && isNaN(casted.getTime()))
@@ -51094,7 +51380,6 @@ var init_src10 = __esm({
51094
51380
  init_ProducerEngine();
51095
51381
  init_ProducerManager();
51096
51382
  init_PushdownEngine();
51097
- init_SampleEngine();
51098
51383
  init_TransformationEngine();
51099
51384
  init_TypeCaster();
51100
51385
  init_LookupRegistry();
@@ -51264,14 +51549,14 @@ var init_OutputExecutor = __esm({
51264
51549
  });
51265
51550
 
51266
51551
  // ../../packages/executors/src/ConsumerExecutor.ts
51267
- var import_path23, import_fs17, import_readline6, import_promises13, import_crypto6, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
51552
+ var import_path23, import_fs17, import_readline6, import_promises14, import_crypto7, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
51268
51553
  var init_ConsumerExecutor = __esm({
51269
51554
  "../../packages/executors/src/ConsumerExecutor.ts"() {
51270
51555
  import_path23 = __toESM(require("path"));
51271
51556
  import_fs17 = __toESM(require("fs"));
51272
51557
  import_readline6 = __toESM(require("readline"));
51273
- import_promises13 = __toESM(require("fs/promises"));
51274
- import_crypto6 = __toESM(require("crypto"));
51558
+ import_promises14 = __toESM(require("fs/promises"));
51559
+ import_crypto7 = __toESM(require("crypto"));
51275
51560
  init_src10();
51276
51561
  init_src10();
51277
51562
  init_src10();
@@ -51457,8 +51742,8 @@ var init_ConsumerExecutor = __esm({
51457
51742
  reader.destroy();
51458
51743
  });
51459
51744
  }
51460
- await import_promises13.default.unlink(datasetPath);
51461
- await import_promises13.default.rename(tempWorkPath, datasetPath);
51745
+ await import_promises14.default.unlink(datasetPath);
51746
+ await import_promises14.default.rename(tempWorkPath, datasetPath);
51462
51747
  return processedLineCount;
51463
51748
  };
51464
51749
  /**
@@ -51476,7 +51761,7 @@ var init_ConsumerExecutor = __esm({
51476
51761
  this.processLimit = async (datasetPath, limit) => {
51477
51762
  Affirm_default(datasetPath, "Invalid dataset path");
51478
51763
  Affirm_default(limit > 0, `Invalid record limit ${limit}`);
51479
- const handle = await import_promises13.default.open(datasetPath, "r+");
51764
+ const handle = await import_promises14.default.open(datasetPath, "r+");
51480
51765
  try {
51481
51766
  const buffer = Buffer.allocUnsafe(LIMIT_SCAN_BYTES);
51482
51767
  let newlines = 0;
@@ -51510,7 +51795,7 @@ var init_ConsumerExecutor = __esm({
51510
51795
  let newLineCount = 0;
51511
51796
  const seen = /* @__PURE__ */ new Set();
51512
51797
  for await (const line of lineReader) {
51513
- const hash = import_crypto6.default.createHash("sha1").update(line).digest("base64");
51798
+ const hash = import_crypto7.default.createHash("sha1").update(line).digest("base64");
51514
51799
  if (!seen.has(hash)) {
51515
51800
  seen.add(hash);
51516
51801
  if (!writer.write(line + "\n"))
@@ -51530,8 +51815,8 @@ var init_ConsumerExecutor = __esm({
51530
51815
  reader.destroy();
51531
51816
  });
51532
51817
  }
51533
- await import_promises13.default.unlink(datasetPath);
51534
- await import_promises13.default.rename(tempWorkPath, datasetPath);
51818
+ await import_promises14.default.unlink(datasetPath);
51819
+ await import_promises14.default.rename(tempWorkPath, datasetPath);
51535
51820
  return newLineCount;
51536
51821
  };
51537
51822
  this.processDistinctOn = async (consumer, datasetPath) => {
@@ -51575,8 +51860,8 @@ var init_ConsumerExecutor = __esm({
51575
51860
  reader.destroy();
51576
51861
  });
51577
51862
  }
51578
- await import_promises13.default.unlink(datasetPath);
51579
- await import_promises13.default.rename(tempWorkPath, datasetPath);
51863
+ await import_promises14.default.unlink(datasetPath);
51864
+ await import_promises14.default.rename(tempWorkPath, datasetPath);
51580
51865
  return winners.size;
51581
51866
  };
51582
51867
  this.processPivot = async (consumer, datasetPath) => {
@@ -51682,12 +51967,12 @@ var init_ConsumerExecutor = __esm({
51682
51967
  reader.destroy();
51683
51968
  });
51684
51969
  }
51685
- await import_promises13.default.unlink(datasetPath);
51686
- await import_promises13.default.rename(tempWorkPath, datasetPath);
51970
+ await import_promises14.default.unlink(datasetPath);
51971
+ await import_promises14.default.rename(tempWorkPath, datasetPath);
51687
51972
  return outputCount;
51688
51973
  };
51689
- this._parseLine = (line, format3, fields) => {
51690
- return format3 === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
51974
+ this._parseLine = (line, format4, fields) => {
51975
+ return format4 === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
51691
51976
  };
51692
51977
  /**
51693
51978
  * Determines if the new record should replace the existing record based on the resolution strategy
@@ -51786,8 +52071,6 @@ var init_ProducerExecutor = __esm({
51786
52071
  init_src8();
51787
52072
  init_src4();
51788
52073
  init_src10();
51789
- init_src();
51790
- init_src10();
51791
52074
  init_src10();
51792
52075
  import_path24 = __toESM(require("path"));
51793
52076
  init_src2();
@@ -51818,20 +52101,14 @@ var init_ProducerExecutor = __esm({
51818
52101
  * the source file name or a mask, and what that file name is. None of it depends on the line, so
51819
52102
  * it is compiled once per chunk instead of rebuilt per line — including the mask's own
51820
52103
  * validation, which used to run once per dimension per line.
52104
+ *
52105
+ * The fills themselves come from `ProducerManager`, which is also where the sample path gets
52106
+ * them: one producer stage, reached from two places.
51821
52107
  */
51822
52108
  this.compileLine = (dimensions, chunk) => {
51823
52109
  Affirm_default(dimensions, "Invalid producer dimensions");
51824
52110
  Affirm_default(chunk, "Invalid chunk");
51825
- const fills = [];
51826
- for (const dimension of dimensions) {
51827
- const { prodDimension } = dimension;
51828
- const mask = ProducerManager_default.getMask(prodDimension);
51829
- const sourceFilename = prodDimension.sourceFilename === true;
51830
- if (!sourceFilename && !Algo_default.hasVal(mask))
51831
- continue;
51832
- fills.push({ key: dimension.name, mask, sourceFilename, type: prodDimension.type });
51833
- }
51834
- return { fills, sourceFilename: import_path24.default.basename(chunk.fileUri) };
52111
+ return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename: import_path24.default.basename(chunk.fileUri) };
51835
52112
  };
51836
52113
  /**
51837
52114
  * The record(s) one source line produces. Almost always exactly one — a producer with a group
@@ -51852,14 +52129,8 @@ var init_ProducerExecutor = __esm({
51852
52129
  if (plan.fills.length === 0)
51853
52130
  return records;
51854
52131
  counter = performance.now();
51855
- for (const record of records) {
51856
- for (const fill2 of plan.fills) {
51857
- if (fill2.sourceFilename)
51858
- record[fill2.key] = plan.sourceFilename;
51859
- if (Algo_default.hasVal(fill2.mask))
51860
- record[fill2.key] = CryptoEngine_default.hashValue(fill2.mask, record[fill2.key]?.toString(), fill2.type);
51861
- }
51862
- }
52132
+ for (const record of records)
52133
+ ProducerManager_default.applyFills(record, plan.fills, plan.sourceFilename);
51863
52134
  tracker.measure("process-line:dimensions-filename-mask", performance.now() - counter);
51864
52135
  return records;
51865
52136
  };
@@ -52170,11 +52441,11 @@ var init_OrchestratorHelper = __esm({
52170
52441
  });
52171
52442
 
52172
52443
  // ../../packages/executors/src/LookupResolver.ts
52173
- var import_fs18, import_promises14, import_path26, import_readline8, LookupResolverClass, LookupResolver, LookupResolver_default;
52444
+ var import_fs18, import_promises15, import_path26, import_readline8, LookupResolverClass, LookupResolver, LookupResolver_default;
52174
52445
  var init_LookupResolver = __esm({
52175
52446
  "../../packages/executors/src/LookupResolver.ts"() {
52176
52447
  import_fs18 = __toESM(require("fs"));
52177
- import_promises14 = __toESM(require("fs/promises"));
52448
+ import_promises15 = __toESM(require("fs/promises"));
52178
52449
  import_path26 = __toESM(require("path"));
52179
52450
  import_readline8 = __toESM(require("readline"));
52180
52451
  init_src();
@@ -52215,11 +52486,11 @@ var init_LookupResolver = __esm({
52215
52486
  this.cache.set(id, await this.buildDict(lookupConsumer, capturePath, ref));
52216
52487
  }
52217
52488
  ExecutorScope_default.ensurePath(outPath);
52218
- await import_promises14.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
52489
+ await import_promises15.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
52219
52490
  manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path26.default.resolve(outPath) });
52220
52491
  }
52221
52492
  for (const capturePath of captureByConsumer.values())
52222
- await import_promises14.default.rm(capturePath, { force: true });
52493
+ await import_promises15.default.rm(capturePath, { force: true });
52223
52494
  return manifest;
52224
52495
  };
52225
52496
  // Runs the source consumer through the real pipeline, capturing its merged output before cleanup.
@@ -52237,7 +52508,7 @@ var init_LookupResolver = __esm({
52237
52508
  };
52238
52509
  // Projects the captured dataset to a { code: value } dict, bounded by MAX_LOOKUP_ROWS.
52239
52510
  this.buildDict = async (lookupConsumer, capturePath, ref) => {
52240
- const format3 = OutputExecutor_default._getInternalRecordFormat(lookupConsumer);
52511
+ const format4 = OutputExecutor_default._getInternalRecordFormat(lookupConsumer);
52241
52512
  const visibleFields = ConsumerManager_default.getExpandedFields(lookupConsumer).filter((f7) => !f7.cField.hidden);
52242
52513
  const finalKeys = visibleFields.map((f7) => f7.finalKey);
52243
52514
  Affirm_default(finalKeys.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
@@ -52250,7 +52521,7 @@ var init_LookupResolver = __esm({
52250
52521
  if (!line.length) continue;
52251
52522
  rowsRead++;
52252
52523
  Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
52253
- const record = this.parseLine(line, format3, visibleFields);
52524
+ const record = this.parseLine(line, format4, visibleFields);
52254
52525
  const code = String(record[ref.key]);
52255
52526
  if (code in dict && !warnedDup) {
52256
52527
  Logger_default.warn(`code_lookup: duplicate key(s) in consumer "${ref.consumer}" for "${ref.key}"; last value wins`);
@@ -52261,8 +52532,8 @@ var init_LookupResolver = __esm({
52261
52532
  Logger_default.log(`[lookup] Built table "${ref.consumer}" (${ref.key}->${ref.value}): ${Object.keys(dict).length} entr(ies) from ${rowsRead} row(s)`);
52262
52533
  return dict;
52263
52534
  };
52264
- this.parseLine = (line, format3, visibleFields) => {
52265
- if (format3 === "CSV") return LineParser_default._internalParseCSV(line, visibleFields, ",");
52535
+ this.parseLine = (line, format4, visibleFields) => {
52536
+ if (format4 === "CSV") return LineParser_default._internalParseCSV(line, visibleFields, ",");
52266
52537
  return JSON.parse(line);
52267
52538
  };
52268
52539
  this.getCapturePath = (scope, consumerName) => {
@@ -52289,11 +52560,11 @@ var init_LookupResolver = __esm({
52289
52560
  });
52290
52561
 
52291
52562
  // ../../packages/executors/src/ConsumerInputResolver.ts
52292
- var import_fs19, import_promises15, import_path27, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
52563
+ var import_fs19, import_promises16, import_path27, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
52293
52564
  var init_ConsumerInputResolver = __esm({
52294
52565
  "../../packages/executors/src/ConsumerInputResolver.ts"() {
52295
52566
  import_fs19 = __toESM(require("fs"));
52296
- import_promises15 = __toESM(require("fs/promises"));
52567
+ import_promises16 = __toESM(require("fs/promises"));
52297
52568
  import_path27 = __toESM(require("path"));
52298
52569
  init_src();
52299
52570
  init_src3();
@@ -52324,7 +52595,7 @@ var init_ConsumerInputResolver = __esm({
52324
52595
  const capturePath = ExecutorScope_default.getConsumerInputPath(scope, name);
52325
52596
  ExecutorScope_default.ensurePath(capturePath);
52326
52597
  await this.runUpstreamConsumer(upstream, capturePath, stack, request);
52327
- const { size } = await import_promises15.default.stat(capturePath);
52598
+ const { size } = await import_promises16.default.stat(capturePath);
52328
52599
  this.cache.set(name, { consumer: name, path: import_path27.default.resolve(capturePath), fileSize: size });
52329
52600
  }
52330
52601
  manifest.set(name, this.cache.get(name));
@@ -52387,11 +52658,11 @@ var ExecutorOrchestrator_exports = {};
52387
52658
  __export(ExecutorOrchestrator_exports, {
52388
52659
  default: () => ExecutorOrchestrator_default
52389
52660
  });
52390
- var import_fs20, import_promises16, import_path28, import_readline9, import_workerpool, import_promises17, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
52661
+ var import_fs20, import_promises17, import_path28, import_readline9, import_workerpool, import_promises18, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
52391
52662
  var init_ExecutorOrchestrator = __esm({
52392
52663
  "../../packages/executors/src/ExecutorOrchestrator.ts"() {
52393
52664
  import_fs20 = __toESM(require("fs"));
52394
- import_promises16 = __toESM(require("fs/promises"));
52665
+ import_promises17 = __toESM(require("fs/promises"));
52395
52666
  import_path28 = __toESM(require("path"));
52396
52667
  import_readline9 = __toESM(require("readline"));
52397
52668
  import_workerpool = __toESM(require("workerpool"));
@@ -52409,7 +52680,7 @@ var init_ExecutorOrchestrator = __esm({
52409
52680
  init_src();
52410
52681
  init_src10();
52411
52682
  init_ExecutorWriter();
52412
- import_promises17 = require("stream/promises");
52683
+ import_promises18 = require("stream/promises");
52413
52684
  init_src2();
52414
52685
  init_ExecutorProgress2();
52415
52686
  init_src6();
@@ -52627,7 +52898,8 @@ var init_ExecutorOrchestrator = __esm({
52627
52898
  } catch (error) {
52628
52899
  Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
52629
52900
  Logger_default.error(error);
52630
- await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
52901
+ if (!request.skipExport)
52902
+ await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
52631
52903
  Logger_default.log(`[${usageId}] Running cleanup after failure`);
52632
52904
  await this.performCleanupOperations(scope, tracker);
52633
52905
  UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
@@ -52925,10 +53197,10 @@ var init_ExecutorOrchestrator = __esm({
52925
53197
  * format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
52926
53198
  */
52927
53199
  this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
52928
- await import_promises16.default.mkdir(import_path28.default.dirname(targetPath), { recursive: true });
53200
+ await import_promises17.default.mkdir(import_path28.default.dirname(targetPath), { recursive: true });
52929
53201
  const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
52930
53202
  if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
52931
- await import_promises16.default.copyFile(sourcePath, targetPath);
53203
+ await import_promises17.default.copyFile(sourcePath, targetPath);
52932
53204
  return;
52933
53205
  }
52934
53206
  const visibleFields = fields.filter((field) => !field.cField.hidden);
@@ -53031,22 +53303,22 @@ var init_ExecutorOrchestrator = __esm({
53031
53303
  startRow: prod.settings.startRow,
53032
53304
  startColumn: prod.settings.startColumn
53033
53305
  });
53034
- await (0, import_promises17.pipeline)(
53306
+ await (0, import_promises18.pipeline)(
53035
53307
  xlsCsvStream,
53036
53308
  import_fs20.default.createWriteStream(decodedPath)
53037
53309
  );
53038
- const fileStats = await import_promises16.default.stat(decodedPath);
53310
+ const fileStats = await import_promises17.default.stat(decodedPath);
53039
53311
  decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
53040
53312
  decodedCount++;
53041
53313
  reportFile();
53042
53314
  continue;
53043
53315
  }
53044
53316
  if (inferredType === "XML") {
53045
- const fileContent = await import_promises16.default.readFile(file.fullUri, "utf-8");
53317
+ const fileContent = await import_promises17.default.readFile(file.fullUri, "utf-8");
53046
53318
  const jsonData = XMLParser_default.xmlToJson(fileContent);
53047
53319
  const records = normalizeXmlRows(jsonData);
53048
53320
  if (records.length === 0) {
53049
- await import_promises16.default.writeFile(decodedPath, "", "utf-8");
53321
+ await import_promises17.default.writeFile(decodedPath, "", "utf-8");
53050
53322
  } else {
53051
53323
  const columns = [];
53052
53324
  for (const record of records) {
@@ -53062,9 +53334,9 @@ var init_ExecutorOrchestrator = __esm({
53062
53334
  const row = columns.map((column) => csvSafeValue(record[column]));
53063
53335
  lines.push(CSVParser_default.stringifyRow(row));
53064
53336
  }
53065
- await import_promises16.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
53337
+ await import_promises17.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
53066
53338
  }
53067
- const fileStats = await import_promises16.default.stat(decodedPath);
53339
+ const fileStats = await import_promises17.default.stat(decodedPath);
53068
53340
  decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
53069
53341
  decodedCount++;
53070
53342
  reportFile();
@@ -53092,20 +53364,20 @@ var init_ExecutorOrchestrator = __esm({
53092
53364
  Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
53093
53365
  const perf = performance.now();
53094
53366
  for (const workerResult of executorResults) {
53095
- await (0, import_promises17.pipeline)(
53367
+ await (0, import_promises18.pipeline)(
53096
53368
  import_fs20.default.createReadStream(workerResult.resultUri),
53097
53369
  import_fs20.default.createWriteStream(mainPath, { flags: "a" })
53098
53370
  );
53099
- await import_promises16.default.unlink(workerResult.resultUri);
53371
+ await import_promises17.default.unlink(workerResult.resultUri);
53100
53372
  }
53101
53373
  tracker.measure("merge-workers", performance.now() - perf);
53102
53374
  Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
53103
53375
  } else if (executorResults.length === 1) {
53104
53376
  Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
53105
- await import_promises16.default.rename(executorResults[0].resultUri, mainPath);
53377
+ await import_promises17.default.rename(executorResults[0].resultUri, mainPath);
53106
53378
  } else {
53107
53379
  Logger_default.log(`[${scope.id}] No worker output \u2014 writing an empty dataset to ${mainPath}`);
53108
- await import_promises16.default.writeFile(mainPath, "");
53380
+ await import_promises17.default.writeFile(mainPath, "");
53109
53381
  }
53110
53382
  return mainPath;
53111
53383
  };
@@ -53826,6 +54098,164 @@ init_src4();
53826
54098
  init_OutputExecutor();
53827
54099
  init_ProducerExecutor();
53828
54100
 
54101
+ // ../../packages/executors/src/SampleRunner.ts
54102
+ var import_fs21 = __toESM(require("fs"));
54103
+ var import_promises19 = __toESM(require("fs/promises"));
54104
+ var import_path29 = __toESM(require("path"));
54105
+ var import_readline10 = __toESM(require("readline"));
54106
+ init_src();
54107
+ init_src3();
54108
+ init_src4();
54109
+ init_src8();
54110
+ init_src10();
54111
+ init_src5();
54112
+ init_src2();
54113
+ init_ExecutorOrchestrator();
54114
+ var MAX_WALK_DEPTH = 20;
54115
+ var SAMPLE_USER = { user: { _id: "sample", name: "sample", type: "licence" } };
54116
+ var SampleRunnerClass = class {
54117
+ constructor() {
54118
+ /**
54119
+ * @param resourceName a producer or consumer name, as declared.
54120
+ * @param rows the most rows to return, capped at `MAX_SAMPLE_ROWS`.
54121
+ * @throws when the name is neither, when a source cannot be read, or when the pipeline itself
54122
+ * fails — a consumer whose transformation throws is a sample that reports the reason.
54123
+ */
54124
+ this.read = async (resourceName, rows = 10, details) => {
54125
+ Affirm_default(resourceName, "Missing resource name to sample");
54126
+ Affirm_default(rows > 0, "Sample size must be greater than 0");
54127
+ const size = Math.min(rows, Constants_default.defaults.MAX_SAMPLE_ROWS);
54128
+ const producer = Environment_default.getProducer(resourceName);
54129
+ if (producer) {
54130
+ const records = await ProducerEngine_default.readSampleData(producer, size, false);
54131
+ return {
54132
+ kind: "producer",
54133
+ name: resourceName,
54134
+ records,
54135
+ note: `${records.length} row(s) read from the source now, with this producer's dimensions, types and masks applied.`
54136
+ };
54137
+ }
54138
+ const consumer = Environment_default.getConsumer(resourceName);
54139
+ Affirm_default(consumer, `Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`);
54140
+ return this._runConsumer(consumer, size, details);
54141
+ };
54142
+ /**
54143
+ * Runs the consumer for real and reads back what it produced.
54144
+ *
54145
+ * The output is captured instead of exported, so nothing reaches the consumer's destination and
54146
+ * no onSuccess action fires: looking at data must never be indistinguishable from publishing it.
54147
+ */
54148
+ this._runConsumer = async (consumer, size, details) => {
54149
+ this._assertStageable(consumer);
54150
+ const capturePath = import_path29.default.resolve(import_path29.default.join(ExecutorScope_default.getBasePath(), `sample_${consumer.name}_${Helper_default.uuid()}.jsonl`));
54151
+ ExecutorScope_default.ensurePath(capturePath);
54152
+ try {
54153
+ const result = await ExecutorOrchestrator_default.launch({
54154
+ consumer,
54155
+ // `SAMPLE`, so the run it records is distinguishable from one the project did. The
54156
+ // executor treats it like any other invocation; only `RunOverlay` reads the marker.
54157
+ details: { ...details ?? SAMPLE_USER, invokedBy: "SAMPLE" },
54158
+ // The rows asked for, bounded the way `remora run -l` bounds them — pushed to the
54159
+ // source and stopped early per worker wherever nothing downstream can drop a record.
54160
+ options: { limit: size },
54161
+ // Nothing reaches the destination and no finish action fires; the merged dataset
54162
+ // comes back through the same capture the lookup and consumer-input runs use.
54163
+ skipExport: true,
54164
+ captureMainDatasetTo: capturePath,
54165
+ captureFormat: "JSONL"
54166
+ });
54167
+ const captured = await this._readCapture(capturePath, size);
54168
+ const records = captured.map((record) => this._inFieldOrder(consumer, record));
54169
+ Logger_default.log(`[sample] Consumer "${consumer.name}": ${records.length} row(s) from ${result.inputCount} input line(s)`);
54170
+ return {
54171
+ kind: "consumer",
54172
+ name: consumer.name,
54173
+ records,
54174
+ note: `${records.length} row(s) from the real pipeline \u2014 filters, transformations, masking and validation all applied, exactly as in a run. Nothing was written to the destination.`
54175
+ };
54176
+ } finally {
54177
+ await import_promises19.default.rm(capturePath, { force: true });
54178
+ }
54179
+ };
54180
+ /**
54181
+ * Refuses, naming the source, when a run could not read one of this consumer's producers.
54182
+ *
54183
+ * `RedshiftDriver.ready` and `HttpApiSourceDriver.ready` are unimplemented, so the pipeline
54184
+ * cannot reach those sources at all. Checked up front because the alternative is a run that
54185
+ * spins up a worker pool and then surfaces "Not implemented yet" from three layers down, which
54186
+ * names neither the producer nor the source that caused it.
54187
+ */
54188
+ this._assertStageable = (consumer) => {
54189
+ for (const producer of this._inputProducers(consumer)) {
54190
+ const source = Environment_default.getSource(producer.source);
54191
+ Affirm_default(source, `Producer "${producer.name}" of consumer "${consumer.name}" has no source "${producer.source}"`);
54192
+ if (!DriverFactory_default.canStage(source.engine))
54193
+ throw new Error(`Cannot sample consumer "${consumer.name}": its producer "${producer.name}" reads source "${source.name}", and a "${source.engine}" source cannot be run through the pipeline yet. Sample the producer "${producer.name}" instead to see its rows.`);
54194
+ }
54195
+ };
54196
+ /**
54197
+ * Every producer a run of this consumer would stage — its own, plus those of the consumers it
54198
+ * reads as input or consults as a code_lookup table, since each of those is executed too.
54199
+ */
54200
+ this._inputProducers = (consumer, visited = []) => {
54201
+ Affirm_default(!visited.includes(consumer.name), `Consumer "${consumer.name}" reads itself through ${visited.join(" \u2192 ")}`);
54202
+ Affirm_default(visited.length < MAX_WALK_DEPTH, `Consumer chain from "${visited[0]}" is more than ${MAX_WALK_DEPTH} deep`);
54203
+ const seen = [...visited, consumer.name];
54204
+ const upstream = [
54205
+ ...consumer.producers.map((entry) => entry.name),
54206
+ ...ConsumerManager_default.collectLookupRefs(ConsumerManager_default.getExpandedFields(consumer)).map((ref) => ref.consumer)
54207
+ ];
54208
+ return upstream.flatMap((name) => {
54209
+ const producer = Environment_default.getProducer(name);
54210
+ if (producer) return [producer];
54211
+ const nested = Environment_default.getConsumer(name);
54212
+ Affirm_default(nested, `Producer or consumer "${name}" not found for consumer "${consumer.name}"`);
54213
+ return this._inputProducers(nested, seen);
54214
+ });
54215
+ };
54216
+ /**
54217
+ * One row's keys in the order the consumer DECLARES its fields.
54218
+ *
54219
+ * A record comes out of a worker keyed in the order the producer's dimensions were read, which is
54220
+ * the source's layout and not the consumer's — so a consumer declaring country, plan, is_active
54221
+ * over a producer that stores is_active first would be shown its own columns in someone else's
54222
+ * order. Invisible in an exported JSON object; the whole shape of the table here.
54223
+ *
54224
+ * Anything the record carries that the field list does not name is appended rather than dropped:
54225
+ * a column that turns up unexpectedly is exactly the one worth seeing.
54226
+ */
54227
+ this._inFieldOrder = (consumer, record) => {
54228
+ const declared = ConsumerManager_default.getExpandedFields(consumer).filter((field) => !field.cField.hidden).map((field) => field.finalKey);
54229
+ const ordered = {};
54230
+ for (const key of declared) {
54231
+ if (key in record) ordered[key] = record[key];
54232
+ }
54233
+ for (const key of Object.keys(record)) {
54234
+ if (!(key in ordered)) ordered[key] = record[key];
54235
+ }
54236
+ return ordered;
54237
+ };
54238
+ /** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
54239
+ this._readCapture = async (capturePath, size) => {
54240
+ if (!import_fs21.default.existsSync(capturePath)) return [];
54241
+ const records = [];
54242
+ const lines = import_readline10.default.createInterface({ input: import_fs21.default.createReadStream(capturePath), crlfDelay: Infinity });
54243
+ try {
54244
+ for await (const line of lines) {
54245
+ if (!line.length) continue;
54246
+ records.push(JSON.parse(line));
54247
+ if (records.length >= size) break;
54248
+ }
54249
+ } finally {
54250
+ lines.close();
54251
+ }
54252
+ return records;
54253
+ };
54254
+ }
54255
+ };
54256
+ var SampleRunner = new SampleRunnerClass();
54257
+ var SampleRunner_default = SampleRunner;
54258
+
53829
54259
  // src/actions/run.ts
53830
54260
  init_src4();
53831
54261
  init_src2();
@@ -53966,10 +54396,10 @@ init_src5();
53966
54396
  init_src2();
53967
54397
  var FORMATS = ["mermaid", "dot", "json"];
53968
54398
  var graph = async (options) => {
53969
- const format3 = options?.format ?? "mermaid";
54399
+ const format4 = options?.format ?? "mermaid";
53970
54400
  try {
53971
- if (!FORMATS.includes(format3)) {
53972
- console.error(import_chalk9.default.red(`Unknown format "${format3}". Use one of: ${FORMATS.join(", ")}.`));
54401
+ if (!FORMATS.includes(format4)) {
54402
+ console.error(import_chalk9.default.red(`Unknown format "${format4}". Use one of: ${FORMATS.join(", ")}.`));
53973
54403
  await Logger_default.flush();
53974
54404
  process.exit(1);
53975
54405
  }
@@ -53979,10 +54409,10 @@ var graph = async (options) => {
53979
54409
  generatedBy: `remora ${Constants_default.cliVersion}`,
53980
54410
  loadDiagnostics: Diagnostics_default.fromLoadProblems(problems),
53981
54411
  // The json format IS the viewer's contract, so every resource view ships with it.
53982
- resources: format3 === "json" ? "inline" : "lazy"
54412
+ resources: format4 === "json" ? "inline" : "lazy"
53983
54413
  });
53984
- process.stdout.write(render(format3, snapshot) + "\n");
53985
- if (format3 !== "json")
54414
+ process.stdout.write(render(format4, snapshot) + "\n");
54415
+ if (format4 !== "json")
53986
54416
  reportDiagnostics(snapshot.diagnostics);
53987
54417
  } catch (err2) {
53988
54418
  const myErr = Helper_default.asError(err2);
@@ -53992,9 +54422,9 @@ var graph = async (options) => {
53992
54422
  process.exit(1);
53993
54423
  }
53994
54424
  };
53995
- var render = (format3, snapshot) => {
53996
- if (format3 === "json") return JSON.stringify(snapshot, null, 4);
53997
- if (format3 === "dot") return LineageRenderer_default.toDot(snapshot.graph);
54425
+ var render = (format4, snapshot) => {
54426
+ if (format4 === "json") return JSON.stringify(snapshot, null, 4);
54427
+ if (format4 === "dot") return LineageRenderer_default.toDot(snapshot.graph);
53998
54428
  return LineageRenderer_default.toMermaid(snapshot.graph);
53999
54429
  };
54000
54430
  var reportDiagnostics = (diagnostics) => {
@@ -54018,6 +54448,7 @@ init_src5();
54018
54448
  init_src2();
54019
54449
  var WATCH_INTERVAL_MS = 1e3;
54020
54450
  var SAMPLEABLE_KINDS = ["producer", "consumer"];
54451
+ var PIPELINE_SAMPLE_KINDS = ["consumer"];
54021
54452
  var EDITABLE_KINDS = {
54022
54453
  source: "source",
54023
54454
  producer: "producer",
@@ -54054,11 +54485,15 @@ var PreviewProject = class {
54054
54485
  this._files = [];
54055
54486
  this._hash = null;
54056
54487
  /**
54057
- * The consumer a run started from this page is executing, or null.
54488
+ * What this page is currently executing, or null — a run, or a consumer sample.
54489
+ *
54490
+ * One at a time, deliberately: `remora run` runs consumers sequentially, and two concurrent
54491
+ * executions would each spin up a worker pool in this one process. Refusing the second with the
54492
+ * name of the first is a better answer than thrashing the machine.
54058
54493
  *
54059
- * One at a time, deliberately: `remora run` runs consumers sequentially, and two concurrent runs
54060
- * would each spin up a worker pool in this one process. Refusing the second with the name of the
54061
- * first is a better answer than thrashing the machine.
54494
+ * A sample takes the same lock because it IS an execution: bounded at the input and exporting
54495
+ * nothing, but a worker pool either way. That also means a sample and a run cannot overlap, which
54496
+ * is what stops a click on one panel from slowing the run someone is watching on another.
54062
54497
  */
54063
54498
  this._running = null;
54064
54499
  /** @returns true when the project changed and the snapshot was rebuilt. */
@@ -54104,18 +54539,47 @@ var PreviewProject = class {
54104
54539
  ]
54105
54540
  };
54106
54541
  },
54107
- sample: async (nodeId, rows) => {
54108
- if (!this._has(nodeId)) return null;
54109
- const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
54110
- if (!SAMPLEABLE_KINDS.includes(kind)) return null;
54111
- const { records } = await SampleEngine_default.read(name, rows);
54112
- return ResourceView_default.buildSampleTable(records);
54113
- },
54542
+ sample: async (nodeId, rows) => this._sample(nodeId, rows),
54114
54543
  run: async (nodeId) => this._run(nodeId),
54115
54544
  file: (nodeId) => this._file(nodeId),
54116
54545
  save: async (nodeId, content) => this._save(nodeId, content),
54117
54546
  logs: (lines) => this._logs(lines)
54118
54547
  });
54548
+ /**
54549
+ * The rows a resource actually holds, as a table.
54550
+ *
54551
+ * A producer answers from a bounded read of its source. A consumer answers by RUNNING — every
54552
+ * filter, transformation, mask, union, distinct, pivot and validation it declares, over a
54553
+ * bounded slice of each producer's input, exporting nothing and recording nothing. Which of the
54554
+ * two happened, and over how much input, is the note `SampleRunner` returns; it becomes the
54555
+ * table's title, so the panel states what it is showing instead of leaving it to be assumed.
54556
+ *
54557
+ * Awaited, unlike a run: a sample is bounded, and its ANSWER is the rows — there is nothing for
54558
+ * the run overlay to show, because a sample deliberately leaves no trace on it.
54559
+ */
54560
+ this._sample = async (nodeId, rows) => {
54561
+ if (!this._has(nodeId)) return null;
54562
+ const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
54563
+ if (!SAMPLEABLE_KINDS.includes(kind)) return null;
54564
+ const pipeline6 = PIPELINE_SAMPLE_KINDS.includes(kind);
54565
+ if (pipeline6 && this._running)
54566
+ throw new Error(`"${this._running}" is still running; one execution at a time`);
54567
+ const blocking = pipeline6 ? this.snapshot.diagnostics.find((x2) => x2.node === nodeId && x2.severity === "error") : null;
54568
+ if (blocking) throw new Error(`Refusing to sample "${name}": ${blocking.message}`);
54569
+ if (pipeline6) this._running = name;
54570
+ try {
54571
+ const licence = LicenceManager_default.validate(ProcessENVManager_default.getEnvVariable("REMORA_LICENCE_KEY"));
54572
+ const { records, note } = await SampleRunner_default.read(name, rows, {
54573
+ // A sample from the preview is a sample from the CLI. It records no usage, so this
54574
+ // only ever names the run in the log.
54575
+ invokedBy: "CLI",
54576
+ user: { _id: licence.customer, name: licence.customer, type: "licence" }
54577
+ });
54578
+ return ResourceView_default.buildSampleTable(records, note);
54579
+ } finally {
54580
+ if (pipeline6) this._running = null;
54581
+ }
54582
+ };
54119
54583
  /**
54120
54584
  * Executes a consumer, and answers as soon as it has STARTED.
54121
54585
  *
@@ -54127,7 +54591,7 @@ var PreviewProject = class {
54127
54591
  if (!this._has(nodeId)) return { started: false, message: `No resource "${nodeId}" in this project` };
54128
54592
  const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
54129
54593
  if (kind !== "consumer") return { started: false, message: `Only a consumer can be run; "${name}" is a ${kind}` };
54130
- if (this._running) return { started: false, message: `"${this._running}" is still running; one run at a time` };
54594
+ if (this._running) return { started: false, message: `"${this._running}" is still running; one execution at a time` };
54131
54595
  const blocking = this.snapshot.diagnostics.find((x2) => x2.node === nodeId && x2.severity === "error");
54132
54596
  if (blocking) return { started: false, message: `Refusing to run "${name}": ${blocking.message}` };
54133
54597
  const consumer = Environment_default.getConsumer(name);
@@ -54251,22 +54715,22 @@ var stopOnSignal = (server, watch) => {
54251
54715
 
54252
54716
  // src/actions/create_producer.ts
54253
54717
  var import_chalk11 = __toESM(require("chalk"));
54254
- var import_fs21 = __toESM(require("fs"));
54255
- var import_path29 = __toESM(require("path"));
54718
+ var import_fs22 = __toESM(require("fs"));
54719
+ var import_path30 = __toESM(require("path"));
54256
54720
  init_src2();
54257
54721
  var create_producer = async (name) => {
54258
54722
  try {
54259
- if (!import_fs21.default.existsSync("./remora/producers")) {
54723
+ if (!import_fs22.default.existsSync("./remora/producers")) {
54260
54724
  throw new Error(import_chalk11.default.red("Missing directory: ") + import_chalk11.default.yellow("./remora/producers"));
54261
54725
  }
54262
- const defaultProducerTemplate = import_fs21.default.readFileSync(
54263
- import_path29.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
54726
+ const defaultProducerTemplate = import_fs22.default.readFileSync(
54727
+ import_path30.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
54264
54728
  "utf-8"
54265
54729
  );
54266
54730
  const defaultProducer = JSON.parse(defaultProducerTemplate);
54267
54731
  defaultProducer.name = name;
54268
- const producerPath = import_path29.default.join("remora/producers", `${name}.json`);
54269
- import_fs21.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
54732
+ const producerPath = import_path30.default.join("remora/producers", `${name}.json`);
54733
+ import_fs22.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
54270
54734
  console.log(import_chalk11.default.green(`\u2705 Created producer config at ${producerPath}`));
54271
54735
  console.log(import_chalk11.default.blue("Remember to:"));
54272
54736
  console.log(import_chalk11.default.blue("1. Set the correct source name"));
@@ -54281,26 +54745,26 @@ var create_producer = async (name) => {
54281
54745
 
54282
54746
  // src/actions/create_consumer.ts
54283
54747
  var import_chalk12 = __toESM(require("chalk"));
54284
- var import_fs22 = __toESM(require("fs"));
54285
- var import_path30 = __toESM(require("path"));
54748
+ var import_fs23 = __toESM(require("fs"));
54749
+ var import_path31 = __toESM(require("path"));
54286
54750
  init_src2();
54287
54751
  var create_consumer = async (name, producerName) => {
54288
54752
  try {
54289
- if (!import_fs22.default.existsSync("./remora/consumers")) {
54753
+ if (!import_fs23.default.existsSync("./remora/consumers")) {
54290
54754
  throw new Error(import_chalk12.default.red("Missing directory: ") + import_chalk12.default.yellow("./remora/consumers"));
54291
54755
  }
54292
- const defaultConsumerTemplate = import_fs22.default.readFileSync(
54293
- import_path30.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
54756
+ const defaultConsumerTemplate = import_fs23.default.readFileSync(
54757
+ import_path31.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
54294
54758
  "utf-8"
54295
54759
  );
54296
54760
  const defaultConsumer = JSON.parse(defaultConsumerTemplate);
54297
54761
  defaultConsumer.name = name;
54298
54762
  if (producerName) {
54299
- const producerPath = import_path30.default.join("remora/producers", `${producerName}.json`);
54300
- if (!import_fs22.default.existsSync(producerPath)) {
54763
+ const producerPath = import_path31.default.join("remora/producers", `${producerName}.json`);
54764
+ if (!import_fs23.default.existsSync(producerPath)) {
54301
54765
  throw new Error(import_chalk12.default.red("Producer not found: ") + import_chalk12.default.yellow(producerPath));
54302
54766
  }
54303
- const producerConfig = JSON.parse(import_fs22.default.readFileSync(producerPath, "utf-8"));
54767
+ const producerConfig = JSON.parse(import_fs23.default.readFileSync(producerPath, "utf-8"));
54304
54768
  defaultConsumer.producers = [{ name: producerName }];
54305
54769
  defaultConsumer.fields = producerConfig.dimensions.map((dim) => ({
54306
54770
  key: dim.name,
@@ -54319,8 +54783,8 @@ var create_consumer = async (name, producerName) => {
54319
54783
  defaultConsumer.filters = void 0;
54320
54784
  defaultConsumer.metadata = void 0;
54321
54785
  }
54322
- const consumerPath = import_path30.default.join("remora/consumers", `${name}.json`);
54323
- import_fs22.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
54786
+ const consumerPath = import_path31.default.join("remora/consumers", `${name}.json`);
54787
+ import_fs23.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
54324
54788
  console.log(import_chalk12.default.green(`\u2705 Created consumer config at ${consumerPath}`));
54325
54789
  if (!producerName) {
54326
54790
  console.log(import_chalk12.default.blue("Remember to:"));
@@ -54348,8 +54812,8 @@ init_src5();
54348
54812
  var import_chalk13 = __toESM(require("chalk"));
54349
54813
  var import_ora5 = __toESM(require("ora"));
54350
54814
  init_src10();
54351
- var import_fs23 = __toESM(require("fs"));
54352
- var import_path31 = __toESM(require("path"));
54815
+ var import_fs24 = __toESM(require("fs"));
54816
+ var import_path32 = __toESM(require("path"));
54353
54817
  init_src4();
54354
54818
  init_src10();
54355
54819
  init_src2();
@@ -54378,14 +54842,14 @@ var automap = async (producerName, schemaNames) => {
54378
54842
  const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
54379
54843
  for (const producer2 of mapResult.producers) {
54380
54844
  const producerPath = Environment_default.resolveResourceFilePath("producers", producer2.name);
54381
- import_fs23.default.mkdirSync(import_path31.default.dirname(producerPath), { recursive: true });
54382
- import_fs23.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
54845
+ import_fs24.default.mkdirSync(import_path32.default.dirname(producerPath), { recursive: true });
54846
+ import_fs24.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
54383
54847
  console.log(import_chalk13.default.blue(`Created producer: ${producer2.name}`));
54384
54848
  }
54385
54849
  for (const consumer of mapResult.consumers) {
54386
54850
  const consumerPath = Environment_default.resolveResourceFilePath("consumers", consumer.name);
54387
- import_fs23.default.mkdirSync(import_path31.default.dirname(consumerPath), { recursive: true });
54388
- import_fs23.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
54851
+ import_fs24.default.mkdirSync(import_path32.default.dirname(consumerPath), { recursive: true });
54852
+ import_fs24.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
54389
54853
  console.log(import_chalk13.default.blue(`Created consumer: ${consumer.name}`));
54390
54854
  }
54391
54855
  spinner.succeed("Producer has been successfully mapped");
@@ -54405,7 +54869,6 @@ var automap = async (producerName, schemaNames) => {
54405
54869
  // src/actions/sample.ts
54406
54870
  var import_chalk14 = __toESM(require("chalk"));
54407
54871
  var import_ora6 = __toESM(require("ora"));
54408
- init_src10();
54409
54872
  init_src5();
54410
54873
  init_src2();
54411
54874
  var sample = async (resourceName, sampleSize = 10) => {
@@ -54413,8 +54876,9 @@ var sample = async (resourceName, sampleSize = 10) => {
54413
54876
  compile();
54414
54877
  console.log();
54415
54878
  const spinner = (0, import_ora6.default)(import_chalk14.default.blue(`Sampling "${resourceName}"...`)).start();
54416
- const { kind, records } = await SampleEngine_default.read(resourceName, sampleSize);
54879
+ const { kind, records, note } = await SampleRunner_default.read(resourceName, sampleSize);
54417
54880
  spinner.succeed(import_chalk14.default.green(`Sample data retrieved from ${kind} "${resourceName}"`));
54881
+ console.log(import_chalk14.default.gray(` ${note}`));
54418
54882
  if (records.length === 0) {
54419
54883
  console.log(import_chalk14.default.yellow("No data found in the dataset."));
54420
54884
  return;