@forzalabs/remora 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8538,7 +8538,7 @@ var require_node2 = __commonJS({
8538
8538
  var require_tail_file = __commonJS({
8539
8539
  "../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
8540
8540
  "use strict";
8541
- var fs32 = require("fs");
8541
+ var fs33 = require("fs");
8542
8542
  var { StringDecoder } = require("string_decoder");
8543
8543
  var { Stream } = require_readable();
8544
8544
  function noop() {
@@ -8559,7 +8559,7 @@ var require_tail_file = __commonJS({
8559
8559
  stream.emit("end");
8560
8560
  stream.emit("close");
8561
8561
  };
8562
- fs32.open(options2.file, "a+", "0644", (err2, fd) => {
8562
+ fs33.open(options2.file, "a+", "0644", (err2, fd) => {
8563
8563
  if (err2) {
8564
8564
  if (!iter) {
8565
8565
  stream.emit("error", err2);
@@ -8571,10 +8571,10 @@ var require_tail_file = __commonJS({
8571
8571
  }
8572
8572
  (function read() {
8573
8573
  if (stream.destroyed) {
8574
- fs32.close(fd, noop);
8574
+ fs33.close(fd, noop);
8575
8575
  return;
8576
8576
  }
8577
- return fs32.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
8577
+ return fs33.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
8578
8578
  if (error) {
8579
8579
  if (!iter) {
8580
8580
  stream.emit("error", error);
@@ -8633,7 +8633,7 @@ var require_tail_file = __commonJS({
8633
8633
  var require_file = __commonJS({
8634
8634
  "../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
8635
8635
  "use strict";
8636
- var fs32 = require("fs");
8636
+ var fs33 = require("fs");
8637
8637
  var path22 = require("path");
8638
8638
  var asyncSeries = require_series();
8639
8639
  var zlib3 = require("zlib");
@@ -8838,7 +8838,7 @@ var require_file = __commonJS({
8838
8838
  let buff = "";
8839
8839
  let results = [];
8840
8840
  let row = 0;
8841
- const stream = fs32.createReadStream(file, {
8841
+ const stream = fs33.createReadStream(file, {
8842
8842
  encoding: "utf8"
8843
8843
  });
8844
8844
  stream.on("error", (err2) => {
@@ -8990,7 +8990,7 @@ var require_file = __commonJS({
8990
8990
  stat(callback) {
8991
8991
  const target = this._getFile();
8992
8992
  const fullpath = path22.join(this.dirname, target);
8993
- fs32.stat(fullpath, (err2, stat) => {
8993
+ fs33.stat(fullpath, (err2, stat) => {
8994
8994
  if (err2 && err2.code === "ENOENT") {
8995
8995
  debug("ENOENT\xA0ok", fullpath);
8996
8996
  this.filename = target;
@@ -9095,7 +9095,7 @@ var require_file = __commonJS({
9095
9095
  _createStream(source) {
9096
9096
  const fullpath = path22.join(this.dirname, this.filename);
9097
9097
  debug("create stream start", fullpath, this.options);
9098
- const dest = fs32.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
9098
+ const dest = fs33.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
9099
9099
  debug("file open ok", fullpath);
9100
9100
  this.emit("open", fullpath);
9101
9101
  source.pipe(dest);
@@ -9174,7 +9174,7 @@ var require_file = __commonJS({
9174
9174
  const isZipped = this.zippedArchive ? ".gz" : "";
9175
9175
  const filePath = `${basename}${isOldest}${ext}${isZipped}`;
9176
9176
  const target = path22.join(this.dirname, filePath);
9177
- fs32.unlink(target, callback);
9177
+ fs33.unlink(target, callback);
9178
9178
  }
9179
9179
  /**
9180
9180
  * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -9197,17 +9197,17 @@ var require_file = __commonJS({
9197
9197
  tasks.push(function(i6, cb) {
9198
9198
  let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
9199
9199
  const tmppath = path22.join(this.dirname, fileName);
9200
- fs32.exists(tmppath, (exists) => {
9200
+ fs33.exists(tmppath, (exists) => {
9201
9201
  if (!exists) {
9202
9202
  return cb(null);
9203
9203
  }
9204
9204
  fileName = `${basename}${i6}${ext}${isZipped}`;
9205
- fs32.rename(tmppath, path22.join(this.dirname, fileName), cb);
9205
+ fs33.rename(tmppath, path22.join(this.dirname, fileName), cb);
9206
9206
  });
9207
9207
  }.bind(this, x2));
9208
9208
  }
9209
9209
  asyncSeries(tasks, () => {
9210
- fs32.rename(
9210
+ fs33.rename(
9211
9211
  path22.join(this.dirname, `${basename}${ext}${isZipped}`),
9212
9212
  path22.join(this.dirname, `${basename}1${ext}${isZipped}`),
9213
9213
  callback
@@ -9223,22 +9223,22 @@ var require_file = __commonJS({
9223
9223
  * @private
9224
9224
  */
9225
9225
  _compressFile(src, dest, callback) {
9226
- fs32.access(src, fs32.F_OK, (err2) => {
9226
+ fs33.access(src, fs33.F_OK, (err2) => {
9227
9227
  if (err2) {
9228
9228
  return callback();
9229
9229
  }
9230
9230
  var gzip = zlib3.createGzip();
9231
- var inp = fs32.createReadStream(src);
9232
- var out = fs32.createWriteStream(dest);
9231
+ var inp = fs33.createReadStream(src);
9232
+ var out = fs33.createWriteStream(dest);
9233
9233
  out.on("finish", () => {
9234
- fs32.unlink(src, callback);
9234
+ fs33.unlink(src, callback);
9235
9235
  });
9236
9236
  inp.pipe(gzip).pipe(out);
9237
9237
  });
9238
9238
  }
9239
9239
  _createLogDirIfNotExist(dirPath) {
9240
- if (!fs32.existsSync(dirPath)) {
9241
- fs32.mkdirSync(dirPath, { recursive: true });
9240
+ if (!fs33.existsSync(dirPath)) {
9241
+ fs33.mkdirSync(dirPath, { recursive: true });
9242
9242
  }
9243
9243
  }
9244
9244
  };
@@ -11809,7 +11809,7 @@ var CONSTANTS, Constants_default;
11809
11809
  var init_Constants = __esm({
11810
11810
  "../../packages/constants/src/Constants.ts"() {
11811
11811
  CONSTANTS = {
11812
- cliVersion: "2.4.0",
11812
+ cliVersion: "2.6.0",
11813
11813
  backendVersion: 1,
11814
11814
  backendPort: 5088,
11815
11815
  workerVersion: 2,
@@ -11934,6 +11934,13 @@ var init_Constants = __esm({
11934
11934
  * mode when its next line arrives — its byte counter lives in the bucket, not the stream.
11935
11935
  */
11936
11936
  MAX_OPEN_SPLIT_STREAMS: 64,
11937
+ /**
11938
+ * Record layouts one FIXED output may write into a single file, which is also how many write
11939
+ * streams its block-ordering pass holds open. A copybook of this size is already unusual —
11940
+ * the NASCO POD feed writes five transaction types — and the bound stops a generated config
11941
+ * from opening a descriptor per distinct value of the wrong field.
11942
+ */
11943
+ MAX_FIXED_LAYOUTS: 64,
11937
11944
  /**
11938
11945
  * Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
11939
11946
  * export of it — `remora sample -s` and the preview's Data tab are both held to this.
@@ -13176,7 +13183,7 @@ var init_Validator = __esm({
13176
13183
  errors.push(`Consumer "${consumer.name}" declares ${consumer.outputs.length} outputs (${consumer.outputs.map((x2) => x2.format).join(", ")}). A consumer writes exactly one output \u2014 only the first was ever written \u2014 so split it into one consumer per output.`);
13177
13184
  for (const output of consumer.outputs) {
13178
13185
  const format4 = output.format.toUpperCase();
13179
- if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
13186
+ if (format4 === "CSV" || format4 === "FIXED" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
13180
13187
  if (!output.exportDestination)
13181
13188
  errors.push(`A static file output must have an export destination set (${format4})`);
13182
13189
  else if (!Environment_default.getSource(output.exportDestination))
@@ -13186,6 +13193,7 @@ var init_Validator = __esm({
13186
13193
  errors.push(...this.validateOmitNull(consumer));
13187
13194
  errors.push(...this.validateHeader(consumer));
13188
13195
  errors.push(...this.validateSplit(consumer));
13196
+ errors.push(...this.validateFixedOutput(consumer));
13189
13197
  const dimensionFields = consumer.fields.filter((x2) => x2.key !== "*" && !x2.fixed && !x2.copyFrom && !x2.lineCount);
13190
13198
  const dimensionKeys = dimensionFields.map((x2) => `${x2.from ?? "_default_"}::${x2.key}`);
13191
13199
  const duplicateDimensionKeys = dimensionKeys.filter((k7, i6) => dimensionKeys.indexOf(k7) !== i6);
@@ -13254,7 +13262,7 @@ var init_Validator = __esm({
13254
13262
  this.validateSplit = (consumer) => {
13255
13263
  const errors = [];
13256
13264
  const { split: split2 } = consumer;
13257
- const positionalFormats = ["CSV", "TXT", "PARQUET"];
13265
+ const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
13258
13266
  if (!Algo_default.hasVal(split2)) return errors;
13259
13267
  if (Algo_default.hasVal(consumer.maximumFileSize))
13260
13268
  errors.push(`Consumer "${consumer.name}" sets both "maximumFileSize" and "split". They are the same cap: move the value to "split": { "maxFileSizeGb": ${consumer.maximumFileSize} } and remove "maximumFileSize".`);
@@ -13285,7 +13293,7 @@ var init_Validator = __esm({
13285
13293
  */
13286
13294
  this.validateOmitNull = (consumer) => {
13287
13295
  const errors = [];
13288
- const positionalFormats = ["CSV", "TXT", "PARQUET"];
13296
+ const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
13289
13297
  for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
13290
13298
  const format4 = output.format.toUpperCase();
13291
13299
  if (positionalFormats.includes(format4))
@@ -13305,8 +13313,47 @@ var init_Validator = __esm({
13305
13313
  for (const output of (consumer.outputs ?? []).filter((x2) => Algo_default.hasVal(x2.header))) {
13306
13314
  const format4 = output.format.toUpperCase();
13307
13315
  if (!headerFormats.includes(format4))
13308
- errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: a ${format4} ${format4 === "PARQUET" ? "file carries its own schema" : "record already carries its own field names"}.`);
13316
+ errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: ${this.headerlessReason(format4)}.`);
13317
+ }
13318
+ return errors;
13319
+ };
13320
+ /** Why the format in hand has no header line, as the clause that finishes the sentence above. */
13321
+ this.headerlessReason = (format4) => {
13322
+ if (format4 === "FIXED") return "a positional line has no room for one, and a field name would not sit on the columns it describes";
13323
+ if (format4 === "PARQUET") return `a ${format4} file carries its own schema`;
13324
+ return `a ${format4} record already carries its own field names`;
13325
+ };
13326
+ /**
13327
+ * Checks a consumer's FIXED output from its configuration alone: the properties only a
13328
+ * positional file has, the ones it cannot have, and whether anything downstream tries to read
13329
+ * one back. The layout itself — widths, columns and record lengths — is resolved against the
13330
+ * dataset's columns, which the producers define, so it is checked by `compile` and again before
13331
+ * a run starts rather than here.
13332
+ *
13333
+ * A property declared on the wrong format is rejected rather than ignored, the way a `position`
13334
+ * on a non-FIXED producer is: silently dropping a rule the author wrote is how a file ships
13335
+ * looking almost right.
13336
+ */
13337
+ this.validateFixedOutput = (consumer) => {
13338
+ Affirm_default(consumer, "Invalid consumer");
13339
+ const errors = [];
13340
+ const output = consumer.outputs?.[0];
13341
+ if (!output) return errors;
13342
+ const format4 = output.format?.toUpperCase();
13343
+ const positioned = (consumer.fields ?? []).filter((x2) => x2.position).map((x2) => `"${x2.alias ?? x2.key}"`);
13344
+ if (format4 !== "FIXED") {
13345
+ const fixedOnly = ["recordLength", "defaults", "lineEnding", "layouts"];
13346
+ const declared = fixedOnly.filter((name) => Algo_default.hasVal(output[name]));
13347
+ if (declared.length > 0)
13348
+ errors.push(`The ${format4} output of consumer "${consumer.name}" declares ${declared.map((x2) => `"${x2}"`).join(", ")}, which only ${declared.length > 1 ? "apply" : "applies"} to a FIXED output, where ${declared.length > 1 ? "they are" : "it is"} ignored.`);
13349
+ if (positioned.length > 0)
13350
+ errors.push(`Consumer "${consumer.name}" declares "position" on field(s) ${positioned.join(", ")} but its output format is not "FIXED", where positions are ignored.`);
13351
+ return errors;
13309
13352
  }
13353
+ if (Algo_default.hasVal(output.delimiter))
13354
+ errors.push(`The FIXED output of consumer "${consumer.name}" declares a "delimiter". A positional line has none: its fields are found by the columns they occupy.`);
13355
+ if ((consumer.fields ?? []).some((x2) => x2.key === "*"))
13356
+ errors.push(`Consumer "${consumer.name}" selects fields with the wildcard key "*", which a FIXED output can't write: every field of a positional line declares its own "width", so the field list has to be explicit.`);
13310
13357
  return errors;
13311
13358
  };
13312
13359
  /**
@@ -14889,7 +14936,7 @@ A consumer takes the data from one or more producers and changes it's shape to t
14889
14936
  # RULES
14890
14937
  - If a field is not needed, do not add it e.g.
14891
14938
  - Only import a producer once
14892
- - Awlays include this exact property as the first -> $schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
14939
+ - Awlays include this exact property as the first -> "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
14893
14940
  - Use "API" as the only valid output format.
14894
14941
  - The "from" must contain only the name of the producer
14895
14942
 
@@ -33698,21 +33745,29 @@ var init_CSVParser = __esm({
33698
33745
  });
33699
33746
 
33700
33747
  // ../../packages/parsing/src/DatasetCodec.ts
33701
- var NULL_FIELD, NEEDS_ESCAPE, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
33748
+ var NULL_FIELD, EMPTY_FIELD, NEEDS_ESCAPE, QUOTE_CODE, QUOTE_TESTS, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
33702
33749
  var init_DatasetCodec = __esm({
33703
33750
  "../../packages/parsing/src/DatasetCodec.ts"() {
33704
33751
  init_src();
33705
33752
  NULL_FIELD = "";
33753
+ EMPTY_FIELD = '""';
33706
33754
  NEEDS_ESCAPE = /["\n\r]/;
33755
+ QUOTE_CODE = 34;
33756
+ QUOTE_TESTS = /* @__PURE__ */ new Map([[",", /^\s|\s$|[",\n\r]/]]);
33707
33757
  DatasetCodecClass = class {
33708
33758
  constructor() {
33709
33759
  /**
33710
- * One value as an internal positional field: quoted, with `"`, `\n` and `\r` escaped - or the
33711
- * unquoted null marker when there is no value.
33760
+ * One value as an internal positional field. Bare where it can be, quoted where it must be -
33761
+ * around the delimiter, a quote, a line break or an edge space - or the unquoted null marker
33762
+ * when there is no value. The delimiter is a parameter rather than a comma because `redelimit`
33763
+ * re-encodes a staged line with the export's own: a tab is safe bare in the staged file and is
33764
+ * not once that file is re-emitted tab-delimited.
33712
33765
  */
33713
- this.encodeField = (value) => {
33766
+ this.encodeField = (value, delimiter = ",") => {
33714
33767
  if (value === null || value === void 0) return NULL_FIELD;
33715
33768
  const text = typeof value === "string" ? value : String(value);
33769
+ if (text === "") return EMPTY_FIELD;
33770
+ if (!this._quoteTest(delimiter).test(text)) return text;
33716
33771
  if (!NEEDS_ESCAPE.test(text)) return `"${text}"`;
33717
33772
  return `"${text.replaceAll('"', '""').replaceAll("\r", "\\r").replaceAll("\n", "\\n")}"`;
33718
33773
  };
@@ -33724,7 +33779,7 @@ var init_DatasetCodec = __esm({
33724
33779
  for (let index = 0; index < values.length; index++) {
33725
33780
  if (index > 0)
33726
33781
  row += delimiter;
33727
- row += this.encodeField(values[index]);
33782
+ row += this.encodeField(values[index], delimiter);
33728
33783
  }
33729
33784
  return row;
33730
33785
  };
@@ -33744,26 +33799,26 @@ var init_DatasetCodec = __esm({
33744
33799
  let index = 0;
33745
33800
  while (index < length) {
33746
33801
  const char = line.charCodeAt(index);
33747
- if (char === 34) {
33802
+ if (char === QUOTE_CODE) {
33748
33803
  if (!inQuotes) {
33749
33804
  inQuotes = true;
33750
33805
  hasQuotes = true;
33751
33806
  fieldStart = index + 1;
33752
- } else if (line.charCodeAt(index + 1) === 34) {
33807
+ } else if (line.charCodeAt(index + 1) === QUOTE_CODE) {
33753
33808
  index++;
33754
33809
  } else {
33755
33810
  inQuotes = false;
33756
33811
  fieldEnd = index;
33757
33812
  }
33758
33813
  } else if (char === delimiterCode && !inQuotes) {
33759
- fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
33814
+ fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : index, hasQuotes));
33760
33815
  fieldStart = index + 1;
33761
33816
  fieldEnd = 0;
33762
33817
  hasQuotes = false;
33763
33818
  }
33764
33819
  index++;
33765
33820
  }
33766
- fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
33821
+ fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : length, hasQuotes));
33767
33822
  return fields;
33768
33823
  };
33769
33824
  /**
@@ -33774,9 +33829,24 @@ var init_DatasetCodec = __esm({
33774
33829
  Affirm_default(delimiter, "Invalid delimiter");
33775
33830
  return this.encodeRow(this.decodeRow(line), delimiter);
33776
33831
  };
33777
- /** One field's text, or null for the unquoted marker. Only a quoted token carries a value. */
33832
+ /**
33833
+ * The quoting test for one delimiter, built on first use. The four characters escaped into the
33834
+ * class are the ones a character class reads as syntax; anything else stands for itself there.
33835
+ */
33836
+ this._quoteTest = (delimiter) => {
33837
+ let test = QUOTE_TESTS.get(delimiter);
33838
+ if (!test) {
33839
+ test = new RegExp(`^\\s|\\s$|["\\n\\r${delimiter.replace(/[\\\]^-]/g, "\\$&")}]`);
33840
+ QUOTE_TESTS.set(delimiter, test);
33841
+ }
33842
+ return test;
33843
+ };
33844
+ /**
33845
+ * One field's text, or null for the unquoted empty marker. A bare token carries its value as it
33846
+ * stands - nothing bare can hold a quote, so there is nothing to unescape in it.
33847
+ */
33778
33848
  this._field = (line, start, end, hasQuotes) => {
33779
- if (!hasQuotes) return null;
33849
+ if (!hasQuotes) return start === end ? null : line.slice(start, end);
33780
33850
  const raw = line.slice(start, end);
33781
33851
  return raw.indexOf('"') === -1 ? raw : raw.replaceAll('""', '"');
33782
33852
  };
@@ -33941,6 +34011,307 @@ var init_FixedWidthParser = __esm({
33941
34011
  }
33942
34012
  });
33943
34013
 
34014
+ // ../../packages/parsing/src/FixedWidthWriter.ts
34015
+ var BASE_FILL, TYPE_FILL, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
34016
+ var init_FixedWidthWriter = __esm({
34017
+ "../../packages/parsing/src/FixedWidthWriter.ts"() {
34018
+ init_src();
34019
+ init_src3();
34020
+ BASE_FILL = { align: "left", pad: " ", nullFill: " ", overflow: "fail" };
34021
+ TYPE_FILL = {
34022
+ string: BASE_FILL,
34023
+ number: { ...BASE_FILL, align: "right", pad: "0" },
34024
+ datetime: BASE_FILL,
34025
+ boolean: BASE_FILL
34026
+ };
34027
+ FixedWidthWriterClass = class {
34028
+ constructor() {
34029
+ /**
34030
+ * Resolves one record layout for writing: every column's fill rules decided, its columns
34031
+ * computed, and the record length agreed with what the output declares.
34032
+ *
34033
+ * Called once per layout when an export starts, never per record. `columns` is the line as the
34034
+ * dataset schema has it — line order, visible fields only — and `positions` carries each one's
34035
+ * `position` block, keyed by the same name the column has.
34036
+ *
34037
+ * `context` is the phrase that names this layout in an error, e.g. `the FIXED output of consumer
34038
+ * "x"`, so a message reads as one sentence wherever the layout came from.
34039
+ */
34040
+ this.compilePlan = (columns, positions, output, context = "the FIXED output") => {
34041
+ Affirm_default(output, "Invalid consumer output");
34042
+ const selected = columns?.map((column, index) => ({ column, index }));
34043
+ return this.plan(selected, positions, output.defaults, output.recordLength, context);
34044
+ };
34045
+ /**
34046
+ * A FIXED output resolved for INSPECTION rather than for writing: every problem with it, and the
34047
+ * layouts it would write. What `compile` reports and prints.
34048
+ *
34049
+ * The rules are not restated here. The layouts are resolved by the same code the export runs,
34050
+ * and what that throws is reported rather than raised — so a layout `compile` accepts is a
34051
+ * layout the export can write. What review adds is that every field's own declaration is
34052
+ * checked first, across the whole output: an author with six fields missing a width sees six
34053
+ * lines from one `compile`, not one line from each of six.
34054
+ *
34055
+ * A field named by a layout that the dataset has no column for is left to the resolution below,
34056
+ * which names it as the layout's error rather than as a field with no width.
34057
+ */
34058
+ this.review = (dsSchema, positions, output, consumerName) => {
34059
+ Affirm_default(dsSchema, "Invalid dataset schema");
34060
+ Affirm_default(output, "Invalid consumer output");
34061
+ const context = `the FIXED output${consumerName ? ` of consumer "${consumerName}"` : ""}`;
34062
+ const declared = this.declarationErrors(dsSchema, positions, output, context);
34063
+ if (declared.length > 0) return { errors: declared, layouts: [] };
34064
+ try {
34065
+ if (!output.layouts)
34066
+ return { errors: [], layouts: [{ value: null, plan: this.compilePlan(dsSchema.columns, positions, output, context) }] };
34067
+ const layouts = this.compileLayouts(dsSchema, positions, output, consumerName);
34068
+ return { errors: [], layouts: layouts.order.map((value) => ({ value, plan: layouts.plans.get(value) })) };
34069
+ } catch (e7) {
34070
+ return { errors: [this.reason(e7)], layouts: [] };
34071
+ }
34072
+ };
34073
+ /** Every field's own declaration, over every field the output writes, before any offset is computed. */
34074
+ this.declarationErrors = (dsSchema, positions, output, context) => {
34075
+ const written = output.layouts ? [...new Set((output.layouts.order ?? []).flatMap((value) => output.layouts.values?.[value]?.fields ?? []))] : dsSchema.columns.map((column) => column.name);
34076
+ const errors = [];
34077
+ for (const name of written) {
34078
+ const column = dsSchema.columns.find((x2) => x2.name === name);
34079
+ if (column) errors.push(...this.fieldErrors(column, positions?.[name], output.defaults, context));
34080
+ }
34081
+ return errors;
34082
+ };
34083
+ /**
34084
+ * What an affirm said, as a sentence a report can start a line with: without the prefix it
34085
+ * carries for a log, and with its first letter upper-cased — the messages are written to be
34086
+ * embedded ("... of the FIXED output of consumer x ..."), so some begin with that context
34087
+ * rather than with a capital.
34088
+ */
34089
+ this.reason = (error) => {
34090
+ const prefix2 = "Affirm failed: ";
34091
+ const message = error instanceof Error ? error.message : String(error);
34092
+ const sentence = message.startsWith(prefix2) ? message.slice(prefix2.length) : message;
34093
+ return sentence.charAt(0).toUpperCase() + sentence.slice(1);
34094
+ };
34095
+ /**
34096
+ * The record layouts of a multi-layout FIXED output, compiled in `order` so the export writes
34097
+ * its blocks in the order the author declared.
34098
+ *
34099
+ * Each layout names a subset of the dataset's columns, in its own column order, so a field's
34100
+ * place on the written line has nothing to do with its place on the staged one — which is what
34101
+ * `IFixedWriteField.index` carries. The output's `defaults` stand under every layout's own.
34102
+ */
34103
+ this.compileLayouts = (dsSchema, positions, output, consumerName) => {
34104
+ Affirm_default(dsSchema, "Invalid dataset schema");
34105
+ Affirm_default(output?.layouts, "Invalid fixed-width layouts");
34106
+ const { discriminator, order, values } = output.layouts;
34107
+ const where = consumerName ? ` of consumer "${consumerName}"` : "";
34108
+ Affirm_default(discriminator, `The FIXED layouts${where} name no discriminator field`);
34109
+ Affirm_default.hasItems(order, `The FIXED layouts${where} declare no "order", which is the order their record blocks are written in`);
34110
+ Affirm_default.noDupes(order, `The FIXED layouts${where} name a discriminator value twice in "order"`);
34111
+ Affirm_default(
34112
+ order.length <= Constants_default.defaults.MAX_FIXED_LAYOUTS,
34113
+ `The FIXED layouts${where} declare ${order.length} record layouts, more than the ${Constants_default.defaults.MAX_FIXED_LAYOUTS} one output may write`
34114
+ );
34115
+ Affirm_default(
34116
+ output.recordLength === void 0 || output.recordLength === null,
34117
+ `The FIXED output${where} declares both "layouts" and a "recordLength". Each layout declares its own; the output has no single one.`
34118
+ );
34119
+ const index = dsSchema.columns.findIndex((column) => column.name === discriminator);
34120
+ Affirm_default(index >= 0, `The FIXED discriminator "${discriminator}"${where} is not one of the consumer's visible fields`);
34121
+ const plans = /* @__PURE__ */ new Map();
34122
+ for (const value of order)
34123
+ plans.set(value, this.layoutPlan(values?.[value], dsSchema, positions, output, value, consumerName));
34124
+ return { discriminator, index, order, plans };
34125
+ };
34126
+ /** One entry of the `layouts` map as a plan: its named fields, found on the staged line, in its order. */
34127
+ this.layoutPlan = (layout, dsSchema, positions, output, value, consumerName) => {
34128
+ const of = consumerName ? ` of consumer "${consumerName}"` : "";
34129
+ const where = `the FIXED layout "${value}"${of}`;
34130
+ Affirm_default(layout, `The FIXED "order"${of} names the discriminator value "${value}", which has no layout under "values"`);
34131
+ Affirm_default.hasItems(layout.fields, `${where} writes no field`);
34132
+ Affirm_default.noDupes(layout.fields, `${where} writes a field twice`);
34133
+ const selected = layout.fields.map((name) => {
34134
+ const index = dsSchema.columns.findIndex((column) => column.name === name);
34135
+ Affirm_default(index >= 0, `The field "${name}" of ${where} is not one of the consumer's visible fields`);
34136
+ return { column: dsSchema.columns[index], index };
34137
+ });
34138
+ return this.plan(selected, positions, this.mergeDefaults(output.defaults, layout.defaults), layout.recordLength, where);
34139
+ };
34140
+ /**
34141
+ * One record layout ready to write: every column's fill rules decided, its columns computed from
34142
+ * the widths before it, and the record length agreed with what the layout declares.
34143
+ */
34144
+ this.plan = (selected, positions, defaults, recordLength, context) => {
34145
+ Affirm_default.hasItems(selected, `${context} has no field to write`);
34146
+ Affirm_default(positions, "Invalid fixed-width field positions");
34147
+ const fields = [];
34148
+ let start = 1;
34149
+ for (const { column, index } of selected) {
34150
+ const field = this.resolveField(column, positions[column.name], defaults, start, context);
34151
+ fields.push({ ...field, index });
34152
+ start = field.end + 1;
34153
+ }
34154
+ const composed = start - 1;
34155
+ Affirm_default(
34156
+ recordLength === void 0 || recordLength === null || recordLength === composed,
34157
+ `${context} declares a recordLength of ${recordLength}, but its ${fields.length} field widths sum to ${composed}. A field, a filler or a width is missing.`
34158
+ );
34159
+ return { fields, recordLength: composed };
34160
+ };
34161
+ /**
34162
+ * The layout's per-type fill rules over the output's, type by type — so an output states a rule
34163
+ * once for the whole file and a layout only names where it differs.
34164
+ */
34165
+ this.mergeDefaults = (output, layout) => {
34166
+ if (!layout) return output;
34167
+ if (!output) return layout;
34168
+ const merged = { ...output };
34169
+ for (const type of Object.keys(layout))
34170
+ merged[type] = this.override(output[type] ?? {}, layout[type]);
34171
+ return merged;
34172
+ };
34173
+ /**
34174
+ * The layout a record is written with, by the value of the discriminator field on its staged
34175
+ * line. A value with no layout stops the run: it is a record the file has no block for, and
34176
+ * writing it under another layout would put its fields at the wrong columns.
34177
+ */
34178
+ this.planFor = (values, layouts) => {
34179
+ const value = values?.[layouts.index];
34180
+ const plan = value === null || value === void 0 ? null : layouts.plans.get(String(value));
34181
+ Affirm_default(plan, `No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`);
34182
+ return plan;
34183
+ };
34184
+ /**
34185
+ * Everything wrong with the way ONE field is declared, as messages rather than as a throw: no
34186
+ * `position` at all, a width that is not a whole number of columns, a fill character that is not
34187
+ * one character.
34188
+ *
34189
+ * Not the field's place on the line — that is only meaningful once every field before it stands
34190
+ * on its own, so `resolveField` checks it where it computes it. This is the one statement of the
34191
+ * rules: the export affirms them, and `review` collects them for every field of an output so
34192
+ * `compile` reports six missing widths in one run rather than one per run.
34193
+ */
34194
+ this.fieldErrors = (column, position, defaults, context) => {
34195
+ Affirm_default(column, "Invalid dataset column");
34196
+ const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
34197
+ if (!position)
34198
+ return [`Missing "position" on ${where}: every visible field of a FIXED output declares the width it occupies`];
34199
+ const errors = [];
34200
+ if (!Number.isInteger(position.width) || position.width < 1)
34201
+ errors.push(`"position.width" of ${where} must be a whole number of 1 or greater`);
34202
+ const fill2 = this.fillOf(column.type, position, defaults);
34203
+ if (fill2.pad.length !== 1)
34204
+ errors.push(`"position.pad" of ${where} must be a single character, not "${fill2.pad}"`);
34205
+ if (fill2.nullFill.length !== 1)
34206
+ errors.push(`"position.nullFill" of ${where} must be a single character, not "${fill2.nullFill}"`);
34207
+ return errors;
34208
+ };
34209
+ /**
34210
+ * One column with its fill rules resolved and its place on the line computed. The field's own
34211
+ * `position` wins over the output's `defaults` for its type, which wins over the type's own rule,
34212
+ * so a wide layout states a rule once and names only its exceptions.
34213
+ */
34214
+ this.resolveField = (column, position, defaults, start, context) => {
34215
+ const declaration = this.fieldErrors(column, position, defaults, context);
34216
+ Affirm_default(declaration.length === 0, declaration.join(" "));
34217
+ const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
34218
+ const fill2 = this.fillOf(column.type, position, defaults);
34219
+ const end = start + position.width - 1;
34220
+ Affirm_default(
34221
+ position.start === void 0 || position.start === null || position.start === start,
34222
+ `"position.start" of ${where} says column ${position.start}, but the widths before it place the field at column ${start}. A field or a filler is missing, or two are out of order.`
34223
+ );
34224
+ return {
34225
+ name: column.name,
34226
+ width: position.width,
34227
+ start,
34228
+ end,
34229
+ align: fill2.align,
34230
+ pad: fill2.pad,
34231
+ nullText: fill2.nullFill.repeat(position.width),
34232
+ overflow: fill2.overflow
34233
+ };
34234
+ };
34235
+ /** The three fill sources folded into one, nearest declaration first. */
34236
+ this.fillOf = (type, position, defaults) => {
34237
+ const base = TYPE_FILL[type] ?? TYPE_FILL.string;
34238
+ return this.override(this.override(base, defaults?.[type]), position);
34239
+ };
34240
+ /** `fill` with whatever `override` actually declares. An absent rule leaves the one below it standing. */
34241
+ this.override = (fill2, override) => {
34242
+ if (!override) return fill2;
34243
+ const merged = { ...fill2 };
34244
+ if (override.align) merged.align = override.align;
34245
+ if (override.pad) merged.pad = override.pad;
34246
+ if (override.nullFill) merged.nullFill = override.nullFill;
34247
+ if (override.overflow) merged.overflow = override.overflow;
34248
+ return merged;
34249
+ };
34250
+ /**
34251
+ * One record as its positional line, with no delimiter, no quoting and no terminator. The values
34252
+ * are the record's fields as the STAGED line holds them, each plan field reading the one it was
34253
+ * compiled against; a value the array does not reach is written as a null, which is what a
34254
+ * staged line does when its last fields carry nothing.
34255
+ */
34256
+ this.composeLine = (values, plan) => {
34257
+ Affirm_default(values, "Invalid record values");
34258
+ Affirm_default(plan, "Invalid fixed-width write plan");
34259
+ let line = "";
34260
+ for (const field of plan.fields)
34261
+ line += this.writeField(values[field.index], field);
34262
+ Affirm_default(line.length === plan.recordLength, `A fixed-width record composed to ${line.length} characters, not the declared recordLength of ${plan.recordLength}`);
34263
+ return line;
34264
+ };
34265
+ /**
34266
+ * One value as one field: its text padded out to the field's width on the side `align` names.
34267
+ *
34268
+ * A null is the full width in `nullFill`, and an EMPTY STRING is a null — the staged encoding
34269
+ * keeps the two apart and a positional line cannot. A value too long for its field stops the
34270
+ * run; see `overflowed`.
34271
+ */
34272
+ this.writeField = (value, field) => {
34273
+ if (value === null || value === void 0) return field.nullText;
34274
+ const text = typeof value === "string" ? value : String(value);
34275
+ if (text.length === 0) return field.nullText;
34276
+ if (text.length > field.width) return this.overflowed(text, field);
34277
+ return field.align === "right" ? text.padStart(field.width, field.pad) : text.padEnd(field.width, field.pad);
34278
+ };
34279
+ /**
34280
+ * A value longer than its field. It fails the run by default: a truncated account number is a
34281
+ * plausible-looking account number, and silent corruption in a feed nobody re-reads is the one
34282
+ * failure mode a positional writer must not have.
34283
+ *
34284
+ * Where the layout genuinely specifies a cut, the end that survives follows `align`, which is
34285
+ * what a COBOL MOVE does: a left-aligned value keeps its head, a right-aligned one keeps its
34286
+ * low-order tail.
34287
+ */
34288
+ this.overflowed = (text, field) => {
34289
+ Affirm_default(
34290
+ field.overflow === "truncate",
34291
+ `Value "${text}" is ${text.length} characters, longer than the ${field.width} of field "${field.name}" at columns ${field.start}-${field.end}. Widen the field, or set its "overflow" to "truncate" if the layout really specifies a cut.`
34292
+ );
34293
+ return field.align === "right" ? text.slice(text.length - field.width) : text.slice(0, field.width);
34294
+ };
34295
+ /**
34296
+ * What a record carries after its last field, BEFORE the newline the destination writes. Every
34297
+ * destination driver ends each transformed line with one `\n`, the last line included, so an
34298
+ * `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
34299
+ * CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
34300
+ *
34301
+ * A real positional feed picks one terminator and holds it: the NASCO POD files are CRLF on
34302
+ * every line including the last, and `lf` is the default everywhere else.
34303
+ */
34304
+ this.lineSuffix = (output) => {
34305
+ Affirm_default(output, "Invalid consumer output");
34306
+ return output.lineEnding === "crlf" ? "\r" : "";
34307
+ };
34308
+ }
34309
+ };
34310
+ FixedWidthWriter = new FixedWidthWriterClass();
34311
+ FixedWidthWriter_default = FixedWidthWriter;
34312
+ }
34313
+ });
34314
+
33944
34315
  // ../../node_modules/pend/index.js
33945
34316
  var require_pend = __commonJS({
33946
34317
  "../../node_modules/pend/index.js"(exports2, module2) {
@@ -33999,7 +34370,7 @@ var require_pend = __commonJS({
33999
34370
  // ../../node_modules/yauzl/fd-slicer.js
34000
34371
  var require_fd_slicer = __commonJS({
34001
34372
  "../../node_modules/yauzl/fd-slicer.js"(exports2) {
34002
- var fs32 = require("fs");
34373
+ var fs33 = require("fs");
34003
34374
  var util = require("util");
34004
34375
  var stream = require("stream");
34005
34376
  var Readable10 = stream.Readable;
@@ -34019,7 +34390,7 @@ var require_fd_slicer = __commonJS({
34019
34390
  FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
34020
34391
  var self2 = this;
34021
34392
  self2.pend.go(function(cb) {
34022
- fs32.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
34393
+ fs33.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
34023
34394
  cb();
34024
34395
  callback(err2, bytesRead, buffer2);
34025
34396
  });
@@ -34036,7 +34407,7 @@ var require_fd_slicer = __commonJS({
34036
34407
  self2.refCount -= 1;
34037
34408
  if (self2.refCount < 0) throw new Error("invalid unref");
34038
34409
  if (self2.refCount > 0) return;
34039
- fs32.close(self2.fd, onCloseDone);
34410
+ fs33.close(self2.fd, onCloseDone);
34040
34411
  function onCloseDone(err2) {
34041
34412
  if (err2) {
34042
34413
  self2.emit("error", err2);
@@ -34068,7 +34439,7 @@ var require_fd_slicer = __commonJS({
34068
34439
  }
34069
34440
  self2.context.pend.go(function(cb) {
34070
34441
  var buffer = Buffer.allocUnsafe(toRead);
34071
- fs32.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
34442
+ fs33.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
34072
34443
  if (err2) {
34073
34444
  self2.destroy(err2);
34074
34445
  } else if (bytesRead === 0) {
@@ -34428,7 +34799,7 @@ var require_crc32 = __commonJS({
34428
34799
  // ../../node_modules/yauzl/index.js
34429
34800
  var require_yauzl = __commonJS({
34430
34801
  "../../node_modules/yauzl/index.js"(exports2) {
34431
- var fs32 = require("fs");
34802
+ var fs33 = require("fs");
34432
34803
  var zlib3 = require("zlib");
34433
34804
  var fd_slicer = require_fd_slicer();
34434
34805
  var util = require("util");
@@ -34497,10 +34868,10 @@ var require_yauzl = __commonJS({
34497
34868
  if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
34498
34869
  if (options2.strictFileNames == null) options2.strictFileNames = false;
34499
34870
  if (callback == null) callback = defaultCallback;
34500
- fs32.open(path22, "r", function(err2, fd) {
34871
+ fs33.open(path22, "r", function(err2, fd) {
34501
34872
  if (err2) return callback(err2);
34502
34873
  fromFd(fd, options2, function(err3, zipfile) {
34503
- if (err3) fs32.close(fd, defaultCallback);
34874
+ if (err3) fs33.close(fd, defaultCallback);
34504
34875
  callback(err3, zipfile);
34505
34876
  });
34506
34877
  });
@@ -34517,7 +34888,7 @@ var require_yauzl = __commonJS({
34517
34888
  if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
34518
34889
  if (options2.strictFileNames == null) options2.strictFileNames = false;
34519
34890
  if (callback == null) callback = defaultCallback;
34520
- fs32.fstat(fd, function(err2, stats) {
34891
+ fs33.fstat(fd, function(err2, stats) {
34521
34892
  if (err2) return callback(err2);
34522
34893
  var reader = new fd_slicer.FdSlicer(fd);
34523
34894
  fromRandomAccessReader(reader, stats.size, options2, callback);
@@ -38598,6 +38969,7 @@ var init_src5 = __esm({
38598
38969
  init_CSVParser();
38599
38970
  init_DatasetCodec();
38600
38971
  init_FixedWidthParser();
38972
+ init_FixedWidthWriter();
38601
38973
  init_ParseCompression();
38602
38974
  init_ParseHelper();
38603
38975
  init_XLSParser();
@@ -46233,8 +46605,8 @@ var init_LocalSourceDriver = __esm({
46233
46605
  Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
46234
46606
  const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
46235
46607
  const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
46236
- const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
46237
- const results = await Promise.all(promises2);
46608
+ const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
46609
+ const results = await Promise.all(promises3);
46238
46610
  return results.flat();
46239
46611
  } else {
46240
46612
  return await this._get(request, "");
@@ -46254,8 +46626,8 @@ var init_LocalSourceDriver = __esm({
46254
46626
  Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
46255
46627
  const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
46256
46628
  const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
46257
- const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
46258
- const results = await Promise.all(promises2);
46629
+ const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
46630
+ const results = await Promise.all(promises3);
46259
46631
  return results.flat();
46260
46632
  } else {
46261
46633
  return await this._get(request, "");
@@ -46683,8 +47055,8 @@ var init_S3SourceDriver = __esm({
46683
47055
  if (fileKey.includes("%")) {
46684
47056
  const allFileKeys = await this.listFiles(fileKey);
46685
47057
  Affirm_default(allFileKeys.length < 50, `Pattern ${fileKey} of producer requested to S3 matches more than 50 files (${allFileKeys.length}), this is more than the S3 allowed limit. Please refine your pattern, remove some files or use a separate bucket.`);
46686
- const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
46687
- const results = await Promise.all(promises2);
47058
+ const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
47059
+ const results = await Promise.all(promises3);
46688
47060
  return results.flat();
46689
47061
  } else {
46690
47062
  return await this._get(request);
@@ -46697,8 +47069,8 @@ var init_S3SourceDriver = __esm({
46697
47069
  const { fileKey } = request;
46698
47070
  if (fileKey.includes("%")) {
46699
47071
  const allFileKeys = await this.listFiles(fileKey);
46700
- const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
46701
- const results = await Promise.all(promises2);
47072
+ const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
47073
+ const results = await Promise.all(promises3);
46702
47074
  return results.flat();
46703
47075
  } else {
46704
47076
  return await this._get(request);
@@ -56102,6 +56474,8 @@ var init_DatasetSchema = __esm({
56102
56474
  return "JSON";
56103
56475
  case "TXT":
56104
56476
  return "CSV";
56477
+ case "FIXED":
56478
+ return "CSV";
56105
56479
  case "CSV":
56106
56480
  return "CSV";
56107
56481
  default:
@@ -56181,6 +56555,112 @@ var init_DatasetSchema = __esm({
56181
56555
  }
56182
56556
  });
56183
56557
 
56558
+ // ../../packages/executors/src/FixedBlockOrder.ts
56559
+ var fs25, FixedBlockOrderClass, FixedBlockOrder, FixedBlockOrder_default;
56560
+ var init_FixedBlockOrder = __esm({
56561
+ "../../packages/executors/src/FixedBlockOrder.ts"() {
56562
+ fs25 = __toESM(require("fs"));
56563
+ init_src();
56564
+ init_src5();
56565
+ init_src2();
56566
+ FixedBlockOrderClass = class {
56567
+ constructor() {
56568
+ /** The file one block's records are collected in. Named by the value's place in `order`, not by
56569
+ * the value itself: a discriminator value is data and may hold anything a path may not. */
56570
+ this.blockPath = (sourcePath, position) => `${sourcePath}.block_${position}`;
56571
+ /**
56572
+ * Routes the staged dataset into one file per layout and concatenates them in `layouts.order`,
56573
+ * returning the path of the block-ordered dataset the export then transforms.
56574
+ */
56575
+ this.reorder = async (request) => {
56576
+ Affirm_default(request, "Invalid block order request");
56577
+ Affirm_default(request.sourcePath, "Missing dataset to order into blocks");
56578
+ Affirm_default(request.layouts, "Invalid fixed-width layouts");
56579
+ Affirm_default.hasItems(request.layouts.order, "The fixed-width layouts declare no block order");
56580
+ const { sourcePath, layouts, consumerName } = request;
56581
+ const orderedPath = `${sourcePath}.blocks`;
56582
+ const counts = await this._route(request);
56583
+ await fs25.promises.unlink(sourcePath);
56584
+ await this._concat(sourcePath, layouts, orderedPath);
56585
+ Logger_default.info(`Consumer "${consumerName}" writes its FIXED record blocks as ${layouts.order.map((value, position) => `${value} x${counts[position]}`).join(", ")}`);
56586
+ return orderedPath;
56587
+ };
56588
+ /**
56589
+ * The routing pass: every line read once, appended to the file of the block its discriminator
56590
+ * names. One stream per layout, all open at once — `MAX_FIXED_LAYOUTS` is what bounds them, so
56591
+ * there is no eviction to do and a block's file is only ever written by its own stream.
56592
+ *
56593
+ * Returns the records each block took, in `order`, which is what the log line reports.
56594
+ */
56595
+ this._route = async (request) => {
56596
+ const { sourcePath, layouts } = request;
56597
+ const streams = layouts.order.map((_3, position) => fs25.createWriteStream(this.blockPath(sourcePath, position)));
56598
+ const positions = new Map(layouts.order.map((value, position) => [value, position]));
56599
+ const counts = layouts.order.map(() => 0);
56600
+ const readStream = fs25.createReadStream(sourcePath);
56601
+ try {
56602
+ for await (const line of readLines(readStream)) {
56603
+ const position = this._positionOf(line, layouts, positions);
56604
+ counts[position]++;
56605
+ const stream = streams[position];
56606
+ if (!stream.write(line + "\n"))
56607
+ await new Promise((resolve) => stream.once("drain", resolve));
56608
+ }
56609
+ } finally {
56610
+ for (const stream of streams)
56611
+ await this._closeStream(stream);
56612
+ readStream.destroy();
56613
+ }
56614
+ return counts;
56615
+ };
56616
+ /** The block a staged line belongs to. A record no layout covers stops the run; see `planFor`. */
56617
+ this._positionOf = (line, layouts, positions) => {
56618
+ const value = DatasetCodec_default.decodeRow(line)[layouts.index];
56619
+ const position = value === null || value === void 0 ? void 0 : positions.get(String(value));
56620
+ Affirm_default(
56621
+ position !== void 0,
56622
+ `No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`
56623
+ );
56624
+ return position;
56625
+ };
56626
+ /** The blocks appended into one dataset, in `order`, each file removed as it is consumed. */
56627
+ this._concat = async (sourcePath, layouts, orderedPath) => {
56628
+ const writer = fs25.createWriteStream(orderedPath);
56629
+ try {
56630
+ for (let position = 0; position < layouts.order.length; position++) {
56631
+ const blockPath = this.blockPath(sourcePath, position);
56632
+ await this._append(blockPath, writer);
56633
+ await fs25.promises.unlink(blockPath);
56634
+ }
56635
+ } finally {
56636
+ await this._closeStream(writer);
56637
+ }
56638
+ };
56639
+ /** One block's bytes onto the end of the ordered dataset. Piped, so a block is never held in memory. */
56640
+ this._append = (blockPath, writer) => {
56641
+ return new Promise((resolve, reject) => {
56642
+ const reader = fs25.createReadStream(blockPath);
56643
+ reader.on("error", reject);
56644
+ reader.on("end", resolve);
56645
+ reader.pipe(writer, { end: false });
56646
+ });
56647
+ };
56648
+ /** Flushes and releases a stream's descriptor. Awaited: the read that follows must not race the flush. */
56649
+ this._closeStream = async (stream) => {
56650
+ if (!stream) return;
56651
+ stream.end();
56652
+ await new Promise((resolve, reject) => {
56653
+ stream.on("finish", resolve);
56654
+ stream.on("error", reject);
56655
+ });
56656
+ };
56657
+ }
56658
+ };
56659
+ FixedBlockOrder = new FixedBlockOrderClass();
56660
+ FixedBlockOrder_default = FixedBlockOrder;
56661
+ }
56662
+ });
56663
+
56184
56664
  // ../../packages/executors/src/OutputExecutor.ts
56185
56665
  var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
56186
56666
  var init_OutputExecutor = __esm({
@@ -56193,6 +56673,7 @@ var init_OutputExecutor = __esm({
56193
56673
  init_src5();
56194
56674
  init_src2();
56195
56675
  init_DatasetSchema();
56676
+ init_FixedBlockOrder();
56196
56677
  OutputExecutorClass = class {
56197
56678
  constructor() {
56198
56679
  /**
@@ -56229,7 +56710,7 @@ var init_OutputExecutor = __esm({
56229
56710
  for (let index = 0; index < visibleKeys.length; index++) {
56230
56711
  if (index > 0)
56231
56712
  row += delimiter;
56232
- row += DatasetCodec_default.encodeField(record[visibleKeys[index]]);
56713
+ row += DatasetCodec_default.encodeField(record[visibleKeys[index]], delimiter);
56233
56714
  }
56234
56715
  return row;
56235
56716
  };
@@ -56283,6 +56764,8 @@ var init_OutputExecutor = __esm({
56283
56764
  const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
56284
56765
  Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
56285
56766
  const header = this._composeHeader(output, dsSchema);
56767
+ const fixedLayouts = output.format === "FIXED" && output.layouts ? FixedWidthWriter_default.compileLayouts(dsSchema, this._fixedPositions(consumer), output, consumer.name) : null;
56768
+ const fixedLine = output.format === "FIXED" ? this._fixedLineWriter(consumer, dsSchema, output, fixedLayouts) : null;
56286
56769
  const results = [];
56287
56770
  for (const { path: startingPath, label } of splitFiles) {
56288
56771
  const destinationPath = this.getCompletedPath(destinationName, label);
@@ -56303,6 +56786,11 @@ var init_OutputExecutor = __esm({
56303
56786
  );
56304
56787
  break;
56305
56788
  }
56789
+ case "FIXED": {
56790
+ const fixedPath = fixedLayouts ? await FixedBlockOrder_default.reorder({ sourcePath: startingPath, layouts: fixedLayouts, consumerName: consumer.name }) : startingPath;
56791
+ results.push(await driver.transformAndMove(fixedPath, fixedLine, destinationPath));
56792
+ break;
56793
+ }
56306
56794
  case "PARQUET":
56307
56795
  results.push(await this._exportParquet({
56308
56796
  sourcePath: startingPath,
@@ -56318,6 +56806,60 @@ var init_OutputExecutor = __esm({
56318
56806
  }
56319
56807
  return this._unifingResult(results);
56320
56808
  };
56809
+ /**
56810
+ * A consumer's FIXED output resolved without running it: the layouts the export would compile,
56811
+ * and every problem with them. `compile` prints what it resolves and fails on what it reports,
56812
+ * and a run resolves it before reading a producer — so a width that does not add up is caught
56813
+ * where it was written rather than after the whole dataset has been built.
56814
+ *
56815
+ * Null for a consumer that writes anything else: there is no layout to resolve.
56816
+ */
56817
+ this.reviewFixedOutput = (consumer, dsSchema) => {
56818
+ Affirm_default(consumer, "Invalid consumer");
56819
+ const output = consumer.outputs?.[0];
56820
+ if (output?.format !== "FIXED") return null;
56821
+ const schema = dsSchema ?? DatasetSchema_default.compile(consumer, ConsumerManager_default.getExpandedFields(consumer));
56822
+ return FixedWidthWriter_default.review(schema, this._fixedPositions(consumer), output, consumer.name);
56823
+ };
56824
+ /**
56825
+ * The line function a FIXED export hands its destination driver: one staged record in, the
56826
+ * positional line it becomes out. The layout is resolved here, once, and closed over — so the
56827
+ * per-record cost is a decode of the staged line and a pad per field, with no config lookup.
56828
+ *
56829
+ * With `layouts` declared the record's own discriminator picks its plan, which is the only
56830
+ * per-record branch on the path; the records are already grouped into their blocks by then.
56831
+ *
56832
+ * No header is passed alongside it: a positional file has no room for one, and the field names
56833
+ * would not sit on the columns they describe.
56834
+ */
56835
+ this._fixedLineWriter = (consumer, dsSchema, output, layouts) => {
56836
+ const suffix = FixedWidthWriter_default.lineSuffix(output);
56837
+ if (layouts) {
56838
+ for (const value of layouts.order)
56839
+ Logger_default.info(`Consumer "${consumer.name}" writes its FIXED "${value}" record as ${layouts.plans.get(value).recordLength} characters over ${layouts.plans.get(value).fields.length} field(s)`);
56840
+ return (line) => {
56841
+ const values = DatasetCodec_default.decodeRow(line);
56842
+ return FixedWidthWriter_default.composeLine(values, FixedWidthWriter_default.planFor(values, layouts)) + suffix;
56843
+ };
56844
+ }
56845
+ const plan = FixedWidthWriter_default.compilePlan(dsSchema.columns, this._fixedPositions(consumer), output, `the FIXED output of consumer "${consumer.name}"`);
56846
+ Logger_default.info(`Consumer "${consumer.name}" writes a FIXED record of ${plan.recordLength} characters over ${plan.fields.length} field(s)`);
56847
+ return (line) => FixedWidthWriter_default.composeLine(DatasetCodec_default.decodeRow(line), plan) + suffix;
56848
+ };
56849
+ /**
56850
+ * Every visible field's `position` block, keyed by the name its column carries on the staged
56851
+ * line. Read from the EXPANDED fields rather than the consumer's own, so a glob key brings the
56852
+ * `position` it was declared with to each of the columns it stands for, and a field the dataset
56853
+ * has no `position` for is left absent for `compilePlan` to name.
56854
+ */
56855
+ this._fixedPositions = (consumer) => {
56856
+ const positions = {};
56857
+ for (const field of ConsumerManager_default.getExpandedFields(consumer)) {
56858
+ if (field.cField.hidden) continue;
56859
+ positions[field.finalKey] = field.cField.position;
56860
+ }
56861
+ return positions;
56862
+ };
56321
56863
  /**
56322
56864
  * Ships one split file as parquet. A parquet file is a columnar container whose footer can only
56323
56865
  * be written once the last row group is encoded, so there is no per-line mapping to hand a
@@ -56353,17 +56895,19 @@ var init_OutputExecutor = __esm({
56353
56895
  return resultList[0];
56354
56896
  };
56355
56897
  /**
56356
- * The header line of a positional export: the visible field names, in column order, quoted and
56357
- * delimited exactly as the rows under them — so the header parses with the same reader the data
56358
- * does. Null for a JSON/API output, where a record already carries its own field names, and for
56359
- * an output that turns the header off.
56898
+ * The header line of a positional export: the visible field names, in column order, written
56899
+ * through the SAME codec as the rows under them — so a name is quoted on the same rule a value
56900
+ * is, and the header parses with the reader the data does. Null for a JSON/API output, where a
56901
+ * record already carries its own field names, for a
56902
+ * FIXED one, where a name does not fit the column it describes and the receiver reads by
56903
+ * position anyway, and for an output that turns the header off.
56360
56904
  *
56361
56905
  * The names come from the dataset's own schema, so a `pivot` is described by the header.
56362
56906
  */
56363
56907
  this._composeHeader = (output, dsSchema) => {
56364
56908
  if (output.header === false) return null;
56365
56909
  if (output.format !== "CSV" && output.format !== "TXT") return null;
56366
- return CSVParser_default.stringifyRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
56910
+ return DatasetCodec_default.encodeRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
56367
56911
  };
56368
56912
  /** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
56369
56913
  this._getDelimiter = (output) => {
@@ -57010,10 +57554,10 @@ var init_ExecutorPerformance = __esm({
57010
57554
  });
57011
57555
 
57012
57556
  // ../../packages/executors/src/ExecutorWriter.ts
57013
- var fs27, ExecutorWriter, ExecutorWriter_default;
57557
+ var fs28, ExecutorWriter, ExecutorWriter_default;
57014
57558
  var init_ExecutorWriter = __esm({
57015
57559
  "../../packages/executors/src/ExecutorWriter.ts"() {
57016
- fs27 = __toESM(require("fs"));
57560
+ fs28 = __toESM(require("fs"));
57017
57561
  init_src();
57018
57562
  init_src3();
57019
57563
  init_src10();
@@ -57065,7 +57609,7 @@ var init_ExecutorWriter = __esm({
57065
57609
  await this._route(pass, dsSchema);
57066
57610
  if (pass.buckets.size === 0)
57067
57611
  return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
57068
- await fs27.promises.unlink(sourcePath);
57612
+ await fs28.promises.unlink(sourcePath);
57069
57613
  return this._listFiles(pass);
57070
57614
  };
57071
57615
  /**
@@ -57073,7 +57617,7 @@ var init_ExecutorWriter = __esm({
57073
57617
  */
57074
57618
  this._route = async (pass, dsSchema) => {
57075
57619
  const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
57076
- const readStream = fs27.createReadStream(pass.sourcePath);
57620
+ const readStream = fs28.createReadStream(pass.sourcePath);
57077
57621
  try {
57078
57622
  for await (const line of readLines(readStream)) {
57079
57623
  const bucket = this._bucketFor(line, pass, parseLine);
@@ -57177,7 +57721,7 @@ var init_ExecutorWriter = __esm({
57177
57721
  }
57178
57722
  if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
57179
57723
  await this._closeStream(pass.openStreams.shift());
57180
- bucket.stream = fs27.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
57724
+ bucket.stream = fs28.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
57181
57725
  pass.openStreams.push(bucket);
57182
57726
  return bucket.stream;
57183
57727
  };
@@ -57762,6 +58306,11 @@ var init_ExecutorOrchestrator = __esm({
57762
58306
  const start = performance.now();
57763
58307
  const executorResults = [];
57764
58308
  Logger_default.info(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length}${OrchestratorHelper_default.formatRequestOptions(options2)}) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
58309
+ const fixedReview = request.skipExport ? null : OutputExecutor_default.reviewFixedOutput(consumer, dsSchema);
58310
+ Affirm_default(
58311
+ !fixedReview || fixedReview.errors.length === 0,
58312
+ `Consumer "${consumer.name}" cannot write its FIXED output: ${fixedReview?.errors.join(" ")}`
58313
+ );
57765
58314
  const lookups = await LookupResolver_default.materialize(consumer, expandedFields, scope, request);
57766
58315
  const consumerInputs = await ConsumerInputResolver_default.materialize(consumer, scope, request);
57767
58316
  let counter = performance.now();