@forzalabs/remora 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/docs/Remora - Code Sets.md +268 -0
- package/docs/Remora - Consumers.md +2232 -0
- package/docs/Remora - Producers.md +1658 -0
- package/docs/Remora - Sources.md +1150 -0
- package/documentation/README.md +262 -130
- package/documentation/default_resources/mock_data.csv +2 -2
- package/index.js +2215 -1637
- package/json_schemas/consumer-schema.json +146 -1
- package/package.json +1 -1
- package/workers/ExecutorWorker.js +611 -62
|
@@ -8538,7 +8538,7 @@ var require_node2 = __commonJS({
|
|
|
8538
8538
|
var require_tail_file = __commonJS({
|
|
8539
8539
|
"../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
|
|
8540
8540
|
"use strict";
|
|
8541
|
-
var
|
|
8541
|
+
var fs33 = require("fs");
|
|
8542
8542
|
var { StringDecoder } = require("string_decoder");
|
|
8543
8543
|
var { Stream } = require_readable();
|
|
8544
8544
|
function noop() {
|
|
@@ -8559,7 +8559,7 @@ var require_tail_file = __commonJS({
|
|
|
8559
8559
|
stream.emit("end");
|
|
8560
8560
|
stream.emit("close");
|
|
8561
8561
|
};
|
|
8562
|
-
|
|
8562
|
+
fs33.open(options2.file, "a+", "0644", (err2, fd) => {
|
|
8563
8563
|
if (err2) {
|
|
8564
8564
|
if (!iter) {
|
|
8565
8565
|
stream.emit("error", err2);
|
|
@@ -8571,10 +8571,10 @@ var require_tail_file = __commonJS({
|
|
|
8571
8571
|
}
|
|
8572
8572
|
(function read() {
|
|
8573
8573
|
if (stream.destroyed) {
|
|
8574
|
-
|
|
8574
|
+
fs33.close(fd, noop);
|
|
8575
8575
|
return;
|
|
8576
8576
|
}
|
|
8577
|
-
return
|
|
8577
|
+
return fs33.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
|
|
8578
8578
|
if (error) {
|
|
8579
8579
|
if (!iter) {
|
|
8580
8580
|
stream.emit("error", error);
|
|
@@ -8633,7 +8633,7 @@ var require_tail_file = __commonJS({
|
|
|
8633
8633
|
var require_file = __commonJS({
|
|
8634
8634
|
"../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
|
|
8635
8635
|
"use strict";
|
|
8636
|
-
var
|
|
8636
|
+
var fs33 = require("fs");
|
|
8637
8637
|
var path22 = require("path");
|
|
8638
8638
|
var asyncSeries = require_series();
|
|
8639
8639
|
var zlib3 = require("zlib");
|
|
@@ -8838,7 +8838,7 @@ var require_file = __commonJS({
|
|
|
8838
8838
|
let buff = "";
|
|
8839
8839
|
let results = [];
|
|
8840
8840
|
let row = 0;
|
|
8841
|
-
const stream =
|
|
8841
|
+
const stream = fs33.createReadStream(file, {
|
|
8842
8842
|
encoding: "utf8"
|
|
8843
8843
|
});
|
|
8844
8844
|
stream.on("error", (err2) => {
|
|
@@ -8990,7 +8990,7 @@ var require_file = __commonJS({
|
|
|
8990
8990
|
stat(callback) {
|
|
8991
8991
|
const target = this._getFile();
|
|
8992
8992
|
const fullpath = path22.join(this.dirname, target);
|
|
8993
|
-
|
|
8993
|
+
fs33.stat(fullpath, (err2, stat) => {
|
|
8994
8994
|
if (err2 && err2.code === "ENOENT") {
|
|
8995
8995
|
debug("ENOENT\xA0ok", fullpath);
|
|
8996
8996
|
this.filename = target;
|
|
@@ -9095,7 +9095,7 @@ var require_file = __commonJS({
|
|
|
9095
9095
|
_createStream(source) {
|
|
9096
9096
|
const fullpath = path22.join(this.dirname, this.filename);
|
|
9097
9097
|
debug("create stream start", fullpath, this.options);
|
|
9098
|
-
const dest =
|
|
9098
|
+
const dest = fs33.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
|
|
9099
9099
|
debug("file open ok", fullpath);
|
|
9100
9100
|
this.emit("open", fullpath);
|
|
9101
9101
|
source.pipe(dest);
|
|
@@ -9174,7 +9174,7 @@ var require_file = __commonJS({
|
|
|
9174
9174
|
const isZipped = this.zippedArchive ? ".gz" : "";
|
|
9175
9175
|
const filePath = `${basename}${isOldest}${ext}${isZipped}`;
|
|
9176
9176
|
const target = path22.join(this.dirname, filePath);
|
|
9177
|
-
|
|
9177
|
+
fs33.unlink(target, callback);
|
|
9178
9178
|
}
|
|
9179
9179
|
/**
|
|
9180
9180
|
* Roll files forward based on integer, up to maxFiles. e.g. if base if
|
|
@@ -9197,17 +9197,17 @@ var require_file = __commonJS({
|
|
|
9197
9197
|
tasks.push(function(i6, cb) {
|
|
9198
9198
|
let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
|
|
9199
9199
|
const tmppath = path22.join(this.dirname, fileName);
|
|
9200
|
-
|
|
9200
|
+
fs33.exists(tmppath, (exists) => {
|
|
9201
9201
|
if (!exists) {
|
|
9202
9202
|
return cb(null);
|
|
9203
9203
|
}
|
|
9204
9204
|
fileName = `${basename}${i6}${ext}${isZipped}`;
|
|
9205
|
-
|
|
9205
|
+
fs33.rename(tmppath, path22.join(this.dirname, fileName), cb);
|
|
9206
9206
|
});
|
|
9207
9207
|
}.bind(this, x2));
|
|
9208
9208
|
}
|
|
9209
9209
|
asyncSeries(tasks, () => {
|
|
9210
|
-
|
|
9210
|
+
fs33.rename(
|
|
9211
9211
|
path22.join(this.dirname, `${basename}${ext}${isZipped}`),
|
|
9212
9212
|
path22.join(this.dirname, `${basename}1${ext}${isZipped}`),
|
|
9213
9213
|
callback
|
|
@@ -9223,22 +9223,22 @@ var require_file = __commonJS({
|
|
|
9223
9223
|
* @private
|
|
9224
9224
|
*/
|
|
9225
9225
|
_compressFile(src, dest, callback) {
|
|
9226
|
-
|
|
9226
|
+
fs33.access(src, fs33.F_OK, (err2) => {
|
|
9227
9227
|
if (err2) {
|
|
9228
9228
|
return callback();
|
|
9229
9229
|
}
|
|
9230
9230
|
var gzip = zlib3.createGzip();
|
|
9231
|
-
var inp =
|
|
9232
|
-
var out =
|
|
9231
|
+
var inp = fs33.createReadStream(src);
|
|
9232
|
+
var out = fs33.createWriteStream(dest);
|
|
9233
9233
|
out.on("finish", () => {
|
|
9234
|
-
|
|
9234
|
+
fs33.unlink(src, callback);
|
|
9235
9235
|
});
|
|
9236
9236
|
inp.pipe(gzip).pipe(out);
|
|
9237
9237
|
});
|
|
9238
9238
|
}
|
|
9239
9239
|
_createLogDirIfNotExist(dirPath) {
|
|
9240
|
-
if (!
|
|
9241
|
-
|
|
9240
|
+
if (!fs33.existsSync(dirPath)) {
|
|
9241
|
+
fs33.mkdirSync(dirPath, { recursive: true });
|
|
9242
9242
|
}
|
|
9243
9243
|
}
|
|
9244
9244
|
};
|
|
@@ -11809,7 +11809,7 @@ var CONSTANTS, Constants_default;
|
|
|
11809
11809
|
var init_Constants = __esm({
|
|
11810
11810
|
"../../packages/constants/src/Constants.ts"() {
|
|
11811
11811
|
CONSTANTS = {
|
|
11812
|
-
cliVersion: "2.
|
|
11812
|
+
cliVersion: "2.6.0",
|
|
11813
11813
|
backendVersion: 1,
|
|
11814
11814
|
backendPort: 5088,
|
|
11815
11815
|
workerVersion: 2,
|
|
@@ -11934,6 +11934,13 @@ var init_Constants = __esm({
|
|
|
11934
11934
|
* mode when its next line arrives — its byte counter lives in the bucket, not the stream.
|
|
11935
11935
|
*/
|
|
11936
11936
|
MAX_OPEN_SPLIT_STREAMS: 64,
|
|
11937
|
+
/**
|
|
11938
|
+
* Record layouts one FIXED output may write into a single file, which is also how many write
|
|
11939
|
+
* streams its block-ordering pass holds open. A copybook of this size is already unusual —
|
|
11940
|
+
* the NASCO POD feed writes five transaction types — and the bound stops a generated config
|
|
11941
|
+
* from opening a descriptor per distinct value of the wrong field.
|
|
11942
|
+
*/
|
|
11943
|
+
MAX_FIXED_LAYOUTS: 64,
|
|
11937
11944
|
/**
|
|
11938
11945
|
* Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
|
|
11939
11946
|
* export of it — `remora sample -s` and the preview's Data tab are both held to this.
|
|
@@ -13176,7 +13183,7 @@ var init_Validator = __esm({
|
|
|
13176
13183
|
errors.push(`Consumer "${consumer.name}" declares ${consumer.outputs.length} outputs (${consumer.outputs.map((x2) => x2.format).join(", ")}). A consumer writes exactly one output \u2014 only the first was ever written \u2014 so split it into one consumer per output.`);
|
|
13177
13184
|
for (const output of consumer.outputs) {
|
|
13178
13185
|
const format4 = output.format.toUpperCase();
|
|
13179
|
-
if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
|
|
13186
|
+
if (format4 === "CSV" || format4 === "FIXED" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
|
|
13180
13187
|
if (!output.exportDestination)
|
|
13181
13188
|
errors.push(`A static file output must have an export destination set (${format4})`);
|
|
13182
13189
|
else if (!Environment_default.getSource(output.exportDestination))
|
|
@@ -13186,6 +13193,7 @@ var init_Validator = __esm({
|
|
|
13186
13193
|
errors.push(...this.validateOmitNull(consumer));
|
|
13187
13194
|
errors.push(...this.validateHeader(consumer));
|
|
13188
13195
|
errors.push(...this.validateSplit(consumer));
|
|
13196
|
+
errors.push(...this.validateFixedOutput(consumer));
|
|
13189
13197
|
const dimensionFields = consumer.fields.filter((x2) => x2.key !== "*" && !x2.fixed && !x2.copyFrom && !x2.lineCount);
|
|
13190
13198
|
const dimensionKeys = dimensionFields.map((x2) => `${x2.from ?? "_default_"}::${x2.key}`);
|
|
13191
13199
|
const duplicateDimensionKeys = dimensionKeys.filter((k7, i6) => dimensionKeys.indexOf(k7) !== i6);
|
|
@@ -13254,7 +13262,7 @@ var init_Validator = __esm({
|
|
|
13254
13262
|
this.validateSplit = (consumer) => {
|
|
13255
13263
|
const errors = [];
|
|
13256
13264
|
const { split: split2 } = consumer;
|
|
13257
|
-
const positionalFormats = ["CSV", "TXT", "PARQUET"];
|
|
13265
|
+
const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
|
|
13258
13266
|
if (!Algo_default.hasVal(split2)) return errors;
|
|
13259
13267
|
if (Algo_default.hasVal(consumer.maximumFileSize))
|
|
13260
13268
|
errors.push(`Consumer "${consumer.name}" sets both "maximumFileSize" and "split". They are the same cap: move the value to "split": { "maxFileSizeGb": ${consumer.maximumFileSize} } and remove "maximumFileSize".`);
|
|
@@ -13285,7 +13293,7 @@ var init_Validator = __esm({
|
|
|
13285
13293
|
*/
|
|
13286
13294
|
this.validateOmitNull = (consumer) => {
|
|
13287
13295
|
const errors = [];
|
|
13288
|
-
const positionalFormats = ["CSV", "TXT", "PARQUET"];
|
|
13296
|
+
const positionalFormats = ["CSV", "FIXED", "TXT", "PARQUET"];
|
|
13289
13297
|
for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
|
|
13290
13298
|
const format4 = output.format.toUpperCase();
|
|
13291
13299
|
if (positionalFormats.includes(format4))
|
|
@@ -13305,8 +13313,47 @@ var init_Validator = __esm({
|
|
|
13305
13313
|
for (const output of (consumer.outputs ?? []).filter((x2) => Algo_default.hasVal(x2.header))) {
|
|
13306
13314
|
const format4 = output.format.toUpperCase();
|
|
13307
13315
|
if (!headerFormats.includes(format4))
|
|
13308
|
-
errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output:
|
|
13316
|
+
errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: ${this.headerlessReason(format4)}.`);
|
|
13317
|
+
}
|
|
13318
|
+
return errors;
|
|
13319
|
+
};
|
|
13320
|
+
/** Why the format in hand has no header line, as the clause that finishes the sentence above. */
|
|
13321
|
+
this.headerlessReason = (format4) => {
|
|
13322
|
+
if (format4 === "FIXED") return "a positional line has no room for one, and a field name would not sit on the columns it describes";
|
|
13323
|
+
if (format4 === "PARQUET") return `a ${format4} file carries its own schema`;
|
|
13324
|
+
return `a ${format4} record already carries its own field names`;
|
|
13325
|
+
};
|
|
13326
|
+
/**
|
|
13327
|
+
* Checks a consumer's FIXED output from its configuration alone: the properties only a
|
|
13328
|
+
* positional file has, the ones it cannot have, and whether anything downstream tries to read
|
|
13329
|
+
* one back. The layout itself — widths, columns and record lengths — is resolved against the
|
|
13330
|
+
* dataset's columns, which the producers define, so it is checked by `compile` and again before
|
|
13331
|
+
* a run starts rather than here.
|
|
13332
|
+
*
|
|
13333
|
+
* A property declared on the wrong format is rejected rather than ignored, the way a `position`
|
|
13334
|
+
* on a non-FIXED producer is: silently dropping a rule the author wrote is how a file ships
|
|
13335
|
+
* looking almost right.
|
|
13336
|
+
*/
|
|
13337
|
+
this.validateFixedOutput = (consumer) => {
|
|
13338
|
+
Affirm_default(consumer, "Invalid consumer");
|
|
13339
|
+
const errors = [];
|
|
13340
|
+
const output = consumer.outputs?.[0];
|
|
13341
|
+
if (!output) return errors;
|
|
13342
|
+
const format4 = output.format?.toUpperCase();
|
|
13343
|
+
const positioned = (consumer.fields ?? []).filter((x2) => x2.position).map((x2) => `"${x2.alias ?? x2.key}"`);
|
|
13344
|
+
if (format4 !== "FIXED") {
|
|
13345
|
+
const fixedOnly = ["recordLength", "defaults", "lineEnding", "layouts"];
|
|
13346
|
+
const declared = fixedOnly.filter((name) => Algo_default.hasVal(output[name]));
|
|
13347
|
+
if (declared.length > 0)
|
|
13348
|
+
errors.push(`The ${format4} output of consumer "${consumer.name}" declares ${declared.map((x2) => `"${x2}"`).join(", ")}, which only ${declared.length > 1 ? "apply" : "applies"} to a FIXED output, where ${declared.length > 1 ? "they are" : "it is"} ignored.`);
|
|
13349
|
+
if (positioned.length > 0)
|
|
13350
|
+
errors.push(`Consumer "${consumer.name}" declares "position" on field(s) ${positioned.join(", ")} but its output format is not "FIXED", where positions are ignored.`);
|
|
13351
|
+
return errors;
|
|
13309
13352
|
}
|
|
13353
|
+
if (Algo_default.hasVal(output.delimiter))
|
|
13354
|
+
errors.push(`The FIXED output of consumer "${consumer.name}" declares a "delimiter". A positional line has none: its fields are found by the columns they occupy.`);
|
|
13355
|
+
if ((consumer.fields ?? []).some((x2) => x2.key === "*"))
|
|
13356
|
+
errors.push(`Consumer "${consumer.name}" selects fields with the wildcard key "*", which a FIXED output can't write: every field of a positional line declares its own "width", so the field list has to be explicit.`);
|
|
13310
13357
|
return errors;
|
|
13311
13358
|
};
|
|
13312
13359
|
/**
|
|
@@ -14889,7 +14936,7 @@ A consumer takes the data from one or more producers and changes it's shape to t
|
|
|
14889
14936
|
# RULES
|
|
14890
14937
|
- If a field is not needed, do not add it e.g.
|
|
14891
14938
|
- Only import a producer once
|
|
14892
|
-
- Awlays include this exact property as the first -> $schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
|
|
14939
|
+
- Awlays include this exact property as the first -> "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
|
|
14893
14940
|
- Use "API" as the only valid output format.
|
|
14894
14941
|
- The "from" must contain only the name of the producer
|
|
14895
14942
|
|
|
@@ -33698,21 +33745,29 @@ var init_CSVParser = __esm({
|
|
|
33698
33745
|
});
|
|
33699
33746
|
|
|
33700
33747
|
// ../../packages/parsing/src/DatasetCodec.ts
|
|
33701
|
-
var NULL_FIELD, NEEDS_ESCAPE, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
|
|
33748
|
+
var NULL_FIELD, EMPTY_FIELD, NEEDS_ESCAPE, QUOTE_CODE, QUOTE_TESTS, DatasetCodecClass, DatasetCodec, DatasetCodec_default;
|
|
33702
33749
|
var init_DatasetCodec = __esm({
|
|
33703
33750
|
"../../packages/parsing/src/DatasetCodec.ts"() {
|
|
33704
33751
|
init_src();
|
|
33705
33752
|
NULL_FIELD = "";
|
|
33753
|
+
EMPTY_FIELD = '""';
|
|
33706
33754
|
NEEDS_ESCAPE = /["\n\r]/;
|
|
33755
|
+
QUOTE_CODE = 34;
|
|
33756
|
+
QUOTE_TESTS = /* @__PURE__ */ new Map([[",", /^\s|\s$|[",\n\r]/]]);
|
|
33707
33757
|
DatasetCodecClass = class {
|
|
33708
33758
|
constructor() {
|
|
33709
33759
|
/**
|
|
33710
|
-
* One value as an internal positional field
|
|
33711
|
-
*
|
|
33760
|
+
* One value as an internal positional field. Bare where it can be, quoted where it must be -
|
|
33761
|
+
* around the delimiter, a quote, a line break or an edge space - or the unquoted null marker
|
|
33762
|
+
* when there is no value. The delimiter is a parameter rather than a comma because `redelimit`
|
|
33763
|
+
* re-encodes a staged line with the export's own: a tab is safe bare in the staged file and is
|
|
33764
|
+
* not once that file is re-emitted tab-delimited.
|
|
33712
33765
|
*/
|
|
33713
|
-
this.encodeField = (value) => {
|
|
33766
|
+
this.encodeField = (value, delimiter = ",") => {
|
|
33714
33767
|
if (value === null || value === void 0) return NULL_FIELD;
|
|
33715
33768
|
const text = typeof value === "string" ? value : String(value);
|
|
33769
|
+
if (text === "") return EMPTY_FIELD;
|
|
33770
|
+
if (!this._quoteTest(delimiter).test(text)) return text;
|
|
33716
33771
|
if (!NEEDS_ESCAPE.test(text)) return `"${text}"`;
|
|
33717
33772
|
return `"${text.replaceAll('"', '""').replaceAll("\r", "\\r").replaceAll("\n", "\\n")}"`;
|
|
33718
33773
|
};
|
|
@@ -33724,7 +33779,7 @@ var init_DatasetCodec = __esm({
|
|
|
33724
33779
|
for (let index = 0; index < values.length; index++) {
|
|
33725
33780
|
if (index > 0)
|
|
33726
33781
|
row += delimiter;
|
|
33727
|
-
row += this.encodeField(values[index]);
|
|
33782
|
+
row += this.encodeField(values[index], delimiter);
|
|
33728
33783
|
}
|
|
33729
33784
|
return row;
|
|
33730
33785
|
};
|
|
@@ -33744,26 +33799,26 @@ var init_DatasetCodec = __esm({
|
|
|
33744
33799
|
let index = 0;
|
|
33745
33800
|
while (index < length) {
|
|
33746
33801
|
const char = line.charCodeAt(index);
|
|
33747
|
-
if (char ===
|
|
33802
|
+
if (char === QUOTE_CODE) {
|
|
33748
33803
|
if (!inQuotes) {
|
|
33749
33804
|
inQuotes = true;
|
|
33750
33805
|
hasQuotes = true;
|
|
33751
33806
|
fieldStart = index + 1;
|
|
33752
|
-
} else if (line.charCodeAt(index + 1) ===
|
|
33807
|
+
} else if (line.charCodeAt(index + 1) === QUOTE_CODE) {
|
|
33753
33808
|
index++;
|
|
33754
33809
|
} else {
|
|
33755
33810
|
inQuotes = false;
|
|
33756
33811
|
fieldEnd = index;
|
|
33757
33812
|
}
|
|
33758
33813
|
} else if (char === delimiterCode && !inQuotes) {
|
|
33759
|
-
fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
|
|
33814
|
+
fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : index, hasQuotes));
|
|
33760
33815
|
fieldStart = index + 1;
|
|
33761
33816
|
fieldEnd = 0;
|
|
33762
33817
|
hasQuotes = false;
|
|
33763
33818
|
}
|
|
33764
33819
|
index++;
|
|
33765
33820
|
}
|
|
33766
|
-
fields.push(this._field(line, fieldStart, fieldEnd, hasQuotes));
|
|
33821
|
+
fields.push(this._field(line, fieldStart, hasQuotes ? fieldEnd : length, hasQuotes));
|
|
33767
33822
|
return fields;
|
|
33768
33823
|
};
|
|
33769
33824
|
/**
|
|
@@ -33774,9 +33829,24 @@ var init_DatasetCodec = __esm({
|
|
|
33774
33829
|
Affirm_default(delimiter, "Invalid delimiter");
|
|
33775
33830
|
return this.encodeRow(this.decodeRow(line), delimiter);
|
|
33776
33831
|
};
|
|
33777
|
-
/**
|
|
33832
|
+
/**
|
|
33833
|
+
* The quoting test for one delimiter, built on first use. The four characters escaped into the
|
|
33834
|
+
* class are the ones a character class reads as syntax; anything else stands for itself there.
|
|
33835
|
+
*/
|
|
33836
|
+
this._quoteTest = (delimiter) => {
|
|
33837
|
+
let test = QUOTE_TESTS.get(delimiter);
|
|
33838
|
+
if (!test) {
|
|
33839
|
+
test = new RegExp(`^\\s|\\s$|["\\n\\r${delimiter.replace(/[\\\]^-]/g, "\\$&")}]`);
|
|
33840
|
+
QUOTE_TESTS.set(delimiter, test);
|
|
33841
|
+
}
|
|
33842
|
+
return test;
|
|
33843
|
+
};
|
|
33844
|
+
/**
|
|
33845
|
+
* One field's text, or null for the unquoted empty marker. A bare token carries its value as it
|
|
33846
|
+
* stands - nothing bare can hold a quote, so there is nothing to unescape in it.
|
|
33847
|
+
*/
|
|
33778
33848
|
this._field = (line, start, end, hasQuotes) => {
|
|
33779
|
-
if (!hasQuotes) return null;
|
|
33849
|
+
if (!hasQuotes) return start === end ? null : line.slice(start, end);
|
|
33780
33850
|
const raw = line.slice(start, end);
|
|
33781
33851
|
return raw.indexOf('"') === -1 ? raw : raw.replaceAll('""', '"');
|
|
33782
33852
|
};
|
|
@@ -33941,6 +34011,307 @@ var init_FixedWidthParser = __esm({
|
|
|
33941
34011
|
}
|
|
33942
34012
|
});
|
|
33943
34013
|
|
|
34014
|
+
// ../../packages/parsing/src/FixedWidthWriter.ts
|
|
34015
|
+
var BASE_FILL, TYPE_FILL, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
|
|
34016
|
+
var init_FixedWidthWriter = __esm({
|
|
34017
|
+
"../../packages/parsing/src/FixedWidthWriter.ts"() {
|
|
34018
|
+
init_src();
|
|
34019
|
+
init_src3();
|
|
34020
|
+
BASE_FILL = { align: "left", pad: " ", nullFill: " ", overflow: "fail" };
|
|
34021
|
+
TYPE_FILL = {
|
|
34022
|
+
string: BASE_FILL,
|
|
34023
|
+
number: { ...BASE_FILL, align: "right", pad: "0" },
|
|
34024
|
+
datetime: BASE_FILL,
|
|
34025
|
+
boolean: BASE_FILL
|
|
34026
|
+
};
|
|
34027
|
+
FixedWidthWriterClass = class {
|
|
34028
|
+
constructor() {
|
|
34029
|
+
/**
|
|
34030
|
+
* Resolves one record layout for writing: every column's fill rules decided, its columns
|
|
34031
|
+
* computed, and the record length agreed with what the output declares.
|
|
34032
|
+
*
|
|
34033
|
+
* Called once per layout when an export starts, never per record. `columns` is the line as the
|
|
34034
|
+
* dataset schema has it — line order, visible fields only — and `positions` carries each one's
|
|
34035
|
+
* `position` block, keyed by the same name the column has.
|
|
34036
|
+
*
|
|
34037
|
+
* `context` is the phrase that names this layout in an error, e.g. `the FIXED output of consumer
|
|
34038
|
+
* "x"`, so a message reads as one sentence wherever the layout came from.
|
|
34039
|
+
*/
|
|
34040
|
+
this.compilePlan = (columns, positions, output, context = "the FIXED output") => {
|
|
34041
|
+
Affirm_default(output, "Invalid consumer output");
|
|
34042
|
+
const selected = columns?.map((column, index) => ({ column, index }));
|
|
34043
|
+
return this.plan(selected, positions, output.defaults, output.recordLength, context);
|
|
34044
|
+
};
|
|
34045
|
+
/**
|
|
34046
|
+
* A FIXED output resolved for INSPECTION rather than for writing: every problem with it, and the
|
|
34047
|
+
* layouts it would write. What `compile` reports and prints.
|
|
34048
|
+
*
|
|
34049
|
+
* The rules are not restated here. The layouts are resolved by the same code the export runs,
|
|
34050
|
+
* and what that throws is reported rather than raised — so a layout `compile` accepts is a
|
|
34051
|
+
* layout the export can write. What review adds is that every field's own declaration is
|
|
34052
|
+
* checked first, across the whole output: an author with six fields missing a width sees six
|
|
34053
|
+
* lines from one `compile`, not one line from each of six.
|
|
34054
|
+
*
|
|
34055
|
+
* A field named by a layout that the dataset has no column for is left to the resolution below,
|
|
34056
|
+
* which names it as the layout's error rather than as a field with no width.
|
|
34057
|
+
*/
|
|
34058
|
+
this.review = (dsSchema, positions, output, consumerName) => {
|
|
34059
|
+
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
34060
|
+
Affirm_default(output, "Invalid consumer output");
|
|
34061
|
+
const context = `the FIXED output${consumerName ? ` of consumer "${consumerName}"` : ""}`;
|
|
34062
|
+
const declared = this.declarationErrors(dsSchema, positions, output, context);
|
|
34063
|
+
if (declared.length > 0) return { errors: declared, layouts: [] };
|
|
34064
|
+
try {
|
|
34065
|
+
if (!output.layouts)
|
|
34066
|
+
return { errors: [], layouts: [{ value: null, plan: this.compilePlan(dsSchema.columns, positions, output, context) }] };
|
|
34067
|
+
const layouts = this.compileLayouts(dsSchema, positions, output, consumerName);
|
|
34068
|
+
return { errors: [], layouts: layouts.order.map((value) => ({ value, plan: layouts.plans.get(value) })) };
|
|
34069
|
+
} catch (e7) {
|
|
34070
|
+
return { errors: [this.reason(e7)], layouts: [] };
|
|
34071
|
+
}
|
|
34072
|
+
};
|
|
34073
|
+
/** Every field's own declaration, over every field the output writes, before any offset is computed. */
|
|
34074
|
+
this.declarationErrors = (dsSchema, positions, output, context) => {
|
|
34075
|
+
const written = output.layouts ? [...new Set((output.layouts.order ?? []).flatMap((value) => output.layouts.values?.[value]?.fields ?? []))] : dsSchema.columns.map((column) => column.name);
|
|
34076
|
+
const errors = [];
|
|
34077
|
+
for (const name of written) {
|
|
34078
|
+
const column = dsSchema.columns.find((x2) => x2.name === name);
|
|
34079
|
+
if (column) errors.push(...this.fieldErrors(column, positions?.[name], output.defaults, context));
|
|
34080
|
+
}
|
|
34081
|
+
return errors;
|
|
34082
|
+
};
|
|
34083
|
+
/**
|
|
34084
|
+
* What an affirm said, as a sentence a report can start a line with: without the prefix it
|
|
34085
|
+
* carries for a log, and with its first letter upper-cased — the messages are written to be
|
|
34086
|
+
* embedded ("... of the FIXED output of consumer x ..."), so some begin with that context
|
|
34087
|
+
* rather than with a capital.
|
|
34088
|
+
*/
|
|
34089
|
+
this.reason = (error) => {
|
|
34090
|
+
const prefix2 = "Affirm failed: ";
|
|
34091
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
34092
|
+
const sentence = message.startsWith(prefix2) ? message.slice(prefix2.length) : message;
|
|
34093
|
+
return sentence.charAt(0).toUpperCase() + sentence.slice(1);
|
|
34094
|
+
};
|
|
34095
|
+
/**
|
|
34096
|
+
* The record layouts of a multi-layout FIXED output, compiled in `order` so the export writes
|
|
34097
|
+
* its blocks in the order the author declared.
|
|
34098
|
+
*
|
|
34099
|
+
* Each layout names a subset of the dataset's columns, in its own column order, so a field's
|
|
34100
|
+
* place on the written line has nothing to do with its place on the staged one — which is what
|
|
34101
|
+
* `IFixedWriteField.index` carries. The output's `defaults` stand under every layout's own.
|
|
34102
|
+
*/
|
|
34103
|
+
this.compileLayouts = (dsSchema, positions, output, consumerName) => {
|
|
34104
|
+
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
34105
|
+
Affirm_default(output?.layouts, "Invalid fixed-width layouts");
|
|
34106
|
+
const { discriminator, order, values } = output.layouts;
|
|
34107
|
+
const where = consumerName ? ` of consumer "${consumerName}"` : "";
|
|
34108
|
+
Affirm_default(discriminator, `The FIXED layouts${where} name no discriminator field`);
|
|
34109
|
+
Affirm_default.hasItems(order, `The FIXED layouts${where} declare no "order", which is the order their record blocks are written in`);
|
|
34110
|
+
Affirm_default.noDupes(order, `The FIXED layouts${where} name a discriminator value twice in "order"`);
|
|
34111
|
+
Affirm_default(
|
|
34112
|
+
order.length <= Constants_default.defaults.MAX_FIXED_LAYOUTS,
|
|
34113
|
+
`The FIXED layouts${where} declare ${order.length} record layouts, more than the ${Constants_default.defaults.MAX_FIXED_LAYOUTS} one output may write`
|
|
34114
|
+
);
|
|
34115
|
+
Affirm_default(
|
|
34116
|
+
output.recordLength === void 0 || output.recordLength === null,
|
|
34117
|
+
`The FIXED output${where} declares both "layouts" and a "recordLength". Each layout declares its own; the output has no single one.`
|
|
34118
|
+
);
|
|
34119
|
+
const index = dsSchema.columns.findIndex((column) => column.name === discriminator);
|
|
34120
|
+
Affirm_default(index >= 0, `The FIXED discriminator "${discriminator}"${where} is not one of the consumer's visible fields`);
|
|
34121
|
+
const plans = /* @__PURE__ */ new Map();
|
|
34122
|
+
for (const value of order)
|
|
34123
|
+
plans.set(value, this.layoutPlan(values?.[value], dsSchema, positions, output, value, consumerName));
|
|
34124
|
+
return { discriminator, index, order, plans };
|
|
34125
|
+
};
|
|
34126
|
+
/** One entry of the `layouts` map as a plan: its named fields, found on the staged line, in its order. */
|
|
34127
|
+
this.layoutPlan = (layout, dsSchema, positions, output, value, consumerName) => {
|
|
34128
|
+
const of = consumerName ? ` of consumer "${consumerName}"` : "";
|
|
34129
|
+
const where = `the FIXED layout "${value}"${of}`;
|
|
34130
|
+
Affirm_default(layout, `The FIXED "order"${of} names the discriminator value "${value}", which has no layout under "values"`);
|
|
34131
|
+
Affirm_default.hasItems(layout.fields, `${where} writes no field`);
|
|
34132
|
+
Affirm_default.noDupes(layout.fields, `${where} writes a field twice`);
|
|
34133
|
+
const selected = layout.fields.map((name) => {
|
|
34134
|
+
const index = dsSchema.columns.findIndex((column) => column.name === name);
|
|
34135
|
+
Affirm_default(index >= 0, `The field "${name}" of ${where} is not one of the consumer's visible fields`);
|
|
34136
|
+
return { column: dsSchema.columns[index], index };
|
|
34137
|
+
});
|
|
34138
|
+
return this.plan(selected, positions, this.mergeDefaults(output.defaults, layout.defaults), layout.recordLength, where);
|
|
34139
|
+
};
|
|
34140
|
+
/**
|
|
34141
|
+
* One record layout ready to write: every column's fill rules decided, its columns computed from
|
|
34142
|
+
* the widths before it, and the record length agreed with what the layout declares.
|
|
34143
|
+
*/
|
|
34144
|
+
this.plan = (selected, positions, defaults, recordLength, context) => {
|
|
34145
|
+
Affirm_default.hasItems(selected, `${context} has no field to write`);
|
|
34146
|
+
Affirm_default(positions, "Invalid fixed-width field positions");
|
|
34147
|
+
const fields = [];
|
|
34148
|
+
let start = 1;
|
|
34149
|
+
for (const { column, index } of selected) {
|
|
34150
|
+
const field = this.resolveField(column, positions[column.name], defaults, start, context);
|
|
34151
|
+
fields.push({ ...field, index });
|
|
34152
|
+
start = field.end + 1;
|
|
34153
|
+
}
|
|
34154
|
+
const composed = start - 1;
|
|
34155
|
+
Affirm_default(
|
|
34156
|
+
recordLength === void 0 || recordLength === null || recordLength === composed,
|
|
34157
|
+
`${context} declares a recordLength of ${recordLength}, but its ${fields.length} field widths sum to ${composed}. A field, a filler or a width is missing.`
|
|
34158
|
+
);
|
|
34159
|
+
return { fields, recordLength: composed };
|
|
34160
|
+
};
|
|
34161
|
+
/**
|
|
34162
|
+
* The layout's per-type fill rules over the output's, type by type — so an output states a rule
|
|
34163
|
+
* once for the whole file and a layout only names where it differs.
|
|
34164
|
+
*/
|
|
34165
|
+
this.mergeDefaults = (output, layout) => {
|
|
34166
|
+
if (!layout) return output;
|
|
34167
|
+
if (!output) return layout;
|
|
34168
|
+
const merged = { ...output };
|
|
34169
|
+
for (const type of Object.keys(layout))
|
|
34170
|
+
merged[type] = this.override(output[type] ?? {}, layout[type]);
|
|
34171
|
+
return merged;
|
|
34172
|
+
};
|
|
34173
|
+
/**
|
|
34174
|
+
* The layout a record is written with, by the value of the discriminator field on its staged
|
|
34175
|
+
* line. A value with no layout stops the run: it is a record the file has no block for, and
|
|
34176
|
+
* writing it under another layout would put its fields at the wrong columns.
|
|
34177
|
+
*/
|
|
34178
|
+
this.planFor = (values, layouts) => {
|
|
34179
|
+
const value = values?.[layouts.index];
|
|
34180
|
+
const plan = value === null || value === void 0 ? null : layouts.plans.get(String(value));
|
|
34181
|
+
Affirm_default(plan, `No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`);
|
|
34182
|
+
return plan;
|
|
34183
|
+
};
|
|
34184
|
+
/**
|
|
34185
|
+
* Everything wrong with the way ONE field is declared, as messages rather than as a throw: no
|
|
34186
|
+
* `position` at all, a width that is not a whole number of columns, a fill character that is not
|
|
34187
|
+
* one character.
|
|
34188
|
+
*
|
|
34189
|
+
* Not the field's place on the line — that is only meaningful once every field before it stands
|
|
34190
|
+
* on its own, so `resolveField` checks it where it computes it. This is the one statement of the
|
|
34191
|
+
* rules: the export affirms them, and `review` collects them for every field of an output so
|
|
34192
|
+
* `compile` reports six missing widths in one run rather than one per run.
|
|
34193
|
+
*/
|
|
34194
|
+
this.fieldErrors = (column, position, defaults, context) => {
|
|
34195
|
+
Affirm_default(column, "Invalid dataset column");
|
|
34196
|
+
const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
|
|
34197
|
+
if (!position)
|
|
34198
|
+
return [`Missing "position" on ${where}: every visible field of a FIXED output declares the width it occupies`];
|
|
34199
|
+
const errors = [];
|
|
34200
|
+
if (!Number.isInteger(position.width) || position.width < 1)
|
|
34201
|
+
errors.push(`"position.width" of ${where} must be a whole number of 1 or greater`);
|
|
34202
|
+
const fill2 = this.fillOf(column.type, position, defaults);
|
|
34203
|
+
if (fill2.pad.length !== 1)
|
|
34204
|
+
errors.push(`"position.pad" of ${where} must be a single character, not "${fill2.pad}"`);
|
|
34205
|
+
if (fill2.nullFill.length !== 1)
|
|
34206
|
+
errors.push(`"position.nullFill" of ${where} must be a single character, not "${fill2.nullFill}"`);
|
|
34207
|
+
return errors;
|
|
34208
|
+
};
|
|
34209
|
+
/**
|
|
34210
|
+
* One column with its fill rules resolved and its place on the line computed. The field's own
|
|
34211
|
+
* `position` wins over the output's `defaults` for its type, which wins over the type's own rule,
|
|
34212
|
+
* so a wide layout states a rule once and names only its exceptions.
|
|
34213
|
+
*/
|
|
34214
|
+
this.resolveField = (column, position, defaults, start, context) => {
|
|
34215
|
+
const declaration = this.fieldErrors(column, position, defaults, context);
|
|
34216
|
+
Affirm_default(declaration.length === 0, declaration.join(" "));
|
|
34217
|
+
const where = `field "${column.name}"${context ? ` of ${context}` : ""}`;
|
|
34218
|
+
const fill2 = this.fillOf(column.type, position, defaults);
|
|
34219
|
+
const end = start + position.width - 1;
|
|
34220
|
+
Affirm_default(
|
|
34221
|
+
position.start === void 0 || position.start === null || position.start === start,
|
|
34222
|
+
`"position.start" of ${where} says column ${position.start}, but the widths before it place the field at column ${start}. A field or a filler is missing, or two are out of order.`
|
|
34223
|
+
);
|
|
34224
|
+
return {
|
|
34225
|
+
name: column.name,
|
|
34226
|
+
width: position.width,
|
|
34227
|
+
start,
|
|
34228
|
+
end,
|
|
34229
|
+
align: fill2.align,
|
|
34230
|
+
pad: fill2.pad,
|
|
34231
|
+
nullText: fill2.nullFill.repeat(position.width),
|
|
34232
|
+
overflow: fill2.overflow
|
|
34233
|
+
};
|
|
34234
|
+
};
|
|
34235
|
+
/** The three fill sources folded into one, nearest declaration first. */
|
|
34236
|
+
this.fillOf = (type, position, defaults) => {
|
|
34237
|
+
const base = TYPE_FILL[type] ?? TYPE_FILL.string;
|
|
34238
|
+
return this.override(this.override(base, defaults?.[type]), position);
|
|
34239
|
+
};
|
|
34240
|
+
/** `fill` with whatever `override` actually declares. An absent rule leaves the one below it standing. */
|
|
34241
|
+
this.override = (fill2, override) => {
|
|
34242
|
+
if (!override) return fill2;
|
|
34243
|
+
const merged = { ...fill2 };
|
|
34244
|
+
if (override.align) merged.align = override.align;
|
|
34245
|
+
if (override.pad) merged.pad = override.pad;
|
|
34246
|
+
if (override.nullFill) merged.nullFill = override.nullFill;
|
|
34247
|
+
if (override.overflow) merged.overflow = override.overflow;
|
|
34248
|
+
return merged;
|
|
34249
|
+
};
|
|
34250
|
+
/**
|
|
34251
|
+
* One record as its positional line, with no delimiter, no quoting and no terminator. The values
|
|
34252
|
+
* are the record's fields as the STAGED line holds them, each plan field reading the one it was
|
|
34253
|
+
* compiled against; a value the array does not reach is written as a null, which is what a
|
|
34254
|
+
* staged line does when its last fields carry nothing.
|
|
34255
|
+
*/
|
|
34256
|
+
this.composeLine = (values, plan) => {
|
|
34257
|
+
Affirm_default(values, "Invalid record values");
|
|
34258
|
+
Affirm_default(plan, "Invalid fixed-width write plan");
|
|
34259
|
+
let line = "";
|
|
34260
|
+
for (const field of plan.fields)
|
|
34261
|
+
line += this.writeField(values[field.index], field);
|
|
34262
|
+
Affirm_default(line.length === plan.recordLength, `A fixed-width record composed to ${line.length} characters, not the declared recordLength of ${plan.recordLength}`);
|
|
34263
|
+
return line;
|
|
34264
|
+
};
|
|
34265
|
+
/**
|
|
34266
|
+
* One value as one field: its text padded out to the field's width on the side `align` names.
|
|
34267
|
+
*
|
|
34268
|
+
* A null is the full width in `nullFill`, and an EMPTY STRING is a null — the staged encoding
|
|
34269
|
+
* keeps the two apart and a positional line cannot. A value too long for its field stops the
|
|
34270
|
+
* run; see `overflowed`.
|
|
34271
|
+
*/
|
|
34272
|
+
this.writeField = (value, field) => {
|
|
34273
|
+
if (value === null || value === void 0) return field.nullText;
|
|
34274
|
+
const text = typeof value === "string" ? value : String(value);
|
|
34275
|
+
if (text.length === 0) return field.nullText;
|
|
34276
|
+
if (text.length > field.width) return this.overflowed(text, field);
|
|
34277
|
+
return field.align === "right" ? text.padStart(field.width, field.pad) : text.padEnd(field.width, field.pad);
|
|
34278
|
+
};
|
|
34279
|
+
/**
|
|
34280
|
+
* A value longer than its field. It fails the run by default: a truncated account number is a
|
|
34281
|
+
* plausible-looking account number, and silent corruption in a feed nobody re-reads is the one
|
|
34282
|
+
* failure mode a positional writer must not have.
|
|
34283
|
+
*
|
|
34284
|
+
* Where the layout genuinely specifies a cut, the end that survives follows `align`, which is
|
|
34285
|
+
* what a COBOL MOVE does: a left-aligned value keeps its head, a right-aligned one keeps its
|
|
34286
|
+
* low-order tail.
|
|
34287
|
+
*/
|
|
34288
|
+
this.overflowed = (text, field) => {
|
|
34289
|
+
Affirm_default(
|
|
34290
|
+
field.overflow === "truncate",
|
|
34291
|
+
`Value "${text}" is ${text.length} characters, longer than the ${field.width} of field "${field.name}" at columns ${field.start}-${field.end}. Widen the field, or set its "overflow" to "truncate" if the layout really specifies a cut.`
|
|
34292
|
+
);
|
|
34293
|
+
return field.align === "right" ? text.slice(text.length - field.width) : text.slice(0, field.width);
|
|
34294
|
+
};
|
|
34295
|
+
/**
|
|
34296
|
+
* What a record carries after its last field, BEFORE the newline the destination writes. Every
|
|
34297
|
+
* destination driver ends each transformed line with one `\n`, the last line included, so an
|
|
34298
|
+
* `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
|
|
34299
|
+
* CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
|
|
34300
|
+
*
|
|
34301
|
+
* A real positional feed picks one terminator and holds it: the NASCO POD files are CRLF on
|
|
34302
|
+
* every line including the last, and `lf` is the default everywhere else.
|
|
34303
|
+
*/
|
|
34304
|
+
this.lineSuffix = (output) => {
|
|
34305
|
+
Affirm_default(output, "Invalid consumer output");
|
|
34306
|
+
return output.lineEnding === "crlf" ? "\r" : "";
|
|
34307
|
+
};
|
|
34308
|
+
}
|
|
34309
|
+
};
|
|
34310
|
+
FixedWidthWriter = new FixedWidthWriterClass();
|
|
34311
|
+
FixedWidthWriter_default = FixedWidthWriter;
|
|
34312
|
+
}
|
|
34313
|
+
});
|
|
34314
|
+
|
|
33944
34315
|
// ../../node_modules/pend/index.js
|
|
33945
34316
|
var require_pend = __commonJS({
|
|
33946
34317
|
"../../node_modules/pend/index.js"(exports2, module2) {
|
|
@@ -33999,7 +34370,7 @@ var require_pend = __commonJS({
|
|
|
33999
34370
|
// ../../node_modules/yauzl/fd-slicer.js
|
|
34000
34371
|
var require_fd_slicer = __commonJS({
|
|
34001
34372
|
"../../node_modules/yauzl/fd-slicer.js"(exports2) {
|
|
34002
|
-
var
|
|
34373
|
+
var fs33 = require("fs");
|
|
34003
34374
|
var util = require("util");
|
|
34004
34375
|
var stream = require("stream");
|
|
34005
34376
|
var Readable10 = stream.Readable;
|
|
@@ -34019,7 +34390,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34019
34390
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
34020
34391
|
var self2 = this;
|
|
34021
34392
|
self2.pend.go(function(cb) {
|
|
34022
|
-
|
|
34393
|
+
fs33.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
|
|
34023
34394
|
cb();
|
|
34024
34395
|
callback(err2, bytesRead, buffer2);
|
|
34025
34396
|
});
|
|
@@ -34036,7 +34407,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34036
34407
|
self2.refCount -= 1;
|
|
34037
34408
|
if (self2.refCount < 0) throw new Error("invalid unref");
|
|
34038
34409
|
if (self2.refCount > 0) return;
|
|
34039
|
-
|
|
34410
|
+
fs33.close(self2.fd, onCloseDone);
|
|
34040
34411
|
function onCloseDone(err2) {
|
|
34041
34412
|
if (err2) {
|
|
34042
34413
|
self2.emit("error", err2);
|
|
@@ -34068,7 +34439,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34068
34439
|
}
|
|
34069
34440
|
self2.context.pend.go(function(cb) {
|
|
34070
34441
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
34071
|
-
|
|
34442
|
+
fs33.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
|
|
34072
34443
|
if (err2) {
|
|
34073
34444
|
self2.destroy(err2);
|
|
34074
34445
|
} else if (bytesRead === 0) {
|
|
@@ -34428,7 +34799,7 @@ var require_crc32 = __commonJS({
|
|
|
34428
34799
|
// ../../node_modules/yauzl/index.js
|
|
34429
34800
|
var require_yauzl = __commonJS({
|
|
34430
34801
|
"../../node_modules/yauzl/index.js"(exports2) {
|
|
34431
|
-
var
|
|
34802
|
+
var fs33 = require("fs");
|
|
34432
34803
|
var zlib3 = require("zlib");
|
|
34433
34804
|
var fd_slicer = require_fd_slicer();
|
|
34434
34805
|
var util = require("util");
|
|
@@ -34497,10 +34868,10 @@ var require_yauzl = __commonJS({
|
|
|
34497
34868
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34498
34869
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34499
34870
|
if (callback == null) callback = defaultCallback;
|
|
34500
|
-
|
|
34871
|
+
fs33.open(path22, "r", function(err2, fd) {
|
|
34501
34872
|
if (err2) return callback(err2);
|
|
34502
34873
|
fromFd(fd, options2, function(err3, zipfile) {
|
|
34503
|
-
if (err3)
|
|
34874
|
+
if (err3) fs33.close(fd, defaultCallback);
|
|
34504
34875
|
callback(err3, zipfile);
|
|
34505
34876
|
});
|
|
34506
34877
|
});
|
|
@@ -34517,7 +34888,7 @@ var require_yauzl = __commonJS({
|
|
|
34517
34888
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34518
34889
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34519
34890
|
if (callback == null) callback = defaultCallback;
|
|
34520
|
-
|
|
34891
|
+
fs33.fstat(fd, function(err2, stats) {
|
|
34521
34892
|
if (err2) return callback(err2);
|
|
34522
34893
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
34523
34894
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -38598,6 +38969,7 @@ var init_src5 = __esm({
|
|
|
38598
38969
|
init_CSVParser();
|
|
38599
38970
|
init_DatasetCodec();
|
|
38600
38971
|
init_FixedWidthParser();
|
|
38972
|
+
init_FixedWidthWriter();
|
|
38601
38973
|
init_ParseCompression();
|
|
38602
38974
|
init_ParseHelper();
|
|
38603
38975
|
init_XLSParser();
|
|
@@ -46233,8 +46605,8 @@ var init_LocalSourceDriver = __esm({
|
|
|
46233
46605
|
Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
|
|
46234
46606
|
const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
|
|
46235
46607
|
const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
|
|
46236
|
-
const
|
|
46237
|
-
const results = await Promise.all(
|
|
46608
|
+
const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
|
|
46609
|
+
const results = await Promise.all(promises3);
|
|
46238
46610
|
return results.flat();
|
|
46239
46611
|
} else {
|
|
46240
46612
|
return await this._get(request, "");
|
|
@@ -46254,8 +46626,8 @@ var init_LocalSourceDriver = __esm({
|
|
|
46254
46626
|
Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
|
|
46255
46627
|
const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
|
|
46256
46628
|
const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
|
|
46257
|
-
const
|
|
46258
|
-
const results = await Promise.all(
|
|
46629
|
+
const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
|
|
46630
|
+
const results = await Promise.all(promises3);
|
|
46259
46631
|
return results.flat();
|
|
46260
46632
|
} else {
|
|
46261
46633
|
return await this._get(request, "");
|
|
@@ -46683,8 +47055,8 @@ var init_S3SourceDriver = __esm({
|
|
|
46683
47055
|
if (fileKey.includes("%")) {
|
|
46684
47056
|
const allFileKeys = await this.listFiles(fileKey);
|
|
46685
47057
|
Affirm_default(allFileKeys.length < 50, `Pattern ${fileKey} of producer requested to S3 matches more than 50 files (${allFileKeys.length}), this is more than the S3 allowed limit. Please refine your pattern, remove some files or use a separate bucket.`);
|
|
46686
|
-
const
|
|
46687
|
-
const results = await Promise.all(
|
|
47058
|
+
const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
|
|
47059
|
+
const results = await Promise.all(promises3);
|
|
46688
47060
|
return results.flat();
|
|
46689
47061
|
} else {
|
|
46690
47062
|
return await this._get(request);
|
|
@@ -46697,8 +47069,8 @@ var init_S3SourceDriver = __esm({
|
|
|
46697
47069
|
const { fileKey } = request;
|
|
46698
47070
|
if (fileKey.includes("%")) {
|
|
46699
47071
|
const allFileKeys = await this.listFiles(fileKey);
|
|
46700
|
-
const
|
|
46701
|
-
const results = await Promise.all(
|
|
47072
|
+
const promises3 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, i6));
|
|
47073
|
+
const results = await Promise.all(promises3);
|
|
46702
47074
|
return results.flat();
|
|
46703
47075
|
} else {
|
|
46704
47076
|
return await this._get(request);
|
|
@@ -56102,6 +56474,8 @@ var init_DatasetSchema = __esm({
|
|
|
56102
56474
|
return "JSON";
|
|
56103
56475
|
case "TXT":
|
|
56104
56476
|
return "CSV";
|
|
56477
|
+
case "FIXED":
|
|
56478
|
+
return "CSV";
|
|
56105
56479
|
case "CSV":
|
|
56106
56480
|
return "CSV";
|
|
56107
56481
|
default:
|
|
@@ -56181,6 +56555,112 @@ var init_DatasetSchema = __esm({
|
|
|
56181
56555
|
}
|
|
56182
56556
|
});
|
|
56183
56557
|
|
|
56558
|
+
// ../../packages/executors/src/FixedBlockOrder.ts
|
|
56559
|
+
var fs25, FixedBlockOrderClass, FixedBlockOrder, FixedBlockOrder_default;
|
|
56560
|
+
var init_FixedBlockOrder = __esm({
|
|
56561
|
+
"../../packages/executors/src/FixedBlockOrder.ts"() {
|
|
56562
|
+
fs25 = __toESM(require("fs"));
|
|
56563
|
+
init_src();
|
|
56564
|
+
init_src5();
|
|
56565
|
+
init_src2();
|
|
56566
|
+
FixedBlockOrderClass = class {
|
|
56567
|
+
constructor() {
|
|
56568
|
+
/** The file one block's records are collected in. Named by the value's place in `order`, not by
|
|
56569
|
+
* the value itself: a discriminator value is data and may hold anything a path may not. */
|
|
56570
|
+
this.blockPath = (sourcePath, position) => `${sourcePath}.block_${position}`;
|
|
56571
|
+
/**
|
|
56572
|
+
* Routes the staged dataset into one file per layout and concatenates them in `layouts.order`,
|
|
56573
|
+
* returning the path of the block-ordered dataset the export then transforms.
|
|
56574
|
+
*/
|
|
56575
|
+
this.reorder = async (request) => {
|
|
56576
|
+
Affirm_default(request, "Invalid block order request");
|
|
56577
|
+
Affirm_default(request.sourcePath, "Missing dataset to order into blocks");
|
|
56578
|
+
Affirm_default(request.layouts, "Invalid fixed-width layouts");
|
|
56579
|
+
Affirm_default.hasItems(request.layouts.order, "The fixed-width layouts declare no block order");
|
|
56580
|
+
const { sourcePath, layouts, consumerName } = request;
|
|
56581
|
+
const orderedPath = `${sourcePath}.blocks`;
|
|
56582
|
+
const counts = await this._route(request);
|
|
56583
|
+
await fs25.promises.unlink(sourcePath);
|
|
56584
|
+
await this._concat(sourcePath, layouts, orderedPath);
|
|
56585
|
+
Logger_default.info(`Consumer "${consumerName}" writes its FIXED record blocks as ${layouts.order.map((value, position) => `${value} x${counts[position]}`).join(", ")}`);
|
|
56586
|
+
return orderedPath;
|
|
56587
|
+
};
|
|
56588
|
+
/**
|
|
56589
|
+
* The routing pass: every line read once, appended to the file of the block its discriminator
|
|
56590
|
+
* names. One stream per layout, all open at once — `MAX_FIXED_LAYOUTS` is what bounds them, so
|
|
56591
|
+
* there is no eviction to do and a block's file is only ever written by its own stream.
|
|
56592
|
+
*
|
|
56593
|
+
* Returns the records each block took, in `order`, which is what the log line reports.
|
|
56594
|
+
*/
|
|
56595
|
+
this._route = async (request) => {
|
|
56596
|
+
const { sourcePath, layouts } = request;
|
|
56597
|
+
const streams = layouts.order.map((_3, position) => fs25.createWriteStream(this.blockPath(sourcePath, position)));
|
|
56598
|
+
const positions = new Map(layouts.order.map((value, position) => [value, position]));
|
|
56599
|
+
const counts = layouts.order.map(() => 0);
|
|
56600
|
+
const readStream = fs25.createReadStream(sourcePath);
|
|
56601
|
+
try {
|
|
56602
|
+
for await (const line of readLines(readStream)) {
|
|
56603
|
+
const position = this._positionOf(line, layouts, positions);
|
|
56604
|
+
counts[position]++;
|
|
56605
|
+
const stream = streams[position];
|
|
56606
|
+
if (!stream.write(line + "\n"))
|
|
56607
|
+
await new Promise((resolve) => stream.once("drain", resolve));
|
|
56608
|
+
}
|
|
56609
|
+
} finally {
|
|
56610
|
+
for (const stream of streams)
|
|
56611
|
+
await this._closeStream(stream);
|
|
56612
|
+
readStream.destroy();
|
|
56613
|
+
}
|
|
56614
|
+
return counts;
|
|
56615
|
+
};
|
|
56616
|
+
/** The block a staged line belongs to. A record no layout covers stops the run; see `planFor`. */
|
|
56617
|
+
this._positionOf = (line, layouts, positions) => {
|
|
56618
|
+
const value = DatasetCodec_default.decodeRow(line)[layouts.index];
|
|
56619
|
+
const position = value === null || value === void 0 ? void 0 : positions.get(String(value));
|
|
56620
|
+
Affirm_default(
|
|
56621
|
+
position !== void 0,
|
|
56622
|
+
`No FIXED layout for the value ${value === null || value === void 0 ? "null" : `"${value}"`} of the discriminator field "${layouts.discriminator}". The layouts cover ${layouts.order.map((x2) => `"${x2}"`).join(", ")}.`
|
|
56623
|
+
);
|
|
56624
|
+
return position;
|
|
56625
|
+
};
|
|
56626
|
+
/** The blocks appended into one dataset, in `order`, each file removed as it is consumed. */
|
|
56627
|
+
this._concat = async (sourcePath, layouts, orderedPath) => {
|
|
56628
|
+
const writer = fs25.createWriteStream(orderedPath);
|
|
56629
|
+
try {
|
|
56630
|
+
for (let position = 0; position < layouts.order.length; position++) {
|
|
56631
|
+
const blockPath = this.blockPath(sourcePath, position);
|
|
56632
|
+
await this._append(blockPath, writer);
|
|
56633
|
+
await fs25.promises.unlink(blockPath);
|
|
56634
|
+
}
|
|
56635
|
+
} finally {
|
|
56636
|
+
await this._closeStream(writer);
|
|
56637
|
+
}
|
|
56638
|
+
};
|
|
56639
|
+
/** One block's bytes onto the end of the ordered dataset. Piped, so a block is never held in memory. */
|
|
56640
|
+
this._append = (blockPath, writer) => {
|
|
56641
|
+
return new Promise((resolve, reject) => {
|
|
56642
|
+
const reader = fs25.createReadStream(blockPath);
|
|
56643
|
+
reader.on("error", reject);
|
|
56644
|
+
reader.on("end", resolve);
|
|
56645
|
+
reader.pipe(writer, { end: false });
|
|
56646
|
+
});
|
|
56647
|
+
};
|
|
56648
|
+
/** Flushes and releases a stream's descriptor. Awaited: the read that follows must not race the flush. */
|
|
56649
|
+
this._closeStream = async (stream) => {
|
|
56650
|
+
if (!stream) return;
|
|
56651
|
+
stream.end();
|
|
56652
|
+
await new Promise((resolve, reject) => {
|
|
56653
|
+
stream.on("finish", resolve);
|
|
56654
|
+
stream.on("error", reject);
|
|
56655
|
+
});
|
|
56656
|
+
};
|
|
56657
|
+
}
|
|
56658
|
+
};
|
|
56659
|
+
FixedBlockOrder = new FixedBlockOrderClass();
|
|
56660
|
+
FixedBlockOrder_default = FixedBlockOrder;
|
|
56661
|
+
}
|
|
56662
|
+
});
|
|
56663
|
+
|
|
56184
56664
|
// ../../packages/executors/src/OutputExecutor.ts
|
|
56185
56665
|
var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
|
|
56186
56666
|
var init_OutputExecutor = __esm({
|
|
@@ -56193,6 +56673,7 @@ var init_OutputExecutor = __esm({
|
|
|
56193
56673
|
init_src5();
|
|
56194
56674
|
init_src2();
|
|
56195
56675
|
init_DatasetSchema();
|
|
56676
|
+
init_FixedBlockOrder();
|
|
56196
56677
|
OutputExecutorClass = class {
|
|
56197
56678
|
constructor() {
|
|
56198
56679
|
/**
|
|
@@ -56229,7 +56710,7 @@ var init_OutputExecutor = __esm({
|
|
|
56229
56710
|
for (let index = 0; index < visibleKeys.length; index++) {
|
|
56230
56711
|
if (index > 0)
|
|
56231
56712
|
row += delimiter;
|
|
56232
|
-
row += DatasetCodec_default.encodeField(record[visibleKeys[index]]);
|
|
56713
|
+
row += DatasetCodec_default.encodeField(record[visibleKeys[index]], delimiter);
|
|
56233
56714
|
}
|
|
56234
56715
|
return row;
|
|
56235
56716
|
};
|
|
@@ -56283,6 +56764,8 @@ var init_OutputExecutor = __esm({
|
|
|
56283
56764
|
const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
|
|
56284
56765
|
Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
|
|
56285
56766
|
const header = this._composeHeader(output, dsSchema);
|
|
56767
|
+
const fixedLayouts = output.format === "FIXED" && output.layouts ? FixedWidthWriter_default.compileLayouts(dsSchema, this._fixedPositions(consumer), output, consumer.name) : null;
|
|
56768
|
+
const fixedLine = output.format === "FIXED" ? this._fixedLineWriter(consumer, dsSchema, output, fixedLayouts) : null;
|
|
56286
56769
|
const results = [];
|
|
56287
56770
|
for (const { path: startingPath, label } of splitFiles) {
|
|
56288
56771
|
const destinationPath = this.getCompletedPath(destinationName, label);
|
|
@@ -56303,6 +56786,11 @@ var init_OutputExecutor = __esm({
|
|
|
56303
56786
|
);
|
|
56304
56787
|
break;
|
|
56305
56788
|
}
|
|
56789
|
+
case "FIXED": {
|
|
56790
|
+
const fixedPath = fixedLayouts ? await FixedBlockOrder_default.reorder({ sourcePath: startingPath, layouts: fixedLayouts, consumerName: consumer.name }) : startingPath;
|
|
56791
|
+
results.push(await driver.transformAndMove(fixedPath, fixedLine, destinationPath));
|
|
56792
|
+
break;
|
|
56793
|
+
}
|
|
56306
56794
|
case "PARQUET":
|
|
56307
56795
|
results.push(await this._exportParquet({
|
|
56308
56796
|
sourcePath: startingPath,
|
|
@@ -56318,6 +56806,60 @@ var init_OutputExecutor = __esm({
|
|
|
56318
56806
|
}
|
|
56319
56807
|
return this._unifingResult(results);
|
|
56320
56808
|
};
|
|
56809
|
+
/**
|
|
56810
|
+
* A consumer's FIXED output resolved without running it: the layouts the export would compile,
|
|
56811
|
+
* and every problem with them. `compile` prints what it resolves and fails on what it reports,
|
|
56812
|
+
* and a run resolves it before reading a producer — so a width that does not add up is caught
|
|
56813
|
+
* where it was written rather than after the whole dataset has been built.
|
|
56814
|
+
*
|
|
56815
|
+
* Null for a consumer that writes anything else: there is no layout to resolve.
|
|
56816
|
+
*/
|
|
56817
|
+
this.reviewFixedOutput = (consumer, dsSchema) => {
|
|
56818
|
+
Affirm_default(consumer, "Invalid consumer");
|
|
56819
|
+
const output = consumer.outputs?.[0];
|
|
56820
|
+
if (output?.format !== "FIXED") return null;
|
|
56821
|
+
const schema = dsSchema ?? DatasetSchema_default.compile(consumer, ConsumerManager_default.getExpandedFields(consumer));
|
|
56822
|
+
return FixedWidthWriter_default.review(schema, this._fixedPositions(consumer), output, consumer.name);
|
|
56823
|
+
};
|
|
56824
|
+
/**
|
|
56825
|
+
* The line function a FIXED export hands its destination driver: one staged record in, the
|
|
56826
|
+
* positional line it becomes out. The layout is resolved here, once, and closed over — so the
|
|
56827
|
+
* per-record cost is a decode of the staged line and a pad per field, with no config lookup.
|
|
56828
|
+
*
|
|
56829
|
+
* With `layouts` declared the record's own discriminator picks its plan, which is the only
|
|
56830
|
+
* per-record branch on the path; the records are already grouped into their blocks by then.
|
|
56831
|
+
*
|
|
56832
|
+
* No header is passed alongside it: a positional file has no room for one, and the field names
|
|
56833
|
+
* would not sit on the columns they describe.
|
|
56834
|
+
*/
|
|
56835
|
+
this._fixedLineWriter = (consumer, dsSchema, output, layouts) => {
|
|
56836
|
+
const suffix = FixedWidthWriter_default.lineSuffix(output);
|
|
56837
|
+
if (layouts) {
|
|
56838
|
+
for (const value of layouts.order)
|
|
56839
|
+
Logger_default.info(`Consumer "${consumer.name}" writes its FIXED "${value}" record as ${layouts.plans.get(value).recordLength} characters over ${layouts.plans.get(value).fields.length} field(s)`);
|
|
56840
|
+
return (line) => {
|
|
56841
|
+
const values = DatasetCodec_default.decodeRow(line);
|
|
56842
|
+
return FixedWidthWriter_default.composeLine(values, FixedWidthWriter_default.planFor(values, layouts)) + suffix;
|
|
56843
|
+
};
|
|
56844
|
+
}
|
|
56845
|
+
const plan = FixedWidthWriter_default.compilePlan(dsSchema.columns, this._fixedPositions(consumer), output, `the FIXED output of consumer "${consumer.name}"`);
|
|
56846
|
+
Logger_default.info(`Consumer "${consumer.name}" writes a FIXED record of ${plan.recordLength} characters over ${plan.fields.length} field(s)`);
|
|
56847
|
+
return (line) => FixedWidthWriter_default.composeLine(DatasetCodec_default.decodeRow(line), plan) + suffix;
|
|
56848
|
+
};
|
|
56849
|
+
/**
|
|
56850
|
+
* Every visible field's `position` block, keyed by the name its column carries on the staged
|
|
56851
|
+
* line. Read from the EXPANDED fields rather than the consumer's own, so a glob key brings the
|
|
56852
|
+
* `position` it was declared with to each of the columns it stands for, and a field the dataset
|
|
56853
|
+
* has no `position` for is left absent for `compilePlan` to name.
|
|
56854
|
+
*/
|
|
56855
|
+
this._fixedPositions = (consumer) => {
|
|
56856
|
+
const positions = {};
|
|
56857
|
+
for (const field of ConsumerManager_default.getExpandedFields(consumer)) {
|
|
56858
|
+
if (field.cField.hidden) continue;
|
|
56859
|
+
positions[field.finalKey] = field.cField.position;
|
|
56860
|
+
}
|
|
56861
|
+
return positions;
|
|
56862
|
+
};
|
|
56321
56863
|
/**
|
|
56322
56864
|
* Ships one split file as parquet. A parquet file is a columnar container whose footer can only
|
|
56323
56865
|
* be written once the last row group is encoded, so there is no per-line mapping to hand a
|
|
@@ -56353,17 +56895,19 @@ var init_OutputExecutor = __esm({
|
|
|
56353
56895
|
return resultList[0];
|
|
56354
56896
|
};
|
|
56355
56897
|
/**
|
|
56356
|
-
* The header line of a positional export: the visible field names, in column order,
|
|
56357
|
-
*
|
|
56358
|
-
* does. Null for a JSON/API output, where a
|
|
56359
|
-
*
|
|
56898
|
+
* The header line of a positional export: the visible field names, in column order, written
|
|
56899
|
+
* through the SAME codec as the rows under them — so a name is quoted on the same rule a value
|
|
56900
|
+
* is, and the header parses with the reader the data does. Null for a JSON/API output, where a
|
|
56901
|
+
* record already carries its own field names, for a
|
|
56902
|
+
* FIXED one, where a name does not fit the column it describes and the receiver reads by
|
|
56903
|
+
* position anyway, and for an output that turns the header off.
|
|
56360
56904
|
*
|
|
56361
56905
|
* The names come from the dataset's own schema, so a `pivot` is described by the header.
|
|
56362
56906
|
*/
|
|
56363
56907
|
this._composeHeader = (output, dsSchema) => {
|
|
56364
56908
|
if (output.header === false) return null;
|
|
56365
56909
|
if (output.format !== "CSV" && output.format !== "TXT") return null;
|
|
56366
|
-
return
|
|
56910
|
+
return DatasetCodec_default.encodeRow(DatasetSchema_default.names(dsSchema), this._getDelimiter(output));
|
|
56367
56911
|
};
|
|
56368
56912
|
/** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
|
|
56369
56913
|
this._getDelimiter = (output) => {
|
|
@@ -57010,10 +57554,10 @@ var init_ExecutorPerformance = __esm({
|
|
|
57010
57554
|
});
|
|
57011
57555
|
|
|
57012
57556
|
// ../../packages/executors/src/ExecutorWriter.ts
|
|
57013
|
-
var
|
|
57557
|
+
var fs28, ExecutorWriter, ExecutorWriter_default;
|
|
57014
57558
|
var init_ExecutorWriter = __esm({
|
|
57015
57559
|
"../../packages/executors/src/ExecutorWriter.ts"() {
|
|
57016
|
-
|
|
57560
|
+
fs28 = __toESM(require("fs"));
|
|
57017
57561
|
init_src();
|
|
57018
57562
|
init_src3();
|
|
57019
57563
|
init_src10();
|
|
@@ -57065,7 +57609,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57065
57609
|
await this._route(pass, dsSchema);
|
|
57066
57610
|
if (pass.buckets.size === 0)
|
|
57067
57611
|
return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
|
|
57068
|
-
await
|
|
57612
|
+
await fs28.promises.unlink(sourcePath);
|
|
57069
57613
|
return this._listFiles(pass);
|
|
57070
57614
|
};
|
|
57071
57615
|
/**
|
|
@@ -57073,7 +57617,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57073
57617
|
*/
|
|
57074
57618
|
this._route = async (pass, dsSchema) => {
|
|
57075
57619
|
const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
|
|
57076
|
-
const readStream =
|
|
57620
|
+
const readStream = fs28.createReadStream(pass.sourcePath);
|
|
57077
57621
|
try {
|
|
57078
57622
|
for await (const line of readLines(readStream)) {
|
|
57079
57623
|
const bucket = this._bucketFor(line, pass, parseLine);
|
|
@@ -57177,7 +57721,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57177
57721
|
}
|
|
57178
57722
|
if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
|
|
57179
57723
|
await this._closeStream(pass.openStreams.shift());
|
|
57180
|
-
bucket.stream =
|
|
57724
|
+
bucket.stream = fs28.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
|
|
57181
57725
|
pass.openStreams.push(bucket);
|
|
57182
57726
|
return bucket.stream;
|
|
57183
57727
|
};
|
|
@@ -57762,6 +58306,11 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
57762
58306
|
const start = performance.now();
|
|
57763
58307
|
const executorResults = [];
|
|
57764
58308
|
Logger_default.info(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length}${OrchestratorHelper_default.formatRequestOptions(options2)}) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
|
|
58309
|
+
const fixedReview = request.skipExport ? null : OutputExecutor_default.reviewFixedOutput(consumer, dsSchema);
|
|
58310
|
+
Affirm_default(
|
|
58311
|
+
!fixedReview || fixedReview.errors.length === 0,
|
|
58312
|
+
`Consumer "${consumer.name}" cannot write its FIXED output: ${fixedReview?.errors.join(" ")}`
|
|
58313
|
+
);
|
|
57765
58314
|
const lookups = await LookupResolver_default.materialize(consumer, expandedFields, scope, request);
|
|
57766
58315
|
const consumerInputs = await ConsumerInputResolver_default.materialize(consumer, scope, request);
|
|
57767
58316
|
let counter = performance.now();
|