@forzalabs/remora 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENCE.md +52 -17
- package/README.md +11 -1
- package/RELEASE_NOTES.md +192 -0
- package/docs/Remora - Code Sets.md +22 -1
- package/docs/Remora - Consumers.md +27 -2
- package/docs/Remora - Producers.md +160 -5
- package/documentation/README.md +2 -1
- package/index.js +1491 -411
- package/json_schemas/consumer-schema.json +12 -1
- package/json_schemas/producer-schema.json +22 -3
- package/package.json +2 -2
- package/workers/ExecutorWorker.js +1121 -138
- package/CHANGELOG.md +0 -557
|
@@ -8538,7 +8538,7 @@ var require_node2 = __commonJS({
|
|
|
8538
8538
|
var require_tail_file = __commonJS({
|
|
8539
8539
|
"../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
|
|
8540
8540
|
"use strict";
|
|
8541
|
-
var
|
|
8541
|
+
var fs34 = require("fs");
|
|
8542
8542
|
var { StringDecoder } = require("string_decoder");
|
|
8543
8543
|
var { Stream } = require_readable();
|
|
8544
8544
|
function noop() {
|
|
@@ -8559,7 +8559,7 @@ var require_tail_file = __commonJS({
|
|
|
8559
8559
|
stream.emit("end");
|
|
8560
8560
|
stream.emit("close");
|
|
8561
8561
|
};
|
|
8562
|
-
|
|
8562
|
+
fs34.open(options2.file, "a+", "0644", (err2, fd) => {
|
|
8563
8563
|
if (err2) {
|
|
8564
8564
|
if (!iter) {
|
|
8565
8565
|
stream.emit("error", err2);
|
|
@@ -8571,10 +8571,10 @@ var require_tail_file = __commonJS({
|
|
|
8571
8571
|
}
|
|
8572
8572
|
(function read() {
|
|
8573
8573
|
if (stream.destroyed) {
|
|
8574
|
-
|
|
8574
|
+
fs34.close(fd, noop);
|
|
8575
8575
|
return;
|
|
8576
8576
|
}
|
|
8577
|
-
return
|
|
8577
|
+
return fs34.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
|
|
8578
8578
|
if (error) {
|
|
8579
8579
|
if (!iter) {
|
|
8580
8580
|
stream.emit("error", error);
|
|
@@ -8633,7 +8633,7 @@ var require_tail_file = __commonJS({
|
|
|
8633
8633
|
var require_file = __commonJS({
|
|
8634
8634
|
"../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
|
|
8635
8635
|
"use strict";
|
|
8636
|
-
var
|
|
8636
|
+
var fs34 = require("fs");
|
|
8637
8637
|
var path22 = require("path");
|
|
8638
8638
|
var asyncSeries = require_series();
|
|
8639
8639
|
var zlib3 = require("zlib");
|
|
@@ -8838,7 +8838,7 @@ var require_file = __commonJS({
|
|
|
8838
8838
|
let buff = "";
|
|
8839
8839
|
let results = [];
|
|
8840
8840
|
let row = 0;
|
|
8841
|
-
const stream =
|
|
8841
|
+
const stream = fs34.createReadStream(file, {
|
|
8842
8842
|
encoding: "utf8"
|
|
8843
8843
|
});
|
|
8844
8844
|
stream.on("error", (err2) => {
|
|
@@ -8990,7 +8990,7 @@ var require_file = __commonJS({
|
|
|
8990
8990
|
stat(callback) {
|
|
8991
8991
|
const target = this._getFile();
|
|
8992
8992
|
const fullpath = path22.join(this.dirname, target);
|
|
8993
|
-
|
|
8993
|
+
fs34.stat(fullpath, (err2, stat) => {
|
|
8994
8994
|
if (err2 && err2.code === "ENOENT") {
|
|
8995
8995
|
debug("ENOENT\xA0ok", fullpath);
|
|
8996
8996
|
this.filename = target;
|
|
@@ -9095,7 +9095,7 @@ var require_file = __commonJS({
|
|
|
9095
9095
|
_createStream(source) {
|
|
9096
9096
|
const fullpath = path22.join(this.dirname, this.filename);
|
|
9097
9097
|
debug("create stream start", fullpath, this.options);
|
|
9098
|
-
const dest =
|
|
9098
|
+
const dest = fs34.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
|
|
9099
9099
|
debug("file open ok", fullpath);
|
|
9100
9100
|
this.emit("open", fullpath);
|
|
9101
9101
|
source.pipe(dest);
|
|
@@ -9174,7 +9174,7 @@ var require_file = __commonJS({
|
|
|
9174
9174
|
const isZipped = this.zippedArchive ? ".gz" : "";
|
|
9175
9175
|
const filePath = `${basename}${isOldest}${ext}${isZipped}`;
|
|
9176
9176
|
const target = path22.join(this.dirname, filePath);
|
|
9177
|
-
|
|
9177
|
+
fs34.unlink(target, callback);
|
|
9178
9178
|
}
|
|
9179
9179
|
/**
|
|
9180
9180
|
* Roll files forward based on integer, up to maxFiles. e.g. if base if
|
|
@@ -9197,17 +9197,17 @@ var require_file = __commonJS({
|
|
|
9197
9197
|
tasks.push(function(i6, cb) {
|
|
9198
9198
|
let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
|
|
9199
9199
|
const tmppath = path22.join(this.dirname, fileName);
|
|
9200
|
-
|
|
9200
|
+
fs34.exists(tmppath, (exists) => {
|
|
9201
9201
|
if (!exists) {
|
|
9202
9202
|
return cb(null);
|
|
9203
9203
|
}
|
|
9204
9204
|
fileName = `${basename}${i6}${ext}${isZipped}`;
|
|
9205
|
-
|
|
9205
|
+
fs34.rename(tmppath, path22.join(this.dirname, fileName), cb);
|
|
9206
9206
|
});
|
|
9207
9207
|
}.bind(this, x2));
|
|
9208
9208
|
}
|
|
9209
9209
|
asyncSeries(tasks, () => {
|
|
9210
|
-
|
|
9210
|
+
fs34.rename(
|
|
9211
9211
|
path22.join(this.dirname, `${basename}${ext}${isZipped}`),
|
|
9212
9212
|
path22.join(this.dirname, `${basename}1${ext}${isZipped}`),
|
|
9213
9213
|
callback
|
|
@@ -9223,22 +9223,22 @@ var require_file = __commonJS({
|
|
|
9223
9223
|
* @private
|
|
9224
9224
|
*/
|
|
9225
9225
|
_compressFile(src, dest, callback) {
|
|
9226
|
-
|
|
9226
|
+
fs34.access(src, fs34.F_OK, (err2) => {
|
|
9227
9227
|
if (err2) {
|
|
9228
9228
|
return callback();
|
|
9229
9229
|
}
|
|
9230
9230
|
var gzip = zlib3.createGzip();
|
|
9231
|
-
var inp =
|
|
9232
|
-
var out =
|
|
9231
|
+
var inp = fs34.createReadStream(src);
|
|
9232
|
+
var out = fs34.createWriteStream(dest);
|
|
9233
9233
|
out.on("finish", () => {
|
|
9234
|
-
|
|
9234
|
+
fs34.unlink(src, callback);
|
|
9235
9235
|
});
|
|
9236
9236
|
inp.pipe(gzip).pipe(out);
|
|
9237
9237
|
});
|
|
9238
9238
|
}
|
|
9239
9239
|
_createLogDirIfNotExist(dirPath) {
|
|
9240
|
-
if (!
|
|
9241
|
-
|
|
9240
|
+
if (!fs34.existsSync(dirPath)) {
|
|
9241
|
+
fs34.mkdirSync(dirPath, { recursive: true });
|
|
9242
9242
|
}
|
|
9243
9243
|
}
|
|
9244
9244
|
};
|
|
@@ -11809,7 +11809,7 @@ var CONSTANTS, Constants_default;
|
|
|
11809
11809
|
var init_Constants = __esm({
|
|
11810
11810
|
"../../packages/constants/src/Constants.ts"() {
|
|
11811
11811
|
CONSTANTS = {
|
|
11812
|
-
cliVersion: "2.
|
|
11812
|
+
cliVersion: "2.7.0",
|
|
11813
11813
|
backendVersion: 1,
|
|
11814
11814
|
backendPort: 5088,
|
|
11815
11815
|
workerVersion: 2,
|
|
@@ -11846,6 +11846,13 @@ var init_Constants = __esm({
|
|
|
11846
11846
|
defaults: {
|
|
11847
11847
|
REMORA_PATH: "./remora",
|
|
11848
11848
|
PRODUCER_TEMP_FOLDER: ".temp",
|
|
11849
|
+
/**
|
|
11850
|
+
* Where `synth` writes its exports, inside the remora directory, when none is given with
|
|
11851
|
+
* `-o`. A synthetic run never writes to the consumer's configured destination, so it needs
|
|
11852
|
+
* a folder of its own — and one inside the project, so the files it leaves sit next to the
|
|
11853
|
+
* configuration that produced them rather than somewhere in the user's tree.
|
|
11854
|
+
*/
|
|
11855
|
+
SYNTH_OUTPUT_FOLDER: ".synth",
|
|
11849
11856
|
SQL_MAX_QUERY_ROWS: 1e4,
|
|
11850
11857
|
/**
|
|
11851
11858
|
* Rows a SQL source hands over per round trip while STAGING a producer (a `FETCH` from a
|
|
@@ -11937,7 +11944,7 @@ var init_Constants = __esm({
|
|
|
11937
11944
|
/**
|
|
11938
11945
|
* Record layouts one FIXED output may write into a single file, which is also how many write
|
|
11939
11946
|
* streams its block-ordering pass holds open. A copybook of this size is already unusual —
|
|
11940
|
-
*
|
|
11947
|
+
* a POD-style transaction feed may write five transaction types — and the bound stops a generated config
|
|
11941
11948
|
* from opening a descriptor per distinct value of the wrong field.
|
|
11942
11949
|
*/
|
|
11943
11950
|
MAX_FIXED_LAYOUTS: 64,
|
|
@@ -12228,7 +12235,7 @@ var init_ProcessShutdownManager = __esm({
|
|
|
12228
12235
|
const shutdownState = this._shutdownState;
|
|
12229
12236
|
const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
|
|
12230
12237
|
if (scopeCount === 0) {
|
|
12231
|
-
Logger_default.
|
|
12238
|
+
Logger_default.log(`Detected ${shutdownDescription} for ${this._runtimeName}. No executor scopes to clean up.`);
|
|
12232
12239
|
return;
|
|
12233
12240
|
}
|
|
12234
12241
|
Logger_default.warn(`Detected ${shutdownDescription} for ${this._runtimeName}. Cleaning up ${scopeCount} executor scope${scopeCount === 1 ? "" : "s"}.`);
|
|
@@ -12741,7 +12748,7 @@ var init_ResourcesUtils = __esm({
|
|
|
12741
12748
|
});
|
|
12742
12749
|
|
|
12743
12750
|
// ../../packages/common/src/validation/Validator.ts
|
|
12744
|
-
var ValidatorClass, Validator, Validator_default;
|
|
12751
|
+
var MAX_SCALE, SIGN_MODES, USAGE_MODES, ValidatorClass, Validator, Validator_default;
|
|
12745
12752
|
var init_Validator = __esm({
|
|
12746
12753
|
"../../packages/common/src/validation/Validator.ts"() {
|
|
12747
12754
|
init_src();
|
|
@@ -12751,6 +12758,13 @@ var init_Validator = __esm({
|
|
|
12751
12758
|
init_ProcessENVManager();
|
|
12752
12759
|
init_SecretManager();
|
|
12753
12760
|
init_ResourcesUtils();
|
|
12761
|
+
MAX_SCALE = 15;
|
|
12762
|
+
SIGN_MODES = ["overpunch", "leading", "trailing"];
|
|
12763
|
+
USAGE_MODES = {
|
|
12764
|
+
"display": null,
|
|
12765
|
+
"comp-3": "packed decimal (two digits per byte, the sign in the last nibble)",
|
|
12766
|
+
"comp": "binary (a two's-complement integer of 2, 4 or 8 bytes)"
|
|
12767
|
+
};
|
|
12754
12768
|
ValidatorClass = class {
|
|
12755
12769
|
constructor() {
|
|
12756
12770
|
this.validateSources = (sources) => {
|
|
@@ -12852,6 +12866,9 @@ var init_Validator = __esm({
|
|
|
12852
12866
|
errors.push(`Producer "${producer.name}" has dimension(s) ${typeless.join(", ")} with no "type". Only a repeating group (a dimension declaring nested "dimensions") may omit it.`);
|
|
12853
12867
|
errors.push(...this.validateLegacySynthFields(producer));
|
|
12854
12868
|
errors.push(...this.validateFixedWidthLayout(producer));
|
|
12869
|
+
errors.push(...this.validateDimensionScale(producer));
|
|
12870
|
+
errors.push(...this.validateDimensionSign(producer));
|
|
12871
|
+
errors.push(...this.validateDimensionUsage(producer));
|
|
12855
12872
|
errors.push(...this.validateDimensionMasks(producer));
|
|
12856
12873
|
const sourceFilenameDimensions = producer.dimensions.filter((x2) => x2.sourceFilename === true);
|
|
12857
12874
|
if (sourceFilenameDimensions.length > 1) {
|
|
@@ -12875,6 +12892,67 @@ var init_Validator = __esm({
|
|
|
12875
12892
|
}
|
|
12876
12893
|
return errors;
|
|
12877
12894
|
};
|
|
12895
|
+
/**
|
|
12896
|
+
* Checks the implied decimals of a dimension. A wrong `scale` is the one layout error nothing
|
|
12897
|
+
* downstream can catch — the value it produces is a perfectly plausible number, 10^n off — so
|
|
12898
|
+
* every way of declaring one that cannot mean what it says is rejected here rather than read.
|
|
12899
|
+
*/
|
|
12900
|
+
this.validateDimensionScale = (producer) => {
|
|
12901
|
+
Affirm_default(producer, "Invalid producer");
|
|
12902
|
+
const errors = [];
|
|
12903
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12904
|
+
if (!Algo_default.hasVal(dim.scale)) continue;
|
|
12905
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12906
|
+
if (dim.type !== "number")
|
|
12907
|
+
errors.push(`${where} declares "scale" on a "${dim.type ?? "typeless"}" dimension: implied decimals only describe a "number".`);
|
|
12908
|
+
if (!Number.isInteger(dim.scale) || dim.scale < 1)
|
|
12909
|
+
errors.push(`${where} declares "scale" ${JSON.stringify(dim.scale)}: it counts implied decimal places, so it must be a whole number of 1 or more. A field written with its decimal point, or with no fraction at all, simply omits it.`);
|
|
12910
|
+
else if (dim.scale > MAX_SCALE)
|
|
12911
|
+
errors.push(`${where} declares "scale" ${dim.scale}, more than the ${MAX_SCALE} decimal places a number can carry without losing digits.`);
|
|
12912
|
+
}
|
|
12913
|
+
return errors;
|
|
12914
|
+
};
|
|
12915
|
+
/**
|
|
12916
|
+
* Checks how a dimension says its sign is written. A mode Remora does not implement would leave
|
|
12917
|
+
* the sign unread — a value of the right magnitude and the wrong direction, which is the one
|
|
12918
|
+
* error a reconciliation finds and a run never does — so a typo fails `verify` rather than a run.
|
|
12919
|
+
*/
|
|
12920
|
+
this.validateDimensionSign = (producer) => {
|
|
12921
|
+
Affirm_default(producer, "Invalid producer");
|
|
12922
|
+
const errors = [];
|
|
12923
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12924
|
+
if (!Algo_default.hasVal(dim.sign)) continue;
|
|
12925
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12926
|
+
if (dim.type !== "number")
|
|
12927
|
+
errors.push(`${where} declares "sign" on a "${dim.type ?? "typeless"}" dimension: a written sign only describes a "number".`);
|
|
12928
|
+
if (!SIGN_MODES.includes(dim.sign))
|
|
12929
|
+
errors.push(`${where} declares "sign" ${JSON.stringify(dim.sign)}, which is not a way a sign can be written. Use ${SIGN_MODES.map((x2) => `"${x2}"`).join(", ")}, or omit it for a plain leading "-".`);
|
|
12930
|
+
}
|
|
12931
|
+
return errors;
|
|
12932
|
+
};
|
|
12933
|
+
/**
|
|
12934
|
+
* Rejects a field the copybook says is stored as BYTES. `import-copybook` emits the `USAGE` it
|
|
12935
|
+
* read rather than dropping it, because the field's digits are the ones a positional read would
|
|
12936
|
+
* slice characters out of and get garbage from: the feed has to expand them to display before it
|
|
12937
|
+
* is sent. Saying so here, naming the field and the conversion, is how the customer learns at
|
|
12938
|
+
* compile instead of from a column of NaN after a run (docs/plans/mainframe.md open question 3).
|
|
12939
|
+
*/
|
|
12940
|
+
this.validateDimensionUsage = (producer) => {
|
|
12941
|
+
Affirm_default(producer, "Invalid producer");
|
|
12942
|
+
const errors = [];
|
|
12943
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12944
|
+
if (!Algo_default.hasVal(dim.usage)) continue;
|
|
12945
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12946
|
+
if (!(dim.usage in USAGE_MODES)) {
|
|
12947
|
+
errors.push(`${where} declares "usage" ${JSON.stringify(dim.usage)}, which is not a storage mode. Use ${Object.keys(USAGE_MODES).map((x2) => `"${x2}"`).join(", ")}.`);
|
|
12948
|
+
continue;
|
|
12949
|
+
}
|
|
12950
|
+
const stored = USAGE_MODES[dim.usage];
|
|
12951
|
+
if (stored)
|
|
12952
|
+
errors.push(`${where} declares "usage": "${dim.usage}": the field is stored as ${stored}, not as characters, so a positional read cannot address it. Have the feed convert it to display before it is sent (the DFSORT/OUTREC step that converts the file also expands packed and binary fields), then remove "usage" from this dimension.`);
|
|
12953
|
+
}
|
|
12954
|
+
return errors;
|
|
12955
|
+
};
|
|
12878
12956
|
/**
|
|
12879
12957
|
* Rejects a masking mode that is not one Remora implements. The producer schema can't: the value
|
|
12880
12958
|
* may be a `{VAR}` reference that only resolves at runtime. Checked here so a typo fails `verify`
|
|
@@ -13510,7 +13588,7 @@ var init_Validator = __esm({
|
|
|
13510
13588
|
});
|
|
13511
13589
|
|
|
13512
13590
|
// ../../packages/common/src/ProducerExpander.ts
|
|
13513
|
-
var MAX_OCCURS, DEFAULT_GROUP_NAMING, DEFAULT_ROWS_NAMING, DEFAULT_OCCURRENCE_FIELD, DEFAULT_FLAG_NAMING, GROUP_NAMING_TOKENS, FLAG_NAMING_TOKENS, GROUP_FORBIDDEN_PROPERTIES, GROUP_ONLY_PROPERTIES, IDENTIFIER_PATTERN, ProducerExpanderClass, ProducerExpander, ProducerExpander_default;
|
|
13591
|
+
var MAX_OCCURS, DEFAULT_GROUP_NAMING, DEFAULT_ROWS_NAMING, DEFAULT_OCCURRENCE_FIELD, DEFAULT_FLAG_NAMING, GROUP_NAMING_TOKENS, FLAG_NAMING_TOKENS, INDEX_TOKEN_PATTERN, GROUP_FORBIDDEN_PROPERTIES, GROUP_ONLY_PROPERTIES, NESTED_FORBIDDEN_PROPERTIES, IDENTIFIER_PATTERN, ProducerExpanderClass, ProducerExpander, ProducerExpander_default;
|
|
13514
13592
|
var init_ProducerExpander = __esm({
|
|
13515
13593
|
"../../packages/common/src/ProducerExpander.ts"() {
|
|
13516
13594
|
init_src();
|
|
@@ -13523,8 +13601,10 @@ var init_ProducerExpander = __esm({
|
|
|
13523
13601
|
DEFAULT_FLAG_NAMING = "{field}{Flag}";
|
|
13524
13602
|
GROUP_NAMING_TOKENS = ["{group}", "{index}", "{field}", "{Field}"];
|
|
13525
13603
|
FLAG_NAMING_TOKENS = ["{field}", "{Field}", "{flag}", "{Flag}"];
|
|
13526
|
-
|
|
13604
|
+
INDEX_TOKEN_PATTERN = /{index(\d*)}/g;
|
|
13605
|
+
GROUP_FORBIDDEN_PROPERTIES = ["type", "alias", "pk", "mask", "synth", "sourceFilename", "flags", "format", "scale", "sign", "usage"];
|
|
13527
13606
|
GROUP_ONLY_PROPERTIES = ["occurs", "indexFrom", "emit", "skipEmptyOccurrences", "occurrenceField"];
|
|
13607
|
+
NESTED_FORBIDDEN_PROPERTIES = ["emit", "skipEmptyOccurrences", "occurrenceField"];
|
|
13528
13608
|
IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
13529
13609
|
ProducerExpanderClass = class {
|
|
13530
13610
|
constructor() {
|
|
@@ -13603,22 +13683,25 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13603
13683
|
this.isGroup = (dimension) => {
|
|
13604
13684
|
return Array.isArray(dimension?.dimensions);
|
|
13605
13685
|
};
|
|
13606
|
-
this._validateGroup = (producer, declared, group3) => {
|
|
13686
|
+
this._validateGroup = (producer, declared, group3, nesting = null) => {
|
|
13607
13687
|
const errors = [];
|
|
13608
|
-
const where = `group dimension "${group3.name}" of producer "${producer.name}"`;
|
|
13688
|
+
const where = nesting ? `nested group dimension "${group3.name}" of ${nesting.where}` : `group dimension "${group3.name}" of producer "${producer.name}"`;
|
|
13609
13689
|
if (!IDENTIFIER_PATTERN.test(group3.name ?? ""))
|
|
13610
13690
|
errors.push(`The name of ${where} is used as a prefix for the names it generates, so it must be a valid identifier: letters, digits and underscores, not starting with a digit.`);
|
|
13611
13691
|
const forbidden = GROUP_FORBIDDEN_PROPERTIES.filter((property) => group3[property] !== void 0);
|
|
13612
13692
|
if (forbidden.length > 0)
|
|
13613
13693
|
errors.push(`${where} declares ${forbidden.map((x2) => `"${x2}"`).join(", ")}: a group produces no column of its own, so ${forbidden.length > 1 ? "they" : "it"} would be silently ignored. Declare ${forbidden.length > 1 ? "them" : "it"} on the child dimension(s) instead.`);
|
|
13614
|
-
errors.push(...Validator_default.validateFieldPosition(group3.position, `${where} (the extent of ONE occurrence)`));
|
|
13694
|
+
errors.push(...Validator_default.validateFieldPosition(group3.position, `${where} (the extent of ONE occurrence${nesting ? ", RELATIVE to one occurrence of the group it sits in" : ""})`));
|
|
13615
13695
|
const occurrenceWidth = this._isPositionValid(group3.position) ? this._width(group3.position) : null;
|
|
13616
|
-
errors.push(...this.
|
|
13696
|
+
errors.push(...this._validateNesting(group3, occurrenceWidth, nesting, where));
|
|
13697
|
+
if (!nesting || typeof group3.occurs !== "object")
|
|
13698
|
+
errors.push(...this._validateOccurs(declared, group3, occurrenceWidth, where));
|
|
13617
13699
|
if (Algo_default.hasVal(group3.indexFrom) && group3.indexFrom !== 0 && group3.indexFrom !== 1)
|
|
13618
13700
|
errors.push(`"indexFrom" on ${where} is ${JSON.stringify(group3.indexFrom)}: it must be 0 or 1.`);
|
|
13619
|
-
|
|
13701
|
+
if (!nesting)
|
|
13702
|
+
errors.push(...this._validateEmit(group3, where));
|
|
13620
13703
|
if (Algo_default.hasVal(group3.naming)) {
|
|
13621
|
-
errors.push(...this._validateNaming(group3.naming,
|
|
13704
|
+
errors.push(...this._validateNaming(group3.naming, this._groupNamingTokens(nesting?.depth ?? 0), where));
|
|
13622
13705
|
const columns = this._occurrences(group3).max;
|
|
13623
13706
|
if (columns > 1 && group3.emit !== "rows" && !group3.naming.includes("{index}"))
|
|
13624
13707
|
errors.push(`"naming" on ${where} ("${group3.naming}") has no "{index}" token, so all ${columns} occurrences would generate the same dimension name.`);
|
|
@@ -13631,16 +13714,39 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13631
13714
|
const childDupes = Algo_default.uniq(children.map((x2) => x2.name).filter((name, index, all) => all.indexOf(name) !== index));
|
|
13632
13715
|
if (childDupes.length > 0)
|
|
13633
13716
|
errors.push(`${where} declares duplicate child dimension name(s): ${childDupes.map((x2) => `"${x2}"`).join(", ")}.`);
|
|
13634
|
-
|
|
13635
|
-
|
|
13717
|
+
const inner = { depth: (nesting?.depth ?? 0) + 1, width: occurrenceWidth, where };
|
|
13718
|
+
for (const child of children) {
|
|
13719
|
+
errors.push(...this.isGroup(child) ? this._validateGroup(producer, declared, child, inner) : this._validateGroupChild(child, occurrenceWidth, `child dimension "${child.name}" of ${where}`));
|
|
13720
|
+
}
|
|
13636
13721
|
return errors;
|
|
13637
13722
|
};
|
|
13638
|
-
|
|
13723
|
+
/**
|
|
13724
|
+
* What a group may only do at the top level. A nested group's occurrences are laid out inside one
|
|
13725
|
+
* occurrence of its parent, which is itself placed by the line, so the inner count has to be
|
|
13726
|
+
* statically known — a per-line count on both axes has no single place to be read from.
|
|
13727
|
+
*/
|
|
13728
|
+
this._validateNesting = (group3, occurrenceWidth, nesting, where) => {
|
|
13729
|
+
if (!nesting) return [];
|
|
13639
13730
|
const errors = [];
|
|
13640
|
-
|
|
13641
|
-
|
|
13731
|
+
const forbidden = NESTED_FORBIDDEN_PROPERTIES.filter((property) => Algo_default.hasVal(group3[property]));
|
|
13732
|
+
if (forbidden.length > 0)
|
|
13733
|
+
errors.push(`${where} declares ${forbidden.map((x2) => `"${x2}"`).join(", ")}: ${forbidden.length > 1 ? "those properties describe" : "that property describes"} fanning a LINE out to one record per occurrence, which only the outermost group can do.`);
|
|
13734
|
+
if (typeof group3.occurs === "object")
|
|
13735
|
+
errors.push(`"occurs" on ${where} varies per line: a nested group repeats a fixed number of times inside one occurrence of the group it sits in, so its count cannot depend on the line. Declare a whole number, or lift the group out to the top level.`);
|
|
13736
|
+
const { max } = this._occurrences(group3);
|
|
13737
|
+
if (occurrenceWidth === null || !Number.isInteger(max) || max < 1 || !Algo_default.hasVal(nesting.width))
|
|
13642
13738
|
return errors;
|
|
13643
|
-
|
|
13739
|
+
const start = group3.position.start ?? 1;
|
|
13740
|
+
const end = start + max * occurrenceWidth - 1;
|
|
13741
|
+
if (end > nesting.width)
|
|
13742
|
+
errors.push(`${where} spans columns ${start}-${end} of an occurrence that is only ${nesting.width} character(s) wide. A nested group's position is RELATIVE to one occurrence of the group it sits in, and all ${max} of its own occurrences have to fit inside it.`);
|
|
13743
|
+
return errors;
|
|
13744
|
+
};
|
|
13745
|
+
this._validateGroupChild = (child, occurrenceWidth, where) => {
|
|
13746
|
+
const errors = [];
|
|
13747
|
+
const groupOnly = GROUP_ONLY_PROPERTIES.filter((property) => Algo_default.hasVal(child[property]));
|
|
13748
|
+
if (groupOnly.length > 0)
|
|
13749
|
+
errors.push(`${where} declares ${groupOnly.map((x2) => `"${x2}"`).join(", ")} but has no "dimensions": ${groupOnly.length > 1 ? "those properties describe" : "that property describes"} a repeating group, and would be silently ignored here.`);
|
|
13644
13750
|
if (child.sourceFilename === true)
|
|
13645
13751
|
errors.push(`${where} declares "sourceFilename": that dimension isn't in the line at all, so repeating it inside a group is meaningless. Declare it outside the group.`);
|
|
13646
13752
|
if (!child.type)
|
|
@@ -13753,9 +13859,9 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13753
13859
|
const errors = [];
|
|
13754
13860
|
const groupNames = declared.filter((x2) => this.isGroup(x2)).map((x2) => x2.name);
|
|
13755
13861
|
for (const [index, dimension] of declared.entries()) {
|
|
13756
|
-
for (const child of dimension
|
|
13862
|
+
for (const { child, parent } of this._descendants(dimension)) {
|
|
13757
13863
|
if (!child.position?.after) continue;
|
|
13758
|
-
errors.push(`"position.after" on child dimension "${child.name}" of group "${
|
|
13864
|
+
errors.push(`"position.after" on child dimension "${child.name}" of group "${parent.name}" (producer "${producer.name}"): a child's position is RELATIVE to one occurrence, so only the group itself can be anchored.`);
|
|
13759
13865
|
}
|
|
13760
13866
|
const after = dimension.position?.after;
|
|
13761
13867
|
if (!after) continue;
|
|
@@ -13798,6 +13904,20 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13798
13904
|
errors.push(`"flags" on ${where} lists ${flags.length} flag(s) but the dimension spans ${width} character(s): one flag decodes exactly one character.`);
|
|
13799
13905
|
return errors;
|
|
13800
13906
|
};
|
|
13907
|
+
/** Every dimension declared inside this one, at any depth, with the group that declares it. */
|
|
13908
|
+
this._descendants = (dimension) => {
|
|
13909
|
+
const found = [];
|
|
13910
|
+
for (const child of dimension.dimensions ?? [])
|
|
13911
|
+
found.push({ child, parent: dimension }, ...this._descendants(child));
|
|
13912
|
+
return found;
|
|
13913
|
+
};
|
|
13914
|
+
/**
|
|
13915
|
+
* The naming tokens a group at this depth may use: its own `{index}`, plus one `{indexN}` per
|
|
13916
|
+
* enclosing group, so a template can reach outward as far as the nesting goes and no further.
|
|
13917
|
+
*/
|
|
13918
|
+
this._groupNamingTokens = (depth) => {
|
|
13919
|
+
return [...GROUP_NAMING_TOKENS, ...Array.from({ length: depth }, (_3, level) => `{index${level + 1}}`)];
|
|
13920
|
+
};
|
|
13801
13921
|
this._validateNaming = (naming, allowed, where) => {
|
|
13802
13922
|
const unknown = (naming.match(/{[^}]*}/g) ?? []).filter((token2) => !allowed.includes(token2));
|
|
13803
13923
|
if (unknown.length === 0)
|
|
@@ -13906,16 +14026,12 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13906
14026
|
this._expandGroup = (group3, layout) => {
|
|
13907
14027
|
if (layout.isRows)
|
|
13908
14028
|
return this._expandRowsGroup(group3);
|
|
13909
|
-
|
|
13910
|
-
|
|
13911
|
-
const isPlanned = this._isPlanned(layout);
|
|
14029
|
+
if (!this._isPlanned(layout))
|
|
14030
|
+
return this._groupLeaves(group3, layout.start - 1, []);
|
|
13912
14031
|
const leaves = [];
|
|
13913
14032
|
for (let occurrence = 0; occurrence < layout.max; occurrence++) {
|
|
13914
|
-
const
|
|
13915
|
-
|
|
13916
|
-
const position = isPlanned ? { start: child.position.start, length: this._width(child.position), group: group3.name, occurrence } : { start: base + child.position.start - 1, length: this._width(child.position) };
|
|
13917
|
-
leaves.push(this._groupLeaf(child, group3, this._applyGroupNaming(naming, group3.name, occurrence + indexFrom, child.name), position));
|
|
13918
|
-
}
|
|
14033
|
+
for (const leaf of this._occurrenceLeaves(group3, occurrence, []))
|
|
14034
|
+
leaves.push({ ...leaf, position: { ...leaf.position, group: group3.name, occurrence } });
|
|
13919
14035
|
}
|
|
13920
14036
|
return leaves;
|
|
13921
14037
|
};
|
|
@@ -13925,7 +14041,6 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13925
14041
|
* no occurrence fixed — the read path picks it per record.
|
|
13926
14042
|
*/
|
|
13927
14043
|
this._expandRowsGroup = (group3) => {
|
|
13928
|
-
const naming = group3.naming ?? DEFAULT_ROWS_NAMING;
|
|
13929
14044
|
const occurrenceField = {
|
|
13930
14045
|
name: this._occurrenceFieldName(group3),
|
|
13931
14046
|
description: `Which occurrence of "${group3.name}" this record carries`,
|
|
@@ -13933,14 +14048,41 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13933
14048
|
occurrenceOf: group3.name,
|
|
13934
14049
|
indexFrom: group3.indexFrom ?? 0
|
|
13935
14050
|
};
|
|
13936
|
-
const leaves = group3.
|
|
13937
|
-
child,
|
|
13938
|
-
group3,
|
|
13939
|
-
this._applyGroupNaming(naming, group3.name, 0, child.name),
|
|
13940
|
-
{ start: child.position.start, length: this._width(child.position), group: group3.name }
|
|
13941
|
-
));
|
|
14051
|
+
const leaves = this._occurrenceLeaves(group3, 0, []).map((leaf) => ({ ...leaf, position: { ...leaf.position, group: group3.name } }));
|
|
13942
14052
|
return [occurrenceField, ...leaves];
|
|
13943
14053
|
};
|
|
14054
|
+
/**
|
|
14055
|
+
* Every leaf a group generates, with each `position.start` 1-based and RELATIVE to the region the
|
|
14056
|
+
* group sits in: the line for a top-level group, one occurrence of the parent for a nested one.
|
|
14057
|
+
* `base` is that region's 0-based offset, and `ancestors` the enclosing occurrence indices,
|
|
14058
|
+
* nearest first, which the naming templates read as `{index1}`...`{indexN}`.
|
|
14059
|
+
*/
|
|
14060
|
+
this._groupLeaves = (group3, base, ancestors) => {
|
|
14061
|
+
const width = this._width(group3.position);
|
|
14062
|
+
const leaves = [];
|
|
14063
|
+
for (let occurrence = 0; occurrence < this._occurrences(group3).max; occurrence++) {
|
|
14064
|
+
const offset = base + occurrence * width;
|
|
14065
|
+
for (const leaf of this._occurrenceLeaves(group3, occurrence, ancestors))
|
|
14066
|
+
leaves.push({ ...leaf, position: { ...leaf.position, start: leaf.position.start + offset } });
|
|
14067
|
+
}
|
|
14068
|
+
return leaves;
|
|
14069
|
+
};
|
|
14070
|
+
/**
|
|
14071
|
+
* The leaves of ONE occurrence of a group, positioned relative to that occurrence's own start. A
|
|
14072
|
+
* child that is itself a group contributes its own leaves instead, already flattened — so the
|
|
14073
|
+
* outer template names what the inner one generated, and nesting costs the pipeline nothing.
|
|
14074
|
+
*/
|
|
14075
|
+
this._occurrenceLeaves = (group3, occurrence, ancestors) => {
|
|
14076
|
+
const naming = group3.naming ?? (group3.emit === "rows" ? DEFAULT_ROWS_NAMING : DEFAULT_GROUP_NAMING);
|
|
14077
|
+
const indices = [occurrence + (group3.indexFrom ?? 0), ...ancestors];
|
|
14078
|
+
const leaves = [];
|
|
14079
|
+
for (const child of group3.dimensions) {
|
|
14080
|
+
const inner = this.isGroup(child) ? this._groupLeaves(child, (child.position.start ?? 1) - 1, indices) : [{ ...child, position: { start: child.position.start ?? 1, length: this._width(child.position) } }];
|
|
14081
|
+
for (const leaf of inner)
|
|
14082
|
+
leaves.push(this._groupLeaf(leaf, group3, this._applyGroupNaming(naming, group3.name, indices, leaf.name), leaf.position));
|
|
14083
|
+
}
|
|
14084
|
+
return leaves;
|
|
14085
|
+
};
|
|
13944
14086
|
this._groupLeaf = (child, group3, name, position) => {
|
|
13945
14087
|
const leaf = { ...child, name, position };
|
|
13946
14088
|
if (!leaf.description && group3.description)
|
|
@@ -13975,8 +14117,9 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13975
14117
|
rebased.after = position.after;
|
|
13976
14118
|
return rebased;
|
|
13977
14119
|
};
|
|
13978
|
-
|
|
13979
|
-
|
|
14120
|
+
/** `indices[0]` is the group's own occurrence, `indices[n]` the one n levels out. */
|
|
14121
|
+
this._applyGroupNaming = (naming, group3, indices, field) => {
|
|
14122
|
+
return naming.replace(/{group}/g, group3).replace(INDEX_TOKEN_PATTERN, (token2, level) => String(indices[level === "" ? 0 : Number(level)] ?? token2)).replace(/{field}/g, field).replace(/{Field}/g, this._capitalize(field));
|
|
13980
14123
|
};
|
|
13981
14124
|
this._applyFlagNaming = (naming, field, flag) => {
|
|
13982
14125
|
return naming.replace(/{field}/g, field).replace(/{Field}/g, this._capitalize(field)).replace(/{flag}/g, flag).replace(/{Flag}/g, this._capitalize(flag));
|
|
@@ -34011,12 +34154,107 @@ var init_FixedWidthParser = __esm({
|
|
|
34011
34154
|
}
|
|
34012
34155
|
});
|
|
34013
34156
|
|
|
34157
|
+
// ../../packages/parsing/src/NumericCodec.ts
|
|
34158
|
+
var SIGN_MODES2, OVERPUNCH_POSITIVE, OVERPUNCH_NEGATIVE, NumericCodecClass, NumericCodec, NumericCodec_default;
|
|
34159
|
+
var init_NumericCodec = __esm({
|
|
34160
|
+
"../../packages/parsing/src/NumericCodec.ts"() {
|
|
34161
|
+
SIGN_MODES2 = {
|
|
34162
|
+
none: { pattern: /^([+-]?)(\d+(?:\.\d+)?)$/, shape: 'digits, with an optional leading "+" or "-"' },
|
|
34163
|
+
leading: { pattern: /^([+-]?)(\d+(?:\.\d+)?)$/, shape: 'an optional leading "+" or "-", then digits' },
|
|
34164
|
+
trailing: { pattern: /^(\d+(?:\.\d+)?)([+-]?)$/, shape: 'digits, then an optional "+" or "-"' },
|
|
34165
|
+
overpunch: { pattern: /^(\d*)([0-9{}A-R])$/, shape: 'digits whose last byte carries the sign: "{" or "A"-"I" positive, "}" or "J"-"R" negative' }
|
|
34166
|
+
};
|
|
34167
|
+
OVERPUNCH_POSITIVE = "{ABCDEFGHI";
|
|
34168
|
+
OVERPUNCH_NEGATIVE = "}JKLMNOPQR";
|
|
34169
|
+
NumericCodecClass = class {
|
|
34170
|
+
constructor() {
|
|
34171
|
+
/**
|
|
34172
|
+
* Reads a number out of such a field. `"0012345"` at `scale: 2` is `123.45`; `"123J"` under
|
|
34173
|
+
* `sign: "overpunch"` is `-1231`. Both misreadings are silent — `12345` is as plausible an
|
|
34174
|
+
* amount as `123.45`, and `NaN` is what a run gets today — so the decoding is declared by the
|
|
34175
|
+
* layout and anything else is rejected.
|
|
34176
|
+
*
|
|
34177
|
+
* A value that is ALREADY a number has been decoded by whoever produced it (a JSON source, the
|
|
34178
|
+
* synthetic input, a re-read staged value) and passes through untouched: both properties
|
|
34179
|
+
* describe how the number is spelled in text, not the number.
|
|
34180
|
+
*/
|
|
34181
|
+
this.parse = (value, field) => {
|
|
34182
|
+
if (typeof value === "number") return value;
|
|
34183
|
+
const text = String(value).trim();
|
|
34184
|
+
const { digits, negative } = this.readSign(text, field.sign);
|
|
34185
|
+
if (!field.scale) return Number(negative ? `-${digits}` : digits);
|
|
34186
|
+
if (digits.includes("."))
|
|
34187
|
+
throw new Error(`Cannot read "${text}" as a number with ${field.scale} implied decimal(s): the last ${field.scale} digits are the fraction, so the value cannot also carry a decimal point.`);
|
|
34188
|
+
const padded = digits.padStart(field.scale + 1, "0");
|
|
34189
|
+
const split2 = padded.length - field.scale;
|
|
34190
|
+
return Number(`${negative ? "-" : ""}${padded.slice(0, split2)}.${padded.slice(split2)}`);
|
|
34191
|
+
};
|
|
34192
|
+
/**
|
|
34193
|
+
* Writes a number the way the field spells it — the mirror of `parse`, so a value read from such
|
|
34194
|
+
* a field can always be written back to one. `123.45` at `scale: 2` is `"12345"`, and `-1231`
|
|
34195
|
+
* under `sign: "overpunch"` is `"123J"`. The result is unpadded, since the width is the field's
|
|
34196
|
+
* to fill.
|
|
34197
|
+
*
|
|
34198
|
+
* Scaled through `toFixed`, which rounds in decimal: `(123.45).toFixed(2)` is `"123.45"` where
|
|
34199
|
+
* `123.45 * 100` is `12344.999999999998`.
|
|
34200
|
+
*/
|
|
34201
|
+
this.format = (value, field) => {
|
|
34202
|
+
const parsed = typeof value === "number" ? value : Number(value);
|
|
34203
|
+
if (!Number.isFinite(parsed))
|
|
34204
|
+
throw new Error(`Cannot write "${value}" as a number: it is not a number.`);
|
|
34205
|
+
const magnitude = Math.abs(parsed);
|
|
34206
|
+
const digits = field.scale ? magnitude.toFixed(field.scale).replace(".", "") : String(magnitude);
|
|
34207
|
+
return this.writeSign(digits, parsed < 0, field.sign);
|
|
34208
|
+
};
|
|
34209
|
+
/** Splits the written sign off a field, leaving the digits and whether the value is negative. */
|
|
34210
|
+
this.readSign = (text, sign2) => {
|
|
34211
|
+
const mode = SIGN_MODES2[sign2 ?? "none"];
|
|
34212
|
+
const match = mode.pattern.exec(text);
|
|
34213
|
+
if (!match)
|
|
34214
|
+
throw new Error(`Cannot read "${text}" as a number: this field holds ${mode.shape}.`);
|
|
34215
|
+
if (sign2 === "overpunch") return this.readOverpunch(match[1], match[2]);
|
|
34216
|
+
const [digits, mark] = sign2 === "trailing" ? [match[1], match[2]] : [match[2], match[1]];
|
|
34217
|
+
return { digits, negative: mark === "-" };
|
|
34218
|
+
};
|
|
34219
|
+
/** The last byte of an overpunched field, read back as its digit and its sign. */
|
|
34220
|
+
this.readOverpunch = (leading, last) => {
|
|
34221
|
+
const negative = OVERPUNCH_NEGATIVE.includes(last);
|
|
34222
|
+
const digit = negative ? OVERPUNCH_NEGATIVE.indexOf(last) : OVERPUNCH_POSITIVE.indexOf(last);
|
|
34223
|
+
return { digits: leading + (digit < 0 ? last : digit), negative };
|
|
34224
|
+
};
|
|
34225
|
+
/**
|
|
34226
|
+
* Puts the sign back the way the field carries it. A positive overpunched value is written in its
|
|
34227
|
+
* letter form (`123A`), which is what a signed DISPLAY field holds — the plain-digit form is the
|
|
34228
|
+
* unsigned one, and a field declaring `sign` is signed.
|
|
34229
|
+
*/
|
|
34230
|
+
this.writeSign = (digits, negative, sign2) => {
|
|
34231
|
+
switch (sign2) {
|
|
34232
|
+
case "overpunch": {
|
|
34233
|
+
const zone = negative ? OVERPUNCH_NEGATIVE : OVERPUNCH_POSITIVE;
|
|
34234
|
+
return digits.slice(0, -1) + zone[Number(digits.slice(-1))];
|
|
34235
|
+
}
|
|
34236
|
+
case "leading":
|
|
34237
|
+
return `${negative ? "-" : "+"}${digits}`;
|
|
34238
|
+
case "trailing":
|
|
34239
|
+
return `${digits}${negative ? "-" : "+"}`;
|
|
34240
|
+
default:
|
|
34241
|
+
return negative ? `-${digits}` : digits;
|
|
34242
|
+
}
|
|
34243
|
+
};
|
|
34244
|
+
}
|
|
34245
|
+
};
|
|
34246
|
+
NumericCodec = new NumericCodecClass();
|
|
34247
|
+
NumericCodec_default = NumericCodec;
|
|
34248
|
+
}
|
|
34249
|
+
});
|
|
34250
|
+
|
|
34014
34251
|
// ../../packages/parsing/src/FixedWidthWriter.ts
|
|
34015
|
-
var BASE_FILL, TYPE_FILL, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
|
|
34252
|
+
var BASE_FILL, TYPE_FILL, MAX_SCALE2, SIGN_MODES3, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
|
|
34016
34253
|
var init_FixedWidthWriter = __esm({
|
|
34017
34254
|
"../../packages/parsing/src/FixedWidthWriter.ts"() {
|
|
34018
34255
|
init_src();
|
|
34019
34256
|
init_src3();
|
|
34257
|
+
init_NumericCodec();
|
|
34020
34258
|
BASE_FILL = { align: "left", pad: " ", nullFill: " ", overflow: "fail" };
|
|
34021
34259
|
TYPE_FILL = {
|
|
34022
34260
|
string: BASE_FILL,
|
|
@@ -34024,6 +34262,8 @@ var init_FixedWidthWriter = __esm({
|
|
|
34024
34262
|
datetime: BASE_FILL,
|
|
34025
34263
|
boolean: BASE_FILL
|
|
34026
34264
|
};
|
|
34265
|
+
MAX_SCALE2 = 15;
|
|
34266
|
+
SIGN_MODES3 = ["overpunch", "leading", "trailing"];
|
|
34027
34267
|
FixedWidthWriterClass = class {
|
|
34028
34268
|
constructor() {
|
|
34029
34269
|
/**
|
|
@@ -34204,6 +34444,24 @@ var init_FixedWidthWriter = __esm({
|
|
|
34204
34444
|
errors.push(`"position.pad" of ${where} must be a single character, not "${fill2.pad}"`);
|
|
34205
34445
|
if (fill2.nullFill.length !== 1)
|
|
34206
34446
|
errors.push(`"position.nullFill" of ${where} must be a single character, not "${fill2.nullFill}"`);
|
|
34447
|
+
errors.push(...this.spellingErrors(column, position, where));
|
|
34448
|
+
return errors;
|
|
34449
|
+
};
|
|
34450
|
+
/**
|
|
34451
|
+
* How the field says its number is written, checked before a record ever reaches it. A wrong
|
|
34452
|
+
* `scale` writes a plausible amount that is 10^n off and a `sign` the mode cannot write throws
|
|
34453
|
+
* on the first record, so both are rejected at compile — on the column's own type, since neither
|
|
34454
|
+
* describes anything a string, a date or a boolean is spelled with.
|
|
34455
|
+
*/
|
|
34456
|
+
this.spellingErrors = (column, position, where) => {
|
|
34457
|
+
const errors = [];
|
|
34458
|
+
const declared = [position.scale === void 0 || position.scale === null ? null : "scale", position.sign ? "sign" : null].filter(Boolean);
|
|
34459
|
+
if (declared.length > 0 && column.type !== "number")
|
|
34460
|
+
errors.push(`${declared.map((x2) => `"position.${x2}"`).join(" and ")} of ${where} ${declared.length > 1 ? "describe" : "describes"} how a number is written, but the field is a "${column.type ?? "typeless"}"`);
|
|
34461
|
+
if (position.scale !== void 0 && position.scale !== null && (!Number.isInteger(position.scale) || position.scale < 1 || position.scale > MAX_SCALE2))
|
|
34462
|
+
errors.push(`"position.scale" of ${where} counts implied decimal places, so it must be a whole number between 1 and ${MAX_SCALE2}, not ${JSON.stringify(position.scale)}. A field written with its decimal point simply omits it.`);
|
|
34463
|
+
if (position.sign && !SIGN_MODES3.includes(position.sign))
|
|
34464
|
+
errors.push(`"position.sign" of ${where} is ${JSON.stringify(position.sign)}, which is not a way a sign can be written. Use ${SIGN_MODES3.map((x2) => `"${x2}"`).join(", ")}, or omit it for a plain leading "-".`);
|
|
34207
34465
|
return errors;
|
|
34208
34466
|
};
|
|
34209
34467
|
/**
|
|
@@ -34229,7 +34487,9 @@ var init_FixedWidthWriter = __esm({
|
|
|
34229
34487
|
align: fill2.align,
|
|
34230
34488
|
pad: fill2.pad,
|
|
34231
34489
|
nullText: fill2.nullFill.repeat(position.width),
|
|
34232
|
-
overflow: fill2.overflow
|
|
34490
|
+
overflow: fill2.overflow,
|
|
34491
|
+
scale: position.scale,
|
|
34492
|
+
sign: position.sign
|
|
34233
34493
|
};
|
|
34234
34494
|
};
|
|
34235
34495
|
/** The three fill sources folded into one, nearest declaration first. */
|
|
@@ -34270,11 +34530,33 @@ var init_FixedWidthWriter = __esm({
|
|
|
34270
34530
|
* run; see `overflowed`.
|
|
34271
34531
|
*/
|
|
34272
34532
|
this.writeField = (value, field) => {
|
|
34273
|
-
if (value === null || value === void 0) return field.nullText;
|
|
34274
|
-
const text =
|
|
34275
|
-
if (text.length === 0) return field.nullText;
|
|
34533
|
+
if (value === null || value === void 0 || value === "") return field.nullText;
|
|
34534
|
+
const text = this.fieldText(value, field);
|
|
34276
34535
|
if (text.length > field.width) return this.overflowed(text, field);
|
|
34277
|
-
return
|
|
34536
|
+
return this.filled(text, field);
|
|
34537
|
+
};
|
|
34538
|
+
/**
|
|
34539
|
+
* The value as the text of its field. A field declaring `scale` or `sign` is spelled the way the
|
|
34540
|
+
* layout says a number is spelled — `123.45` at `scale: 2` written back as `12345`, `-1231`
|
|
34541
|
+
* under `sign: "overpunch"` as `123J` — and everything else is the value as it stands.
|
|
34542
|
+
*
|
|
34543
|
+
* Without this a read-and-write-back is silently wrong in both directions: a field read as
|
|
34544
|
+
* `0012345` would be written `123.45`, which is the right number spelled the wrong way, in the
|
|
34545
|
+
* one place nothing downstream re-reads.
|
|
34546
|
+
*/
|
|
34547
|
+
this.fieldText = (value, field) => {
|
|
34548
|
+
if (field.scale || field.sign) return NumericCodec_default.format(value, field);
|
|
34549
|
+
return typeof value === "string" ? value : String(value);
|
|
34550
|
+
};
|
|
34551
|
+
/**
|
|
34552
|
+
* The text filled out to the field's width, on the side away from `align`. A `leading` sign keeps
|
|
34553
|
+
* the first column to itself and the fill goes between it and the digits — `-0001234`, the way a
|
|
34554
|
+
* SIGN LEADING SEPARATE field is written — rather than ahead of a sign nothing would then find.
|
|
34555
|
+
*/
|
|
34556
|
+
this.filled = (text, field) => {
|
|
34557
|
+
if (field.align !== "right") return text.padEnd(field.width, field.pad);
|
|
34558
|
+
if (field.sign === "leading") return text.charAt(0) + text.slice(1).padStart(field.width - 1, field.pad);
|
|
34559
|
+
return text.padStart(field.width, field.pad);
|
|
34278
34560
|
};
|
|
34279
34561
|
/**
|
|
34280
34562
|
* A value longer than its field. It fails the run by default: a truncated account number is a
|
|
@@ -34283,14 +34565,17 @@ var init_FixedWidthWriter = __esm({
|
|
|
34283
34565
|
*
|
|
34284
34566
|
* Where the layout genuinely specifies a cut, the end that survives follows `align`, which is
|
|
34285
34567
|
* what a COBOL MOVE does: a left-aligned value keeps its head, a right-aligned one keeps its
|
|
34286
|
-
* low-order tail.
|
|
34568
|
+
* low-order tail. A `leading` sign survives either way — a cut that drops it writes the
|
|
34569
|
+
* magnitude with the direction reversed, which is worse than the cut itself.
|
|
34287
34570
|
*/
|
|
34288
34571
|
this.overflowed = (text, field) => {
|
|
34289
34572
|
Affirm_default(
|
|
34290
34573
|
field.overflow === "truncate",
|
|
34291
34574
|
`Value "${text}" is ${text.length} characters, longer than the ${field.width} of field "${field.name}" at columns ${field.start}-${field.end}. Widen the field, or set its "overflow" to "truncate" if the layout really specifies a cut.`
|
|
34292
34575
|
);
|
|
34293
|
-
|
|
34576
|
+
if (field.align !== "right") return text.slice(0, field.width);
|
|
34577
|
+
if (field.sign === "leading") return text.charAt(0) + text.slice(text.length - field.width + 1);
|
|
34578
|
+
return text.slice(text.length - field.width);
|
|
34294
34579
|
};
|
|
34295
34580
|
/**
|
|
34296
34581
|
* What a record carries after its last field, BEFORE the newline the destination writes. Every
|
|
@@ -34298,7 +34583,7 @@ var init_FixedWidthWriter = __esm({
|
|
|
34298
34583
|
* `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
|
|
34299
34584
|
* CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
|
|
34300
34585
|
*
|
|
34301
|
-
* A real positional feed picks one terminator and holds it:
|
|
34586
|
+
* A real positional feed picks one terminator and holds it: some are CRLF on
|
|
34302
34587
|
* every line including the last, and `lf` is the default everywhere else.
|
|
34303
34588
|
*/
|
|
34304
34589
|
this.lineSuffix = (output) => {
|
|
@@ -34370,7 +34655,7 @@ var require_pend = __commonJS({
|
|
|
34370
34655
|
// ../../node_modules/yauzl/fd-slicer.js
|
|
34371
34656
|
var require_fd_slicer = __commonJS({
|
|
34372
34657
|
"../../node_modules/yauzl/fd-slicer.js"(exports2) {
|
|
34373
|
-
var
|
|
34658
|
+
var fs34 = require("fs");
|
|
34374
34659
|
var util = require("util");
|
|
34375
34660
|
var stream = require("stream");
|
|
34376
34661
|
var Readable10 = stream.Readable;
|
|
@@ -34390,7 +34675,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34390
34675
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
34391
34676
|
var self2 = this;
|
|
34392
34677
|
self2.pend.go(function(cb) {
|
|
34393
|
-
|
|
34678
|
+
fs34.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
|
|
34394
34679
|
cb();
|
|
34395
34680
|
callback(err2, bytesRead, buffer2);
|
|
34396
34681
|
});
|
|
@@ -34407,7 +34692,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34407
34692
|
self2.refCount -= 1;
|
|
34408
34693
|
if (self2.refCount < 0) throw new Error("invalid unref");
|
|
34409
34694
|
if (self2.refCount > 0) return;
|
|
34410
|
-
|
|
34695
|
+
fs34.close(self2.fd, onCloseDone);
|
|
34411
34696
|
function onCloseDone(err2) {
|
|
34412
34697
|
if (err2) {
|
|
34413
34698
|
self2.emit("error", err2);
|
|
@@ -34439,7 +34724,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34439
34724
|
}
|
|
34440
34725
|
self2.context.pend.go(function(cb) {
|
|
34441
34726
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
34442
|
-
|
|
34727
|
+
fs34.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
|
|
34443
34728
|
if (err2) {
|
|
34444
34729
|
self2.destroy(err2);
|
|
34445
34730
|
} else if (bytesRead === 0) {
|
|
@@ -34799,7 +35084,7 @@ var require_crc32 = __commonJS({
|
|
|
34799
35084
|
// ../../node_modules/yauzl/index.js
|
|
34800
35085
|
var require_yauzl = __commonJS({
|
|
34801
35086
|
"../../node_modules/yauzl/index.js"(exports2) {
|
|
34802
|
-
var
|
|
35087
|
+
var fs34 = require("fs");
|
|
34803
35088
|
var zlib3 = require("zlib");
|
|
34804
35089
|
var fd_slicer = require_fd_slicer();
|
|
34805
35090
|
var util = require("util");
|
|
@@ -34868,10 +35153,10 @@ var require_yauzl = __commonJS({
|
|
|
34868
35153
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34869
35154
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34870
35155
|
if (callback == null) callback = defaultCallback;
|
|
34871
|
-
|
|
35156
|
+
fs34.open(path22, "r", function(err2, fd) {
|
|
34872
35157
|
if (err2) return callback(err2);
|
|
34873
35158
|
fromFd(fd, options2, function(err3, zipfile) {
|
|
34874
|
-
if (err3)
|
|
35159
|
+
if (err3) fs34.close(fd, defaultCallback);
|
|
34875
35160
|
callback(err3, zipfile);
|
|
34876
35161
|
});
|
|
34877
35162
|
});
|
|
@@ -34888,7 +35173,7 @@ var require_yauzl = __commonJS({
|
|
|
34888
35173
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34889
35174
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34890
35175
|
if (callback == null) callback = defaultCallback;
|
|
34891
|
-
|
|
35176
|
+
fs34.fstat(fd, function(err2, stats) {
|
|
34892
35177
|
if (err2) return callback(err2);
|
|
34893
35178
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
34894
35179
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -38970,6 +39255,7 @@ var init_src5 = __esm({
|
|
|
38970
39255
|
init_DatasetCodec();
|
|
38971
39256
|
init_FixedWidthParser();
|
|
38972
39257
|
init_FixedWidthWriter();
|
|
39258
|
+
init_NumericCodec();
|
|
38973
39259
|
init_ParseCompression();
|
|
38974
39260
|
init_ParseHelper();
|
|
38975
39261
|
init_XLSParser();
|
|
@@ -46858,6 +47144,16 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46858
47144
|
* sibling of the destination on purpose: rename is only atomic within one filesystem.
|
|
46859
47145
|
*/
|
|
46860
47146
|
this._stagingPath = (toFilePath) => `${toFilePath}.${(0, import_crypto4.randomUUID)()}.partial`;
|
|
47147
|
+
/**
|
|
47148
|
+
* Creates the folder an export is about to be written into.
|
|
47149
|
+
*
|
|
47150
|
+
* An `exportName` may carry a prefix — it is written for an S3 key, where `outbound/feed.csv`
|
|
47151
|
+
* needs no folder to exist — so a local destination has to make one. `saveFile` always did;
|
|
47152
|
+
* `move` did not, and failed with a bare ENOENT naming the staging file rather than the folder.
|
|
47153
|
+
*/
|
|
47154
|
+
this._ensureDirectory = async (toFilePath) => {
|
|
47155
|
+
await import_promises10.default.mkdir(import_path11.default.dirname(toFilePath), { recursive: true });
|
|
47156
|
+
};
|
|
46861
47157
|
/** Removes a staging file a failed export left behind; never masks the original error. */
|
|
46862
47158
|
this._discardStaging = async (stagingPath) => {
|
|
46863
47159
|
try {
|
|
@@ -46869,6 +47165,7 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46869
47165
|
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46870
47166
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46871
47167
|
try {
|
|
47168
|
+
await this._ensureDirectory(toFilePath);
|
|
46872
47169
|
if (header)
|
|
46873
47170
|
await DriverHelper_default.copyWithHeader(fromPath, stagingPath, header);
|
|
46874
47171
|
else
|
|
@@ -46886,6 +47183,7 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46886
47183
|
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46887
47184
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46888
47185
|
try {
|
|
47186
|
+
await this._ensureDirectory(toFilePath);
|
|
46889
47187
|
const reader = fs15.createReadStream(fromPath);
|
|
46890
47188
|
const writer = new BatchWriter_default(fs15.createWriteStream(stagingPath));
|
|
46891
47189
|
if (header)
|
|
@@ -48021,6 +48319,7 @@ var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMA
|
|
|
48021
48319
|
var init_TypeCaster = __esm({
|
|
48022
48320
|
"../../packages/engines/src/transform/TypeCaster.ts"() {
|
|
48023
48321
|
init_src();
|
|
48322
|
+
init_src5();
|
|
48024
48323
|
import_dayjs = __toESM(require("dayjs"), 1);
|
|
48025
48324
|
import_customParseFormat = __toESM(require("dayjs/plugin/customParseFormat"), 1);
|
|
48026
48325
|
import_utc = __toESM(require("dayjs/plugin/utc"), 1);
|
|
@@ -48062,6 +48361,7 @@ var init_TypeCaster = __esm({
|
|
|
48062
48361
|
this.castDimension = (value, dimension) => {
|
|
48063
48362
|
if (dimension.type && this.isMissing(value, dimension.type))
|
|
48064
48363
|
return dimension.nullable === false ? TYPE_DEFAULTS[dimension.type] : null;
|
|
48364
|
+
if (dimension.scale || dimension.sign) return NumericCodec_default.parse(value, dimension);
|
|
48065
48365
|
return this.cast(value, dimension.type, dimension.format);
|
|
48066
48366
|
};
|
|
48067
48367
|
/** null/undefined for any type, plus the empty string for every type but `string`. */
|
|
@@ -51470,7 +51770,10 @@ var init_SyntheticInputEngine = __esm({
|
|
|
51470
51770
|
name,
|
|
51471
51771
|
type: dim.type ?? "string",
|
|
51472
51772
|
format: dim.format,
|
|
51473
|
-
|
|
51773
|
+
// An implied-decimal field holds exactly `scale` decimals, so that is what a generated
|
|
51774
|
+
// value gets unless `synth` asks for another: a 4-decimal value in a `scale: 2` column
|
|
51775
|
+
// is one the layout cannot write back.
|
|
51776
|
+
decimals: synth.decimals ?? dim.scale,
|
|
51474
51777
|
min,
|
|
51475
51778
|
max,
|
|
51476
51779
|
isPk: dim.pk === true,
|
|
@@ -51632,6 +51935,650 @@ var init_SyntheticInputEngine = __esm({
|
|
|
51632
51935
|
}
|
|
51633
51936
|
});
|
|
51634
51937
|
|
|
51938
|
+
// ../../packages/engines/src/copybook/CopybookParser.ts
|
|
51939
|
+
var MAX_RECORD_WIDTH, USAGES, UNSUPPORTED_USAGES, CLAUSE_KEYWORDS, PICTURE_CHARS, EDIT_CHARS, DIGIT_CHARS, COMMENT_INDICATORS, INDICATORS, CopybookParserClass, CopybookParser;
|
|
51940
|
+
var init_CopybookParser = __esm({
|
|
51941
|
+
"../../packages/engines/src/copybook/CopybookParser.ts"() {
|
|
51942
|
+
init_src();
|
|
51943
|
+
MAX_RECORD_WIDTH = 32767;
|
|
51944
|
+
USAGES = {
|
|
51945
|
+
"DISPLAY": "display",
|
|
51946
|
+
"COMP": "comp",
|
|
51947
|
+
"COMPUTATIONAL": "comp",
|
|
51948
|
+
"COMP-4": "comp",
|
|
51949
|
+
"COMPUTATIONAL-4": "comp",
|
|
51950
|
+
"COMP-5": "comp",
|
|
51951
|
+
"COMPUTATIONAL-5": "comp",
|
|
51952
|
+
"BINARY": "comp",
|
|
51953
|
+
"COMP-3": "comp-3",
|
|
51954
|
+
"COMPUTATIONAL-3": "comp-3",
|
|
51955
|
+
"PACKED-DECIMAL": "comp-3"
|
|
51956
|
+
};
|
|
51957
|
+
UNSUPPORTED_USAGES = {
|
|
51958
|
+
"COMP-1": "single-precision hexadecimal floating point",
|
|
51959
|
+
"COMPUTATIONAL-1": "single-precision hexadecimal floating point",
|
|
51960
|
+
"COMP-2": "double-precision hexadecimal floating point",
|
|
51961
|
+
"COMPUTATIONAL-2": "double-precision hexadecimal floating point",
|
|
51962
|
+
"INDEX": "a subscript, which holds no data of the record",
|
|
51963
|
+
"POINTER": "an address, which holds no data of the record",
|
|
51964
|
+
"DISPLAY-1": "double-byte characters"
|
|
51965
|
+
};
|
|
51966
|
+
CLAUSE_KEYWORDS = /* @__PURE__ */ new Set([
|
|
51967
|
+
"PIC",
|
|
51968
|
+
"PICTURE",
|
|
51969
|
+
"USAGE",
|
|
51970
|
+
"REDEFINES",
|
|
51971
|
+
"OCCURS",
|
|
51972
|
+
"SIGN",
|
|
51973
|
+
"VALUE",
|
|
51974
|
+
"VALUES",
|
|
51975
|
+
"JUST",
|
|
51976
|
+
"JUSTIFIED",
|
|
51977
|
+
"SYNC",
|
|
51978
|
+
"SYNCHRONIZED",
|
|
51979
|
+
"BLANK",
|
|
51980
|
+
"ASCENDING",
|
|
51981
|
+
"DESCENDING",
|
|
51982
|
+
"INDEXED",
|
|
51983
|
+
"EXTERNAL",
|
|
51984
|
+
"GLOBAL",
|
|
51985
|
+
...Object.keys(USAGES),
|
|
51986
|
+
...Object.keys(UNSUPPORTED_USAGES)
|
|
51987
|
+
]);
|
|
51988
|
+
PICTURE_CHARS = "XA9VSPZ*$,B0/.+-";
|
|
51989
|
+
EDIT_CHARS = "Z*$,B0/";
|
|
51990
|
+
DIGIT_CHARS = "9Z*";
|
|
51991
|
+
COMMENT_INDICATORS = "*/";
|
|
51992
|
+
INDICATORS = " */-";
|
|
51993
|
+
CopybookParserClass = class {
|
|
51994
|
+
constructor() {
|
|
51995
|
+
/**
|
|
51996
|
+
* Parses copybook text into one record layout. Throws on the first clause it cannot read, naming
|
|
51997
|
+
* the line it is on.
|
|
51998
|
+
*/
|
|
51999
|
+
this.parse = (text) => {
|
|
52000
|
+
Affirm_default(typeof text === "string", "Missing required parameter text");
|
|
52001
|
+
const statements = this._statements(this._lines(text));
|
|
52002
|
+
if (statements.length === 0)
|
|
52003
|
+
throw new Error("The copybook holds no data description entries.");
|
|
52004
|
+
const entries = statements.map((statement) => this._entry(statement)).filter((entry) => entry);
|
|
52005
|
+
const [record, ...alternates] = this._tree(entries);
|
|
52006
|
+
if (!record)
|
|
52007
|
+
throw new Error("The copybook declares no record: there is no 01 level item.");
|
|
52008
|
+
for (const alternate of alternates) {
|
|
52009
|
+
if (!alternate.redefines)
|
|
52010
|
+
this._fail(alternate.line, `"${alternate.name}" is a second record description, and a producer reads one record layout. Split the copybook, and read each record type with its own producer and a "recordSelector".`);
|
|
52011
|
+
}
|
|
52012
|
+
const fields = record.fields ?? [record];
|
|
52013
|
+
const width = this._place(fields);
|
|
52014
|
+
const redefinedBy = alternates.map((x2) => x2.name);
|
|
52015
|
+
return { name: record.name, line: record.line, width, fields, redefinedBy: redefinedBy.length > 0 ? redefinedBy : void 0 };
|
|
52016
|
+
};
|
|
52017
|
+
/** Throws about the copybook line a problem is on, which is the only way to locate it. */
|
|
52018
|
+
this._fail = (line, message) => {
|
|
52019
|
+
throw new Error(`Copybook line ${line}: ${message}`);
|
|
52020
|
+
};
|
|
52021
|
+
/**
|
|
52022
|
+
* The lines that hold definitions, with their comments and their sequence and identification
|
|
52023
|
+
* areas removed. Fixed-format and free-format copybooks are both read, decided per line: a line
|
|
52024
|
+
* whose first six characters are a sequence number (or blank) and whose seventh is an indicator
|
|
52025
|
+
* is read as columns 8-72, and any other line is read whole.
|
|
52026
|
+
*/
|
|
52027
|
+
this._lines = (text) => {
|
|
52028
|
+
const lines = [];
|
|
52029
|
+
text.split(/\r?\n/).forEach((raw, index) => {
|
|
52030
|
+
const fixed = /^(\d{6}| {6})$/.test(raw.slice(0, 6)) && INDICATORS.includes(raw.charAt(6));
|
|
52031
|
+
if (fixed && COMMENT_INDICATORS.includes(raw.charAt(6))) return;
|
|
52032
|
+
const content = fixed ? raw.slice(7, 72) : raw;
|
|
52033
|
+
if (content.trim().length === 0 || content.trimStart().startsWith("*")) return;
|
|
52034
|
+
lines.push({ text: content, line: index + 1 });
|
|
52035
|
+
});
|
|
52036
|
+
return lines;
|
|
52037
|
+
};
|
|
52038
|
+
/**
|
|
52039
|
+
* The data description entries, split on the periods that terminate them. A period only
|
|
52040
|
+
* terminates where it is followed by a space or the end of the line, so the one inside
|
|
52041
|
+
* `PIC 9(5).99` stays part of the picture.
|
|
52042
|
+
*/
|
|
52043
|
+
this._statements = (lines) => {
|
|
52044
|
+
const statements = [];
|
|
52045
|
+
let current = null;
|
|
52046
|
+
for (const { text, line } of lines) {
|
|
52047
|
+
for (const token2 of this._tokens(text, line)) {
|
|
52048
|
+
if (!current) current = { tokens: [], line };
|
|
52049
|
+
if (token2 !== ".") {
|
|
52050
|
+
current.tokens.push(token2);
|
|
52051
|
+
continue;
|
|
52052
|
+
}
|
|
52053
|
+
if (current.tokens.length > 0) statements.push(current);
|
|
52054
|
+
current = null;
|
|
52055
|
+
}
|
|
52056
|
+
}
|
|
52057
|
+
if (current)
|
|
52058
|
+
this._fail(current.line, `the entry starting "${current.tokens.slice(0, 3).join(" ")}" is never terminated by a period.`);
|
|
52059
|
+
return statements;
|
|
52060
|
+
};
|
|
52061
|
+
/** One line's tokens: quoted literals whole, a terminating period on its own, words otherwise. */
|
|
52062
|
+
this._tokens = (content, line) => {
|
|
52063
|
+
const tokens = [];
|
|
52064
|
+
let index = 0;
|
|
52065
|
+
while (index < content.length) {
|
|
52066
|
+
const char = content.charAt(index);
|
|
52067
|
+
if (/\s/.test(char)) {
|
|
52068
|
+
index++;
|
|
52069
|
+
continue;
|
|
52070
|
+
}
|
|
52071
|
+
if (char === '"' || char === "'") {
|
|
52072
|
+
const close = content.indexOf(char, index + 1);
|
|
52073
|
+
if (close < 0) this._fail(line, `a ${char === '"' ? "double" : "single"}-quoted literal is never closed.`);
|
|
52074
|
+
tokens.push(content.slice(index, close + 1));
|
|
52075
|
+
index = close + 1;
|
|
52076
|
+
continue;
|
|
52077
|
+
}
|
|
52078
|
+
if (this._terminates(content, index)) {
|
|
52079
|
+
tokens.push(".");
|
|
52080
|
+
index++;
|
|
52081
|
+
continue;
|
|
52082
|
+
}
|
|
52083
|
+
let end = index;
|
|
52084
|
+
while (end < content.length && !/\s/.test(content.charAt(end)) && !this._terminates(content, end)) end++;
|
|
52085
|
+
tokens.push(content.slice(index, end));
|
|
52086
|
+
index = end;
|
|
52087
|
+
}
|
|
52088
|
+
return tokens;
|
|
52089
|
+
};
|
|
52090
|
+
/** Whether the character at `index` is a period that ends an entry rather than one inside it. */
|
|
52091
|
+
this._terminates = (content, index) => {
|
|
52092
|
+
if (content.charAt(index) !== ".") return false;
|
|
52093
|
+
const next = content.charAt(index + 1);
|
|
52094
|
+
return next === "" || /\s/.test(next);
|
|
52095
|
+
};
|
|
52096
|
+
/**
|
|
52097
|
+
* One data description entry: its level, its name and its clauses. Returns `null` for a level 88
|
|
52098
|
+
* condition name, which describes values of the item above it and occupies no storage.
|
|
52099
|
+
*/
|
|
52100
|
+
this._entry = (statement) => {
|
|
52101
|
+
const { tokens, line } = statement;
|
|
52102
|
+
const level = Number(tokens[0]);
|
|
52103
|
+
if (!Number.isInteger(level) || !/^\d{1,2}$/.test(tokens[0]))
|
|
52104
|
+
this._fail(line, `"${tokens[0]}" is not a level number: an entry starts with one, written as 01-49, 66, 77 or 88.`);
|
|
52105
|
+
if (level === 88) return null;
|
|
52106
|
+
if (level === 66)
|
|
52107
|
+
this._fail(line, `level 66 (RENAMES) regroups fields that are already described: read them under their own names instead.`);
|
|
52108
|
+
if (level < 1 || level > 49 && level !== 77)
|
|
52109
|
+
this._fail(line, `level ${tokens[0]} is not a level a data item can be declared at (01-49, 66, 77 or 88).`);
|
|
52110
|
+
const named = tokens.length > 1 && !CLAUSE_KEYWORDS.has(tokens[1].toUpperCase());
|
|
52111
|
+
const name = named ? tokens[1] : "FILLER";
|
|
52112
|
+
const clauses = {};
|
|
52113
|
+
let index = named ? 2 : 1;
|
|
52114
|
+
while (index < tokens.length)
|
|
52115
|
+
index = this._clause(clauses, tokens, index, statement);
|
|
52116
|
+
return this._field(level, name, clauses, line);
|
|
52117
|
+
};
|
|
52118
|
+
/**
|
|
52119
|
+
* Reads the clause starting at `index` into `clauses` and answers where the next one begins. The
|
|
52120
|
+
* branches live in one place so that an unreadable clause is reported from one place too.
|
|
52121
|
+
*/
|
|
52122
|
+
this._clause = (clauses, tokens, index, statement) => {
|
|
52123
|
+
const keyword = tokens[index].toUpperCase();
|
|
52124
|
+
const { line } = statement;
|
|
52125
|
+
switch (keyword) {
|
|
52126
|
+
case "PIC":
|
|
52127
|
+
case "PICTURE":
|
|
52128
|
+
return this._picture(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52129
|
+
case "USAGE":
|
|
52130
|
+
return this._usage(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52131
|
+
case "REDEFINES":
|
|
52132
|
+
return this._redefines(clauses, tokens, index + 1, line);
|
|
52133
|
+
case "OCCURS":
|
|
52134
|
+
return this._occurs(clauses, tokens, index + 1, line);
|
|
52135
|
+
case "SIGN":
|
|
52136
|
+
return this._sign(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52137
|
+
case "VALUE":
|
|
52138
|
+
case "VALUES":
|
|
52139
|
+
case "JUST":
|
|
52140
|
+
case "JUSTIFIED":
|
|
52141
|
+
case "SYNC":
|
|
52142
|
+
case "SYNCHRONIZED":
|
|
52143
|
+
case "BLANK":
|
|
52144
|
+
case "ASCENDING":
|
|
52145
|
+
case "DESCENDING":
|
|
52146
|
+
case "INDEXED":
|
|
52147
|
+
case "EXTERNAL":
|
|
52148
|
+
case "GLOBAL":
|
|
52149
|
+
return this._ignore(tokens, index + 1);
|
|
52150
|
+
default:
|
|
52151
|
+
if (USAGES[keyword] || UNSUPPORTED_USAGES[keyword]) return this._usage(clauses, tokens, index, line);
|
|
52152
|
+
return this._fail(line, `"${tokens[index]}" is not a clause this reads, in the definition of "${tokens[1] ?? tokens[0]}".`);
|
|
52153
|
+
}
|
|
52154
|
+
};
|
|
52155
|
+
/** Steps past an optional noise word (`IS`, `TO`, `BY`, `ON`), which changes no meaning. */
|
|
52156
|
+
this._skip = (tokens, index, word2) => {
|
|
52157
|
+
return tokens[index]?.toUpperCase() === word2 ? index + 1 : index;
|
|
52158
|
+
};
|
|
52159
|
+
/** Consumes a clause whose content changes no position: its operands, up to the next clause. */
|
|
52160
|
+
this._ignore = (tokens, index) => {
|
|
52161
|
+
let next = index;
|
|
52162
|
+
while (next < tokens.length && !CLAUSE_KEYWORDS.has(tokens[next].toUpperCase())) next++;
|
|
52163
|
+
return next;
|
|
52164
|
+
};
|
|
52165
|
+
this._picture = (clauses, tokens, index, line) => {
|
|
52166
|
+
if (!tokens[index]) this._fail(line, "a PIC clause states no picture.");
|
|
52167
|
+
if (clauses.picture) this._fail(line, "the entry declares two PIC clauses.");
|
|
52168
|
+
clauses.picture = tokens[index];
|
|
52169
|
+
return index + 1;
|
|
52170
|
+
};
|
|
52171
|
+
this._usage = (clauses, tokens, index, line) => {
|
|
52172
|
+
const word2 = tokens[index]?.toUpperCase() ?? "";
|
|
52173
|
+
if (UNSUPPORTED_USAGES[word2])
|
|
52174
|
+
this._fail(line, `USAGE ${word2} is ${UNSUPPORTED_USAGES[word2]}, which Remora does not read.`);
|
|
52175
|
+
if (!USAGES[word2])
|
|
52176
|
+
this._fail(line, `"${tokens[index]}" is not a storage mode a USAGE clause can name.`);
|
|
52177
|
+
clauses.usage = USAGES[word2];
|
|
52178
|
+
return index + 1;
|
|
52179
|
+
};
|
|
52180
|
+
this._redefines = (clauses, tokens, index, line) => {
|
|
52181
|
+
if (!tokens[index]) this._fail(line, "a REDEFINES clause names no item.");
|
|
52182
|
+
clauses.redefines = tokens[index];
|
|
52183
|
+
return index + 1;
|
|
52184
|
+
};
|
|
52185
|
+
/** `OCCURS n [TO m] [TIMES] [DEPENDING [ON] name]`, the fixed and the varying repeat counts. */
|
|
52186
|
+
this._occurs = (clauses, tokens, index, line) => {
|
|
52187
|
+
const count = Number(tokens[index]);
|
|
52188
|
+
if (!Number.isInteger(count) || count < 1)
|
|
52189
|
+
this._fail(line, `"${tokens[index]}" is not a number of occurrences: OCCURS takes a whole number of 1 or more.`);
|
|
52190
|
+
let next = index + 1;
|
|
52191
|
+
clauses.occurs = count;
|
|
52192
|
+
if (tokens[next]?.toUpperCase() === "TO") {
|
|
52193
|
+
const max = Number(tokens[next + 1]);
|
|
52194
|
+
if (!Number.isInteger(max) || max < count)
|
|
52195
|
+
this._fail(line, `"${tokens[next + 1]}" is not an upper bound of 'OCCURS ${count} TO': it must be a whole number of ${count} or more.`);
|
|
52196
|
+
clauses.occursMin = count;
|
|
52197
|
+
clauses.occurs = max;
|
|
52198
|
+
next += 2;
|
|
52199
|
+
}
|
|
52200
|
+
next = this._skip(tokens, next, "TIMES");
|
|
52201
|
+
if (tokens[next]?.toUpperCase() !== "DEPENDING") return next;
|
|
52202
|
+
next = this._skip(tokens, next + 1, "ON");
|
|
52203
|
+
if (!tokens[next]) this._fail(line, "an OCCURS DEPENDING ON clause names no counting field.");
|
|
52204
|
+
clauses.dependingOn = tokens[next];
|
|
52205
|
+
return next + 1;
|
|
52206
|
+
};
|
|
52207
|
+
/** `SIGN [IS] LEADING|TRAILING [SEPARATE [CHARACTER]]`, which says where a written sign sits. */
|
|
52208
|
+
this._sign = (clauses, tokens, index, line) => {
|
|
52209
|
+
const word2 = tokens[index]?.toUpperCase() ?? "";
|
|
52210
|
+
if (word2 !== "LEADING" && word2 !== "TRAILING")
|
|
52211
|
+
this._fail(line, `"${tokens[index]}" is not a sign position: a SIGN clause takes LEADING or TRAILING.`);
|
|
52212
|
+
clauses.signPosition = word2 === "LEADING" ? "leading" : "trailing";
|
|
52213
|
+
const next = this._skip(tokens, index + 1, "SEPARATE");
|
|
52214
|
+
if (next === index + 1) return next;
|
|
52215
|
+
clauses.signSeparate = true;
|
|
52216
|
+
return this._skip(tokens, next, "CHARACTER");
|
|
52217
|
+
};
|
|
52218
|
+
/** The parsed entry, with the picture and the sign clause finally read against each other. */
|
|
52219
|
+
this._field = (level, name, clauses, line) => {
|
|
52220
|
+
const field = {
|
|
52221
|
+
name,
|
|
52222
|
+
level,
|
|
52223
|
+
line,
|
|
52224
|
+
offset: 0,
|
|
52225
|
+
width: 0,
|
|
52226
|
+
occurs: clauses.occurs ?? 1,
|
|
52227
|
+
occursMin: clauses.occursMin,
|
|
52228
|
+
dependingOn: clauses.dependingOn,
|
|
52229
|
+
redefines: clauses.redefines,
|
|
52230
|
+
filler: name.toUpperCase() === "FILLER",
|
|
52231
|
+
picture: clauses.picture,
|
|
52232
|
+
usage: clauses.usage ?? "display"
|
|
52233
|
+
};
|
|
52234
|
+
if (!clauses.picture) return field;
|
|
52235
|
+
return { ...field, ...this._measure(clauses, line) };
|
|
52236
|
+
};
|
|
52237
|
+
/**
|
|
52238
|
+
* What a picture says the field is: how wide it is in characters, what its value reads as, where
|
|
52239
|
+
* its decimal point is implied and how its sign is written. Every one of those is silent when it
|
|
52240
|
+
* is wrong — a mis-measured width shifts every field after it — so an unreadable picture fails.
|
|
52241
|
+
*/
|
|
52242
|
+
this._measure = (clauses, line) => {
|
|
52243
|
+
const chars2 = this._expand(clauses.picture, line);
|
|
52244
|
+
const signed = chars2[0] === "S";
|
|
52245
|
+
const body = signed ? chars2.slice(1) : chars2;
|
|
52246
|
+
if (body.includes("S")) this._fail(line, `"${clauses.picture}": 'S' can only be a picture's first character.`);
|
|
52247
|
+
if (body.includes("P")) this._fail(line, `"${clauses.picture}": 'P' scales a value by a power of ten with no digit to show for it, which no Remora dimension can state.`);
|
|
52248
|
+
const point = body.indexOf("V");
|
|
52249
|
+
if (point !== body.lastIndexOf("V")) this._fail(line, `"${clauses.picture}" states more than one implied decimal point.`);
|
|
52250
|
+
if (point >= 0 && body.includes(".")) this._fail(line, `"${clauses.picture}" states both an implied ('V') and a real ('.') decimal point.`);
|
|
52251
|
+
const width = body.filter((x2) => x2 !== "V").length + (signed && clauses.signSeparate ? 1 : 0);
|
|
52252
|
+
if (body.some((x2) => x2 === "X" || x2 === "A")) {
|
|
52253
|
+
if (signed || point >= 0)
|
|
52254
|
+
this._fail(line, `"${clauses.picture}" mixes character positions with 'S' or 'V', which only describe a number.`);
|
|
52255
|
+
return { width, type: "string" };
|
|
52256
|
+
}
|
|
52257
|
+
if (!body.some((x2) => DIGIT_CHARS.includes(x2)))
|
|
52258
|
+
this._fail(line, `"${clauses.picture}" has no digit and no character position: there is nothing for the field to hold.`);
|
|
52259
|
+
const scale = point < 0 ? 0 : body.slice(point).filter((x2) => DIGIT_CHARS.includes(x2)).length;
|
|
52260
|
+
return { width, scale: scale > 0 ? scale : void 0, ...this._numeric(body, clauses, signed, line) };
|
|
52261
|
+
};
|
|
52262
|
+
/**
|
|
52263
|
+
* How a numeric field's value is written. An EDITED picture holds insertion characters as well
|
|
52264
|
+
* as digits (`ZZ,ZZ9.99`), so its field is text: reading it as a number would mean deciding what
|
|
52265
|
+
* the insertions mean, and deciding is what this parser does not do.
|
|
52266
|
+
*/
|
|
52267
|
+
this._numeric = (body, clauses, signed, line) => {
|
|
52268
|
+
const edits = body.filter((x2) => EDIT_CHARS.includes(x2));
|
|
52269
|
+
const signs = body.filter((x2) => x2 === "+" || x2 === "-");
|
|
52270
|
+
const floating = signs.length > 1 || signs.length === 1 && body[0] !== signs[0] && body[body.length - 1] !== signs[0];
|
|
52271
|
+
if (edits.length > 0 || floating)
|
|
52272
|
+
return { type: "string", edited: `its picture "${clauses.picture}" is edited (${Algo_default.uniq([...edits, ...signs]).join(" ")}), so the field holds those characters as well as digits` };
|
|
52273
|
+
if (signs.length === 1)
|
|
52274
|
+
return { type: "number", sign: body[0] === signs[0] ? "leading" : "trailing" };
|
|
52275
|
+
if (!signed)
|
|
52276
|
+
return { type: "number" };
|
|
52277
|
+
if (clauses.signSeparate)
|
|
52278
|
+
return { type: "number", sign: clauses.signPosition ?? "trailing" };
|
|
52279
|
+
if (clauses.signPosition === "leading")
|
|
52280
|
+
this._fail(line, `"${clauses.picture}" with SIGN IS LEADING carries its sign in the zone of the FIRST byte, and Remora reads an overpunched sign from the last. Ask for SIGN IS LEADING SEPARATE, or for a trailing sign.`);
|
|
52281
|
+
return { type: "number", sign: "overpunch" };
|
|
52282
|
+
};
|
|
52283
|
+
/** A picture with its repeat counts expanded: `S9(3)V99` becomes `S 9 9 9 V 9 9`. */
|
|
52284
|
+
this._expand = (picture, line) => {
|
|
52285
|
+
const text = picture.toUpperCase().replace(/CR|DB/g, "BB");
|
|
52286
|
+
const chars2 = [];
|
|
52287
|
+
let index = 0;
|
|
52288
|
+
while (index < text.length) {
|
|
52289
|
+
const char = text.charAt(index);
|
|
52290
|
+
if (char !== "(") {
|
|
52291
|
+
if (!PICTURE_CHARS.includes(char)) this._fail(line, `"${picture}" holds "${char}", which is not a picture character.`);
|
|
52292
|
+
chars2.push(char);
|
|
52293
|
+
index++;
|
|
52294
|
+
continue;
|
|
52295
|
+
}
|
|
52296
|
+
const close = text.indexOf(")", index);
|
|
52297
|
+
const count = Number(text.slice(index + 1, close));
|
|
52298
|
+
if (close < 0 || !Number.isInteger(count) || count < 1)
|
|
52299
|
+
this._fail(line, `"${picture}" has a repeat count that is not a whole number of 1 or more.`);
|
|
52300
|
+
if (chars2.length === 0) this._fail(line, `"${picture}" opens with a repeat count, which repeats nothing.`);
|
|
52301
|
+
if (chars2.length + count > MAX_RECORD_WIDTH) this._fail(line, `"${picture}" expands past the ${MAX_RECORD_WIDTH}-character maximum.`);
|
|
52302
|
+
const repeated = chars2[chars2.length - 1];
|
|
52303
|
+
for (let i6 = 1; i6 < count; i6++) chars2.push(repeated);
|
|
52304
|
+
index = close + 1;
|
|
52305
|
+
}
|
|
52306
|
+
return chars2;
|
|
52307
|
+
};
|
|
52308
|
+
/**
|
|
52309
|
+
* The entries nested by their level numbers: each one belongs to the last entry declared at a
|
|
52310
|
+
* lower level. `01` and `77` always start a record of their own, whatever precedes them.
|
|
52311
|
+
*/
|
|
52312
|
+
this._tree = (entries) => {
|
|
52313
|
+
const roots = [];
|
|
52314
|
+
const stack = [];
|
|
52315
|
+
for (const entry of entries) {
|
|
52316
|
+
if (entry.level === 1 || entry.level === 77) stack.length = 0;
|
|
52317
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= entry.level) stack.pop();
|
|
52318
|
+
const parent = stack[stack.length - 1];
|
|
52319
|
+
if (!parent) roots.push(entry);
|
|
52320
|
+
else parent.fields = [...parent.fields ?? [], entry];
|
|
52321
|
+
stack.push(entry);
|
|
52322
|
+
}
|
|
52323
|
+
return roots;
|
|
52324
|
+
};
|
|
52325
|
+
/**
|
|
52326
|
+
* Places every item: an offset inside the item that contains it, and a width of one occurrence.
|
|
52327
|
+
* This is the whole point of reading the copybook rather than a spreadsheet of columns — an
|
|
52328
|
+
* offset nobody typed is an offset nobody mistyped. Answers the width of the items as a whole.
|
|
52329
|
+
*/
|
|
52330
|
+
this._place = (fields) => {
|
|
52331
|
+
let cursor = 1;
|
|
52332
|
+
for (const field of fields) {
|
|
52333
|
+
if (field.fields && field.picture)
|
|
52334
|
+
this._fail(field.line, `"${field.name}" has both a PIC clause and items declared under it.`);
|
|
52335
|
+
if (field.fields) field.width = this._place(field.fields);
|
|
52336
|
+
if (!field.width)
|
|
52337
|
+
this._fail(field.line, `"${field.name}" has no PIC clause and no items under it, so its width is unknown.`);
|
|
52338
|
+
if (field.redefines) {
|
|
52339
|
+
this._redefine(fields, field);
|
|
52340
|
+
continue;
|
|
52341
|
+
}
|
|
52342
|
+
field.offset = cursor;
|
|
52343
|
+
cursor += field.width * field.occurs;
|
|
52344
|
+
if (cursor - 1 > MAX_RECORD_WIDTH)
|
|
52345
|
+
this._fail(field.line, `the record passes the ${MAX_RECORD_WIDTH}-character maximum at "${field.name}".`);
|
|
52346
|
+
}
|
|
52347
|
+
return cursor - 1;
|
|
52348
|
+
};
|
|
52349
|
+
/** Lays a REDEFINES item over the item it redefines, which is where its bytes already are. */
|
|
52350
|
+
this._redefine = (siblings, field) => {
|
|
52351
|
+
const target = siblings.find((x2) => x2 !== field && x2.name === field.redefines);
|
|
52352
|
+
if (!target)
|
|
52353
|
+
this._fail(field.line, `"${field.name}" REDEFINES "${field.redefines}", which is not an item declared beside it.`);
|
|
52354
|
+
if (field.width > target.width)
|
|
52355
|
+
this._fail(field.line, `"${field.name}" is ${field.width} characters and REDEFINES "${target.name}", which is ${target.width}: a redefinition cannot be wider than what it redefines.`);
|
|
52356
|
+
field.offset = target.offset;
|
|
52357
|
+
target.redefinedBy = [...target.redefinedBy ?? [], field.name];
|
|
52358
|
+
};
|
|
52359
|
+
}
|
|
52360
|
+
};
|
|
52361
|
+
CopybookParser = new CopybookParserClass();
|
|
52362
|
+
}
|
|
52363
|
+
});
|
|
52364
|
+
|
|
52365
|
+
// ../../packages/engines/src/copybook/CopybookEmitter.ts
|
|
52366
|
+
var DEFAULT_SOURCE, REPEATED_FIELD_NAMING, CopybookEmitterClass, CopybookEmitter;
|
|
52367
|
+
var init_CopybookEmitter = __esm({
|
|
52368
|
+
"../../packages/engines/src/copybook/CopybookEmitter.ts"() {
|
|
52369
|
+
init_src();
|
|
52370
|
+
init_src4();
|
|
52371
|
+
DEFAULT_SOURCE = "s_default";
|
|
52372
|
+
REPEATED_FIELD_NAMING = "{group}{index}";
|
|
52373
|
+
CopybookEmitterClass = class {
|
|
52374
|
+
constructor() {
|
|
52375
|
+
/**
|
|
52376
|
+
* Emits the producer that reads the record this layout describes, with the notes a reader has
|
|
52377
|
+
* to act on. Throws rather than emitting a layout the config loader would reject: a layout that
|
|
52378
|
+
* is 95% right reads plausible values out of the wrong columns.
|
|
52379
|
+
*/
|
|
52380
|
+
this.toProducer = (layout, options2) => {
|
|
52381
|
+
Affirm_default(layout, "Invalid copybook layout");
|
|
52382
|
+
Affirm_default(layout.fields?.length > 0, "The copybook layout holds no fields");
|
|
52383
|
+
Affirm_default(options2?.name, "Missing required parameter name");
|
|
52384
|
+
const emitted = this._items(layout.fields, 0);
|
|
52385
|
+
if (emitted.length === 0)
|
|
52386
|
+
throw new Error(`Record "${layout.name}" has no field to read: every item in it is FILLER or a redefinition.`);
|
|
52387
|
+
this._names(emitted);
|
|
52388
|
+
this._counts(emitted);
|
|
52389
|
+
this._anchor(emitted);
|
|
52390
|
+
const producer = {
|
|
52391
|
+
name: options2.name,
|
|
52392
|
+
description: `Imported from a COBOL copybook: record ${layout.name}, ${layout.width} characters.`,
|
|
52393
|
+
source: options2.source ?? DEFAULT_SOURCE,
|
|
52394
|
+
dimensions: emitted.map((x2) => x2.dimension),
|
|
52395
|
+
settings: { fileType: "FIXED", fileKey: options2.fileKey ?? `${options2.name}.txt` },
|
|
52396
|
+
_version: 1
|
|
52397
|
+
};
|
|
52398
|
+
this._check(producer);
|
|
52399
|
+
return { producer, notes: this._notes(layout) };
|
|
52400
|
+
};
|
|
52401
|
+
/**
|
|
52402
|
+
* The dimensions a list of copybook items emits, in layout order. A group that does not repeat
|
|
52403
|
+
* is TRANSPARENT: it structures the copybook but generates no column, so its items are emitted
|
|
52404
|
+
* in its place with their offsets carried down into the coordinate space of its own parent.
|
|
52405
|
+
*/
|
|
52406
|
+
this._items = (fields, base) => {
|
|
52407
|
+
const emitted = [];
|
|
52408
|
+
for (const field of fields) {
|
|
52409
|
+
if (field.redefines || field.filler) continue;
|
|
52410
|
+
if (field.fields && !this._repeats(field)) {
|
|
52411
|
+
const inner = this._items(field.fields, base + field.offset - 1);
|
|
52412
|
+
emitted.push(...inner.map((x2) => ({ ...x2, qualifier: x2.qualifier || this._name(field.name) })));
|
|
52413
|
+
continue;
|
|
52414
|
+
}
|
|
52415
|
+
emitted.push(this._item(field, base + field.offset));
|
|
52416
|
+
}
|
|
52417
|
+
return emitted;
|
|
52418
|
+
};
|
|
52419
|
+
/** Whether an item generates one dimension per occurrence rather than a single dimension. */
|
|
52420
|
+
this._repeats = (field) => {
|
|
52421
|
+
return field.occurs > 1 || !!field.dependingOn;
|
|
52422
|
+
};
|
|
52423
|
+
/** One copybook item as one dimension: a leaf, a repeating leaf, or a group of its own. */
|
|
52424
|
+
this._item = (field, start) => {
|
|
52425
|
+
const position = { start, length: field.width };
|
|
52426
|
+
const span = field.width * field.occurs;
|
|
52427
|
+
if (!field.fields && !this._repeats(field))
|
|
52428
|
+
return { dimension: this._leaf(field, position), field, qualifier: "", start, span };
|
|
52429
|
+
const children = field.fields ? this._items(field.fields, 0) : [{ dimension: this._leaf(field, { start: 1, length: field.width }), field, qualifier: "", start: 1, span: field.width }];
|
|
52430
|
+
this._names(children);
|
|
52431
|
+
this._rejectNestedCounts(field);
|
|
52432
|
+
const dimension = {
|
|
52433
|
+
name: this._name(field.name),
|
|
52434
|
+
description: this._describe(field),
|
|
52435
|
+
position,
|
|
52436
|
+
occurs: field.occurs,
|
|
52437
|
+
// A copybook's occurrences are numbered from 1, and the names read against the copybook.
|
|
52438
|
+
indexFrom: 1,
|
|
52439
|
+
naming: field.fields ? void 0 : REPEATED_FIELD_NAMING,
|
|
52440
|
+
dimensions: children.map((x2) => x2.dimension)
|
|
52441
|
+
};
|
|
52442
|
+
return { dimension, field, qualifier: "", start, span };
|
|
52443
|
+
};
|
|
52444
|
+
/** A dimension that produces a column: its type, and how its number is spelled, if it is one. */
|
|
52445
|
+
this._leaf = (field, position) => {
|
|
52446
|
+
return {
|
|
52447
|
+
name: this._name(field.name),
|
|
52448
|
+
description: this._describe(field),
|
|
52449
|
+
type: field.type,
|
|
52450
|
+
position,
|
|
52451
|
+
scale: field.scale,
|
|
52452
|
+
sign: field.sign,
|
|
52453
|
+
usage: field.usage === "display" ? void 0 : field.usage
|
|
52454
|
+
};
|
|
52455
|
+
};
|
|
52456
|
+
/** What the copybook says about an item, kept beside the dimension it became. */
|
|
52457
|
+
this._describe = (field) => {
|
|
52458
|
+
const clauses = [field.name];
|
|
52459
|
+
if (field.picture) clauses.push(`PIC ${field.picture}`);
|
|
52460
|
+
if (field.usage !== "display") clauses.push(`USAGE ${field.usage.toUpperCase()}`);
|
|
52461
|
+
if (field.dependingOn) clauses.push(`OCCURS ${field.occursMin ?? 1} TO ${field.occurs} DEPENDING ON ${field.dependingOn}`);
|
|
52462
|
+
else if (field.occurs > 1) clauses.push(`OCCURS ${field.occurs}`);
|
|
52463
|
+
const notes = [clauses.join(" ")];
|
|
52464
|
+
if (field.edited) notes.push(`read as text because ${field.edited}`);
|
|
52465
|
+
if (field.redefinedBy?.length > 0) notes.push(`${field.redefinedBy.join(", ")} redefine the same bytes and are not read`);
|
|
52466
|
+
return notes.join("; ");
|
|
52467
|
+
};
|
|
52468
|
+
/**
|
|
52469
|
+
* A per-line occurrence count is read from the LINE, and a nested group's occurrences sit inside
|
|
52470
|
+
* one occurrence of its parent — there is no second axis to count along, and nowhere to record
|
|
52471
|
+
* the count (docs/plans/mainframe.md open question 5).
|
|
52472
|
+
*/
|
|
52473
|
+
this._rejectNestedCounts = (group3) => {
|
|
52474
|
+
for (const field of this._walk(group3.fields ?? [])) {
|
|
52475
|
+
if (!field.dependingOn) continue;
|
|
52476
|
+
throw new Error(`Copybook line ${field.line}: "${field.name}" has OCCURS DEPENDING ON inside "${group3.name}", which repeats as well. A varying count is scanned along the line, so only the outermost group can have one.`);
|
|
52477
|
+
}
|
|
52478
|
+
};
|
|
52479
|
+
/**
|
|
52480
|
+
* Gives every dimension of one list a name no sibling has. A copybook qualifies two items of the
|
|
52481
|
+
* same name by the group they sit in, and a flattened layout has to do the same: the group's
|
|
52482
|
+
* name goes in front, and a number only if that still collides.
|
|
52483
|
+
*/
|
|
52484
|
+
this._names = (emitted) => {
|
|
52485
|
+
const taken = /* @__PURE__ */ new Set();
|
|
52486
|
+
for (const item of emitted) {
|
|
52487
|
+
const base = item.dimension.name;
|
|
52488
|
+
const qualified = item.qualifier ? `${item.qualifier}${this._capitalize(base)}` : "";
|
|
52489
|
+
let name = [base, qualified].find((x2) => x2 && !taken.has(x2));
|
|
52490
|
+
for (let suffix = 2; !name && suffix <= emitted.length + 2; suffix++)
|
|
52491
|
+
if (!taken.has(`${base}${suffix}`)) name = `${base}${suffix}`;
|
|
52492
|
+
Affirm_default(name, `Cannot name the dimension read from copybook item "${item.field.name}"`);
|
|
52493
|
+
item.dimension.name = name;
|
|
52494
|
+
taken.add(name);
|
|
52495
|
+
}
|
|
52496
|
+
};
|
|
52497
|
+
/**
|
|
52498
|
+
* Resolves each `OCCURS ... DEPENDING ON` against the dimension that holds the count. The count
|
|
52499
|
+
* is read per line, so the field holding it has to be read before the occurrences it counts.
|
|
52500
|
+
*/
|
|
52501
|
+
this._counts = (emitted) => {
|
|
52502
|
+
for (const [index, item] of emitted.entries()) {
|
|
52503
|
+
const counter = item.field.dependingOn;
|
|
52504
|
+
if (!counter) continue;
|
|
52505
|
+
const source = emitted.find((x2, at3) => at3 < index && !x2.field.fields && x2.field.name === counter);
|
|
52506
|
+
if (!source)
|
|
52507
|
+
throw new Error(`Copybook line ${item.field.line}: "${item.field.name}" has OCCURS DEPENDING ON "${counter}", which is not a field read before it. The count has to be a field of this record, declared ahead of the group and outside every other one.`);
|
|
52508
|
+
item.dimension.occurs = {
|
|
52509
|
+
max: item.field.occurs,
|
|
52510
|
+
min: item.field.occursMin || void 0,
|
|
52511
|
+
from: `$${source.dimension.name}`
|
|
52512
|
+
};
|
|
52513
|
+
}
|
|
52514
|
+
};
|
|
52515
|
+
/**
|
|
52516
|
+
* Anchors everything that follows a group whose occurrence count varies. That group's span
|
|
52517
|
+
* differs line by line, so an absolute position past it reads the wrong characters on every line
|
|
52518
|
+
* but the longest — `position.after` measures from where the occurrences actually ended.
|
|
52519
|
+
*/
|
|
52520
|
+
this._anchor = (emitted) => {
|
|
52521
|
+
let anchor = null;
|
|
52522
|
+
for (const item of emitted) {
|
|
52523
|
+
if (anchor) {
|
|
52524
|
+
const offset = item.start - anchor.end;
|
|
52525
|
+
if (offset < 1)
|
|
52526
|
+
throw new Error(`Copybook line ${item.field.line}: "${item.field.name}" starts inside the occurrences of "${anchor.group}", which it is laid out after.`);
|
|
52527
|
+
item.dimension.position = { after: anchor.group, start: offset, length: item.field.width };
|
|
52528
|
+
}
|
|
52529
|
+
if (item.field.dependingOn) anchor = { group: item.dimension.name, end: item.start + item.span - 1 };
|
|
52530
|
+
}
|
|
52531
|
+
};
|
|
52532
|
+
/**
|
|
52533
|
+
* Everything the copybook states that the producer cannot carry out on its own. Written against
|
|
52534
|
+
* the copybook's own names and lines, because that is the document the reader checks it against.
|
|
52535
|
+
*/
|
|
52536
|
+
this._notes = (layout) => {
|
|
52537
|
+
const notes = [];
|
|
52538
|
+
for (const field of this._walk(layout.fields)) {
|
|
52539
|
+
if (field.usage !== "display")
|
|
52540
|
+
notes.push(`${field.name} (line ${field.line}) is USAGE ${field.usage.toUpperCase()}: its digits are stored as bytes, so "remora compile" rejects it until the feed converts the field to display. It is laid out as the ${field.width} characters its PIC clause states.`);
|
|
52541
|
+
if (field.edited)
|
|
52542
|
+
notes.push(`${field.name} (line ${field.line}) reads as text: ${field.edited}.`);
|
|
52543
|
+
if (field.redefinedBy?.length > 0)
|
|
52544
|
+
notes.push(`${field.name} (line ${field.line}) is read as declared; ${field.redefinedBy.join(", ")} redefine the same bytes and were not emitted.`);
|
|
52545
|
+
}
|
|
52546
|
+
if (layout.redefinedBy?.length > 0)
|
|
52547
|
+
notes.push(`Record ${layout.name} is read as declared; ${layout.redefinedBy.join(", ")} redefine the whole record and were not emitted.`);
|
|
52548
|
+
return notes;
|
|
52549
|
+
};
|
|
52550
|
+
/** Every item of a layout, at any depth, in declaration order. */
|
|
52551
|
+
this._walk = (fields) => {
|
|
52552
|
+
return fields.flatMap((field) => [field, ...this._walk(field.fields ?? [])]);
|
|
52553
|
+
};
|
|
52554
|
+
/**
|
|
52555
|
+
* Refuses to emit a layout the config loader would reject or expand into colliding names. The
|
|
52556
|
+
* importer exists so that nobody transcribes a layout by hand; emitting one that does not load
|
|
52557
|
+
* would hand the transcription back.
|
|
52558
|
+
*/
|
|
52559
|
+
this._check = (producer) => {
|
|
52560
|
+
const errors = ProducerExpander_default.validate(producer);
|
|
52561
|
+
if (errors.length > 0)
|
|
52562
|
+
throw new Error(`The copybook describes a layout Remora cannot read:
|
|
52563
|
+
${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
52564
|
+
ProducerExpander_default.expand(Algo_default.deepClone(producer));
|
|
52565
|
+
};
|
|
52566
|
+
/** A COBOL data name as a dimension name: `CLM-TOT-AMT` reads as `clmTotAmt`. */
|
|
52567
|
+
this._name = (cobol) => {
|
|
52568
|
+
const parts = cobol.toLowerCase().split(/[^a-z0-9]+/).filter((x2) => x2.length > 0);
|
|
52569
|
+
Affirm_default(parts.length > 0, `Cannot read a dimension name from the copybook item "${cobol}"`);
|
|
52570
|
+
const name = parts.map((part, index) => index === 0 ? part : this._capitalize(part)).join("");
|
|
52571
|
+
return /^\d/.test(name) ? `f${name}` : name;
|
|
52572
|
+
};
|
|
52573
|
+
this._capitalize = (text) => {
|
|
52574
|
+
return text.charAt(0).toUpperCase() + text.slice(1);
|
|
52575
|
+
};
|
|
52576
|
+
}
|
|
52577
|
+
};
|
|
52578
|
+
CopybookEmitter = new CopybookEmitterClass();
|
|
52579
|
+
}
|
|
52580
|
+
});
|
|
52581
|
+
|
|
51635
52582
|
// ../../packages/engines/src/consumer/ConsumerManager.ts
|
|
51636
52583
|
var ConsumerManagerClass, ConsumerManager, ConsumerManager_default;
|
|
51637
52584
|
var init_ConsumerManager = __esm({
|
|
@@ -51795,21 +52742,21 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51795
52742
|
init_src2();
|
|
51796
52743
|
ConsumerOnFinishManagerClass = class {
|
|
51797
52744
|
constructor() {
|
|
51798
|
-
this.onConsumerSuccess = async (consumer, executionId, runConsumer) => {
|
|
52745
|
+
this.onConsumerSuccess = async (consumer, executionId, runConsumer, options2) => {
|
|
51799
52746
|
void executionId;
|
|
51800
52747
|
for (const output of consumer.outputs) {
|
|
51801
52748
|
if (output.onSuccess)
|
|
51802
|
-
await this.performOnSuccessActions(consumer, output, runConsumer);
|
|
52749
|
+
await this.performOnSuccessActions(consumer, output, runConsumer, options2);
|
|
51803
52750
|
}
|
|
51804
52751
|
};
|
|
51805
|
-
this.onConsumerError = async (consumer, executionId, runConsumer) => {
|
|
52752
|
+
this.onConsumerError = async (consumer, executionId, runConsumer, options2) => {
|
|
51806
52753
|
void executionId;
|
|
51807
52754
|
for (const output of consumer.outputs) {
|
|
51808
52755
|
if (output.onError)
|
|
51809
|
-
await this.performOnErrorActions(consumer, output, runConsumer);
|
|
52756
|
+
await this.performOnErrorActions(consumer, output, runConsumer, options2);
|
|
51810
52757
|
}
|
|
51811
52758
|
};
|
|
51812
|
-
this.performOnSuccessActions = async (consumer, output, runConsumer) => {
|
|
52759
|
+
this.performOnSuccessActions = async (consumer, output, runConsumer, options2) => {
|
|
51813
52760
|
Affirm_default(consumer, "Invalid consumer");
|
|
51814
52761
|
Affirm_default(output, "Invalid output");
|
|
51815
52762
|
if (!output.onSuccess || output.onSuccess.length === 0)
|
|
@@ -51817,6 +52764,8 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51817
52764
|
for (const onSuccess of output.onSuccess) {
|
|
51818
52765
|
switch (onSuccess.action) {
|
|
51819
52766
|
case "move-file": {
|
|
52767
|
+
if (this._skipped(consumer, options2))
|
|
52768
|
+
break;
|
|
51820
52769
|
await this.moveSourceFiles(consumer, onSuccess.moveToDestination);
|
|
51821
52770
|
break;
|
|
51822
52771
|
}
|
|
@@ -51829,6 +52778,12 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51829
52778
|
}
|
|
51830
52779
|
}
|
|
51831
52780
|
};
|
|
52781
|
+
/** Whether this run declines a move-file action, logged so the skip is visible in the run's log. */
|
|
52782
|
+
this._skipped = (consumer, options2) => {
|
|
52783
|
+
if (!options2?.skipFileMoves) return false;
|
|
52784
|
+
Logger_default.info(`Skipping the move-file action of consumer "${consumer.name}": this run did not read its real sources`);
|
|
52785
|
+
return true;
|
|
52786
|
+
};
|
|
51832
52787
|
/**
|
|
51833
52788
|
* Runs the downstream consumer(s) named by a run-consumer action, sequentially in declared order.
|
|
51834
52789
|
* A downstream failure (unknown consumer or a failed run) is logged and swallowed so it does not
|
|
@@ -51947,7 +52902,7 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51947
52902
|
await destinationDriver.copyFromS3(sourceSource.authentication["bucket"], fileKey, fileKey);
|
|
51948
52903
|
await sourceDriver.deleteFile(fileKey);
|
|
51949
52904
|
};
|
|
51950
|
-
this.performOnErrorActions = async (consumer, output, runConsumer) => {
|
|
52905
|
+
this.performOnErrorActions = async (consumer, output, runConsumer, options2) => {
|
|
51951
52906
|
Affirm_default(consumer, "Invalid consumer");
|
|
51952
52907
|
Affirm_default(output, "Invalid output");
|
|
51953
52908
|
if (!output.onError || output.onError.length === 0)
|
|
@@ -51955,6 +52910,8 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51955
52910
|
for (const onError of output.onError) {
|
|
51956
52911
|
switch (onError.action) {
|
|
51957
52912
|
case "move-file": {
|
|
52913
|
+
if (this._skipped(consumer, options2))
|
|
52914
|
+
break;
|
|
51958
52915
|
await this.moveSourceFiles(consumer, onError.moveToDestination);
|
|
51959
52916
|
break;
|
|
51960
52917
|
}
|
|
@@ -56422,6 +57379,8 @@ var init_src10 = __esm({
|
|
|
56422
57379
|
init_DeveloperEngine();
|
|
56423
57380
|
init_LLM();
|
|
56424
57381
|
init_SyntheticInputEngine();
|
|
57382
|
+
init_CopybookParser();
|
|
57383
|
+
init_CopybookEmitter();
|
|
56425
57384
|
init_ConsumerManager();
|
|
56426
57385
|
init_ConsumerOnFinishManager();
|
|
56427
57386
|
init_RequestExecutor();
|
|
@@ -56533,9 +57492,10 @@ var init_DatasetSchema = __esm({
|
|
|
56533
57492
|
return dsSchema.columns.map((column) => column.name);
|
|
56534
57493
|
};
|
|
56535
57494
|
/**
|
|
56536
|
-
* One column, typed from the dimension behind it. `type`, `format` and `nullable` are
|
|
56537
|
-
*
|
|
56538
|
-
*
|
|
57495
|
+
* One column, typed from the dimension behind it. `type`, `format` and `nullable` are what a cast
|
|
57496
|
+
* reads off a staged value, so carrying those lets a reader cast it back without going near the
|
|
57497
|
+
* consumer config. A dimension's `scale` and `sign` deliberately do NOT travel: they say how the
|
|
57498
|
+
* SOURCE spelled the number, and what is staged here is the number itself.
|
|
56539
57499
|
*/
|
|
56540
57500
|
this._column = (field) => {
|
|
56541
57501
|
if (ConsumerManager_default.isLineCountField(field.cField))
|
|
@@ -56662,9 +57622,10 @@ var init_FixedBlockOrder = __esm({
|
|
|
56662
57622
|
});
|
|
56663
57623
|
|
|
56664
57624
|
// ../../packages/executors/src/OutputExecutor.ts
|
|
56665
|
-
var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
|
|
57625
|
+
var import_fs21, OutputExecutorClass, OutputExecutor, OutputExecutor_default;
|
|
56666
57626
|
var init_OutputExecutor = __esm({
|
|
56667
57627
|
"../../packages/executors/src/OutputExecutor.ts"() {
|
|
57628
|
+
import_fs21 = __toESM(require("fs"));
|
|
56668
57629
|
init_src();
|
|
56669
57630
|
init_src();
|
|
56670
57631
|
init_src8();
|
|
@@ -56752,17 +57713,17 @@ var init_OutputExecutor = __esm({
|
|
|
56752
57713
|
* One output per consumer (`maxItems: 1` in consumer-schema.json). This used to loop, but only
|
|
56753
57714
|
* ever shipped the first entry, and could not have done more: `move` unlinks what it ships.
|
|
56754
57715
|
*/
|
|
56755
|
-
this.exportResult = async (consumer, dsSchema, splitFiles) => {
|
|
57716
|
+
this.exportResult = async (consumer, dsSchema, splitFiles, localPath) => {
|
|
56756
57717
|
Affirm_default.hasItems(splitFiles, `Consumer "${consumer.name}" has no output file to export`);
|
|
56757
57718
|
Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output to export to`);
|
|
56758
57719
|
Affirm_default(dsSchema, `Consumer "${consumer.name}" has no dataset schema to export against`);
|
|
56759
57720
|
const output = consumer.outputs[0];
|
|
56760
57721
|
const internalFormat = dsSchema.encoding;
|
|
56761
|
-
const destination =
|
|
56762
|
-
Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
|
|
57722
|
+
const destination = this._resolveDestination(consumer, output, localPath);
|
|
56763
57723
|
const driver = await DriverFactory_default.instantiateDestination(destination);
|
|
56764
57724
|
const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
|
|
56765
|
-
|
|
57725
|
+
const target = localPath ?? output.exportDestination;
|
|
57726
|
+
Logger_default.info(`Exporting consumer "${consumer.name}" to "${target}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
|
|
56766
57727
|
const header = this._composeHeader(output, dsSchema);
|
|
56767
57728
|
const fixedLayouts = output.format === "FIXED" && output.layouts ? FixedWidthWriter_default.compileLayouts(dsSchema, this._fixedPositions(consumer), output, consumer.name) : null;
|
|
56768
57729
|
const fixedLine = output.format === "FIXED" ? this._fixedLineWriter(consumer, dsSchema, output, fixedLayouts) : null;
|
|
@@ -56925,6 +57886,25 @@ var init_OutputExecutor = __esm({
|
|
|
56925
57886
|
return "txt";
|
|
56926
57887
|
}
|
|
56927
57888
|
};
|
|
57889
|
+
/**
|
|
57890
|
+
* Where this export writes: the consumer's configured destination, or — in synth mode — a local
|
|
57891
|
+
* folder standing in for it, as a `local` source built on the spot.
|
|
57892
|
+
*
|
|
57893
|
+
* The real destination is never even resolved for a synthetic run, which is what lets a consumer
|
|
57894
|
+
* pointed at a production bucket be synthesized from a machine that holds no credentials for it.
|
|
57895
|
+
*
|
|
57896
|
+
* Named here, not left to the driver: an unresolvable destination used to surface as a TypeError
|
|
57897
|
+
* three layers down, naming neither the consumer nor the destination.
|
|
57898
|
+
*/
|
|
57899
|
+
this._resolveDestination = (consumer, output, localPath) => {
|
|
57900
|
+
if (localPath) {
|
|
57901
|
+
import_fs21.default.mkdirSync(localPath, { recursive: true });
|
|
57902
|
+
return { name: localPath, engine: "local", authentication: { method: "none", path: localPath }, _version: 1 };
|
|
57903
|
+
}
|
|
57904
|
+
const destination = Environment_default.getSource(output.exportDestination);
|
|
57905
|
+
Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
|
|
57906
|
+
return destination;
|
|
57907
|
+
};
|
|
56928
57908
|
this._composeFileName = (consumer, output, extension, executionId) => {
|
|
56929
57909
|
if (output.exportName && output.exportName.trim().length > 0) {
|
|
56930
57910
|
const sanitized = output.exportName.replace(/\.[^.]+$/, "");
|
|
@@ -56942,11 +57922,11 @@ var init_OutputExecutor = __esm({
|
|
|
56942
57922
|
});
|
|
56943
57923
|
|
|
56944
57924
|
// ../../packages/executors/src/ConsumerExecutor.ts
|
|
56945
|
-
var import_path19,
|
|
57925
|
+
var import_path19, import_fs22, import_promises16, import_crypto10, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
|
|
56946
57926
|
var init_ConsumerExecutor = __esm({
|
|
56947
57927
|
"../../packages/executors/src/ConsumerExecutor.ts"() {
|
|
56948
57928
|
import_path19 = __toESM(require("path"));
|
|
56949
|
-
|
|
57929
|
+
import_fs22 = __toESM(require("fs"));
|
|
56950
57930
|
import_promises16 = __toESM(require("fs/promises"));
|
|
56951
57931
|
import_crypto10 = __toESM(require("crypto"));
|
|
56952
57932
|
init_src10();
|
|
@@ -56963,10 +57943,10 @@ var init_ConsumerExecutor = __esm({
|
|
|
56963
57943
|
constructor() {
|
|
56964
57944
|
this._ensurePath = (pathUri) => {
|
|
56965
57945
|
const dir = import_path19.default.dirname(pathUri);
|
|
56966
|
-
if (!
|
|
56967
|
-
|
|
56968
|
-
if (!
|
|
56969
|
-
|
|
57946
|
+
if (!import_fs22.default.existsSync(dir))
|
|
57947
|
+
import_fs22.default.mkdirSync(dir, { recursive: true });
|
|
57948
|
+
if (!import_fs22.default.existsSync(pathUri))
|
|
57949
|
+
import_fs22.default.writeFileSync(pathUri, "");
|
|
56970
57950
|
};
|
|
56971
57951
|
/**
|
|
56972
57952
|
* Shapes one parsed producer record into one consumer record, following a plan compiled once per
|
|
@@ -57094,8 +58074,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
57094
58074
|
const parseLine = LineParser_default.datasetParser(dsSchema);
|
|
57095
58075
|
const outputPlan = OutputExecutor_default.compileOutput(consumer, dsSchema);
|
|
57096
58076
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57097
|
-
const reader =
|
|
57098
|
-
const writer =
|
|
58077
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
58078
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57099
58079
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57100
58080
|
let workerIndex = 0;
|
|
57101
58081
|
let currentWorker = executorResults[workerIndex] ?? null;
|
|
@@ -57186,9 +58166,9 @@ var init_ConsumerExecutor = __esm({
|
|
|
57186
58166
|
};
|
|
57187
58167
|
this.processDistinct = async (dsSchema, datasetPath) => {
|
|
57188
58168
|
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
57189
|
-
const reader =
|
|
58169
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57190
58170
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57191
|
-
const writer =
|
|
58171
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57192
58172
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57193
58173
|
let newLineCount = 0;
|
|
57194
58174
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -57218,7 +58198,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57218
58198
|
};
|
|
57219
58199
|
this.processDistinctOn = async (consumer, dsSchema, datasetPath) => {
|
|
57220
58200
|
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
57221
|
-
const reader =
|
|
58201
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57222
58202
|
const { distinctOn } = consumer.options;
|
|
57223
58203
|
const { keys, resolution } = distinctOn;
|
|
57224
58204
|
const { strategy, orderBy, direction = "asc" } = resolution;
|
|
@@ -57238,7 +58218,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57238
58218
|
}
|
|
57239
58219
|
}
|
|
57240
58220
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57241
|
-
const writer =
|
|
58221
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57242
58222
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57243
58223
|
for (const { line } of winners.values()) {
|
|
57244
58224
|
if (!writer.write(line + "\n"))
|
|
@@ -57275,7 +58255,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57275
58255
|
if (!pivotValues) {
|
|
57276
58256
|
pivotValues = [];
|
|
57277
58257
|
const discoverySet = /* @__PURE__ */ new Set();
|
|
57278
|
-
const discoverReader =
|
|
58258
|
+
const discoverReader = import_fs22.default.createReadStream(datasetPath);
|
|
57279
58259
|
for await (const line of readLines(discoverReader)) {
|
|
57280
58260
|
const record = parseLine(line);
|
|
57281
58261
|
const val = String(record[pivotColumn] ?? "");
|
|
@@ -57292,7 +58272,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57292
58272
|
}
|
|
57293
58273
|
}
|
|
57294
58274
|
const groups = /* @__PURE__ */ new Map();
|
|
57295
|
-
const reader =
|
|
58275
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57296
58276
|
for await (const line of readLines(reader)) {
|
|
57297
58277
|
const record = parseLine(line);
|
|
57298
58278
|
const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
|
|
@@ -57316,7 +58296,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57316
58296
|
const pivotedDsSchema = DatasetSchema_default.forPivot(dsSchema, rowKeys, pivotValues.map((pv) => columnPrefix + pv), groups.size);
|
|
57317
58297
|
const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedDsSchema);
|
|
57318
58298
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57319
|
-
const writer =
|
|
58299
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57320
58300
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57321
58301
|
let outputCount = 0;
|
|
57322
58302
|
for (const { rowRecord, cells } of groups.values()) {
|
|
@@ -57407,7 +58387,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57407
58387
|
for (const fieldKey of uniqueFieldKeys) {
|
|
57408
58388
|
fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
|
|
57409
58389
|
}
|
|
57410
|
-
const reader =
|
|
58390
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57411
58391
|
for await (const line of readLines(reader)) {
|
|
57412
58392
|
rowCount++;
|
|
57413
58393
|
if (checkDuplicateRows) {
|
|
@@ -57554,10 +58534,10 @@ var init_ExecutorPerformance = __esm({
|
|
|
57554
58534
|
});
|
|
57555
58535
|
|
|
57556
58536
|
// ../../packages/executors/src/ExecutorWriter.ts
|
|
57557
|
-
var
|
|
58537
|
+
var fs29, ExecutorWriter, ExecutorWriter_default;
|
|
57558
58538
|
var init_ExecutorWriter = __esm({
|
|
57559
58539
|
"../../packages/executors/src/ExecutorWriter.ts"() {
|
|
57560
|
-
|
|
58540
|
+
fs29 = __toESM(require("fs"));
|
|
57561
58541
|
init_src();
|
|
57562
58542
|
init_src3();
|
|
57563
58543
|
init_src10();
|
|
@@ -57609,7 +58589,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57609
58589
|
await this._route(pass, dsSchema);
|
|
57610
58590
|
if (pass.buckets.size === 0)
|
|
57611
58591
|
return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
|
|
57612
|
-
await
|
|
58592
|
+
await fs29.promises.unlink(sourcePath);
|
|
57613
58593
|
return this._listFiles(pass);
|
|
57614
58594
|
};
|
|
57615
58595
|
/**
|
|
@@ -57617,7 +58597,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57617
58597
|
*/
|
|
57618
58598
|
this._route = async (pass, dsSchema) => {
|
|
57619
58599
|
const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
|
|
57620
|
-
const readStream =
|
|
58600
|
+
const readStream = fs29.createReadStream(pass.sourcePath);
|
|
57621
58601
|
try {
|
|
57622
58602
|
for await (const line of readLines(readStream)) {
|
|
57623
58603
|
const bucket = this._bucketFor(line, pass, parseLine);
|
|
@@ -57721,7 +58701,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57721
58701
|
}
|
|
57722
58702
|
if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
|
|
57723
58703
|
await this._closeStream(pass.openStreams.shift());
|
|
57724
|
-
bucket.stream =
|
|
58704
|
+
bucket.stream = fs29.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
|
|
57725
58705
|
pass.openStreams.push(bucket);
|
|
57726
58706
|
return bucket.stream;
|
|
57727
58707
|
};
|
|
@@ -57999,10 +58979,10 @@ var init_OrchestratorHelper = __esm({
|
|
|
57999
58979
|
});
|
|
58000
58980
|
|
|
58001
58981
|
// ../../packages/executors/src/LookupResolver.ts
|
|
58002
|
-
var
|
|
58982
|
+
var import_fs24, import_promises17, import_path22, LookupResolverClass, LookupResolver, LookupResolver_default;
|
|
58003
58983
|
var init_LookupResolver = __esm({
|
|
58004
58984
|
"../../packages/executors/src/LookupResolver.ts"() {
|
|
58005
|
-
|
|
58985
|
+
import_fs24 = __toESM(require("fs"));
|
|
58006
58986
|
import_promises17 = __toESM(require("fs/promises"));
|
|
58007
58987
|
import_path22 = __toESM(require("path"));
|
|
58008
58988
|
init_src();
|
|
@@ -58080,7 +59060,7 @@ var init_LookupResolver = __esm({
|
|
|
58080
59060
|
const dict = {};
|
|
58081
59061
|
let rowsRead = 0;
|
|
58082
59062
|
let warnedDup = false;
|
|
58083
|
-
for await (const line of readLines(
|
|
59063
|
+
for await (const line of readLines(import_fs24.default.createReadStream(capture.path))) {
|
|
58084
59064
|
if (!line.length) continue;
|
|
58085
59065
|
rowsRead++;
|
|
58086
59066
|
Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
|
|
@@ -58119,10 +59099,10 @@ var init_LookupResolver = __esm({
|
|
|
58119
59099
|
});
|
|
58120
59100
|
|
|
58121
59101
|
// ../../packages/executors/src/ConsumerInputResolver.ts
|
|
58122
|
-
var
|
|
59102
|
+
var import_fs25, import_promises18, import_path23, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
|
|
58123
59103
|
var init_ConsumerInputResolver = __esm({
|
|
58124
59104
|
"../../packages/executors/src/ConsumerInputResolver.ts"() {
|
|
58125
|
-
|
|
59105
|
+
import_fs25 = __toESM(require("fs"));
|
|
58126
59106
|
import_promises18 = __toESM(require("fs/promises"));
|
|
58127
59107
|
import_path23 = __toESM(require("path"));
|
|
58128
59108
|
init_src();
|
|
@@ -58146,7 +59126,7 @@ var init_ConsumerInputResolver = __esm({
|
|
|
58146
59126
|
Affirm_default(stack.length <= Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH, `consumer-as-input nesting exceeds MAX_CONSUMER_INPUT_DEPTH (${Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH}): ${stack.join(" -> ")}`);
|
|
58147
59127
|
for (const name of upstreamNames) {
|
|
58148
59128
|
const cached = this.cache.get(name);
|
|
58149
|
-
if (!cached || !
|
|
59129
|
+
if (!cached || !import_fs25.default.existsSync(cached.path)) {
|
|
58150
59130
|
Affirm_default(!stack.includes(name), `consumer-as-input cycle detected: ${[...stack, name].join(" -> ")}`);
|
|
58151
59131
|
const upstream = Environment_default.getConsumer(name);
|
|
58152
59132
|
Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
|
|
@@ -58225,10 +59205,10 @@ var ExecutorOrchestrator_exports = {};
|
|
|
58225
59205
|
__export(ExecutorOrchestrator_exports, {
|
|
58226
59206
|
default: () => ExecutorOrchestrator_default
|
|
58227
59207
|
});
|
|
58228
|
-
var
|
|
59208
|
+
var import_fs26, import_promises19, import_path24, import_workerpool, import_promises20, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
|
|
58229
59209
|
var init_ExecutorOrchestrator = __esm({
|
|
58230
59210
|
"../../packages/executors/src/ExecutorOrchestrator.ts"() {
|
|
58231
|
-
|
|
59211
|
+
import_fs26 = __toESM(require("fs"));
|
|
58232
59212
|
import_promises19 = __toESM(require("fs/promises"));
|
|
58233
59213
|
import_path24 = __toESM(require("path"));
|
|
58234
59214
|
import_workerpool = __toESM(require("workerpool"));
|
|
@@ -58417,15 +59397,17 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58417
59397
|
_progress.update({ phase: "Exporting result", progress: 0 });
|
|
58418
59398
|
counter = performance.now();
|
|
58419
59399
|
Logger_default.info(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
|
|
58420
|
-
const
|
|
59400
|
+
const localExportPath = request.syntheticExportPath ? import_path24.default.join(request.syntheticExportPath, consumer.name) : void 0;
|
|
59401
|
+
const exportRes = await OutputExecutor_default.exportResult(consumer, dsSchema, splitFiles, localExportPath);
|
|
58421
59402
|
tracker.measure("export-result", performance.now() - counter);
|
|
58422
|
-
|
|
59403
|
+
const writtenTo = localExportPath ?? consumer.outputs.map((x2) => x2.exportDestination).join(", ");
|
|
59404
|
+
Affirm_default(exportRes.res, `Consumer "${consumer.name}" failed to write its output to "${writtenTo}" \u2014 see the logged driver error for the cause`);
|
|
58423
59405
|
Logger_default.info(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
|
|
58424
59406
|
_progress.update({ phase: "Exporting result", progress: 1 });
|
|
58425
59407
|
if (consumer.outputs.some((x2) => x2.onSuccess)) {
|
|
58426
59408
|
Logger_default.log(`[${usageId}] Running on-success actions`);
|
|
58427
59409
|
counter = performance.now();
|
|
58428
|
-
await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
|
|
59410
|
+
await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId), { skipFileMoves: !!request.syntheticExportPath });
|
|
58429
59411
|
tracker.measure("on-success-actions", performance.now() - counter);
|
|
58430
59412
|
Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
|
|
58431
59413
|
}
|
|
@@ -58444,7 +59426,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58444
59426
|
Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
|
|
58445
59427
|
Logger_default.error(error);
|
|
58446
59428
|
if (!request.skipExport)
|
|
58447
|
-
await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
|
|
59429
|
+
await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId), { skipFileMoves: !!request.syntheticExportPath });
|
|
58448
59430
|
Logger_default.log(`[${usageId}] Running cleanup after failure`);
|
|
58449
59431
|
await this.performCleanupOperations(scope, tracker);
|
|
58450
59432
|
UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
|
|
@@ -58465,7 +59447,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58465
59447
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": source returned no files, skipping`);
|
|
58466
59448
|
continue;
|
|
58467
59449
|
}
|
|
58468
|
-
if (!
|
|
59450
|
+
if (!import_fs26.default.existsSync(response.files[0].fullUri)) {
|
|
58469
59451
|
if (!cProd.isOptional)
|
|
58470
59452
|
throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
|
|
58471
59453
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
|
|
@@ -58620,7 +59602,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58620
59602
|
details: { invokedBy: "ON_FINISH", user: request.details.user },
|
|
58621
59603
|
logProgress: request.logProgress,
|
|
58622
59604
|
_onFinishStack: chain2,
|
|
58623
|
-
syntheticProducerInputs: request.syntheticProducerInputs
|
|
59605
|
+
syntheticProducerInputs: request.syntheticProducerInputs,
|
|
59606
|
+
syntheticExportPath: request.syntheticExportPath
|
|
58624
59607
|
});
|
|
58625
59608
|
};
|
|
58626
59609
|
};
|
|
@@ -58630,7 +59613,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58630
59613
|
* Returns a single chunk for small files where parallelism overhead isn't worth it.
|
|
58631
59614
|
*/
|
|
58632
59615
|
this.scopeWork = (fileUri, numChunks) => {
|
|
58633
|
-
const fileSize =
|
|
59616
|
+
const fileSize = import_fs26.default.statSync(fileUri).size;
|
|
58634
59617
|
if (fileSize === 0) return [];
|
|
58635
59618
|
if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
|
|
58636
59619
|
return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
@@ -58640,7 +59623,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58640
59623
|
const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
|
|
58641
59624
|
if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
58642
59625
|
const targetChunkSize = Math.floor(fileSize / effectiveChunks);
|
|
58643
|
-
const fd =
|
|
59626
|
+
const fd = import_fs26.default.openSync(fileUri, "r");
|
|
58644
59627
|
try {
|
|
58645
59628
|
const offsets = [];
|
|
58646
59629
|
let currentStart = 0;
|
|
@@ -58658,7 +59641,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58658
59641
|
}
|
|
58659
59642
|
return offsets;
|
|
58660
59643
|
} finally {
|
|
58661
|
-
|
|
59644
|
+
import_fs26.default.closeSync(fd);
|
|
58662
59645
|
}
|
|
58663
59646
|
};
|
|
58664
59647
|
/**
|
|
@@ -58726,7 +59709,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58726
59709
|
let currentPos = position;
|
|
58727
59710
|
while (currentPos < fileSize) {
|
|
58728
59711
|
const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
|
|
58729
|
-
const bytesRead =
|
|
59712
|
+
const bytesRead = import_fs26.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
|
|
58730
59713
|
if (bytesRead === 0) break;
|
|
58731
59714
|
for (let i6 = 0; i6 < bytesRead; i6++) {
|
|
58732
59715
|
if (buffer[i6] === 10) {
|
|
@@ -58751,9 +59734,9 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58751
59734
|
return;
|
|
58752
59735
|
}
|
|
58753
59736
|
const parseLine = LineParser_default.datasetParser(dsSchema);
|
|
58754
|
-
const output =
|
|
59737
|
+
const output = import_fs26.default.createWriteStream(targetPath);
|
|
58755
59738
|
try {
|
|
58756
|
-
for await (const line of readLines(
|
|
59739
|
+
for await (const line of readLines(import_fs26.default.createReadStream(sourcePath))) {
|
|
58757
59740
|
if (!line.length) continue;
|
|
58758
59741
|
const record = parseLine(line);
|
|
58759
59742
|
if (!output.write(JSON.stringify(record) + "\n"))
|
|
@@ -58851,7 +59834,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58851
59834
|
});
|
|
58852
59835
|
await (0, import_promises20.pipeline)(
|
|
58853
59836
|
xlsCsvStream,
|
|
58854
|
-
|
|
59837
|
+
import_fs26.default.createWriteStream(decodedPath)
|
|
58855
59838
|
);
|
|
58856
59839
|
const fileStats = await import_promises19.default.stat(decodedPath);
|
|
58857
59840
|
decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
|
|
@@ -58911,8 +59894,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58911
59894
|
const perf = performance.now();
|
|
58912
59895
|
for (const workerResult of executorResults) {
|
|
58913
59896
|
await (0, import_promises20.pipeline)(
|
|
58914
|
-
|
|
58915
|
-
|
|
59897
|
+
import_fs26.default.createReadStream(workerResult.resultUri),
|
|
59898
|
+
import_fs26.default.createWriteStream(mainPath, { flags: "a" })
|
|
58916
59899
|
);
|
|
58917
59900
|
await import_promises19.default.unlink(workerResult.resultUri);
|
|
58918
59901
|
}
|
|
@@ -58994,7 +59977,7 @@ init_ConsumerExecutor();
|
|
|
58994
59977
|
init_DatasetSchema();
|
|
58995
59978
|
|
|
58996
59979
|
// ../../packages/executors/src/Executor.ts
|
|
58997
|
-
var
|
|
59980
|
+
var import_fs23 = __toESM(require("fs"));
|
|
58998
59981
|
init_ProducerExecutor();
|
|
58999
59982
|
init_ConsumerExecutor();
|
|
59000
59983
|
|
|
@@ -59288,18 +60271,18 @@ var Executor = class {
|
|
|
59288
60271
|
if (!lookups?.length) return;
|
|
59289
60272
|
for (const entry of lookups) {
|
|
59290
60273
|
if (LookupRegistry_default.has(entry.id)) continue;
|
|
59291
|
-
const table = JSON.parse(
|
|
60274
|
+
const table = JSON.parse(import_fs23.default.readFileSync(entry.path, "utf-8"));
|
|
59292
60275
|
LookupRegistry_default.set(entry.id, table);
|
|
59293
60276
|
Logger_default.log(`[lookup] Hydrated table "${entry.consumer}" (${entry.key}->${entry.value}) from ${entry.path}`);
|
|
59294
60277
|
}
|
|
59295
60278
|
};
|
|
59296
60279
|
this.openReadStream = (chunk) => {
|
|
59297
60280
|
const { end, fileUri, start } = chunk;
|
|
59298
|
-
return
|
|
60281
|
+
return import_fs23.default.createReadStream(fileUri, { start, end: end - 1 });
|
|
59299
60282
|
};
|
|
59300
60283
|
this.openWriteStream = (scope, workerId) => {
|
|
59301
60284
|
const workerPath = ExecutorScope_default.getWorkerPath(scope, workerId);
|
|
59302
|
-
return new BatchWriter_default(
|
|
60285
|
+
return new BatchWriter_default(import_fs23.default.createWriteStream(workerPath));
|
|
59303
60286
|
};
|
|
59304
60287
|
// The first line of the first chunk is data unless the file type carries a header there.
|
|
59305
60288
|
this.shouldProcessFirstLine = (producer) => {
|
|
@@ -59414,7 +60397,7 @@ init_OutputExecutor();
|
|
|
59414
60397
|
init_ProducerExecutor();
|
|
59415
60398
|
|
|
59416
60399
|
// ../../packages/executors/src/SampleRunner.ts
|
|
59417
|
-
var
|
|
60400
|
+
var import_fs27 = __toESM(require("fs"));
|
|
59418
60401
|
var import_promises21 = __toESM(require("fs/promises"));
|
|
59419
60402
|
var import_path25 = __toESM(require("path"));
|
|
59420
60403
|
init_src();
|
|
@@ -59528,9 +60511,9 @@ var SampleRunnerClass = class {
|
|
|
59528
60511
|
};
|
|
59529
60512
|
/** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
|
|
59530
60513
|
this._readCapture = async (capturePath, size) => {
|
|
59531
|
-
if (!
|
|
60514
|
+
if (!import_fs27.default.existsSync(capturePath)) return [];
|
|
59532
60515
|
const records = [];
|
|
59533
|
-
for await (const line of readLines(
|
|
60516
|
+
for await (const line of readLines(import_fs27.default.createReadStream(capturePath))) {
|
|
59534
60517
|
if (!line.length) continue;
|
|
59535
60518
|
records.push(JSON.parse(line));
|
|
59536
60519
|
if (records.length >= size) break;
|