@forzalabs/remora 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENCE.md +52 -17
- package/README.md +11 -1
- package/RELEASE_NOTES.md +192 -0
- package/docs/Remora - Code Sets.md +22 -1
- package/docs/Remora - Consumers.md +27 -2
- package/docs/Remora - Producers.md +160 -5
- package/documentation/README.md +2 -1
- package/index.js +1491 -411
- package/json_schemas/consumer-schema.json +12 -1
- package/json_schemas/producer-schema.json +22 -3
- package/package.json +2 -2
- package/workers/ExecutorWorker.js +1121 -138
- package/CHANGELOG.md +0 -557
package/index.js
CHANGED
|
@@ -7852,15 +7852,15 @@ var require_index_cjs = __commonJS({
|
|
|
7852
7852
|
};
|
|
7853
7853
|
}
|
|
7854
7854
|
function wrapConversion(toModel, graph2) {
|
|
7855
|
-
const
|
|
7855
|
+
const path32 = [graph2[toModel].parent, toModel];
|
|
7856
7856
|
let fn = convert$1[graph2[toModel].parent][toModel];
|
|
7857
7857
|
let cur = graph2[toModel].parent;
|
|
7858
7858
|
while (graph2[cur].parent) {
|
|
7859
|
-
|
|
7859
|
+
path32.unshift(graph2[cur].parent);
|
|
7860
7860
|
fn = link(convert$1[graph2[cur].parent][cur], fn);
|
|
7861
7861
|
cur = graph2[cur].parent;
|
|
7862
7862
|
}
|
|
7863
|
-
fn.conversion =
|
|
7863
|
+
fn.conversion = path32;
|
|
7864
7864
|
return fn;
|
|
7865
7865
|
}
|
|
7866
7866
|
function route(fromModel) {
|
|
@@ -8490,7 +8490,7 @@ var require_node2 = __commonJS({
|
|
|
8490
8490
|
var require_tail_file = __commonJS({
|
|
8491
8491
|
"../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
|
|
8492
8492
|
"use strict";
|
|
8493
|
-
var
|
|
8493
|
+
var fs43 = require("fs");
|
|
8494
8494
|
var { StringDecoder } = require("string_decoder");
|
|
8495
8495
|
var { Stream } = require_readable();
|
|
8496
8496
|
function noop() {
|
|
@@ -8511,7 +8511,7 @@ var require_tail_file = __commonJS({
|
|
|
8511
8511
|
stream.emit("end");
|
|
8512
8512
|
stream.emit("close");
|
|
8513
8513
|
};
|
|
8514
|
-
|
|
8514
|
+
fs43.open(options2.file, "a+", "0644", (err2, fd) => {
|
|
8515
8515
|
if (err2) {
|
|
8516
8516
|
if (!iter) {
|
|
8517
8517
|
stream.emit("error", err2);
|
|
@@ -8523,10 +8523,10 @@ var require_tail_file = __commonJS({
|
|
|
8523
8523
|
}
|
|
8524
8524
|
(function read() {
|
|
8525
8525
|
if (stream.destroyed) {
|
|
8526
|
-
|
|
8526
|
+
fs43.close(fd, noop);
|
|
8527
8527
|
return;
|
|
8528
8528
|
}
|
|
8529
|
-
return
|
|
8529
|
+
return fs43.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
|
|
8530
8530
|
if (error) {
|
|
8531
8531
|
if (!iter) {
|
|
8532
8532
|
stream.emit("error", error);
|
|
@@ -8585,8 +8585,8 @@ var require_tail_file = __commonJS({
|
|
|
8585
8585
|
var require_file = __commonJS({
|
|
8586
8586
|
"../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
|
|
8587
8587
|
"use strict";
|
|
8588
|
-
var
|
|
8589
|
-
var
|
|
8588
|
+
var fs43 = require("fs");
|
|
8589
|
+
var path32 = require("path");
|
|
8590
8590
|
var asyncSeries = require_series();
|
|
8591
8591
|
var zlib3 = require("zlib");
|
|
8592
8592
|
var { MESSAGE } = require_triple_beam();
|
|
@@ -8616,14 +8616,14 @@ var require_file = __commonJS({
|
|
|
8616
8616
|
this._onError = this._onError.bind(this);
|
|
8617
8617
|
if (options2.filename || options2.dirname) {
|
|
8618
8618
|
throwIf("filename or dirname", "stream");
|
|
8619
|
-
this._basename = this.filename = options2.filename ?
|
|
8620
|
-
this.dirname = options2.dirname ||
|
|
8619
|
+
this._basename = this.filename = options2.filename ? path32.basename(options2.filename) : "winston.log";
|
|
8620
|
+
this.dirname = options2.dirname || path32.dirname(options2.filename);
|
|
8621
8621
|
this.options = options2.options || { flags: "a" };
|
|
8622
8622
|
} else if (options2.stream) {
|
|
8623
8623
|
console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
|
|
8624
8624
|
throwIf("stream", "filename", "maxsize");
|
|
8625
8625
|
this._dest = this._stream.pipe(this._setupStream(options2.stream));
|
|
8626
|
-
this.dirname =
|
|
8626
|
+
this.dirname = path32.dirname(this._dest.path);
|
|
8627
8627
|
} else {
|
|
8628
8628
|
throw new Error("Cannot log to file without filename or stream.");
|
|
8629
8629
|
}
|
|
@@ -8786,11 +8786,11 @@ var require_file = __commonJS({
|
|
|
8786
8786
|
options2 = {};
|
|
8787
8787
|
}
|
|
8788
8788
|
options2 = normalizeQuery(options2);
|
|
8789
|
-
const file =
|
|
8789
|
+
const file = path32.join(this.dirname, this.filename);
|
|
8790
8790
|
let buff = "";
|
|
8791
8791
|
let results = [];
|
|
8792
8792
|
let row = 0;
|
|
8793
|
-
const stream =
|
|
8793
|
+
const stream = fs43.createReadStream(file, {
|
|
8794
8794
|
encoding: "utf8"
|
|
8795
8795
|
});
|
|
8796
8796
|
stream.on("error", (err2) => {
|
|
@@ -8826,7 +8826,7 @@ var require_file = __commonJS({
|
|
|
8826
8826
|
function add(buff2, attempt) {
|
|
8827
8827
|
try {
|
|
8828
8828
|
const log = JSON.parse(buff2);
|
|
8829
|
-
if (
|
|
8829
|
+
if (check(log)) {
|
|
8830
8830
|
push(log);
|
|
8831
8831
|
}
|
|
8832
8832
|
} catch (e7) {
|
|
@@ -8855,7 +8855,7 @@ var require_file = __commonJS({
|
|
|
8855
8855
|
}
|
|
8856
8856
|
results.push(log);
|
|
8857
8857
|
}
|
|
8858
|
-
function
|
|
8858
|
+
function check(log) {
|
|
8859
8859
|
if (!log) {
|
|
8860
8860
|
return;
|
|
8861
8861
|
}
|
|
@@ -8891,7 +8891,7 @@ var require_file = __commonJS({
|
|
|
8891
8891
|
* TODO: Refactor me.
|
|
8892
8892
|
*/
|
|
8893
8893
|
stream(options2 = {}) {
|
|
8894
|
-
const file =
|
|
8894
|
+
const file = path32.join(this.dirname, this.filename);
|
|
8895
8895
|
const stream = new Stream();
|
|
8896
8896
|
const tail = {
|
|
8897
8897
|
file,
|
|
@@ -8941,8 +8941,8 @@ var require_file = __commonJS({
|
|
|
8941
8941
|
*/
|
|
8942
8942
|
stat(callback) {
|
|
8943
8943
|
const target = this._getFile();
|
|
8944
|
-
const fullpath =
|
|
8945
|
-
|
|
8944
|
+
const fullpath = path32.join(this.dirname, target);
|
|
8945
|
+
fs43.stat(fullpath, (err2, stat) => {
|
|
8946
8946
|
if (err2 && err2.code === "ENOENT") {
|
|
8947
8947
|
debug("ENOENT\xA0ok", fullpath);
|
|
8948
8948
|
this.filename = target;
|
|
@@ -9045,9 +9045,9 @@ var require_file = __commonJS({
|
|
|
9045
9045
|
* @returns {WritableStream} Stream that writes to disk for the active file.
|
|
9046
9046
|
*/
|
|
9047
9047
|
_createStream(source) {
|
|
9048
|
-
const fullpath =
|
|
9048
|
+
const fullpath = path32.join(this.dirname, this.filename);
|
|
9049
9049
|
debug("create stream start", fullpath, this.options);
|
|
9050
|
-
const dest =
|
|
9050
|
+
const dest = fs43.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
|
|
9051
9051
|
debug("file open ok", fullpath);
|
|
9052
9052
|
this.emit("open", fullpath);
|
|
9053
9053
|
source.pipe(dest);
|
|
@@ -9070,16 +9070,16 @@ var require_file = __commonJS({
|
|
|
9070
9070
|
*/
|
|
9071
9071
|
_incFile(callback) {
|
|
9072
9072
|
debug("_incFile", this.filename);
|
|
9073
|
-
const ext =
|
|
9074
|
-
const basename =
|
|
9073
|
+
const ext = path32.extname(this._basename);
|
|
9074
|
+
const basename = path32.basename(this._basename, ext);
|
|
9075
9075
|
const tasks = [];
|
|
9076
9076
|
if (this.zippedArchive) {
|
|
9077
9077
|
tasks.push(
|
|
9078
9078
|
function(cb) {
|
|
9079
9079
|
const num = this._created > 0 && !this.tailable ? this._created : "";
|
|
9080
9080
|
this._compressFile(
|
|
9081
|
-
|
|
9082
|
-
|
|
9081
|
+
path32.join(this.dirname, `${basename}${num}${ext}`),
|
|
9082
|
+
path32.join(this.dirname, `${basename}${num}${ext}.gz`),
|
|
9083
9083
|
cb
|
|
9084
9084
|
);
|
|
9085
9085
|
}.bind(this)
|
|
@@ -9104,8 +9104,8 @@ var require_file = __commonJS({
|
|
|
9104
9104
|
* @private
|
|
9105
9105
|
*/
|
|
9106
9106
|
_getFile() {
|
|
9107
|
-
const ext =
|
|
9108
|
-
const basename =
|
|
9107
|
+
const ext = path32.extname(this._basename);
|
|
9108
|
+
const basename = path32.basename(this._basename, ext);
|
|
9109
9109
|
const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
|
|
9110
9110
|
return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
|
|
9111
9111
|
}
|
|
@@ -9125,8 +9125,8 @@ var require_file = __commonJS({
|
|
|
9125
9125
|
const isOldest = oldest !== 0 ? oldest : "";
|
|
9126
9126
|
const isZipped = this.zippedArchive ? ".gz" : "";
|
|
9127
9127
|
const filePath = `${basename}${isOldest}${ext}${isZipped}`;
|
|
9128
|
-
const target =
|
|
9129
|
-
|
|
9128
|
+
const target = path32.join(this.dirname, filePath);
|
|
9129
|
+
fs43.unlink(target, callback);
|
|
9130
9130
|
}
|
|
9131
9131
|
/**
|
|
9132
9132
|
* Roll files forward based on integer, up to maxFiles. e.g. if base if
|
|
@@ -9148,20 +9148,20 @@ var require_file = __commonJS({
|
|
|
9148
9148
|
for (let x2 = this.maxFiles - 1; x2 > 1; x2--) {
|
|
9149
9149
|
tasks.push(function(i6, cb) {
|
|
9150
9150
|
let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
|
|
9151
|
-
const tmppath =
|
|
9152
|
-
|
|
9151
|
+
const tmppath = path32.join(this.dirname, fileName);
|
|
9152
|
+
fs43.exists(tmppath, (exists) => {
|
|
9153
9153
|
if (!exists) {
|
|
9154
9154
|
return cb(null);
|
|
9155
9155
|
}
|
|
9156
9156
|
fileName = `${basename}${i6}${ext}${isZipped}`;
|
|
9157
|
-
|
|
9157
|
+
fs43.rename(tmppath, path32.join(this.dirname, fileName), cb);
|
|
9158
9158
|
});
|
|
9159
9159
|
}.bind(this, x2));
|
|
9160
9160
|
}
|
|
9161
9161
|
asyncSeries(tasks, () => {
|
|
9162
|
-
|
|
9163
|
-
|
|
9164
|
-
|
|
9162
|
+
fs43.rename(
|
|
9163
|
+
path32.join(this.dirname, `${basename}${ext}${isZipped}`),
|
|
9164
|
+
path32.join(this.dirname, `${basename}1${ext}${isZipped}`),
|
|
9165
9165
|
callback
|
|
9166
9166
|
);
|
|
9167
9167
|
});
|
|
@@ -9175,22 +9175,22 @@ var require_file = __commonJS({
|
|
|
9175
9175
|
* @private
|
|
9176
9176
|
*/
|
|
9177
9177
|
_compressFile(src, dest, callback) {
|
|
9178
|
-
|
|
9178
|
+
fs43.access(src, fs43.F_OK, (err2) => {
|
|
9179
9179
|
if (err2) {
|
|
9180
9180
|
return callback();
|
|
9181
9181
|
}
|
|
9182
9182
|
var gzip = zlib3.createGzip();
|
|
9183
|
-
var inp =
|
|
9184
|
-
var out =
|
|
9183
|
+
var inp = fs43.createReadStream(src);
|
|
9184
|
+
var out = fs43.createWriteStream(dest);
|
|
9185
9185
|
out.on("finish", () => {
|
|
9186
|
-
|
|
9186
|
+
fs43.unlink(src, callback);
|
|
9187
9187
|
});
|
|
9188
9188
|
inp.pipe(gzip).pipe(out);
|
|
9189
9189
|
});
|
|
9190
9190
|
}
|
|
9191
9191
|
_createLogDirIfNotExist(dirPath) {
|
|
9192
|
-
if (!
|
|
9193
|
-
|
|
9192
|
+
if (!fs43.existsSync(dirPath)) {
|
|
9193
|
+
fs43.mkdirSync(dirPath, { recursive: true });
|
|
9194
9194
|
}
|
|
9195
9195
|
}
|
|
9196
9196
|
};
|
|
@@ -9274,9 +9274,9 @@ var require_http = __commonJS({
|
|
|
9274
9274
|
};
|
|
9275
9275
|
const auth = options2.params.auth || null;
|
|
9276
9276
|
delete options2.params.auth;
|
|
9277
|
-
const
|
|
9277
|
+
const path32 = options2.params.path || null;
|
|
9278
9278
|
delete options2.params.path;
|
|
9279
|
-
this._request(options2, auth,
|
|
9279
|
+
this._request(options2, auth, path32, (err2, res, body) => {
|
|
9280
9280
|
if (res && res.statusCode !== 200) {
|
|
9281
9281
|
err2 = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
|
|
9282
9282
|
}
|
|
@@ -9304,12 +9304,12 @@ var require_http = __commonJS({
|
|
|
9304
9304
|
method: "stream",
|
|
9305
9305
|
params: options2
|
|
9306
9306
|
};
|
|
9307
|
-
const
|
|
9307
|
+
const path32 = options2.params.path || null;
|
|
9308
9308
|
delete options2.params.path;
|
|
9309
9309
|
const auth = options2.params.auth || null;
|
|
9310
9310
|
delete options2.params.auth;
|
|
9311
9311
|
let buff = "";
|
|
9312
|
-
const req = this._request(options2, auth,
|
|
9312
|
+
const req = this._request(options2, auth, path32);
|
|
9313
9313
|
stream.destroy = () => req.destroy();
|
|
9314
9314
|
req.on("data", (data) => {
|
|
9315
9315
|
data = (buff + data).split(/\n+/);
|
|
@@ -9335,14 +9335,14 @@ var require_http = __commonJS({
|
|
|
9335
9335
|
* @param {string} path - request path
|
|
9336
9336
|
* @param {function} callback - Continuation to respond to when complete.
|
|
9337
9337
|
*/
|
|
9338
|
-
_request(options2, auth,
|
|
9338
|
+
_request(options2, auth, path32, callback) {
|
|
9339
9339
|
options2 = options2 || {};
|
|
9340
9340
|
auth = auth || this.auth;
|
|
9341
|
-
|
|
9341
|
+
path32 = path32 || this.path || "";
|
|
9342
9342
|
if (this.batch) {
|
|
9343
|
-
this._doBatch(options2, callback, auth,
|
|
9343
|
+
this._doBatch(options2, callback, auth, path32);
|
|
9344
9344
|
} else {
|
|
9345
|
-
this._doRequest(options2, callback, auth,
|
|
9345
|
+
this._doRequest(options2, callback, auth, path32);
|
|
9346
9346
|
}
|
|
9347
9347
|
}
|
|
9348
9348
|
/**
|
|
@@ -9352,18 +9352,18 @@ var require_http = __commonJS({
|
|
|
9352
9352
|
* @param {Object?} auth - authentication options
|
|
9353
9353
|
* @param {string} path - request path
|
|
9354
9354
|
*/
|
|
9355
|
-
_doBatch(options2, callback, auth,
|
|
9355
|
+
_doBatch(options2, callback, auth, path32) {
|
|
9356
9356
|
this.batchOptions.push(options2);
|
|
9357
9357
|
if (this.batchOptions.length === 1) {
|
|
9358
9358
|
const me3 = this;
|
|
9359
9359
|
this.batchCallback = callback;
|
|
9360
9360
|
this.batchTimeoutID = setTimeout(function() {
|
|
9361
9361
|
me3.batchTimeoutID = -1;
|
|
9362
|
-
me3._doBatchRequest(me3.batchCallback, auth,
|
|
9362
|
+
me3._doBatchRequest(me3.batchCallback, auth, path32);
|
|
9363
9363
|
}, this.batchInterval);
|
|
9364
9364
|
}
|
|
9365
9365
|
if (this.batchOptions.length === this.batchCount) {
|
|
9366
|
-
this._doBatchRequest(this.batchCallback, auth,
|
|
9366
|
+
this._doBatchRequest(this.batchCallback, auth, path32);
|
|
9367
9367
|
}
|
|
9368
9368
|
}
|
|
9369
9369
|
/**
|
|
@@ -9372,14 +9372,14 @@ var require_http = __commonJS({
|
|
|
9372
9372
|
* @param {Object?} auth - authentication options
|
|
9373
9373
|
* @param {string} path - request path
|
|
9374
9374
|
*/
|
|
9375
|
-
_doBatchRequest(callback, auth,
|
|
9375
|
+
_doBatchRequest(callback, auth, path32) {
|
|
9376
9376
|
if (this.batchTimeoutID > 0) {
|
|
9377
9377
|
clearTimeout(this.batchTimeoutID);
|
|
9378
9378
|
this.batchTimeoutID = -1;
|
|
9379
9379
|
}
|
|
9380
9380
|
const batchOptionsCopy = this.batchOptions.slice();
|
|
9381
9381
|
this.batchOptions = [];
|
|
9382
|
-
this._doRequest(batchOptionsCopy, callback, auth,
|
|
9382
|
+
this._doRequest(batchOptionsCopy, callback, auth, path32);
|
|
9383
9383
|
}
|
|
9384
9384
|
/**
|
|
9385
9385
|
* Make a request to a winstond server or any http server which can
|
|
@@ -9389,7 +9389,7 @@ var require_http = __commonJS({
|
|
|
9389
9389
|
* @param {Object?} auth - authentication options
|
|
9390
9390
|
* @param {string} path - request path
|
|
9391
9391
|
*/
|
|
9392
|
-
_doRequest(options2, callback, auth,
|
|
9392
|
+
_doRequest(options2, callback, auth, path32) {
|
|
9393
9393
|
const headers = Object.assign({}, this.headers);
|
|
9394
9394
|
if (auth && auth.bearer) {
|
|
9395
9395
|
headers.Authorization = `Bearer ${auth.bearer}`;
|
|
@@ -9399,7 +9399,7 @@ var require_http = __commonJS({
|
|
|
9399
9399
|
method: "POST",
|
|
9400
9400
|
host: this.host,
|
|
9401
9401
|
port: this.port,
|
|
9402
|
-
path: `/${
|
|
9402
|
+
path: `/${path32.replace(/^\//, "")}`,
|
|
9403
9403
|
headers,
|
|
9404
9404
|
auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
|
|
9405
9405
|
agent: this.agent
|
|
@@ -11761,7 +11761,7 @@ var CONSTANTS, Constants_default;
|
|
|
11761
11761
|
var init_Constants = __esm({
|
|
11762
11762
|
"../../packages/constants/src/Constants.ts"() {
|
|
11763
11763
|
CONSTANTS = {
|
|
11764
|
-
cliVersion: "2.
|
|
11764
|
+
cliVersion: "2.7.0",
|
|
11765
11765
|
backendVersion: 1,
|
|
11766
11766
|
backendPort: 5088,
|
|
11767
11767
|
workerVersion: 2,
|
|
@@ -11798,6 +11798,13 @@ var init_Constants = __esm({
|
|
|
11798
11798
|
defaults: {
|
|
11799
11799
|
REMORA_PATH: "./remora",
|
|
11800
11800
|
PRODUCER_TEMP_FOLDER: ".temp",
|
|
11801
|
+
/**
|
|
11802
|
+
* Where `synth` writes its exports, inside the remora directory, when none is given with
|
|
11803
|
+
* `-o`. A synthetic run never writes to the consumer's configured destination, so it needs
|
|
11804
|
+
* a folder of its own — and one inside the project, so the files it leaves sit next to the
|
|
11805
|
+
* configuration that produced them rather than somewhere in the user's tree.
|
|
11806
|
+
*/
|
|
11807
|
+
SYNTH_OUTPUT_FOLDER: ".synth",
|
|
11801
11808
|
SQL_MAX_QUERY_ROWS: 1e4,
|
|
11802
11809
|
/**
|
|
11803
11810
|
* Rows a SQL source hands over per round trip while STAGING a producer (a `FETCH` from a
|
|
@@ -11889,7 +11896,7 @@ var init_Constants = __esm({
|
|
|
11889
11896
|
/**
|
|
11890
11897
|
* Record layouts one FIXED output may write into a single file, which is also how many write
|
|
11891
11898
|
* streams its block-ordering pass holds open. A copybook of this size is already unusual —
|
|
11892
|
-
*
|
|
11899
|
+
* a POD-style transaction feed may write five transaction types — and the bound stops a generated config
|
|
11893
11900
|
* from opening a descriptor per distinct value of the wrong field.
|
|
11894
11901
|
*/
|
|
11895
11902
|
MAX_FIXED_LAYOUTS: 64,
|
|
@@ -12180,7 +12187,7 @@ var init_ProcessShutdownManager = __esm({
|
|
|
12180
12187
|
const shutdownState = this._shutdownState;
|
|
12181
12188
|
const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
|
|
12182
12189
|
if (scopeCount === 0) {
|
|
12183
|
-
Logger_default.
|
|
12190
|
+
Logger_default.log(`Detected ${shutdownDescription} for ${this._runtimeName}. No executor scopes to clean up.`);
|
|
12184
12191
|
return;
|
|
12185
12192
|
}
|
|
12186
12193
|
Logger_default.warn(`Detected ${shutdownDescription} for ${this._runtimeName}. Cleaning up ${scopeCount} executor scope${scopeCount === 1 ? "" : "s"}.`);
|
|
@@ -12694,7 +12701,7 @@ var init_ResourcesUtils = __esm({
|
|
|
12694
12701
|
});
|
|
12695
12702
|
|
|
12696
12703
|
// ../../packages/common/src/validation/Validator.ts
|
|
12697
|
-
var ValidatorClass, Validator, Validator_default;
|
|
12704
|
+
var MAX_SCALE, SIGN_MODES, USAGE_MODES, ValidatorClass, Validator, Validator_default;
|
|
12698
12705
|
var init_Validator = __esm({
|
|
12699
12706
|
"../../packages/common/src/validation/Validator.ts"() {
|
|
12700
12707
|
init_src();
|
|
@@ -12704,6 +12711,13 @@ var init_Validator = __esm({
|
|
|
12704
12711
|
init_ProcessENVManager();
|
|
12705
12712
|
init_SecretManager();
|
|
12706
12713
|
init_ResourcesUtils();
|
|
12714
|
+
MAX_SCALE = 15;
|
|
12715
|
+
SIGN_MODES = ["overpunch", "leading", "trailing"];
|
|
12716
|
+
USAGE_MODES = {
|
|
12717
|
+
"display": null,
|
|
12718
|
+
"comp-3": "packed decimal (two digits per byte, the sign in the last nibble)",
|
|
12719
|
+
"comp": "binary (a two's-complement integer of 2, 4 or 8 bytes)"
|
|
12720
|
+
};
|
|
12707
12721
|
ValidatorClass = class {
|
|
12708
12722
|
constructor() {
|
|
12709
12723
|
this.validateSources = (sources) => {
|
|
@@ -12805,6 +12819,9 @@ var init_Validator = __esm({
|
|
|
12805
12819
|
errors.push(`Producer "${producer.name}" has dimension(s) ${typeless.join(", ")} with no "type". Only a repeating group (a dimension declaring nested "dimensions") may omit it.`);
|
|
12806
12820
|
errors.push(...this.validateLegacySynthFields(producer));
|
|
12807
12821
|
errors.push(...this.validateFixedWidthLayout(producer));
|
|
12822
|
+
errors.push(...this.validateDimensionScale(producer));
|
|
12823
|
+
errors.push(...this.validateDimensionSign(producer));
|
|
12824
|
+
errors.push(...this.validateDimensionUsage(producer));
|
|
12808
12825
|
errors.push(...this.validateDimensionMasks(producer));
|
|
12809
12826
|
const sourceFilenameDimensions = producer.dimensions.filter((x2) => x2.sourceFilename === true);
|
|
12810
12827
|
if (sourceFilenameDimensions.length > 1) {
|
|
@@ -12828,6 +12845,67 @@ var init_Validator = __esm({
|
|
|
12828
12845
|
}
|
|
12829
12846
|
return errors;
|
|
12830
12847
|
};
|
|
12848
|
+
/**
|
|
12849
|
+
* Checks the implied decimals of a dimension. A wrong `scale` is the one layout error nothing
|
|
12850
|
+
* downstream can catch — the value it produces is a perfectly plausible number, 10^n off — so
|
|
12851
|
+
* every way of declaring one that cannot mean what it says is rejected here rather than read.
|
|
12852
|
+
*/
|
|
12853
|
+
this.validateDimensionScale = (producer) => {
|
|
12854
|
+
Affirm_default(producer, "Invalid producer");
|
|
12855
|
+
const errors = [];
|
|
12856
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12857
|
+
if (!Algo_default.hasVal(dim.scale)) continue;
|
|
12858
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12859
|
+
if (dim.type !== "number")
|
|
12860
|
+
errors.push(`${where} declares "scale" on a "${dim.type ?? "typeless"}" dimension: implied decimals only describe a "number".`);
|
|
12861
|
+
if (!Number.isInteger(dim.scale) || dim.scale < 1)
|
|
12862
|
+
errors.push(`${where} declares "scale" ${JSON.stringify(dim.scale)}: it counts implied decimal places, so it must be a whole number of 1 or more. A field written with its decimal point, or with no fraction at all, simply omits it.`);
|
|
12863
|
+
else if (dim.scale > MAX_SCALE)
|
|
12864
|
+
errors.push(`${where} declares "scale" ${dim.scale}, more than the ${MAX_SCALE} decimal places a number can carry without losing digits.`);
|
|
12865
|
+
}
|
|
12866
|
+
return errors;
|
|
12867
|
+
};
|
|
12868
|
+
/**
|
|
12869
|
+
* Checks how a dimension says its sign is written. A mode Remora does not implement would leave
|
|
12870
|
+
* the sign unread — a value of the right magnitude and the wrong direction, which is the one
|
|
12871
|
+
* error a reconciliation finds and a run never does — so a typo fails `verify` rather than a run.
|
|
12872
|
+
*/
|
|
12873
|
+
this.validateDimensionSign = (producer) => {
|
|
12874
|
+
Affirm_default(producer, "Invalid producer");
|
|
12875
|
+
const errors = [];
|
|
12876
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12877
|
+
if (!Algo_default.hasVal(dim.sign)) continue;
|
|
12878
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12879
|
+
if (dim.type !== "number")
|
|
12880
|
+
errors.push(`${where} declares "sign" on a "${dim.type ?? "typeless"}" dimension: a written sign only describes a "number".`);
|
|
12881
|
+
if (!SIGN_MODES.includes(dim.sign))
|
|
12882
|
+
errors.push(`${where} declares "sign" ${JSON.stringify(dim.sign)}, which is not a way a sign can be written. Use ${SIGN_MODES.map((x2) => `"${x2}"`).join(", ")}, or omit it for a plain leading "-".`);
|
|
12883
|
+
}
|
|
12884
|
+
return errors;
|
|
12885
|
+
};
|
|
12886
|
+
/**
|
|
12887
|
+
* Rejects a field the copybook says is stored as BYTES. `import-copybook` emits the `USAGE` it
|
|
12888
|
+
* read rather than dropping it, because the field's digits are the ones a positional read would
|
|
12889
|
+
* slice characters out of and get garbage from: the feed has to expand them to display before it
|
|
12890
|
+
* is sent. Saying so here, naming the field and the conversion, is how the customer learns at
|
|
12891
|
+
* compile instead of from a column of NaN after a run (docs/plans/mainframe.md open question 3).
|
|
12892
|
+
*/
|
|
12893
|
+
this.validateDimensionUsage = (producer) => {
|
|
12894
|
+
Affirm_default(producer, "Invalid producer");
|
|
12895
|
+
const errors = [];
|
|
12896
|
+
for (const dim of producer.dimensions ?? []) {
|
|
12897
|
+
if (!Algo_default.hasVal(dim.usage)) continue;
|
|
12898
|
+
const where = `Dimension "${dim.name}" of producer "${producer.name}"`;
|
|
12899
|
+
if (!(dim.usage in USAGE_MODES)) {
|
|
12900
|
+
errors.push(`${where} declares "usage" ${JSON.stringify(dim.usage)}, which is not a storage mode. Use ${Object.keys(USAGE_MODES).map((x2) => `"${x2}"`).join(", ")}.`);
|
|
12901
|
+
continue;
|
|
12902
|
+
}
|
|
12903
|
+
const stored = USAGE_MODES[dim.usage];
|
|
12904
|
+
if (stored)
|
|
12905
|
+
errors.push(`${where} declares "usage": "${dim.usage}": the field is stored as ${stored}, not as characters, so a positional read cannot address it. Have the feed convert it to display before it is sent (the DFSORT/OUTREC step that converts the file also expands packed and binary fields), then remove "usage" from this dimension.`);
|
|
12906
|
+
}
|
|
12907
|
+
return errors;
|
|
12908
|
+
};
|
|
12831
12909
|
/**
|
|
12832
12910
|
* Rejects a masking mode that is not one Remora implements. The producer schema can't: the value
|
|
12833
12911
|
* may be a `{VAR}` reference that only resolves at runtime. Checked here so a typo fails `verify`
|
|
@@ -13384,7 +13462,7 @@ var init_Validator = __esm({
|
|
|
13384
13462
|
*/
|
|
13385
13463
|
this.detectOnFinishRecursion = (consumer) => {
|
|
13386
13464
|
const errors = [];
|
|
13387
|
-
const
|
|
13465
|
+
const path32 = [];
|
|
13388
13466
|
const targetsOf = (cons) => {
|
|
13389
13467
|
const names = [];
|
|
13390
13468
|
for (const output of cons.outputs ?? []) {
|
|
@@ -13396,17 +13474,17 @@ var init_Validator = __esm({
|
|
|
13396
13474
|
return names;
|
|
13397
13475
|
};
|
|
13398
13476
|
const walk = (name) => {
|
|
13399
|
-
if (
|
|
13400
|
-
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...
|
|
13477
|
+
if (path32.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
|
|
13478
|
+
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path32, name].join(" -> ")}`);
|
|
13401
13479
|
return true;
|
|
13402
13480
|
}
|
|
13403
13481
|
const cons = Environment_default.getConsumer(name);
|
|
13404
13482
|
if (!cons) return false;
|
|
13405
|
-
|
|
13483
|
+
path32.push(name);
|
|
13406
13484
|
for (const target of targetsOf(cons)) {
|
|
13407
13485
|
if (walk(target)) return true;
|
|
13408
13486
|
}
|
|
13409
|
-
|
|
13487
|
+
path32.pop();
|
|
13410
13488
|
return false;
|
|
13411
13489
|
};
|
|
13412
13490
|
walk(consumer.name);
|
|
@@ -13463,7 +13541,7 @@ var init_Validator = __esm({
|
|
|
13463
13541
|
});
|
|
13464
13542
|
|
|
13465
13543
|
// ../../packages/common/src/ProducerExpander.ts
|
|
13466
|
-
var MAX_OCCURS, DEFAULT_GROUP_NAMING, DEFAULT_ROWS_NAMING, DEFAULT_OCCURRENCE_FIELD, DEFAULT_FLAG_NAMING, GROUP_NAMING_TOKENS, FLAG_NAMING_TOKENS, GROUP_FORBIDDEN_PROPERTIES, GROUP_ONLY_PROPERTIES, IDENTIFIER_PATTERN, ProducerExpanderClass, ProducerExpander, ProducerExpander_default;
|
|
13544
|
+
var MAX_OCCURS, DEFAULT_GROUP_NAMING, DEFAULT_ROWS_NAMING, DEFAULT_OCCURRENCE_FIELD, DEFAULT_FLAG_NAMING, GROUP_NAMING_TOKENS, FLAG_NAMING_TOKENS, INDEX_TOKEN_PATTERN, GROUP_FORBIDDEN_PROPERTIES, GROUP_ONLY_PROPERTIES, NESTED_FORBIDDEN_PROPERTIES, IDENTIFIER_PATTERN, ProducerExpanderClass, ProducerExpander, ProducerExpander_default;
|
|
13467
13545
|
var init_ProducerExpander = __esm({
|
|
13468
13546
|
"../../packages/common/src/ProducerExpander.ts"() {
|
|
13469
13547
|
init_src();
|
|
@@ -13476,8 +13554,10 @@ var init_ProducerExpander = __esm({
|
|
|
13476
13554
|
DEFAULT_FLAG_NAMING = "{field}{Flag}";
|
|
13477
13555
|
GROUP_NAMING_TOKENS = ["{group}", "{index}", "{field}", "{Field}"];
|
|
13478
13556
|
FLAG_NAMING_TOKENS = ["{field}", "{Field}", "{flag}", "{Flag}"];
|
|
13479
|
-
|
|
13557
|
+
INDEX_TOKEN_PATTERN = /{index(\d*)}/g;
|
|
13558
|
+
GROUP_FORBIDDEN_PROPERTIES = ["type", "alias", "pk", "mask", "synth", "sourceFilename", "flags", "format", "scale", "sign", "usage"];
|
|
13480
13559
|
GROUP_ONLY_PROPERTIES = ["occurs", "indexFrom", "emit", "skipEmptyOccurrences", "occurrenceField"];
|
|
13560
|
+
NESTED_FORBIDDEN_PROPERTIES = ["emit", "skipEmptyOccurrences", "occurrenceField"];
|
|
13481
13561
|
IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
13482
13562
|
ProducerExpanderClass = class {
|
|
13483
13563
|
constructor() {
|
|
@@ -13556,22 +13636,25 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13556
13636
|
this.isGroup = (dimension) => {
|
|
13557
13637
|
return Array.isArray(dimension?.dimensions);
|
|
13558
13638
|
};
|
|
13559
|
-
this._validateGroup = (producer, declared, group3) => {
|
|
13639
|
+
this._validateGroup = (producer, declared, group3, nesting = null) => {
|
|
13560
13640
|
const errors = [];
|
|
13561
|
-
const where = `group dimension "${group3.name}" of producer "${producer.name}"`;
|
|
13641
|
+
const where = nesting ? `nested group dimension "${group3.name}" of ${nesting.where}` : `group dimension "${group3.name}" of producer "${producer.name}"`;
|
|
13562
13642
|
if (!IDENTIFIER_PATTERN.test(group3.name ?? ""))
|
|
13563
13643
|
errors.push(`The name of ${where} is used as a prefix for the names it generates, so it must be a valid identifier: letters, digits and underscores, not starting with a digit.`);
|
|
13564
13644
|
const forbidden = GROUP_FORBIDDEN_PROPERTIES.filter((property) => group3[property] !== void 0);
|
|
13565
13645
|
if (forbidden.length > 0)
|
|
13566
13646
|
errors.push(`${where} declares ${forbidden.map((x2) => `"${x2}"`).join(", ")}: a group produces no column of its own, so ${forbidden.length > 1 ? "they" : "it"} would be silently ignored. Declare ${forbidden.length > 1 ? "them" : "it"} on the child dimension(s) instead.`);
|
|
13567
|
-
errors.push(...Validator_default.validateFieldPosition(group3.position, `${where} (the extent of ONE occurrence)`));
|
|
13647
|
+
errors.push(...Validator_default.validateFieldPosition(group3.position, `${where} (the extent of ONE occurrence${nesting ? ", RELATIVE to one occurrence of the group it sits in" : ""})`));
|
|
13568
13648
|
const occurrenceWidth = this._isPositionValid(group3.position) ? this._width(group3.position) : null;
|
|
13569
|
-
errors.push(...this.
|
|
13649
|
+
errors.push(...this._validateNesting(group3, occurrenceWidth, nesting, where));
|
|
13650
|
+
if (!nesting || typeof group3.occurs !== "object")
|
|
13651
|
+
errors.push(...this._validateOccurs(declared, group3, occurrenceWidth, where));
|
|
13570
13652
|
if (Algo_default.hasVal(group3.indexFrom) && group3.indexFrom !== 0 && group3.indexFrom !== 1)
|
|
13571
13653
|
errors.push(`"indexFrom" on ${where} is ${JSON.stringify(group3.indexFrom)}: it must be 0 or 1.`);
|
|
13572
|
-
|
|
13654
|
+
if (!nesting)
|
|
13655
|
+
errors.push(...this._validateEmit(group3, where));
|
|
13573
13656
|
if (Algo_default.hasVal(group3.naming)) {
|
|
13574
|
-
errors.push(...this._validateNaming(group3.naming,
|
|
13657
|
+
errors.push(...this._validateNaming(group3.naming, this._groupNamingTokens(nesting?.depth ?? 0), where));
|
|
13575
13658
|
const columns = this._occurrences(group3).max;
|
|
13576
13659
|
if (columns > 1 && group3.emit !== "rows" && !group3.naming.includes("{index}"))
|
|
13577
13660
|
errors.push(`"naming" on ${where} ("${group3.naming}") has no "{index}" token, so all ${columns} occurrences would generate the same dimension name.`);
|
|
@@ -13584,16 +13667,39 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13584
13667
|
const childDupes = Algo_default.uniq(children.map((x2) => x2.name).filter((name, index, all) => all.indexOf(name) !== index));
|
|
13585
13668
|
if (childDupes.length > 0)
|
|
13586
13669
|
errors.push(`${where} declares duplicate child dimension name(s): ${childDupes.map((x2) => `"${x2}"`).join(", ")}.`);
|
|
13587
|
-
|
|
13588
|
-
|
|
13670
|
+
const inner = { depth: (nesting?.depth ?? 0) + 1, width: occurrenceWidth, where };
|
|
13671
|
+
for (const child of children) {
|
|
13672
|
+
errors.push(...this.isGroup(child) ? this._validateGroup(producer, declared, child, inner) : this._validateGroupChild(child, occurrenceWidth, `child dimension "${child.name}" of ${where}`));
|
|
13673
|
+
}
|
|
13589
13674
|
return errors;
|
|
13590
13675
|
};
|
|
13591
|
-
|
|
13676
|
+
/**
|
|
13677
|
+
* What a group may only do at the top level. A nested group's occurrences are laid out inside one
|
|
13678
|
+
* occurrence of its parent, which is itself placed by the line, so the inner count has to be
|
|
13679
|
+
* statically known — a per-line count on both axes has no single place to be read from.
|
|
13680
|
+
*/
|
|
13681
|
+
this._validateNesting = (group3, occurrenceWidth, nesting, where) => {
|
|
13682
|
+
if (!nesting) return [];
|
|
13592
13683
|
const errors = [];
|
|
13593
|
-
|
|
13594
|
-
|
|
13684
|
+
const forbidden = NESTED_FORBIDDEN_PROPERTIES.filter((property) => Algo_default.hasVal(group3[property]));
|
|
13685
|
+
if (forbidden.length > 0)
|
|
13686
|
+
errors.push(`${where} declares ${forbidden.map((x2) => `"${x2}"`).join(", ")}: ${forbidden.length > 1 ? "those properties describe" : "that property describes"} fanning a LINE out to one record per occurrence, which only the outermost group can do.`);
|
|
13687
|
+
if (typeof group3.occurs === "object")
|
|
13688
|
+
errors.push(`"occurs" on ${where} varies per line: a nested group repeats a fixed number of times inside one occurrence of the group it sits in, so its count cannot depend on the line. Declare a whole number, or lift the group out to the top level.`);
|
|
13689
|
+
const { max } = this._occurrences(group3);
|
|
13690
|
+
if (occurrenceWidth === null || !Number.isInteger(max) || max < 1 || !Algo_default.hasVal(nesting.width))
|
|
13595
13691
|
return errors;
|
|
13596
|
-
|
|
13692
|
+
const start = group3.position.start ?? 1;
|
|
13693
|
+
const end = start + max * occurrenceWidth - 1;
|
|
13694
|
+
if (end > nesting.width)
|
|
13695
|
+
errors.push(`${where} spans columns ${start}-${end} of an occurrence that is only ${nesting.width} character(s) wide. A nested group's position is RELATIVE to one occurrence of the group it sits in, and all ${max} of its own occurrences have to fit inside it.`);
|
|
13696
|
+
return errors;
|
|
13697
|
+
};
|
|
13698
|
+
this._validateGroupChild = (child, occurrenceWidth, where) => {
|
|
13699
|
+
const errors = [];
|
|
13700
|
+
const groupOnly = GROUP_ONLY_PROPERTIES.filter((property) => Algo_default.hasVal(child[property]));
|
|
13701
|
+
if (groupOnly.length > 0)
|
|
13702
|
+
errors.push(`${where} declares ${groupOnly.map((x2) => `"${x2}"`).join(", ")} but has no "dimensions": ${groupOnly.length > 1 ? "those properties describe" : "that property describes"} a repeating group, and would be silently ignored here.`);
|
|
13597
13703
|
if (child.sourceFilename === true)
|
|
13598
13704
|
errors.push(`${where} declares "sourceFilename": that dimension isn't in the line at all, so repeating it inside a group is meaningless. Declare it outside the group.`);
|
|
13599
13705
|
if (!child.type)
|
|
@@ -13706,9 +13812,9 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13706
13812
|
const errors = [];
|
|
13707
13813
|
const groupNames = declared.filter((x2) => this.isGroup(x2)).map((x2) => x2.name);
|
|
13708
13814
|
for (const [index, dimension] of declared.entries()) {
|
|
13709
|
-
for (const child of dimension
|
|
13815
|
+
for (const { child, parent } of this._descendants(dimension)) {
|
|
13710
13816
|
if (!child.position?.after) continue;
|
|
13711
|
-
errors.push(`"position.after" on child dimension "${child.name}" of group "${
|
|
13817
|
+
errors.push(`"position.after" on child dimension "${child.name}" of group "${parent.name}" (producer "${producer.name}"): a child's position is RELATIVE to one occurrence, so only the group itself can be anchored.`);
|
|
13712
13818
|
}
|
|
13713
13819
|
const after = dimension.position?.after;
|
|
13714
13820
|
if (!after) continue;
|
|
@@ -13751,6 +13857,20 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13751
13857
|
errors.push(`"flags" on ${where} lists ${flags.length} flag(s) but the dimension spans ${width} character(s): one flag decodes exactly one character.`);
|
|
13752
13858
|
return errors;
|
|
13753
13859
|
};
|
|
13860
|
+
/** Every dimension declared inside this one, at any depth, with the group that declares it. */
|
|
13861
|
+
this._descendants = (dimension) => {
|
|
13862
|
+
const found = [];
|
|
13863
|
+
for (const child of dimension.dimensions ?? [])
|
|
13864
|
+
found.push({ child, parent: dimension }, ...this._descendants(child));
|
|
13865
|
+
return found;
|
|
13866
|
+
};
|
|
13867
|
+
/**
|
|
13868
|
+
* The naming tokens a group at this depth may use: its own `{index}`, plus one `{indexN}` per
|
|
13869
|
+
* enclosing group, so a template can reach outward as far as the nesting goes and no further.
|
|
13870
|
+
*/
|
|
13871
|
+
this._groupNamingTokens = (depth) => {
|
|
13872
|
+
return [...GROUP_NAMING_TOKENS, ...Array.from({ length: depth }, (_3, level) => `{index${level + 1}}`)];
|
|
13873
|
+
};
|
|
13754
13874
|
this._validateNaming = (naming, allowed, where) => {
|
|
13755
13875
|
const unknown = (naming.match(/{[^}]*}/g) ?? []).filter((token2) => !allowed.includes(token2));
|
|
13756
13876
|
if (unknown.length === 0)
|
|
@@ -13859,16 +13979,12 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13859
13979
|
this._expandGroup = (group3, layout) => {
|
|
13860
13980
|
if (layout.isRows)
|
|
13861
13981
|
return this._expandRowsGroup(group3);
|
|
13862
|
-
|
|
13863
|
-
|
|
13864
|
-
const isPlanned = this._isPlanned(layout);
|
|
13982
|
+
if (!this._isPlanned(layout))
|
|
13983
|
+
return this._groupLeaves(group3, layout.start - 1, []);
|
|
13865
13984
|
const leaves = [];
|
|
13866
13985
|
for (let occurrence = 0; occurrence < layout.max; occurrence++) {
|
|
13867
|
-
const
|
|
13868
|
-
|
|
13869
|
-
const position = isPlanned ? { start: child.position.start, length: this._width(child.position), group: group3.name, occurrence } : { start: base + child.position.start - 1, length: this._width(child.position) };
|
|
13870
|
-
leaves.push(this._groupLeaf(child, group3, this._applyGroupNaming(naming, group3.name, occurrence + indexFrom, child.name), position));
|
|
13871
|
-
}
|
|
13986
|
+
for (const leaf of this._occurrenceLeaves(group3, occurrence, []))
|
|
13987
|
+
leaves.push({ ...leaf, position: { ...leaf.position, group: group3.name, occurrence } });
|
|
13872
13988
|
}
|
|
13873
13989
|
return leaves;
|
|
13874
13990
|
};
|
|
@@ -13878,7 +13994,6 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13878
13994
|
* no occurrence fixed — the read path picks it per record.
|
|
13879
13995
|
*/
|
|
13880
13996
|
this._expandRowsGroup = (group3) => {
|
|
13881
|
-
const naming = group3.naming ?? DEFAULT_ROWS_NAMING;
|
|
13882
13997
|
const occurrenceField = {
|
|
13883
13998
|
name: this._occurrenceFieldName(group3),
|
|
13884
13999
|
description: `Which occurrence of "${group3.name}" this record carries`,
|
|
@@ -13886,14 +14001,41 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13886
14001
|
occurrenceOf: group3.name,
|
|
13887
14002
|
indexFrom: group3.indexFrom ?? 0
|
|
13888
14003
|
};
|
|
13889
|
-
const leaves = group3.
|
|
13890
|
-
child,
|
|
13891
|
-
group3,
|
|
13892
|
-
this._applyGroupNaming(naming, group3.name, 0, child.name),
|
|
13893
|
-
{ start: child.position.start, length: this._width(child.position), group: group3.name }
|
|
13894
|
-
));
|
|
14004
|
+
const leaves = this._occurrenceLeaves(group3, 0, []).map((leaf) => ({ ...leaf, position: { ...leaf.position, group: group3.name } }));
|
|
13895
14005
|
return [occurrenceField, ...leaves];
|
|
13896
14006
|
};
|
|
14007
|
+
/**
|
|
14008
|
+
* Every leaf a group generates, with each `position.start` 1-based and RELATIVE to the region the
|
|
14009
|
+
* group sits in: the line for a top-level group, one occurrence of the parent for a nested one.
|
|
14010
|
+
* `base` is that region's 0-based offset, and `ancestors` the enclosing occurrence indices,
|
|
14011
|
+
* nearest first, which the naming templates read as `{index1}`...`{indexN}`.
|
|
14012
|
+
*/
|
|
14013
|
+
this._groupLeaves = (group3, base, ancestors) => {
|
|
14014
|
+
const width = this._width(group3.position);
|
|
14015
|
+
const leaves = [];
|
|
14016
|
+
for (let occurrence = 0; occurrence < this._occurrences(group3).max; occurrence++) {
|
|
14017
|
+
const offset = base + occurrence * width;
|
|
14018
|
+
for (const leaf of this._occurrenceLeaves(group3, occurrence, ancestors))
|
|
14019
|
+
leaves.push({ ...leaf, position: { ...leaf.position, start: leaf.position.start + offset } });
|
|
14020
|
+
}
|
|
14021
|
+
return leaves;
|
|
14022
|
+
};
|
|
14023
|
+
/**
|
|
14024
|
+
* The leaves of ONE occurrence of a group, positioned relative to that occurrence's own start. A
|
|
14025
|
+
* child that is itself a group contributes its own leaves instead, already flattened — so the
|
|
14026
|
+
* outer template names what the inner one generated, and nesting costs the pipeline nothing.
|
|
14027
|
+
*/
|
|
14028
|
+
this._occurrenceLeaves = (group3, occurrence, ancestors) => {
|
|
14029
|
+
const naming = group3.naming ?? (group3.emit === "rows" ? DEFAULT_ROWS_NAMING : DEFAULT_GROUP_NAMING);
|
|
14030
|
+
const indices = [occurrence + (group3.indexFrom ?? 0), ...ancestors];
|
|
14031
|
+
const leaves = [];
|
|
14032
|
+
for (const child of group3.dimensions) {
|
|
14033
|
+
const inner = this.isGroup(child) ? this._groupLeaves(child, (child.position.start ?? 1) - 1, indices) : [{ ...child, position: { start: child.position.start ?? 1, length: this._width(child.position) } }];
|
|
14034
|
+
for (const leaf of inner)
|
|
14035
|
+
leaves.push(this._groupLeaf(leaf, group3, this._applyGroupNaming(naming, group3.name, indices, leaf.name), leaf.position));
|
|
14036
|
+
}
|
|
14037
|
+
return leaves;
|
|
14038
|
+
};
|
|
13897
14039
|
this._groupLeaf = (child, group3, name, position) => {
|
|
13898
14040
|
const leaf = { ...child, name, position };
|
|
13899
14041
|
if (!leaf.description && group3.description)
|
|
@@ -13928,8 +14070,9 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13928
14070
|
rebased.after = position.after;
|
|
13929
14071
|
return rebased;
|
|
13930
14072
|
};
|
|
13931
|
-
|
|
13932
|
-
|
|
14073
|
+
/** `indices[0]` is the group's own occurrence, `indices[n]` the one n levels out. */
|
|
14074
|
+
this._applyGroupNaming = (naming, group3, indices, field) => {
|
|
14075
|
+
return naming.replace(/{group}/g, group3).replace(INDEX_TOKEN_PATTERN, (token2, level) => String(indices[level === "" ? 0 : Number(level)] ?? token2)).replace(/{field}/g, field).replace(/{Field}/g, this._capitalize(field));
|
|
13933
14076
|
};
|
|
13934
14077
|
this._applyFlagNaming = (naming, field, flag) => {
|
|
13935
14078
|
return naming.replace(/{field}/g, field).replace(/{Field}/g, this._capitalize(field)).replace(/{flag}/g, flag).replace(/{Flag}/g, this._capitalize(flag));
|
|
@@ -14957,8 +15100,8 @@ var init_HttpClient = __esm({
|
|
|
14957
15100
|
throw new Error(err2.message ?? "An error occurred on GET.");
|
|
14958
15101
|
}
|
|
14959
15102
|
};
|
|
14960
|
-
this.getWorkerUrl = (
|
|
14961
|
-
const cleanPath =
|
|
15103
|
+
this.getWorkerUrl = (path32) => {
|
|
15104
|
+
const cleanPath = path32.startsWith("/") ? path32.substring(1) : path32;
|
|
14962
15105
|
return `${this.workerHost}/${cleanPath}`;
|
|
14963
15106
|
};
|
|
14964
15107
|
this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
|
|
@@ -15785,14 +15928,14 @@ var init_readFile = __esm({
|
|
|
15785
15928
|
import_promises3 = require("fs/promises");
|
|
15786
15929
|
filePromises = {};
|
|
15787
15930
|
fileIntercept = {};
|
|
15788
|
-
readFile2 = (
|
|
15789
|
-
if (fileIntercept[
|
|
15790
|
-
return fileIntercept[
|
|
15931
|
+
readFile2 = (path32, options2) => {
|
|
15932
|
+
if (fileIntercept[path32] !== void 0) {
|
|
15933
|
+
return fileIntercept[path32];
|
|
15791
15934
|
}
|
|
15792
|
-
if (!filePromises[
|
|
15793
|
-
filePromises[
|
|
15935
|
+
if (!filePromises[path32] || options2?.ignoreCache) {
|
|
15936
|
+
filePromises[path32] = (0, import_promises3.readFile)(path32, "utf8");
|
|
15794
15937
|
}
|
|
15795
|
-
return filePromises[
|
|
15938
|
+
return filePromises[path32];
|
|
15796
15939
|
};
|
|
15797
15940
|
}
|
|
15798
15941
|
});
|
|
@@ -15904,8 +16047,8 @@ var init_externalDataInterceptor = __esm({
|
|
|
15904
16047
|
getFileRecord() {
|
|
15905
16048
|
return fileIntercept;
|
|
15906
16049
|
},
|
|
15907
|
-
interceptFile(
|
|
15908
|
-
fileIntercept[
|
|
16050
|
+
interceptFile(path32, contents) {
|
|
16051
|
+
fileIntercept[path32] = Promise.resolve(contents);
|
|
15909
16052
|
},
|
|
15910
16053
|
getTokenRecord() {
|
|
15911
16054
|
return tokenIntercept;
|
|
@@ -17815,8 +17958,8 @@ var init_checkRegion = __esm({
|
|
|
17815
17958
|
"../../node_modules/@smithy/core/dist-es/submodules/config/config-resolver/regionConfig/checkRegion.js"() {
|
|
17816
17959
|
init_transport();
|
|
17817
17960
|
validRegions = /* @__PURE__ */ new Set();
|
|
17818
|
-
checkRegion = (region,
|
|
17819
|
-
if (!validRegions.has(region) && !
|
|
17961
|
+
checkRegion = (region, check = isValidHostLabel) => {
|
|
17962
|
+
if (!validRegions.has(region) && !check(region)) {
|
|
17820
17963
|
if (region === "*") {
|
|
17821
17964
|
console.warn(`@smithy/config-resolver WARN - Please use the caller region instead of "*". See "sigv4a" in https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/CLIENTS.md.`);
|
|
17822
17965
|
} else {
|
|
@@ -18985,8 +19128,8 @@ var init_createConfigValueProvider = __esm({
|
|
|
18985
19128
|
return endpoint.url.href;
|
|
18986
19129
|
}
|
|
18987
19130
|
if ("hostname" in endpoint) {
|
|
18988
|
-
const { protocol, hostname, port, path:
|
|
18989
|
-
return `${protocol}//${hostname}${port ? ":" + port : ""}${
|
|
19131
|
+
const { protocol, hostname, port, path: path32 } = endpoint;
|
|
19132
|
+
return `${protocol}//${hostname}${port ? ":" + port : ""}${path32}`;
|
|
18990
19133
|
}
|
|
18991
19134
|
}
|
|
18992
19135
|
return endpoint;
|
|
@@ -19361,18 +19504,18 @@ var getAttrPathList;
|
|
|
19361
19504
|
var init_getAttrPathList = __esm({
|
|
19362
19505
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
|
|
19363
19506
|
init_types2();
|
|
19364
|
-
getAttrPathList = (
|
|
19365
|
-
const parts =
|
|
19507
|
+
getAttrPathList = (path32) => {
|
|
19508
|
+
const parts = path32.split(".");
|
|
19366
19509
|
const pathList = [];
|
|
19367
19510
|
for (const part of parts) {
|
|
19368
19511
|
const squareBracketIndex = part.indexOf("[");
|
|
19369
19512
|
if (squareBracketIndex !== -1) {
|
|
19370
19513
|
if (part.indexOf("]") !== part.length - 1) {
|
|
19371
|
-
throw new EndpointError(`Path: '${
|
|
19514
|
+
throw new EndpointError(`Path: '${path32}' does not end with ']'`);
|
|
19372
19515
|
}
|
|
19373
19516
|
const arrayIndex = part.slice(squareBracketIndex + 1, -1);
|
|
19374
19517
|
if (Number.isNaN(parseInt(arrayIndex))) {
|
|
19375
|
-
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${
|
|
19518
|
+
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path32}'`);
|
|
19376
19519
|
}
|
|
19377
19520
|
if (squareBracketIndex !== 0) {
|
|
19378
19521
|
pathList.push(part.slice(0, squareBracketIndex));
|
|
@@ -19393,9 +19536,9 @@ var init_getAttr = __esm({
|
|
|
19393
19536
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttr.js"() {
|
|
19394
19537
|
init_types2();
|
|
19395
19538
|
init_getAttrPathList();
|
|
19396
|
-
getAttr = (value,
|
|
19539
|
+
getAttr = (value, path32) => getAttrPathList(path32).reduce((acc, index) => {
|
|
19397
19540
|
if (typeof acc !== "object") {
|
|
19398
|
-
throw new EndpointError(`Index '${index}' in '${
|
|
19541
|
+
throw new EndpointError(`Index '${index}' in '${path32}' not found in '${JSON.stringify(value)}'`);
|
|
19399
19542
|
} else if (Array.isArray(acc)) {
|
|
19400
19543
|
const i6 = parseInt(index);
|
|
19401
19544
|
return acc[i6 < 0 ? acc.length + i6 : i6];
|
|
@@ -19456,8 +19599,8 @@ var init_parseURL = __esm({
|
|
|
19456
19599
|
return value;
|
|
19457
19600
|
}
|
|
19458
19601
|
if (typeof value === "object" && "hostname" in value) {
|
|
19459
|
-
const { hostname: hostname2, port, protocol: protocol2 = "", path:
|
|
19460
|
-
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${
|
|
19602
|
+
const { hostname: hostname2, port, protocol: protocol2 = "", path: path32 = "", query = {} } = value;
|
|
19603
|
+
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path32}`);
|
|
19461
19604
|
url.search = Object.entries(query).map(([k7, v3]) => `${k7}=${v3}`).join("&");
|
|
19462
19605
|
return url;
|
|
19463
19606
|
}
|
|
@@ -21801,11 +21944,11 @@ var init_HttpBindingProtocol = __esm({
|
|
|
21801
21944
|
const opTraits = translateTraits(operationSchema.traits);
|
|
21802
21945
|
if (opTraits.http) {
|
|
21803
21946
|
request.method = opTraits.http[0];
|
|
21804
|
-
const [
|
|
21947
|
+
const [path32, search] = opTraits.http[1].split("?");
|
|
21805
21948
|
if (request.path == "/") {
|
|
21806
|
-
request.path =
|
|
21949
|
+
request.path = path32;
|
|
21807
21950
|
} else {
|
|
21808
|
-
request.path +=
|
|
21951
|
+
request.path += path32;
|
|
21809
21952
|
}
|
|
21810
21953
|
const traitSearchParams = new URLSearchParams(search ?? "");
|
|
21811
21954
|
for (const [key, value] of traitSearchParams) {
|
|
@@ -25518,12 +25661,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
25518
25661
|
const password = request.password ?? "";
|
|
25519
25662
|
auth = `${username}:${password}`;
|
|
25520
25663
|
}
|
|
25521
|
-
let
|
|
25664
|
+
let path32 = request.path;
|
|
25522
25665
|
if (queryString) {
|
|
25523
|
-
|
|
25666
|
+
path32 += `?${queryString}`;
|
|
25524
25667
|
}
|
|
25525
25668
|
if (request.fragment) {
|
|
25526
|
-
|
|
25669
|
+
path32 += `#${request.fragment}`;
|
|
25527
25670
|
}
|
|
25528
25671
|
let hostname = request.hostname ?? "";
|
|
25529
25672
|
if (hostname[0] === "[" && hostname.endsWith("]")) {
|
|
@@ -25535,7 +25678,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
25535
25678
|
headers: request.headers,
|
|
25536
25679
|
host: hostname,
|
|
25537
25680
|
method: request.method,
|
|
25538
|
-
path:
|
|
25681
|
+
path: path32,
|
|
25539
25682
|
port: request.port,
|
|
25540
25683
|
agent,
|
|
25541
25684
|
auth
|
|
@@ -26444,10 +26587,10 @@ ${longDate}
|
|
|
26444
26587
|
${credentialScope}
|
|
26445
26588
|
${toHex(hashedRequest)}`;
|
|
26446
26589
|
}
|
|
26447
|
-
getCanonicalPath({ path:
|
|
26590
|
+
getCanonicalPath({ path: path32 }) {
|
|
26448
26591
|
if (this.uriEscapePath) {
|
|
26449
26592
|
const normalizedPathSegments = [];
|
|
26450
|
-
for (const pathSegment of
|
|
26593
|
+
for (const pathSegment of path32.split("/")) {
|
|
26451
26594
|
if (pathSegment?.length === 0)
|
|
26452
26595
|
continue;
|
|
26453
26596
|
if (pathSegment === ".")
|
|
@@ -26458,11 +26601,11 @@ ${toHex(hashedRequest)}`;
|
|
|
26458
26601
|
normalizedPathSegments.push(pathSegment);
|
|
26459
26602
|
}
|
|
26460
26603
|
}
|
|
26461
|
-
const normalizedPath = `${
|
|
26604
|
+
const normalizedPath = `${path32?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path32?.endsWith("/") ? "/" : ""}`;
|
|
26462
26605
|
const doubleEncoded = escapeUri(normalizedPath);
|
|
26463
26606
|
return doubleEncoded.replace(/%2F/g, "/");
|
|
26464
26607
|
}
|
|
26465
|
-
return
|
|
26608
|
+
return path32;
|
|
26466
26609
|
}
|
|
26467
26610
|
validateResolvedCredentials(credentials) {
|
|
26468
26611
|
if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
|
|
@@ -34228,12 +34371,107 @@ var init_FixedWidthParser = __esm({
|
|
|
34228
34371
|
}
|
|
34229
34372
|
});
|
|
34230
34373
|
|
|
34374
|
+
// ../../packages/parsing/src/NumericCodec.ts
|
|
34375
|
+
var SIGN_MODES2, OVERPUNCH_POSITIVE, OVERPUNCH_NEGATIVE, NumericCodecClass, NumericCodec, NumericCodec_default;
|
|
34376
|
+
var init_NumericCodec = __esm({
|
|
34377
|
+
"../../packages/parsing/src/NumericCodec.ts"() {
|
|
34378
|
+
SIGN_MODES2 = {
|
|
34379
|
+
none: { pattern: /^([+-]?)(\d+(?:\.\d+)?)$/, shape: 'digits, with an optional leading "+" or "-"' },
|
|
34380
|
+
leading: { pattern: /^([+-]?)(\d+(?:\.\d+)?)$/, shape: 'an optional leading "+" or "-", then digits' },
|
|
34381
|
+
trailing: { pattern: /^(\d+(?:\.\d+)?)([+-]?)$/, shape: 'digits, then an optional "+" or "-"' },
|
|
34382
|
+
overpunch: { pattern: /^(\d*)([0-9{}A-R])$/, shape: 'digits whose last byte carries the sign: "{" or "A"-"I" positive, "}" or "J"-"R" negative' }
|
|
34383
|
+
};
|
|
34384
|
+
OVERPUNCH_POSITIVE = "{ABCDEFGHI";
|
|
34385
|
+
OVERPUNCH_NEGATIVE = "}JKLMNOPQR";
|
|
34386
|
+
NumericCodecClass = class {
|
|
34387
|
+
constructor() {
|
|
34388
|
+
/**
|
|
34389
|
+
* Reads a number out of such a field. `"0012345"` at `scale: 2` is `123.45`; `"123J"` under
|
|
34390
|
+
* `sign: "overpunch"` is `-1231`. Both misreadings are silent — `12345` is as plausible an
|
|
34391
|
+
* amount as `123.45`, and `NaN` is what a run gets today — so the decoding is declared by the
|
|
34392
|
+
* layout and anything else is rejected.
|
|
34393
|
+
*
|
|
34394
|
+
* A value that is ALREADY a number has been decoded by whoever produced it (a JSON source, the
|
|
34395
|
+
* synthetic input, a re-read staged value) and passes through untouched: both properties
|
|
34396
|
+
* describe how the number is spelled in text, not the number.
|
|
34397
|
+
*/
|
|
34398
|
+
this.parse = (value, field) => {
|
|
34399
|
+
if (typeof value === "number") return value;
|
|
34400
|
+
const text = String(value).trim();
|
|
34401
|
+
const { digits, negative } = this.readSign(text, field.sign);
|
|
34402
|
+
if (!field.scale) return Number(negative ? `-${digits}` : digits);
|
|
34403
|
+
if (digits.includes("."))
|
|
34404
|
+
throw new Error(`Cannot read "${text}" as a number with ${field.scale} implied decimal(s): the last ${field.scale} digits are the fraction, so the value cannot also carry a decimal point.`);
|
|
34405
|
+
const padded = digits.padStart(field.scale + 1, "0");
|
|
34406
|
+
const split2 = padded.length - field.scale;
|
|
34407
|
+
return Number(`${negative ? "-" : ""}${padded.slice(0, split2)}.${padded.slice(split2)}`);
|
|
34408
|
+
};
|
|
34409
|
+
/**
|
|
34410
|
+
* Writes a number the way the field spells it — the mirror of `parse`, so a value read from such
|
|
34411
|
+
* a field can always be written back to one. `123.45` at `scale: 2` is `"12345"`, and `-1231`
|
|
34412
|
+
* under `sign: "overpunch"` is `"123J"`. The result is unpadded, since the width is the field's
|
|
34413
|
+
* to fill.
|
|
34414
|
+
*
|
|
34415
|
+
* Scaled through `toFixed`, which rounds in decimal: `(123.45).toFixed(2)` is `"123.45"` where
|
|
34416
|
+
* `123.45 * 100` is `12344.999999999998`.
|
|
34417
|
+
*/
|
|
34418
|
+
this.format = (value, field) => {
|
|
34419
|
+
const parsed = typeof value === "number" ? value : Number(value);
|
|
34420
|
+
if (!Number.isFinite(parsed))
|
|
34421
|
+
throw new Error(`Cannot write "${value}" as a number: it is not a number.`);
|
|
34422
|
+
const magnitude = Math.abs(parsed);
|
|
34423
|
+
const digits = field.scale ? magnitude.toFixed(field.scale).replace(".", "") : String(magnitude);
|
|
34424
|
+
return this.writeSign(digits, parsed < 0, field.sign);
|
|
34425
|
+
};
|
|
34426
|
+
/** Splits the written sign off a field, leaving the digits and whether the value is negative. */
|
|
34427
|
+
this.readSign = (text, sign2) => {
|
|
34428
|
+
const mode = SIGN_MODES2[sign2 ?? "none"];
|
|
34429
|
+
const match = mode.pattern.exec(text);
|
|
34430
|
+
if (!match)
|
|
34431
|
+
throw new Error(`Cannot read "${text}" as a number: this field holds ${mode.shape}.`);
|
|
34432
|
+
if (sign2 === "overpunch") return this.readOverpunch(match[1], match[2]);
|
|
34433
|
+
const [digits, mark] = sign2 === "trailing" ? [match[1], match[2]] : [match[2], match[1]];
|
|
34434
|
+
return { digits, negative: mark === "-" };
|
|
34435
|
+
};
|
|
34436
|
+
/** The last byte of an overpunched field, read back as its digit and its sign. */
|
|
34437
|
+
this.readOverpunch = (leading, last) => {
|
|
34438
|
+
const negative = OVERPUNCH_NEGATIVE.includes(last);
|
|
34439
|
+
const digit = negative ? OVERPUNCH_NEGATIVE.indexOf(last) : OVERPUNCH_POSITIVE.indexOf(last);
|
|
34440
|
+
return { digits: leading + (digit < 0 ? last : digit), negative };
|
|
34441
|
+
};
|
|
34442
|
+
/**
|
|
34443
|
+
* Puts the sign back the way the field carries it. A positive overpunched value is written in its
|
|
34444
|
+
* letter form (`123A`), which is what a signed DISPLAY field holds — the plain-digit form is the
|
|
34445
|
+
* unsigned one, and a field declaring `sign` is signed.
|
|
34446
|
+
*/
|
|
34447
|
+
this.writeSign = (digits, negative, sign2) => {
|
|
34448
|
+
switch (sign2) {
|
|
34449
|
+
case "overpunch": {
|
|
34450
|
+
const zone = negative ? OVERPUNCH_NEGATIVE : OVERPUNCH_POSITIVE;
|
|
34451
|
+
return digits.slice(0, -1) + zone[Number(digits.slice(-1))];
|
|
34452
|
+
}
|
|
34453
|
+
case "leading":
|
|
34454
|
+
return `${negative ? "-" : "+"}${digits}`;
|
|
34455
|
+
case "trailing":
|
|
34456
|
+
return `${digits}${negative ? "-" : "+"}`;
|
|
34457
|
+
default:
|
|
34458
|
+
return negative ? `-${digits}` : digits;
|
|
34459
|
+
}
|
|
34460
|
+
};
|
|
34461
|
+
}
|
|
34462
|
+
};
|
|
34463
|
+
NumericCodec = new NumericCodecClass();
|
|
34464
|
+
NumericCodec_default = NumericCodec;
|
|
34465
|
+
}
|
|
34466
|
+
});
|
|
34467
|
+
|
|
34231
34468
|
// ../../packages/parsing/src/FixedWidthWriter.ts
|
|
34232
|
-
var BASE_FILL, TYPE_FILL, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
|
|
34469
|
+
var BASE_FILL, TYPE_FILL, MAX_SCALE2, SIGN_MODES3, FixedWidthWriterClass, FixedWidthWriter, FixedWidthWriter_default;
|
|
34233
34470
|
var init_FixedWidthWriter = __esm({
|
|
34234
34471
|
"../../packages/parsing/src/FixedWidthWriter.ts"() {
|
|
34235
34472
|
init_src();
|
|
34236
34473
|
init_src3();
|
|
34474
|
+
init_NumericCodec();
|
|
34237
34475
|
BASE_FILL = { align: "left", pad: " ", nullFill: " ", overflow: "fail" };
|
|
34238
34476
|
TYPE_FILL = {
|
|
34239
34477
|
string: BASE_FILL,
|
|
@@ -34241,6 +34479,8 @@ var init_FixedWidthWriter = __esm({
|
|
|
34241
34479
|
datetime: BASE_FILL,
|
|
34242
34480
|
boolean: BASE_FILL
|
|
34243
34481
|
};
|
|
34482
|
+
MAX_SCALE2 = 15;
|
|
34483
|
+
SIGN_MODES3 = ["overpunch", "leading", "trailing"];
|
|
34244
34484
|
FixedWidthWriterClass = class {
|
|
34245
34485
|
constructor() {
|
|
34246
34486
|
/**
|
|
@@ -34421,6 +34661,24 @@ var init_FixedWidthWriter = __esm({
|
|
|
34421
34661
|
errors.push(`"position.pad" of ${where} must be a single character, not "${fill2.pad}"`);
|
|
34422
34662
|
if (fill2.nullFill.length !== 1)
|
|
34423
34663
|
errors.push(`"position.nullFill" of ${where} must be a single character, not "${fill2.nullFill}"`);
|
|
34664
|
+
errors.push(...this.spellingErrors(column2, position, where));
|
|
34665
|
+
return errors;
|
|
34666
|
+
};
|
|
34667
|
+
/**
|
|
34668
|
+
* How the field says its number is written, checked before a record ever reaches it. A wrong
|
|
34669
|
+
* `scale` writes a plausible amount that is 10^n off and a `sign` the mode cannot write throws
|
|
34670
|
+
* on the first record, so both are rejected at compile — on the column's own type, since neither
|
|
34671
|
+
* describes anything a string, a date or a boolean is spelled with.
|
|
34672
|
+
*/
|
|
34673
|
+
this.spellingErrors = (column2, position, where) => {
|
|
34674
|
+
const errors = [];
|
|
34675
|
+
const declared = [position.scale === void 0 || position.scale === null ? null : "scale", position.sign ? "sign" : null].filter(Boolean);
|
|
34676
|
+
if (declared.length > 0 && column2.type !== "number")
|
|
34677
|
+
errors.push(`${declared.map((x2) => `"position.${x2}"`).join(" and ")} of ${where} ${declared.length > 1 ? "describe" : "describes"} how a number is written, but the field is a "${column2.type ?? "typeless"}"`);
|
|
34678
|
+
if (position.scale !== void 0 && position.scale !== null && (!Number.isInteger(position.scale) || position.scale < 1 || position.scale > MAX_SCALE2))
|
|
34679
|
+
errors.push(`"position.scale" of ${where} counts implied decimal places, so it must be a whole number between 1 and ${MAX_SCALE2}, not ${JSON.stringify(position.scale)}. A field written with its decimal point simply omits it.`);
|
|
34680
|
+
if (position.sign && !SIGN_MODES3.includes(position.sign))
|
|
34681
|
+
errors.push(`"position.sign" of ${where} is ${JSON.stringify(position.sign)}, which is not a way a sign can be written. Use ${SIGN_MODES3.map((x2) => `"${x2}"`).join(", ")}, or omit it for a plain leading "-".`);
|
|
34424
34682
|
return errors;
|
|
34425
34683
|
};
|
|
34426
34684
|
/**
|
|
@@ -34446,7 +34704,9 @@ var init_FixedWidthWriter = __esm({
|
|
|
34446
34704
|
align: fill2.align,
|
|
34447
34705
|
pad: fill2.pad,
|
|
34448
34706
|
nullText: fill2.nullFill.repeat(position.width),
|
|
34449
|
-
overflow: fill2.overflow
|
|
34707
|
+
overflow: fill2.overflow,
|
|
34708
|
+
scale: position.scale,
|
|
34709
|
+
sign: position.sign
|
|
34450
34710
|
};
|
|
34451
34711
|
};
|
|
34452
34712
|
/** The three fill sources folded into one, nearest declaration first. */
|
|
@@ -34487,11 +34747,33 @@ var init_FixedWidthWriter = __esm({
|
|
|
34487
34747
|
* run; see `overflowed`.
|
|
34488
34748
|
*/
|
|
34489
34749
|
this.writeField = (value, field) => {
|
|
34490
|
-
if (value === null || value === void 0) return field.nullText;
|
|
34491
|
-
const text =
|
|
34492
|
-
if (text.length === 0) return field.nullText;
|
|
34750
|
+
if (value === null || value === void 0 || value === "") return field.nullText;
|
|
34751
|
+
const text = this.fieldText(value, field);
|
|
34493
34752
|
if (text.length > field.width) return this.overflowed(text, field);
|
|
34494
|
-
return
|
|
34753
|
+
return this.filled(text, field);
|
|
34754
|
+
};
|
|
34755
|
+
/**
|
|
34756
|
+
* The value as the text of its field. A field declaring `scale` or `sign` is spelled the way the
|
|
34757
|
+
* layout says a number is spelled — `123.45` at `scale: 2` written back as `12345`, `-1231`
|
|
34758
|
+
* under `sign: "overpunch"` as `123J` — and everything else is the value as it stands.
|
|
34759
|
+
*
|
|
34760
|
+
* Without this a read-and-write-back is silently wrong in both directions: a field read as
|
|
34761
|
+
* `0012345` would be written `123.45`, which is the right number spelled the wrong way, in the
|
|
34762
|
+
* one place nothing downstream re-reads.
|
|
34763
|
+
*/
|
|
34764
|
+
this.fieldText = (value, field) => {
|
|
34765
|
+
if (field.scale || field.sign) return NumericCodec_default.format(value, field);
|
|
34766
|
+
return typeof value === "string" ? value : String(value);
|
|
34767
|
+
};
|
|
34768
|
+
/**
|
|
34769
|
+
* The text filled out to the field's width, on the side away from `align`. A `leading` sign keeps
|
|
34770
|
+
* the first column to itself and the fill goes between it and the digits — `-0001234`, the way a
|
|
34771
|
+
* SIGN LEADING SEPARATE field is written — rather than ahead of a sign nothing would then find.
|
|
34772
|
+
*/
|
|
34773
|
+
this.filled = (text, field) => {
|
|
34774
|
+
if (field.align !== "right") return text.padEnd(field.width, field.pad);
|
|
34775
|
+
if (field.sign === "leading") return text.charAt(0) + text.slice(1).padStart(field.width - 1, field.pad);
|
|
34776
|
+
return text.padStart(field.width, field.pad);
|
|
34495
34777
|
};
|
|
34496
34778
|
/**
|
|
34497
34779
|
* A value longer than its field. It fails the run by default: a truncated account number is a
|
|
@@ -34500,14 +34782,17 @@ var init_FixedWidthWriter = __esm({
|
|
|
34500
34782
|
*
|
|
34501
34783
|
* Where the layout genuinely specifies a cut, the end that survives follows `align`, which is
|
|
34502
34784
|
* what a COBOL MOVE does: a left-aligned value keeps its head, a right-aligned one keeps its
|
|
34503
|
-
* low-order tail.
|
|
34785
|
+
* low-order tail. A `leading` sign survives either way — a cut that drops it writes the
|
|
34786
|
+
* magnitude with the direction reversed, which is worse than the cut itself.
|
|
34504
34787
|
*/
|
|
34505
34788
|
this.overflowed = (text, field) => {
|
|
34506
34789
|
Affirm_default(
|
|
34507
34790
|
field.overflow === "truncate",
|
|
34508
34791
|
`Value "${text}" is ${text.length} characters, longer than the ${field.width} of field "${field.name}" at columns ${field.start}-${field.end}. Widen the field, or set its "overflow" to "truncate" if the layout really specifies a cut.`
|
|
34509
34792
|
);
|
|
34510
|
-
|
|
34793
|
+
if (field.align !== "right") return text.slice(0, field.width);
|
|
34794
|
+
if (field.sign === "leading") return text.charAt(0) + text.slice(text.length - field.width + 1);
|
|
34795
|
+
return text.slice(text.length - field.width);
|
|
34511
34796
|
};
|
|
34512
34797
|
/**
|
|
34513
34798
|
* What a record carries after its last field, BEFORE the newline the destination writes. Every
|
|
@@ -34515,7 +34800,7 @@ var init_FixedWidthWriter = __esm({
|
|
|
34515
34800
|
* `lf` output needs nothing here and a `crlf` one needs the `\r` that turns that newline into a
|
|
34516
34801
|
* CRLF — which is exactly what a CRLF file looks like to anything that splits on `\n`.
|
|
34517
34802
|
*
|
|
34518
|
-
* A real positional feed picks one terminator and holds it:
|
|
34803
|
+
* A real positional feed picks one terminator and holds it: some are CRLF on
|
|
34519
34804
|
* every line including the last, and `lf` is the default everywhere else.
|
|
34520
34805
|
*/
|
|
34521
34806
|
this.lineSuffix = (output) => {
|
|
@@ -34587,7 +34872,7 @@ var require_pend = __commonJS({
|
|
|
34587
34872
|
// ../../node_modules/yauzl/fd-slicer.js
|
|
34588
34873
|
var require_fd_slicer = __commonJS({
|
|
34589
34874
|
"../../node_modules/yauzl/fd-slicer.js"(exports2) {
|
|
34590
|
-
var
|
|
34875
|
+
var fs43 = require("fs");
|
|
34591
34876
|
var util = require("util");
|
|
34592
34877
|
var stream = require("stream");
|
|
34593
34878
|
var Readable10 = stream.Readable;
|
|
@@ -34607,7 +34892,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34607
34892
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
34608
34893
|
var self2 = this;
|
|
34609
34894
|
self2.pend.go(function(cb) {
|
|
34610
|
-
|
|
34895
|
+
fs43.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
|
|
34611
34896
|
cb();
|
|
34612
34897
|
callback(err2, bytesRead, buffer2);
|
|
34613
34898
|
});
|
|
@@ -34624,7 +34909,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34624
34909
|
self2.refCount -= 1;
|
|
34625
34910
|
if (self2.refCount < 0) throw new Error("invalid unref");
|
|
34626
34911
|
if (self2.refCount > 0) return;
|
|
34627
|
-
|
|
34912
|
+
fs43.close(self2.fd, onCloseDone);
|
|
34628
34913
|
function onCloseDone(err2) {
|
|
34629
34914
|
if (err2) {
|
|
34630
34915
|
self2.emit("error", err2);
|
|
@@ -34656,7 +34941,7 @@ var require_fd_slicer = __commonJS({
|
|
|
34656
34941
|
}
|
|
34657
34942
|
self2.context.pend.go(function(cb) {
|
|
34658
34943
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
34659
|
-
|
|
34944
|
+
fs43.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
|
|
34660
34945
|
if (err2) {
|
|
34661
34946
|
self2.destroy(err2);
|
|
34662
34947
|
} else if (bytesRead === 0) {
|
|
@@ -35016,7 +35301,7 @@ var require_crc32 = __commonJS({
|
|
|
35016
35301
|
// ../../node_modules/yauzl/index.js
|
|
35017
35302
|
var require_yauzl = __commonJS({
|
|
35018
35303
|
"../../node_modules/yauzl/index.js"(exports2) {
|
|
35019
|
-
var
|
|
35304
|
+
var fs43 = require("fs");
|
|
35020
35305
|
var zlib3 = require("zlib");
|
|
35021
35306
|
var fd_slicer = require_fd_slicer();
|
|
35022
35307
|
var util = require("util");
|
|
@@ -35041,9 +35326,9 @@ var require_yauzl = __commonJS({
|
|
|
35041
35326
|
exports2.Entry = Entry;
|
|
35042
35327
|
exports2.LocalFileHeader = LocalFileHeader;
|
|
35043
35328
|
exports2.RandomAccessReader = RandomAccessReader;
|
|
35044
|
-
function openPromise(
|
|
35329
|
+
function openPromise(path32, options2) {
|
|
35045
35330
|
return new Promise((resolve, reject) => {
|
|
35046
|
-
open(
|
|
35331
|
+
open(path32, { ...options2, lazyEntries: true }, function(err2, zipfile) {
|
|
35047
35332
|
if (err2) return reject(err2);
|
|
35048
35333
|
resolve(zipfile);
|
|
35049
35334
|
});
|
|
@@ -35073,7 +35358,7 @@ var require_yauzl = __commonJS({
|
|
|
35073
35358
|
});
|
|
35074
35359
|
});
|
|
35075
35360
|
}
|
|
35076
|
-
function open(
|
|
35361
|
+
function open(path32, options2, callback) {
|
|
35077
35362
|
if (typeof options2 === "function") {
|
|
35078
35363
|
callback = options2;
|
|
35079
35364
|
options2 = null;
|
|
@@ -35085,10 +35370,10 @@ var require_yauzl = __commonJS({
|
|
|
35085
35370
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
35086
35371
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
35087
35372
|
if (callback == null) callback = defaultCallback;
|
|
35088
|
-
|
|
35373
|
+
fs43.open(path32, "r", function(err2, fd) {
|
|
35089
35374
|
if (err2) return callback(err2);
|
|
35090
35375
|
fromFd(fd, options2, function(err3, zipfile) {
|
|
35091
|
-
if (err3)
|
|
35376
|
+
if (err3) fs43.close(fd, defaultCallback);
|
|
35092
35377
|
callback(err3, zipfile);
|
|
35093
35378
|
});
|
|
35094
35379
|
});
|
|
@@ -35105,7 +35390,7 @@ var require_yauzl = __commonJS({
|
|
|
35105
35390
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
35106
35391
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
35107
35392
|
if (callback == null) callback = defaultCallback;
|
|
35108
|
-
|
|
35393
|
+
fs43.fstat(fd, function(err2, stats) {
|
|
35109
35394
|
if (err2) return callback(err2);
|
|
35110
35395
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
35111
35396
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -39187,6 +39472,7 @@ var init_src6 = __esm({
|
|
|
39187
39472
|
init_DatasetCodec();
|
|
39188
39473
|
init_FixedWidthParser();
|
|
39189
39474
|
init_FixedWidthWriter();
|
|
39475
|
+
init_NumericCodec();
|
|
39190
39476
|
init_ParseCompression();
|
|
39191
39477
|
init_ParseHelper();
|
|
39192
39478
|
init_XLSParser();
|
|
@@ -46861,6 +47147,16 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46861
47147
|
* sibling of the destination on purpose: rename is only atomic within one filesystem.
|
|
46862
47148
|
*/
|
|
46863
47149
|
this._stagingPath = (toFilePath) => `${toFilePath}.${(0, import_crypto4.randomUUID)()}.partial`;
|
|
47150
|
+
/**
|
|
47151
|
+
* Creates the folder an export is about to be written into.
|
|
47152
|
+
*
|
|
47153
|
+
* An `exportName` may carry a prefix — it is written for an S3 key, where `outbound/feed.csv`
|
|
47154
|
+
* needs no folder to exist — so a local destination has to make one. `saveFile` always did;
|
|
47155
|
+
* `move` did not, and failed with a bare ENOENT naming the staging file rather than the folder.
|
|
47156
|
+
*/
|
|
47157
|
+
this._ensureDirectory = async (toFilePath) => {
|
|
47158
|
+
await import_promises10.default.mkdir(import_path11.default.dirname(toFilePath), { recursive: true });
|
|
47159
|
+
};
|
|
46864
47160
|
/** Removes a staging file a failed export left behind; never masks the original error. */
|
|
46865
47161
|
this._discardStaging = async (stagingPath) => {
|
|
46866
47162
|
try {
|
|
@@ -46872,6 +47168,7 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46872
47168
|
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46873
47169
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46874
47170
|
try {
|
|
47171
|
+
await this._ensureDirectory(toFilePath);
|
|
46875
47172
|
if (header)
|
|
46876
47173
|
await DriverHelper_default.copyWithHeader(fromPath, stagingPath, header);
|
|
46877
47174
|
else
|
|
@@ -46889,6 +47186,7 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46889
47186
|
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46890
47187
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46891
47188
|
try {
|
|
47189
|
+
await this._ensureDirectory(toFilePath);
|
|
46892
47190
|
const reader = fs15.createReadStream(fromPath);
|
|
46893
47191
|
const writer = new BatchWriter_default(fs15.createWriteStream(stagingPath));
|
|
46894
47192
|
if (header)
|
|
@@ -47512,9 +47810,9 @@ var init_UrlMinter = __esm({
|
|
|
47512
47810
|
return config;
|
|
47513
47811
|
};
|
|
47514
47812
|
/** Walks a dot path (`data.uploadUrl`) into the response, answering undefined rather than throwing. */
|
|
47515
|
-
this._resolve = (payload,
|
|
47813
|
+
this._resolve = (payload, path32) => {
|
|
47516
47814
|
let current = payload;
|
|
47517
|
-
for (const segment of
|
|
47815
|
+
for (const segment of path32.split(".")) {
|
|
47518
47816
|
if (current === null || typeof current !== "object")
|
|
47519
47817
|
return void 0;
|
|
47520
47818
|
current = current[segment];
|
|
@@ -48024,6 +48322,7 @@ var import_dayjs, import_customParseFormat, import_utc, DAY_OF_YEAR_TOKEN, FORMA
|
|
|
48024
48322
|
var init_TypeCaster = __esm({
|
|
48025
48323
|
"../../packages/engines/src/transform/TypeCaster.ts"() {
|
|
48026
48324
|
init_src();
|
|
48325
|
+
init_src6();
|
|
48027
48326
|
import_dayjs = __toESM(require("dayjs"), 1);
|
|
48028
48327
|
import_customParseFormat = __toESM(require("dayjs/plugin/customParseFormat"), 1);
|
|
48029
48328
|
import_utc = __toESM(require("dayjs/plugin/utc"), 1);
|
|
@@ -48065,6 +48364,7 @@ var init_TypeCaster = __esm({
|
|
|
48065
48364
|
this.castDimension = (value, dimension) => {
|
|
48066
48365
|
if (dimension.type && this.isMissing(value, dimension.type))
|
|
48067
48366
|
return dimension.nullable === false ? TYPE_DEFAULTS[dimension.type] : null;
|
|
48367
|
+
if (dimension.scale || dimension.sign) return NumericCodec_default.parse(value, dimension);
|
|
48068
48368
|
return this.cast(value, dimension.type, dimension.format);
|
|
48069
48369
|
};
|
|
48070
48370
|
/** null/undefined for any type, plus the empty string for every type but `string`. */
|
|
@@ -51473,7 +51773,10 @@ var init_SyntheticInputEngine = __esm({
|
|
|
51473
51773
|
name,
|
|
51474
51774
|
type: dim.type ?? "string",
|
|
51475
51775
|
format: dim.format,
|
|
51476
|
-
|
|
51776
|
+
// An implied-decimal field holds exactly `scale` decimals, so that is what a generated
|
|
51777
|
+
// value gets unless `synth` asks for another: a 4-decimal value in a `scale: 2` column
|
|
51778
|
+
// is one the layout cannot write back.
|
|
51779
|
+
decimals: synth2.decimals ?? dim.scale,
|
|
51477
51780
|
min,
|
|
51478
51781
|
max,
|
|
51479
51782
|
isPk: dim.pk === true,
|
|
@@ -51636,6 +51939,652 @@ var init_SyntheticInputEngine = __esm({
|
|
|
51636
51939
|
}
|
|
51637
51940
|
});
|
|
51638
51941
|
|
|
51942
|
+
// ../../packages/engines/src/copybook/CopybookParser.ts
|
|
51943
|
+
var MAX_RECORD_WIDTH, USAGES, UNSUPPORTED_USAGES, CLAUSE_KEYWORDS, PICTURE_CHARS, EDIT_CHARS, DIGIT_CHARS, COMMENT_INDICATORS, INDICATORS, CopybookParserClass, CopybookParser, CopybookParser_default;
|
|
51944
|
+
var init_CopybookParser = __esm({
|
|
51945
|
+
"../../packages/engines/src/copybook/CopybookParser.ts"() {
|
|
51946
|
+
init_src();
|
|
51947
|
+
MAX_RECORD_WIDTH = 32767;
|
|
51948
|
+
USAGES = {
|
|
51949
|
+
"DISPLAY": "display",
|
|
51950
|
+
"COMP": "comp",
|
|
51951
|
+
"COMPUTATIONAL": "comp",
|
|
51952
|
+
"COMP-4": "comp",
|
|
51953
|
+
"COMPUTATIONAL-4": "comp",
|
|
51954
|
+
"COMP-5": "comp",
|
|
51955
|
+
"COMPUTATIONAL-5": "comp",
|
|
51956
|
+
"BINARY": "comp",
|
|
51957
|
+
"COMP-3": "comp-3",
|
|
51958
|
+
"COMPUTATIONAL-3": "comp-3",
|
|
51959
|
+
"PACKED-DECIMAL": "comp-3"
|
|
51960
|
+
};
|
|
51961
|
+
UNSUPPORTED_USAGES = {
|
|
51962
|
+
"COMP-1": "single-precision hexadecimal floating point",
|
|
51963
|
+
"COMPUTATIONAL-1": "single-precision hexadecimal floating point",
|
|
51964
|
+
"COMP-2": "double-precision hexadecimal floating point",
|
|
51965
|
+
"COMPUTATIONAL-2": "double-precision hexadecimal floating point",
|
|
51966
|
+
"INDEX": "a subscript, which holds no data of the record",
|
|
51967
|
+
"POINTER": "an address, which holds no data of the record",
|
|
51968
|
+
"DISPLAY-1": "double-byte characters"
|
|
51969
|
+
};
|
|
51970
|
+
CLAUSE_KEYWORDS = /* @__PURE__ */ new Set([
|
|
51971
|
+
"PIC",
|
|
51972
|
+
"PICTURE",
|
|
51973
|
+
"USAGE",
|
|
51974
|
+
"REDEFINES",
|
|
51975
|
+
"OCCURS",
|
|
51976
|
+
"SIGN",
|
|
51977
|
+
"VALUE",
|
|
51978
|
+
"VALUES",
|
|
51979
|
+
"JUST",
|
|
51980
|
+
"JUSTIFIED",
|
|
51981
|
+
"SYNC",
|
|
51982
|
+
"SYNCHRONIZED",
|
|
51983
|
+
"BLANK",
|
|
51984
|
+
"ASCENDING",
|
|
51985
|
+
"DESCENDING",
|
|
51986
|
+
"INDEXED",
|
|
51987
|
+
"EXTERNAL",
|
|
51988
|
+
"GLOBAL",
|
|
51989
|
+
...Object.keys(USAGES),
|
|
51990
|
+
...Object.keys(UNSUPPORTED_USAGES)
|
|
51991
|
+
]);
|
|
51992
|
+
PICTURE_CHARS = "XA9VSPZ*$,B0/.+-";
|
|
51993
|
+
EDIT_CHARS = "Z*$,B0/";
|
|
51994
|
+
DIGIT_CHARS = "9Z*";
|
|
51995
|
+
COMMENT_INDICATORS = "*/";
|
|
51996
|
+
INDICATORS = " */-";
|
|
51997
|
+
CopybookParserClass = class {
|
|
51998
|
+
constructor() {
|
|
51999
|
+
/**
|
|
52000
|
+
* Parses copybook text into one record layout. Throws on the first clause it cannot read, naming
|
|
52001
|
+
* the line it is on.
|
|
52002
|
+
*/
|
|
52003
|
+
this.parse = (text) => {
|
|
52004
|
+
Affirm_default(typeof text === "string", "Missing required parameter text");
|
|
52005
|
+
const statements = this._statements(this._lines(text));
|
|
52006
|
+
if (statements.length === 0)
|
|
52007
|
+
throw new Error("The copybook holds no data description entries.");
|
|
52008
|
+
const entries = statements.map((statement) => this._entry(statement)).filter((entry) => entry);
|
|
52009
|
+
const [record, ...alternates] = this._tree(entries);
|
|
52010
|
+
if (!record)
|
|
52011
|
+
throw new Error("The copybook declares no record: there is no 01 level item.");
|
|
52012
|
+
for (const alternate of alternates) {
|
|
52013
|
+
if (!alternate.redefines)
|
|
52014
|
+
this._fail(alternate.line, `"${alternate.name}" is a second record description, and a producer reads one record layout. Split the copybook, and read each record type with its own producer and a "recordSelector".`);
|
|
52015
|
+
}
|
|
52016
|
+
const fields = record.fields ?? [record];
|
|
52017
|
+
const width = this._place(fields);
|
|
52018
|
+
const redefinedBy = alternates.map((x2) => x2.name);
|
|
52019
|
+
return { name: record.name, line: record.line, width, fields, redefinedBy: redefinedBy.length > 0 ? redefinedBy : void 0 };
|
|
52020
|
+
};
|
|
52021
|
+
/** Throws about the copybook line a problem is on, which is the only way to locate it. */
|
|
52022
|
+
this._fail = (line, message) => {
|
|
52023
|
+
throw new Error(`Copybook line ${line}: ${message}`);
|
|
52024
|
+
};
|
|
52025
|
+
/**
|
|
52026
|
+
* The lines that hold definitions, with their comments and their sequence and identification
|
|
52027
|
+
* areas removed. Fixed-format and free-format copybooks are both read, decided per line: a line
|
|
52028
|
+
* whose first six characters are a sequence number (or blank) and whose seventh is an indicator
|
|
52029
|
+
* is read as columns 8-72, and any other line is read whole.
|
|
52030
|
+
*/
|
|
52031
|
+
this._lines = (text) => {
|
|
52032
|
+
const lines = [];
|
|
52033
|
+
text.split(/\r?\n/).forEach((raw, index) => {
|
|
52034
|
+
const fixed = /^(\d{6}| {6})$/.test(raw.slice(0, 6)) && INDICATORS.includes(raw.charAt(6));
|
|
52035
|
+
if (fixed && COMMENT_INDICATORS.includes(raw.charAt(6))) return;
|
|
52036
|
+
const content = fixed ? raw.slice(7, 72) : raw;
|
|
52037
|
+
if (content.trim().length === 0 || content.trimStart().startsWith("*")) return;
|
|
52038
|
+
lines.push({ text: content, line: index + 1 });
|
|
52039
|
+
});
|
|
52040
|
+
return lines;
|
|
52041
|
+
};
|
|
52042
|
+
/**
|
|
52043
|
+
* The data description entries, split on the periods that terminate them. A period only
|
|
52044
|
+
* terminates where it is followed by a space or the end of the line, so the one inside
|
|
52045
|
+
* `PIC 9(5).99` stays part of the picture.
|
|
52046
|
+
*/
|
|
52047
|
+
this._statements = (lines) => {
|
|
52048
|
+
const statements = [];
|
|
52049
|
+
let current = null;
|
|
52050
|
+
for (const { text, line } of lines) {
|
|
52051
|
+
for (const token2 of this._tokens(text, line)) {
|
|
52052
|
+
if (!current) current = { tokens: [], line };
|
|
52053
|
+
if (token2 !== ".") {
|
|
52054
|
+
current.tokens.push(token2);
|
|
52055
|
+
continue;
|
|
52056
|
+
}
|
|
52057
|
+
if (current.tokens.length > 0) statements.push(current);
|
|
52058
|
+
current = null;
|
|
52059
|
+
}
|
|
52060
|
+
}
|
|
52061
|
+
if (current)
|
|
52062
|
+
this._fail(current.line, `the entry starting "${current.tokens.slice(0, 3).join(" ")}" is never terminated by a period.`);
|
|
52063
|
+
return statements;
|
|
52064
|
+
};
|
|
52065
|
+
/** One line's tokens: quoted literals whole, a terminating period on its own, words otherwise. */
|
|
52066
|
+
this._tokens = (content, line) => {
|
|
52067
|
+
const tokens = [];
|
|
52068
|
+
let index = 0;
|
|
52069
|
+
while (index < content.length) {
|
|
52070
|
+
const char = content.charAt(index);
|
|
52071
|
+
if (/\s/.test(char)) {
|
|
52072
|
+
index++;
|
|
52073
|
+
continue;
|
|
52074
|
+
}
|
|
52075
|
+
if (char === '"' || char === "'") {
|
|
52076
|
+
const close = content.indexOf(char, index + 1);
|
|
52077
|
+
if (close < 0) this._fail(line, `a ${char === '"' ? "double" : "single"}-quoted literal is never closed.`);
|
|
52078
|
+
tokens.push(content.slice(index, close + 1));
|
|
52079
|
+
index = close + 1;
|
|
52080
|
+
continue;
|
|
52081
|
+
}
|
|
52082
|
+
if (this._terminates(content, index)) {
|
|
52083
|
+
tokens.push(".");
|
|
52084
|
+
index++;
|
|
52085
|
+
continue;
|
|
52086
|
+
}
|
|
52087
|
+
let end = index;
|
|
52088
|
+
while (end < content.length && !/\s/.test(content.charAt(end)) && !this._terminates(content, end)) end++;
|
|
52089
|
+
tokens.push(content.slice(index, end));
|
|
52090
|
+
index = end;
|
|
52091
|
+
}
|
|
52092
|
+
return tokens;
|
|
52093
|
+
};
|
|
52094
|
+
/** Whether the character at `index` is a period that ends an entry rather than one inside it. */
|
|
52095
|
+
this._terminates = (content, index) => {
|
|
52096
|
+
if (content.charAt(index) !== ".") return false;
|
|
52097
|
+
const next = content.charAt(index + 1);
|
|
52098
|
+
return next === "" || /\s/.test(next);
|
|
52099
|
+
};
|
|
52100
|
+
/**
|
|
52101
|
+
* One data description entry: its level, its name and its clauses. Returns `null` for a level 88
|
|
52102
|
+
* condition name, which describes values of the item above it and occupies no storage.
|
|
52103
|
+
*/
|
|
52104
|
+
this._entry = (statement) => {
|
|
52105
|
+
const { tokens, line } = statement;
|
|
52106
|
+
const level = Number(tokens[0]);
|
|
52107
|
+
if (!Number.isInteger(level) || !/^\d{1,2}$/.test(tokens[0]))
|
|
52108
|
+
this._fail(line, `"${tokens[0]}" is not a level number: an entry starts with one, written as 01-49, 66, 77 or 88.`);
|
|
52109
|
+
if (level === 88) return null;
|
|
52110
|
+
if (level === 66)
|
|
52111
|
+
this._fail(line, `level 66 (RENAMES) regroups fields that are already described: read them under their own names instead.`);
|
|
52112
|
+
if (level < 1 || level > 49 && level !== 77)
|
|
52113
|
+
this._fail(line, `level ${tokens[0]} is not a level a data item can be declared at (01-49, 66, 77 or 88).`);
|
|
52114
|
+
const named = tokens.length > 1 && !CLAUSE_KEYWORDS.has(tokens[1].toUpperCase());
|
|
52115
|
+
const name = named ? tokens[1] : "FILLER";
|
|
52116
|
+
const clauses = {};
|
|
52117
|
+
let index = named ? 2 : 1;
|
|
52118
|
+
while (index < tokens.length)
|
|
52119
|
+
index = this._clause(clauses, tokens, index, statement);
|
|
52120
|
+
return this._field(level, name, clauses, line);
|
|
52121
|
+
};
|
|
52122
|
+
/**
|
|
52123
|
+
* Reads the clause starting at `index` into `clauses` and answers where the next one begins. The
|
|
52124
|
+
* branches live in one place so that an unreadable clause is reported from one place too.
|
|
52125
|
+
*/
|
|
52126
|
+
this._clause = (clauses, tokens, index, statement) => {
|
|
52127
|
+
const keyword = tokens[index].toUpperCase();
|
|
52128
|
+
const { line } = statement;
|
|
52129
|
+
switch (keyword) {
|
|
52130
|
+
case "PIC":
|
|
52131
|
+
case "PICTURE":
|
|
52132
|
+
return this._picture(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52133
|
+
case "USAGE":
|
|
52134
|
+
return this._usage(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52135
|
+
case "REDEFINES":
|
|
52136
|
+
return this._redefines(clauses, tokens, index + 1, line);
|
|
52137
|
+
case "OCCURS":
|
|
52138
|
+
return this._occurs(clauses, tokens, index + 1, line);
|
|
52139
|
+
case "SIGN":
|
|
52140
|
+
return this._sign(clauses, tokens, this._skip(tokens, index + 1, "IS"), line);
|
|
52141
|
+
case "VALUE":
|
|
52142
|
+
case "VALUES":
|
|
52143
|
+
case "JUST":
|
|
52144
|
+
case "JUSTIFIED":
|
|
52145
|
+
case "SYNC":
|
|
52146
|
+
case "SYNCHRONIZED":
|
|
52147
|
+
case "BLANK":
|
|
52148
|
+
case "ASCENDING":
|
|
52149
|
+
case "DESCENDING":
|
|
52150
|
+
case "INDEXED":
|
|
52151
|
+
case "EXTERNAL":
|
|
52152
|
+
case "GLOBAL":
|
|
52153
|
+
return this._ignore(tokens, index + 1);
|
|
52154
|
+
default:
|
|
52155
|
+
if (USAGES[keyword] || UNSUPPORTED_USAGES[keyword]) return this._usage(clauses, tokens, index, line);
|
|
52156
|
+
return this._fail(line, `"${tokens[index]}" is not a clause this reads, in the definition of "${tokens[1] ?? tokens[0]}".`);
|
|
52157
|
+
}
|
|
52158
|
+
};
|
|
52159
|
+
/** Steps past an optional noise word (`IS`, `TO`, `BY`, `ON`), which changes no meaning. */
|
|
52160
|
+
this._skip = (tokens, index, word2) => {
|
|
52161
|
+
return tokens[index]?.toUpperCase() === word2 ? index + 1 : index;
|
|
52162
|
+
};
|
|
52163
|
+
/** Consumes a clause whose content changes no position: its operands, up to the next clause. */
|
|
52164
|
+
this._ignore = (tokens, index) => {
|
|
52165
|
+
let next = index;
|
|
52166
|
+
while (next < tokens.length && !CLAUSE_KEYWORDS.has(tokens[next].toUpperCase())) next++;
|
|
52167
|
+
return next;
|
|
52168
|
+
};
|
|
52169
|
+
this._picture = (clauses, tokens, index, line) => {
|
|
52170
|
+
if (!tokens[index]) this._fail(line, "a PIC clause states no picture.");
|
|
52171
|
+
if (clauses.picture) this._fail(line, "the entry declares two PIC clauses.");
|
|
52172
|
+
clauses.picture = tokens[index];
|
|
52173
|
+
return index + 1;
|
|
52174
|
+
};
|
|
52175
|
+
this._usage = (clauses, tokens, index, line) => {
|
|
52176
|
+
const word2 = tokens[index]?.toUpperCase() ?? "";
|
|
52177
|
+
if (UNSUPPORTED_USAGES[word2])
|
|
52178
|
+
this._fail(line, `USAGE ${word2} is ${UNSUPPORTED_USAGES[word2]}, which Remora does not read.`);
|
|
52179
|
+
if (!USAGES[word2])
|
|
52180
|
+
this._fail(line, `"${tokens[index]}" is not a storage mode a USAGE clause can name.`);
|
|
52181
|
+
clauses.usage = USAGES[word2];
|
|
52182
|
+
return index + 1;
|
|
52183
|
+
};
|
|
52184
|
+
this._redefines = (clauses, tokens, index, line) => {
|
|
52185
|
+
if (!tokens[index]) this._fail(line, "a REDEFINES clause names no item.");
|
|
52186
|
+
clauses.redefines = tokens[index];
|
|
52187
|
+
return index + 1;
|
|
52188
|
+
};
|
|
52189
|
+
/** `OCCURS n [TO m] [TIMES] [DEPENDING [ON] name]`, the fixed and the varying repeat counts. */
|
|
52190
|
+
this._occurs = (clauses, tokens, index, line) => {
|
|
52191
|
+
const count = Number(tokens[index]);
|
|
52192
|
+
if (!Number.isInteger(count) || count < 1)
|
|
52193
|
+
this._fail(line, `"${tokens[index]}" is not a number of occurrences: OCCURS takes a whole number of 1 or more.`);
|
|
52194
|
+
let next = index + 1;
|
|
52195
|
+
clauses.occurs = count;
|
|
52196
|
+
if (tokens[next]?.toUpperCase() === "TO") {
|
|
52197
|
+
const max = Number(tokens[next + 1]);
|
|
52198
|
+
if (!Number.isInteger(max) || max < count)
|
|
52199
|
+
this._fail(line, `"${tokens[next + 1]}" is not an upper bound of 'OCCURS ${count} TO': it must be a whole number of ${count} or more.`);
|
|
52200
|
+
clauses.occursMin = count;
|
|
52201
|
+
clauses.occurs = max;
|
|
52202
|
+
next += 2;
|
|
52203
|
+
}
|
|
52204
|
+
next = this._skip(tokens, next, "TIMES");
|
|
52205
|
+
if (tokens[next]?.toUpperCase() !== "DEPENDING") return next;
|
|
52206
|
+
next = this._skip(tokens, next + 1, "ON");
|
|
52207
|
+
if (!tokens[next]) this._fail(line, "an OCCURS DEPENDING ON clause names no counting field.");
|
|
52208
|
+
clauses.dependingOn = tokens[next];
|
|
52209
|
+
return next + 1;
|
|
52210
|
+
};
|
|
52211
|
+
/** `SIGN [IS] LEADING|TRAILING [SEPARATE [CHARACTER]]`, which says where a written sign sits. */
|
|
52212
|
+
this._sign = (clauses, tokens, index, line) => {
|
|
52213
|
+
const word2 = tokens[index]?.toUpperCase() ?? "";
|
|
52214
|
+
if (word2 !== "LEADING" && word2 !== "TRAILING")
|
|
52215
|
+
this._fail(line, `"${tokens[index]}" is not a sign position: a SIGN clause takes LEADING or TRAILING.`);
|
|
52216
|
+
clauses.signPosition = word2 === "LEADING" ? "leading" : "trailing";
|
|
52217
|
+
const next = this._skip(tokens, index + 1, "SEPARATE");
|
|
52218
|
+
if (next === index + 1) return next;
|
|
52219
|
+
clauses.signSeparate = true;
|
|
52220
|
+
return this._skip(tokens, next, "CHARACTER");
|
|
52221
|
+
};
|
|
52222
|
+
/** The parsed entry, with the picture and the sign clause finally read against each other. */
|
|
52223
|
+
this._field = (level, name, clauses, line) => {
|
|
52224
|
+
const field = {
|
|
52225
|
+
name,
|
|
52226
|
+
level,
|
|
52227
|
+
line,
|
|
52228
|
+
offset: 0,
|
|
52229
|
+
width: 0,
|
|
52230
|
+
occurs: clauses.occurs ?? 1,
|
|
52231
|
+
occursMin: clauses.occursMin,
|
|
52232
|
+
dependingOn: clauses.dependingOn,
|
|
52233
|
+
redefines: clauses.redefines,
|
|
52234
|
+
filler: name.toUpperCase() === "FILLER",
|
|
52235
|
+
picture: clauses.picture,
|
|
52236
|
+
usage: clauses.usage ?? "display"
|
|
52237
|
+
};
|
|
52238
|
+
if (!clauses.picture) return field;
|
|
52239
|
+
return { ...field, ...this._measure(clauses, line) };
|
|
52240
|
+
};
|
|
52241
|
+
/**
|
|
52242
|
+
* What a picture says the field is: how wide it is in characters, what its value reads as, where
|
|
52243
|
+
* its decimal point is implied and how its sign is written. Every one of those is silent when it
|
|
52244
|
+
* is wrong — a mis-measured width shifts every field after it — so an unreadable picture fails.
|
|
52245
|
+
*/
|
|
52246
|
+
this._measure = (clauses, line) => {
|
|
52247
|
+
const chars2 = this._expand(clauses.picture, line);
|
|
52248
|
+
const signed = chars2[0] === "S";
|
|
52249
|
+
const body = signed ? chars2.slice(1) : chars2;
|
|
52250
|
+
if (body.includes("S")) this._fail(line, `"${clauses.picture}": 'S' can only be a picture's first character.`);
|
|
52251
|
+
if (body.includes("P")) this._fail(line, `"${clauses.picture}": 'P' scales a value by a power of ten with no digit to show for it, which no Remora dimension can state.`);
|
|
52252
|
+
const point = body.indexOf("V");
|
|
52253
|
+
if (point !== body.lastIndexOf("V")) this._fail(line, `"${clauses.picture}" states more than one implied decimal point.`);
|
|
52254
|
+
if (point >= 0 && body.includes(".")) this._fail(line, `"${clauses.picture}" states both an implied ('V') and a real ('.') decimal point.`);
|
|
52255
|
+
const width = body.filter((x2) => x2 !== "V").length + (signed && clauses.signSeparate ? 1 : 0);
|
|
52256
|
+
if (body.some((x2) => x2 === "X" || x2 === "A")) {
|
|
52257
|
+
if (signed || point >= 0)
|
|
52258
|
+
this._fail(line, `"${clauses.picture}" mixes character positions with 'S' or 'V', which only describe a number.`);
|
|
52259
|
+
return { width, type: "string" };
|
|
52260
|
+
}
|
|
52261
|
+
if (!body.some((x2) => DIGIT_CHARS.includes(x2)))
|
|
52262
|
+
this._fail(line, `"${clauses.picture}" has no digit and no character position: there is nothing for the field to hold.`);
|
|
52263
|
+
const scale = point < 0 ? 0 : body.slice(point).filter((x2) => DIGIT_CHARS.includes(x2)).length;
|
|
52264
|
+
return { width, scale: scale > 0 ? scale : void 0, ...this._numeric(body, clauses, signed, line) };
|
|
52265
|
+
};
|
|
52266
|
+
/**
|
|
52267
|
+
* How a numeric field's value is written. An EDITED picture holds insertion characters as well
|
|
52268
|
+
* as digits (`ZZ,ZZ9.99`), so its field is text: reading it as a number would mean deciding what
|
|
52269
|
+
* the insertions mean, and deciding is what this parser does not do.
|
|
52270
|
+
*/
|
|
52271
|
+
this._numeric = (body, clauses, signed, line) => {
|
|
52272
|
+
const edits = body.filter((x2) => EDIT_CHARS.includes(x2));
|
|
52273
|
+
const signs = body.filter((x2) => x2 === "+" || x2 === "-");
|
|
52274
|
+
const floating = signs.length > 1 || signs.length === 1 && body[0] !== signs[0] && body[body.length - 1] !== signs[0];
|
|
52275
|
+
if (edits.length > 0 || floating)
|
|
52276
|
+
return { type: "string", edited: `its picture "${clauses.picture}" is edited (${Algo_default.uniq([...edits, ...signs]).join(" ")}), so the field holds those characters as well as digits` };
|
|
52277
|
+
if (signs.length === 1)
|
|
52278
|
+
return { type: "number", sign: body[0] === signs[0] ? "leading" : "trailing" };
|
|
52279
|
+
if (!signed)
|
|
52280
|
+
return { type: "number" };
|
|
52281
|
+
if (clauses.signSeparate)
|
|
52282
|
+
return { type: "number", sign: clauses.signPosition ?? "trailing" };
|
|
52283
|
+
if (clauses.signPosition === "leading")
|
|
52284
|
+
this._fail(line, `"${clauses.picture}" with SIGN IS LEADING carries its sign in the zone of the FIRST byte, and Remora reads an overpunched sign from the last. Ask for SIGN IS LEADING SEPARATE, or for a trailing sign.`);
|
|
52285
|
+
return { type: "number", sign: "overpunch" };
|
|
52286
|
+
};
|
|
52287
|
+
/** A picture with its repeat counts expanded: `S9(3)V99` becomes `S 9 9 9 V 9 9`. */
|
|
52288
|
+
this._expand = (picture, line) => {
|
|
52289
|
+
const text = picture.toUpperCase().replace(/CR|DB/g, "BB");
|
|
52290
|
+
const chars2 = [];
|
|
52291
|
+
let index = 0;
|
|
52292
|
+
while (index < text.length) {
|
|
52293
|
+
const char = text.charAt(index);
|
|
52294
|
+
if (char !== "(") {
|
|
52295
|
+
if (!PICTURE_CHARS.includes(char)) this._fail(line, `"${picture}" holds "${char}", which is not a picture character.`);
|
|
52296
|
+
chars2.push(char);
|
|
52297
|
+
index++;
|
|
52298
|
+
continue;
|
|
52299
|
+
}
|
|
52300
|
+
const close = text.indexOf(")", index);
|
|
52301
|
+
const count = Number(text.slice(index + 1, close));
|
|
52302
|
+
if (close < 0 || !Number.isInteger(count) || count < 1)
|
|
52303
|
+
this._fail(line, `"${picture}" has a repeat count that is not a whole number of 1 or more.`);
|
|
52304
|
+
if (chars2.length === 0) this._fail(line, `"${picture}" opens with a repeat count, which repeats nothing.`);
|
|
52305
|
+
if (chars2.length + count > MAX_RECORD_WIDTH) this._fail(line, `"${picture}" expands past the ${MAX_RECORD_WIDTH}-character maximum.`);
|
|
52306
|
+
const repeated = chars2[chars2.length - 1];
|
|
52307
|
+
for (let i6 = 1; i6 < count; i6++) chars2.push(repeated);
|
|
52308
|
+
index = close + 1;
|
|
52309
|
+
}
|
|
52310
|
+
return chars2;
|
|
52311
|
+
};
|
|
52312
|
+
/**
|
|
52313
|
+
* The entries nested by their level numbers: each one belongs to the last entry declared at a
|
|
52314
|
+
* lower level. `01` and `77` always start a record of their own, whatever precedes them.
|
|
52315
|
+
*/
|
|
52316
|
+
this._tree = (entries) => {
|
|
52317
|
+
const roots = [];
|
|
52318
|
+
const stack = [];
|
|
52319
|
+
for (const entry of entries) {
|
|
52320
|
+
if (entry.level === 1 || entry.level === 77) stack.length = 0;
|
|
52321
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= entry.level) stack.pop();
|
|
52322
|
+
const parent = stack[stack.length - 1];
|
|
52323
|
+
if (!parent) roots.push(entry);
|
|
52324
|
+
else parent.fields = [...parent.fields ?? [], entry];
|
|
52325
|
+
stack.push(entry);
|
|
52326
|
+
}
|
|
52327
|
+
return roots;
|
|
52328
|
+
};
|
|
52329
|
+
/**
|
|
52330
|
+
* Places every item: an offset inside the item that contains it, and a width of one occurrence.
|
|
52331
|
+
* This is the whole point of reading the copybook rather than a spreadsheet of columns — an
|
|
52332
|
+
* offset nobody typed is an offset nobody mistyped. Answers the width of the items as a whole.
|
|
52333
|
+
*/
|
|
52334
|
+
this._place = (fields) => {
|
|
52335
|
+
let cursor = 1;
|
|
52336
|
+
for (const field of fields) {
|
|
52337
|
+
if (field.fields && field.picture)
|
|
52338
|
+
this._fail(field.line, `"${field.name}" has both a PIC clause and items declared under it.`);
|
|
52339
|
+
if (field.fields) field.width = this._place(field.fields);
|
|
52340
|
+
if (!field.width)
|
|
52341
|
+
this._fail(field.line, `"${field.name}" has no PIC clause and no items under it, so its width is unknown.`);
|
|
52342
|
+
if (field.redefines) {
|
|
52343
|
+
this._redefine(fields, field);
|
|
52344
|
+
continue;
|
|
52345
|
+
}
|
|
52346
|
+
field.offset = cursor;
|
|
52347
|
+
cursor += field.width * field.occurs;
|
|
52348
|
+
if (cursor - 1 > MAX_RECORD_WIDTH)
|
|
52349
|
+
this._fail(field.line, `the record passes the ${MAX_RECORD_WIDTH}-character maximum at "${field.name}".`);
|
|
52350
|
+
}
|
|
52351
|
+
return cursor - 1;
|
|
52352
|
+
};
|
|
52353
|
+
/** Lays a REDEFINES item over the item it redefines, which is where its bytes already are. */
|
|
52354
|
+
this._redefine = (siblings, field) => {
|
|
52355
|
+
const target = siblings.find((x2) => x2 !== field && x2.name === field.redefines);
|
|
52356
|
+
if (!target)
|
|
52357
|
+
this._fail(field.line, `"${field.name}" REDEFINES "${field.redefines}", which is not an item declared beside it.`);
|
|
52358
|
+
if (field.width > target.width)
|
|
52359
|
+
this._fail(field.line, `"${field.name}" is ${field.width} characters and REDEFINES "${target.name}", which is ${target.width}: a redefinition cannot be wider than what it redefines.`);
|
|
52360
|
+
field.offset = target.offset;
|
|
52361
|
+
target.redefinedBy = [...target.redefinedBy ?? [], field.name];
|
|
52362
|
+
};
|
|
52363
|
+
}
|
|
52364
|
+
};
|
|
52365
|
+
CopybookParser = new CopybookParserClass();
|
|
52366
|
+
CopybookParser_default = CopybookParser;
|
|
52367
|
+
}
|
|
52368
|
+
});
|
|
52369
|
+
|
|
52370
|
+
// ../../packages/engines/src/copybook/CopybookEmitter.ts
|
|
52371
|
+
var DEFAULT_SOURCE, REPEATED_FIELD_NAMING, CopybookEmitterClass, CopybookEmitter, CopybookEmitter_default;
|
|
52372
|
+
var init_CopybookEmitter = __esm({
|
|
52373
|
+
"../../packages/engines/src/copybook/CopybookEmitter.ts"() {
|
|
52374
|
+
init_src();
|
|
52375
|
+
init_src4();
|
|
52376
|
+
DEFAULT_SOURCE = "s_default";
|
|
52377
|
+
REPEATED_FIELD_NAMING = "{group}{index}";
|
|
52378
|
+
CopybookEmitterClass = class {
|
|
52379
|
+
constructor() {
|
|
52380
|
+
/**
|
|
52381
|
+
* Emits the producer that reads the record this layout describes, with the notes a reader has
|
|
52382
|
+
* to act on. Throws rather than emitting a layout the config loader would reject: a layout that
|
|
52383
|
+
* is 95% right reads plausible values out of the wrong columns.
|
|
52384
|
+
*/
|
|
52385
|
+
this.toProducer = (layout, options2) => {
|
|
52386
|
+
Affirm_default(layout, "Invalid copybook layout");
|
|
52387
|
+
Affirm_default(layout.fields?.length > 0, "The copybook layout holds no fields");
|
|
52388
|
+
Affirm_default(options2?.name, "Missing required parameter name");
|
|
52389
|
+
const emitted = this._items(layout.fields, 0);
|
|
52390
|
+
if (emitted.length === 0)
|
|
52391
|
+
throw new Error(`Record "${layout.name}" has no field to read: every item in it is FILLER or a redefinition.`);
|
|
52392
|
+
this._names(emitted);
|
|
52393
|
+
this._counts(emitted);
|
|
52394
|
+
this._anchor(emitted);
|
|
52395
|
+
const producer = {
|
|
52396
|
+
name: options2.name,
|
|
52397
|
+
description: `Imported from a COBOL copybook: record ${layout.name}, ${layout.width} characters.`,
|
|
52398
|
+
source: options2.source ?? DEFAULT_SOURCE,
|
|
52399
|
+
dimensions: emitted.map((x2) => x2.dimension),
|
|
52400
|
+
settings: { fileType: "FIXED", fileKey: options2.fileKey ?? `${options2.name}.txt` },
|
|
52401
|
+
_version: 1
|
|
52402
|
+
};
|
|
52403
|
+
this._check(producer);
|
|
52404
|
+
return { producer, notes: this._notes(layout) };
|
|
52405
|
+
};
|
|
52406
|
+
/**
|
|
52407
|
+
* The dimensions a list of copybook items emits, in layout order. A group that does not repeat
|
|
52408
|
+
* is TRANSPARENT: it structures the copybook but generates no column, so its items are emitted
|
|
52409
|
+
* in its place with their offsets carried down into the coordinate space of its own parent.
|
|
52410
|
+
*/
|
|
52411
|
+
this._items = (fields, base) => {
|
|
52412
|
+
const emitted = [];
|
|
52413
|
+
for (const field of fields) {
|
|
52414
|
+
if (field.redefines || field.filler) continue;
|
|
52415
|
+
if (field.fields && !this._repeats(field)) {
|
|
52416
|
+
const inner = this._items(field.fields, base + field.offset - 1);
|
|
52417
|
+
emitted.push(...inner.map((x2) => ({ ...x2, qualifier: x2.qualifier || this._name(field.name) })));
|
|
52418
|
+
continue;
|
|
52419
|
+
}
|
|
52420
|
+
emitted.push(this._item(field, base + field.offset));
|
|
52421
|
+
}
|
|
52422
|
+
return emitted;
|
|
52423
|
+
};
|
|
52424
|
+
/** Whether an item generates one dimension per occurrence rather than a single dimension. */
|
|
52425
|
+
this._repeats = (field) => {
|
|
52426
|
+
return field.occurs > 1 || !!field.dependingOn;
|
|
52427
|
+
};
|
|
52428
|
+
/** One copybook item as one dimension: a leaf, a repeating leaf, or a group of its own. */
|
|
52429
|
+
this._item = (field, start) => {
|
|
52430
|
+
const position = { start, length: field.width };
|
|
52431
|
+
const span = field.width * field.occurs;
|
|
52432
|
+
if (!field.fields && !this._repeats(field))
|
|
52433
|
+
return { dimension: this._leaf(field, position), field, qualifier: "", start, span };
|
|
52434
|
+
const children = field.fields ? this._items(field.fields, 0) : [{ dimension: this._leaf(field, { start: 1, length: field.width }), field, qualifier: "", start: 1, span: field.width }];
|
|
52435
|
+
this._names(children);
|
|
52436
|
+
this._rejectNestedCounts(field);
|
|
52437
|
+
const dimension = {
|
|
52438
|
+
name: this._name(field.name),
|
|
52439
|
+
description: this._describe(field),
|
|
52440
|
+
position,
|
|
52441
|
+
occurs: field.occurs,
|
|
52442
|
+
// A copybook's occurrences are numbered from 1, and the names read against the copybook.
|
|
52443
|
+
indexFrom: 1,
|
|
52444
|
+
naming: field.fields ? void 0 : REPEATED_FIELD_NAMING,
|
|
52445
|
+
dimensions: children.map((x2) => x2.dimension)
|
|
52446
|
+
};
|
|
52447
|
+
return { dimension, field, qualifier: "", start, span };
|
|
52448
|
+
};
|
|
52449
|
+
/** A dimension that produces a column: its type, and how its number is spelled, if it is one. */
|
|
52450
|
+
this._leaf = (field, position) => {
|
|
52451
|
+
return {
|
|
52452
|
+
name: this._name(field.name),
|
|
52453
|
+
description: this._describe(field),
|
|
52454
|
+
type: field.type,
|
|
52455
|
+
position,
|
|
52456
|
+
scale: field.scale,
|
|
52457
|
+
sign: field.sign,
|
|
52458
|
+
usage: field.usage === "display" ? void 0 : field.usage
|
|
52459
|
+
};
|
|
52460
|
+
};
|
|
52461
|
+
/** What the copybook says about an item, kept beside the dimension it became. */
|
|
52462
|
+
this._describe = (field) => {
|
|
52463
|
+
const clauses = [field.name];
|
|
52464
|
+
if (field.picture) clauses.push(`PIC ${field.picture}`);
|
|
52465
|
+
if (field.usage !== "display") clauses.push(`USAGE ${field.usage.toUpperCase()}`);
|
|
52466
|
+
if (field.dependingOn) clauses.push(`OCCURS ${field.occursMin ?? 1} TO ${field.occurs} DEPENDING ON ${field.dependingOn}`);
|
|
52467
|
+
else if (field.occurs > 1) clauses.push(`OCCURS ${field.occurs}`);
|
|
52468
|
+
const notes = [clauses.join(" ")];
|
|
52469
|
+
if (field.edited) notes.push(`read as text because ${field.edited}`);
|
|
52470
|
+
if (field.redefinedBy?.length > 0) notes.push(`${field.redefinedBy.join(", ")} redefine the same bytes and are not read`);
|
|
52471
|
+
return notes.join("; ");
|
|
52472
|
+
};
|
|
52473
|
+
/**
|
|
52474
|
+
* A per-line occurrence count is read from the LINE, and a nested group's occurrences sit inside
|
|
52475
|
+
* one occurrence of its parent — there is no second axis to count along, and nowhere to record
|
|
52476
|
+
* the count (docs/plans/mainframe.md open question 5).
|
|
52477
|
+
*/
|
|
52478
|
+
this._rejectNestedCounts = (group3) => {
|
|
52479
|
+
for (const field of this._walk(group3.fields ?? [])) {
|
|
52480
|
+
if (!field.dependingOn) continue;
|
|
52481
|
+
throw new Error(`Copybook line ${field.line}: "${field.name}" has OCCURS DEPENDING ON inside "${group3.name}", which repeats as well. A varying count is scanned along the line, so only the outermost group can have one.`);
|
|
52482
|
+
}
|
|
52483
|
+
};
|
|
52484
|
+
/**
|
|
52485
|
+
* Gives every dimension of one list a name no sibling has. A copybook qualifies two items of the
|
|
52486
|
+
* same name by the group they sit in, and a flattened layout has to do the same: the group's
|
|
52487
|
+
* name goes in front, and a number only if that still collides.
|
|
52488
|
+
*/
|
|
52489
|
+
this._names = (emitted) => {
|
|
52490
|
+
const taken = /* @__PURE__ */ new Set();
|
|
52491
|
+
for (const item of emitted) {
|
|
52492
|
+
const base = item.dimension.name;
|
|
52493
|
+
const qualified = item.qualifier ? `${item.qualifier}${this._capitalize(base)}` : "";
|
|
52494
|
+
let name = [base, qualified].find((x2) => x2 && !taken.has(x2));
|
|
52495
|
+
for (let suffix = 2; !name && suffix <= emitted.length + 2; suffix++)
|
|
52496
|
+
if (!taken.has(`${base}${suffix}`)) name = `${base}${suffix}`;
|
|
52497
|
+
Affirm_default(name, `Cannot name the dimension read from copybook item "${item.field.name}"`);
|
|
52498
|
+
item.dimension.name = name;
|
|
52499
|
+
taken.add(name);
|
|
52500
|
+
}
|
|
52501
|
+
};
|
|
52502
|
+
/**
|
|
52503
|
+
* Resolves each `OCCURS ... DEPENDING ON` against the dimension that holds the count. The count
|
|
52504
|
+
* is read per line, so the field holding it has to be read before the occurrences it counts.
|
|
52505
|
+
*/
|
|
52506
|
+
this._counts = (emitted) => {
|
|
52507
|
+
for (const [index, item] of emitted.entries()) {
|
|
52508
|
+
const counter = item.field.dependingOn;
|
|
52509
|
+
if (!counter) continue;
|
|
52510
|
+
const source = emitted.find((x2, at3) => at3 < index && !x2.field.fields && x2.field.name === counter);
|
|
52511
|
+
if (!source)
|
|
52512
|
+
throw new Error(`Copybook line ${item.field.line}: "${item.field.name}" has OCCURS DEPENDING ON "${counter}", which is not a field read before it. The count has to be a field of this record, declared ahead of the group and outside every other one.`);
|
|
52513
|
+
item.dimension.occurs = {
|
|
52514
|
+
max: item.field.occurs,
|
|
52515
|
+
min: item.field.occursMin || void 0,
|
|
52516
|
+
from: `$${source.dimension.name}`
|
|
52517
|
+
};
|
|
52518
|
+
}
|
|
52519
|
+
};
|
|
52520
|
+
/**
|
|
52521
|
+
* Anchors everything that follows a group whose occurrence count varies. That group's span
|
|
52522
|
+
* differs line by line, so an absolute position past it reads the wrong characters on every line
|
|
52523
|
+
* but the longest — `position.after` measures from where the occurrences actually ended.
|
|
52524
|
+
*/
|
|
52525
|
+
this._anchor = (emitted) => {
|
|
52526
|
+
let anchor = null;
|
|
52527
|
+
for (const item of emitted) {
|
|
52528
|
+
if (anchor) {
|
|
52529
|
+
const offset = item.start - anchor.end;
|
|
52530
|
+
if (offset < 1)
|
|
52531
|
+
throw new Error(`Copybook line ${item.field.line}: "${item.field.name}" starts inside the occurrences of "${anchor.group}", which it is laid out after.`);
|
|
52532
|
+
item.dimension.position = { after: anchor.group, start: offset, length: item.field.width };
|
|
52533
|
+
}
|
|
52534
|
+
if (item.field.dependingOn) anchor = { group: item.dimension.name, end: item.start + item.span - 1 };
|
|
52535
|
+
}
|
|
52536
|
+
};
|
|
52537
|
+
/**
|
|
52538
|
+
* Everything the copybook states that the producer cannot carry out on its own. Written against
|
|
52539
|
+
* the copybook's own names and lines, because that is the document the reader checks it against.
|
|
52540
|
+
*/
|
|
52541
|
+
this._notes = (layout) => {
|
|
52542
|
+
const notes = [];
|
|
52543
|
+
for (const field of this._walk(layout.fields)) {
|
|
52544
|
+
if (field.usage !== "display")
|
|
52545
|
+
notes.push(`${field.name} (line ${field.line}) is USAGE ${field.usage.toUpperCase()}: its digits are stored as bytes, so "remora compile" rejects it until the feed converts the field to display. It is laid out as the ${field.width} characters its PIC clause states.`);
|
|
52546
|
+
if (field.edited)
|
|
52547
|
+
notes.push(`${field.name} (line ${field.line}) reads as text: ${field.edited}.`);
|
|
52548
|
+
if (field.redefinedBy?.length > 0)
|
|
52549
|
+
notes.push(`${field.name} (line ${field.line}) is read as declared; ${field.redefinedBy.join(", ")} redefine the same bytes and were not emitted.`);
|
|
52550
|
+
}
|
|
52551
|
+
if (layout.redefinedBy?.length > 0)
|
|
52552
|
+
notes.push(`Record ${layout.name} is read as declared; ${layout.redefinedBy.join(", ")} redefine the whole record and were not emitted.`);
|
|
52553
|
+
return notes;
|
|
52554
|
+
};
|
|
52555
|
+
/** Every item of a layout, at any depth, in declaration order. */
|
|
52556
|
+
this._walk = (fields) => {
|
|
52557
|
+
return fields.flatMap((field) => [field, ...this._walk(field.fields ?? [])]);
|
|
52558
|
+
};
|
|
52559
|
+
/**
|
|
52560
|
+
* Refuses to emit a layout the config loader would reject or expand into colliding names. The
|
|
52561
|
+
* importer exists so that nobody transcribes a layout by hand; emitting one that does not load
|
|
52562
|
+
* would hand the transcription back.
|
|
52563
|
+
*/
|
|
52564
|
+
this._check = (producer) => {
|
|
52565
|
+
const errors = ProducerExpander_default.validate(producer);
|
|
52566
|
+
if (errors.length > 0)
|
|
52567
|
+
throw new Error(`The copybook describes a layout Remora cannot read:
|
|
52568
|
+
${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
52569
|
+
ProducerExpander_default.expand(Algo_default.deepClone(producer));
|
|
52570
|
+
};
|
|
52571
|
+
/** A COBOL data name as a dimension name: `CLM-TOT-AMT` reads as `clmTotAmt`. */
|
|
52572
|
+
this._name = (cobol) => {
|
|
52573
|
+
const parts = cobol.toLowerCase().split(/[^a-z0-9]+/).filter((x2) => x2.length > 0);
|
|
52574
|
+
Affirm_default(parts.length > 0, `Cannot read a dimension name from the copybook item "${cobol}"`);
|
|
52575
|
+
const name = parts.map((part, index) => index === 0 ? part : this._capitalize(part)).join("");
|
|
52576
|
+
return /^\d/.test(name) ? `f${name}` : name;
|
|
52577
|
+
};
|
|
52578
|
+
this._capitalize = (text) => {
|
|
52579
|
+
return text.charAt(0).toUpperCase() + text.slice(1);
|
|
52580
|
+
};
|
|
52581
|
+
}
|
|
52582
|
+
};
|
|
52583
|
+
CopybookEmitter = new CopybookEmitterClass();
|
|
52584
|
+
CopybookEmitter_default = CopybookEmitter;
|
|
52585
|
+
}
|
|
52586
|
+
});
|
|
52587
|
+
|
|
51639
52588
|
// ../../packages/engines/src/consumer/ConsumerManager.ts
|
|
51640
52589
|
var ConsumerManagerClass, ConsumerManager, ConsumerManager_default;
|
|
51641
52590
|
var init_ConsumerManager = __esm({
|
|
@@ -51799,21 +52748,21 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51799
52748
|
init_src2();
|
|
51800
52749
|
ConsumerOnFinishManagerClass = class {
|
|
51801
52750
|
constructor() {
|
|
51802
|
-
this.onConsumerSuccess = async (consumer, executionId, runConsumer2) => {
|
|
52751
|
+
this.onConsumerSuccess = async (consumer, executionId, runConsumer2, options2) => {
|
|
51803
52752
|
void executionId;
|
|
51804
52753
|
for (const output of consumer.outputs) {
|
|
51805
52754
|
if (output.onSuccess)
|
|
51806
|
-
await this.performOnSuccessActions(consumer, output, runConsumer2);
|
|
52755
|
+
await this.performOnSuccessActions(consumer, output, runConsumer2, options2);
|
|
51807
52756
|
}
|
|
51808
52757
|
};
|
|
51809
|
-
this.onConsumerError = async (consumer, executionId, runConsumer2) => {
|
|
52758
|
+
this.onConsumerError = async (consumer, executionId, runConsumer2, options2) => {
|
|
51810
52759
|
void executionId;
|
|
51811
52760
|
for (const output of consumer.outputs) {
|
|
51812
52761
|
if (output.onError)
|
|
51813
|
-
await this.performOnErrorActions(consumer, output, runConsumer2);
|
|
52762
|
+
await this.performOnErrorActions(consumer, output, runConsumer2, options2);
|
|
51814
52763
|
}
|
|
51815
52764
|
};
|
|
51816
|
-
this.performOnSuccessActions = async (consumer, output, runConsumer2) => {
|
|
52765
|
+
this.performOnSuccessActions = async (consumer, output, runConsumer2, options2) => {
|
|
51817
52766
|
Affirm_default(consumer, "Invalid consumer");
|
|
51818
52767
|
Affirm_default(output, "Invalid output");
|
|
51819
52768
|
if (!output.onSuccess || output.onSuccess.length === 0)
|
|
@@ -51821,6 +52770,8 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51821
52770
|
for (const onSuccess of output.onSuccess) {
|
|
51822
52771
|
switch (onSuccess.action) {
|
|
51823
52772
|
case "move-file": {
|
|
52773
|
+
if (this._skipped(consumer, options2))
|
|
52774
|
+
break;
|
|
51824
52775
|
await this.moveSourceFiles(consumer, onSuccess.moveToDestination);
|
|
51825
52776
|
break;
|
|
51826
52777
|
}
|
|
@@ -51833,6 +52784,12 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51833
52784
|
}
|
|
51834
52785
|
}
|
|
51835
52786
|
};
|
|
52787
|
+
/** Whether this run declines a move-file action, logged so the skip is visible in the run's log. */
|
|
52788
|
+
this._skipped = (consumer, options2) => {
|
|
52789
|
+
if (!options2?.skipFileMoves) return false;
|
|
52790
|
+
Logger_default.info(`Skipping the move-file action of consumer "${consumer.name}": this run did not read its real sources`);
|
|
52791
|
+
return true;
|
|
52792
|
+
};
|
|
51836
52793
|
/**
|
|
51837
52794
|
* Runs the downstream consumer(s) named by a run-consumer action, sequentially in declared order.
|
|
51838
52795
|
* A downstream failure (unknown consumer or a failed run) is logged and swallowed so it does not
|
|
@@ -51951,7 +52908,7 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51951
52908
|
await destinationDriver.copyFromS3(sourceSource.authentication["bucket"], fileKey, fileKey);
|
|
51952
52909
|
await sourceDriver.deleteFile(fileKey);
|
|
51953
52910
|
};
|
|
51954
|
-
this.performOnErrorActions = async (consumer, output, runConsumer2) => {
|
|
52911
|
+
this.performOnErrorActions = async (consumer, output, runConsumer2, options2) => {
|
|
51955
52912
|
Affirm_default(consumer, "Invalid consumer");
|
|
51956
52913
|
Affirm_default(output, "Invalid output");
|
|
51957
52914
|
if (!output.onError || output.onError.length === 0)
|
|
@@ -51959,6 +52916,8 @@ var init_ConsumerOnFinishManager = __esm({
|
|
|
51959
52916
|
for (const onError of output.onError) {
|
|
51960
52917
|
switch (onError.action) {
|
|
51961
52918
|
case "move-file": {
|
|
52919
|
+
if (this._skipped(consumer, options2))
|
|
52920
|
+
break;
|
|
51962
52921
|
await this.moveSourceFiles(consumer, onError.moveToDestination);
|
|
51963
52922
|
break;
|
|
51964
52923
|
}
|
|
@@ -52235,33 +53194,33 @@ var init_RequestValidator = __esm({
|
|
|
52235
53194
|
this.resolveMembers = (consumer) => {
|
|
52236
53195
|
return new Set(ConsumerManager_default.getExpandedFields(consumer).map((x2) => x2.finalKey));
|
|
52237
53196
|
};
|
|
52238
|
-
this.checkFilters = (value, context,
|
|
53197
|
+
this.checkFilters = (value, context, path32, depth) => {
|
|
52239
53198
|
const { MAX_REQUEST_FILTERS, MAX_REQUEST_FILTER_DEPTH } = Constants_default.defaults;
|
|
52240
53199
|
if (!Array.isArray(value))
|
|
52241
|
-
throw new Error(`Invalid "${
|
|
53200
|
+
throw new Error(`Invalid "${path32}": expected an array of filters, got ${this.describe(value)}`);
|
|
52242
53201
|
if (value.length > MAX_REQUEST_FILTERS)
|
|
52243
|
-
throw new Error(`Invalid "${
|
|
53202
|
+
throw new Error(`Invalid "${path32}": ${value.length} filters is above the maximum of ${MAX_REQUEST_FILTERS}`);
|
|
52244
53203
|
if (depth > MAX_REQUEST_FILTER_DEPTH)
|
|
52245
|
-
throw new Error(`Invalid filter at ${
|
|
52246
|
-
return value.map((x2, index) => this.checkFilter(x2, context, `${
|
|
53204
|
+
throw new Error(`Invalid filter at ${path32}: filters may not nest more than ${MAX_REQUEST_FILTER_DEPTH} deep`);
|
|
53205
|
+
return value.map((x2, index) => this.checkFilter(x2, context, `${path32}[${index}]`, depth));
|
|
52247
53206
|
};
|
|
52248
|
-
this.checkFilter = (value, context,
|
|
53207
|
+
this.checkFilter = (value, context, path32, depth) => {
|
|
52249
53208
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
52250
|
-
throw new Error(`Invalid filter at ${
|
|
53209
|
+
throw new Error(`Invalid filter at ${path32}: expected an object, got ${this.describe(value)}`);
|
|
52251
53210
|
const filter = value;
|
|
52252
53211
|
const unknownKey = Object.keys(filter).find((key) => !FILTER_KEYS.includes(key));
|
|
52253
53212
|
if (unknownKey)
|
|
52254
|
-
throw new Error(`Invalid filter at ${
|
|
52255
|
-
const operator = this.checkOperator(filter.operator,
|
|
53213
|
+
throw new Error(`Invalid filter at ${path32}: unknown property "${unknownKey}". Supported: ${FILTER_KEYS.join(", ")}`);
|
|
53214
|
+
const operator = this.checkOperator(filter.operator, path32);
|
|
52256
53215
|
const checked = {
|
|
52257
|
-
member: this.checkMember(filter.member, context,
|
|
53216
|
+
member: this.checkMember(filter.member, context, path32),
|
|
52258
53217
|
operator,
|
|
52259
|
-
values: this.checkValues(filter.values, operator,
|
|
53218
|
+
values: this.checkValues(filter.values, operator, path32)
|
|
52260
53219
|
};
|
|
52261
53220
|
if (Algo_default.hasVal(filter.and))
|
|
52262
|
-
checked.and = this.checkFilters(filter.and, context, `${
|
|
53221
|
+
checked.and = this.checkFilters(filter.and, context, `${path32}.and`, depth + 1);
|
|
52263
53222
|
if (Algo_default.hasVal(filter.or))
|
|
52264
|
-
checked.or = this.checkFilters(filter.or, context, `${
|
|
53223
|
+
checked.or = this.checkFilters(filter.or, context, `${path32}.or`, depth + 1);
|
|
52265
53224
|
return checked;
|
|
52266
53225
|
};
|
|
52267
53226
|
/**
|
|
@@ -52269,18 +53228,18 @@ var init_RequestValidator = __esm({
|
|
|
52269
53228
|
* own field names by the time filters run, so any other name reads `undefined` and quietly keeps
|
|
52270
53229
|
* every row (or drops every row, for a negated operator).
|
|
52271
53230
|
*/
|
|
52272
|
-
this.checkMember = (value, context,
|
|
53231
|
+
this.checkMember = (value, context, path32) => {
|
|
52273
53232
|
if (typeof value !== "string" || value.length === 0)
|
|
52274
|
-
throw new Error(`Invalid filter at ${
|
|
53233
|
+
throw new Error(`Invalid filter at ${path32}: "member" must be a non-empty string, got ${this.describe(value)}`);
|
|
52275
53234
|
if (!context.members.has(value))
|
|
52276
|
-
throw new Error(`Invalid filter at ${
|
|
53235
|
+
throw new Error(`Invalid filter at ${path32}: consumer "${context.consumer.name}" has no field named "${value}"`);
|
|
52277
53236
|
return value;
|
|
52278
53237
|
};
|
|
52279
|
-
this.checkOperator = (value,
|
|
53238
|
+
this.checkOperator = (value, path32) => {
|
|
52280
53239
|
if (typeof value !== "string" || !VALUES_BY_OPERATOR.has(value))
|
|
52281
|
-
throw new Error(`Invalid filter at ${
|
|
53240
|
+
throw new Error(`Invalid filter at ${path32}: unknown operator ${this.describe(value)}. Supported: ${this.supportedOperators()}`);
|
|
52282
53241
|
if (VALUES_BY_OPERATOR.get(value) === "unsupported")
|
|
52283
|
-
throw new Error(`Invalid filter at ${
|
|
53242
|
+
throw new Error(`Invalid filter at ${path32}: operator "${value}" is not supported by a run`);
|
|
52284
53243
|
return value;
|
|
52285
53244
|
};
|
|
52286
53245
|
/**
|
|
@@ -52288,24 +53247,24 @@ var init_RequestValidator = __esm({
|
|
|
52288
53247
|
* own type and the filter's is read back into it, so a number or a boolean here is a client bug
|
|
52289
53248
|
* that reads as a type mismatch (matching nothing) rather than as the comparison it meant.
|
|
52290
53249
|
*/
|
|
52291
|
-
this.checkValues = (value, operator,
|
|
53250
|
+
this.checkValues = (value, operator, path32) => {
|
|
52292
53251
|
const values = Algo_default.hasVal(value) ? value : [];
|
|
52293
53252
|
if (!Array.isArray(values))
|
|
52294
|
-
throw new Error(`Invalid filter at ${
|
|
53253
|
+
throw new Error(`Invalid filter at ${path32}: "values" must be an array of strings, got ${this.describe(value)}`);
|
|
52295
53254
|
const wrongIndex = values.findIndex((x2) => typeof x2 !== "string");
|
|
52296
53255
|
if (wrongIndex >= 0)
|
|
52297
|
-
throw new Error(`Invalid filter at ${
|
|
52298
|
-
return this.checkArity(values, operator,
|
|
53256
|
+
throw new Error(`Invalid filter at ${path32}: "values[${wrongIndex}]" must be a string, got ${this.describe(values[wrongIndex])}`);
|
|
53257
|
+
return this.checkArity(values, operator, path32);
|
|
52299
53258
|
};
|
|
52300
|
-
this.checkArity = (values, operator,
|
|
53259
|
+
this.checkArity = (values, operator, path32) => {
|
|
52301
53260
|
const expected = VALUES_BY_OPERATOR.get(operator);
|
|
52302
53261
|
const reason = `operator "${operator}"`;
|
|
52303
53262
|
if (expected === "none" && values.length > 0)
|
|
52304
|
-
throw new Error(`Invalid filter at ${
|
|
53263
|
+
throw new Error(`Invalid filter at ${path32}: ${reason} reads no values, ${values.length} given`);
|
|
52305
53264
|
if (expected === "many" && values.length === 0)
|
|
52306
|
-
throw new Error(`Invalid filter at ${
|
|
53265
|
+
throw new Error(`Invalid filter at ${path32}: ${reason} needs at least one value, none given`);
|
|
52307
53266
|
if (typeof expected === "number" && values.length !== expected)
|
|
52308
|
-
throw new Error(`Invalid filter at ${
|
|
53267
|
+
throw new Error(`Invalid filter at ${path32}: ${reason} needs exactly ${expected} ${expected === 1 ? "value" : "values"}, ${values.length} given`);
|
|
52309
53268
|
return values;
|
|
52310
53269
|
};
|
|
52311
53270
|
/** Every operator a request may ask for, in the order they are declared. */
|
|
@@ -52646,23 +53605,23 @@ var init_Diagnostics = __esm({
|
|
|
52646
53605
|
return [.../* @__PURE__ */ new Set([...inputs, ...chained])];
|
|
52647
53606
|
};
|
|
52648
53607
|
this._walk = (start, next, onCycle) => {
|
|
52649
|
-
const
|
|
53608
|
+
const path32 = [];
|
|
52650
53609
|
const onPath = /* @__PURE__ */ new Set();
|
|
52651
53610
|
const stack = [{ name: start, done: false }];
|
|
52652
53611
|
const visited = /* @__PURE__ */ new Set();
|
|
52653
53612
|
while (stack.length > 0) {
|
|
52654
53613
|
const frame = stack.pop();
|
|
52655
53614
|
if (frame.done) {
|
|
52656
|
-
onPath.delete(
|
|
53615
|
+
onPath.delete(path32.pop());
|
|
52657
53616
|
continue;
|
|
52658
53617
|
}
|
|
52659
53618
|
if (onPath.has(frame.name)) {
|
|
52660
|
-
|
|
53619
|
+
path32.slice(path32.indexOf(frame.name)).forEach((name) => onCycle.add(name));
|
|
52661
53620
|
continue;
|
|
52662
53621
|
}
|
|
52663
53622
|
if (visited.has(frame.name)) continue;
|
|
52664
53623
|
visited.add(frame.name);
|
|
52665
|
-
|
|
53624
|
+
path32.push(frame.name);
|
|
52666
53625
|
onPath.add(frame.name);
|
|
52667
53626
|
stack.push({ name: frame.name, done: true });
|
|
52668
53627
|
const successors = next.get(frame.name) ?? [];
|
|
@@ -53341,8 +54300,8 @@ var init_ResourceView = __esm({
|
|
|
53341
54300
|
if (!source || origin.path.length < 2) return { value: "declared here", muted: true };
|
|
53342
54301
|
return { value: `${source.resource}.${source.field}`, link: source.node };
|
|
53343
54302
|
};
|
|
53344
|
-
this._pathText = (
|
|
53345
|
-
return
|
|
54303
|
+
this._pathText = (path32) => {
|
|
54304
|
+
return path32.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
|
|
53346
54305
|
};
|
|
53347
54306
|
this._consumerOutputsTable = (env2, consumer) => {
|
|
53348
54307
|
const declared = new Set((env2.sources ?? []).map((x2) => x2.name));
|
|
@@ -54377,17 +55336,17 @@ var init_BugReportBundle = __esm({
|
|
|
54377
55336
|
}
|
|
54378
55337
|
};
|
|
54379
55338
|
/** Write one file into the bundle, under the section that produced it. */
|
|
54380
|
-
this.add = (name,
|
|
54381
|
-
Affirm_default(
|
|
54382
|
-
Affirm_default(content !== void 0 && content !== null, `No content for bundle entry ${
|
|
55339
|
+
this.add = (name, path32, content) => {
|
|
55340
|
+
Affirm_default(path32, "Missing required parameter path");
|
|
55341
|
+
Affirm_default(content !== void 0 && content !== null, `No content for bundle entry ${path32}`);
|
|
54383
55342
|
const section = this._section(name);
|
|
54384
55343
|
Affirm_default(
|
|
54385
55344
|
this._zip.getEntries().length < MAX_ENTRIES,
|
|
54386
|
-
`Bug report reached its ${MAX_ENTRIES} file limit while writing ${
|
|
55345
|
+
`Bug report reached its ${MAX_ENTRIES} file limit while writing ${path32}`
|
|
54387
55346
|
);
|
|
54388
|
-
this._zip.addFile(
|
|
55347
|
+
this._zip.addFile(path32, Buffer.isBuffer(content) ? content : Buffer.from(content, "utf8"));
|
|
54389
55348
|
section.included = true;
|
|
54390
|
-
section.files.push(
|
|
55349
|
+
section.files.push(path32);
|
|
54391
55350
|
};
|
|
54392
55351
|
/** Record that a section contributed nothing, and why. Absence without a reason reads as a bug. */
|
|
54393
55352
|
this.skip = (name, reason) => {
|
|
@@ -56430,6 +57389,8 @@ var init_src10 = __esm({
|
|
|
56430
57389
|
init_DeveloperEngine();
|
|
56431
57390
|
init_LLM();
|
|
56432
57391
|
init_SyntheticInputEngine();
|
|
57392
|
+
init_CopybookParser();
|
|
57393
|
+
init_CopybookEmitter();
|
|
56433
57394
|
init_ConsumerManager();
|
|
56434
57395
|
init_ConsumerOnFinishManager();
|
|
56435
57396
|
init_RequestExecutor();
|
|
@@ -56541,9 +57502,10 @@ var init_DatasetSchema = __esm({
|
|
|
56541
57502
|
return dsSchema.columns.map((column2) => column2.name);
|
|
56542
57503
|
};
|
|
56543
57504
|
/**
|
|
56544
|
-
* One column, typed from the dimension behind it. `type`, `format` and `nullable` are
|
|
56545
|
-
*
|
|
56546
|
-
*
|
|
57505
|
+
* One column, typed from the dimension behind it. `type`, `format` and `nullable` are what a cast
|
|
57506
|
+
* reads off a staged value, so carrying those lets a reader cast it back without going near the
|
|
57507
|
+
* consumer config. A dimension's `scale` and `sign` deliberately do NOT travel: they say how the
|
|
57508
|
+
* SOURCE spelled the number, and what is staged here is the number itself.
|
|
56547
57509
|
*/
|
|
56548
57510
|
this._column = (field) => {
|
|
56549
57511
|
if (ConsumerManager_default.isLineCountField(field.cField))
|
|
@@ -56670,9 +57632,10 @@ var init_FixedBlockOrder = __esm({
|
|
|
56670
57632
|
});
|
|
56671
57633
|
|
|
56672
57634
|
// ../../packages/executors/src/OutputExecutor.ts
|
|
56673
|
-
var OutputExecutorClass, OutputExecutor, OutputExecutor_default;
|
|
57635
|
+
var import_fs21, OutputExecutorClass, OutputExecutor, OutputExecutor_default;
|
|
56674
57636
|
var init_OutputExecutor = __esm({
|
|
56675
57637
|
"../../packages/executors/src/OutputExecutor.ts"() {
|
|
57638
|
+
import_fs21 = __toESM(require("fs"));
|
|
56676
57639
|
init_src();
|
|
56677
57640
|
init_src();
|
|
56678
57641
|
init_src8();
|
|
@@ -56760,17 +57723,17 @@ var init_OutputExecutor = __esm({
|
|
|
56760
57723
|
* One output per consumer (`maxItems: 1` in consumer-schema.json). This used to loop, but only
|
|
56761
57724
|
* ever shipped the first entry, and could not have done more: `move` unlinks what it ships.
|
|
56762
57725
|
*/
|
|
56763
|
-
this.exportResult = async (consumer, dsSchema, splitFiles) => {
|
|
57726
|
+
this.exportResult = async (consumer, dsSchema, splitFiles, localPath) => {
|
|
56764
57727
|
Affirm_default.hasItems(splitFiles, `Consumer "${consumer.name}" has no output file to export`);
|
|
56765
57728
|
Affirm_default.hasItems(consumer.outputs, `Consumer "${consumer.name}" declares no output to export to`);
|
|
56766
57729
|
Affirm_default(dsSchema, `Consumer "${consumer.name}" has no dataset schema to export against`);
|
|
56767
57730
|
const output = consumer.outputs[0];
|
|
56768
57731
|
const internalFormat = dsSchema.encoding;
|
|
56769
|
-
const destination =
|
|
56770
|
-
Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
|
|
57732
|
+
const destination = this._resolveDestination(consumer, output, localPath);
|
|
56771
57733
|
const driver = await DriverFactory_default.instantiateDestination(destination);
|
|
56772
57734
|
const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
|
|
56773
|
-
|
|
57735
|
+
const target = localPath ?? output.exportDestination;
|
|
57736
|
+
Logger_default.info(`Exporting consumer "${consumer.name}" to "${target}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
|
|
56774
57737
|
const header = this._composeHeader(output, dsSchema);
|
|
56775
57738
|
const fixedLayouts = output.format === "FIXED" && output.layouts ? FixedWidthWriter_default.compileLayouts(dsSchema, this._fixedPositions(consumer), output, consumer.name) : null;
|
|
56776
57739
|
const fixedLine = output.format === "FIXED" ? this._fixedLineWriter(consumer, dsSchema, output, fixedLayouts) : null;
|
|
@@ -56933,6 +57896,25 @@ var init_OutputExecutor = __esm({
|
|
|
56933
57896
|
return "txt";
|
|
56934
57897
|
}
|
|
56935
57898
|
};
|
|
57899
|
+
/**
|
|
57900
|
+
* Where this export writes: the consumer's configured destination, or — in synth mode — a local
|
|
57901
|
+
* folder standing in for it, as a `local` source built on the spot.
|
|
57902
|
+
*
|
|
57903
|
+
* The real destination is never even resolved for a synthetic run, which is what lets a consumer
|
|
57904
|
+
* pointed at a production bucket be synthesized from a machine that holds no credentials for it.
|
|
57905
|
+
*
|
|
57906
|
+
* Named here, not left to the driver: an unresolvable destination used to surface as a TypeError
|
|
57907
|
+
* three layers down, naming neither the consumer nor the destination.
|
|
57908
|
+
*/
|
|
57909
|
+
this._resolveDestination = (consumer, output, localPath) => {
|
|
57910
|
+
if (localPath) {
|
|
57911
|
+
import_fs21.default.mkdirSync(localPath, { recursive: true });
|
|
57912
|
+
return { name: localPath, engine: "local", authentication: { method: "none", path: localPath }, _version: 1 };
|
|
57913
|
+
}
|
|
57914
|
+
const destination = Environment_default.getSource(output.exportDestination);
|
|
57915
|
+
Affirm_default(destination, `The export destination "${output.exportDestination}" of consumer "${consumer.name}" is not a source of this project`);
|
|
57916
|
+
return destination;
|
|
57917
|
+
};
|
|
56936
57918
|
this._composeFileName = (consumer, output, extension, executionId) => {
|
|
56937
57919
|
if (output.exportName && output.exportName.trim().length > 0) {
|
|
56938
57920
|
const sanitized = output.exportName.replace(/\.[^.]+$/, "");
|
|
@@ -56950,11 +57932,11 @@ var init_OutputExecutor = __esm({
|
|
|
56950
57932
|
});
|
|
56951
57933
|
|
|
56952
57934
|
// ../../packages/executors/src/ConsumerExecutor.ts
|
|
56953
|
-
var import_path19,
|
|
57935
|
+
var import_path19, import_fs22, import_promises16, import_crypto10, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
|
|
56954
57936
|
var init_ConsumerExecutor = __esm({
|
|
56955
57937
|
"../../packages/executors/src/ConsumerExecutor.ts"() {
|
|
56956
57938
|
import_path19 = __toESM(require("path"));
|
|
56957
|
-
|
|
57939
|
+
import_fs22 = __toESM(require("fs"));
|
|
56958
57940
|
import_promises16 = __toESM(require("fs/promises"));
|
|
56959
57941
|
import_crypto10 = __toESM(require("crypto"));
|
|
56960
57942
|
init_src10();
|
|
@@ -56971,10 +57953,10 @@ var init_ConsumerExecutor = __esm({
|
|
|
56971
57953
|
constructor() {
|
|
56972
57954
|
this._ensurePath = (pathUri) => {
|
|
56973
57955
|
const dir = import_path19.default.dirname(pathUri);
|
|
56974
|
-
if (!
|
|
56975
|
-
|
|
56976
|
-
if (!
|
|
56977
|
-
|
|
57956
|
+
if (!import_fs22.default.existsSync(dir))
|
|
57957
|
+
import_fs22.default.mkdirSync(dir, { recursive: true });
|
|
57958
|
+
if (!import_fs22.default.existsSync(pathUri))
|
|
57959
|
+
import_fs22.default.writeFileSync(pathUri, "");
|
|
56978
57960
|
};
|
|
56979
57961
|
/**
|
|
56980
57962
|
* Shapes one parsed producer record into one consumer record, following a plan compiled once per
|
|
@@ -57102,8 +58084,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
57102
58084
|
const parseLine = LineParser_default.datasetParser(dsSchema);
|
|
57103
58085
|
const outputPlan = OutputExecutor_default.compileOutput(consumer, dsSchema);
|
|
57104
58086
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57105
|
-
const reader =
|
|
57106
|
-
const writer =
|
|
58087
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
58088
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57107
58089
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57108
58090
|
let workerIndex = 0;
|
|
57109
58091
|
let currentWorker = executorResults[workerIndex] ?? null;
|
|
@@ -57194,9 +58176,9 @@ var init_ConsumerExecutor = __esm({
|
|
|
57194
58176
|
};
|
|
57195
58177
|
this.processDistinct = async (dsSchema, datasetPath) => {
|
|
57196
58178
|
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
57197
|
-
const reader =
|
|
58179
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57198
58180
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57199
|
-
const writer =
|
|
58181
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57200
58182
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57201
58183
|
let newLineCount = 0;
|
|
57202
58184
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -57226,7 +58208,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57226
58208
|
};
|
|
57227
58209
|
this.processDistinctOn = async (consumer, dsSchema, datasetPath) => {
|
|
57228
58210
|
Affirm_default(dsSchema, "Invalid dataset schema");
|
|
57229
|
-
const reader =
|
|
58211
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57230
58212
|
const { distinctOn } = consumer.options;
|
|
57231
58213
|
const { keys, resolution } = distinctOn;
|
|
57232
58214
|
const { strategy, orderBy, direction = "asc" } = resolution;
|
|
@@ -57246,7 +58228,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57246
58228
|
}
|
|
57247
58229
|
}
|
|
57248
58230
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57249
|
-
const writer =
|
|
58231
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57250
58232
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57251
58233
|
for (const { line } of winners.values()) {
|
|
57252
58234
|
if (!writer.write(line + "\n"))
|
|
@@ -57283,7 +58265,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57283
58265
|
if (!pivotValues) {
|
|
57284
58266
|
pivotValues = [];
|
|
57285
58267
|
const discoverySet = /* @__PURE__ */ new Set();
|
|
57286
|
-
const discoverReader =
|
|
58268
|
+
const discoverReader = import_fs22.default.createReadStream(datasetPath);
|
|
57287
58269
|
for await (const line of readLines(discoverReader)) {
|
|
57288
58270
|
const record = parseLine(line);
|
|
57289
58271
|
const val = String(record[pivotColumn] ?? "");
|
|
@@ -57300,7 +58282,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57300
58282
|
}
|
|
57301
58283
|
}
|
|
57302
58284
|
const groups = /* @__PURE__ */ new Map();
|
|
57303
|
-
const reader =
|
|
58285
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57304
58286
|
for await (const line of readLines(reader)) {
|
|
57305
58287
|
const record = parseLine(line);
|
|
57306
58288
|
const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
|
|
@@ -57324,7 +58306,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57324
58306
|
const pivotedDsSchema = DatasetSchema_default.forPivot(dsSchema, rowKeys, pivotValues.map((pv) => columnPrefix + pv), groups.size);
|
|
57325
58307
|
const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedDsSchema);
|
|
57326
58308
|
const tempWorkPath = datasetPath + "_tmp";
|
|
57327
|
-
const writer =
|
|
58309
|
+
const writer = import_fs22.default.createWriteStream(tempWorkPath);
|
|
57328
58310
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
57329
58311
|
let outputCount = 0;
|
|
57330
58312
|
for (const { rowRecord, cells } of groups.values()) {
|
|
@@ -57415,7 +58397,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
57415
58397
|
for (const fieldKey of uniqueFieldKeys) {
|
|
57416
58398
|
fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
|
|
57417
58399
|
}
|
|
57418
|
-
const reader =
|
|
58400
|
+
const reader = import_fs22.default.createReadStream(datasetPath);
|
|
57419
58401
|
for await (const line of readLines(reader)) {
|
|
57420
58402
|
rowCount++;
|
|
57421
58403
|
if (checkDuplicateRows) {
|
|
@@ -57562,10 +58544,10 @@ var init_ExecutorPerformance = __esm({
|
|
|
57562
58544
|
});
|
|
57563
58545
|
|
|
57564
58546
|
// ../../packages/executors/src/ExecutorWriter.ts
|
|
57565
|
-
var
|
|
58547
|
+
var fs28, ExecutorWriter, ExecutorWriter_default;
|
|
57566
58548
|
var init_ExecutorWriter = __esm({
|
|
57567
58549
|
"../../packages/executors/src/ExecutorWriter.ts"() {
|
|
57568
|
-
|
|
58550
|
+
fs28 = __toESM(require("fs"));
|
|
57569
58551
|
init_src();
|
|
57570
58552
|
init_src3();
|
|
57571
58553
|
init_src10();
|
|
@@ -57617,7 +58599,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57617
58599
|
await this._route(pass, dsSchema);
|
|
57618
58600
|
if (pass.buckets.size === 0)
|
|
57619
58601
|
return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
|
|
57620
|
-
await
|
|
58602
|
+
await fs28.promises.unlink(sourcePath);
|
|
57621
58603
|
return this._listFiles(pass);
|
|
57622
58604
|
};
|
|
57623
58605
|
/**
|
|
@@ -57625,7 +58607,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57625
58607
|
*/
|
|
57626
58608
|
this._route = async (pass, dsSchema) => {
|
|
57627
58609
|
const parseLine = pass.keys.length === 0 ? null : LineParser_default.datasetParser(dsSchema);
|
|
57628
|
-
const readStream =
|
|
58610
|
+
const readStream = fs28.createReadStream(pass.sourcePath);
|
|
57629
58611
|
try {
|
|
57630
58612
|
for await (const line of readLines(readStream)) {
|
|
57631
58613
|
const bucket = this._bucketFor(line, pass, parseLine);
|
|
@@ -57729,7 +58711,7 @@ var init_ExecutorWriter = __esm({
|
|
|
57729
58711
|
}
|
|
57730
58712
|
if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
|
|
57731
58713
|
await this._closeStream(pass.openStreams.shift());
|
|
57732
|
-
bucket.stream =
|
|
58714
|
+
bucket.stream = fs28.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
|
|
57733
58715
|
pass.openStreams.push(bucket);
|
|
57734
58716
|
return bucket.stream;
|
|
57735
58717
|
};
|
|
@@ -58007,10 +58989,10 @@ var init_OrchestratorHelper = __esm({
|
|
|
58007
58989
|
});
|
|
58008
58990
|
|
|
58009
58991
|
// ../../packages/executors/src/LookupResolver.ts
|
|
58010
|
-
var
|
|
58992
|
+
var import_fs23, import_promises17, import_path22, LookupResolverClass, LookupResolver, LookupResolver_default;
|
|
58011
58993
|
var init_LookupResolver = __esm({
|
|
58012
58994
|
"../../packages/executors/src/LookupResolver.ts"() {
|
|
58013
|
-
|
|
58995
|
+
import_fs23 = __toESM(require("fs"));
|
|
58014
58996
|
import_promises17 = __toESM(require("fs/promises"));
|
|
58015
58997
|
import_path22 = __toESM(require("path"));
|
|
58016
58998
|
init_src();
|
|
@@ -58088,7 +59070,7 @@ var init_LookupResolver = __esm({
|
|
|
58088
59070
|
const dict = {};
|
|
58089
59071
|
let rowsRead = 0;
|
|
58090
59072
|
let warnedDup = false;
|
|
58091
|
-
for await (const line of readLines(
|
|
59073
|
+
for await (const line of readLines(import_fs23.default.createReadStream(capture.path))) {
|
|
58092
59074
|
if (!line.length) continue;
|
|
58093
59075
|
rowsRead++;
|
|
58094
59076
|
Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
|
|
@@ -58127,10 +59109,10 @@ var init_LookupResolver = __esm({
|
|
|
58127
59109
|
});
|
|
58128
59110
|
|
|
58129
59111
|
// ../../packages/executors/src/ConsumerInputResolver.ts
|
|
58130
|
-
var
|
|
59112
|
+
var import_fs24, import_promises18, import_path23, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
|
|
58131
59113
|
var init_ConsumerInputResolver = __esm({
|
|
58132
59114
|
"../../packages/executors/src/ConsumerInputResolver.ts"() {
|
|
58133
|
-
|
|
59115
|
+
import_fs24 = __toESM(require("fs"));
|
|
58134
59116
|
import_promises18 = __toESM(require("fs/promises"));
|
|
58135
59117
|
import_path23 = __toESM(require("path"));
|
|
58136
59118
|
init_src();
|
|
@@ -58154,7 +59136,7 @@ var init_ConsumerInputResolver = __esm({
|
|
|
58154
59136
|
Affirm_default(stack.length <= Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH, `consumer-as-input nesting exceeds MAX_CONSUMER_INPUT_DEPTH (${Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH}): ${stack.join(" -> ")}`);
|
|
58155
59137
|
for (const name of upstreamNames) {
|
|
58156
59138
|
const cached = this.cache.get(name);
|
|
58157
|
-
if (!cached || !
|
|
59139
|
+
if (!cached || !import_fs24.default.existsSync(cached.path)) {
|
|
58158
59140
|
Affirm_default(!stack.includes(name), `consumer-as-input cycle detected: ${[...stack, name].join(" -> ")}`);
|
|
58159
59141
|
const upstream = Environment_default.getConsumer(name);
|
|
58160
59142
|
Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
|
|
@@ -58233,10 +59215,10 @@ var ExecutorOrchestrator_exports = {};
|
|
|
58233
59215
|
__export(ExecutorOrchestrator_exports, {
|
|
58234
59216
|
default: () => ExecutorOrchestrator_default
|
|
58235
59217
|
});
|
|
58236
|
-
var
|
|
59218
|
+
var import_fs25, import_promises19, import_path24, import_workerpool, import_promises20, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
|
|
58237
59219
|
var init_ExecutorOrchestrator = __esm({
|
|
58238
59220
|
"../../packages/executors/src/ExecutorOrchestrator.ts"() {
|
|
58239
|
-
|
|
59221
|
+
import_fs25 = __toESM(require("fs"));
|
|
58240
59222
|
import_promises19 = __toESM(require("fs/promises"));
|
|
58241
59223
|
import_path24 = __toESM(require("path"));
|
|
58242
59224
|
import_workerpool = __toESM(require("workerpool"));
|
|
@@ -58425,15 +59407,17 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58425
59407
|
_progress.update({ phase: "Exporting result", progress: 0 });
|
|
58426
59408
|
counter = performance.now();
|
|
58427
59409
|
Logger_default.info(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
|
|
58428
|
-
const
|
|
59410
|
+
const localExportPath = request.syntheticExportPath ? import_path24.default.join(request.syntheticExportPath, consumer.name) : void 0;
|
|
59411
|
+
const exportRes = await OutputExecutor_default.exportResult(consumer, dsSchema, splitFiles, localExportPath);
|
|
58429
59412
|
tracker.measure("export-result", performance.now() - counter);
|
|
58430
|
-
|
|
59413
|
+
const writtenTo = localExportPath ?? consumer.outputs.map((x2) => x2.exportDestination).join(", ");
|
|
59414
|
+
Affirm_default(exportRes.res, `Consumer "${consumer.name}" failed to write its output to "${writtenTo}" \u2014 see the logged driver error for the cause`);
|
|
58431
59415
|
Logger_default.info(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
|
|
58432
59416
|
_progress.update({ phase: "Exporting result", progress: 1 });
|
|
58433
59417
|
if (consumer.outputs.some((x2) => x2.onSuccess)) {
|
|
58434
59418
|
Logger_default.log(`[${usageId}] Running on-success actions`);
|
|
58435
59419
|
counter = performance.now();
|
|
58436
|
-
await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
|
|
59420
|
+
await ConsumerOnFinishManager_default.onConsumerSuccess(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId), { skipFileMoves: !!request.syntheticExportPath });
|
|
58437
59421
|
tracker.measure("on-success-actions", performance.now() - counter);
|
|
58438
59422
|
Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
|
|
58439
59423
|
}
|
|
@@ -58452,7 +59436,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58452
59436
|
Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
|
|
58453
59437
|
Logger_default.error(error);
|
|
58454
59438
|
if (!request.skipExport)
|
|
58455
|
-
await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
|
|
59439
|
+
await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId), { skipFileMoves: !!request.syntheticExportPath });
|
|
58456
59440
|
Logger_default.log(`[${usageId}] Running cleanup after failure`);
|
|
58457
59441
|
await this.performCleanupOperations(scope, tracker);
|
|
58458
59442
|
UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
|
|
@@ -58473,7 +59457,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58473
59457
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": source returned no files, skipping`);
|
|
58474
59458
|
continue;
|
|
58475
59459
|
}
|
|
58476
|
-
if (!
|
|
59460
|
+
if (!import_fs25.default.existsSync(response.files[0].fullUri)) {
|
|
58477
59461
|
if (!cProd.isOptional)
|
|
58478
59462
|
throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
|
|
58479
59463
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
|
|
@@ -58628,7 +59612,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58628
59612
|
details: { invokedBy: "ON_FINISH", user: request.details.user },
|
|
58629
59613
|
logProgress: request.logProgress,
|
|
58630
59614
|
_onFinishStack: chain2,
|
|
58631
|
-
syntheticProducerInputs: request.syntheticProducerInputs
|
|
59615
|
+
syntheticProducerInputs: request.syntheticProducerInputs,
|
|
59616
|
+
syntheticExportPath: request.syntheticExportPath
|
|
58632
59617
|
});
|
|
58633
59618
|
};
|
|
58634
59619
|
};
|
|
@@ -58638,7 +59623,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58638
59623
|
* Returns a single chunk for small files where parallelism overhead isn't worth it.
|
|
58639
59624
|
*/
|
|
58640
59625
|
this.scopeWork = (fileUri, numChunks) => {
|
|
58641
|
-
const fileSize =
|
|
59626
|
+
const fileSize = import_fs25.default.statSync(fileUri).size;
|
|
58642
59627
|
if (fileSize === 0) return [];
|
|
58643
59628
|
if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
|
|
58644
59629
|
return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
@@ -58648,7 +59633,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58648
59633
|
const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
|
|
58649
59634
|
if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
58650
59635
|
const targetChunkSize = Math.floor(fileSize / effectiveChunks);
|
|
58651
|
-
const fd =
|
|
59636
|
+
const fd = import_fs25.default.openSync(fileUri, "r");
|
|
58652
59637
|
try {
|
|
58653
59638
|
const offsets = [];
|
|
58654
59639
|
let currentStart = 0;
|
|
@@ -58666,7 +59651,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58666
59651
|
}
|
|
58667
59652
|
return offsets;
|
|
58668
59653
|
} finally {
|
|
58669
|
-
|
|
59654
|
+
import_fs25.default.closeSync(fd);
|
|
58670
59655
|
}
|
|
58671
59656
|
};
|
|
58672
59657
|
/**
|
|
@@ -58734,7 +59719,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58734
59719
|
let currentPos = position;
|
|
58735
59720
|
while (currentPos < fileSize) {
|
|
58736
59721
|
const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
|
|
58737
|
-
const bytesRead =
|
|
59722
|
+
const bytesRead = import_fs25.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
|
|
58738
59723
|
if (bytesRead === 0) break;
|
|
58739
59724
|
for (let i6 = 0; i6 < bytesRead; i6++) {
|
|
58740
59725
|
if (buffer[i6] === 10) {
|
|
@@ -58759,9 +59744,9 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58759
59744
|
return;
|
|
58760
59745
|
}
|
|
58761
59746
|
const parseLine = LineParser_default.datasetParser(dsSchema);
|
|
58762
|
-
const output =
|
|
59747
|
+
const output = import_fs25.default.createWriteStream(targetPath);
|
|
58763
59748
|
try {
|
|
58764
|
-
for await (const line of readLines(
|
|
59749
|
+
for await (const line of readLines(import_fs25.default.createReadStream(sourcePath))) {
|
|
58765
59750
|
if (!line.length) continue;
|
|
58766
59751
|
const record = parseLine(line);
|
|
58767
59752
|
if (!output.write(JSON.stringify(record) + "\n"))
|
|
@@ -58859,7 +59844,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58859
59844
|
});
|
|
58860
59845
|
await (0, import_promises20.pipeline)(
|
|
58861
59846
|
xlsCsvStream,
|
|
58862
|
-
|
|
59847
|
+
import_fs25.default.createWriteStream(decodedPath)
|
|
58863
59848
|
);
|
|
58864
59849
|
const fileStats = await import_promises19.default.stat(decodedPath);
|
|
58865
59850
|
decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
|
|
@@ -58919,8 +59904,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
58919
59904
|
const perf = performance.now();
|
|
58920
59905
|
for (const workerResult of executorResults) {
|
|
58921
59906
|
await (0, import_promises20.pipeline)(
|
|
58922
|
-
|
|
58923
|
-
|
|
59907
|
+
import_fs25.default.createReadStream(workerResult.resultUri),
|
|
59908
|
+
import_fs25.default.createWriteStream(mainPath, { flags: "a" })
|
|
58924
59909
|
);
|
|
58925
59910
|
await import_promises19.default.unlink(workerResult.resultUri);
|
|
58926
59911
|
}
|
|
@@ -59176,7 +60161,7 @@ init_OutputExecutor();
|
|
|
59176
60161
|
init_ProducerExecutor();
|
|
59177
60162
|
|
|
59178
60163
|
// ../../packages/executors/src/SampleRunner.ts
|
|
59179
|
-
var
|
|
60164
|
+
var import_fs26 = __toESM(require("fs"));
|
|
59180
60165
|
var import_promises21 = __toESM(require("fs/promises"));
|
|
59181
60166
|
var import_path25 = __toESM(require("path"));
|
|
59182
60167
|
init_src();
|
|
@@ -59290,9 +60275,9 @@ var SampleRunnerClass = class {
|
|
|
59290
60275
|
};
|
|
59291
60276
|
/** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
|
|
59292
60277
|
this._readCapture = async (capturePath, size) => {
|
|
59293
|
-
if (!
|
|
60278
|
+
if (!import_fs26.default.existsSync(capturePath)) return [];
|
|
59294
60279
|
const records = [];
|
|
59295
|
-
for await (const line of readLines(
|
|
60280
|
+
for await (const line of readLines(import_fs26.default.createReadStream(capturePath))) {
|
|
59296
60281
|
if (!line.length) continue;
|
|
59297
60282
|
records.push(JSON.parse(line));
|
|
59298
60283
|
if (records.length >= size) break;
|
|
@@ -59309,11 +60294,11 @@ init_src2();
|
|
|
59309
60294
|
|
|
59310
60295
|
// src/paths.ts
|
|
59311
60296
|
var import_path26 = __toESM(require("path"));
|
|
59312
|
-
var
|
|
60297
|
+
var import_fs27 = __toESM(require("fs"));
|
|
59313
60298
|
init_src3();
|
|
59314
60299
|
var resolveAssetDir = (name) => {
|
|
59315
60300
|
const bundledPath = import_path26.default.join(__dirname, name);
|
|
59316
|
-
if (
|
|
60301
|
+
if (import_fs27.default.existsSync(bundledPath)) return bundledPath;
|
|
59317
60302
|
return import_path26.default.join(__dirname, "..", name);
|
|
59318
60303
|
};
|
|
59319
60304
|
var DOCUMENTATION_DIR = resolveAssetDir("documentation");
|
|
@@ -59375,9 +60360,13 @@ var printFixedLayout = (report4) => {
|
|
|
59375
60360
|
console.log(import_chalk2.default.blue(` ${name}: ${layout.plan.recordLength} characters, ${layout.plan.fields.length} field(s)`));
|
|
59376
60361
|
console.log(import_chalk2.default.gray(` ${column("start")} ${column("end")} ${column("width")} field`));
|
|
59377
60362
|
for (const field of layout.plan.fields)
|
|
59378
|
-
console.log(import_chalk2.default.gray(` ${column(field.start)} ${column(field.end)} ${column(field.width)} `) + field.name);
|
|
60363
|
+
console.log(import_chalk2.default.gray(` ${column(field.start)} ${column(field.end)} ${column(field.width)} `) + field.name + import_chalk2.default.gray(spelling(field)));
|
|
59379
60364
|
}
|
|
59380
60365
|
};
|
|
60366
|
+
var spelling = (field) => {
|
|
60367
|
+
const parts = [field.scale ? `${field.scale} implied decimal(s)` : null, field.sign ? `${field.sign} sign` : null].filter(Boolean);
|
|
60368
|
+
return parts.length > 0 ? ` (${parts.join(", ")})` : "";
|
|
60369
|
+
};
|
|
59381
60370
|
var column = (value) => String(value).padStart(5);
|
|
59382
60371
|
|
|
59383
60372
|
// src/actions/verify.ts
|
|
@@ -59437,7 +60426,7 @@ var verify = async (options2) => {
|
|
|
59437
60426
|
|
|
59438
60427
|
// src/actions/deploy.ts
|
|
59439
60428
|
var import_chalk5 = __toESM(require("chalk"));
|
|
59440
|
-
var
|
|
60429
|
+
var import_fs29 = __toESM(require("fs"));
|
|
59441
60430
|
var import_path28 = __toESM(require("path"));
|
|
59442
60431
|
init_src3();
|
|
59443
60432
|
init_src4();
|
|
@@ -59445,7 +60434,7 @@ init_src2();
|
|
|
59445
60434
|
|
|
59446
60435
|
// src/actions/package_config.ts
|
|
59447
60436
|
var import_chalk4 = __toESM(require("chalk"));
|
|
59448
|
-
var
|
|
60437
|
+
var import_fs28 = __toESM(require("fs"));
|
|
59449
60438
|
var import_path27 = __toESM(require("path"));
|
|
59450
60439
|
init_src4();
|
|
59451
60440
|
init_src2();
|
|
@@ -59454,7 +60443,7 @@ var package_config = async (output) => {
|
|
|
59454
60443
|
try {
|
|
59455
60444
|
await compile();
|
|
59456
60445
|
const target = import_path27.default.resolve(output || DEFAULT_OUTPUT);
|
|
59457
|
-
const replaced =
|
|
60446
|
+
const replaced = import_fs28.default.existsSync(target);
|
|
59458
60447
|
const built = ConfigPackager_default.write("./remora", target);
|
|
59459
60448
|
const sizeKB = (built.bytes / 1024).toFixed(1);
|
|
59460
60449
|
console.log(import_chalk4.default.green(`\u2705 ${replaced ? "Replaced" : "Created"} configuration package: `) + import_chalk4.default.yellow(built.path));
|
|
@@ -59479,7 +60468,7 @@ var deploy = async (options2) => {
|
|
|
59479
60468
|
const tempZipPath = import_path28.default.join(process.cwd(), "temp_deployment.zip");
|
|
59480
60469
|
ConfigPackager_default.write("./remora", tempZipPath);
|
|
59481
60470
|
try {
|
|
59482
|
-
const zipBuffer =
|
|
60471
|
+
const zipBuffer = import_fs29.default.readFileSync(tempZipPath);
|
|
59483
60472
|
const host = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST");
|
|
59484
60473
|
const version = Constants_default.workerVersion;
|
|
59485
60474
|
const workerAPI = `${host}/cli/v${version}/uploaddeployment`;
|
|
@@ -59501,7 +60490,7 @@ var deploy = async (options2) => {
|
|
|
59501
60490
|
}
|
|
59502
60491
|
console.log(import_chalk5.default.green("\u2705 Deployment package uploaded successfully"));
|
|
59503
60492
|
} finally {
|
|
59504
|
-
|
|
60493
|
+
import_fs29.default.unlinkSync(tempZipPath);
|
|
59505
60494
|
}
|
|
59506
60495
|
} catch (error) {
|
|
59507
60496
|
console.error(import_chalk5.default.red("\u274C Deployment error:"), error instanceof Error ? error.message : String(error));
|
|
@@ -59552,16 +60541,17 @@ var ensureEnvRuntimeContext = async () => {
|
|
|
59552
60541
|
await import_fs_extra2.default.appendFile(envPath, `${prefix2}${line}
|
|
59553
60542
|
`);
|
|
59554
60543
|
};
|
|
60544
|
+
var GENERATED_FOLDERS = ["remora/.temp", "remora/.synth"];
|
|
59555
60545
|
var ensureGitignoreEntry = async () => {
|
|
59556
60546
|
const gitignorePath = ".gitignore";
|
|
59557
|
-
const entry = "remora/.temp";
|
|
59558
60547
|
const exists = await import_fs_extra2.default.pathExists(gitignorePath);
|
|
59559
60548
|
const current = exists ? await import_fs_extra2.default.readFile(gitignorePath, "utf-8") : "";
|
|
59560
|
-
const
|
|
59561
|
-
|
|
60549
|
+
const rows = current.split("\n").map((row) => row.trim());
|
|
60550
|
+
const missing = GENERATED_FOLDERS.filter((entry) => !rows.includes(entry));
|
|
60551
|
+
if (missing.length === 0)
|
|
59562
60552
|
return;
|
|
59563
60553
|
const prefix2 = current.length && !current.endsWith("\n") ? "\n" : "";
|
|
59564
|
-
await import_fs_extra2.default.appendFile(gitignorePath, `${prefix2}${
|
|
60554
|
+
await import_fs_extra2.default.appendFile(gitignorePath, `${prefix2}${missing.join("\n")}
|
|
59565
60555
|
`);
|
|
59566
60556
|
};
|
|
59567
60557
|
var init = async () => {
|
|
@@ -59607,7 +60597,7 @@ var init = async () => {
|
|
|
59607
60597
|
console.log(import_chalk6.default.green("\u{1F4C1} A new folder named ") + import_chalk6.default.cyan("remora/") + import_chalk6.default.green(" has been created in your project."));
|
|
59608
60598
|
console.log(import_chalk6.default.green("\u270F\uFE0F Customize the generated configuration files inside this folder to setup Remora."));
|
|
59609
60599
|
console.log(import_chalk6.default.gray(" \u2022 REMORA_RUNTIME_CONTEXT=cli was added to your .env"));
|
|
59610
|
-
console.log(import_chalk6.default.gray(
|
|
60600
|
+
console.log(import_chalk6.default.gray(` \u2022 ${GENERATED_FOLDERS.join(" and ")} were added to your .gitignore`));
|
|
59611
60601
|
console.log(import_chalk6.default.green("\n\u{1F463} Next step: set your environment variables (add these to your .env file):"));
|
|
59612
60602
|
console.log(import_chalk6.default.blue("\u2022 REMORA_LICENCE_KEY") + import_chalk6.default.gray(" \u2192 Your valid Remora licence key (required to run any command)"));
|
|
59613
60603
|
console.log(import_chalk6.default.blue("\u2022 OPENAI_API_KEY") + import_chalk6.default.gray(" \u2192 Optional, enables AI features (discover, automap)"));
|
|
@@ -59620,10 +60610,15 @@ var init = async () => {
|
|
|
59620
60610
|
};
|
|
59621
60611
|
|
|
59622
60612
|
// src/actions/run.ts
|
|
59623
|
-
var
|
|
60613
|
+
var import_chalk8 = __toESM(require("chalk"));
|
|
59624
60614
|
init_src4();
|
|
59625
60615
|
init_src5();
|
|
59626
60616
|
|
|
60617
|
+
// src/licence.ts
|
|
60618
|
+
var import_chalk7 = __toESM(require("chalk"));
|
|
60619
|
+
var import_dayjs5 = __toESM(require("dayjs"));
|
|
60620
|
+
init_src4();
|
|
60621
|
+
|
|
59627
60622
|
// ../../packages/auth/src/AdminManager.ts
|
|
59628
60623
|
init_src();
|
|
59629
60624
|
init_src9();
|
|
@@ -59798,8 +60793,64 @@ var LicenceManagerClass = class {
|
|
|
59798
60793
|
var LicenceManager = new LicenceManagerClass();
|
|
59799
60794
|
var LicenceManager_default = LicenceManager;
|
|
59800
60795
|
|
|
60796
|
+
// src/licence.ts
|
|
60797
|
+
var readKey = () => {
|
|
60798
|
+
try {
|
|
60799
|
+
return ProcessENVManager_default.getEnvVariable("REMORA_LICENCE_KEY");
|
|
60800
|
+
} catch {
|
|
60801
|
+
return null;
|
|
60802
|
+
}
|
|
60803
|
+
};
|
|
60804
|
+
var print = (headline, body) => {
|
|
60805
|
+
console.error("");
|
|
60806
|
+
console.error(` ${import_chalk7.default.red("\u2716")} ${import_chalk7.default.red.bold(headline)}`);
|
|
60807
|
+
console.error("");
|
|
60808
|
+
for (const line of body)
|
|
60809
|
+
console.error(line ? ` ${line}` : "");
|
|
60810
|
+
console.error("");
|
|
60811
|
+
};
|
|
60812
|
+
var explainMissing = () => print("Remora needs a licence key to run.", [
|
|
60813
|
+
`${import_chalk7.default.cyan("REMORA_LICENCE_KEY")} is not set in this environment.`,
|
|
60814
|
+
"",
|
|
60815
|
+
`Add it to the ${import_chalk7.default.cyan(".env")} file beside your project folder:`,
|
|
60816
|
+
"",
|
|
60817
|
+
import_chalk7.default.gray(" REMORA_LICENCE_KEY=your-licence-key"),
|
|
60818
|
+
"",
|
|
60819
|
+
"or export it in your shell:",
|
|
60820
|
+
"",
|
|
60821
|
+
import_chalk7.default.gray(" export REMORA_LICENCE_KEY=your-licence-key"),
|
|
60822
|
+
"",
|
|
60823
|
+
import_chalk7.default.gray("No key yet? Ask whoever set Remora up for your team.")
|
|
60824
|
+
]);
|
|
60825
|
+
var explainExpired = (expiry) => print(`Your Remora licence expired on ${(0, import_dayjs5.default)(expiry).format("D MMMM YYYY")}.`, [
|
|
60826
|
+
"Renew the licence, then replace the key held in",
|
|
60827
|
+
`${import_chalk7.default.cyan("REMORA_LICENCE_KEY")} with the new one.`
|
|
60828
|
+
]);
|
|
60829
|
+
var explainUnrecognised = () => print("This Remora licence key is not recognised.", [
|
|
60830
|
+
`The key in ${import_chalk7.default.cyan("REMORA_LICENCE_KEY")} is not signed by Forza Labs.`,
|
|
60831
|
+
"Check it was copied whole \u2014 a licence key is one unbroken line,",
|
|
60832
|
+
"with no spaces and no line breaks."
|
|
60833
|
+
]);
|
|
60834
|
+
var explainMalformed = () => print("This Remora licence key cannot be read.", [
|
|
60835
|
+
`The value of ${import_chalk7.default.cyan("REMORA_LICENCE_KEY")} is not a well-formed key.`,
|
|
60836
|
+
"Check it was copied whole \u2014 a licence key is one unbroken line,",
|
|
60837
|
+
"with no spaces and no line breaks."
|
|
60838
|
+
]);
|
|
60839
|
+
var assertLicence = () => {
|
|
60840
|
+
const key = readKey();
|
|
60841
|
+
if (!key) {
|
|
60842
|
+
explainMissing();
|
|
60843
|
+
process.exit(1);
|
|
60844
|
+
}
|
|
60845
|
+
const check = LicenceManager_default.validate(key);
|
|
60846
|
+
if (check.valid) return check;
|
|
60847
|
+
if (check.expiryDate) explainExpired(check.expiryDate);
|
|
60848
|
+
else if (check.reason === "Invalid license signature") explainUnrecognised();
|
|
60849
|
+
else explainMalformed();
|
|
60850
|
+
process.exit(1);
|
|
60851
|
+
};
|
|
60852
|
+
|
|
59801
60853
|
// src/actions/run.ts
|
|
59802
|
-
init_src4();
|
|
59803
60854
|
init_src2();
|
|
59804
60855
|
var run = async (consumerName, options2) => {
|
|
59805
60856
|
try {
|
|
@@ -59822,29 +60873,24 @@ var run = async (consumerName, options2) => {
|
|
|
59822
60873
|
consumersToExecute.push(...Environment_default._env.consumers);
|
|
59823
60874
|
}
|
|
59824
60875
|
if (consumersToExecute.length === 1)
|
|
59825
|
-
console.log(
|
|
60876
|
+
console.log(import_chalk8.default.blue(`Running consumer ${consumersToExecute[0].name}...
|
|
59826
60877
|
`));
|
|
59827
60878
|
else
|
|
59828
|
-
console.log(
|
|
60879
|
+
console.log(import_chalk8.default.blue(`Running consumers ${consumersToExecute.map((x2) => x2.name).join(", ")}...
|
|
59829
60880
|
`));
|
|
59830
60881
|
if (options2?.limit)
|
|
59831
|
-
console.log(
|
|
60882
|
+
console.log(import_chalk8.default.gray(`Limiting output to ${options2.limit} records
|
|
59832
60883
|
`));
|
|
59833
60884
|
const results = [];
|
|
59834
60885
|
for (let i6 = 0; i6 < consumersToExecute.length; i6++) {
|
|
59835
60886
|
const consumer = consumersToExecute[i6];
|
|
59836
60887
|
try {
|
|
59837
|
-
const
|
|
59838
|
-
const check2 = LicenceManager_default.validate(remoraLicenceKey2);
|
|
59839
|
-
if (!check2.valid) {
|
|
59840
|
-
console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
|
|
59841
|
-
process.exit(1);
|
|
59842
|
-
}
|
|
60888
|
+
const check = assertLicence();
|
|
59843
60889
|
const response = await ExecutorOrchestrator_default.launch({
|
|
59844
60890
|
consumer,
|
|
59845
60891
|
details: {
|
|
59846
60892
|
invokedBy: "CLI",
|
|
59847
|
-
user: { _id:
|
|
60893
|
+
user: { _id: check.customer, name: check.customer, type: "licence" }
|
|
59848
60894
|
},
|
|
59849
60895
|
options: options2?.limit ? { limit: options2.limit } : void 0,
|
|
59850
60896
|
logProgress: true
|
|
@@ -59865,20 +60911,20 @@ var run = async (consumerName, options2) => {
|
|
|
59865
60911
|
const { elapsedMS, outputCount, resultUri } = response;
|
|
59866
60912
|
const rowCount = outputCount;
|
|
59867
60913
|
const duration = Helper_default.formatDuration(elapsedMS);
|
|
59868
|
-
const performanceInfo =
|
|
60914
|
+
const performanceInfo = import_chalk8.default.gray(` (${rowCount} rows, ${duration})`);
|
|
59869
60915
|
if (resultUri)
|
|
59870
|
-
console.log(
|
|
60916
|
+
console.log(import_chalk8.default.green(`\u2022 Consumer ${consumer.name} -> ${resultUri}`) + performanceInfo);
|
|
59871
60917
|
else
|
|
59872
|
-
console.log(
|
|
60918
|
+
console.log(import_chalk8.default.green(`\u2022 Consumer ${consumer.name} `) + performanceInfo);
|
|
59873
60919
|
} else {
|
|
59874
|
-
console.log(
|
|
60920
|
+
console.log(import_chalk8.default.red(`\u2022 Consumer ${consumer.name} -> Failed`));
|
|
59875
60921
|
}
|
|
59876
60922
|
});
|
|
59877
60923
|
const failedResults = results.filter((x2) => !x2.success);
|
|
59878
60924
|
if (failedResults.length > 0) {
|
|
59879
|
-
console.log(
|
|
60925
|
+
console.log(import_chalk8.default.red("\nError details:"));
|
|
59880
60926
|
failedResults.forEach((result, index) => {
|
|
59881
|
-
console.log(
|
|
60927
|
+
console.log(import_chalk8.default.red(` ${index + 1}. ${result.consumer.name}: ${result.error}`));
|
|
59882
60928
|
});
|
|
59883
60929
|
}
|
|
59884
60930
|
const successfulResults = results.filter((x2) => x2.success);
|
|
@@ -59888,17 +60934,17 @@ var run = async (consumerName, options2) => {
|
|
|
59888
60934
|
const totalDuration = successfulResults.reduce((sum, result) => {
|
|
59889
60935
|
return sum + (result.response?.elapsedMS || 0);
|
|
59890
60936
|
}, 0);
|
|
59891
|
-
const totalsInfo =
|
|
60937
|
+
const totalsInfo = import_chalk8.default.gray(` (${totalRows} rows, ${Helper_default.formatDuration(totalDuration)})`);
|
|
59892
60938
|
const hasFailures = results.some((x2) => !x2.success);
|
|
59893
60939
|
if (hasFailures)
|
|
59894
|
-
console.log(
|
|
60940
|
+
console.log(import_chalk8.default.blueBright("\n\u2139\uFE0F Run completed with errors") + totalsInfo);
|
|
59895
60941
|
else
|
|
59896
|
-
console.log(
|
|
60942
|
+
console.log(import_chalk8.default.green("\n\u2705 Run complete!") + totalsInfo);
|
|
59897
60943
|
await Logger_default.flush();
|
|
59898
60944
|
process.exit(hasFailures ? 1 : 0);
|
|
59899
60945
|
} catch (err2) {
|
|
59900
60946
|
const myErr = Helper_default.asError(err2);
|
|
59901
|
-
console.error(
|
|
60947
|
+
console.error(import_chalk8.default.red.bold("\n\u274C Unexpected error during run:"), myErr.message);
|
|
59902
60948
|
Logger_default.error(myErr);
|
|
59903
60949
|
await Logger_default.flush();
|
|
59904
60950
|
process.exit(1);
|
|
@@ -59906,24 +60952,24 @@ var run = async (consumerName, options2) => {
|
|
|
59906
60952
|
};
|
|
59907
60953
|
|
|
59908
60954
|
// src/actions/report.ts
|
|
59909
|
-
var
|
|
60955
|
+
var import_chalk9 = __toESM(require("chalk"));
|
|
59910
60956
|
init_src10();
|
|
59911
60957
|
init_src5();
|
|
59912
60958
|
init_src2();
|
|
59913
60959
|
var report = async (runId, options2 = {}) => {
|
|
59914
60960
|
try {
|
|
59915
|
-
console.log(
|
|
60961
|
+
console.log(import_chalk9.default.blue(runId ? `Collecting a bug report for run ${runId}...` : "Collecting a bug report for the last recorded run..."));
|
|
59916
60962
|
const result = BugReportService_default.collect({ runId, outputPath: options2.output });
|
|
59917
60963
|
const { manifest } = result;
|
|
59918
|
-
console.log(
|
|
59919
|
-
console.log(
|
|
59920
|
-
console.log(
|
|
60964
|
+
console.log(import_chalk9.default.green("\n\u2705 Bug report written: ") + import_chalk9.default.yellow(result.path));
|
|
60965
|
+
console.log(import_chalk9.default.blue(` ${manifest.files.length} files, ${(result.bytes / 1024).toFixed(1)} KB, bundle ${manifest.bundleId}`));
|
|
60966
|
+
console.log(import_chalk9.default.blue(" Run: ") + (manifest.runId ? import_chalk9.default.yellow(`${manifest.consumer} (${manifest.runId})`) : import_chalk9.default.yellow("none \u2014 no local run history, so the bundle reports the project as it stands")));
|
|
59921
60967
|
printSections(manifest.sections);
|
|
59922
|
-
console.log(
|
|
59923
|
-
console.log(
|
|
60968
|
+
console.log(import_chalk9.default.blue("\nIt holds no customer data: only configuration travels, and the log is scrubbed of credential shapes."));
|
|
60969
|
+
console.log(import_chalk9.default.blue("Read ") + import_chalk9.default.yellow("report.md") + import_chalk9.default.blue(" inside it, then attach the archive where you report the problem."));
|
|
59924
60970
|
await Logger_default.flush();
|
|
59925
60971
|
} catch (error) {
|
|
59926
|
-
console.error(
|
|
60972
|
+
console.error(import_chalk9.default.red.bold("\n\u274C Could not write the bug report:"), Helper_default.asError(error).message);
|
|
59927
60973
|
Logger_default.error(error);
|
|
59928
60974
|
await Logger_default.flush();
|
|
59929
60975
|
process.exit(1);
|
|
@@ -59932,14 +60978,14 @@ var report = async (runId, options2 = {}) => {
|
|
|
59932
60978
|
var printSections = (sections) => {
|
|
59933
60979
|
console.log("");
|
|
59934
60980
|
for (const section of sections) {
|
|
59935
|
-
const mark = section.included ?
|
|
59936
|
-
const reason = section.reason ?
|
|
59937
|
-
console.log(mark +
|
|
60981
|
+
const mark = section.included ? import_chalk9.default.green(" \u2713 ") : import_chalk9.default.gray(" \xB7 ");
|
|
60982
|
+
const reason = section.reason ? import_chalk9.default.gray(` \u2014 ${section.reason}`) : "";
|
|
60983
|
+
console.log(mark + import_chalk9.default.white(section.name.padEnd(12)) + reason);
|
|
59938
60984
|
}
|
|
59939
60985
|
};
|
|
59940
60986
|
|
|
59941
60987
|
// src/actions/discover.ts
|
|
59942
|
-
var
|
|
60988
|
+
var import_chalk10 = __toESM(require("chalk"));
|
|
59943
60989
|
var import_ora4 = __toESM(require("ora"));
|
|
59944
60990
|
init_src4();
|
|
59945
60991
|
init_src10();
|
|
@@ -59947,17 +60993,17 @@ init_src2();
|
|
|
59947
60993
|
var discover = async (producerName, options2) => {
|
|
59948
60994
|
try {
|
|
59949
60995
|
compile();
|
|
59950
|
-
const spinner = (0, import_ora4.default)(
|
|
60996
|
+
const spinner = (0, import_ora4.default)(import_chalk10.default.blue("Discovering producer...\n")).start();
|
|
59951
60997
|
const producer = Environment_default.getProducer(producerName);
|
|
59952
60998
|
const mode = options2?.replace ? "replace" : "merge";
|
|
59953
60999
|
const result = await DeveloperEngine_default.discover(producer, mode);
|
|
59954
61000
|
spinner.succeed("Producer has been successfully discovered");
|
|
59955
61001
|
reportDiscovery(result.report, mode);
|
|
59956
|
-
console.log(
|
|
61002
|
+
console.log(import_chalk10.default.green("\n\u2705 Discovery complete!"));
|
|
59957
61003
|
await Logger_default.flush();
|
|
59958
61004
|
process.exit(0);
|
|
59959
61005
|
} catch (err2) {
|
|
59960
|
-
console.error(
|
|
61006
|
+
console.error(import_chalk10.default.red.bold("\n\u274C Unexpected error during discover:"), err2 instanceof Error ? err2.message : String(err2));
|
|
59961
61007
|
Logger_default.error(err2);
|
|
59962
61008
|
await Logger_default.flush();
|
|
59963
61009
|
process.exit(1);
|
|
@@ -59966,21 +61012,21 @@ var discover = async (producerName, options2) => {
|
|
|
59966
61012
|
var reportDiscovery = (report4, mode) => {
|
|
59967
61013
|
console.log("");
|
|
59968
61014
|
if (report4.added.length > 0)
|
|
59969
|
-
console.log(
|
|
61015
|
+
console.log(import_chalk10.default.green(` + ${report4.added.length} new dimension(s): ${report4.added.join(", ")}`));
|
|
59970
61016
|
for (const { name, properties } of report4.filled)
|
|
59971
|
-
console.log(
|
|
61017
|
+
console.log(import_chalk10.default.blue(` ~ ${name}: filled in ${properties.join(", ")}`));
|
|
59972
61018
|
const onMismatch = mode === "replace" ? "rewritten to the source's" : "left as declared";
|
|
59973
61019
|
for (const { name, declared, discovered } of report4.mismatched)
|
|
59974
|
-
console.log(
|
|
61020
|
+
console.log(import_chalk10.default.yellow(` ! ${name} is declared "${declared}" but the source reports "${discovered}" \u2014 ${onMismatch}`));
|
|
59975
61021
|
if (report4.orphaned.length === 0) return;
|
|
59976
61022
|
const verb = mode === "replace" ? "dropped" : "kept";
|
|
59977
|
-
console.log(
|
|
61023
|
+
console.log(import_chalk10.default.yellow(` ! ${report4.orphaned.length} dimension(s) the source has no field for, ${verb}: ${report4.orphaned.join(", ")}`));
|
|
59978
61024
|
if (mode !== "replace")
|
|
59979
|
-
console.log(
|
|
61025
|
+
console.log(import_chalk10.default.gray(" Re-run with --replace to rewrite the dimensions from the source instead."));
|
|
59980
61026
|
};
|
|
59981
61027
|
|
|
59982
61028
|
// src/actions/graph.ts
|
|
59983
|
-
var
|
|
61029
|
+
var import_chalk11 = __toESM(require("chalk"));
|
|
59984
61030
|
init_src4();
|
|
59985
61031
|
init_src3();
|
|
59986
61032
|
init_src10();
|
|
@@ -59991,7 +61037,7 @@ var graph = async (options2) => {
|
|
|
59991
61037
|
const format4 = options2?.format ?? "mermaid";
|
|
59992
61038
|
try {
|
|
59993
61039
|
if (!FORMATS.includes(format4)) {
|
|
59994
|
-
console.error(
|
|
61040
|
+
console.error(import_chalk11.default.red(`Unknown format "${format4}". Use one of: ${FORMATS.join(", ")}.`));
|
|
59995
61041
|
await Logger_default.flush();
|
|
59996
61042
|
process.exit(1);
|
|
59997
61043
|
}
|
|
@@ -60008,7 +61054,7 @@ var graph = async (options2) => {
|
|
|
60008
61054
|
reportDiagnostics(snapshot.diagnostics);
|
|
60009
61055
|
} catch (err2) {
|
|
60010
61056
|
const myErr = Helper_default.asError(err2);
|
|
60011
|
-
console.error(
|
|
61057
|
+
console.error(import_chalk11.default.red.bold("\n\u274C Error while building the project graph:"), myErr.message);
|
|
60012
61058
|
Logger_default.error(err2);
|
|
60013
61059
|
await Logger_default.flush();
|
|
60014
61060
|
process.exit(1);
|
|
@@ -60023,14 +61069,14 @@ var reportDiagnostics = (diagnostics) => {
|
|
|
60023
61069
|
const errors = diagnostics.filter((x2) => x2.severity === "error");
|
|
60024
61070
|
const warnings = diagnostics.filter((x2) => x2.severity === "warning");
|
|
60025
61071
|
if (errors.length === 0 && warnings.length === 0) return;
|
|
60026
|
-
console.error(
|
|
61072
|
+
console.error(import_chalk11.default.gray(`
|
|
60027
61073
|
${errors.length} error(s), ${warnings.length} warning(s):`));
|
|
60028
|
-
errors.forEach((x2) => console.error(
|
|
60029
|
-
warnings.forEach((x2) => console.error(
|
|
61074
|
+
errors.forEach((x2) => console.error(import_chalk11.default.red(` \u2022 ${x2.message}`)));
|
|
61075
|
+
warnings.forEach((x2) => console.error(import_chalk11.default.yellow(` \u2022 ${x2.message}`)));
|
|
60030
61076
|
};
|
|
60031
61077
|
|
|
60032
61078
|
// src/actions/ui_preview.ts
|
|
60033
|
-
var
|
|
61079
|
+
var import_chalk12 = __toESM(require("chalk"));
|
|
60034
61080
|
var import_node_fs11 = __toESM(require("fs"));
|
|
60035
61081
|
var import_node_path16 = __toESM(require("path"));
|
|
60036
61082
|
init_src4();
|
|
@@ -60062,7 +61108,7 @@ var ui_preview = async (options2) => {
|
|
|
60062
61108
|
stopOnSignal(server, watch);
|
|
60063
61109
|
} catch (err2) {
|
|
60064
61110
|
const myErr = Helper_default.asError(err2);
|
|
60065
|
-
console.error(
|
|
61111
|
+
console.error(import_chalk12.default.red.bold("\n\u274C Could not start the project preview:"), myErr.message);
|
|
60066
61112
|
Logger_default.error(err2);
|
|
60067
61113
|
await Logger_default.flush();
|
|
60068
61114
|
process.exit(1);
|
|
@@ -60289,10 +61335,10 @@ var readPort = (value) => {
|
|
|
60289
61335
|
var report2 = (server, snapshot) => {
|
|
60290
61336
|
const errors = snapshot.diagnostics.filter((x2) => x2.severity === "error").length;
|
|
60291
61337
|
const warnings = snapshot.diagnostics.filter((x2) => x2.severity === "warning").length;
|
|
60292
|
-
console.log(
|
|
60293
|
-
\u{1F50D} Preview of "${snapshot.project}" on ${
|
|
60294
|
-
console.log(
|
|
60295
|
-
console.log(
|
|
61338
|
+
console.log(import_chalk12.default.bold(`
|
|
61339
|
+
\u{1F50D} Preview of "${snapshot.project}" on ${import_chalk12.default.cyan(server.url)}`));
|
|
61340
|
+
console.log(import_chalk12.default.gray(` ${snapshot.graph.nodes.length} node(s), ${snapshot.graph.edges.length} edge(s), ${errors} error(s), ${warnings} warning(s)`));
|
|
61341
|
+
console.log(import_chalk12.default.gray(" Watching the configuration and the run history for changes. Press Ctrl+C to stop.\n"));
|
|
60296
61342
|
};
|
|
60297
61343
|
var stopOnSignal = (server, watch) => {
|
|
60298
61344
|
const stop = async () => {
|
|
@@ -60306,47 +61352,47 @@ var stopOnSignal = (server, watch) => {
|
|
|
60306
61352
|
};
|
|
60307
61353
|
|
|
60308
61354
|
// src/actions/create_producer.ts
|
|
60309
|
-
var
|
|
60310
|
-
var
|
|
61355
|
+
var import_chalk13 = __toESM(require("chalk"));
|
|
61356
|
+
var import_fs30 = __toESM(require("fs"));
|
|
60311
61357
|
var import_path30 = __toESM(require("path"));
|
|
60312
61358
|
init_src2();
|
|
60313
61359
|
var create_producer = async (name) => {
|
|
60314
61360
|
try {
|
|
60315
|
-
if (!
|
|
60316
|
-
throw new Error(
|
|
61361
|
+
if (!import_fs30.default.existsSync(PRODUCERS_DIR)) {
|
|
61362
|
+
throw new Error(import_chalk13.default.red("Missing directory: ") + import_chalk13.default.yellow(PRODUCERS_DIR));
|
|
60317
61363
|
}
|
|
60318
|
-
const defaultProducerTemplate =
|
|
61364
|
+
const defaultProducerTemplate = import_fs30.default.readFileSync(
|
|
60319
61365
|
import_path30.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
|
|
60320
61366
|
"utf-8"
|
|
60321
61367
|
);
|
|
60322
61368
|
const defaultProducer = JSON.parse(defaultProducerTemplate);
|
|
60323
61369
|
defaultProducer.name = name;
|
|
60324
61370
|
const producerPath = import_path30.default.join(PRODUCERS_DIR, `${name}.json`);
|
|
60325
|
-
|
|
60326
|
-
console.log(
|
|
60327
|
-
console.log(
|
|
60328
|
-
console.log(
|
|
60329
|
-
console.log(
|
|
60330
|
-
console.log(
|
|
60331
|
-
console.log(
|
|
61371
|
+
import_fs30.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
|
|
61372
|
+
console.log(import_chalk13.default.green(`\u2705 Created producer config at ${producerPath}`));
|
|
61373
|
+
console.log(import_chalk13.default.blue("Remember to:"));
|
|
61374
|
+
console.log(import_chalk13.default.blue("1. Set the correct source name"));
|
|
61375
|
+
console.log(import_chalk13.default.blue(`2. Configure the dimensions, or run "remora discover ${name} --replace" to`));
|
|
61376
|
+
console.log(import_chalk13.default.blue(" write them from the source over the placeholders this template ships with"));
|
|
61377
|
+
console.log(import_chalk13.default.blue("3. Update the settings as needed"));
|
|
60332
61378
|
} catch (error) {
|
|
60333
|
-
console.error(
|
|
61379
|
+
console.error(import_chalk13.default.red("\u274C Error creating producer:"), error instanceof Error ? error.message : String(error));
|
|
60334
61380
|
Logger_default.error(error);
|
|
60335
61381
|
process.exit(1);
|
|
60336
61382
|
}
|
|
60337
61383
|
};
|
|
60338
61384
|
|
|
60339
61385
|
// src/actions/create_consumer.ts
|
|
60340
|
-
var
|
|
60341
|
-
var
|
|
61386
|
+
var import_chalk14 = __toESM(require("chalk"));
|
|
61387
|
+
var import_fs31 = __toESM(require("fs"));
|
|
60342
61388
|
var import_path31 = __toESM(require("path"));
|
|
60343
61389
|
init_src2();
|
|
60344
61390
|
var create_consumer = async (name, producerName) => {
|
|
60345
61391
|
try {
|
|
60346
|
-
if (!
|
|
60347
|
-
throw new Error(
|
|
61392
|
+
if (!import_fs31.default.existsSync(CONSUMERS_DIR)) {
|
|
61393
|
+
throw new Error(import_chalk14.default.red("Missing directory: ") + import_chalk14.default.yellow(CONSUMERS_DIR));
|
|
60348
61394
|
}
|
|
60349
|
-
const defaultConsumerTemplate =
|
|
61395
|
+
const defaultConsumerTemplate = import_fs31.default.readFileSync(
|
|
60350
61396
|
import_path31.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
|
|
60351
61397
|
"utf-8"
|
|
60352
61398
|
);
|
|
@@ -60354,10 +61400,10 @@ var create_consumer = async (name, producerName) => {
|
|
|
60354
61400
|
defaultConsumer.name = name;
|
|
60355
61401
|
if (producerName) {
|
|
60356
61402
|
const producerPath = import_path31.default.join(PRODUCERS_DIR, `${producerName}.json`);
|
|
60357
|
-
if (!
|
|
60358
|
-
throw new Error(
|
|
61403
|
+
if (!import_fs31.default.existsSync(producerPath)) {
|
|
61404
|
+
throw new Error(import_chalk14.default.red("Producer not found: ") + import_chalk14.default.yellow(producerPath));
|
|
60359
61405
|
}
|
|
60360
|
-
const producerConfig = JSON.parse(
|
|
61406
|
+
const producerConfig = JSON.parse(import_fs31.default.readFileSync(producerPath, "utf-8"));
|
|
60361
61407
|
defaultConsumer.producers = [{ name: producerName }];
|
|
60362
61408
|
defaultConsumer.fields = producerConfig.dimensions.map((dim) => ({
|
|
60363
61409
|
key: dim.name,
|
|
@@ -60370,43 +61416,79 @@ var create_consumer = async (name, producerName) => {
|
|
|
60370
61416
|
defaultConsumer.metadata = void 0;
|
|
60371
61417
|
}
|
|
60372
61418
|
const consumerPath = import_path31.default.join(CONSUMERS_DIR, `${name}.json`);
|
|
60373
|
-
|
|
60374
|
-
console.log(
|
|
61419
|
+
import_fs31.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
|
|
61420
|
+
console.log(import_chalk14.default.green(`\u2705 Created consumer config at ${consumerPath}`));
|
|
60375
61421
|
if (!producerName) {
|
|
60376
|
-
console.log(
|
|
60377
|
-
console.log(
|
|
60378
|
-
console.log(
|
|
60379
|
-
console.log(
|
|
61422
|
+
console.log(import_chalk14.default.blue("Remember to:"));
|
|
61423
|
+
console.log(import_chalk14.default.blue("1. Set the correct producer name"));
|
|
61424
|
+
console.log(import_chalk14.default.blue("2. Configure the fields you want to include"));
|
|
61425
|
+
console.log(import_chalk14.default.blue("3. Set up the desired output formats"));
|
|
60380
61426
|
} else {
|
|
60381
|
-
console.log(
|
|
60382
|
-
console.log(
|
|
60383
|
-
console.log(
|
|
60384
|
-
console.log(
|
|
61427
|
+
console.log(import_chalk14.default.blue("Consumer is configured to match all fields from producer:"), import_chalk14.default.yellow(producerName));
|
|
61428
|
+
console.log(import_chalk14.default.blue("Remember to:"));
|
|
61429
|
+
console.log(import_chalk14.default.blue("1. Review the included fields"));
|
|
61430
|
+
console.log(import_chalk14.default.blue("2. Set up the desired output formats"));
|
|
60385
61431
|
}
|
|
60386
61432
|
} catch (error) {
|
|
60387
|
-
console.error(
|
|
61433
|
+
console.error(import_chalk14.default.red("\u274C Error creating consumer:"), error instanceof Error ? error.message : String(error));
|
|
60388
61434
|
Logger_default.error(error);
|
|
60389
61435
|
process.exit(1);
|
|
60390
61436
|
}
|
|
60391
61437
|
};
|
|
60392
61438
|
|
|
61439
|
+
// src/actions/import_copybook.ts
|
|
61440
|
+
var import_chalk15 = __toESM(require("chalk"));
|
|
61441
|
+
var import_fs32 = __toESM(require("fs"));
|
|
61442
|
+
var import_path32 = __toESM(require("path"));
|
|
61443
|
+
init_src10();
|
|
61444
|
+
init_src2();
|
|
61445
|
+
var PRODUCER_SCHEMA = "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json";
|
|
61446
|
+
var import_copybook = async (file, options2 = {}) => {
|
|
61447
|
+
try {
|
|
61448
|
+
if (!import_fs32.default.existsSync(file))
|
|
61449
|
+
throw new Error(`No copybook at ${file}`);
|
|
61450
|
+
if (!import_fs32.default.existsSync(PRODUCERS_DIR))
|
|
61451
|
+
throw new Error(`Missing directory: ${PRODUCERS_DIR}`);
|
|
61452
|
+
const name = options2.name || defaultName(file);
|
|
61453
|
+
const producerPath = import_path32.default.join(PRODUCERS_DIR, `${name}.json`);
|
|
61454
|
+
if (import_fs32.default.existsSync(producerPath))
|
|
61455
|
+
throw new Error(`${producerPath} already exists. Pass a different name with -n, or remove the file to replace it.`);
|
|
61456
|
+
const layout = CopybookParser_default.parse(import_fs32.default.readFileSync(file, "utf-8"));
|
|
61457
|
+
const { producer, notes } = CopybookEmitter_default.toProducer(layout, { name });
|
|
61458
|
+
import_fs32.default.writeFileSync(producerPath, JSON.stringify({ $schema: PRODUCER_SCHEMA, ...producer }, null, 4));
|
|
61459
|
+
console.log(import_chalk15.default.green(`\u2705 Created producer config at ${producerPath}`));
|
|
61460
|
+
console.log(import_chalk15.default.blue(`Record ${layout.name}: ${layout.width} characters, ${producer.dimensions.length} dimension(s).`));
|
|
61461
|
+
notes.forEach((note) => console.log(import_chalk15.default.yellow(`\u2022 ${note}`)));
|
|
61462
|
+
console.log(import_chalk15.default.blue('Remember to set the source and the fileKey, then run "remora compile".'));
|
|
61463
|
+
} catch (error) {
|
|
61464
|
+
console.error(import_chalk15.default.red("\u274C Error importing copybook:"), error instanceof Error ? error.message : String(error));
|
|
61465
|
+
Logger_default.error(error);
|
|
61466
|
+
process.exit(1);
|
|
61467
|
+
}
|
|
61468
|
+
};
|
|
61469
|
+
var defaultName = (file) => {
|
|
61470
|
+
const base = import_path32.default.basename(file, import_path32.default.extname(file)).toLowerCase().replace(/[^a-z0-9]+/g, "_");
|
|
61471
|
+
const trimmed = base.replace(/^_+|_+$/g, "") || "copybook";
|
|
61472
|
+
return trimmed.startsWith("p_") ? trimmed : `p_${trimmed}`;
|
|
61473
|
+
};
|
|
61474
|
+
|
|
60393
61475
|
// src/index.ts
|
|
60394
61476
|
init_src3();
|
|
60395
61477
|
init_src5();
|
|
60396
61478
|
|
|
60397
61479
|
// src/actions/automap.ts
|
|
60398
|
-
var
|
|
61480
|
+
var import_chalk16 = __toESM(require("chalk"));
|
|
60399
61481
|
var import_ora5 = __toESM(require("ora"));
|
|
60400
61482
|
init_src10();
|
|
60401
|
-
var
|
|
60402
|
-
var
|
|
61483
|
+
var import_fs33 = __toESM(require("fs"));
|
|
61484
|
+
var import_path33 = __toESM(require("path"));
|
|
60403
61485
|
init_src4();
|
|
60404
61486
|
init_src10();
|
|
60405
61487
|
init_src2();
|
|
60406
61488
|
var automap = async (producerName, schemaNames) => {
|
|
60407
61489
|
try {
|
|
60408
61490
|
compile();
|
|
60409
|
-
const spinner = (0, import_ora5.default)(
|
|
61491
|
+
const spinner = (0, import_ora5.default)(import_chalk16.default.blue("Auto-mapping producer data...\n")).start();
|
|
60410
61492
|
const producer = Environment_default.getProducer(producerName);
|
|
60411
61493
|
if (!producer) {
|
|
60412
61494
|
throw new Error(`Producer ${producerName} not found`);
|
|
@@ -60428,24 +61510,24 @@ var automap = async (producerName, schemaNames) => {
|
|
|
60428
61510
|
const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
|
|
60429
61511
|
for (const producer2 of mapResult.producers) {
|
|
60430
61512
|
const producerPath = Environment_default.resolveResourceFilePath("producers", producer2.name);
|
|
60431
|
-
|
|
60432
|
-
|
|
60433
|
-
console.log(
|
|
61513
|
+
import_fs33.default.mkdirSync(import_path33.default.dirname(producerPath), { recursive: true });
|
|
61514
|
+
import_fs33.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
|
|
61515
|
+
console.log(import_chalk16.default.blue(`Created producer: ${producer2.name}`));
|
|
60434
61516
|
}
|
|
60435
61517
|
for (const consumer of mapResult.consumers) {
|
|
60436
61518
|
const consumerPath = Environment_default.resolveResourceFilePath("consumers", consumer.name);
|
|
60437
|
-
|
|
60438
|
-
|
|
60439
|
-
console.log(
|
|
61519
|
+
import_fs33.default.mkdirSync(import_path33.default.dirname(consumerPath), { recursive: true });
|
|
61520
|
+
import_fs33.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
|
|
61521
|
+
console.log(import_chalk16.default.blue(`Created consumer: ${consumer.name}`));
|
|
60440
61522
|
}
|
|
60441
61523
|
spinner.succeed("Producer has been successfully mapped");
|
|
60442
|
-
console.log(
|
|
61524
|
+
console.log(import_chalk16.default.green(`
|
|
60443
61525
|
\u2705 Created ${mapResult.producers.length} producers!`));
|
|
60444
|
-
console.log(
|
|
61526
|
+
console.log(import_chalk16.default.green(`\u2705 Created ${mapResult.consumers.length} consumers!`));
|
|
60445
61527
|
await Logger_default.flush();
|
|
60446
61528
|
process.exit(0);
|
|
60447
61529
|
} catch (err2) {
|
|
60448
|
-
console.error(
|
|
61530
|
+
console.error(import_chalk16.default.red.bold("\n\u274C Unexpected error during automapping:"), err2 instanceof Error ? err2.message : String(err2));
|
|
60449
61531
|
Logger_default.error(err2);
|
|
60450
61532
|
await Logger_default.flush();
|
|
60451
61533
|
process.exit(1);
|
|
@@ -60453,7 +61535,7 @@ var automap = async (producerName, schemaNames) => {
|
|
|
60453
61535
|
};
|
|
60454
61536
|
|
|
60455
61537
|
// src/actions/sample.ts
|
|
60456
|
-
var
|
|
61538
|
+
var import_chalk17 = __toESM(require("chalk"));
|
|
60457
61539
|
var import_ora6 = __toESM(require("ora"));
|
|
60458
61540
|
init_src5();
|
|
60459
61541
|
init_src2();
|
|
@@ -60461,23 +61543,23 @@ var sample = async (resourceName, sampleSize = 10) => {
|
|
|
60461
61543
|
try {
|
|
60462
61544
|
compile();
|
|
60463
61545
|
console.log();
|
|
60464
|
-
const spinner = (0, import_ora6.default)(
|
|
61546
|
+
const spinner = (0, import_ora6.default)(import_chalk17.default.blue(`Sampling "${resourceName}"...`)).start();
|
|
60465
61547
|
const { kind, records, note } = await SampleRunner_default.read(resourceName, sampleSize);
|
|
60466
|
-
spinner.succeed(
|
|
60467
|
-
console.log(
|
|
61548
|
+
spinner.succeed(import_chalk17.default.green(`Sample data retrieved from ${kind} "${resourceName}"`));
|
|
61549
|
+
console.log(import_chalk17.default.gray(` ${note}`));
|
|
60468
61550
|
if (records.length === 0) {
|
|
60469
|
-
console.log(
|
|
61551
|
+
console.log(import_chalk17.default.yellow("No data found in the dataset."));
|
|
60470
61552
|
return;
|
|
60471
61553
|
}
|
|
60472
|
-
console.log(
|
|
61554
|
+
console.log(import_chalk17.default.cyan(`
|
|
60473
61555
|
\u{1F4CA} Sample Data (showing ${records.length} rows):`));
|
|
60474
|
-
console.log(
|
|
61556
|
+
console.log(import_chalk17.default.gray("\u2500".repeat(80)));
|
|
60475
61557
|
displayDataAsTable(records);
|
|
60476
|
-
console.log(
|
|
60477
|
-
console.log(
|
|
61558
|
+
console.log(import_chalk17.default.gray("\u2500".repeat(80)));
|
|
61559
|
+
console.log(import_chalk17.default.green(`\u2705 Successfully sampled ${records.length} rows from ${kind} "${resourceName}"`));
|
|
60478
61560
|
} catch (err2) {
|
|
60479
61561
|
const myErr = Helper_default.asError(err2);
|
|
60480
|
-
console.error(
|
|
61562
|
+
console.error(import_chalk17.default.red.bold("\n\u274C Error during sampling:"), myErr.message);
|
|
60481
61563
|
Logger_default.error(err2);
|
|
60482
61564
|
await Logger_default.flush();
|
|
60483
61565
|
process.exit(1);
|
|
@@ -60506,7 +61588,7 @@ var displayDataAsTable = (data) => {
|
|
|
60506
61588
|
columnWidths[field] = Math.min(columnWidths[field], maxColumnWidth);
|
|
60507
61589
|
});
|
|
60508
61590
|
const headerRow = fields.map(
|
|
60509
|
-
(field) =>
|
|
61591
|
+
(field) => import_chalk17.default.bold(field.padEnd(columnWidths[field]))
|
|
60510
61592
|
).join(" \u2502 ");
|
|
60511
61593
|
console.log("\u2502 " + headerRow + " \u2502");
|
|
60512
61594
|
const separator = fields.map(
|
|
@@ -60521,33 +61603,35 @@ var displayDataAsTable = (data) => {
|
|
|
60521
61603
|
return truncatedValue.padEnd(columnWidths[field]);
|
|
60522
61604
|
}).join(" \u2502 ");
|
|
60523
61605
|
if (index % 2 === 0) {
|
|
60524
|
-
console.log("\u2502 " +
|
|
61606
|
+
console.log("\u2502 " + import_chalk17.default.white(dataRow) + " \u2502");
|
|
60525
61607
|
} else {
|
|
60526
|
-
console.log("\u2502 " +
|
|
61608
|
+
console.log("\u2502 " + import_chalk17.default.gray(dataRow) + " \u2502");
|
|
60527
61609
|
}
|
|
60528
61610
|
});
|
|
60529
61611
|
};
|
|
60530
61612
|
var formatValue = (value) => {
|
|
60531
61613
|
if (value === null || value === void 0) {
|
|
60532
|
-
return
|
|
61614
|
+
return import_chalk17.default.dim("null");
|
|
60533
61615
|
}
|
|
60534
61616
|
if (typeof value === "string") {
|
|
60535
61617
|
return value;
|
|
60536
61618
|
}
|
|
60537
61619
|
if (typeof value === "number") {
|
|
60538
|
-
return
|
|
61620
|
+
return import_chalk17.default.cyan(value.toString());
|
|
60539
61621
|
}
|
|
60540
61622
|
if (typeof value === "boolean") {
|
|
60541
|
-
return
|
|
61623
|
+
return import_chalk17.default.yellow(value.toString());
|
|
60542
61624
|
}
|
|
60543
61625
|
if (value instanceof Date) {
|
|
60544
|
-
return
|
|
61626
|
+
return import_chalk17.default.magenta(value.toISOString());
|
|
60545
61627
|
}
|
|
60546
|
-
return
|
|
61628
|
+
return import_chalk17.default.dim(JSON.stringify(value));
|
|
60547
61629
|
};
|
|
60548
61630
|
|
|
60549
61631
|
// src/actions/synth.ts
|
|
60550
|
-
var
|
|
61632
|
+
var import_path34 = __toESM(require("path"));
|
|
61633
|
+
var import_chalk18 = __toESM(require("chalk"));
|
|
61634
|
+
init_src3();
|
|
60551
61635
|
init_src4();
|
|
60552
61636
|
init_src5();
|
|
60553
61637
|
init_src10();
|
|
@@ -60558,19 +61642,21 @@ var synth = async (consumerName, args) => {
|
|
|
60558
61642
|
compile();
|
|
60559
61643
|
assertLicence();
|
|
60560
61644
|
const consumers = resolveConsumers(consumerName, args.project);
|
|
61645
|
+
const outputPath = import_path34.default.resolve(args.output ?? import_path34.default.join(Constants_default.defaults.REMORA_PATH, Constants_default.defaults.SYNTH_OUTPUT_FOLDER));
|
|
60561
61646
|
console.log();
|
|
60562
|
-
console.log(
|
|
61647
|
+
console.log(import_chalk18.default.blue(`Synthesizing ${consumers.length === 1 ? `consumer ${consumers[0].name}` : `${consumers.length} consumers`} with ${args.records} input records/producer (seed "${args.seed}")...`));
|
|
61648
|
+
console.log(import_chalk18.default.gray(`Exports are written to ${outputPath}, not to the consumers' configured destinations.
|
|
60563
61649
|
`));
|
|
60564
61650
|
bundle = await SyntheticInputEngine_default.generate(Environment_default.getAllProducers(), { records: args.records, seed: args.seed }, Environment_default.getAllCodeSets());
|
|
60565
61651
|
const results = [];
|
|
60566
61652
|
for (const consumer of consumers)
|
|
60567
|
-
results.push(await runConsumer(consumer, bundle));
|
|
61653
|
+
results.push(await runConsumer(consumer, bundle, outputPath));
|
|
60568
61654
|
report3(results);
|
|
60569
61655
|
await Logger_default.flush();
|
|
60570
61656
|
process.exit(results.some((x2) => !x2.success) ? 1 : 0);
|
|
60571
61657
|
} catch (err2) {
|
|
60572
61658
|
const myErr = Helper_default.asError(err2);
|
|
60573
|
-
console.error(
|
|
61659
|
+
console.error(import_chalk18.default.red.bold("\n\u274C Unexpected error during synthetic run:"), myErr.message);
|
|
60574
61660
|
Logger_default.error(myErr);
|
|
60575
61661
|
await Logger_default.flush();
|
|
60576
61662
|
process.exit(1);
|
|
@@ -60593,13 +61679,14 @@ var resolveConsumers = (consumerName, project) => {
|
|
|
60593
61679
|
if (all.length === 0) throw new Error("No consumers found in the project.");
|
|
60594
61680
|
return all;
|
|
60595
61681
|
};
|
|
60596
|
-
var runConsumer = async (consumer, bundle) => {
|
|
61682
|
+
var runConsumer = async (consumer, bundle, exportPath) => {
|
|
60597
61683
|
try {
|
|
60598
|
-
const
|
|
61684
|
+
const check = assertLicence();
|
|
60599
61685
|
const response = await ExecutorOrchestrator_default.launch({
|
|
60600
61686
|
consumer,
|
|
60601
|
-
details: { invokedBy: "CLI", user: { _id:
|
|
61687
|
+
details: { invokedBy: "CLI", user: { _id: check.customer, name: check.customer, type: "licence" } },
|
|
60602
61688
|
syntheticProducerInputs: bundle.inputs,
|
|
61689
|
+
syntheticExportPath: exportPath,
|
|
60603
61690
|
logProgress: true
|
|
60604
61691
|
});
|
|
60605
61692
|
return { consumer, success: true, outputCount: response.outputCount, resultUri: response.resultUri, elapsedMS: response.elapsedMS };
|
|
@@ -60609,25 +61696,18 @@ var runConsumer = async (consumer, bundle) => {
|
|
|
60609
61696
|
return { consumer, success: false, error: myErr.message };
|
|
60610
61697
|
}
|
|
60611
61698
|
};
|
|
60612
|
-
var assertLicence = () => {
|
|
60613
|
-
const check2 = LicenceManager_default.validate(ProcessENVManager_default.getEnvVariable("REMORA_LICENCE_KEY"));
|
|
60614
|
-
if (!check2.valid) {
|
|
60615
|
-
console.error(import_chalk16.default.red('Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.'));
|
|
60616
|
-
process.exit(1);
|
|
60617
|
-
}
|
|
60618
|
-
};
|
|
60619
61699
|
var report3 = (results) => {
|
|
60620
|
-
console.log(
|
|
61700
|
+
console.log(import_chalk18.default.green("\n\u2705 Synthetic run complete!\n"));
|
|
60621
61701
|
for (const result of results) {
|
|
60622
61702
|
if (!result.success) {
|
|
60623
|
-
console.log(
|
|
61703
|
+
console.log(import_chalk18.default.red(` \u2717 ${result.consumer.name} \u2014 ${result.error}`));
|
|
60624
61704
|
continue;
|
|
60625
61705
|
}
|
|
60626
|
-
const info =
|
|
61706
|
+
const info = import_chalk18.default.gray(` (${result.outputCount} rows, ${Helper_default.formatDuration(result.elapsedMS ?? 0)})`);
|
|
60627
61707
|
if (result.resultUri)
|
|
60628
|
-
console.log(
|
|
61708
|
+
console.log(import_chalk18.default.green(` \u2022 ${result.consumer.name} \u2192 ${result.resultUri}`) + info);
|
|
60629
61709
|
else
|
|
60630
|
-
console.log(
|
|
61710
|
+
console.log(import_chalk18.default.green(` \u2022 ${result.consumer.name}`) + info);
|
|
60631
61711
|
}
|
|
60632
61712
|
};
|
|
60633
61713
|
|
|
@@ -60646,12 +61726,11 @@ if (!process.env.REMORA_RUNTIME_CONTEXT) {
|
|
|
60646
61726
|
}
|
|
60647
61727
|
ProcessShutdownManager_default.init("Remora CLI");
|
|
60648
61728
|
var program = new import_commander.Command();
|
|
60649
|
-
var
|
|
60650
|
-
|
|
60651
|
-
if (
|
|
60652
|
-
|
|
60653
|
-
|
|
60654
|
-
}
|
|
61729
|
+
var LICENCE_EXEMPT_COMMANDS = ["init", "version"];
|
|
61730
|
+
program.hook("preAction", (_root, command5) => {
|
|
61731
|
+
if (LICENCE_EXEMPT_COMMANDS.includes(command5.name())) return;
|
|
61732
|
+
assertLicence();
|
|
61733
|
+
});
|
|
60655
61734
|
var { heapSizeMB } = Runtime_default.getHeap();
|
|
60656
61735
|
if (heapSizeMB < Constants_default.defaults.MIN_RUNTIME_HEAP_MB)
|
|
60657
61736
|
console.warn(`Remora is running with ${heapSizeMB}MB of runtime heap, which is below the bare minimum of ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB (Recommended: ${Constants_default.defaults.RECOMMENDED_RUNTIME_HEAP_MB}MB).`);
|
|
@@ -60672,6 +61751,7 @@ program.command("discover").description("Discover the data shape of a producer a
|
|
|
60672
61751
|
program.command("graph").description("Print the project lineage (what feeds what) to stdout as a diagram or as JSON").option("-f, --format <format>", "Output format: mermaid, dot or json", "mermaid").action((options2) => graph(options2));
|
|
60673
61752
|
program.command("ui-preview").description("Open a visual preview of the project: the lineage graph, and a table inspector for every resource").option("-p, --port <number>", "Port to serve the preview on (default: 5070, or the next free one)").option("--no-open", "Do not open a browser window").action((options2) => ui_preview(options2));
|
|
60674
61753
|
program.command("create-producer <name>").description("Create a new producer configuration with default settings").action(create_producer);
|
|
61754
|
+
program.command("import-copybook <file>").description("Read a COBOL copybook and write the FIXED producer for the record it describes, computing every position from the PIC widths").option("-n, --name <name>", 'Name of the producer to write (default: the copybook file name, prefixed with "p_")').action((file, options2) => import_copybook(file, options2));
|
|
60675
61755
|
program.command("automap").description("Automatically map a producer to consumers using specified schemas.").argument("<producer>", "The producer to analyze").argument("<schemas...>", "One or more schema names to map against").action(automap);
|
|
60676
61756
|
program.command("create-consumer <name>").description("Create a new consumer configuration with default settings").option("-p, --producer <name>", "Producer to create a one-to-one mapping from").action((name, options2) => create_consumer(name, options2.producer));
|
|
60677
61757
|
program.command("sample <name>").description("Sample data from a producer or consumer and display it in a formatted table").option("-s, --size <number>", "Number of sample rows to display (default: 10)", "10").action((name, options2) => {
|
|
@@ -60682,13 +61762,13 @@ program.command("sample <name>").description("Sample data from a producer or con
|
|
|
60682
61762
|
}
|
|
60683
61763
|
sample(name, sampleSize);
|
|
60684
61764
|
});
|
|
60685
|
-
program.command("synth [consumer]").description("Run the full pipeline on synthetic data: generate fake producer input, then execute one consumer, all consumers of a project (--project), or all consumers,
|
|
61765
|
+
program.command("synth [consumer]").description("Run the full pipeline on synthetic data: generate fake producer input, then execute one consumer, all consumers of a project (--project), or all consumers, writing the exports to a local folder instead of their configured destinations").option("-r, --records <number>", "Number of synthetic INPUT records to generate per producer", "100").option("-s, --seed <string>", "Seed for deterministic generation (same seed => same data)", "remora").option("-p, --project <name>", "Run all consumers belonging to the specified project").option("-o, --output <dir>", `Folder to write the exports into, one subfolder per consumer (default: ${Constants_default.defaults.REMORA_PATH}/${Constants_default.defaults.SYNTH_OUTPUT_FOLDER})`).action((consumer, options2) => {
|
|
60686
61766
|
const records = parseInt(options2.records, 10);
|
|
60687
61767
|
if (isNaN(records) || records <= 0) {
|
|
60688
61768
|
console.error("Record count must be a positive number");
|
|
60689
61769
|
process.exit(1);
|
|
60690
61770
|
}
|
|
60691
|
-
synth(consumer, { records, seed: options2.seed, project: options2.project });
|
|
61771
|
+
synth(consumer, { records, seed: options2.seed, project: options2.project, output: options2.output });
|
|
60692
61772
|
});
|
|
60693
61773
|
program.parse(process.argv);
|
|
60694
61774
|
/*! Bundled license information:
|