@forzalabs/remora 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/index.js +2033 -1072
- package/json_schemas/consumer-schema.json +4 -0
- package/package.json +1 -1
- package/workers/ExecutorWorker.js +1881 -892
|
@@ -544,7 +544,7 @@ var require_has_flag = __commonJS({
|
|
|
544
544
|
var require_supports_colors = __commonJS({
|
|
545
545
|
"../../node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
|
|
546
546
|
"use strict";
|
|
547
|
-
var
|
|
547
|
+
var os5 = require("os");
|
|
548
548
|
var hasFlag = require_has_flag();
|
|
549
549
|
var env2 = process.env;
|
|
550
550
|
var forceColor = void 0;
|
|
@@ -582,7 +582,7 @@ var require_supports_colors = __commonJS({
|
|
|
582
582
|
}
|
|
583
583
|
var min = forceColor ? 1 : 0;
|
|
584
584
|
if (process.platform === "win32") {
|
|
585
|
-
var osRelease =
|
|
585
|
+
var osRelease = os5.release().split(".");
|
|
586
586
|
if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
|
|
587
587
|
return Number(osRelease[2]) >= 14931 ? 3 : 2;
|
|
588
588
|
}
|
|
@@ -5827,7 +5827,7 @@ var require_winston_transport = __commonJS({
|
|
|
5827
5827
|
var require_console = __commonJS({
|
|
5828
5828
|
"../../node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
|
|
5829
5829
|
"use strict";
|
|
5830
|
-
var
|
|
5830
|
+
var os5 = require("os");
|
|
5831
5831
|
var { LEVEL, MESSAGE } = require_triple_beam();
|
|
5832
5832
|
var TransportStream = require_winston_transport();
|
|
5833
5833
|
module2.exports = class Console extends TransportStream {
|
|
@@ -5841,7 +5841,7 @@ var require_console = __commonJS({
|
|
|
5841
5841
|
this.name = options2.name || "console";
|
|
5842
5842
|
this.stderrLevels = this._stringArrayToSet(options2.stderrLevels);
|
|
5843
5843
|
this.consoleWarnLevels = this._stringArrayToSet(options2.consoleWarnLevels);
|
|
5844
|
-
this.eol = typeof options2.eol === "string" ? options2.eol :
|
|
5844
|
+
this.eol = typeof options2.eol === "string" ? options2.eol : os5.EOL;
|
|
5845
5845
|
this.forceConsole = options2.forceConsole || false;
|
|
5846
5846
|
this._consoleLog = console.log.bind(console);
|
|
5847
5847
|
this._consoleWarn = console.warn.bind(console);
|
|
@@ -7900,15 +7900,15 @@ var require_index_cjs = __commonJS({
|
|
|
7900
7900
|
};
|
|
7901
7901
|
}
|
|
7902
7902
|
function wrapConversion(toModel, graph) {
|
|
7903
|
-
const
|
|
7903
|
+
const path22 = [graph[toModel].parent, toModel];
|
|
7904
7904
|
let fn = convert$1[graph[toModel].parent][toModel];
|
|
7905
7905
|
let cur = graph[toModel].parent;
|
|
7906
7906
|
while (graph[cur].parent) {
|
|
7907
|
-
|
|
7907
|
+
path22.unshift(graph[cur].parent);
|
|
7908
7908
|
fn = link(convert$1[graph[cur].parent][cur], fn);
|
|
7909
7909
|
cur = graph[cur].parent;
|
|
7910
7910
|
}
|
|
7911
|
-
fn.conversion =
|
|
7911
|
+
fn.conversion = path22;
|
|
7912
7912
|
return fn;
|
|
7913
7913
|
}
|
|
7914
7914
|
function route(fromModel) {
|
|
@@ -8538,7 +8538,7 @@ var require_node2 = __commonJS({
|
|
|
8538
8538
|
var require_tail_file = __commonJS({
|
|
8539
8539
|
"../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
|
|
8540
8540
|
"use strict";
|
|
8541
|
-
var
|
|
8541
|
+
var fs31 = require("fs");
|
|
8542
8542
|
var { StringDecoder } = require("string_decoder");
|
|
8543
8543
|
var { Stream } = require_readable();
|
|
8544
8544
|
function noop() {
|
|
@@ -8559,7 +8559,7 @@ var require_tail_file = __commonJS({
|
|
|
8559
8559
|
stream.emit("end");
|
|
8560
8560
|
stream.emit("close");
|
|
8561
8561
|
};
|
|
8562
|
-
|
|
8562
|
+
fs31.open(options2.file, "a+", "0644", (err2, fd) => {
|
|
8563
8563
|
if (err2) {
|
|
8564
8564
|
if (!iter) {
|
|
8565
8565
|
stream.emit("error", err2);
|
|
@@ -8571,10 +8571,10 @@ var require_tail_file = __commonJS({
|
|
|
8571
8571
|
}
|
|
8572
8572
|
(function read() {
|
|
8573
8573
|
if (stream.destroyed) {
|
|
8574
|
-
|
|
8574
|
+
fs31.close(fd, noop);
|
|
8575
8575
|
return;
|
|
8576
8576
|
}
|
|
8577
|
-
return
|
|
8577
|
+
return fs31.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
|
|
8578
8578
|
if (error) {
|
|
8579
8579
|
if (!iter) {
|
|
8580
8580
|
stream.emit("error", error);
|
|
@@ -8633,15 +8633,15 @@ var require_tail_file = __commonJS({
|
|
|
8633
8633
|
var require_file = __commonJS({
|
|
8634
8634
|
"../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
|
|
8635
8635
|
"use strict";
|
|
8636
|
-
var
|
|
8637
|
-
var
|
|
8636
|
+
var fs31 = require("fs");
|
|
8637
|
+
var path22 = require("path");
|
|
8638
8638
|
var asyncSeries = require_series();
|
|
8639
8639
|
var zlib3 = require("zlib");
|
|
8640
8640
|
var { MESSAGE } = require_triple_beam();
|
|
8641
8641
|
var { Stream, PassThrough } = require_readable();
|
|
8642
8642
|
var TransportStream = require_winston_transport();
|
|
8643
8643
|
var debug = require_node2()("winston:file");
|
|
8644
|
-
var
|
|
8644
|
+
var os5 = require("os");
|
|
8645
8645
|
var tailFile = require_tail_file();
|
|
8646
8646
|
module2.exports = class File extends TransportStream {
|
|
8647
8647
|
/**
|
|
@@ -8664,14 +8664,14 @@ var require_file = __commonJS({
|
|
|
8664
8664
|
this._onError = this._onError.bind(this);
|
|
8665
8665
|
if (options2.filename || options2.dirname) {
|
|
8666
8666
|
throwIf("filename or dirname", "stream");
|
|
8667
|
-
this._basename = this.filename = options2.filename ?
|
|
8668
|
-
this.dirname = options2.dirname ||
|
|
8667
|
+
this._basename = this.filename = options2.filename ? path22.basename(options2.filename) : "winston.log";
|
|
8668
|
+
this.dirname = options2.dirname || path22.dirname(options2.filename);
|
|
8669
8669
|
this.options = options2.options || { flags: "a" };
|
|
8670
8670
|
} else if (options2.stream) {
|
|
8671
8671
|
console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
|
|
8672
8672
|
throwIf("stream", "filename", "maxsize");
|
|
8673
8673
|
this._dest = this._stream.pipe(this._setupStream(options2.stream));
|
|
8674
|
-
this.dirname =
|
|
8674
|
+
this.dirname = path22.dirname(this._dest.path);
|
|
8675
8675
|
} else {
|
|
8676
8676
|
throw new Error("Cannot log to file without filename or stream.");
|
|
8677
8677
|
}
|
|
@@ -8679,7 +8679,7 @@ var require_file = __commonJS({
|
|
|
8679
8679
|
this.rotationFormat = options2.rotationFormat || false;
|
|
8680
8680
|
this.zippedArchive = options2.zippedArchive || false;
|
|
8681
8681
|
this.maxFiles = options2.maxFiles || null;
|
|
8682
|
-
this.eol = typeof options2.eol === "string" ? options2.eol :
|
|
8682
|
+
this.eol = typeof options2.eol === "string" ? options2.eol : os5.EOL;
|
|
8683
8683
|
this.tailable = options2.tailable || false;
|
|
8684
8684
|
this.lazy = options2.lazy || false;
|
|
8685
8685
|
this._size = 0;
|
|
@@ -8834,11 +8834,11 @@ var require_file = __commonJS({
|
|
|
8834
8834
|
options2 = {};
|
|
8835
8835
|
}
|
|
8836
8836
|
options2 = normalizeQuery(options2);
|
|
8837
|
-
const file =
|
|
8837
|
+
const file = path22.join(this.dirname, this.filename);
|
|
8838
8838
|
let buff = "";
|
|
8839
8839
|
let results = [];
|
|
8840
8840
|
let row = 0;
|
|
8841
|
-
const stream =
|
|
8841
|
+
const stream = fs31.createReadStream(file, {
|
|
8842
8842
|
encoding: "utf8"
|
|
8843
8843
|
});
|
|
8844
8844
|
stream.on("error", (err2) => {
|
|
@@ -8939,7 +8939,7 @@ var require_file = __commonJS({
|
|
|
8939
8939
|
* TODO: Refactor me.
|
|
8940
8940
|
*/
|
|
8941
8941
|
stream(options2 = {}) {
|
|
8942
|
-
const file =
|
|
8942
|
+
const file = path22.join(this.dirname, this.filename);
|
|
8943
8943
|
const stream = new Stream();
|
|
8944
8944
|
const tail = {
|
|
8945
8945
|
file,
|
|
@@ -8989,8 +8989,8 @@ var require_file = __commonJS({
|
|
|
8989
8989
|
*/
|
|
8990
8990
|
stat(callback) {
|
|
8991
8991
|
const target = this._getFile();
|
|
8992
|
-
const fullpath =
|
|
8993
|
-
|
|
8992
|
+
const fullpath = path22.join(this.dirname, target);
|
|
8993
|
+
fs31.stat(fullpath, (err2, stat) => {
|
|
8994
8994
|
if (err2 && err2.code === "ENOENT") {
|
|
8995
8995
|
debug("ENOENT\xA0ok", fullpath);
|
|
8996
8996
|
this.filename = target;
|
|
@@ -9093,9 +9093,9 @@ var require_file = __commonJS({
|
|
|
9093
9093
|
* @returns {WritableStream} Stream that writes to disk for the active file.
|
|
9094
9094
|
*/
|
|
9095
9095
|
_createStream(source) {
|
|
9096
|
-
const fullpath =
|
|
9096
|
+
const fullpath = path22.join(this.dirname, this.filename);
|
|
9097
9097
|
debug("create stream start", fullpath, this.options);
|
|
9098
|
-
const dest =
|
|
9098
|
+
const dest = fs31.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
|
|
9099
9099
|
debug("file open ok", fullpath);
|
|
9100
9100
|
this.emit("open", fullpath);
|
|
9101
9101
|
source.pipe(dest);
|
|
@@ -9118,16 +9118,16 @@ var require_file = __commonJS({
|
|
|
9118
9118
|
*/
|
|
9119
9119
|
_incFile(callback) {
|
|
9120
9120
|
debug("_incFile", this.filename);
|
|
9121
|
-
const ext =
|
|
9122
|
-
const basename =
|
|
9121
|
+
const ext = path22.extname(this._basename);
|
|
9122
|
+
const basename = path22.basename(this._basename, ext);
|
|
9123
9123
|
const tasks = [];
|
|
9124
9124
|
if (this.zippedArchive) {
|
|
9125
9125
|
tasks.push(
|
|
9126
9126
|
function(cb) {
|
|
9127
9127
|
const num = this._created > 0 && !this.tailable ? this._created : "";
|
|
9128
9128
|
this._compressFile(
|
|
9129
|
-
|
|
9130
|
-
|
|
9129
|
+
path22.join(this.dirname, `${basename}${num}${ext}`),
|
|
9130
|
+
path22.join(this.dirname, `${basename}${num}${ext}.gz`),
|
|
9131
9131
|
cb
|
|
9132
9132
|
);
|
|
9133
9133
|
}.bind(this)
|
|
@@ -9152,8 +9152,8 @@ var require_file = __commonJS({
|
|
|
9152
9152
|
* @private
|
|
9153
9153
|
*/
|
|
9154
9154
|
_getFile() {
|
|
9155
|
-
const ext =
|
|
9156
|
-
const basename =
|
|
9155
|
+
const ext = path22.extname(this._basename);
|
|
9156
|
+
const basename = path22.basename(this._basename, ext);
|
|
9157
9157
|
const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
|
|
9158
9158
|
return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
|
|
9159
9159
|
}
|
|
@@ -9173,8 +9173,8 @@ var require_file = __commonJS({
|
|
|
9173
9173
|
const isOldest = oldest !== 0 ? oldest : "";
|
|
9174
9174
|
const isZipped = this.zippedArchive ? ".gz" : "";
|
|
9175
9175
|
const filePath = `${basename}${isOldest}${ext}${isZipped}`;
|
|
9176
|
-
const target =
|
|
9177
|
-
|
|
9176
|
+
const target = path22.join(this.dirname, filePath);
|
|
9177
|
+
fs31.unlink(target, callback);
|
|
9178
9178
|
}
|
|
9179
9179
|
/**
|
|
9180
9180
|
* Roll files forward based on integer, up to maxFiles. e.g. if base if
|
|
@@ -9196,20 +9196,20 @@ var require_file = __commonJS({
|
|
|
9196
9196
|
for (let x2 = this.maxFiles - 1; x2 > 1; x2--) {
|
|
9197
9197
|
tasks.push(function(i6, cb) {
|
|
9198
9198
|
let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
|
|
9199
|
-
const tmppath =
|
|
9200
|
-
|
|
9199
|
+
const tmppath = path22.join(this.dirname, fileName);
|
|
9200
|
+
fs31.exists(tmppath, (exists) => {
|
|
9201
9201
|
if (!exists) {
|
|
9202
9202
|
return cb(null);
|
|
9203
9203
|
}
|
|
9204
9204
|
fileName = `${basename}${i6}${ext}${isZipped}`;
|
|
9205
|
-
|
|
9205
|
+
fs31.rename(tmppath, path22.join(this.dirname, fileName), cb);
|
|
9206
9206
|
});
|
|
9207
9207
|
}.bind(this, x2));
|
|
9208
9208
|
}
|
|
9209
9209
|
asyncSeries(tasks, () => {
|
|
9210
|
-
|
|
9211
|
-
|
|
9212
|
-
|
|
9210
|
+
fs31.rename(
|
|
9211
|
+
path22.join(this.dirname, `${basename}${ext}${isZipped}`),
|
|
9212
|
+
path22.join(this.dirname, `${basename}1${ext}${isZipped}`),
|
|
9213
9213
|
callback
|
|
9214
9214
|
);
|
|
9215
9215
|
});
|
|
@@ -9223,22 +9223,22 @@ var require_file = __commonJS({
|
|
|
9223
9223
|
* @private
|
|
9224
9224
|
*/
|
|
9225
9225
|
_compressFile(src, dest, callback) {
|
|
9226
|
-
|
|
9226
|
+
fs31.access(src, fs31.F_OK, (err2) => {
|
|
9227
9227
|
if (err2) {
|
|
9228
9228
|
return callback();
|
|
9229
9229
|
}
|
|
9230
9230
|
var gzip = zlib3.createGzip();
|
|
9231
|
-
var inp =
|
|
9232
|
-
var out =
|
|
9231
|
+
var inp = fs31.createReadStream(src);
|
|
9232
|
+
var out = fs31.createWriteStream(dest);
|
|
9233
9233
|
out.on("finish", () => {
|
|
9234
|
-
|
|
9234
|
+
fs31.unlink(src, callback);
|
|
9235
9235
|
});
|
|
9236
9236
|
inp.pipe(gzip).pipe(out);
|
|
9237
9237
|
});
|
|
9238
9238
|
}
|
|
9239
9239
|
_createLogDirIfNotExist(dirPath) {
|
|
9240
|
-
if (!
|
|
9241
|
-
|
|
9240
|
+
if (!fs31.existsSync(dirPath)) {
|
|
9241
|
+
fs31.mkdirSync(dirPath, { recursive: true });
|
|
9242
9242
|
}
|
|
9243
9243
|
}
|
|
9244
9244
|
};
|
|
@@ -9322,9 +9322,9 @@ var require_http = __commonJS({
|
|
|
9322
9322
|
};
|
|
9323
9323
|
const auth = options2.params.auth || null;
|
|
9324
9324
|
delete options2.params.auth;
|
|
9325
|
-
const
|
|
9325
|
+
const path22 = options2.params.path || null;
|
|
9326
9326
|
delete options2.params.path;
|
|
9327
|
-
this._request(options2, auth,
|
|
9327
|
+
this._request(options2, auth, path22, (err2, res, body) => {
|
|
9328
9328
|
if (res && res.statusCode !== 200) {
|
|
9329
9329
|
err2 = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
|
|
9330
9330
|
}
|
|
@@ -9352,12 +9352,12 @@ var require_http = __commonJS({
|
|
|
9352
9352
|
method: "stream",
|
|
9353
9353
|
params: options2
|
|
9354
9354
|
};
|
|
9355
|
-
const
|
|
9355
|
+
const path22 = options2.params.path || null;
|
|
9356
9356
|
delete options2.params.path;
|
|
9357
9357
|
const auth = options2.params.auth || null;
|
|
9358
9358
|
delete options2.params.auth;
|
|
9359
9359
|
let buff = "";
|
|
9360
|
-
const req = this._request(options2, auth,
|
|
9360
|
+
const req = this._request(options2, auth, path22);
|
|
9361
9361
|
stream.destroy = () => req.destroy();
|
|
9362
9362
|
req.on("data", (data) => {
|
|
9363
9363
|
data = (buff + data).split(/\n+/);
|
|
@@ -9383,14 +9383,14 @@ var require_http = __commonJS({
|
|
|
9383
9383
|
* @param {string} path - request path
|
|
9384
9384
|
* @param {function} callback - Continuation to respond to when complete.
|
|
9385
9385
|
*/
|
|
9386
|
-
_request(options2, auth,
|
|
9386
|
+
_request(options2, auth, path22, callback) {
|
|
9387
9387
|
options2 = options2 || {};
|
|
9388
9388
|
auth = auth || this.auth;
|
|
9389
|
-
|
|
9389
|
+
path22 = path22 || this.path || "";
|
|
9390
9390
|
if (this.batch) {
|
|
9391
|
-
this._doBatch(options2, callback, auth,
|
|
9391
|
+
this._doBatch(options2, callback, auth, path22);
|
|
9392
9392
|
} else {
|
|
9393
|
-
this._doRequest(options2, callback, auth,
|
|
9393
|
+
this._doRequest(options2, callback, auth, path22);
|
|
9394
9394
|
}
|
|
9395
9395
|
}
|
|
9396
9396
|
/**
|
|
@@ -9400,18 +9400,18 @@ var require_http = __commonJS({
|
|
|
9400
9400
|
* @param {Object?} auth - authentication options
|
|
9401
9401
|
* @param {string} path - request path
|
|
9402
9402
|
*/
|
|
9403
|
-
_doBatch(options2, callback, auth,
|
|
9403
|
+
_doBatch(options2, callback, auth, path22) {
|
|
9404
9404
|
this.batchOptions.push(options2);
|
|
9405
9405
|
if (this.batchOptions.length === 1) {
|
|
9406
9406
|
const me3 = this;
|
|
9407
9407
|
this.batchCallback = callback;
|
|
9408
9408
|
this.batchTimeoutID = setTimeout(function() {
|
|
9409
9409
|
me3.batchTimeoutID = -1;
|
|
9410
|
-
me3._doBatchRequest(me3.batchCallback, auth,
|
|
9410
|
+
me3._doBatchRequest(me3.batchCallback, auth, path22);
|
|
9411
9411
|
}, this.batchInterval);
|
|
9412
9412
|
}
|
|
9413
9413
|
if (this.batchOptions.length === this.batchCount) {
|
|
9414
|
-
this._doBatchRequest(this.batchCallback, auth,
|
|
9414
|
+
this._doBatchRequest(this.batchCallback, auth, path22);
|
|
9415
9415
|
}
|
|
9416
9416
|
}
|
|
9417
9417
|
/**
|
|
@@ -9420,14 +9420,14 @@ var require_http = __commonJS({
|
|
|
9420
9420
|
* @param {Object?} auth - authentication options
|
|
9421
9421
|
* @param {string} path - request path
|
|
9422
9422
|
*/
|
|
9423
|
-
_doBatchRequest(callback, auth,
|
|
9423
|
+
_doBatchRequest(callback, auth, path22) {
|
|
9424
9424
|
if (this.batchTimeoutID > 0) {
|
|
9425
9425
|
clearTimeout(this.batchTimeoutID);
|
|
9426
9426
|
this.batchTimeoutID = -1;
|
|
9427
9427
|
}
|
|
9428
9428
|
const batchOptionsCopy = this.batchOptions.slice();
|
|
9429
9429
|
this.batchOptions = [];
|
|
9430
|
-
this._doRequest(batchOptionsCopy, callback, auth,
|
|
9430
|
+
this._doRequest(batchOptionsCopy, callback, auth, path22);
|
|
9431
9431
|
}
|
|
9432
9432
|
/**
|
|
9433
9433
|
* Make a request to a winstond server or any http server which can
|
|
@@ -9437,7 +9437,7 @@ var require_http = __commonJS({
|
|
|
9437
9437
|
* @param {Object?} auth - authentication options
|
|
9438
9438
|
* @param {string} path - request path
|
|
9439
9439
|
*/
|
|
9440
|
-
_doRequest(options2, callback, auth,
|
|
9440
|
+
_doRequest(options2, callback, auth, path22) {
|
|
9441
9441
|
const headers = Object.assign({}, this.headers);
|
|
9442
9442
|
if (auth && auth.bearer) {
|
|
9443
9443
|
headers.Authorization = `Bearer ${auth.bearer}`;
|
|
@@ -9447,7 +9447,7 @@ var require_http = __commonJS({
|
|
|
9447
9447
|
method: "POST",
|
|
9448
9448
|
host: this.host,
|
|
9449
9449
|
port: this.port,
|
|
9450
|
-
path: `/${
|
|
9450
|
+
path: `/${path22.replace(/^\//, "")}`,
|
|
9451
9451
|
headers,
|
|
9452
9452
|
auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
|
|
9453
9453
|
agent: this.agent
|
|
@@ -9482,7 +9482,7 @@ var require_stream2 = __commonJS({
|
|
|
9482
9482
|
"use strict";
|
|
9483
9483
|
var isStream = require_is_stream();
|
|
9484
9484
|
var { MESSAGE } = require_triple_beam();
|
|
9485
|
-
var
|
|
9485
|
+
var os5 = require("os");
|
|
9486
9486
|
var TransportStream = require_winston_transport();
|
|
9487
9487
|
module2.exports = class Stream extends TransportStream {
|
|
9488
9488
|
/**
|
|
@@ -9498,7 +9498,7 @@ var require_stream2 = __commonJS({
|
|
|
9498
9498
|
this._stream = options2.stream;
|
|
9499
9499
|
this._stream.setMaxListeners(Infinity);
|
|
9500
9500
|
this.isObjectMode = options2.stream._writableState.objectMode;
|
|
9501
|
-
this.eol = typeof options2.eol === "string" ? options2.eol :
|
|
9501
|
+
this.eol = typeof options2.eol === "string" ? options2.eol : os5.EOL;
|
|
9502
9502
|
}
|
|
9503
9503
|
/**
|
|
9504
9504
|
* Core logging method exposed to Winston.
|
|
@@ -9881,7 +9881,7 @@ var require_exception_stream = __commonJS({
|
|
|
9881
9881
|
var require_exception_handler = __commonJS({
|
|
9882
9882
|
"../../node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
|
|
9883
9883
|
"use strict";
|
|
9884
|
-
var
|
|
9884
|
+
var os5 = require("os");
|
|
9885
9885
|
var asyncForEach = require_forEach();
|
|
9886
9886
|
var debug = require_node2()("winston:exception");
|
|
9887
9887
|
var once = require_one_time();
|
|
@@ -9976,8 +9976,8 @@ var require_exception_handler = __commonJS({
|
|
|
9976
9976
|
*/
|
|
9977
9977
|
getOsInfo() {
|
|
9978
9978
|
return {
|
|
9979
|
-
loadavg:
|
|
9980
|
-
uptime:
|
|
9979
|
+
loadavg: os5.loadavg(),
|
|
9980
|
+
uptime: os5.uptime()
|
|
9981
9981
|
};
|
|
9982
9982
|
}
|
|
9983
9983
|
/**
|
|
@@ -10119,7 +10119,7 @@ var require_rejection_stream = __commonJS({
|
|
|
10119
10119
|
var require_rejection_handler = __commonJS({
|
|
10120
10120
|
"../../node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
|
|
10121
10121
|
"use strict";
|
|
10122
|
-
var
|
|
10122
|
+
var os5 = require("os");
|
|
10123
10123
|
var asyncForEach = require_forEach();
|
|
10124
10124
|
var debug = require_node2()("winston:rejection");
|
|
10125
10125
|
var once = require_one_time();
|
|
@@ -10216,8 +10216,8 @@ var require_rejection_handler = __commonJS({
|
|
|
10216
10216
|
*/
|
|
10217
10217
|
getOsInfo() {
|
|
10218
10218
|
return {
|
|
10219
|
-
loadavg:
|
|
10220
|
-
uptime:
|
|
10219
|
+
loadavg: os5.loadavg(),
|
|
10220
|
+
uptime: os5.uptime()
|
|
10221
10221
|
};
|
|
10222
10222
|
}
|
|
10223
10223
|
/**
|
|
@@ -11809,7 +11809,7 @@ var CONSTANTS, Constants_default;
|
|
|
11809
11809
|
var init_Constants = __esm({
|
|
11810
11810
|
"../../packages/constants/src/Constants.ts"() {
|
|
11811
11811
|
CONSTANTS = {
|
|
11812
|
-
cliVersion: "2.
|
|
11812
|
+
cliVersion: "2.3.0",
|
|
11813
11813
|
backendVersion: 1,
|
|
11814
11814
|
backendPort: 5088,
|
|
11815
11815
|
workerVersion: 2,
|
|
@@ -11961,11 +11961,67 @@ var init_settings = __esm({
|
|
|
11961
11961
|
}
|
|
11962
11962
|
});
|
|
11963
11963
|
|
|
11964
|
+
// ../../packages/constants/src/environmentSettings.ts
|
|
11965
|
+
var EnvironmentSettingsClass, EnvironmentSettings, ENVIRONMENT_VARIABLES;
|
|
11966
|
+
var init_environmentSettings = __esm({
|
|
11967
|
+
"../../packages/constants/src/environmentSettings.ts"() {
|
|
11968
|
+
EnvironmentSettingsClass = class {
|
|
11969
|
+
constructor() {
|
|
11970
|
+
/**
|
|
11971
|
+
* All the variables foundamental for operativity of remora
|
|
11972
|
+
*/
|
|
11973
|
+
this.FOUNDAMENTAL = ["REMORA_LICENCE_KEY"];
|
|
11974
|
+
/**
|
|
11975
|
+
* All the variables that are not foundamental for the operativity of remora
|
|
11976
|
+
*/
|
|
11977
|
+
this.BASIC = ["REMORA_KEY_SECRET", "REMORA_WORKERS_PATH", "ADMIN_JWT_SECRET", "JWT_SECRET", "ROOT_TEMP_PASSWORD", "REMORA_WORKER_HOST", "MONGO_URI", "REMORA_SALT", "OPENAI_API_KEY", "AWS_DEFAULT_REGION", "AWS_ACCOUNT_ID", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN", "NODE_ENV", "REMORA_RUNTIME_CONTEXT", "REMORA_STATE_ENABLED", "REMORA_STATE_INTERVAL_MS", "REMORA_MAX_WORKERS", "REMORA_LOG_FORMAT"];
|
|
11978
|
+
this.addFoundamentalVariables = (variable) => {
|
|
11979
|
+
this.FOUNDAMENTAL.push(variable);
|
|
11980
|
+
};
|
|
11981
|
+
this.addEnvironmentVariables = (variable) => {
|
|
11982
|
+
this.BASIC.push(variable);
|
|
11983
|
+
};
|
|
11984
|
+
}
|
|
11985
|
+
};
|
|
11986
|
+
EnvironmentSettings = new EnvironmentSettingsClass();
|
|
11987
|
+
ENVIRONMENT_VARIABLES = [
|
|
11988
|
+
"REMORA_STATE_ENABLED",
|
|
11989
|
+
"REMORA_STATE_INTERVAL_MS",
|
|
11990
|
+
"REMORA_KEY_SECRET",
|
|
11991
|
+
"REMORA_LICENCE_KEY",
|
|
11992
|
+
"INSTALLATION_ID",
|
|
11993
|
+
"REMORA_WORKER_HOST",
|
|
11994
|
+
"JWT_SECRET",
|
|
11995
|
+
"ADMIN_JWT_SECRET",
|
|
11996
|
+
"MONGO_URI",
|
|
11997
|
+
"ROOT_TEMP_PASSWORD",
|
|
11998
|
+
"REMORA_SALT",
|
|
11999
|
+
"OPENAI_API_KEY",
|
|
12000
|
+
"AWS_DEFAULT_REGION",
|
|
12001
|
+
"AWS_ACCOUNT_ID",
|
|
12002
|
+
"AWS_ACCESS_KEY_ID",
|
|
12003
|
+
"AWS_SESSION_TOKEN",
|
|
12004
|
+
"AWS_SECRET_ACCESS_KEY",
|
|
12005
|
+
"NODE_ENV",
|
|
12006
|
+
"REMORA_WORKERS_PATH",
|
|
12007
|
+
"REMORA_CORS_ORIGINS",
|
|
12008
|
+
"REMORA_DEBUG_MODE",
|
|
12009
|
+
"REMORA_MASK_IN_DEV",
|
|
12010
|
+
"BACKEND_ENDPOINT",
|
|
12011
|
+
"REMORA_CONFIG_URI",
|
|
12012
|
+
"REMORA_RUNTIME_CONTEXT",
|
|
12013
|
+
"REMORA_MAX_WORKERS",
|
|
12014
|
+
"REMORA_LOG_FORMAT"
|
|
12015
|
+
];
|
|
12016
|
+
}
|
|
12017
|
+
});
|
|
12018
|
+
|
|
11964
12019
|
// ../../packages/constants/src/index.ts
|
|
11965
12020
|
var init_src3 = __esm({
|
|
11966
12021
|
"../../packages/constants/src/index.ts"() {
|
|
11967
12022
|
init_Constants();
|
|
11968
12023
|
init_settings();
|
|
12024
|
+
init_environmentSettings();
|
|
11969
12025
|
}
|
|
11970
12026
|
});
|
|
11971
12027
|
|
|
@@ -12148,13 +12204,22 @@ var init_SecretManager = __esm({
|
|
|
12148
12204
|
init_ProcessENVManager();
|
|
12149
12205
|
SecretManagerClass = class {
|
|
12150
12206
|
constructor() {
|
|
12207
|
+
/**
|
|
12208
|
+
* Whether a configured value NAMES an environment variable rather than holding one.
|
|
12209
|
+
*
|
|
12210
|
+
* The one place that decides what a reference is. `replaceSecret` reads it to know what to
|
|
12211
|
+
* resolve, and `SecretRedactor` reads it to know what to leave alone — a `{AWS_ID}` is
|
|
12212
|
+
* configuration and has to survive into a bug report, where which variable a source reads is
|
|
12213
|
+
* exactly the question being asked.
|
|
12214
|
+
*/
|
|
12215
|
+
this.isReference = (value) => !!value && value.length > 2 && value.startsWith("{") && value.endsWith("}");
|
|
12151
12216
|
/**
|
|
12152
12217
|
* If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
|
|
12153
12218
|
* Starts with "{" and ends with "}".
|
|
12154
12219
|
* e.g. {AWS_ID}
|
|
12155
12220
|
*/
|
|
12156
12221
|
this.replaceSecret = (value) => {
|
|
12157
|
-
if (!
|
|
12222
|
+
if (!this.isReference(value))
|
|
12158
12223
|
return value;
|
|
12159
12224
|
const parsedValue = value.slice(1, value.length - 1);
|
|
12160
12225
|
return ProcessENVManager_default.getEnvVariable(parsedValue);
|
|
@@ -12166,6 +12231,130 @@ var init_SecretManager = __esm({
|
|
|
12166
12231
|
}
|
|
12167
12232
|
});
|
|
12168
12233
|
|
|
12234
|
+
// ../../packages/common/src/SecretRedactor.ts
|
|
12235
|
+
var REDACTED2, TEXT_PATTERNS, unquote, SecretRedactorClass, SecretRedactor, SecretRedactor_default;
|
|
12236
|
+
var init_SecretRedactor = __esm({
|
|
12237
|
+
"../../packages/common/src/SecretRedactor.ts"() {
|
|
12238
|
+
init_SecretManager();
|
|
12239
|
+
REDACTED2 = "[redacted]";
|
|
12240
|
+
TEXT_PATTERNS = [
|
|
12241
|
+
// A JWT in a log line is a working credential until it expires
|
|
12242
|
+
{
|
|
12243
|
+
pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g,
|
|
12244
|
+
replace: () => REDACTED2
|
|
12245
|
+
},
|
|
12246
|
+
// An AWS access key id names the account it belongs to, and pairs with a secret in the same log
|
|
12247
|
+
{
|
|
12248
|
+
pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g,
|
|
12249
|
+
replace: () => REDACTED2
|
|
12250
|
+
},
|
|
12251
|
+
// Any URL query string: a presigned URL carries its signature there, which is the credential
|
|
12252
|
+
{
|
|
12253
|
+
pattern: /(https?:\/\/[^\s"'<>]+?)\?[^\s"'<>]*/g,
|
|
12254
|
+
replace: (_match, url) => `${url}?${REDACTED2}`
|
|
12255
|
+
},
|
|
12256
|
+
// `password=...`, `"apiKey": "..."`, `Authorization: Bearer ...` and the rest of the family.
|
|
12257
|
+
// A bare `{ENV_VAR}` is its own alternative, and first: the general value class stops at `}`
|
|
12258
|
+
// so that a `{"password":"x"}` match cannot swallow the closing brace of its own object, which
|
|
12259
|
+
// left an unquoted reference matched as `{PG_PASSWORD` — not a reference, and redacted as if
|
|
12260
|
+
// it held something.
|
|
12261
|
+
{
|
|
12262
|
+
pattern: /("?\b(?:password|secret|secret_key|secretKey|session_token|sessionToken|bearerToken|client_secret|clientSecret|api_key|apiKey|access_key|accessKey|token|authorization)\b"?\s*[:=]\s*)(Bearer\s+)?(\{[A-Za-z_][A-Za-z0-9_]*\}|"[^"]*"|'[^']*'|[^\s,;}&]+)/gi,
|
|
12263
|
+
replace: (match, prefix2, bearer, value) => SecretManager_default.isReference(unquote(value)) ? match : `${prefix2}${bearer ?? ""}${REDACTED2}`
|
|
12264
|
+
}
|
|
12265
|
+
];
|
|
12266
|
+
unquote = (value) => value?.length > 1 && /^["']/.test(value) ? value.slice(1, -1) : value;
|
|
12267
|
+
SecretRedactorClass = class {
|
|
12268
|
+
constructor() {
|
|
12269
|
+
/**
|
|
12270
|
+
* The text with every credential shape above replaced. Pure: same input, same output, and the
|
|
12271
|
+
* caller's string is never touched.
|
|
12272
|
+
*/
|
|
12273
|
+
this.redactText = (text) => {
|
|
12274
|
+
if (!text) return text;
|
|
12275
|
+
return TEXT_PATTERNS.reduce(
|
|
12276
|
+
(current, { pattern, replace }) => current.replace(pattern, replace),
|
|
12277
|
+
text
|
|
12278
|
+
);
|
|
12279
|
+
};
|
|
12280
|
+
}
|
|
12281
|
+
};
|
|
12282
|
+
SecretRedactor = new SecretRedactorClass();
|
|
12283
|
+
SecretRedactor_default = SecretRedactor;
|
|
12284
|
+
}
|
|
12285
|
+
});
|
|
12286
|
+
|
|
12287
|
+
// ../../packages/common/src/ConfigPackager.ts
|
|
12288
|
+
var import_fs3, import_path3, import_adm_zip, EXCLUDED_ENTRIES, EXCLUDED_EXTENSIONS, ENV_FILE_PATTERN, ConfigPackagerClass, ConfigPackager, ConfigPackager_default;
|
|
12289
|
+
var init_ConfigPackager = __esm({
|
|
12290
|
+
"../../packages/common/src/ConfigPackager.ts"() {
|
|
12291
|
+
import_fs3 = __toESM(require("fs"), 1);
|
|
12292
|
+
import_path3 = __toESM(require("path"), 1);
|
|
12293
|
+
import_adm_zip = __toESM(require("adm-zip"), 1);
|
|
12294
|
+
init_src();
|
|
12295
|
+
EXCLUDED_ENTRIES = ["temp", ".temp", "logs", ".git", "node_modules", ".DS_Store"];
|
|
12296
|
+
EXCLUDED_EXTENSIONS = [".log"];
|
|
12297
|
+
ENV_FILE_PATTERN = /(^|\/)\.env(\.|$)/;
|
|
12298
|
+
ConfigPackagerClass = class {
|
|
12299
|
+
constructor() {
|
|
12300
|
+
/**
|
|
12301
|
+
* Build the deployment archive of a remora project directory.
|
|
12302
|
+
*
|
|
12303
|
+
* Entries are stored relative to the project root (`producers/p.json`, never
|
|
12304
|
+
* `remora/producers/p.json`) because every consumer of this archive — the worker's
|
|
12305
|
+
* `uploaddeployment` endpoint and a remote `REMORA_CONFIG_URI` — extracts it straight
|
|
12306
|
+
* into the target's own `remora` directory.
|
|
12307
|
+
*/
|
|
12308
|
+
this.build = (rootDir) => {
|
|
12309
|
+
Affirm_default(rootDir, "Missing required parameter rootDir");
|
|
12310
|
+
Affirm_default(import_fs3.default.existsSync(rootDir), `Missing configuration directory: ${rootDir}`);
|
|
12311
|
+
const zip = new import_adm_zip.default();
|
|
12312
|
+
this._addDirectory(zip, rootDir, "");
|
|
12313
|
+
Affirm_default(zip.getEntries().length > 0, `No configuration files found in ${rootDir}`);
|
|
12314
|
+
const envFile = zip.getEntries().find((entry) => ENV_FILE_PATTERN.test(entry.entryName));
|
|
12315
|
+
Affirm_default(!envFile, `Environment file inside the configuration directory: ${envFile?.entryName}. It holds the values the configuration only names, and this archive is uploaded to a worker and packaged into bug reports \u2014 move it above the project directory.`);
|
|
12316
|
+
return zip;
|
|
12317
|
+
};
|
|
12318
|
+
/** Build the archive of rootDir and write it to outputPath, replacing any file already there. */
|
|
12319
|
+
this.write = (rootDir, outputPath) => {
|
|
12320
|
+
Affirm_default(rootDir, "Missing required parameter rootDir");
|
|
12321
|
+
Affirm_default(outputPath, "Missing required parameter outputPath");
|
|
12322
|
+
const target = import_path3.default.resolve(outputPath);
|
|
12323
|
+
Affirm_default(
|
|
12324
|
+
!import_fs3.default.existsSync(target) || import_fs3.default.statSync(target).isFile(),
|
|
12325
|
+
`Output path is not a file: ${target}`
|
|
12326
|
+
);
|
|
12327
|
+
const parent = import_path3.default.dirname(target);
|
|
12328
|
+
Affirm_default(import_fs3.default.existsSync(parent), `Missing output directory: ${parent}`);
|
|
12329
|
+
const zip = this.build(rootDir);
|
|
12330
|
+
zip.writeZip(target);
|
|
12331
|
+
return {
|
|
12332
|
+
path: target,
|
|
12333
|
+
files: zip.getEntries().length,
|
|
12334
|
+
bytes: import_fs3.default.statSync(target).size
|
|
12335
|
+
};
|
|
12336
|
+
};
|
|
12337
|
+
/** Whether an entry name is one the package never carries. */
|
|
12338
|
+
this._isExcluded = (name) => EXCLUDED_ENTRIES.includes(name) || EXCLUDED_EXTENSIONS.some((ext) => name.endsWith(ext));
|
|
12339
|
+
/** Add every file under directoryPath to the archive, recursively, under zipPath. */
|
|
12340
|
+
this._addDirectory = (zip, directoryPath, zipPath) => {
|
|
12341
|
+
for (const entry of import_fs3.default.readdirSync(directoryPath, { withFileTypes: true })) {
|
|
12342
|
+
if (this._isExcluded(entry.name)) continue;
|
|
12343
|
+
const fullPath = import_path3.default.join(directoryPath, entry.name);
|
|
12344
|
+
const entryPath = zipPath ? `${zipPath}/${entry.name}` : entry.name;
|
|
12345
|
+
if (entry.isDirectory())
|
|
12346
|
+
this._addDirectory(zip, fullPath, entryPath);
|
|
12347
|
+
else
|
|
12348
|
+
zip.addFile(entryPath, import_fs3.default.readFileSync(fullPath));
|
|
12349
|
+
}
|
|
12350
|
+
};
|
|
12351
|
+
}
|
|
12352
|
+
};
|
|
12353
|
+
ConfigPackager = new ConfigPackagerClass();
|
|
12354
|
+
ConfigPackager_default = ConfigPackager;
|
|
12355
|
+
}
|
|
12356
|
+
});
|
|
12357
|
+
|
|
12169
12358
|
// ../../packages/common/src/schema/SchemaErrorReporter.ts
|
|
12170
12359
|
var MAX_REPORTED_ERRORS, MAX_LISTED_VALUES, MAX_QUOTED_LENGTH, VALUE_KEYWORDS, ITEM_NAME_KEYS, SchemaErrorReporterClass, SchemaErrorReporter, SchemaErrorReporter_default;
|
|
12171
12360
|
var init_SchemaErrorReporter = __esm({
|
|
@@ -12293,13 +12482,13 @@ var init_SchemaErrorReporter = __esm({
|
|
|
12293
12482
|
});
|
|
12294
12483
|
|
|
12295
12484
|
// ../../packages/common/src/schema/SchemaValidator.ts
|
|
12296
|
-
var import_ajv, import_ajv_formats,
|
|
12485
|
+
var import_ajv, import_ajv_formats, import_fs4, import_path4, SchemaValidatorClass, SchemaValidator, SchemaValidator_default;
|
|
12297
12486
|
var init_SchemaValidator = __esm({
|
|
12298
12487
|
"../../packages/common/src/schema/SchemaValidator.ts"() {
|
|
12299
12488
|
import_ajv = __toESM(require("ajv"), 1);
|
|
12300
12489
|
import_ajv_formats = __toESM(require("ajv-formats"), 1);
|
|
12301
|
-
|
|
12302
|
-
|
|
12490
|
+
import_fs4 = __toESM(require("fs"), 1);
|
|
12491
|
+
import_path4 = __toESM(require("path"), 1);
|
|
12303
12492
|
init_src();
|
|
12304
12493
|
init_SchemaErrorReporter();
|
|
12305
12494
|
SchemaValidatorClass = class {
|
|
@@ -12339,8 +12528,8 @@ var init_SchemaValidator = __esm({
|
|
|
12339
12528
|
"code-set-schema"
|
|
12340
12529
|
];
|
|
12341
12530
|
internalSchemas.forEach((sn2) => {
|
|
12342
|
-
const sp =
|
|
12343
|
-
const parsed = JSON.parse(
|
|
12531
|
+
const sp = import_path4.default.join(internalSchemaDir, `${sn2}.json`);
|
|
12532
|
+
const parsed = JSON.parse(import_fs4.default.readFileSync(sp, "utf-8"));
|
|
12344
12533
|
this.addSchema(parsed, sn2);
|
|
12345
12534
|
});
|
|
12346
12535
|
return true;
|
|
@@ -12348,11 +12537,11 @@ var init_SchemaValidator = __esm({
|
|
|
12348
12537
|
this._findSchemaDir = () => {
|
|
12349
12538
|
let dir = __dirname;
|
|
12350
12539
|
for (let i6 = 0; i6 < 5; i6++) {
|
|
12351
|
-
const candidate =
|
|
12352
|
-
if (
|
|
12353
|
-
dir =
|
|
12540
|
+
const candidate = import_path4.default.join(dir, "json_schemas");
|
|
12541
|
+
if (import_fs4.default.existsSync(candidate)) return candidate;
|
|
12542
|
+
dir = import_path4.default.dirname(dir);
|
|
12354
12543
|
}
|
|
12355
|
-
return
|
|
12544
|
+
return import_path4.default.join(__dirname, "json_schemas");
|
|
12356
12545
|
};
|
|
12357
12546
|
this.ajv = new import_ajv.default({
|
|
12358
12547
|
allErrors: true,
|
|
@@ -12961,6 +13150,7 @@ var init_Validator = __esm({
|
|
|
12961
13150
|
}
|
|
12962
13151
|
}
|
|
12963
13152
|
errors.push(...this.validateOmitNull(consumer));
|
|
13153
|
+
errors.push(...this.validateHeader(consumer));
|
|
12964
13154
|
errors.push(...this.validateSplit(consumer));
|
|
12965
13155
|
const dimensionFields = consumer.fields.filter((x2) => x2.key !== "*" && !x2.fixed && !x2.copyFrom && !x2.lineCount);
|
|
12966
13156
|
const dimensionKeys = dimensionFields.map((x2) => `${x2.from ?? "_default_"}::${x2.key}`);
|
|
@@ -13075,6 +13265,22 @@ var init_Validator = __esm({
|
|
|
13075
13265
|
}
|
|
13076
13266
|
return errors;
|
|
13077
13267
|
};
|
|
13268
|
+
/**
|
|
13269
|
+
* Checks `header`, which writes the field names as the first line of an exported file. A JSON
|
|
13270
|
+
* record carries its own field names and an API response is not a file, so the flag means
|
|
13271
|
+
* nothing there — rejected rather than ignored, so a config that asks for a header always gets
|
|
13272
|
+
* one.
|
|
13273
|
+
*/
|
|
13274
|
+
this.validateHeader = (consumer) => {
|
|
13275
|
+
const errors = [];
|
|
13276
|
+
const positionalFormats = ["CSV", "TXT", "PARQUET"];
|
|
13277
|
+
for (const output of (consumer.outputs ?? []).filter((x2) => Algo_default.hasVal(x2.header))) {
|
|
13278
|
+
const format4 = output.format.toUpperCase();
|
|
13279
|
+
if (!positionalFormats.includes(format4))
|
|
13280
|
+
errors.push(`The ${format4} output of consumer "${consumer.name}" declares "header", which only applies to CSV/TXT output: a ${format4} record already carries its own field names.`);
|
|
13281
|
+
}
|
|
13282
|
+
return errors;
|
|
13283
|
+
};
|
|
13078
13284
|
/**
|
|
13079
13285
|
* Checks a field whose key selects columns by pattern ("addr*Street"). A pattern that matches
|
|
13080
13286
|
* nothing is always a mistake — a renamed dimension, or a producer that never had those columns —
|
|
@@ -13150,7 +13356,7 @@ var init_Validator = __esm({
|
|
|
13150
13356
|
*/
|
|
13151
13357
|
this.detectOnFinishRecursion = (consumer) => {
|
|
13152
13358
|
const errors = [];
|
|
13153
|
-
const
|
|
13359
|
+
const path22 = [];
|
|
13154
13360
|
const targetsOf = (cons) => {
|
|
13155
13361
|
const names = [];
|
|
13156
13362
|
for (const output of cons.outputs ?? []) {
|
|
@@ -13162,17 +13368,17 @@ var init_Validator = __esm({
|
|
|
13162
13368
|
return names;
|
|
13163
13369
|
};
|
|
13164
13370
|
const walk = (name) => {
|
|
13165
|
-
if (
|
|
13166
|
-
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...
|
|
13371
|
+
if (path22.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
|
|
13372
|
+
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path22, name].join(" -> ")}`);
|
|
13167
13373
|
return true;
|
|
13168
13374
|
}
|
|
13169
13375
|
const cons = Environment_default.getConsumer(name);
|
|
13170
13376
|
if (!cons) return false;
|
|
13171
|
-
|
|
13377
|
+
path22.push(name);
|
|
13172
13378
|
for (const target of targetsOf(cons)) {
|
|
13173
13379
|
if (walk(target)) return true;
|
|
13174
13380
|
}
|
|
13175
|
-
|
|
13381
|
+
path22.pop();
|
|
13176
13382
|
return false;
|
|
13177
13383
|
};
|
|
13178
13384
|
walk(consumer.name);
|
|
@@ -13757,14 +13963,14 @@ ${errors.map((x2) => ` - ${x2}`).join("\n")}`);
|
|
|
13757
13963
|
});
|
|
13758
13964
|
|
|
13759
13965
|
// ../../packages/common/src/Environment.ts
|
|
13760
|
-
var
|
|
13966
|
+
var import_fs5, import_crypto, import_adm_zip2, import_path5, import_client_s3, RESOURCE_SCHEMA_KEYS, INVALID_RESOURCE_MESSAGES, STAGING_DIRECTORY, EnvironmentClass, Environment, Environment_default;
|
|
13761
13967
|
var init_Environment = __esm({
|
|
13762
13968
|
"../../packages/common/src/Environment.ts"() {
|
|
13763
|
-
|
|
13969
|
+
import_fs5 = __toESM(require("fs"), 1);
|
|
13764
13970
|
import_crypto = __toESM(require("crypto"), 1);
|
|
13765
|
-
|
|
13971
|
+
import_adm_zip2 = __toESM(require("adm-zip"), 1);
|
|
13766
13972
|
init_src();
|
|
13767
|
-
|
|
13973
|
+
import_path5 = __toESM(require("path"), 1);
|
|
13768
13974
|
init_SchemaValidator();
|
|
13769
13975
|
init_SchemaErrorReporter();
|
|
13770
13976
|
init_Validator();
|
|
@@ -13886,16 +14092,16 @@ var init_Environment = __esm({
|
|
|
13886
14092
|
this._saveConfigFile = (buffer, url) => {
|
|
13887
14093
|
Affirm_default(buffer?.length, `Empty configuration archive downloaded from ${url}`);
|
|
13888
14094
|
const zip = this._readArchive(buffer, url);
|
|
13889
|
-
const extractPath =
|
|
13890
|
-
const stagingPath =
|
|
13891
|
-
|
|
14095
|
+
const extractPath = import_path5.default.join(this._localPath, "remora");
|
|
14096
|
+
const stagingPath = import_path5.default.join(this._localPath, STAGING_DIRECTORY);
|
|
14097
|
+
import_fs5.default.rmSync(stagingPath, { recursive: true, force: true });
|
|
13892
14098
|
try {
|
|
13893
14099
|
zip.extractAllTo(stagingPath, true);
|
|
13894
14100
|
this._clearConfigDirectory(extractPath);
|
|
13895
|
-
for (const entry of
|
|
13896
|
-
|
|
14101
|
+
for (const entry of import_fs5.default.readdirSync(stagingPath))
|
|
14102
|
+
import_fs5.default.renameSync(import_path5.default.join(stagingPath, entry), import_path5.default.join(extractPath, entry));
|
|
13897
14103
|
} finally {
|
|
13898
|
-
|
|
14104
|
+
import_fs5.default.rmSync(stagingPath, { recursive: true, force: true });
|
|
13899
14105
|
}
|
|
13900
14106
|
Logger_default.log(`Remote configuration downloaded and extracted from ${url}`);
|
|
13901
14107
|
return true;
|
|
@@ -13907,7 +14113,7 @@ var init_Environment = __esm({
|
|
|
13907
14113
|
*/
|
|
13908
14114
|
this._readArchive = (buffer, url) => {
|
|
13909
14115
|
try {
|
|
13910
|
-
const zip = new
|
|
14116
|
+
const zip = new import_adm_zip2.default(buffer);
|
|
13911
14117
|
Affirm_default.hasItems(zip.getEntries(), `Configuration archive from ${url} contains no files`);
|
|
13912
14118
|
return zip;
|
|
13913
14119
|
} catch (error) {
|
|
@@ -13919,17 +14125,17 @@ var init_Environment = __esm({
|
|
|
13919
14125
|
};
|
|
13920
14126
|
/** Remove everything the previous configuration left behind, keeping the run scratch directory. */
|
|
13921
14127
|
this._clearConfigDirectory = (extractPath) => {
|
|
13922
|
-
if (!
|
|
13923
|
-
|
|
14128
|
+
if (!import_fs5.default.existsSync(extractPath)) {
|
|
14129
|
+
import_fs5.default.mkdirSync(extractPath, { recursive: true });
|
|
13924
14130
|
return;
|
|
13925
14131
|
}
|
|
13926
|
-
for (const file of
|
|
14132
|
+
for (const file of import_fs5.default.readdirSync(extractPath)) {
|
|
13927
14133
|
if (file === "temp") continue;
|
|
13928
|
-
const filePath =
|
|
13929
|
-
if (
|
|
13930
|
-
|
|
14134
|
+
const filePath = import_path5.default.join(extractPath, file);
|
|
14135
|
+
if (import_fs5.default.statSync(filePath).isDirectory())
|
|
14136
|
+
import_fs5.default.rmSync(filePath, { recursive: true, force: true });
|
|
13931
14137
|
else
|
|
13932
|
-
|
|
14138
|
+
import_fs5.default.unlinkSync(filePath);
|
|
13933
14139
|
}
|
|
13934
14140
|
};
|
|
13935
14141
|
/**
|
|
@@ -13988,13 +14194,13 @@ var init_Environment = __esm({
|
|
|
13988
14194
|
* project's own JSON schemas means registering that schema first.
|
|
13989
14195
|
*/
|
|
13990
14196
|
this.inspect = (remoraPath) => {
|
|
13991
|
-
const envPath =
|
|
14197
|
+
const envPath = import_path5.default.join(remoraPath, "remora");
|
|
13992
14198
|
const problems = [];
|
|
13993
14199
|
const projectConfig = this._readProjectConfig(envPath, problems);
|
|
13994
14200
|
if (!projectConfig) return this._emptyInspection(problems);
|
|
13995
14201
|
SchemaValidator_default.unloadSchemas();
|
|
13996
14202
|
const read = (entries) => this._readConfigs(envPath, entries, remoraPath, problems);
|
|
13997
|
-
const optional = (folder) =>
|
|
14203
|
+
const optional = (folder) => import_fs5.default.existsSync(import_path5.default.join(envPath, folder)) ? [folder] : [];
|
|
13998
14204
|
const folders = Constants_default.RESOURCE_FOLDERS;
|
|
13999
14205
|
const schemaFiles = read(optional(folders.schemas));
|
|
14000
14206
|
const schemas = schemaFiles.map((x2) => x2.config);
|
|
@@ -14078,14 +14284,14 @@ var init_Environment = __esm({
|
|
|
14078
14284
|
* picture for the preview.
|
|
14079
14285
|
*/
|
|
14080
14286
|
this._readProjectConfig = (envPath, problems) => {
|
|
14081
|
-
const projectPath =
|
|
14082
|
-
if (!
|
|
14287
|
+
const projectPath = import_path5.default.join(envPath, "project.json");
|
|
14288
|
+
if (!import_fs5.default.existsSync(projectPath)) {
|
|
14083
14289
|
problems.push({ severity: "error", scope: "project", file: projectPath, message: "Project configuration file not found" });
|
|
14084
14290
|
return null;
|
|
14085
14291
|
}
|
|
14086
14292
|
let projectConfig;
|
|
14087
14293
|
try {
|
|
14088
|
-
projectConfig = JSON.parse(
|
|
14294
|
+
projectConfig = JSON.parse(import_fs5.default.readFileSync(projectPath, "utf-8"));
|
|
14089
14295
|
} catch (error) {
|
|
14090
14296
|
problems.push({ severity: "error", scope: "project", file: projectPath, message: this._messageOf(error) });
|
|
14091
14297
|
return null;
|
|
@@ -14132,7 +14338,7 @@ var init_Environment = __esm({
|
|
|
14132
14338
|
for (const configPath of entries ?? []) {
|
|
14133
14339
|
for (const file of this._resolveConfigFiles(envPath, configPath, remoraPath, problems)) {
|
|
14134
14340
|
try {
|
|
14135
|
-
configs.push({ config: JSON.parse(
|
|
14341
|
+
configs.push({ config: JSON.parse(import_fs5.default.readFileSync(file, "utf-8")), file });
|
|
14136
14342
|
} catch (error) {
|
|
14137
14343
|
problems.push({
|
|
14138
14344
|
severity: "error",
|
|
@@ -14147,9 +14353,9 @@ var init_Environment = __esm({
|
|
|
14147
14353
|
};
|
|
14148
14354
|
/** The files one project.json entry resolves to: itself, or every .json under it if it is a directory. */
|
|
14149
14355
|
this._resolveConfigFiles = (envPath, configPath, remoraPath, problems) => {
|
|
14150
|
-
const fullPath =
|
|
14356
|
+
const fullPath = import_path5.default.join(envPath, configPath);
|
|
14151
14357
|
try {
|
|
14152
|
-
return
|
|
14358
|
+
return import_fs5.default.statSync(fullPath).isDirectory() ? this._readJsonFilesRecursively(fullPath) : [fullPath];
|
|
14153
14359
|
} catch (error) {
|
|
14154
14360
|
problems.push({
|
|
14155
14361
|
severity: "error",
|
|
@@ -14162,8 +14368,8 @@ var init_Environment = __esm({
|
|
|
14162
14368
|
};
|
|
14163
14369
|
this._readJsonFilesRecursively = (dirPath) => {
|
|
14164
14370
|
const results = [];
|
|
14165
|
-
for (const entry of
|
|
14166
|
-
const fullEntry =
|
|
14371
|
+
for (const entry of import_fs5.default.readdirSync(dirPath, { withFileTypes: true })) {
|
|
14372
|
+
const fullEntry = import_path5.default.join(dirPath, entry.name);
|
|
14167
14373
|
if (entry.isDirectory())
|
|
14168
14374
|
results.push(...this._readJsonFilesRecursively(fullEntry));
|
|
14169
14375
|
else if (entry.name.endsWith(".json"))
|
|
@@ -14268,7 +14474,7 @@ var init_Environment = __esm({
|
|
|
14268
14474
|
resourceFiles: []
|
|
14269
14475
|
});
|
|
14270
14476
|
/** Kept verbatim from the batch loader this replaced, so `load()` still throws the same string. */
|
|
14271
|
-
this._loadFailureMessage = (remoraPath, configPath, error) => `Error loading from ${
|
|
14477
|
+
this._loadFailureMessage = (remoraPath, configPath, error) => `Error loading from ${import_path5.default.resolve(remoraPath)} configuration from ${configPath}: ${this._messageOf(error)}`;
|
|
14272
14478
|
this._messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
14273
14479
|
/**
|
|
14274
14480
|
* The loaded environment, flattened for transport to a worker thread.
|
|
@@ -14410,14 +14616,14 @@ var init_Environment = __esm({
|
|
|
14410
14616
|
this.resolveResourceFilePath = (folder, resourceName) => {
|
|
14411
14617
|
Affirm_default(folder, "Invalid resource folder");
|
|
14412
14618
|
Affirm_default(resourceName, "Invalid resource name");
|
|
14413
|
-
const baseDir =
|
|
14414
|
-
const defaultPath =
|
|
14619
|
+
const baseDir = import_path5.default.join(this._localPath || "", "remora", folder);
|
|
14620
|
+
const defaultPath = import_path5.default.join(baseDir, `${resourceName}.json`);
|
|
14415
14621
|
return this._findConfigFileByName(baseDir, resourceName) || defaultPath;
|
|
14416
14622
|
};
|
|
14417
14623
|
this._findConfigFileByName = (dirPath, resourceName) => {
|
|
14418
|
-
if (!
|
|
14419
|
-
for (const entry of
|
|
14420
|
-
const fullPath =
|
|
14624
|
+
if (!import_fs5.default.existsSync(dirPath)) return null;
|
|
14625
|
+
for (const entry of import_fs5.default.readdirSync(dirPath, { withFileTypes: true })) {
|
|
14626
|
+
const fullPath = import_path5.default.join(dirPath, entry.name);
|
|
14421
14627
|
if (entry.isDirectory()) {
|
|
14422
14628
|
const nested = this._findConfigFileByName(fullPath, resourceName);
|
|
14423
14629
|
if (nested) return nested;
|
|
@@ -14425,7 +14631,7 @@ var init_Environment = __esm({
|
|
|
14425
14631
|
}
|
|
14426
14632
|
if (!entry.name.endsWith(".json")) continue;
|
|
14427
14633
|
try {
|
|
14428
|
-
const parsed = JSON.parse(
|
|
14634
|
+
const parsed = JSON.parse(import_fs5.default.readFileSync(fullPath, "utf-8"));
|
|
14429
14635
|
if (parsed?.name === resourceName) return fullPath;
|
|
14430
14636
|
} catch {
|
|
14431
14637
|
}
|
|
@@ -14490,18 +14696,18 @@ ${ce3.map((x2) => ` -${x2}
|
|
|
14490
14696
|
* Same digest as the one `load()` records, so the two can never disagree about "changed".
|
|
14491
14697
|
*/
|
|
14492
14698
|
this.configHash = (remoraPath) => {
|
|
14493
|
-
const envPath =
|
|
14494
|
-
if (!
|
|
14699
|
+
const envPath = import_path5.default.join(remoraPath, "remora");
|
|
14700
|
+
if (!import_fs5.default.existsSync(envPath)) return null;
|
|
14495
14701
|
const hash = import_crypto.default.createHash("sha256");
|
|
14496
14702
|
const hashDir = (dirPath) => {
|
|
14497
|
-
if (!
|
|
14498
|
-
for (const entry of
|
|
14703
|
+
if (!import_fs5.default.existsSync(dirPath)) return;
|
|
14704
|
+
for (const entry of import_fs5.default.readdirSync(dirPath, { withFileTypes: true })) {
|
|
14499
14705
|
if (entry.name === "temp" || entry.name === ".temp") continue;
|
|
14500
|
-
const fullEntry =
|
|
14706
|
+
const fullEntry = import_path5.default.join(dirPath, entry.name);
|
|
14501
14707
|
if (entry.isDirectory())
|
|
14502
14708
|
hashDir(fullEntry);
|
|
14503
14709
|
else if (entry.name.endsWith(".json"))
|
|
14504
|
-
hash.update(
|
|
14710
|
+
hash.update(import_fs5.default.readFileSync(fullEntry, "utf-8"));
|
|
14505
14711
|
}
|
|
14506
14712
|
};
|
|
14507
14713
|
hashDir(envPath);
|
|
@@ -14520,6 +14726,8 @@ var init_src4 = __esm({
|
|
|
14520
14726
|
init_ProcessENVManager();
|
|
14521
14727
|
init_ProcessShutdownManager();
|
|
14522
14728
|
init_SecretManager();
|
|
14729
|
+
init_SecretRedactor();
|
|
14730
|
+
init_ConfigPackager();
|
|
14523
14731
|
init_ExecutorScope();
|
|
14524
14732
|
init_Environment();
|
|
14525
14733
|
init_ResourcesUtils();
|
|
@@ -15285,14 +15493,14 @@ var init_readFile = __esm({
|
|
|
15285
15493
|
import_promises3 = require("fs/promises");
|
|
15286
15494
|
filePromises = {};
|
|
15287
15495
|
fileIntercept = {};
|
|
15288
|
-
readFile2 = (
|
|
15289
|
-
if (fileIntercept[
|
|
15290
|
-
return fileIntercept[
|
|
15496
|
+
readFile2 = (path22, options2) => {
|
|
15497
|
+
if (fileIntercept[path22] !== void 0) {
|
|
15498
|
+
return fileIntercept[path22];
|
|
15291
15499
|
}
|
|
15292
|
-
if (!filePromises[
|
|
15293
|
-
filePromises[
|
|
15500
|
+
if (!filePromises[path22] || options2?.ignoreCache) {
|
|
15501
|
+
filePromises[path22] = (0, import_promises3.readFile)(path22, "utf8");
|
|
15294
15502
|
}
|
|
15295
|
-
return filePromises[
|
|
15503
|
+
return filePromises[path22];
|
|
15296
15504
|
};
|
|
15297
15505
|
}
|
|
15298
15506
|
});
|
|
@@ -15404,8 +15612,8 @@ var init_externalDataInterceptor = __esm({
|
|
|
15404
15612
|
getFileRecord() {
|
|
15405
15613
|
return fileIntercept;
|
|
15406
15614
|
},
|
|
15407
|
-
interceptFile(
|
|
15408
|
-
fileIntercept[
|
|
15615
|
+
interceptFile(path22, contents) {
|
|
15616
|
+
fileIntercept[path22] = Promise.resolve(contents);
|
|
15409
15617
|
},
|
|
15410
15618
|
getTokenRecord() {
|
|
15411
15619
|
return tokenIntercept;
|
|
@@ -18485,8 +18693,8 @@ var init_createConfigValueProvider = __esm({
|
|
|
18485
18693
|
return endpoint.url.href;
|
|
18486
18694
|
}
|
|
18487
18695
|
if ("hostname" in endpoint) {
|
|
18488
|
-
const { protocol, hostname, port, path:
|
|
18489
|
-
return `${protocol}//${hostname}${port ? ":" + port : ""}${
|
|
18696
|
+
const { protocol, hostname, port, path: path22 } = endpoint;
|
|
18697
|
+
return `${protocol}//${hostname}${port ? ":" + port : ""}${path22}`;
|
|
18490
18698
|
}
|
|
18491
18699
|
}
|
|
18492
18700
|
return endpoint;
|
|
@@ -18861,18 +19069,18 @@ var getAttrPathList;
|
|
|
18861
19069
|
var init_getAttrPathList = __esm({
|
|
18862
19070
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
|
|
18863
19071
|
init_types2();
|
|
18864
|
-
getAttrPathList = (
|
|
18865
|
-
const parts =
|
|
19072
|
+
getAttrPathList = (path22) => {
|
|
19073
|
+
const parts = path22.split(".");
|
|
18866
19074
|
const pathList = [];
|
|
18867
19075
|
for (const part of parts) {
|
|
18868
19076
|
const squareBracketIndex = part.indexOf("[");
|
|
18869
19077
|
if (squareBracketIndex !== -1) {
|
|
18870
19078
|
if (part.indexOf("]") !== part.length - 1) {
|
|
18871
|
-
throw new EndpointError(`Path: '${
|
|
19079
|
+
throw new EndpointError(`Path: '${path22}' does not end with ']'`);
|
|
18872
19080
|
}
|
|
18873
19081
|
const arrayIndex = part.slice(squareBracketIndex + 1, -1);
|
|
18874
19082
|
if (Number.isNaN(parseInt(arrayIndex))) {
|
|
18875
|
-
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${
|
|
19083
|
+
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path22}'`);
|
|
18876
19084
|
}
|
|
18877
19085
|
if (squareBracketIndex !== 0) {
|
|
18878
19086
|
pathList.push(part.slice(0, squareBracketIndex));
|
|
@@ -18893,9 +19101,9 @@ var init_getAttr = __esm({
|
|
|
18893
19101
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttr.js"() {
|
|
18894
19102
|
init_types2();
|
|
18895
19103
|
init_getAttrPathList();
|
|
18896
|
-
getAttr = (value,
|
|
19104
|
+
getAttr = (value, path22) => getAttrPathList(path22).reduce((acc, index) => {
|
|
18897
19105
|
if (typeof acc !== "object") {
|
|
18898
|
-
throw new EndpointError(`Index '${index}' in '${
|
|
19106
|
+
throw new EndpointError(`Index '${index}' in '${path22}' not found in '${JSON.stringify(value)}'`);
|
|
18899
19107
|
} else if (Array.isArray(acc)) {
|
|
18900
19108
|
const i6 = parseInt(index);
|
|
18901
19109
|
return acc[i6 < 0 ? acc.length + i6 : i6];
|
|
@@ -18956,8 +19164,8 @@ var init_parseURL = __esm({
|
|
|
18956
19164
|
return value;
|
|
18957
19165
|
}
|
|
18958
19166
|
if (typeof value === "object" && "hostname" in value) {
|
|
18959
|
-
const { hostname: hostname2, port, protocol: protocol2 = "", path:
|
|
18960
|
-
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${
|
|
19167
|
+
const { hostname: hostname2, port, protocol: protocol2 = "", path: path22 = "", query = {} } = value;
|
|
19168
|
+
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path22}`);
|
|
18961
19169
|
url.search = Object.entries(query).map(([k7, v3]) => `${k7}=${v3}`).join("&");
|
|
18962
19170
|
return url;
|
|
18963
19171
|
}
|
|
@@ -21301,11 +21509,11 @@ var init_HttpBindingProtocol = __esm({
|
|
|
21301
21509
|
const opTraits = translateTraits(operationSchema.traits);
|
|
21302
21510
|
if (opTraits.http) {
|
|
21303
21511
|
request.method = opTraits.http[0];
|
|
21304
|
-
const [
|
|
21512
|
+
const [path22, search] = opTraits.http[1].split("?");
|
|
21305
21513
|
if (request.path == "/") {
|
|
21306
|
-
request.path =
|
|
21514
|
+
request.path = path22;
|
|
21307
21515
|
} else {
|
|
21308
|
-
request.path +=
|
|
21516
|
+
request.path += path22;
|
|
21309
21517
|
}
|
|
21310
21518
|
const traitSearchParams = new URLSearchParams(search ?? "");
|
|
21311
21519
|
for (const [key, value] of traitSearchParams) {
|
|
@@ -25018,12 +25226,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
25018
25226
|
const password = request.password ?? "";
|
|
25019
25227
|
auth = `${username}:${password}`;
|
|
25020
25228
|
}
|
|
25021
|
-
let
|
|
25229
|
+
let path22 = request.path;
|
|
25022
25230
|
if (queryString) {
|
|
25023
|
-
|
|
25231
|
+
path22 += `?${queryString}`;
|
|
25024
25232
|
}
|
|
25025
25233
|
if (request.fragment) {
|
|
25026
|
-
|
|
25234
|
+
path22 += `#${request.fragment}`;
|
|
25027
25235
|
}
|
|
25028
25236
|
let hostname = request.hostname ?? "";
|
|
25029
25237
|
if (hostname[0] === "[" && hostname.endsWith("]")) {
|
|
@@ -25035,7 +25243,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
25035
25243
|
headers: request.headers,
|
|
25036
25244
|
host: hostname,
|
|
25037
25245
|
method: request.method,
|
|
25038
|
-
path:
|
|
25246
|
+
path: path22,
|
|
25039
25247
|
port: request.port,
|
|
25040
25248
|
agent,
|
|
25041
25249
|
auth
|
|
@@ -25944,10 +26152,10 @@ ${longDate}
|
|
|
25944
26152
|
${credentialScope}
|
|
25945
26153
|
${toHex(hashedRequest)}`;
|
|
25946
26154
|
}
|
|
25947
|
-
getCanonicalPath({ path:
|
|
26155
|
+
getCanonicalPath({ path: path22 }) {
|
|
25948
26156
|
if (this.uriEscapePath) {
|
|
25949
26157
|
const normalizedPathSegments = [];
|
|
25950
|
-
for (const pathSegment of
|
|
26158
|
+
for (const pathSegment of path22.split("/")) {
|
|
25951
26159
|
if (pathSegment?.length === 0)
|
|
25952
26160
|
continue;
|
|
25953
26161
|
if (pathSegment === ".")
|
|
@@ -25958,11 +26166,11 @@ ${toHex(hashedRequest)}`;
|
|
|
25958
26166
|
normalizedPathSegments.push(pathSegment);
|
|
25959
26167
|
}
|
|
25960
26168
|
}
|
|
25961
|
-
const normalizedPath = `${
|
|
26169
|
+
const normalizedPath = `${path22?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path22?.endsWith("/") ? "/" : ""}`;
|
|
25962
26170
|
const doubleEncoded = escapeUri(normalizedPath);
|
|
25963
26171
|
return doubleEncoded.replace(/%2F/g, "/");
|
|
25964
26172
|
}
|
|
25965
|
-
return
|
|
26173
|
+
return path22;
|
|
25966
26174
|
}
|
|
25967
26175
|
validateResolvedCredentials(credentials) {
|
|
25968
26176
|
if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
|
|
@@ -33673,7 +33881,7 @@ var require_pend = __commonJS({
|
|
|
33673
33881
|
// ../../node_modules/yauzl/fd-slicer.js
|
|
33674
33882
|
var require_fd_slicer = __commonJS({
|
|
33675
33883
|
"../../node_modules/yauzl/fd-slicer.js"(exports2) {
|
|
33676
|
-
var
|
|
33884
|
+
var fs31 = require("fs");
|
|
33677
33885
|
var util = require("util");
|
|
33678
33886
|
var stream = require("stream");
|
|
33679
33887
|
var Readable10 = stream.Readable;
|
|
@@ -33693,7 +33901,7 @@ var require_fd_slicer = __commonJS({
|
|
|
33693
33901
|
FdSlicer.prototype.read = function(buffer, offset, length, position, callback) {
|
|
33694
33902
|
var self2 = this;
|
|
33695
33903
|
self2.pend.go(function(cb) {
|
|
33696
|
-
|
|
33904
|
+
fs31.read(self2.fd, buffer, offset, length, position, function(err2, bytesRead, buffer2) {
|
|
33697
33905
|
cb();
|
|
33698
33906
|
callback(err2, bytesRead, buffer2);
|
|
33699
33907
|
});
|
|
@@ -33710,7 +33918,7 @@ var require_fd_slicer = __commonJS({
|
|
|
33710
33918
|
self2.refCount -= 1;
|
|
33711
33919
|
if (self2.refCount < 0) throw new Error("invalid unref");
|
|
33712
33920
|
if (self2.refCount > 0) return;
|
|
33713
|
-
|
|
33921
|
+
fs31.close(self2.fd, onCloseDone);
|
|
33714
33922
|
function onCloseDone(err2) {
|
|
33715
33923
|
if (err2) {
|
|
33716
33924
|
self2.emit("error", err2);
|
|
@@ -33742,7 +33950,7 @@ var require_fd_slicer = __commonJS({
|
|
|
33742
33950
|
}
|
|
33743
33951
|
self2.context.pend.go(function(cb) {
|
|
33744
33952
|
var buffer = Buffer.allocUnsafe(toRead);
|
|
33745
|
-
|
|
33953
|
+
fs31.read(self2.context.fd, buffer, 0, toRead, self2.pos, function(err2, bytesRead) {
|
|
33746
33954
|
if (err2) {
|
|
33747
33955
|
self2.destroy(err2);
|
|
33748
33956
|
} else if (bytesRead === 0) {
|
|
@@ -34102,7 +34310,7 @@ var require_crc32 = __commonJS({
|
|
|
34102
34310
|
// ../../node_modules/yauzl/index.js
|
|
34103
34311
|
var require_yauzl = __commonJS({
|
|
34104
34312
|
"../../node_modules/yauzl/index.js"(exports2) {
|
|
34105
|
-
var
|
|
34313
|
+
var fs31 = require("fs");
|
|
34106
34314
|
var zlib3 = require("zlib");
|
|
34107
34315
|
var fd_slicer = require_fd_slicer();
|
|
34108
34316
|
var util = require("util");
|
|
@@ -34127,9 +34335,9 @@ var require_yauzl = __commonJS({
|
|
|
34127
34335
|
exports2.Entry = Entry;
|
|
34128
34336
|
exports2.LocalFileHeader = LocalFileHeader;
|
|
34129
34337
|
exports2.RandomAccessReader = RandomAccessReader;
|
|
34130
|
-
function openPromise(
|
|
34338
|
+
function openPromise(path22, options2) {
|
|
34131
34339
|
return new Promise((resolve, reject) => {
|
|
34132
|
-
open(
|
|
34340
|
+
open(path22, { ...options2, lazyEntries: true }, function(err2, zipfile) {
|
|
34133
34341
|
if (err2) return reject(err2);
|
|
34134
34342
|
resolve(zipfile);
|
|
34135
34343
|
});
|
|
@@ -34159,7 +34367,7 @@ var require_yauzl = __commonJS({
|
|
|
34159
34367
|
});
|
|
34160
34368
|
});
|
|
34161
34369
|
}
|
|
34162
|
-
function open(
|
|
34370
|
+
function open(path22, options2, callback) {
|
|
34163
34371
|
if (typeof options2 === "function") {
|
|
34164
34372
|
callback = options2;
|
|
34165
34373
|
options2 = null;
|
|
@@ -34171,10 +34379,10 @@ var require_yauzl = __commonJS({
|
|
|
34171
34379
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34172
34380
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34173
34381
|
if (callback == null) callback = defaultCallback;
|
|
34174
|
-
|
|
34382
|
+
fs31.open(path22, "r", function(err2, fd) {
|
|
34175
34383
|
if (err2) return callback(err2);
|
|
34176
34384
|
fromFd(fd, options2, function(err3, zipfile) {
|
|
34177
|
-
if (err3)
|
|
34385
|
+
if (err3) fs31.close(fd, defaultCallback);
|
|
34178
34386
|
callback(err3, zipfile);
|
|
34179
34387
|
});
|
|
34180
34388
|
});
|
|
@@ -34191,7 +34399,7 @@ var require_yauzl = __commonJS({
|
|
|
34191
34399
|
if (options2.validateEntrySizes == null) options2.validateEntrySizes = true;
|
|
34192
34400
|
if (options2.strictFileNames == null) options2.strictFileNames = false;
|
|
34193
34401
|
if (callback == null) callback = defaultCallback;
|
|
34194
|
-
|
|
34402
|
+
fs31.fstat(fd, function(err2, stats) {
|
|
34195
34403
|
if (err2) return callback(err2);
|
|
34196
34404
|
var reader = new fd_slicer.FdSlicer(fd);
|
|
34197
34405
|
fromRandomAccessReader(reader, stats.size, options2, callback);
|
|
@@ -34969,17 +35177,17 @@ function Yn(s5, t2) {
|
|
|
34969
35177
|
function Kn(s5, t2) {
|
|
34970
35178
|
s5.head = new ue(t2, void 0, s5.head, s5), s5.tail || (s5.tail = s5.head), s5.length++;
|
|
34971
35179
|
}
|
|
34972
|
-
var import_events,
|
|
35180
|
+
var import_events, import_fs6, import_node_events, import_node_stream6, import_node_string_decoder, import_node_path7, import_node_fs5, import_path6, import_events2, import_assert, import_buffer, Ps, import_zlib, import_node_path8, import_node_path9, import_fs7, import_fs8, import_path7, import_node_path10, import_path8, import_node_fs6, import_node_assert, import_node_crypto5, import_node_fs7, import_node_path11, import_fs9, import_node_fs8, import_node_path12, import_node_fs9, import_promises5, import_node_path13, import_node_path14, import_node_fs10, import_node_path15, zr, Ur, Ds, Wr, Gr, Zr, Q, J, nt, De, qt, Ne, Ns, Ae, As, z2, Mt, g5, Qt, Bt, b5, N, _, bi, Ie, L, S, _i, Oi, Is, Ti, Z, xi, Ce, Jt, Rt, C, jt, Yr, Kr, Vr, $r, Fe, Li, Xr, qr, A, Jr, ht, H, te, m3, Ni, tt, Ai, ki, vi, ie, ke, Ut, Ht, Ii, Pt, at, U, ot, Y, zt, Ci, j5, ee, Fi, ve, gt, Me, bt, _t, Be, et, Wt, jr, Fs, ks, vs, Ms, Bs, tn, se, K2, sn, M, rn, zs, nn, Bi, Tt, Gt, Pi, re, Pe, ze, Ue, He, We, Ge, Ze, Ye, Ke, Us, hn, an, Hs, ln, cn, Ws, Gs, Hi, ne, dn, Ui, oe, Ve, un, F, mn, xt, Wi, pn, lt, En, wn, Sn, ct, yn, Rn, gn, Gi, bn, Lt, ft, On, Tn, xn, Ln, f5, $e, Dt, Nn, Xi, qi, An, B, Nt, it, Zi, Zs, V, he, dt, Ys, p3, st, ut, Yi, At, w, Xe, qe, Ki, Ks, Vs, ae, Vi, Qe, Yt, $, Je, It, je, ti, $s, In, le, $i, Xs, Cn, rt, mt, vn, Qi, Mn, Bn, Ct, Ji, zn, qs, ce, ei, ji, Un, Hn, ts, Qs, rr, Wn, tr, er, ir, is, sr, fe, ii, ss, si, rs, ns, os, hs, pt, ri, as, es, q3, de, ni, oi, Gn, hi, ue, pi, nr, li, me, W, pe, Et, Ft, Ee, ai, G, ls, ci, or, ds, us, fi, di, hr, cs, ui, lr, fs8, wt, kt, Vn, $n, fr, dr, Xn, qn, Qn, Jn, Er, wr, ur, Sr, yr, Rr, jn, to, eo, mr, ms, ps, Ei, io, Es, so, ws, Se, St, no, gr, Ss, br, oo, _r3, ys, Or, Vt, Tr, ho, ao, lo, yi, Lr, Dr, _s5, Nr, Os, P, Ts, xs, gi, Ar, Ir, Re, Cr, Fr, Rs, yt, O, Ri, kr, $t, gs, bs, Ls, ge, be, _e5, Oe, fo, Te, uo, mo, po, vr, Xt, ye, xe, Eo, wo, So, yo, Ro, go, bo, _o, vt, Oo, To;
|
|
34973
35181
|
var init_index_min = __esm({
|
|
34974
35182
|
"../../packages/parsing/node_modules/tar/dist/esm/index.min.js"() {
|
|
34975
35183
|
import_events = __toESM(require("events"), 1);
|
|
34976
|
-
|
|
35184
|
+
import_fs6 = __toESM(require("fs"), 1);
|
|
34977
35185
|
import_node_events = require("events");
|
|
34978
35186
|
import_node_stream6 = __toESM(require("stream"), 1);
|
|
34979
35187
|
import_node_string_decoder = require("string_decoder");
|
|
34980
35188
|
import_node_path7 = __toESM(require("path"), 1);
|
|
34981
35189
|
import_node_fs5 = __toESM(require("fs"), 1);
|
|
34982
|
-
|
|
35190
|
+
import_path6 = require("path");
|
|
34983
35191
|
import_events2 = require("events");
|
|
34984
35192
|
import_assert = __toESM(require("assert"), 1);
|
|
34985
35193
|
import_buffer = require("buffer");
|
|
@@ -34987,17 +35195,17 @@ var init_index_min = __esm({
|
|
|
34987
35195
|
import_zlib = __toESM(require("zlib"), 1);
|
|
34988
35196
|
import_node_path8 = require("path");
|
|
34989
35197
|
import_node_path9 = require("path");
|
|
34990
|
-
import_fs6 = __toESM(require("fs"), 1);
|
|
34991
35198
|
import_fs7 = __toESM(require("fs"), 1);
|
|
34992
|
-
|
|
34993
|
-
import_node_path10 = require("path");
|
|
35199
|
+
import_fs8 = __toESM(require("fs"), 1);
|
|
34994
35200
|
import_path7 = __toESM(require("path"), 1);
|
|
35201
|
+
import_node_path10 = require("path");
|
|
35202
|
+
import_path8 = __toESM(require("path"), 1);
|
|
34995
35203
|
import_node_fs6 = __toESM(require("fs"), 1);
|
|
34996
35204
|
import_node_assert = __toESM(require("assert"), 1);
|
|
34997
35205
|
import_node_crypto5 = require("crypto");
|
|
34998
35206
|
import_node_fs7 = __toESM(require("fs"), 1);
|
|
34999
35207
|
import_node_path11 = __toESM(require("path"), 1);
|
|
35000
|
-
|
|
35208
|
+
import_fs9 = __toESM(require("fs"), 1);
|
|
35001
35209
|
import_node_fs8 = __toESM(require("fs"), 1);
|
|
35002
35210
|
import_node_path12 = __toESM(require("path"), 1);
|
|
35003
35211
|
import_node_fs9 = __toESM(require("fs"), 1);
|
|
@@ -35349,7 +35557,7 @@ var init_index_min = __esm({
|
|
|
35349
35557
|
return Wr;
|
|
35350
35558
|
}
|
|
35351
35559
|
};
|
|
35352
|
-
Jr =
|
|
35560
|
+
Jr = import_fs6.default.writev;
|
|
35353
35561
|
ht = /* @__PURE__ */ Symbol("_autoClose");
|
|
35354
35562
|
H = /* @__PURE__ */ Symbol("_close");
|
|
35355
35563
|
te = /* @__PURE__ */ Symbol("_ended");
|
|
@@ -35404,7 +35612,7 @@ var init_index_min = __esm({
|
|
|
35404
35612
|
throw new TypeError("this is a readable stream");
|
|
35405
35613
|
}
|
|
35406
35614
|
[at]() {
|
|
35407
|
-
|
|
35615
|
+
import_fs6.default.open(this[U], "r", (t2, e7) => this[Ht](t2, e7));
|
|
35408
35616
|
}
|
|
35409
35617
|
[Ht](t2, e7) {
|
|
35410
35618
|
t2 ? this[Ut](t2) : (this[m3] = e7, this.emit("open", e7), this[zt]());
|
|
@@ -35417,7 +35625,7 @@ var init_index_min = __esm({
|
|
|
35417
35625
|
this[j5] = true;
|
|
35418
35626
|
let t2 = this[vi]();
|
|
35419
35627
|
if (t2.length === 0) return process.nextTick(() => this[Ii](null, 0, t2));
|
|
35420
|
-
|
|
35628
|
+
import_fs6.default.read(this[m3], t2, 0, t2.length, null, (e7, i6, r7) => this[Ii](e7, i6, r7));
|
|
35421
35629
|
}
|
|
35422
35630
|
}
|
|
35423
35631
|
[Ii](t2, e7, i6) {
|
|
@@ -35426,7 +35634,7 @@ var init_index_min = __esm({
|
|
|
35426
35634
|
[H]() {
|
|
35427
35635
|
if (this[ht] && typeof this[m3] == "number") {
|
|
35428
35636
|
let t2 = this[m3];
|
|
35429
|
-
this[m3] = void 0,
|
|
35637
|
+
this[m3] = void 0, import_fs6.default.close(t2, (e7) => e7 ? this.emit("error", e7) : this.emit("close"));
|
|
35430
35638
|
}
|
|
35431
35639
|
}
|
|
35432
35640
|
[Ut](t2) {
|
|
@@ -35454,7 +35662,7 @@ var init_index_min = __esm({
|
|
|
35454
35662
|
[at]() {
|
|
35455
35663
|
let t2 = true;
|
|
35456
35664
|
try {
|
|
35457
|
-
this[Ht](null,
|
|
35665
|
+
this[Ht](null, import_fs6.default.openSync(this[U], "r")), t2 = false;
|
|
35458
35666
|
} finally {
|
|
35459
35667
|
t2 && this[H]();
|
|
35460
35668
|
}
|
|
@@ -35465,7 +35673,7 @@ var init_index_min = __esm({
|
|
|
35465
35673
|
if (!this[j5]) {
|
|
35466
35674
|
this[j5] = true;
|
|
35467
35675
|
do {
|
|
35468
|
-
let e7 = this[vi](), i6 = e7.length === 0 ? 0 :
|
|
35676
|
+
let e7 = this[vi](), i6 = e7.length === 0 ? 0 : import_fs6.default.readSync(this[m3], e7, 0, e7.length, null);
|
|
35469
35677
|
if (!this[ki](i6, e7)) break;
|
|
35470
35678
|
} while (true);
|
|
35471
35679
|
this[j5] = false;
|
|
@@ -35478,7 +35686,7 @@ var init_index_min = __esm({
|
|
|
35478
35686
|
[H]() {
|
|
35479
35687
|
if (this[ht] && typeof this[m3] == "number") {
|
|
35480
35688
|
let t2 = this[m3];
|
|
35481
|
-
this[m3] = void 0,
|
|
35689
|
+
this[m3] = void 0, import_fs6.default.closeSync(t2), this.emit("close");
|
|
35482
35690
|
}
|
|
35483
35691
|
}
|
|
35484
35692
|
};
|
|
@@ -35520,7 +35728,7 @@ var init_index_min = __esm({
|
|
|
35520
35728
|
this[H](), this[gt] = true, this.emit("error", t2);
|
|
35521
35729
|
}
|
|
35522
35730
|
[at]() {
|
|
35523
|
-
|
|
35731
|
+
import_fs6.default.open(this[U], this[tt], this[ie], (t2, e7) => this[Ht](t2, e7));
|
|
35524
35732
|
}
|
|
35525
35733
|
[Ht](t2, e7) {
|
|
35526
35734
|
this[Me] && this[tt] === "r+" && t2 && t2.code === "ENOENT" ? (this[tt] = "w", this[at]()) : t2 ? this[Ut](t2) : (this[m3] = e7, this.emit("open", e7), this[gt] || this[Ai]());
|
|
@@ -35532,7 +35740,7 @@ var init_index_min = __esm({
|
|
|
35532
35740
|
return typeof t2 == "string" && (t2 = Buffer.from(t2, e7)), this[te] ? (this.emit("error", new Error("write() after end()")), false) : this[m3] === void 0 || this[gt] || this[Y].length ? (this[Y].push(t2), this[ke] = true, false) : (this[gt] = true, this[ve](t2), true);
|
|
35533
35741
|
}
|
|
35534
35742
|
[ve](t2) {
|
|
35535
|
-
|
|
35743
|
+
import_fs6.default.write(this[m3], t2, 0, t2.length, this[ot], (e7, i6) => this[Pt](e7, i6));
|
|
35536
35744
|
}
|
|
35537
35745
|
[Pt](t2, e7) {
|
|
35538
35746
|
t2 ? this[Ut](t2) : (this[ot] !== void 0 && typeof e7 == "number" && (this[ot] += e7), this[Y].length ? this[Ai]() : (this[gt] = false, this[te] && !this[Ni] ? (this[Ni] = true, this[H](), this.emit("finish")) : this[ke] && (this[ke] = false, this.emit("drain"))));
|
|
@@ -35548,7 +35756,7 @@ var init_index_min = __esm({
|
|
|
35548
35756
|
[H]() {
|
|
35549
35757
|
if (this[ht] && typeof this[m3] == "number") {
|
|
35550
35758
|
let t2 = this[m3];
|
|
35551
|
-
this[m3] = void 0,
|
|
35759
|
+
this[m3] = void 0, import_fs6.default.close(t2, (e7) => e7 ? this.emit("error", e7) : this.emit("close"));
|
|
35552
35760
|
}
|
|
35553
35761
|
}
|
|
35554
35762
|
};
|
|
@@ -35556,24 +35764,24 @@ var init_index_min = __esm({
|
|
|
35556
35764
|
[at]() {
|
|
35557
35765
|
let t2;
|
|
35558
35766
|
if (this[Me] && this[tt] === "r+") try {
|
|
35559
|
-
t2 =
|
|
35767
|
+
t2 = import_fs6.default.openSync(this[U], this[tt], this[ie]);
|
|
35560
35768
|
} catch (e7) {
|
|
35561
35769
|
if (e7?.code === "ENOENT") return this[tt] = "w", this[at]();
|
|
35562
35770
|
throw e7;
|
|
35563
35771
|
}
|
|
35564
|
-
else t2 =
|
|
35772
|
+
else t2 = import_fs6.default.openSync(this[U], this[tt], this[ie]);
|
|
35565
35773
|
this[Ht](null, t2);
|
|
35566
35774
|
}
|
|
35567
35775
|
[H]() {
|
|
35568
35776
|
if (this[ht] && typeof this[m3] == "number") {
|
|
35569
35777
|
let t2 = this[m3];
|
|
35570
|
-
this[m3] = void 0,
|
|
35778
|
+
this[m3] = void 0, import_fs6.default.closeSync(t2), this.emit("close");
|
|
35571
35779
|
}
|
|
35572
35780
|
}
|
|
35573
35781
|
[ve](t2) {
|
|
35574
35782
|
let e7 = true;
|
|
35575
35783
|
try {
|
|
35576
|
-
this[Pt](null,
|
|
35784
|
+
this[Pt](null, import_fs6.default.writeSync(this[m3], t2, 0, t2.length, this[ot])), e7 = false;
|
|
35577
35785
|
} finally {
|
|
35578
35786
|
if (e7) try {
|
|
35579
35787
|
this[H]();
|
|
@@ -36363,11 +36571,11 @@ var init_index_min = __esm({
|
|
|
36363
36571
|
Qi = (s5, t2) => {
|
|
36364
36572
|
let e7 = new Map(t2.map((o4) => [mt(o4), true])), i6 = s5.filter, r7 = 100, n4 = (o4, h6 = "", a6 = 0) => {
|
|
36365
36573
|
if (a6 >= r7) return e7.set(o4, false), false;
|
|
36366
|
-
let l4 = h6 || (0,
|
|
36574
|
+
let l4 = h6 || (0, import_path6.parse)(o4).root || ".", c6;
|
|
36367
36575
|
if (o4 === l4) c6 = false;
|
|
36368
36576
|
else {
|
|
36369
36577
|
let d6 = e7.get(o4);
|
|
36370
|
-
c6 = d6 !== void 0 ? d6 : n4((0,
|
|
36578
|
+
c6 = d6 !== void 0 ? d6 : n4((0, import_path6.dirname)(o4), l4, a6 + 1);
|
|
36371
36579
|
}
|
|
36372
36580
|
return e7.set(o4, c6), c6;
|
|
36373
36581
|
};
|
|
@@ -36487,7 +36695,7 @@ var init_index_min = __esm({
|
|
|
36487
36695
|
let [o4, h6] = ce(this.path);
|
|
36488
36696
|
o4 && typeof h6 == "string" && (this.path = h6, r7 = o4);
|
|
36489
36697
|
}
|
|
36490
|
-
this.win32 = !!i6.win32 || process.platform === "win32", this.win32 && (this.path = Qs(this.path.replaceAll(/\\/g, "/")), t2 = t2.replaceAll(/\\/g, "/")), this.absolute = f5(i6.absolute ||
|
|
36698
|
+
this.win32 = !!i6.win32 || process.platform === "win32", this.win32 && (this.path = Qs(this.path.replaceAll(/\\/g, "/")), t2 = t2.replaceAll(/\\/g, "/")), this.absolute = f5(i6.absolute || import_path7.default.resolve(this.cwd, t2)), this.path === "" && (this.path = "./"), r7 && this.warn("TAR_ENTRY_INFO", `stripping ${r7} from absolute path`, { entry: this, path: r7 + this.path });
|
|
36491
36699
|
let n4 = this.statCache.get(this.absolute);
|
|
36492
36700
|
n4 ? this[si](n4) : this[ss]();
|
|
36493
36701
|
}
|
|
@@ -36498,7 +36706,7 @@ var init_index_min = __esm({
|
|
|
36498
36706
|
return t2 === "error" && (this.#t = true), super.emit(t2, ...e7);
|
|
36499
36707
|
}
|
|
36500
36708
|
[ss]() {
|
|
36501
|
-
|
|
36709
|
+
import_fs8.default.lstat(this.absolute, (t2, e7) => {
|
|
36502
36710
|
if (t2) return this.emit("error", t2);
|
|
36503
36711
|
this[si](e7);
|
|
36504
36712
|
});
|
|
@@ -36536,7 +36744,7 @@ var init_index_min = __esm({
|
|
|
36536
36744
|
this.path.slice(-1) !== "/" && (this.path += "/"), this.stat.size = 0, this[fe](), this.end();
|
|
36537
36745
|
}
|
|
36538
36746
|
[is]() {
|
|
36539
|
-
|
|
36747
|
+
import_fs8.default.readlink(this.absolute, (t2, e7) => {
|
|
36540
36748
|
if (t2) return this.emit("error", t2);
|
|
36541
36749
|
this[ns](e7);
|
|
36542
36750
|
});
|
|
@@ -36546,7 +36754,7 @@ var init_index_min = __esm({
|
|
|
36546
36754
|
}
|
|
36547
36755
|
[sr](t2) {
|
|
36548
36756
|
if (!this.stat) throw new Error("cannot create link entry without stat");
|
|
36549
|
-
this.type = "Link", this.linkpath = f5(
|
|
36757
|
+
this.type = "Link", this.linkpath = f5(import_path7.default.relative(this.cwd, t2)), this.stat.size = 0, this[fe](), this.end();
|
|
36550
36758
|
}
|
|
36551
36759
|
[er]() {
|
|
36552
36760
|
if (!this.stat) throw new Error("cannot create file entry without stat");
|
|
@@ -36559,7 +36767,7 @@ var init_index_min = __esm({
|
|
|
36559
36767
|
this[os]();
|
|
36560
36768
|
}
|
|
36561
36769
|
[os]() {
|
|
36562
|
-
|
|
36770
|
+
import_fs8.default.open(this.absolute, "r", (t2, e7) => {
|
|
36563
36771
|
if (t2) return this.emit("error", t2);
|
|
36564
36772
|
this[hs](e7);
|
|
36565
36773
|
});
|
|
@@ -36574,14 +36782,14 @@ var init_index_min = __esm({
|
|
|
36574
36782
|
[ii]() {
|
|
36575
36783
|
let { fd: t2, buf: e7, offset: i6, length: r7, pos: n4 } = this;
|
|
36576
36784
|
if (t2 === void 0 || e7 === void 0) throw new Error("cannot read file without first opening");
|
|
36577
|
-
|
|
36785
|
+
import_fs8.default.read(t2, e7, i6, r7, n4, (o4, h6) => {
|
|
36578
36786
|
if (o4) return this[pt](() => this.emit("error", o4));
|
|
36579
36787
|
this[rs](h6);
|
|
36580
36788
|
});
|
|
36581
36789
|
}
|
|
36582
36790
|
[pt](t2 = () => {
|
|
36583
36791
|
}) {
|
|
36584
|
-
this.fd !== void 0 &&
|
|
36792
|
+
this.fd !== void 0 && import_fs8.default.close(this.fd, t2);
|
|
36585
36793
|
}
|
|
36586
36794
|
[rs](t2) {
|
|
36587
36795
|
if (t2 <= 0 && this.remain > 0) {
|
|
@@ -36616,20 +36824,20 @@ var init_index_min = __esm({
|
|
|
36616
36824
|
ni = class extends de {
|
|
36617
36825
|
sync = true;
|
|
36618
36826
|
[ss]() {
|
|
36619
|
-
this[si](
|
|
36827
|
+
this[si](import_fs8.default.lstatSync(this.absolute));
|
|
36620
36828
|
}
|
|
36621
36829
|
[is]() {
|
|
36622
|
-
this[ns](
|
|
36830
|
+
this[ns](import_fs8.default.readlinkSync(this.absolute));
|
|
36623
36831
|
}
|
|
36624
36832
|
[os]() {
|
|
36625
|
-
this[hs](
|
|
36833
|
+
this[hs](import_fs8.default.openSync(this.absolute, "r"));
|
|
36626
36834
|
}
|
|
36627
36835
|
[ii]() {
|
|
36628
36836
|
let t2 = true;
|
|
36629
36837
|
try {
|
|
36630
36838
|
let { fd: e7, buf: i6, offset: r7, length: n4, pos: o4 } = this;
|
|
36631
36839
|
if (e7 === void 0 || i6 === void 0) throw new Error("fd and buf must be set in READ method");
|
|
36632
|
-
let h6 =
|
|
36840
|
+
let h6 = import_fs8.default.readSync(e7, i6, r7, n4, o4);
|
|
36633
36841
|
this[rs](h6), t2 = false;
|
|
36634
36842
|
} finally {
|
|
36635
36843
|
if (t2) try {
|
|
@@ -36644,7 +36852,7 @@ var init_index_min = __esm({
|
|
|
36644
36852
|
}
|
|
36645
36853
|
[pt](t2 = () => {
|
|
36646
36854
|
}) {
|
|
36647
|
-
this.fd !== void 0 &&
|
|
36855
|
+
this.fd !== void 0 && import_fs8.default.closeSync(this.fd), t2();
|
|
36648
36856
|
}
|
|
36649
36857
|
};
|
|
36650
36858
|
oi = class extends A {
|
|
@@ -36898,7 +37106,7 @@ var init_index_min = __esm({
|
|
|
36898
37106
|
cs = /* @__PURE__ */ Symbol("entryOpt");
|
|
36899
37107
|
ui = /* @__PURE__ */ Symbol("writeEntryClass");
|
|
36900
37108
|
lr = /* @__PURE__ */ Symbol("write");
|
|
36901
|
-
|
|
37109
|
+
fs8 = /* @__PURE__ */ Symbol("ondrain");
|
|
36902
37110
|
wt = class extends A {
|
|
36903
37111
|
sync = false;
|
|
36904
37112
|
opt;
|
|
@@ -36932,8 +37140,8 @@ var init_index_min = __esm({
|
|
|
36932
37140
|
if ((t2.gzip ? 1 : 0) + (t2.brotli ? 1 : 0) + (t2.zstd ? 1 : 0) > 1) throw new TypeError("gzip, brotli, zstd are mutually exclusive");
|
|
36933
37141
|
if (t2.gzip && (typeof t2.gzip != "object" && (t2.gzip = {}), this.portable && (t2.gzip.portable = true), this.zip = new ze(t2.gzip)), t2.brotli && (typeof t2.brotli != "object" && (t2.brotli = {}), this.zip = new We(t2.brotli)), t2.zstd && (typeof t2.zstd != "object" && (t2.zstd = {}), this.zip = new Ye(t2.zstd)), !this.zip) throw new Error("impossible");
|
|
36934
37142
|
let e7 = this.zip;
|
|
36935
|
-
e7.on("data", (i6) => super.write(i6)), e7.on("end", () => super.end()), e7.on("drain", () => this[
|
|
36936
|
-
} else this.on("drain", this[
|
|
37143
|
+
e7.on("data", (i6) => super.write(i6)), e7.on("end", () => super.end()), e7.on("drain", () => this[fs8]()), this.on("resume", () => e7.resume());
|
|
37144
|
+
} else this.on("drain", this[fs8]);
|
|
36937
37145
|
this.noDirRecurse = !!t2.noDirRecurse, this.follow = !!t2.follow, this.noMtime = !!t2.noMtime, t2.mtime && (this.mtime = t2.mtime), this.filter = typeof t2.filter == "function" ? t2.filter : () => true, this[W] = new hi(), this[G] = 0, this.jobs = Number(t2.jobs) || 4, this[Ee] = false, this[me] = false;
|
|
36938
37146
|
}
|
|
36939
37147
|
[lr](t2) {
|
|
@@ -36950,7 +37158,7 @@ var init_index_min = __esm({
|
|
|
36950
37158
|
return typeof t2 == "string" ? this[ci](t2) : this[or](t2), this.flowing;
|
|
36951
37159
|
}
|
|
36952
37160
|
[or](t2) {
|
|
36953
|
-
let e7 = f5(
|
|
37161
|
+
let e7 = f5(import_path8.default.resolve(this.cwd, t2.path));
|
|
36954
37162
|
if (!this.filter(t2.path, t2)) t2.resume();
|
|
36955
37163
|
else {
|
|
36956
37164
|
let i6 = new pi(t2.path, e7);
|
|
@@ -36959,13 +37167,13 @@ var init_index_min = __esm({
|
|
|
36959
37167
|
this[Ft]();
|
|
36960
37168
|
}
|
|
36961
37169
|
[ci](t2) {
|
|
36962
|
-
let e7 = f5(
|
|
37170
|
+
let e7 = f5(import_path8.default.resolve(this.cwd, t2));
|
|
36963
37171
|
this[W].push(new pi(t2, e7)), this[Ft]();
|
|
36964
37172
|
}
|
|
36965
37173
|
[ds](t2) {
|
|
36966
37174
|
t2.pending = true, this[G] += 1;
|
|
36967
37175
|
let e7 = this.follow ? "stat" : "lstat";
|
|
36968
|
-
|
|
37176
|
+
import_fs7.default[e7](t2.absolute, (i6, r7) => {
|
|
36969
37177
|
t2.pending = false, this[G] -= 1, i6 ? this.emit("error", i6) : this[li](t2, r7);
|
|
36970
37178
|
});
|
|
36971
37179
|
}
|
|
@@ -36979,7 +37187,7 @@ var init_index_min = __esm({
|
|
|
36979
37187
|
this[Ft]();
|
|
36980
37188
|
}
|
|
36981
37189
|
[us](t2) {
|
|
36982
|
-
t2.pending = true, this[G] += 1,
|
|
37190
|
+
t2.pending = true, this[G] += 1, import_fs7.default.readdir(t2.absolute, (e7, i6) => {
|
|
36983
37191
|
if (t2.pending = false, this[G] -= 1, e7) return this.emit("error", e7);
|
|
36984
37192
|
this[fi](t2, i6);
|
|
36985
37193
|
});
|
|
@@ -37046,7 +37254,7 @@ var init_index_min = __esm({
|
|
|
37046
37254
|
this.emit("error", e7);
|
|
37047
37255
|
}
|
|
37048
37256
|
}
|
|
37049
|
-
[
|
|
37257
|
+
[fs8]() {
|
|
37050
37258
|
this[Et] && this[Et].entry && this[Et].entry.resume();
|
|
37051
37259
|
}
|
|
37052
37260
|
[di](t2) {
|
|
@@ -37080,10 +37288,10 @@ var init_index_min = __esm({
|
|
|
37080
37288
|
}
|
|
37081
37289
|
[ds](t2) {
|
|
37082
37290
|
let e7 = this.follow ? "statSync" : "lstatSync";
|
|
37083
|
-
this[li](t2,
|
|
37291
|
+
this[li](t2, import_fs7.default[e7](t2.absolute));
|
|
37084
37292
|
}
|
|
37085
37293
|
[us](t2) {
|
|
37086
|
-
this[fi](t2,
|
|
37294
|
+
this[fi](t2, import_fs7.default.readdirSync(t2.absolute));
|
|
37087
37295
|
}
|
|
37088
37296
|
[di](t2) {
|
|
37089
37297
|
let e7 = t2.entry, i6 = this.zip;
|
|
@@ -37134,8 +37342,8 @@ var init_index_min = __esm({
|
|
|
37134
37342
|
});
|
|
37135
37343
|
Jn = process.env.__FAKE_PLATFORM__ || process.platform;
|
|
37136
37344
|
Er = Jn === "win32";
|
|
37137
|
-
({ O_CREAT: wr, O_NOFOLLOW: ur, O_TRUNC: Sr, O_WRONLY: yr } =
|
|
37138
|
-
Rr = Number(process.env.__FAKE_FS_O_FILENAME__) ||
|
|
37345
|
+
({ O_CREAT: wr, O_NOFOLLOW: ur, O_TRUNC: Sr, O_WRONLY: yr } = import_fs9.default.constants);
|
|
37346
|
+
Rr = Number(process.env.__FAKE_FS_O_FILENAME__) || import_fs9.default.constants.UV_FS_O_FILEMAP || 0;
|
|
37139
37347
|
jn = Er && !!Rr;
|
|
37140
37348
|
to = 512 * 1024;
|
|
37141
37349
|
eo = Rr | Sr | wr | yr;
|
|
@@ -37933,12 +38141,12 @@ var init_index_min = __esm({
|
|
|
37933
38141
|
});
|
|
37934
38142
|
|
|
37935
38143
|
// ../../packages/parsing/src/ParseCompression.ts
|
|
37936
|
-
var
|
|
38144
|
+
var fs9, import_zlib2, import_path9, import_yauzl, import_promises6, SUPPORTED_FORMATS, FORMAT_SIGNATURES, TRUNCATED_INPUT_CODE, ParseCompressionClass, ParseCompression, ParseCompression_default;
|
|
37937
38145
|
var init_ParseCompression = __esm({
|
|
37938
38146
|
"../../packages/parsing/src/ParseCompression.ts"() {
|
|
37939
|
-
|
|
38147
|
+
fs9 = __toESM(require("fs"), 1);
|
|
37940
38148
|
import_zlib2 = __toESM(require("zlib"), 1);
|
|
37941
|
-
|
|
38149
|
+
import_path9 = __toESM(require("path"), 1);
|
|
37942
38150
|
import_yauzl = __toESM(require_yauzl(), 1);
|
|
37943
38151
|
init_index_min();
|
|
37944
38152
|
import_promises6 = require("stream/promises");
|
|
@@ -37968,7 +38176,7 @@ var init_ParseCompression = __esm({
|
|
|
37968
38176
|
Affirm_default(fileKey, "Missing required parameter fileKey");
|
|
37969
38177
|
Affirm_default(standardPath, "Missing required parameter standardPath");
|
|
37970
38178
|
Affirm_default(executionPath, "Missing required parameter executionPath");
|
|
37971
|
-
const archivePath =
|
|
38179
|
+
const archivePath = import_path9.default.join(standardPath, fileKey);
|
|
37972
38180
|
const format4 = compressionType?.toUpperCase();
|
|
37973
38181
|
if (!format4) {
|
|
37974
38182
|
if ([".zip", ".tar", ".gz"].some((extension) => fileKey.includes(extension)))
|
|
@@ -37986,7 +38194,7 @@ var init_ParseCompression = __esm({
|
|
|
37986
38194
|
return await this._extractTar(archivePath, this._prepareTargetDir(executionPath));
|
|
37987
38195
|
// GZ, the only format the affirm above leaves
|
|
37988
38196
|
default: {
|
|
37989
|
-
|
|
38197
|
+
fs9.mkdirSync(import_path9.default.dirname(executionPath), { recursive: true });
|
|
37990
38198
|
await this._gunzipFile(archivePath, executionPath);
|
|
37991
38199
|
return [executionPath];
|
|
37992
38200
|
}
|
|
@@ -38005,8 +38213,8 @@ var init_ParseCompression = __esm({
|
|
|
38005
38213
|
* since a file being written into one exists at zero bytes before it holds anything.
|
|
38006
38214
|
*/
|
|
38007
38215
|
this._affirmArchive = (archivePath, format4) => {
|
|
38008
|
-
Affirm_default(
|
|
38009
|
-
const { size } =
|
|
38216
|
+
Affirm_default(fs9.existsSync(archivePath), `Source file does not exist: ${archivePath}`);
|
|
38217
|
+
const { size } = fs9.statSync(archivePath);
|
|
38010
38218
|
Affirm_default(size > 0, `Source file "${archivePath}" is empty (0 bytes), so there is no ${format4} archive in it to read. A file still being written into the source folder looks exactly like this \u2014 have the writer stage it under a temporary name and rename it into place, which is atomic, so a run never sees a partial file`);
|
|
38011
38219
|
const signature = FORMAT_SIGNATURES[format4];
|
|
38012
38220
|
if (!signature) return;
|
|
@@ -38015,11 +38223,11 @@ var init_ParseCompression = __esm({
|
|
|
38015
38223
|
/** Whether a file's first bytes are exactly `expected`. Reads only those bytes, never the file. */
|
|
38016
38224
|
this._startsWith = (filePath, expected) => {
|
|
38017
38225
|
const head = Buffer.alloc(expected.length);
|
|
38018
|
-
const handle =
|
|
38226
|
+
const handle = fs9.openSync(filePath, "r");
|
|
38019
38227
|
try {
|
|
38020
|
-
|
|
38228
|
+
fs9.readSync(handle, head, 0, head.length, 0);
|
|
38021
38229
|
} finally {
|
|
38022
|
-
|
|
38230
|
+
fs9.closeSync(handle);
|
|
38023
38231
|
}
|
|
38024
38232
|
return head.equals(Buffer.from(expected));
|
|
38025
38233
|
};
|
|
@@ -38049,9 +38257,9 @@ var init_ParseCompression = __esm({
|
|
|
38049
38257
|
// same dataset path as an empty file (which is what a single-stream GZ wants). Clear that
|
|
38050
38258
|
// placeholder — never a directory, so nothing extracted can be lost — and claim the name.
|
|
38051
38259
|
this._prepareTargetDir = (targetDir) => {
|
|
38052
|
-
if (
|
|
38053
|
-
|
|
38054
|
-
|
|
38260
|
+
if (fs9.existsSync(targetDir) && fs9.statSync(targetDir).isFile())
|
|
38261
|
+
fs9.unlinkSync(targetDir);
|
|
38262
|
+
fs9.mkdirSync(targetDir, { recursive: true });
|
|
38055
38263
|
return targetDir;
|
|
38056
38264
|
};
|
|
38057
38265
|
// Extract a ZIP into targetDir one entry at a time, each entry piped from the archive straight to
|
|
@@ -38066,9 +38274,9 @@ var init_ParseCompression = __esm({
|
|
|
38066
38274
|
if (entry.fileName.endsWith("/")) continue;
|
|
38067
38275
|
Affirm_default(!entry.isEncrypted(), `Encrypted zip entry cannot be read: ${entry.fileName} in ${archivePath}`);
|
|
38068
38276
|
const entryPath = this._resolveEntryPath(targetDir, entry.fileName);
|
|
38069
|
-
|
|
38277
|
+
fs9.mkdirSync(import_path9.default.dirname(entryPath), { recursive: true });
|
|
38070
38278
|
const entryStream = await zip.openReadStreamPromise(entry);
|
|
38071
|
-
await (0, import_promises6.pipeline)(entryStream,
|
|
38279
|
+
await (0, import_promises6.pipeline)(entryStream, fs9.createWriteStream(entryPath));
|
|
38072
38280
|
extractedPaths.push(entryPath);
|
|
38073
38281
|
}
|
|
38074
38282
|
Affirm_default.hasItems(extractedPaths, `Archive ${archivePath} holds no file to read`);
|
|
@@ -38084,7 +38292,7 @@ var init_ParseCompression = __esm({
|
|
|
38084
38292
|
cwd: targetDir,
|
|
38085
38293
|
onReadEntry: (entry) => {
|
|
38086
38294
|
if (!["File", "OldFile", "ContiguousFile"].includes(entry.type)) return;
|
|
38087
|
-
extractedPaths.push(
|
|
38295
|
+
extractedPaths.push(import_path9.default.resolve(targetDir, entry.path));
|
|
38088
38296
|
}
|
|
38089
38297
|
});
|
|
38090
38298
|
const maxEntries = Constants_default.defaults.MAX_ARCHIVE_ENTRIES;
|
|
@@ -38093,17 +38301,17 @@ var init_ParseCompression = __esm({
|
|
|
38093
38301
|
return extractedPaths;
|
|
38094
38302
|
};
|
|
38095
38303
|
this._gunzipFile = async (archivePath, targetPath) => {
|
|
38096
|
-
const fileContents =
|
|
38304
|
+
const fileContents = fs9.createReadStream(archivePath);
|
|
38097
38305
|
const unzip = import_zlib2.default.createGunzip();
|
|
38098
|
-
const writeStream =
|
|
38306
|
+
const writeStream = fs9.createWriteStream(targetPath);
|
|
38099
38307
|
await (0, import_promises6.pipeline)(fileContents, unzip, writeStream);
|
|
38100
38308
|
};
|
|
38101
38309
|
// Where an archive entry lands, refusing any name that escapes targetDir ('../', an absolute
|
|
38102
38310
|
// path, a symlink-shaped name). Resolving first and comparing after is what makes '..' visible.
|
|
38103
38311
|
this._resolveEntryPath = (targetDir, entryName) => {
|
|
38104
|
-
const resolvedTarget =
|
|
38105
|
-
const entryPath =
|
|
38106
|
-
Affirm_default(entryPath.startsWith(resolvedTarget +
|
|
38312
|
+
const resolvedTarget = import_path9.default.resolve(targetDir);
|
|
38313
|
+
const entryPath = import_path9.default.resolve(resolvedTarget, entryName);
|
|
38314
|
+
Affirm_default(entryPath.startsWith(resolvedTarget + import_path9.default.sep), `Blocked path traversal in archive entry: ${entryName}`);
|
|
38107
38315
|
return entryPath;
|
|
38108
38316
|
};
|
|
38109
38317
|
}
|
|
@@ -38400,7 +38608,7 @@ var init_Settings = __esm({
|
|
|
38400
38608
|
});
|
|
38401
38609
|
|
|
38402
38610
|
// ../../packages/helper/src/Runtime.ts
|
|
38403
|
-
var import_node_v8, RuntimeClass, Runtime;
|
|
38611
|
+
var import_node_v8, RuntimeClass, Runtime, Runtime_default;
|
|
38404
38612
|
var init_Runtime = __esm({
|
|
38405
38613
|
"../../packages/helper/src/Runtime.ts"() {
|
|
38406
38614
|
import_node_v8 = __toESM(require("v8"), 1);
|
|
@@ -38420,6 +38628,7 @@ var init_Runtime = __esm({
|
|
|
38420
38628
|
}
|
|
38421
38629
|
};
|
|
38422
38630
|
Runtime = new RuntimeClass();
|
|
38631
|
+
Runtime_default = Runtime;
|
|
38423
38632
|
}
|
|
38424
38633
|
});
|
|
38425
38634
|
|
|
@@ -38469,8 +38678,8 @@ var init_HttpClient = __esm({
|
|
|
38469
38678
|
throw new Error(err2.message ?? "An error occurred on GET.");
|
|
38470
38679
|
}
|
|
38471
38680
|
};
|
|
38472
|
-
this.getWorkerUrl = (
|
|
38473
|
-
const cleanPath =
|
|
38681
|
+
this.getWorkerUrl = (path22) => {
|
|
38682
|
+
const cleanPath = path22.startsWith("/") ? path22.substring(1) : path22;
|
|
38474
38683
|
return `${this.workerHost}/${cleanPath}`;
|
|
38475
38684
|
};
|
|
38476
38685
|
this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
|
|
@@ -43978,10 +44187,10 @@ var init_src7 = __esm({
|
|
|
43978
44187
|
});
|
|
43979
44188
|
|
|
43980
44189
|
// ../../packages/drivers/src/DeltaShareDriver.ts
|
|
43981
|
-
var
|
|
44190
|
+
var fs10, import_promises7, import_stream, import_promises8, MAX_LIMIT_HINT, PARQUET_BATCH_ROWS, MAX_METADATA_LOG_CHARS, DeltaShareSourceDriver, DeltaShareDriver_default;
|
|
43982
44191
|
var init_DeltaShareDriver = __esm({
|
|
43983
44192
|
"../../packages/drivers/src/DeltaShareDriver.ts"() {
|
|
43984
|
-
|
|
44193
|
+
fs10 = __toESM(require("fs"), 1);
|
|
43985
44194
|
import_promises7 = __toESM(require("fs/promises"), 1);
|
|
43986
44195
|
import_stream = require("stream");
|
|
43987
44196
|
import_promises8 = require("stream/promises");
|
|
@@ -44336,7 +44545,7 @@ var init_DeltaShareDriver = __esm({
|
|
|
44336
44545
|
if (!res.ok)
|
|
44337
44546
|
throw new Error(`Error downloading parquet from the delta share: ${res.status} ${res.statusText} (${this._redactUrl(deltaFile.file.url)})`);
|
|
44338
44547
|
Affirm_default(res.body, `Empty response body downloading parquet from ${this._redactUrl(deltaFile.file.url)}`);
|
|
44339
|
-
await (0, import_promises8.pipeline)(import_stream.Readable.fromWeb(res.body),
|
|
44548
|
+
await (0, import_promises8.pipeline)(import_stream.Readable.fromWeb(res.body), fs10.createWriteStream(localPath));
|
|
44340
44549
|
const { size } = await import_promises7.default.stat(localPath);
|
|
44341
44550
|
Logger_default.log(`Delta Share staged ${this._redactUrl(deltaFile.file.url)} (${size} bytes) in ${Date.now() - startedMs}ms`);
|
|
44342
44551
|
};
|
|
@@ -44361,7 +44570,7 @@ var init_DeltaShareDriver = __esm({
|
|
|
44361
44570
|
* disagree on their columns still produces a rectangular dataset.
|
|
44362
44571
|
*/
|
|
44363
44572
|
this._writeDatasetFile = async (localPath, batches, delimiter) => {
|
|
44364
|
-
const writer = new BatchWriter_default(
|
|
44573
|
+
const writer = new BatchWriter_default(fs10.createWriteStream(localPath));
|
|
44365
44574
|
let header = null;
|
|
44366
44575
|
let records = 0;
|
|
44367
44576
|
try {
|
|
@@ -44468,7 +44677,7 @@ var init_DeltaShareDriver = __esm({
|
|
|
44468
44677
|
ExecutorScope_default.ensurePath(localPath);
|
|
44469
44678
|
const written = await this._stageDeltaFile(deltaFile, localPath, delimiter);
|
|
44470
44679
|
if (written > 0) {
|
|
44471
|
-
files.push({ fullUri: localPath, fileSize:
|
|
44680
|
+
files.push({ fullUri: localPath, fileSize: fs10.statSync(localPath).size });
|
|
44472
44681
|
records += written;
|
|
44473
44682
|
} else {
|
|
44474
44683
|
skipped++;
|
|
@@ -44541,10 +44750,10 @@ var init_DeltaShareDriver = __esm({
|
|
|
44541
44750
|
});
|
|
44542
44751
|
|
|
44543
44752
|
// ../../packages/drivers/src/sql/SqlDatasetWriter.ts
|
|
44544
|
-
var
|
|
44753
|
+
var import_fs10, SqlDatasetWriterClass, SqlDatasetWriter, SqlDatasetWriter_default;
|
|
44545
44754
|
var init_SqlDatasetWriter = __esm({
|
|
44546
44755
|
"../../packages/drivers/src/sql/SqlDatasetWriter.ts"() {
|
|
44547
|
-
|
|
44756
|
+
import_fs10 = __toESM(require("fs"), 1);
|
|
44548
44757
|
init_src();
|
|
44549
44758
|
init_src5();
|
|
44550
44759
|
SqlDatasetWriterClass = class {
|
|
@@ -44560,7 +44769,7 @@ var init_SqlDatasetWriter = __esm({
|
|
|
44560
44769
|
Affirm_default(localPath, "Invalid dataset path");
|
|
44561
44770
|
Affirm_default.hasItems(columns, "Invalid dataset columns");
|
|
44562
44771
|
Affirm_default(batches, "Invalid dataset batches");
|
|
44563
|
-
const writer = new BatchWriter_default(
|
|
44772
|
+
const writer = new BatchWriter_default(import_fs10.default.createWriteStream(localPath));
|
|
44564
44773
|
let rows = 0;
|
|
44565
44774
|
try {
|
|
44566
44775
|
if (!writer.write(CSVParser_default.stringifyRow(columns, delimiter)))
|
|
@@ -45007,10 +45216,10 @@ var init_SqlPushdown = __esm({
|
|
|
45007
45216
|
});
|
|
45008
45217
|
|
|
45009
45218
|
// ../../packages/drivers/src/RedshiftDriver.ts
|
|
45010
|
-
var
|
|
45219
|
+
var import_fs11, import_client_redshift_data, RedshiftDriver, RedshiftDriver_default;
|
|
45011
45220
|
var init_RedshiftDriver = __esm({
|
|
45012
45221
|
"../../packages/drivers/src/RedshiftDriver.ts"() {
|
|
45013
|
-
|
|
45222
|
+
import_fs11 = __toESM(require("fs"), 1);
|
|
45014
45223
|
import_client_redshift_data = require("@aws-sdk/client-redshift-data");
|
|
45015
45224
|
init_src();
|
|
45016
45225
|
init_src();
|
|
@@ -45086,7 +45295,7 @@ var init_RedshiftDriver = __esm({
|
|
|
45086
45295
|
const statement = await this.waitForStatement(queryId);
|
|
45087
45296
|
const rows = await SqlDatasetWriter_default.write(localPath, columns, this.streamResultPages(queryId), producer.settings.delimiter ?? ",");
|
|
45088
45297
|
onProgress?.(1);
|
|
45089
|
-
const fileSize =
|
|
45298
|
+
const fileSize = import_fs11.default.statSync(localPath).size;
|
|
45090
45299
|
const reported = statement.ResultRows >= 0 ? statement.ResultRows : null;
|
|
45091
45300
|
Affirm_default(reported === null || reported === rows, `Redshift ready: staged ${rows} row(s) for producer "${producer.name}" but the statement reported ${reported}`);
|
|
45092
45301
|
Logger_default.log(`Redshift ready: staged ${rows} row(s) (${fileSize} bytes) for producer "${producer.name}" from table "${table}"`);
|
|
@@ -45294,10 +45503,10 @@ var init_RedshiftDriver = __esm({
|
|
|
45294
45503
|
});
|
|
45295
45504
|
|
|
45296
45505
|
// ../../packages/drivers/src/sql/PostgresDriver.ts
|
|
45297
|
-
var
|
|
45506
|
+
var import_fs12, import_crypto3, PostgresDriver, PostgresDriver_default;
|
|
45298
45507
|
var init_PostgresDriver = __esm({
|
|
45299
45508
|
"../../packages/drivers/src/sql/PostgresDriver.ts"() {
|
|
45300
|
-
|
|
45509
|
+
import_fs12 = __toESM(require("fs"), 1);
|
|
45301
45510
|
import_crypto3 = require("crypto");
|
|
45302
45511
|
init_src();
|
|
45303
45512
|
init_src4();
|
|
@@ -45350,7 +45559,7 @@ var init_PostgresDriver = __esm({
|
|
|
45350
45559
|
});
|
|
45351
45560
|
}
|
|
45352
45561
|
onProgress?.(1);
|
|
45353
|
-
const fileSize =
|
|
45562
|
+
const fileSize = import_fs12.default.statSync(localPath).size;
|
|
45354
45563
|
Logger_default.log(`Postgres ready: staged ${rows} row(s) (${fileSize} bytes) for producer "${producer.name}" from table "${table}"`);
|
|
45355
45564
|
return { files: [{ fullUri: localPath, fileSize }], filteredAtSource: !!pushdown.where };
|
|
45356
45565
|
};
|
|
@@ -45710,12 +45919,12 @@ var init_HttpApiDriver = __esm({
|
|
|
45710
45919
|
});
|
|
45711
45920
|
|
|
45712
45921
|
// ../../packages/drivers/src/DriverHelper.ts
|
|
45713
|
-
var import_stream2, import_promises9,
|
|
45922
|
+
var import_stream2, import_promises9, import_fs13, DriverHelper, DriverHelper_default;
|
|
45714
45923
|
var init_DriverHelper = __esm({
|
|
45715
45924
|
"../../packages/drivers/src/DriverHelper.ts"() {
|
|
45716
45925
|
import_stream2 = require("stream");
|
|
45717
45926
|
import_promises9 = require("stream/promises");
|
|
45718
|
-
|
|
45927
|
+
import_fs13 = require("fs");
|
|
45719
45928
|
init_src2();
|
|
45720
45929
|
init_src();
|
|
45721
45930
|
DriverHelper = {
|
|
@@ -45806,7 +46015,7 @@ var init_DriverHelper = __esm({
|
|
|
45806
46015
|
return processedLine;
|
|
45807
46016
|
};
|
|
45808
46017
|
const writeOptions = append ? { flags: "a" } : {};
|
|
45809
|
-
const writeStream = (0,
|
|
46018
|
+
const writeStream = (0, import_fs13.createWriteStream)(destinationPath, writeOptions);
|
|
45810
46019
|
await (0, import_promises9.pipeline)(stream, headerValidationTransform, writeStream);
|
|
45811
46020
|
return lineCount;
|
|
45812
46021
|
},
|
|
@@ -45814,7 +46023,7 @@ var init_DriverHelper = __esm({
|
|
|
45814
46023
|
Affirm_default(options2, "Invalid options");
|
|
45815
46024
|
const { append, destinationPath, objects, delimiter } = options2;
|
|
45816
46025
|
const writeOptions = append ? { flags: "a" } : {};
|
|
45817
|
-
const writer = new BatchWriter_default((0,
|
|
46026
|
+
const writer = new BatchWriter_default((0, import_fs13.createWriteStream)(destinationPath, writeOptions));
|
|
45818
46027
|
let lineCount = 0;
|
|
45819
46028
|
const keys = Object.keys(objects[0]);
|
|
45820
46029
|
for (const obj of objects) {
|
|
@@ -45825,8 +46034,21 @@ var init_DriverHelper = __esm({
|
|
|
45825
46034
|
await writer.close();
|
|
45826
46035
|
return lineCount;
|
|
45827
46036
|
},
|
|
46037
|
+
/**
|
|
46038
|
+
* Copies a file behind a header line, as bytes rather than lines: the header goes out first and
|
|
46039
|
+
* the file is piped in behind it. Used by the destination drivers that export a dataset whose
|
|
46040
|
+
* records need no transforming, so adding a header costs a copy and not a parse per record.
|
|
46041
|
+
*/
|
|
46042
|
+
copyWithHeader: async (fromPath, toPath, header) => {
|
|
46043
|
+
Affirm_default(fromPath, "Invalid source path");
|
|
46044
|
+
Affirm_default(toPath, "Invalid destination path");
|
|
46045
|
+
Affirm_default(header, "Invalid header line");
|
|
46046
|
+
const writeStream = (0, import_fs13.createWriteStream)(toPath);
|
|
46047
|
+
writeStream.write(header + "\n");
|
|
46048
|
+
await (0, import_promises9.pipeline)((0, import_fs13.createReadStream)(fromPath), writeStream);
|
|
46049
|
+
},
|
|
45828
46050
|
quickReadFile: async (filePath, lineCount) => {
|
|
45829
|
-
const fileStream = (0,
|
|
46051
|
+
const fileStream = (0, import_fs13.createReadStream)(filePath);
|
|
45830
46052
|
const lines = [];
|
|
45831
46053
|
let counter = 0;
|
|
45832
46054
|
for await (const line of readLines(fileStream)) {
|
|
@@ -45856,11 +46078,11 @@ var init_DriverHelper = __esm({
|
|
|
45856
46078
|
});
|
|
45857
46079
|
|
|
45858
46080
|
// ../../packages/drivers/src/files/LocalSourceDriver.ts
|
|
45859
|
-
var
|
|
46081
|
+
var fs14, import_path10, XLSX2, LocalSourceDriver;
|
|
45860
46082
|
var init_LocalSourceDriver = __esm({
|
|
45861
46083
|
"../../packages/drivers/src/files/LocalSourceDriver.ts"() {
|
|
45862
|
-
|
|
45863
|
-
|
|
46084
|
+
fs14 = __toESM(require("fs"), 1);
|
|
46085
|
+
import_path10 = __toESM(require("path"), 1);
|
|
45864
46086
|
init_src();
|
|
45865
46087
|
init_src();
|
|
45866
46088
|
XLSX2 = __toESM(require("@e965/xlsx"), 1);
|
|
@@ -45876,7 +46098,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
45876
46098
|
this.init = async (source) => {
|
|
45877
46099
|
const fileURL = source.authentication["path"];
|
|
45878
46100
|
Affirm_default(fileURL, `Missing file path in the authentication of source "${source.name}"`);
|
|
45879
|
-
const exist =
|
|
46101
|
+
const exist = fs14.existsSync(fileURL);
|
|
45880
46102
|
Affirm_default(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
|
|
45881
46103
|
this._path = source.authentication["path"];
|
|
45882
46104
|
return this;
|
|
@@ -45890,7 +46112,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
45890
46112
|
if (fileKey.includes("%")) {
|
|
45891
46113
|
const allFileKeys = this.listFiles(fileKey);
|
|
45892
46114
|
Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
|
|
45893
|
-
const firstPath =
|
|
46115
|
+
const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
|
|
45894
46116
|
const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
|
|
45895
46117
|
const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
|
|
45896
46118
|
const results = await Promise.all(promises2);
|
|
@@ -45911,7 +46133,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
45911
46133
|
if (fileKey.includes("%")) {
|
|
45912
46134
|
const allFileKeys = this.listFiles(fileKey);
|
|
45913
46135
|
Logger_default.log(`Matched ${allFileKeys.length} files, copying to locally and creating unified dataset.`);
|
|
45914
|
-
const firstPath =
|
|
46136
|
+
const firstPath = import_path10.default.join(this._path, allFileKeys[0]);
|
|
45915
46137
|
const headerLine = (await DriverHelper_default.quickReadFile(firstPath, 1))[0];
|
|
45916
46138
|
const promises2 = allFileKeys.map((x2, i6) => this._get({ ...request, fileKey: x2 }, headerLine, i6));
|
|
45917
46139
|
const results = await Promise.all(promises2);
|
|
@@ -45929,12 +46151,12 @@ var init_LocalSourceDriver = __esm({
|
|
|
45929
46151
|
const allFileKeys = this.listFiles(fileKey);
|
|
45930
46152
|
return allFileKeys.length > 0;
|
|
45931
46153
|
} else {
|
|
45932
|
-
const fileUrl =
|
|
45933
|
-
return
|
|
46154
|
+
const fileUrl = import_path10.default.join(this._path, fileKey);
|
|
46155
|
+
return fs14.existsSync(fileUrl);
|
|
45934
46156
|
}
|
|
45935
46157
|
};
|
|
45936
46158
|
this._readLines = async (fileUri, lineFrom, lineTo) => {
|
|
45937
|
-
const stream =
|
|
46159
|
+
const stream = fs14.createReadStream(fileUri);
|
|
45938
46160
|
const lines = [];
|
|
45939
46161
|
let lineCounter = 0;
|
|
45940
46162
|
for await (const line of readLines(stream)) {
|
|
@@ -45976,7 +46198,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
45976
46198
|
return lines;
|
|
45977
46199
|
};
|
|
45978
46200
|
this._readXmlLines = async (fileUri, lineFrom, lineTo) => {
|
|
45979
|
-
const fileContent =
|
|
46201
|
+
const fileContent = fs14.readFileSync(fileUri, "utf-8");
|
|
45980
46202
|
const jsonData = XMLParser_default.xmlToJson(fileContent);
|
|
45981
46203
|
let lines = Array.isArray(jsonData) ? jsonData.map((item) => JSON.stringify(item)) : [JSON.stringify(jsonData)];
|
|
45982
46204
|
if (Algo_default.hasVal(lineFrom) && Algo_default.hasVal(lineTo)) {
|
|
@@ -45995,7 +46217,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
45995
46217
|
startRow = options2.startRow;
|
|
45996
46218
|
startColumn = options2.startColumn;
|
|
45997
46219
|
}
|
|
45998
|
-
const fileUrl =
|
|
46220
|
+
const fileUrl = import_path10.default.join(this._path, fileKey);
|
|
45999
46221
|
let lines = [];
|
|
46000
46222
|
switch (fileType) {
|
|
46001
46223
|
case "CSV":
|
|
@@ -46040,11 +46262,11 @@ var init_LocalSourceDriver = __esm({
|
|
|
46040
46262
|
try {
|
|
46041
46263
|
const getAllFiles = (dirPath, basePath = "") => {
|
|
46042
46264
|
const files = [];
|
|
46043
|
-
const items =
|
|
46265
|
+
const items = fs14.readdirSync(dirPath);
|
|
46044
46266
|
for (const item of items) {
|
|
46045
|
-
const fullPath =
|
|
46046
|
-
const relativePath = basePath ?
|
|
46047
|
-
const stats =
|
|
46267
|
+
const fullPath = import_path10.default.join(dirPath, item);
|
|
46268
|
+
const relativePath = basePath ? import_path10.default.join(basePath, item) : item;
|
|
46269
|
+
const stats = fs14.statSync(fullPath);
|
|
46048
46270
|
if (stats.isDirectory()) {
|
|
46049
46271
|
files.push(...getAllFiles(fullPath, relativePath));
|
|
46050
46272
|
} else if (stats.isFile()) {
|
|
@@ -46062,16 +46284,16 @@ var init_LocalSourceDriver = __esm({
|
|
|
46062
46284
|
this.readFile = (fileKey) => {
|
|
46063
46285
|
Affirm_default(this._path, "Path not initialized");
|
|
46064
46286
|
Affirm_default(fileKey, "Invalid file key");
|
|
46065
|
-
const filePath =
|
|
46066
|
-
Affirm_default(
|
|
46067
|
-
return
|
|
46287
|
+
const filePath = import_path10.default.join(this._path, fileKey);
|
|
46288
|
+
Affirm_default(fs14.existsSync(filePath), `Source file does not exist: ${filePath}`);
|
|
46289
|
+
return fs14.readFileSync(filePath);
|
|
46068
46290
|
};
|
|
46069
46291
|
this.deleteFile = (fileKey) => {
|
|
46070
46292
|
Affirm_default(this._path, "Path not initialized");
|
|
46071
46293
|
Affirm_default(fileKey, "Invalid file key");
|
|
46072
|
-
const filePath =
|
|
46073
|
-
if (
|
|
46074
|
-
|
|
46294
|
+
const filePath = import_path10.default.join(this._path, fileKey);
|
|
46295
|
+
if (fs14.existsSync(filePath)) {
|
|
46296
|
+
fs14.unlinkSync(filePath);
|
|
46075
46297
|
}
|
|
46076
46298
|
};
|
|
46077
46299
|
this.ready = async (request) => {
|
|
@@ -46087,7 +46309,7 @@ var init_LocalSourceDriver = __esm({
|
|
|
46087
46309
|
allFilePaths.push(...localPaths);
|
|
46088
46310
|
onProgress?.((index + 1) / allFileKeys.length);
|
|
46089
46311
|
}
|
|
46090
|
-
return { files: allFilePaths.map((x2) => ({ fullUri: x2, fileSize:
|
|
46312
|
+
return { files: allFilePaths.map((x2) => ({ fullUri: x2, fileSize: fs14.statSync(x2).size })) };
|
|
46091
46313
|
};
|
|
46092
46314
|
}
|
|
46093
46315
|
};
|
|
@@ -46095,22 +46317,23 @@ var init_LocalSourceDriver = __esm({
|
|
|
46095
46317
|
});
|
|
46096
46318
|
|
|
46097
46319
|
// ../../packages/drivers/src/files/LocalDestinationDriver.ts
|
|
46098
|
-
var
|
|
46320
|
+
var fs15, import_crypto4, import_promises10, import_path11, LocalDestinationDriver;
|
|
46099
46321
|
var init_LocalDestinationDriver = __esm({
|
|
46100
46322
|
"../../packages/drivers/src/files/LocalDestinationDriver.ts"() {
|
|
46101
|
-
|
|
46323
|
+
fs15 = __toESM(require("fs"), 1);
|
|
46102
46324
|
import_crypto4 = require("crypto");
|
|
46103
46325
|
import_promises10 = __toESM(require("fs/promises"), 1);
|
|
46104
|
-
|
|
46326
|
+
import_path11 = __toESM(require("path"), 1);
|
|
46105
46327
|
init_src();
|
|
46106
46328
|
init_src2();
|
|
46329
|
+
init_DriverHelper();
|
|
46107
46330
|
LocalDestinationDriver = class {
|
|
46108
46331
|
constructor() {
|
|
46109
46332
|
this.init = async (source) => {
|
|
46110
46333
|
Affirm_default(source, `Invalid source`);
|
|
46111
46334
|
const fileURL = source.authentication["path"];
|
|
46112
46335
|
Affirm_default(fileURL, `Missing file path in the authentication of source "${source.name}"`);
|
|
46113
|
-
const exist =
|
|
46336
|
+
const exist = fs15.existsSync(fileURL);
|
|
46114
46337
|
Affirm_default(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
|
|
46115
46338
|
this._path = source.authentication["path"];
|
|
46116
46339
|
return this;
|
|
@@ -46119,12 +46342,12 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46119
46342
|
Affirm_default(this._path, "Path not initialized");
|
|
46120
46343
|
Affirm_default(fileKey, "Invalid file key");
|
|
46121
46344
|
Affirm_default(content, "Invalid content");
|
|
46122
|
-
const filePath =
|
|
46123
|
-
const directory =
|
|
46124
|
-
if (!
|
|
46125
|
-
|
|
46345
|
+
const filePath = import_path11.default.join(this._path, fileKey);
|
|
46346
|
+
const directory = import_path11.default.dirname(filePath);
|
|
46347
|
+
if (!fs15.existsSync(directory)) {
|
|
46348
|
+
fs15.mkdirSync(directory, { recursive: true });
|
|
46126
46349
|
}
|
|
46127
|
-
|
|
46350
|
+
fs15.writeFileSync(filePath, content);
|
|
46128
46351
|
return Promise.resolve();
|
|
46129
46352
|
};
|
|
46130
46353
|
this.copyFromS3 = async (s3Driver, sourceFileKey, destinationFileKey) => {
|
|
@@ -46151,11 +46374,14 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46151
46374
|
} catch {
|
|
46152
46375
|
}
|
|
46153
46376
|
};
|
|
46154
|
-
this.move = async (fromPath, toName) => {
|
|
46155
|
-
const toFilePath =
|
|
46377
|
+
this.move = async (fromPath, toName, header) => {
|
|
46378
|
+
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46156
46379
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46157
46380
|
try {
|
|
46158
|
-
|
|
46381
|
+
if (header)
|
|
46382
|
+
await DriverHelper_default.copyWithHeader(fromPath, stagingPath, header);
|
|
46383
|
+
else
|
|
46384
|
+
await import_promises10.default.copyFile(fromPath, stagingPath);
|
|
46159
46385
|
await import_promises10.default.rename(stagingPath, toFilePath);
|
|
46160
46386
|
await import_promises10.default.unlink(fromPath);
|
|
46161
46387
|
return { bucket: "", key: toFilePath, res: true };
|
|
@@ -46165,12 +46391,14 @@ var init_LocalDestinationDriver = __esm({
|
|
|
46165
46391
|
return { bucket: "", key: "", res: false };
|
|
46166
46392
|
}
|
|
46167
46393
|
};
|
|
46168
|
-
this.transformAndMove = async (fromPath, transform, toName) => {
|
|
46169
|
-
const toFilePath =
|
|
46394
|
+
this.transformAndMove = async (fromPath, transform, toName, header) => {
|
|
46395
|
+
const toFilePath = import_path11.default.join(this._path, toName);
|
|
46170
46396
|
const stagingPath = this._stagingPath(toFilePath);
|
|
46171
46397
|
try {
|
|
46172
|
-
const reader =
|
|
46173
|
-
const writer = new BatchWriter_default(
|
|
46398
|
+
const reader = fs15.createReadStream(fromPath);
|
|
46399
|
+
const writer = new BatchWriter_default(fs15.createWriteStream(stagingPath));
|
|
46400
|
+
if (header)
|
|
46401
|
+
writer.write(header);
|
|
46174
46402
|
for await (const line of readLines(reader)) {
|
|
46175
46403
|
if (!writer.write(transform(line)))
|
|
46176
46404
|
await writer.waitForDrain();
|
|
@@ -46304,7 +46532,7 @@ var init_StreamReader = __esm({
|
|
|
46304
46532
|
});
|
|
46305
46533
|
|
|
46306
46534
|
// ../../packages/drivers/src/s3/S3SourceDriver.ts
|
|
46307
|
-
var import_client_s32, import_stream3, import_promises11,
|
|
46535
|
+
var import_client_s32, import_stream3, import_promises11, import_path12, import_fs14, S3SourceDriver;
|
|
46308
46536
|
var init_S3SourceDriver = __esm({
|
|
46309
46537
|
"../../packages/drivers/src/s3/S3SourceDriver.ts"() {
|
|
46310
46538
|
import_client_s32 = require("@aws-sdk/client-s3");
|
|
@@ -46312,8 +46540,8 @@ var init_S3SourceDriver = __esm({
|
|
|
46312
46540
|
init_AwsCredentials();
|
|
46313
46541
|
import_stream3 = require("stream");
|
|
46314
46542
|
import_promises11 = require("stream/promises");
|
|
46315
|
-
|
|
46316
|
-
|
|
46543
|
+
import_path12 = __toESM(require("path"), 1);
|
|
46544
|
+
import_fs14 = __toESM(require("fs"), 1);
|
|
46317
46545
|
init_src6();
|
|
46318
46546
|
init_StreamReader();
|
|
46319
46547
|
init_src4();
|
|
@@ -46473,9 +46701,9 @@ var init_S3SourceDriver = __esm({
|
|
|
46473
46701
|
});
|
|
46474
46702
|
const response = await this._client.send(command5);
|
|
46475
46703
|
Affirm_default(response.Body, `Failed to fetch object "${s3Key}" from S3`);
|
|
46476
|
-
const fileDir =
|
|
46477
|
-
if (!
|
|
46478
|
-
|
|
46704
|
+
const fileDir = import_path12.default.dirname(localPath);
|
|
46705
|
+
if (!import_fs14.default.existsSync(fileDir)) {
|
|
46706
|
+
import_fs14.default.mkdirSync(fileDir, { recursive: true });
|
|
46479
46707
|
}
|
|
46480
46708
|
const totalBytes = response.ContentLength ?? 0;
|
|
46481
46709
|
let stagedBytes = 0;
|
|
@@ -46487,7 +46715,7 @@ var init_S3SourceDriver = __esm({
|
|
|
46487
46715
|
done(null, chunk);
|
|
46488
46716
|
}
|
|
46489
46717
|
});
|
|
46490
|
-
const writeStream =
|
|
46718
|
+
const writeStream = import_fs14.default.createWriteStream(localPath);
|
|
46491
46719
|
await (0, import_promises11.pipeline)(response.Body, meter, writeStream);
|
|
46492
46720
|
report(1);
|
|
46493
46721
|
};
|
|
@@ -46501,12 +46729,12 @@ var init_S3SourceDriver = __esm({
|
|
|
46501
46729
|
await streamToFile(s3Key, localPath, (fraction) => onProgress?.((index + fraction) / allFileKeys.length));
|
|
46502
46730
|
allFilePaths.push(localPath);
|
|
46503
46731
|
}
|
|
46504
|
-
return { files: allFilePaths.map((x2) => ({ fullUri: x2, fileSize:
|
|
46732
|
+
return { files: allFilePaths.map((x2) => ({ fullUri: x2, fileSize: import_fs14.default.statSync(x2).size })) };
|
|
46505
46733
|
} else {
|
|
46506
46734
|
const localPath = ExecutorScope_default.getProducerPath(scope, producer, fileKey);
|
|
46507
46735
|
ExecutorScope_default.ensurePath(localPath);
|
|
46508
46736
|
await streamToFile(fileKey, localPath, (fraction) => onProgress?.(fraction));
|
|
46509
|
-
return { files: [{ fullUri: localPath, fileSize:
|
|
46737
|
+
return { files: [{ fullUri: localPath, fileSize: import_fs14.default.statSync(localPath).size }] };
|
|
46510
46738
|
}
|
|
46511
46739
|
};
|
|
46512
46740
|
}
|
|
@@ -46515,7 +46743,7 @@ var init_S3SourceDriver = __esm({
|
|
|
46515
46743
|
});
|
|
46516
46744
|
|
|
46517
46745
|
// ../../packages/drivers/src/s3/S3DestinationDriver.ts
|
|
46518
|
-
var import_client_s33,
|
|
46746
|
+
var import_client_s33, import_path13, import_fs15, S3DestinationDriver;
|
|
46519
46747
|
var init_S3DestinationDriver = __esm({
|
|
46520
46748
|
"../../packages/drivers/src/s3/S3DestinationDriver.ts"() {
|
|
46521
46749
|
import_client_s33 = require("@aws-sdk/client-s3");
|
|
@@ -46523,8 +46751,8 @@ var init_S3DestinationDriver = __esm({
|
|
|
46523
46751
|
init_src2();
|
|
46524
46752
|
init_src4();
|
|
46525
46753
|
init_AwsCredentials();
|
|
46526
|
-
|
|
46527
|
-
|
|
46754
|
+
import_path13 = __toESM(require("path"), 1);
|
|
46755
|
+
import_fs15 = __toESM(require("fs"), 1);
|
|
46528
46756
|
S3DestinationDriver = class {
|
|
46529
46757
|
constructor() {
|
|
46530
46758
|
this.init = async (source) => {
|
|
@@ -46557,26 +46785,33 @@ var init_S3DestinationDriver = __esm({
|
|
|
46557
46785
|
Body: content
|
|
46558
46786
|
}));
|
|
46559
46787
|
};
|
|
46560
|
-
this.move = async (fromPath, toName) => {
|
|
46788
|
+
this.move = async (fromPath, toName, header) => {
|
|
46561
46789
|
Affirm_default(fromPath, "Invalid source path");
|
|
46562
46790
|
Affirm_default(toName, "Invalid destination name");
|
|
46563
|
-
Affirm_default(
|
|
46564
|
-
const readStream =
|
|
46565
|
-
return this._multipartUpload(toName, readStream);
|
|
46791
|
+
Affirm_default(import_fs15.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
|
|
46792
|
+
const readStream = import_fs15.default.createReadStream(fromPath);
|
|
46793
|
+
return this._multipartUpload(toName, header ? this._behindHeader(header, readStream) : readStream);
|
|
46566
46794
|
};
|
|
46567
|
-
this.transformAndMove = async (fromPath, transform, toName) => {
|
|
46795
|
+
this.transformAndMove = async (fromPath, transform, toName, header) => {
|
|
46568
46796
|
Affirm_default(fromPath, "Invalid source path");
|
|
46569
46797
|
Affirm_default(transform, "Invalid transform function");
|
|
46570
46798
|
Affirm_default(toName, "Invalid destination name");
|
|
46571
|
-
Affirm_default(
|
|
46572
|
-
const reader =
|
|
46799
|
+
Affirm_default(import_fs15.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
|
|
46800
|
+
const reader = import_fs15.default.createReadStream(fromPath);
|
|
46573
46801
|
const transformedLines = async function* () {
|
|
46802
|
+
if (header)
|
|
46803
|
+
yield header + "\n";
|
|
46574
46804
|
for await (const line of readLines(reader)) {
|
|
46575
46805
|
yield transform(line) + "\n";
|
|
46576
46806
|
}
|
|
46577
46807
|
};
|
|
46578
46808
|
return this._multipartUpload(toName, transformedLines());
|
|
46579
46809
|
};
|
|
46810
|
+
/** The file's own bytes behind its header line, so an untransformed upload stays a byte stream. */
|
|
46811
|
+
this._behindHeader = async function* (header, source) {
|
|
46812
|
+
yield header + "\n";
|
|
46813
|
+
yield* source;
|
|
46814
|
+
};
|
|
46580
46815
|
this._multipartUpload = async (toName, dataSource) => {
|
|
46581
46816
|
let uploadId;
|
|
46582
46817
|
try {
|
|
@@ -46620,7 +46855,7 @@ var init_S3DestinationDriver = __esm({
|
|
|
46620
46855
|
await this._abortUpload(toName, uploadId);
|
|
46621
46856
|
uploadId = void 0;
|
|
46622
46857
|
await this.saveFile(toName, Buffer.alloc(0));
|
|
46623
|
-
return { res: true, key:
|
|
46858
|
+
return { res: true, key: import_path13.default.join(this._bucketName, toName), bucket: this._bucketName };
|
|
46624
46859
|
}
|
|
46625
46860
|
const completeRes = await this._client.send(new import_client_s33.CompleteMultipartUploadCommand({
|
|
46626
46861
|
Bucket: this._bucketName,
|
|
@@ -46631,7 +46866,7 @@ var init_S3DestinationDriver = __esm({
|
|
|
46631
46866
|
}
|
|
46632
46867
|
}));
|
|
46633
46868
|
Affirm_default(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${toName}": status code ${completeRes.$metadata.httpStatusCode}`);
|
|
46634
|
-
return { res: true, key:
|
|
46869
|
+
return { res: true, key: import_path13.default.join(this._bucketName, toName), bucket: this._bucketName };
|
|
46635
46870
|
} catch (error) {
|
|
46636
46871
|
if (uploadId)
|
|
46637
46872
|
await this._abortUpload(toName, uploadId);
|
|
@@ -46783,9 +47018,9 @@ var init_UrlMinter = __esm({
|
|
|
46783
47018
|
return config;
|
|
46784
47019
|
};
|
|
46785
47020
|
/** Walks a dot path (`data.uploadUrl`) into the response, answering undefined rather than throwing. */
|
|
46786
|
-
this._resolve = (payload,
|
|
47021
|
+
this._resolve = (payload, path22) => {
|
|
46787
47022
|
let current = payload;
|
|
46788
|
-
for (const segment of
|
|
47023
|
+
for (const segment of path22.split(".")) {
|
|
46789
47024
|
if (current === null || typeof current !== "object")
|
|
46790
47025
|
return void 0;
|
|
46791
47026
|
current = current[segment];
|
|
@@ -46800,7 +47035,7 @@ var init_UrlMinter = __esm({
|
|
|
46800
47035
|
});
|
|
46801
47036
|
|
|
46802
47037
|
// ../../packages/drivers/src/presigned/PresignedUrlSourceDriver.ts
|
|
46803
|
-
var import_stream4, import_promises12,
|
|
47038
|
+
var import_stream4, import_promises12, import_fs16, MAX_ERROR_BODY_CHARS2, PresignedUrlSourceDriver;
|
|
46804
47039
|
var init_PresignedUrlSourceDriver = __esm({
|
|
46805
47040
|
"../../packages/drivers/src/presigned/PresignedUrlSourceDriver.ts"() {
|
|
46806
47041
|
init_src();
|
|
@@ -46808,7 +47043,7 @@ var init_PresignedUrlSourceDriver = __esm({
|
|
|
46808
47043
|
init_src2();
|
|
46809
47044
|
import_stream4 = require("stream");
|
|
46810
47045
|
import_promises12 = require("stream/promises");
|
|
46811
|
-
|
|
47046
|
+
import_fs16 = __toESM(require("fs"), 1);
|
|
46812
47047
|
init_StreamReader();
|
|
46813
47048
|
init_UrlMinter();
|
|
46814
47049
|
MAX_ERROR_BODY_CHARS2 = 500;
|
|
@@ -46890,9 +47125,9 @@ var init_PresignedUrlSourceDriver = __esm({
|
|
|
46890
47125
|
done(null, chunk);
|
|
46891
47126
|
}
|
|
46892
47127
|
});
|
|
46893
|
-
await (0, import_promises12.pipeline)(import_stream4.Readable.fromWeb(response.body), meter,
|
|
47128
|
+
await (0, import_promises12.pipeline)(import_stream4.Readable.fromWeb(response.body), meter, import_fs16.default.createWriteStream(localPath));
|
|
46894
47129
|
onProgress?.(1);
|
|
46895
|
-
const fileSize =
|
|
47130
|
+
const fileSize = import_fs16.default.statSync(localPath).size;
|
|
46896
47131
|
Logger_default.info(`Staged "${fileKey}" from a signed URL to ${localPath} (${fileSize} bytes)`);
|
|
46897
47132
|
return { files: [{ fullUri: localPath, fileSize }] };
|
|
46898
47133
|
};
|
|
@@ -46932,7 +47167,7 @@ var init_PresignedUrlSourceDriver = __esm({
|
|
|
46932
47167
|
});
|
|
46933
47168
|
|
|
46934
47169
|
// ../../packages/drivers/src/presigned/PresignedUrlDestinationDriver.ts
|
|
46935
|
-
var import_stream5, import_crypto5, import_promises13,
|
|
47170
|
+
var import_stream5, import_crypto5, import_promises13, import_fs17, MAX_ERROR_BODY_CHARS3, PresignedUrlDestinationDriver;
|
|
46936
47171
|
var init_PresignedUrlDestinationDriver = __esm({
|
|
46937
47172
|
"../../packages/drivers/src/presigned/PresignedUrlDestinationDriver.ts"() {
|
|
46938
47173
|
init_src();
|
|
@@ -46941,8 +47176,9 @@ var init_PresignedUrlDestinationDriver = __esm({
|
|
|
46941
47176
|
import_stream5 = require("stream");
|
|
46942
47177
|
import_crypto5 = require("crypto");
|
|
46943
47178
|
import_promises13 = __toESM(require("fs/promises"), 1);
|
|
46944
|
-
|
|
47179
|
+
import_fs17 = __toESM(require("fs"), 1);
|
|
46945
47180
|
init_UrlMinter();
|
|
47181
|
+
init_DriverHelper();
|
|
46946
47182
|
MAX_ERROR_BODY_CHARS3 = 500;
|
|
46947
47183
|
PresignedUrlDestinationDriver = class {
|
|
46948
47184
|
constructor() {
|
|
@@ -46955,25 +47191,31 @@ var init_PresignedUrlDestinationDriver = __esm({
|
|
|
46955
47191
|
Logger_default.log(`Presigned URL destination "${source.name}" initialised: minting endpoint=${UrlMinter_default.safeUrl(SecretManager_default.replaceSecret(source.authentication.url))}`);
|
|
46956
47192
|
return this;
|
|
46957
47193
|
};
|
|
46958
|
-
this.move = async (fromPath, toName) => {
|
|
47194
|
+
this.move = async (fromPath, toName, header) => {
|
|
46959
47195
|
Affirm_default(fromPath, "Invalid source path");
|
|
46960
47196
|
Affirm_default(toName, "Invalid destination name");
|
|
46961
|
-
Affirm_default(
|
|
47197
|
+
Affirm_default(import_fs17.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
|
|
47198
|
+
const stagingPath = `${fromPath}.${(0, import_crypto5.randomUUID)()}.partial`;
|
|
46962
47199
|
try {
|
|
46963
|
-
|
|
47200
|
+
if (!header)
|
|
47201
|
+
return await this._upload(fromPath, toName);
|
|
47202
|
+
await DriverHelper_default.copyWithHeader(fromPath, stagingPath, header);
|
|
47203
|
+
return await this._upload(stagingPath, toName);
|
|
46964
47204
|
} catch (error) {
|
|
46965
47205
|
Logger_default.error(`Presigned upload of "${toName}" from source "${this._source.name}" failed: ${error}`);
|
|
46966
47206
|
return { res: false, key: "", bucket: "" };
|
|
47207
|
+
} finally {
|
|
47208
|
+
await this._discardStaging(stagingPath);
|
|
46967
47209
|
}
|
|
46968
47210
|
};
|
|
46969
|
-
this.transformAndMove = async (fromPath, transform, toName) => {
|
|
47211
|
+
this.transformAndMove = async (fromPath, transform, toName, header) => {
|
|
46970
47212
|
Affirm_default(fromPath, "Invalid source path");
|
|
46971
47213
|
Affirm_default(transform, "Invalid transform function");
|
|
46972
47214
|
Affirm_default(toName, "Invalid destination name");
|
|
46973
|
-
Affirm_default(
|
|
47215
|
+
Affirm_default(import_fs17.default.existsSync(fromPath), `Source file does not exist: ${fromPath}`);
|
|
46974
47216
|
const stagingPath = `${fromPath}.${(0, import_crypto5.randomUUID)()}.partial`;
|
|
46975
47217
|
try {
|
|
46976
|
-
await this._stageTransformed(fromPath, transform, stagingPath);
|
|
47218
|
+
await this._stageTransformed(fromPath, transform, stagingPath, header);
|
|
46977
47219
|
return await this._upload(stagingPath, toName);
|
|
46978
47220
|
} catch (error) {
|
|
46979
47221
|
Logger_default.error(`Presigned upload of "${toName}" from source "${this._source.name}" failed: ${error}`);
|
|
@@ -46984,11 +47226,11 @@ var init_PresignedUrlDestinationDriver = __esm({
|
|
|
46984
47226
|
};
|
|
46985
47227
|
this._upload = async (fromPath, toName) => {
|
|
46986
47228
|
const minted = await UrlMinter_default.mint(this._auth, toName);
|
|
46987
|
-
const fileSize =
|
|
47229
|
+
const fileSize = import_fs17.default.statSync(fromPath).size;
|
|
46988
47230
|
const options2 = {
|
|
46989
47231
|
method: this._auth.signedUrlMethod || "PUT",
|
|
46990
47232
|
headers: { ...minted.headers, "content-length": String(fileSize) },
|
|
46991
|
-
body: import_stream5.Readable.toWeb(
|
|
47233
|
+
body: import_stream5.Readable.toWeb(import_fs17.default.createReadStream(fromPath)),
|
|
46992
47234
|
duplex: "half"
|
|
46993
47235
|
};
|
|
46994
47236
|
const response = await fetch(minted.url, options2);
|
|
@@ -47001,9 +47243,11 @@ var init_PresignedUrlDestinationDriver = __esm({
|
|
|
47001
47243
|
return { res: true, key: toName, bucket: this._host(minted.url) };
|
|
47002
47244
|
};
|
|
47003
47245
|
/** Writes the transformed lines to their own file, so the upload has a size to declare. */
|
|
47004
|
-
this._stageTransformed = async (fromPath, transform, stagingPath) => {
|
|
47005
|
-
const reader =
|
|
47006
|
-
const writer = new BatchWriter_default(
|
|
47246
|
+
this._stageTransformed = async (fromPath, transform, stagingPath, header) => {
|
|
47247
|
+
const reader = import_fs17.default.createReadStream(fromPath);
|
|
47248
|
+
const writer = new BatchWriter_default(import_fs17.default.createWriteStream(stagingPath));
|
|
47249
|
+
if (header)
|
|
47250
|
+
writer.write(header);
|
|
47007
47251
|
for await (const line of readLines(reader)) {
|
|
47008
47252
|
if (!writer.write(transform(line)))
|
|
47009
47253
|
await writer.waitForDrain();
|
|
@@ -47599,10 +47843,10 @@ var init_LineParser = __esm({
|
|
|
47599
47843
|
});
|
|
47600
47844
|
|
|
47601
47845
|
// ../../packages/engines/src/producer/ProducerEngine.ts
|
|
47602
|
-
var
|
|
47846
|
+
var import_path14, import_crypto6, noopTracker, ProducerEngineClass, ProducerEngine, ProducerEngine_default;
|
|
47603
47847
|
var init_ProducerEngine = __esm({
|
|
47604
47848
|
"../../packages/engines/src/producer/ProducerEngine.ts"() {
|
|
47605
|
-
|
|
47849
|
+
import_path14 = __toESM(require("path"), 1);
|
|
47606
47850
|
import_crypto6 = require("crypto");
|
|
47607
47851
|
init_src();
|
|
47608
47852
|
init_src8();
|
|
@@ -47710,13 +47954,13 @@ var init_ProducerEngine = __esm({
|
|
|
47710
47954
|
const driver = await DriverFactory_default.instantiateSource(source);
|
|
47711
47955
|
const readyResult = await driver.ready({ producer, scope });
|
|
47712
47956
|
const firstFile = readyResult.files[0].fullUri;
|
|
47713
|
-
const decompressedDir =
|
|
47957
|
+
const decompressedDir = import_path14.default.dirname(firstFile);
|
|
47714
47958
|
effectiveSource = { ...source, engine: "local", authentication: { ...source.authentication, path: decompressedDir } };
|
|
47715
47959
|
effectiveProducer = {
|
|
47716
47960
|
...producer,
|
|
47717
47961
|
settings: {
|
|
47718
47962
|
...producer.settings,
|
|
47719
|
-
fileKey: readyResult.files.length > 1 ? "%.dataset" :
|
|
47963
|
+
fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path14.default.basename(firstFile),
|
|
47720
47964
|
compressionType: void 0
|
|
47721
47965
|
}
|
|
47722
47966
|
};
|
|
@@ -47742,7 +47986,7 @@ var init_ProducerEngine = __esm({
|
|
|
47742
47986
|
const firstItem = rawData[0];
|
|
47743
47987
|
const isLineData = typeof firstItem === "string";
|
|
47744
47988
|
const fills = discover ? [] : ProducerManager_default.compileFills(this._fillDimensions(producer));
|
|
47745
|
-
const sourceFilename =
|
|
47989
|
+
const sourceFilename = import_path14.default.basename(producer.settings.fileKey ?? producer.settings.sqlTable ?? "");
|
|
47746
47990
|
if (isLineData) {
|
|
47747
47991
|
const lines = rawData;
|
|
47748
47992
|
const dimensions = this.deriveDimensions(lines[0], producer, discover);
|
|
@@ -47949,7 +48193,7 @@ var init_DiscoverMerger = __esm({
|
|
|
47949
48193
|
});
|
|
47950
48194
|
|
|
47951
48195
|
// ../../packages/engines/src/ai/DeveloperEngine.ts
|
|
47952
|
-
var
|
|
48196
|
+
var import_path15, import_promises14, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
|
|
47953
48197
|
var init_DeveloperEngine = __esm({
|
|
47954
48198
|
"../../packages/engines/src/ai/DeveloperEngine.ts"() {
|
|
47955
48199
|
init_src();
|
|
@@ -47957,7 +48201,7 @@ var init_DeveloperEngine = __esm({
|
|
|
47957
48201
|
init_DiscoverMerger();
|
|
47958
48202
|
init_src4();
|
|
47959
48203
|
init_src8();
|
|
47960
|
-
|
|
48204
|
+
import_path15 = __toESM(require("path"), 1);
|
|
47961
48205
|
import_promises14 = __toESM(require("fs/promises"), 1);
|
|
47962
48206
|
import_dayjs2 = __toESM(require("dayjs"), 1);
|
|
47963
48207
|
import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
|
|
@@ -47984,7 +48228,7 @@ var init_DeveloperEngine = __esm({
|
|
|
47984
48228
|
const discovered = typeDefinitions.map(this.toDimension);
|
|
47985
48229
|
const { dimensions, report } = DiscoverMerger_default.merge(authored.dimensions ?? [], discovered, mode);
|
|
47986
48230
|
const mappedProducer = { ...authored, dimensions };
|
|
47987
|
-
await import_promises14.default.mkdir(
|
|
48231
|
+
await import_promises14.default.mkdir(import_path15.default.dirname(producerPath), { recursive: true });
|
|
47988
48232
|
await import_promises14.default.writeFile(producerPath, `${JSON.stringify(mappedProducer, null, 4)}
|
|
47989
48233
|
`, "utf-8");
|
|
47990
48234
|
return { producer: mappedProducer, fields: typeDefinitions, report };
|
|
@@ -50567,12 +50811,12 @@ var init_MockDataFactory = __esm({
|
|
|
50567
50811
|
});
|
|
50568
50812
|
|
|
50569
50813
|
// ../../packages/engines/src/mock/SyntheticInputEngine.ts
|
|
50570
|
-
var import_promises15, import_os,
|
|
50814
|
+
var import_promises15, import_os, import_path16, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine;
|
|
50571
50815
|
var init_SyntheticInputEngine = __esm({
|
|
50572
50816
|
"../../packages/engines/src/mock/SyntheticInputEngine.ts"() {
|
|
50573
50817
|
import_promises15 = __toESM(require("fs/promises"), 1);
|
|
50574
50818
|
import_os = __toESM(require("os"), 1);
|
|
50575
|
-
|
|
50819
|
+
import_path16 = __toESM(require("path"), 1);
|
|
50576
50820
|
import_dayjs4 = __toESM(require("dayjs"), 1);
|
|
50577
50821
|
import_customParseFormat3 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
|
|
50578
50822
|
import_utc3 = __toESM(require("dayjs/plugin/utc"), 1);
|
|
@@ -50596,7 +50840,7 @@ var init_SyntheticInputEngine = __esm({
|
|
|
50596
50840
|
for (const cs2 of [...BUILTIN_CODE_SETS, ...codeSets]) codeSetMap.set(cs2.name.toLowerCase(), cs2);
|
|
50597
50841
|
const producerMap = new Map(producers.map((p5) => [p5.name, p5]));
|
|
50598
50842
|
const refTargets = this._collectReferences(producers, producerMap);
|
|
50599
|
-
const dir = await import_promises15.default.mkdtemp(
|
|
50843
|
+
const dir = await import_promises15.default.mkdtemp(import_path16.default.join(import_os.default.tmpdir(), "remora-synth-"));
|
|
50600
50844
|
const poolSize = Math.min(Math.max(opts.records, 1), POOL_CAP);
|
|
50601
50845
|
const registry = new MockKeyRegistry();
|
|
50602
50846
|
const specs = producers.flatMap((p5) => this._specs(p5, codeSetMap, producerMap, refTargets));
|
|
@@ -50662,7 +50906,7 @@ var init_SyntheticInputEngine = __esm({
|
|
|
50662
50906
|
}
|
|
50663
50907
|
lines.push(JSON.stringify(row) + "\n");
|
|
50664
50908
|
}
|
|
50665
|
-
const filePath =
|
|
50909
|
+
const filePath = import_path16.default.join(dir, `${producer.name}.jsonl`);
|
|
50666
50910
|
await import_promises15.default.writeFile(filePath, lines.join(""), "utf-8");
|
|
50667
50911
|
const { size } = await import_promises15.default.stat(filePath);
|
|
50668
50912
|
return { path: filePath, fileSize: size };
|
|
@@ -51494,33 +51738,33 @@ var init_RequestValidator = __esm({
|
|
|
51494
51738
|
this.resolveMembers = (consumer) => {
|
|
51495
51739
|
return new Set(ConsumerManager_default.getExpandedFields(consumer).map((x2) => x2.finalKey));
|
|
51496
51740
|
};
|
|
51497
|
-
this.checkFilters = (value, context,
|
|
51741
|
+
this.checkFilters = (value, context, path22, depth) => {
|
|
51498
51742
|
const { MAX_REQUEST_FILTERS, MAX_REQUEST_FILTER_DEPTH } = Constants_default.defaults;
|
|
51499
51743
|
if (!Array.isArray(value))
|
|
51500
|
-
throw new Error(`Invalid "${
|
|
51744
|
+
throw new Error(`Invalid "${path22}": expected an array of filters, got ${this.describe(value)}`);
|
|
51501
51745
|
if (value.length > MAX_REQUEST_FILTERS)
|
|
51502
|
-
throw new Error(`Invalid "${
|
|
51746
|
+
throw new Error(`Invalid "${path22}": ${value.length} filters is above the maximum of ${MAX_REQUEST_FILTERS}`);
|
|
51503
51747
|
if (depth > MAX_REQUEST_FILTER_DEPTH)
|
|
51504
|
-
throw new Error(`Invalid filter at ${
|
|
51505
|
-
return value.map((x2, index) => this.checkFilter(x2, context, `${
|
|
51748
|
+
throw new Error(`Invalid filter at ${path22}: filters may not nest more than ${MAX_REQUEST_FILTER_DEPTH} deep`);
|
|
51749
|
+
return value.map((x2, index) => this.checkFilter(x2, context, `${path22}[${index}]`, depth));
|
|
51506
51750
|
};
|
|
51507
|
-
this.checkFilter = (value, context,
|
|
51751
|
+
this.checkFilter = (value, context, path22, depth) => {
|
|
51508
51752
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
51509
|
-
throw new Error(`Invalid filter at ${
|
|
51753
|
+
throw new Error(`Invalid filter at ${path22}: expected an object, got ${this.describe(value)}`);
|
|
51510
51754
|
const filter = value;
|
|
51511
51755
|
const unknownKey = Object.keys(filter).find((key) => !FILTER_KEYS.includes(key));
|
|
51512
51756
|
if (unknownKey)
|
|
51513
|
-
throw new Error(`Invalid filter at ${
|
|
51514
|
-
const operator = this.checkOperator(filter.operator,
|
|
51757
|
+
throw new Error(`Invalid filter at ${path22}: unknown property "${unknownKey}". Supported: ${FILTER_KEYS.join(", ")}`);
|
|
51758
|
+
const operator = this.checkOperator(filter.operator, path22);
|
|
51515
51759
|
const checked = {
|
|
51516
|
-
member: this.checkMember(filter.member, context,
|
|
51760
|
+
member: this.checkMember(filter.member, context, path22),
|
|
51517
51761
|
operator,
|
|
51518
|
-
values: this.checkValues(filter.values, operator,
|
|
51762
|
+
values: this.checkValues(filter.values, operator, path22)
|
|
51519
51763
|
};
|
|
51520
51764
|
if (Algo_default.hasVal(filter.and))
|
|
51521
|
-
checked.and = this.checkFilters(filter.and, context, `${
|
|
51765
|
+
checked.and = this.checkFilters(filter.and, context, `${path22}.and`, depth + 1);
|
|
51522
51766
|
if (Algo_default.hasVal(filter.or))
|
|
51523
|
-
checked.or = this.checkFilters(filter.or, context, `${
|
|
51767
|
+
checked.or = this.checkFilters(filter.or, context, `${path22}.or`, depth + 1);
|
|
51524
51768
|
return checked;
|
|
51525
51769
|
};
|
|
51526
51770
|
/**
|
|
@@ -51528,18 +51772,18 @@ var init_RequestValidator = __esm({
|
|
|
51528
51772
|
* own field names by the time filters run, so any other name reads `undefined` and quietly keeps
|
|
51529
51773
|
* every row (or drops every row, for a negated operator).
|
|
51530
51774
|
*/
|
|
51531
|
-
this.checkMember = (value, context,
|
|
51775
|
+
this.checkMember = (value, context, path22) => {
|
|
51532
51776
|
if (typeof value !== "string" || value.length === 0)
|
|
51533
|
-
throw new Error(`Invalid filter at ${
|
|
51777
|
+
throw new Error(`Invalid filter at ${path22}: "member" must be a non-empty string, got ${this.describe(value)}`);
|
|
51534
51778
|
if (!context.members.has(value))
|
|
51535
|
-
throw new Error(`Invalid filter at ${
|
|
51779
|
+
throw new Error(`Invalid filter at ${path22}: consumer "${context.consumer.name}" has no field named "${value}"`);
|
|
51536
51780
|
return value;
|
|
51537
51781
|
};
|
|
51538
|
-
this.checkOperator = (value,
|
|
51782
|
+
this.checkOperator = (value, path22) => {
|
|
51539
51783
|
if (typeof value !== "string" || !VALUES_BY_OPERATOR.has(value))
|
|
51540
|
-
throw new Error(`Invalid filter at ${
|
|
51784
|
+
throw new Error(`Invalid filter at ${path22}: unknown operator ${this.describe(value)}. Supported: ${this.supportedOperators()}`);
|
|
51541
51785
|
if (VALUES_BY_OPERATOR.get(value) === "unsupported")
|
|
51542
|
-
throw new Error(`Invalid filter at ${
|
|
51786
|
+
throw new Error(`Invalid filter at ${path22}: operator "${value}" is not supported by a run`);
|
|
51543
51787
|
return value;
|
|
51544
51788
|
};
|
|
51545
51789
|
/**
|
|
@@ -51547,24 +51791,24 @@ var init_RequestValidator = __esm({
|
|
|
51547
51791
|
* own type and the filter's is read back into it, so a number or a boolean here is a client bug
|
|
51548
51792
|
* that reads as a type mismatch (matching nothing) rather than as the comparison it meant.
|
|
51549
51793
|
*/
|
|
51550
|
-
this.checkValues = (value, operator,
|
|
51794
|
+
this.checkValues = (value, operator, path22) => {
|
|
51551
51795
|
const values = Algo_default.hasVal(value) ? value : [];
|
|
51552
51796
|
if (!Array.isArray(values))
|
|
51553
|
-
throw new Error(`Invalid filter at ${
|
|
51797
|
+
throw new Error(`Invalid filter at ${path22}: "values" must be an array of strings, got ${this.describe(value)}`);
|
|
51554
51798
|
const wrongIndex = values.findIndex((x2) => typeof x2 !== "string");
|
|
51555
51799
|
if (wrongIndex >= 0)
|
|
51556
|
-
throw new Error(`Invalid filter at ${
|
|
51557
|
-
return this.checkArity(values, operator,
|
|
51800
|
+
throw new Error(`Invalid filter at ${path22}: "values[${wrongIndex}]" must be a string, got ${this.describe(values[wrongIndex])}`);
|
|
51801
|
+
return this.checkArity(values, operator, path22);
|
|
51558
51802
|
};
|
|
51559
|
-
this.checkArity = (values, operator,
|
|
51803
|
+
this.checkArity = (values, operator, path22) => {
|
|
51560
51804
|
const expected = VALUES_BY_OPERATOR.get(operator);
|
|
51561
51805
|
const reason = `operator "${operator}"`;
|
|
51562
51806
|
if (expected === "none" && values.length > 0)
|
|
51563
|
-
throw new Error(`Invalid filter at ${
|
|
51807
|
+
throw new Error(`Invalid filter at ${path22}: ${reason} reads no values, ${values.length} given`);
|
|
51564
51808
|
if (expected === "many" && values.length === 0)
|
|
51565
|
-
throw new Error(`Invalid filter at ${
|
|
51809
|
+
throw new Error(`Invalid filter at ${path22}: ${reason} needs at least one value, none given`);
|
|
51566
51810
|
if (typeof expected === "number" && values.length !== expected)
|
|
51567
|
-
throw new Error(`Invalid filter at ${
|
|
51811
|
+
throw new Error(`Invalid filter at ${path22}: ${reason} needs exactly ${expected} ${expected === 1 ? "value" : "values"}, ${values.length} given`);
|
|
51568
51812
|
return values;
|
|
51569
51813
|
};
|
|
51570
51814
|
/** Every operator a request may ask for, in the order they are declared. */
|
|
@@ -51905,23 +52149,23 @@ var init_Diagnostics = __esm({
|
|
|
51905
52149
|
return [.../* @__PURE__ */ new Set([...inputs, ...chained])];
|
|
51906
52150
|
};
|
|
51907
52151
|
this._walk = (start, next, onCycle) => {
|
|
51908
|
-
const
|
|
52152
|
+
const path22 = [];
|
|
51909
52153
|
const onPath = /* @__PURE__ */ new Set();
|
|
51910
52154
|
const stack = [{ name: start, done: false }];
|
|
51911
52155
|
const visited = /* @__PURE__ */ new Set();
|
|
51912
52156
|
while (stack.length > 0) {
|
|
51913
52157
|
const frame = stack.pop();
|
|
51914
52158
|
if (frame.done) {
|
|
51915
|
-
onPath.delete(
|
|
52159
|
+
onPath.delete(path22.pop());
|
|
51916
52160
|
continue;
|
|
51917
52161
|
}
|
|
51918
52162
|
if (onPath.has(frame.name)) {
|
|
51919
|
-
|
|
52163
|
+
path22.slice(path22.indexOf(frame.name)).forEach((name) => onCycle.add(name));
|
|
51920
52164
|
continue;
|
|
51921
52165
|
}
|
|
51922
52166
|
if (visited.has(frame.name)) continue;
|
|
51923
52167
|
visited.add(frame.name);
|
|
51924
|
-
|
|
52168
|
+
path22.push(frame.name);
|
|
51925
52169
|
onPath.add(frame.name);
|
|
51926
52170
|
stack.push({ name: frame.name, done: true });
|
|
51927
52171
|
const successors = next.get(frame.name) ?? [];
|
|
@@ -52614,8 +52858,8 @@ var init_ResourceView = __esm({
|
|
|
52614
52858
|
if (!source || origin.path.length < 2) return { value: "declared here", muted: true };
|
|
52615
52859
|
return { value: `${source.resource}.${source.field}`, link: source.node };
|
|
52616
52860
|
};
|
|
52617
|
-
this._pathText = (
|
|
52618
|
-
return
|
|
52861
|
+
this._pathText = (path22) => {
|
|
52862
|
+
return path22.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
|
|
52619
52863
|
};
|
|
52620
52864
|
this._consumerOutputsTable = (env2, consumer) => {
|
|
52621
52865
|
const declared = new Set((env2.sources ?? []).map((x2) => x2.name));
|
|
@@ -52839,6 +53083,297 @@ var init_lineage = __esm({
|
|
|
52839
53083
|
}
|
|
52840
53084
|
});
|
|
52841
53085
|
|
|
53086
|
+
// ../../packages/engines/src/usage/LocalUsageDB.ts
|
|
53087
|
+
var import_fs18, import_path17, COLUMNS, CLI_DB_FOLDER, CLI_DB_FILE, CLI_DB_PATH, MAX_USAGE_ROWS, MAX_LISTED_ROWS, EXECUTED_BY_SCHEMA, EXECUTOR_OPERATION_SCHEMA, STATS_SCHEMA, COLUMN_SCHEMAS, LocalUsageDBClass, LocalUsageDB, LocalUsageDB_default;
|
|
53088
|
+
var init_LocalUsageDB = __esm({
|
|
53089
|
+
"../../packages/engines/src/usage/LocalUsageDB.ts"() {
|
|
53090
|
+
init_src();
|
|
53091
|
+
import_fs18 = require("fs");
|
|
53092
|
+
import_path17 = require("path");
|
|
53093
|
+
COLUMNS = [
|
|
53094
|
+
"_id",
|
|
53095
|
+
"consumer",
|
|
53096
|
+
"executedBy",
|
|
53097
|
+
"startedAt",
|
|
53098
|
+
"finishedAt",
|
|
53099
|
+
"itemsCount",
|
|
53100
|
+
"invokedBy",
|
|
53101
|
+
"status",
|
|
53102
|
+
"error",
|
|
53103
|
+
"stats",
|
|
53104
|
+
// Appended last on purpose: rows written before this column existed still decode positionally.
|
|
53105
|
+
"engineVersion"
|
|
53106
|
+
];
|
|
53107
|
+
CLI_DB_FOLDER = "./remora/.temp/usage";
|
|
53108
|
+
CLI_DB_FILE = "remora_usage.csv";
|
|
53109
|
+
CLI_DB_PATH = (0, import_path17.join)(CLI_DB_FOLDER, CLI_DB_FILE);
|
|
53110
|
+
MAX_USAGE_ROWS = 1e4;
|
|
53111
|
+
MAX_LISTED_ROWS = 500;
|
|
53112
|
+
EXECUTED_BY_SCHEMA = {
|
|
53113
|
+
_id: "string",
|
|
53114
|
+
name: "string",
|
|
53115
|
+
type: "string"
|
|
53116
|
+
};
|
|
53117
|
+
EXECUTOR_OPERATION_SCHEMA = {
|
|
53118
|
+
elapsedMS: ["number"],
|
|
53119
|
+
min: "number",
|
|
53120
|
+
max: "number",
|
|
53121
|
+
avg: "number"
|
|
53122
|
+
};
|
|
53123
|
+
STATS_SCHEMA = {
|
|
53124
|
+
executionId: "string",
|
|
53125
|
+
resultUri: "string",
|
|
53126
|
+
elapsedMS: "number",
|
|
53127
|
+
inputCount: "number",
|
|
53128
|
+
outputCount: "number",
|
|
53129
|
+
cycles: "number",
|
|
53130
|
+
workerCount: "number",
|
|
53131
|
+
operations: {
|
|
53132
|
+
"*": EXECUTOR_OPERATION_SCHEMA
|
|
53133
|
+
}
|
|
53134
|
+
};
|
|
53135
|
+
COLUMN_SCHEMAS = {
|
|
53136
|
+
_id: "string",
|
|
53137
|
+
consumer: "string",
|
|
53138
|
+
executedBy: EXECUTED_BY_SCHEMA,
|
|
53139
|
+
startedAt: "date",
|
|
53140
|
+
finishedAt: "date",
|
|
53141
|
+
itemsCount: "number",
|
|
53142
|
+
invokedBy: "string",
|
|
53143
|
+
status: "string",
|
|
53144
|
+
error: "string",
|
|
53145
|
+
stats: STATS_SCHEMA,
|
|
53146
|
+
engineVersion: "string"
|
|
53147
|
+
};
|
|
53148
|
+
LocalUsageDBClass = class {
|
|
53149
|
+
constructor() {
|
|
53150
|
+
this._handleError = (action, error) => {
|
|
53151
|
+
console.error(`LocalUsageDB ${action} failed:`, error);
|
|
53152
|
+
};
|
|
53153
|
+
this._splitSerialized = (value, separator) => {
|
|
53154
|
+
if (!value)
|
|
53155
|
+
return [];
|
|
53156
|
+
const parts = [];
|
|
53157
|
+
let current = "";
|
|
53158
|
+
let quoteOpen = false;
|
|
53159
|
+
let curlyDepth = 0;
|
|
53160
|
+
let squareDepth = 0;
|
|
53161
|
+
for (let index = 0; index < value.length; index++) {
|
|
53162
|
+
const char = value[index];
|
|
53163
|
+
const next = value[index + 1];
|
|
53164
|
+
if (char === '"') {
|
|
53165
|
+
current += char;
|
|
53166
|
+
if (quoteOpen && next === '"') {
|
|
53167
|
+
current += next;
|
|
53168
|
+
index++;
|
|
53169
|
+
continue;
|
|
53170
|
+
}
|
|
53171
|
+
quoteOpen = !quoteOpen;
|
|
53172
|
+
continue;
|
|
53173
|
+
}
|
|
53174
|
+
if (!quoteOpen) {
|
|
53175
|
+
if (char === "{")
|
|
53176
|
+
curlyDepth++;
|
|
53177
|
+
else if (char === "}")
|
|
53178
|
+
curlyDepth--;
|
|
53179
|
+
else if (char === "[")
|
|
53180
|
+
squareDepth++;
|
|
53181
|
+
else if (char === "]")
|
|
53182
|
+
squareDepth--;
|
|
53183
|
+
else if (char === separator && curlyDepth === 0 && squareDepth === 0) {
|
|
53184
|
+
parts.push(current);
|
|
53185
|
+
current = "";
|
|
53186
|
+
continue;
|
|
53187
|
+
}
|
|
53188
|
+
}
|
|
53189
|
+
current += char;
|
|
53190
|
+
}
|
|
53191
|
+
parts.push(current);
|
|
53192
|
+
return parts;
|
|
53193
|
+
};
|
|
53194
|
+
this._splitKeyValue = (value) => {
|
|
53195
|
+
let quoteOpen = false;
|
|
53196
|
+
let curlyDepth = 0;
|
|
53197
|
+
let squareDepth = 0;
|
|
53198
|
+
for (let index = 0; index < value.length; index++) {
|
|
53199
|
+
const char = value[index];
|
|
53200
|
+
const next = value[index + 1];
|
|
53201
|
+
if (char === '"') {
|
|
53202
|
+
if (quoteOpen && next === '"') {
|
|
53203
|
+
index++;
|
|
53204
|
+
continue;
|
|
53205
|
+
}
|
|
53206
|
+
quoteOpen = !quoteOpen;
|
|
53207
|
+
continue;
|
|
53208
|
+
}
|
|
53209
|
+
if (!quoteOpen) {
|
|
53210
|
+
if (char === "{")
|
|
53211
|
+
curlyDepth++;
|
|
53212
|
+
else if (char === "}")
|
|
53213
|
+
curlyDepth--;
|
|
53214
|
+
else if (char === "[")
|
|
53215
|
+
squareDepth++;
|
|
53216
|
+
else if (char === "]")
|
|
53217
|
+
squareDepth--;
|
|
53218
|
+
else if (char === ":" && curlyDepth === 0 && squareDepth === 0)
|
|
53219
|
+
return [value.slice(0, index), value.slice(index + 1)];
|
|
53220
|
+
}
|
|
53221
|
+
}
|
|
53222
|
+
return [value, ""];
|
|
53223
|
+
};
|
|
53224
|
+
this._decodeString = (value) => {
|
|
53225
|
+
if (!value)
|
|
53226
|
+
return "";
|
|
53227
|
+
if (value.startsWith('"') && value.endsWith('"'))
|
|
53228
|
+
return value.slice(1, -1).replaceAll('""', '"');
|
|
53229
|
+
return value;
|
|
53230
|
+
};
|
|
53231
|
+
this._decodeValue = (value, schema) => {
|
|
53232
|
+
if (!value)
|
|
53233
|
+
return null;
|
|
53234
|
+
if (value.startsWith("{") && value.endsWith("}")) {
|
|
53235
|
+
const objectSchema = schema && !Array.isArray(schema) && typeof schema === "object" ? schema : {};
|
|
53236
|
+
const dynamicSchema = objectSchema["*"];
|
|
53237
|
+
const serializedEntries = this._splitSerialized(value.slice(1, -1), ";");
|
|
53238
|
+
const decodedObject = {};
|
|
53239
|
+
serializedEntries.forEach((entry) => {
|
|
53240
|
+
const [rawKey, rawValue] = this._splitKeyValue(entry);
|
|
53241
|
+
const key = this._decodeString(rawKey);
|
|
53242
|
+
decodedObject[key] = this._decodeValue(rawValue, objectSchema[key] ?? dynamicSchema);
|
|
53243
|
+
});
|
|
53244
|
+
return decodedObject;
|
|
53245
|
+
}
|
|
53246
|
+
if (value.startsWith("[") && value.endsWith("]")) {
|
|
53247
|
+
const serializedItems = this._splitSerialized(value.slice(1, -1), ";");
|
|
53248
|
+
const itemSchema = Array.isArray(schema) ? schema[0] : void 0;
|
|
53249
|
+
return serializedItems.map((item) => this._decodeValue(item, itemSchema));
|
|
53250
|
+
}
|
|
53251
|
+
const decodedString = this._decodeString(value);
|
|
53252
|
+
if (schema === "number")
|
|
53253
|
+
return Number(decodedString);
|
|
53254
|
+
if (schema === "date")
|
|
53255
|
+
return decodedString ? new Date(decodedString) : null;
|
|
53256
|
+
return decodedString;
|
|
53257
|
+
};
|
|
53258
|
+
this._encodeValue = (value) => {
|
|
53259
|
+
if (value === null || value === void 0)
|
|
53260
|
+
return "";
|
|
53261
|
+
else if (value instanceof Date)
|
|
53262
|
+
return `"${value.toISOString().replaceAll('"', '""')}"`;
|
|
53263
|
+
else if (Array.isArray(value))
|
|
53264
|
+
return `[${value.map((item) => this._encodeValue(item)).join(";")}]`;
|
|
53265
|
+
else if (typeof value === "object")
|
|
53266
|
+
return `{${Object.keys(value).map((key) => `${this._encodeValue(key)}:${this._encodeValue(value[key])}`).join(";")}}`;
|
|
53267
|
+
else
|
|
53268
|
+
return `"${(value ?? "").toString().replaceAll('"', '""')}"`;
|
|
53269
|
+
};
|
|
53270
|
+
this._toLine = (usage) => {
|
|
53271
|
+
return COLUMNS.map((column) => this._encodeValue(usage[column])).join(",");
|
|
53272
|
+
};
|
|
53273
|
+
this._fromLine = (line) => {
|
|
53274
|
+
const values = this._splitSerialized(line, ",");
|
|
53275
|
+
const usage = {};
|
|
53276
|
+
COLUMNS.forEach((column, index) => {
|
|
53277
|
+
usage[column] = this._decodeValue(values[index] ?? "", COLUMN_SCHEMAS[column]);
|
|
53278
|
+
});
|
|
53279
|
+
return usage;
|
|
53280
|
+
};
|
|
53281
|
+
this._getRows = () => {
|
|
53282
|
+
const lines = (0, import_fs18.readFileSync)(CLI_DB_PATH, "utf8").split("\n").filter(Boolean);
|
|
53283
|
+
lines.shift();
|
|
53284
|
+
return lines;
|
|
53285
|
+
};
|
|
53286
|
+
this._writeRows = (rows) => {
|
|
53287
|
+
const trimmedRows = rows.slice(-MAX_USAGE_ROWS);
|
|
53288
|
+
(0, import_fs18.writeFileSync)(CLI_DB_PATH, `${[COLUMNS.join(","), ...trimmedRows].join("\n")}
|
|
53289
|
+
`);
|
|
53290
|
+
};
|
|
53291
|
+
this.init = () => {
|
|
53292
|
+
try {
|
|
53293
|
+
if (!(0, import_fs18.existsSync)(CLI_DB_FOLDER))
|
|
53294
|
+
(0, import_fs18.mkdirSync)(CLI_DB_FOLDER, { recursive: true });
|
|
53295
|
+
if (!(0, import_fs18.existsSync)(CLI_DB_PATH))
|
|
53296
|
+
(0, import_fs18.writeFileSync)(CLI_DB_PATH, `${COLUMNS.join(",")}
|
|
53297
|
+
`);
|
|
53298
|
+
this._writeRows(this._getRows());
|
|
53299
|
+
} catch (error) {
|
|
53300
|
+
this._handleError("init", error);
|
|
53301
|
+
}
|
|
53302
|
+
};
|
|
53303
|
+
this.insert = (usage) => {
|
|
53304
|
+
try {
|
|
53305
|
+
this.init();
|
|
53306
|
+
this._writeRows([...this._getRows(), this._toLine(usage)]);
|
|
53307
|
+
} catch (error) {
|
|
53308
|
+
this._handleError("insert", error);
|
|
53309
|
+
}
|
|
53310
|
+
};
|
|
53311
|
+
/**
|
|
53312
|
+
* The recorded runs, newest first — the newest `limit` of the rows `keep` accepts.
|
|
53313
|
+
*
|
|
53314
|
+
* Decoded from the end and stopped as soon as that many are held: a reader wants the recent
|
|
53315
|
+
* executions, and decoding ten thousand rows to show ten of them is work nobody asked for.
|
|
53316
|
+
*
|
|
53317
|
+
* `keep` is applied BEFORE the limit, which is the only way "the newest N runs of this kind" can
|
|
53318
|
+
* mean it: filtering afterwards lets the rows a caller does not want displace the ones it does,
|
|
53319
|
+
* so one sample recorded here would silently push a real run out of a history someone is reading.
|
|
53320
|
+
*
|
|
53321
|
+
* Never throws. A missing file reads as no history, and so does a half-written one — this store is
|
|
53322
|
+
* appended to while it is being read, so failing on that would make it unreadable during a run.
|
|
53323
|
+
*/
|
|
53324
|
+
this.list = (limit = MAX_LISTED_ROWS, keep) => {
|
|
53325
|
+
try {
|
|
53326
|
+
if (!(0, import_fs18.existsSync)(CLI_DB_PATH))
|
|
53327
|
+
return [];
|
|
53328
|
+
const rows = this._getRows();
|
|
53329
|
+
const wanted = Math.max(1, limit);
|
|
53330
|
+
const found = [];
|
|
53331
|
+
for (let index = rows.length - 1; index >= 0 && found.length < wanted; index--) {
|
|
53332
|
+
const usage = this._fromLine(rows[index]);
|
|
53333
|
+
if (!keep || keep(usage)) found.push(usage);
|
|
53334
|
+
}
|
|
53335
|
+
return found;
|
|
53336
|
+
} catch (error) {
|
|
53337
|
+
this._handleError("list", error);
|
|
53338
|
+
return [];
|
|
53339
|
+
}
|
|
53340
|
+
};
|
|
53341
|
+
/**
|
|
53342
|
+
* A token that changes whenever the store does, cheap enough to poll: a preview watching for a
|
|
53343
|
+
* finished run must not decode the whole file once a second to notice one.
|
|
53344
|
+
*/
|
|
53345
|
+
this.signature = () => {
|
|
53346
|
+
try {
|
|
53347
|
+
if (!(0, import_fs18.existsSync)(CLI_DB_PATH))
|
|
53348
|
+
return "";
|
|
53349
|
+
const stats = (0, import_fs18.statSync)(CLI_DB_PATH);
|
|
53350
|
+
return `${stats.mtimeMs}:${stats.size}`;
|
|
53351
|
+
} catch (error) {
|
|
53352
|
+
this._handleError("signature", error);
|
|
53353
|
+
return "";
|
|
53354
|
+
}
|
|
53355
|
+
};
|
|
53356
|
+
this.update = (update) => {
|
|
53357
|
+
try {
|
|
53358
|
+
Affirm_default(update?._id, "Invalid usage update _id");
|
|
53359
|
+
this.init();
|
|
53360
|
+
const updatedLines = this._getRows().map((line) => {
|
|
53361
|
+
const current = this._fromLine(line);
|
|
53362
|
+
if (current._id !== update._id) return line;
|
|
53363
|
+
return this._toLine({ ...current, ...update });
|
|
53364
|
+
});
|
|
53365
|
+
this._writeRows(updatedLines);
|
|
53366
|
+
} catch (error) {
|
|
53367
|
+
this._handleError("update", error);
|
|
53368
|
+
}
|
|
53369
|
+
};
|
|
53370
|
+
}
|
|
53371
|
+
};
|
|
53372
|
+
LocalUsageDB = new LocalUsageDBClass();
|
|
53373
|
+
LocalUsageDB_default = LocalUsageDB;
|
|
53374
|
+
}
|
|
53375
|
+
});
|
|
53376
|
+
|
|
52842
53377
|
// ../../packages/database/src/DatabaseEngine.ts
|
|
52843
53378
|
var import_mongodb, DatabaseEngineClass, DatabaseEngine, DatabaseEngine_default;
|
|
52844
53379
|
var init_DatabaseEngine = __esm({
|
|
@@ -53186,6 +53721,481 @@ var init_RunRegistry = __esm({
|
|
|
53186
53721
|
}
|
|
53187
53722
|
});
|
|
53188
53723
|
|
|
53724
|
+
// ../../packages/engines/src/usage/UsageManager.ts
|
|
53725
|
+
var MAX_RECORDED_OPERATIONS, UsageManagerClass, UsageManager, UsageManager_default;
|
|
53726
|
+
var init_UsageManager = __esm({
|
|
53727
|
+
"../../packages/engines/src/usage/UsageManager.ts"() {
|
|
53728
|
+
init_src();
|
|
53729
|
+
init_src3();
|
|
53730
|
+
init_src();
|
|
53731
|
+
init_src4();
|
|
53732
|
+
init_src9();
|
|
53733
|
+
init_src6();
|
|
53734
|
+
init_src6();
|
|
53735
|
+
init_src2();
|
|
53736
|
+
init_RunRegistry();
|
|
53737
|
+
init_LocalUsageDB();
|
|
53738
|
+
MAX_RECORDED_OPERATIONS = 40;
|
|
53739
|
+
UsageManagerClass = class {
|
|
53740
|
+
constructor() {
|
|
53741
|
+
this._shouldSkipDb = () => Helper_default.isDev() || !process.env.MONGO_URI;
|
|
53742
|
+
this._shouldLogCLI = () => process.env.REMORA_RUNTIME_CONTEXT === "cli";
|
|
53743
|
+
/**
|
|
53744
|
+
* Whether this runtime keeps its execution records in the local file rather than in the usage
|
|
53745
|
+
* collection. Public because a READER has to make the same decision the writer just made — two
|
|
53746
|
+
* answers to "where are the runs" is one answer too many.
|
|
53747
|
+
*/
|
|
53748
|
+
this.recordsLocally = () => this._shouldSkipDb() && this._shouldLogCLI();
|
|
53749
|
+
this.startUsage = (consumer, details) => {
|
|
53750
|
+
const { user, invokedBy } = details;
|
|
53751
|
+
const newUsage = {
|
|
53752
|
+
_id: Helper_default.uuid(),
|
|
53753
|
+
consumer: consumer.name,
|
|
53754
|
+
engineVersion: Constants_default.cliVersion,
|
|
53755
|
+
startedAt: DSTE_default.now(),
|
|
53756
|
+
executedBy: user,
|
|
53757
|
+
itemsCount: -1,
|
|
53758
|
+
status: "started",
|
|
53759
|
+
invokedBy: invokedBy ?? "UNKNOWN",
|
|
53760
|
+
_signature: "",
|
|
53761
|
+
stats: null
|
|
53762
|
+
};
|
|
53763
|
+
RunRegistry_default.start({
|
|
53764
|
+
executionId: newUsage._id,
|
|
53765
|
+
consumer: newUsage.consumer,
|
|
53766
|
+
invokedBy: newUsage.invokedBy,
|
|
53767
|
+
userType: user?.type ?? "UNKNOWN",
|
|
53768
|
+
startedAt: newUsage.startedAt
|
|
53769
|
+
});
|
|
53770
|
+
if (this._shouldSkipDb()) {
|
|
53771
|
+
if (this._shouldLogCLI())
|
|
53772
|
+
LocalUsageDB_default.insert(newUsage);
|
|
53773
|
+
return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
|
|
53774
|
+
}
|
|
53775
|
+
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, newUsage._id, newUsage);
|
|
53776
|
+
return { usageId: newUsage._id, usage: updateRes };
|
|
53777
|
+
};
|
|
53778
|
+
this.endUsage = (usageId, itemsCount, stats) => {
|
|
53779
|
+
const update = {
|
|
53780
|
+
itemsCount,
|
|
53781
|
+
status: "success",
|
|
53782
|
+
finishedAt: DSTE_default.now(),
|
|
53783
|
+
stats
|
|
53784
|
+
};
|
|
53785
|
+
this._emitRunRecord(usageId, "success", stats, null);
|
|
53786
|
+
if (this._shouldSkipDb()) {
|
|
53787
|
+
if (this._shouldLogCLI())
|
|
53788
|
+
LocalUsageDB_default.update({ ...update, _id: usageId });
|
|
53789
|
+
return { usageId: null, usage: Promise.resolve(update) };
|
|
53790
|
+
}
|
|
53791
|
+
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, usageId, update);
|
|
53792
|
+
return { usageId, usage: updateRes };
|
|
53793
|
+
};
|
|
53794
|
+
this.failUsage = (usageId, error) => {
|
|
53795
|
+
const update = {
|
|
53796
|
+
status: "failed",
|
|
53797
|
+
error,
|
|
53798
|
+
finishedAt: DSTE_default.now()
|
|
53799
|
+
};
|
|
53800
|
+
this._emitRunRecord(usageId, "failed", null, error);
|
|
53801
|
+
if (this._shouldSkipDb()) {
|
|
53802
|
+
if (this._shouldLogCLI())
|
|
53803
|
+
LocalUsageDB_default.update({ ...update, _id: usageId });
|
|
53804
|
+
return { usageId: null, usage: Promise.resolve(update) };
|
|
53805
|
+
}
|
|
53806
|
+
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, usageId, update);
|
|
53807
|
+
return { usageId, usage: updateRes };
|
|
53808
|
+
};
|
|
53809
|
+
/**
|
|
53810
|
+
* Close out the in-flight entry and write one structured record for the finished run.
|
|
53811
|
+
*
|
|
53812
|
+
* This is the cheapest useful thing in the whole ops surface: the executor already hands us the
|
|
53813
|
+
* complete result, so a single hook makes every run queryable in the log pipeline without a
|
|
53814
|
+
* database. Never throws — reporting on a run must not be able to fail the run.
|
|
53815
|
+
*/
|
|
53816
|
+
this._emitRunRecord = (usageId, status, stats, error) => {
|
|
53817
|
+
Affirm_default(usageId, "Missing usage id");
|
|
53818
|
+
try {
|
|
53819
|
+
const run = RunRegistry_default.finish(usageId);
|
|
53820
|
+
const record = {
|
|
53821
|
+
executionId: usageId,
|
|
53822
|
+
consumer: run?.consumer ?? null,
|
|
53823
|
+
engineVersion: Constants_default.cliVersion,
|
|
53824
|
+
project: Environment_default._env ? Environment_default.getProjectName() : null,
|
|
53825
|
+
invokedBy: run?.invokedBy ?? null,
|
|
53826
|
+
userType: run?.userType ?? null,
|
|
53827
|
+
status,
|
|
53828
|
+
startedAt: run?.startedAt?.toISOString() ?? null,
|
|
53829
|
+
elapsedMS: stats?.elapsedMS ?? (run ? DSTE_default.now().getTime() - run.startedAt.getTime() : -1),
|
|
53830
|
+
inputCount: stats?.inputCount ?? -1,
|
|
53831
|
+
outputCount: stats?.outputCount ?? -1,
|
|
53832
|
+
cycles: stats?.cycles ?? -1,
|
|
53833
|
+
workerCount: stats?.workerCount ?? -1,
|
|
53834
|
+
operations: this._summarizeOperations(stats),
|
|
53835
|
+
error: error ?? null
|
|
53836
|
+
};
|
|
53837
|
+
StateEmitter_default.emit("run", record);
|
|
53838
|
+
} catch {
|
|
53839
|
+
}
|
|
53840
|
+
};
|
|
53841
|
+
/**
|
|
53842
|
+
* Keep only min/max/avg per stage. The raw `elapsedMS` array holds one entry per worker chunk
|
|
53843
|
+
* and can run to thousands of numbers on a large run — useless in a log line and unbounded.
|
|
53844
|
+
*/
|
|
53845
|
+
this._summarizeOperations = (stats) => {
|
|
53846
|
+
const operations = {};
|
|
53847
|
+
if (!stats?.operations)
|
|
53848
|
+
return operations;
|
|
53849
|
+
for (const name of Object.keys(stats.operations).slice(0, MAX_RECORDED_OPERATIONS)) {
|
|
53850
|
+
const operation2 = stats.operations[name];
|
|
53851
|
+
operations[name] = { min: operation2.min, max: operation2.max, avg: operation2.avg };
|
|
53852
|
+
}
|
|
53853
|
+
return operations;
|
|
53854
|
+
};
|
|
53855
|
+
}
|
|
53856
|
+
};
|
|
53857
|
+
UsageManager = new UsageManagerClass();
|
|
53858
|
+
UsageManager_default = UsageManager;
|
|
53859
|
+
}
|
|
53860
|
+
});
|
|
53861
|
+
|
|
53862
|
+
// ../../packages/engines/src/ops/BugReportBundle.ts
|
|
53863
|
+
var import_crypto7, import_fs19, import_adm_zip3, MAX_ENTRIES, BugReportBundle, BugReportBundle_default;
|
|
53864
|
+
var init_BugReportBundle = __esm({
|
|
53865
|
+
"../../packages/engines/src/ops/BugReportBundle.ts"() {
|
|
53866
|
+
import_crypto7 = __toESM(require("crypto"), 1);
|
|
53867
|
+
import_fs19 = __toESM(require("fs"), 1);
|
|
53868
|
+
import_adm_zip3 = __toESM(require("adm-zip"), 1);
|
|
53869
|
+
init_src();
|
|
53870
|
+
init_src2();
|
|
53871
|
+
MAX_ENTRIES = 2e3;
|
|
53872
|
+
BugReportBundle = class {
|
|
53873
|
+
constructor() {
|
|
53874
|
+
this._zip = new import_adm_zip3.default();
|
|
53875
|
+
this._sections = /* @__PURE__ */ new Map();
|
|
53876
|
+
/**
|
|
53877
|
+
* Run one section's collection, and record what came of it.
|
|
53878
|
+
*
|
|
53879
|
+
* The section decides what it writes by calling `add`; this only decides what happens when it
|
|
53880
|
+
* cannot. A thrown error becomes the section's `reason` — which is more useful in the bundle
|
|
53881
|
+
* than it would have been on someone's terminal.
|
|
53882
|
+
*/
|
|
53883
|
+
this.collect = (name, collector) => {
|
|
53884
|
+
Affirm_default(name, "Missing required parameter name");
|
|
53885
|
+
Affirm_default(collector, "Missing required parameter collector");
|
|
53886
|
+
this._section(name);
|
|
53887
|
+
try {
|
|
53888
|
+
collector();
|
|
53889
|
+
} catch (error) {
|
|
53890
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
53891
|
+
this.skip(name, `Collection failed: ${reason}`);
|
|
53892
|
+
Logger_default.warn(`Bug report section "${name}" could not be collected: ${reason}`);
|
|
53893
|
+
}
|
|
53894
|
+
};
|
|
53895
|
+
/** Write one file into the bundle, under the section that produced it. */
|
|
53896
|
+
this.add = (name, path22, content) => {
|
|
53897
|
+
Affirm_default(path22, "Missing required parameter path");
|
|
53898
|
+
Affirm_default(content !== void 0 && content !== null, `No content for bundle entry ${path22}`);
|
|
53899
|
+
const section = this._section(name);
|
|
53900
|
+
Affirm_default(
|
|
53901
|
+
this._zip.getEntries().length < MAX_ENTRIES,
|
|
53902
|
+
`Bug report reached its ${MAX_ENTRIES} file limit while writing ${path22}`
|
|
53903
|
+
);
|
|
53904
|
+
this._zip.addFile(path22, Buffer.isBuffer(content) ? content : Buffer.from(content, "utf8"));
|
|
53905
|
+
section.included = true;
|
|
53906
|
+
section.files.push(path22);
|
|
53907
|
+
};
|
|
53908
|
+
/** Record that a section contributed nothing, and why. Absence without a reason reads as a bug. */
|
|
53909
|
+
this.skip = (name, reason) => {
|
|
53910
|
+
Affirm_default(reason, `No reason given for skipping bug report section ${name}`);
|
|
53911
|
+
const section = this._section(name);
|
|
53912
|
+
section.included = section.files.length > 0;
|
|
53913
|
+
section.reason = reason;
|
|
53914
|
+
};
|
|
53915
|
+
/** Whether a section wrote anything — the condition `report.md` renders its inventory from. */
|
|
53916
|
+
this.has = (name) => this._section(name).included;
|
|
53917
|
+
/** Every section, in the order they were collected. */
|
|
53918
|
+
this.sections = () => [...this._sections.values()];
|
|
53919
|
+
/** Every entry with its size and digest, so a reader can tell a truncated bundle from a short one. */
|
|
53920
|
+
this.files = () => this._zip.getEntries().map((entry) => {
|
|
53921
|
+
const data = entry.getData();
|
|
53922
|
+
return {
|
|
53923
|
+
path: entry.entryName,
|
|
53924
|
+
bytes: data.length,
|
|
53925
|
+
sha256: import_crypto7.default.createHash("sha256").update(data).digest("hex")
|
|
53926
|
+
};
|
|
53927
|
+
});
|
|
53928
|
+
/**
|
|
53929
|
+
* Write the archive and return its size on disk. The manifest is added here rather than by a
|
|
53930
|
+
* section, because it inventories every other file and so has to be the last one in.
|
|
53931
|
+
*/
|
|
53932
|
+
this.write = (outputPath, manifest) => {
|
|
53933
|
+
Affirm_default(outputPath, "Missing required parameter outputPath");
|
|
53934
|
+
Affirm_default(manifest, "Missing required parameter manifest");
|
|
53935
|
+
this._zip.addFile("manifest.json", Buffer.from(JSON.stringify(manifest, null, 4), "utf8"));
|
|
53936
|
+
this._zip.writeZip(outputPath);
|
|
53937
|
+
return import_fs19.default.statSync(outputPath).size;
|
|
53938
|
+
};
|
|
53939
|
+
this._section = (name) => {
|
|
53940
|
+
if (!this._sections.has(name))
|
|
53941
|
+
this._sections.set(name, { name, included: false, files: [] });
|
|
53942
|
+
return this._sections.get(name);
|
|
53943
|
+
};
|
|
53944
|
+
}
|
|
53945
|
+
};
|
|
53946
|
+
BugReportBundle_default = BugReportBundle;
|
|
53947
|
+
}
|
|
53948
|
+
});
|
|
53949
|
+
|
|
53950
|
+
// ../../packages/engines/src/ops/BugReportRenderer.ts
|
|
53951
|
+
var QUOTED_ERRORS, LISTED_OPERATIONS, LISTED_PROBLEMS, LISTED_FILES, BugReportRendererClass, BugReportRenderer, BugReportRenderer_default;
|
|
53952
|
+
var init_BugReportRenderer = __esm({
|
|
53953
|
+
"../../packages/engines/src/ops/BugReportRenderer.ts"() {
|
|
53954
|
+
init_src();
|
|
53955
|
+
init_src6();
|
|
53956
|
+
QUOTED_ERRORS = 3;
|
|
53957
|
+
LISTED_OPERATIONS = 8;
|
|
53958
|
+
LISTED_PROBLEMS = 20;
|
|
53959
|
+
LISTED_FILES = 3;
|
|
53960
|
+
BugReportRendererClass = class {
|
|
53961
|
+
constructor() {
|
|
53962
|
+
/** The one page a dev reads. Markdown, because it renders wherever a ticket does. */
|
|
53963
|
+
this.report = (page) => {
|
|
53964
|
+
Affirm_default(page, "Missing required parameter page");
|
|
53965
|
+
Affirm_default(page.environment, "Missing environment in bug report page");
|
|
53966
|
+
return [
|
|
53967
|
+
this._header(page),
|
|
53968
|
+
this._summary(page),
|
|
53969
|
+
this._failure(page),
|
|
53970
|
+
this._pipeline(page),
|
|
53971
|
+
this._problems(page.environment.config.problems),
|
|
53972
|
+
this._timings(page.run),
|
|
53973
|
+
this._reproduction(page),
|
|
53974
|
+
this._inventory(page.sections)
|
|
53975
|
+
].filter(Boolean).join("\n\n") + "\n";
|
|
53976
|
+
};
|
|
53977
|
+
/**
|
|
53978
|
+
* The commands that reproduce the case, as POSIX sh.
|
|
53979
|
+
*
|
|
53980
|
+
* TEXT, never a file in the bundle. An executable inside an archive that travels by mail or
|
|
53981
|
+
* ticket is what a gateway or an endpoint scanner quarantines, and a bundle that does not
|
|
53982
|
+
* arrive reproduces nothing — so `report.md` carries this in a fenced block and the reader
|
|
53983
|
+
* pastes it. It stays one definition rather than prose that drifts from what actually works,
|
|
53984
|
+
* and it is still the dev tool it always was: save it as a file and run it.
|
|
53985
|
+
*/
|
|
53986
|
+
this.repro = (repro) => {
|
|
53987
|
+
Affirm_default(repro, "Missing required parameter repro");
|
|
53988
|
+
const consumer = repro.consumer ?? "<consumer>";
|
|
53989
|
+
return [
|
|
53990
|
+
"# Run from the directory the bundle was unpacked into, with a valid",
|
|
53991
|
+
"# REMORA_LICENCE_KEY in the environment.",
|
|
53992
|
+
"#",
|
|
53993
|
+
"# The input is SYNTHETIC: no row of the customer's data is in this bundle. The seed is",
|
|
53994
|
+
"# fixed, so the same input is generated here and there \u2014 it reproduces the pipeline,",
|
|
53995
|
+
"# not the values. Ask for a --capture bundle when the bug depends on what the data was.",
|
|
53996
|
+
"set -e",
|
|
53997
|
+
"",
|
|
53998
|
+
"# 1. The configuration is the project",
|
|
53999
|
+
"mkdir -p repro",
|
|
54000
|
+
"cp -r config repro/remora",
|
|
54001
|
+
"cd repro",
|
|
54002
|
+
"",
|
|
54003
|
+
"# 2. A placeholder for every {ENV_VAR} the configuration names.",
|
|
54004
|
+
"# `compile` refuses to run while a reference is unset, and the config came from a",
|
|
54005
|
+
"# machine that had them all \u2014 so without this the repro stops before it starts.",
|
|
54006
|
+
"# Placeholders are enough because `synth` contacts no source; put the real values",
|
|
54007
|
+
"# here before uncommenting the `run` at the end. Uppercase only, which is the",
|
|
54008
|
+
"# environment convention and keeps a producer's own {field}/{index} placeholders",
|
|
54009
|
+
"# out of a file that would otherwise claim the project needs a variable named key.",
|
|
54010
|
+
"grep -rho '{[A-Z][A-Z0-9_]*}' remora/ | tr -d '{}' | sort -u \\",
|
|
54011
|
+
" | sed 's/$/=placeholder/' > .env || true",
|
|
54012
|
+
"",
|
|
54013
|
+
"# 3. The local paths the configuration points at, which were someone else's machine.",
|
|
54014
|
+
"# A destination directory that does not exist fails the export AFTER the run, which",
|
|
54015
|
+
"# reads as a second bug and is not one.",
|
|
54016
|
+
`grep -rho '"path"[[:space:]]*:[[:space:]]*"[^"]*"' remora/sources/ 2>/dev/null \\`,
|
|
54017
|
+
` | sed 's/.*:[[:space:]]*"//; s/"$//' | sort -u \\`,
|
|
54018
|
+
' | while read -r directory; do mkdir -p "$directory" || true; done',
|
|
54019
|
+
"",
|
|
54020
|
+
"# 4. What the configuration says it is",
|
|
54021
|
+
"remora compile",
|
|
54022
|
+
"",
|
|
54023
|
+
"# 5. The failing pipeline, on generated input",
|
|
54024
|
+
`remora synth ${consumer} --records ${repro.records} --seed ${repro.seed}`,
|
|
54025
|
+
"",
|
|
54026
|
+
"# 6. The real thing, once the sources are reachable and the values above are real",
|
|
54027
|
+
`# remora run ${consumer}`,
|
|
54028
|
+
""
|
|
54029
|
+
].join("\n");
|
|
54030
|
+
};
|
|
54031
|
+
this._header = (page) => {
|
|
54032
|
+
const redaction = page.repro.syntheticData ? "no customer data" : "masked customer data";
|
|
54033
|
+
return [
|
|
54034
|
+
"# Remora bug report",
|
|
54035
|
+
"",
|
|
54036
|
+
`Bundle \`${page.bundleId}\` \xB7 collected ${page.createdAt} \xB7 remora ${page.remoraVersion} \xB7 ${redaction}`
|
|
54037
|
+
].join("\n");
|
|
54038
|
+
};
|
|
54039
|
+
this._summary = (page) => {
|
|
54040
|
+
const { remora, platform: platform2, node, config } = page.environment;
|
|
54041
|
+
const run = page.run;
|
|
54042
|
+
const rows = [
|
|
54043
|
+
["Project", `${config.project ?? "(unreadable project.json)"} \`${config.signature ?? "\u2014"}\``],
|
|
54044
|
+
["Consumer", run?.consumer ? `\`${run.consumer}\`` : "none \u2014 no run history"],
|
|
54045
|
+
["Run", run ? `\`${run._id}\` \u2014 **${run.status}**${this._elapsed(run)}` : "none \u2014 no run history"],
|
|
54046
|
+
["Started", run?.startedAt ? this._iso(run.startedAt) : "\u2014"],
|
|
54047
|
+
["Rows", run ? `${this._count(run.stats?.inputCount)} in \u2192 ${this._count(run.itemsCount)} out` : "\u2014"],
|
|
54048
|
+
["Engine", `${run?.engineVersion ?? "unknown"} (collected by ${page.remoraVersion})`],
|
|
54049
|
+
["Machine", `${platform2.hostname ?? "unknown host"} \u2014 ${platform2.osType ?? platform2.platform} ${platform2.release}, ${platform2.arch}`],
|
|
54050
|
+
["CPU / RAM", `${platform2.cpuCount} \xD7 ${platform2.cpuModel ?? "unknown"}, ${platform2.totalMemoryMB} MB total (${platform2.freeMemoryMB} MB free at collection)`],
|
|
54051
|
+
["Time zone", `${platform2.timezone ?? "unknown"} \u2014 a datetime read here was read in this zone`],
|
|
54052
|
+
["Node", `${node.version} (v8 ${node.v8}, icu ${node.icu}, zlib ${node.zlib}, openssl ${node.openssl})`],
|
|
54053
|
+
["Heap", `${platform2.heapLimitMB} MB limit, ${remora.heapUsedMB} MB in use at collection`],
|
|
54054
|
+
["Runtime", this._settings(node)],
|
|
54055
|
+
["Config", config.loaded ? `${config.counts.sources} sources, ${config.counts.producers} producers, ${config.counts.consumers} consumers` : "**did not load** \u2014 see Configuration problems"]
|
|
54056
|
+
];
|
|
54057
|
+
return ["## Summary", "", ...rows.map(([key, value]) => `- **${key}** \u2014 ${value}`)].join("\n");
|
|
54058
|
+
};
|
|
54059
|
+
/**
|
|
54060
|
+
* The behaviour-changing variables, as `NAME=value`. Unset is itself the answer much of the
|
|
54061
|
+
* time — a worker with no `REMORA_MAX_WORKERS` sized itself from memory — so it says so.
|
|
54062
|
+
*/
|
|
54063
|
+
this._settings = (node) => {
|
|
54064
|
+
const entries = Object.entries(node.settings ?? {});
|
|
54065
|
+
if (!entries.length)
|
|
54066
|
+
return "no behaviour-changing variables set \u2014 every default in force";
|
|
54067
|
+
return entries.map(([name, value]) => `\`${name}=${value}\``).join(", ");
|
|
54068
|
+
};
|
|
54069
|
+
this._failure = (page) => {
|
|
54070
|
+
const error = page.run?.error;
|
|
54071
|
+
const logged = page.run?.status === "success" ? [] : page.log?.errors?.slice(-QUOTED_ERRORS) ?? [];
|
|
54072
|
+
if (!error && !logged.length)
|
|
54073
|
+
return page.run ? `## Failure
|
|
54074
|
+
|
|
54075
|
+
The run reported no error \u2014 it finished **${page.run.status}**.` : null;
|
|
54076
|
+
const parts = ["## Failure", ""];
|
|
54077
|
+
if (error)
|
|
54078
|
+
parts.push("```", error, "```");
|
|
54079
|
+
if (logged.length)
|
|
54080
|
+
parts.push("", `Last ${logged.length} error record${logged.length === 1 ? "" : "s"} in the log:`, "", "```", logged.join("\n"), "```");
|
|
54081
|
+
return parts.join("\n");
|
|
54082
|
+
};
|
|
54083
|
+
/**
|
|
54084
|
+
* What the run was made of — consumer, producers, sources and engines. The question behind
|
|
54085
|
+
* nearly every report is "what was it talking to", and the answer is three joins away in
|
|
54086
|
+
* config/, so it is resolved here instead.
|
|
54087
|
+
*/
|
|
54088
|
+
this._pipeline = (page) => {
|
|
54089
|
+
const { sources, producers, consumers } = page.environment.config;
|
|
54090
|
+
const consumer = consumers.find((x2) => x2.name === page.run?.consumer);
|
|
54091
|
+
if (!consumer)
|
|
54092
|
+
return sources.length ? ["## Pipeline", "", ...sources.map((x2) => `- source **${x2.name}** \u2014 \`${x2.engine}\`, auth \`${x2.authMethod ?? "none"}\``)].join("\n") : null;
|
|
54093
|
+
const lines = [`- consumer **${consumer.name}** \u2192 ${consumer.outputs.join(", ") || "no outputs"}`];
|
|
54094
|
+
for (const name of consumer.producers) {
|
|
54095
|
+
const producer = producers.find((x2) => x2.name === name);
|
|
54096
|
+
if (!producer) {
|
|
54097
|
+
lines.push(` - producer **${name}** \u2014 *not found in the configuration*`);
|
|
54098
|
+
continue;
|
|
54099
|
+
}
|
|
54100
|
+
const source = sources.find((x2) => x2.name === producer.source);
|
|
54101
|
+
const described = source ? `source **${source.name}** (\`${source.engine}\`, auth \`${source.authMethod ?? "none"}\`)` : `source **${producer.source}** \u2014 *not found in the configuration*`;
|
|
54102
|
+
lines.push(` - producer **${producer.name}** (${producer.dimensionCount} dimensions) \u2190 ${described}`);
|
|
54103
|
+
}
|
|
54104
|
+
return ["## Pipeline", "", ...lines].join("\n");
|
|
54105
|
+
};
|
|
54106
|
+
this._problems = (problems) => {
|
|
54107
|
+
if (!problems?.length)
|
|
54108
|
+
return "## Configuration problems\n\nNone \u2014 the project read cleanly.";
|
|
54109
|
+
const listed = problems.slice(0, LISTED_PROBLEMS).map((problem) => {
|
|
54110
|
+
const where = problem.resource ? `${problem.resource.kind} \`${problem.resource.name}\`` : problem.file ?? problem.scope;
|
|
54111
|
+
return `- **${problem.severity}** ${where} \u2014 ${problem.message}`;
|
|
54112
|
+
});
|
|
54113
|
+
if (problems.length > LISTED_PROBLEMS)
|
|
54114
|
+
listed.push(`- \u2026and ${problems.length - LISTED_PROBLEMS} more`);
|
|
54115
|
+
return ["## Configuration problems", "", ...listed].join("\n");
|
|
54116
|
+
};
|
|
54117
|
+
this._timings = (run) => {
|
|
54118
|
+
const operations = Object.entries(run?.stats?.operations ?? {});
|
|
54119
|
+
if (!operations.length)
|
|
54120
|
+
return null;
|
|
54121
|
+
const rows = operations.sort((left, right) => (right[1].max ?? 0) - (left[1].max ?? 0)).slice(0, LISTED_OPERATIONS).map(([name, timing2]) => `| \`${name}\` | ${timing2.elapsedMS?.length ?? 0} | ${Helper_default.formatDuration(timing2.min)} | ${Helper_default.formatDuration(timing2.avg)} | ${Helper_default.formatDuration(timing2.max)} |`);
|
|
54122
|
+
return [
|
|
54123
|
+
"## Stage timings",
|
|
54124
|
+
"",
|
|
54125
|
+
"| Stage | Calls | Min | Avg | Max |",
|
|
54126
|
+
"|---|---|---|---|---|",
|
|
54127
|
+
...rows
|
|
54128
|
+
].join("\n");
|
|
54129
|
+
};
|
|
54130
|
+
/**
|
|
54131
|
+
* The reproduction, as a block to paste rather than a script to run.
|
|
54132
|
+
*
|
|
54133
|
+
* The bundle deliberately contains no executable — see `repro()` — so these are the same
|
|
54134
|
+
* commands, here, where a scanner has nothing to quarantine and a reader has nothing to open.
|
|
54135
|
+
*/
|
|
54136
|
+
this._reproduction = (page) => {
|
|
54137
|
+
return [
|
|
54138
|
+
"## Reproduce",
|
|
54139
|
+
"",
|
|
54140
|
+
"Unpack the bundle, then run this from the directory it unpacked into. Paste it, or save",
|
|
54141
|
+
"it as a file and run that \u2014 the bundle ships no script of its own, so that nothing in it",
|
|
54142
|
+
"is executable.",
|
|
54143
|
+
"",
|
|
54144
|
+
"```sh",
|
|
54145
|
+
this.repro(page.repro).trimEnd(),
|
|
54146
|
+
"```",
|
|
54147
|
+
"",
|
|
54148
|
+
"The input is generated, not captured. It reproduces every bug in the SHAPE of the",
|
|
54149
|
+
"pipeline \u2014 joins, pivots, transformations, validation \u2014 and none that depend on what a",
|
|
54150
|
+
"value was. The seed is derived from the bundle id, so it has no relation to the failing",
|
|
54151
|
+
"run beyond being the same for whoever opens this bundle."
|
|
54152
|
+
].join("\n");
|
|
54153
|
+
};
|
|
54154
|
+
this._inventory = (sections) => {
|
|
54155
|
+
const rows = sections.filter((section) => section.name !== "report").map((section) => `| ${section.name} | ${section.included ? "\u2705" : "\u2014"} | ${this._describe(section)} |`);
|
|
54156
|
+
return [
|
|
54157
|
+
"## In this bundle",
|
|
54158
|
+
"",
|
|
54159
|
+
"| Section | Present | Files |",
|
|
54160
|
+
"|---|---|---|",
|
|
54161
|
+
...rows,
|
|
54162
|
+
"",
|
|
54163
|
+
"The configuration travels as authored \u2014 remora config names its credentials (`{ENV_VAR}`)",
|
|
54164
|
+
"rather than holding them, so what is in `config/` is what runs. The log is scrubbed of",
|
|
54165
|
+
"credential shapes (JWTs, access key ids, URL query strings), since a value that reached a",
|
|
54166
|
+
"log line had already been resolved. No row of customer data is in this bundle."
|
|
54167
|
+
].join("\n");
|
|
54168
|
+
};
|
|
54169
|
+
/**
|
|
54170
|
+
* What a section contributed, and what it left out. Past a handful of files the list stops
|
|
54171
|
+
* being readable and becomes a count — the whole of `config/` is seventy file names, which
|
|
54172
|
+
* tells a reader nothing the directory does not.
|
|
54173
|
+
*/
|
|
54174
|
+
this._describe = (section) => {
|
|
54175
|
+
const listed = section.files.length <= LISTED_FILES ? section.files.map((file) => `\`${file}\``).join(", ") : `${section.files.length} files under \`${section.files[0].split("/")[0]}/\``;
|
|
54176
|
+
return [listed, section.reason].filter(Boolean).join(" \u2014 ") || "\u2014";
|
|
54177
|
+
};
|
|
54178
|
+
this._elapsed = (run) => {
|
|
54179
|
+
const started = run.startedAt ? new Date(run.startedAt).getTime() : 0;
|
|
54180
|
+
const finished = run.finishedAt ? new Date(run.finishedAt).getTime() : 0;
|
|
54181
|
+
if (run.stats?.elapsedMS > 0)
|
|
54182
|
+
return ` after ${Helper_default.formatDuration(run.stats.elapsedMS)}`;
|
|
54183
|
+
if (started && finished)
|
|
54184
|
+
return ` after ${Helper_default.formatDuration(finished - started)}`;
|
|
54185
|
+
return run.status === "started" ? " \u2014 never reported a result" : "";
|
|
54186
|
+
};
|
|
54187
|
+
this._count = (value) => value === void 0 || value === null || value < 0 ? "unknown" : String(value);
|
|
54188
|
+
this._iso = (value) => {
|
|
54189
|
+
const date2 = value instanceof Date ? value : new Date(value);
|
|
54190
|
+
return isNaN(date2.getTime()) ? String(value) : date2.toISOString();
|
|
54191
|
+
};
|
|
54192
|
+
}
|
|
54193
|
+
};
|
|
54194
|
+
BugReportRenderer = new BugReportRendererClass();
|
|
54195
|
+
BugReportRenderer_default = BugReportRenderer;
|
|
54196
|
+
}
|
|
54197
|
+
});
|
|
54198
|
+
|
|
53189
54199
|
// ../../packages/engines/src/ops/OpsService.ts
|
|
53190
54200
|
var MAX_HISTORY_LIMIT, DEFAULT_HISTORY_LIMIT, MAX_LISTED_RESOURCES, BYTES_PER_MB, OpsServiceClass, OpsService, OpsService_default;
|
|
53191
54201
|
var init_OpsService = __esm({
|
|
@@ -53370,6 +54380,390 @@ var init_OpsService = __esm({
|
|
|
53370
54380
|
}
|
|
53371
54381
|
});
|
|
53372
54382
|
|
|
54383
|
+
// ../../packages/engines/src/ops/RunLogSlicer.ts
|
|
54384
|
+
var RECORD_HEAD, MAX_INPUT_CHARS, RunLogSlicerClass, RunLogSlicer, RunLogSlicer_default;
|
|
54385
|
+
var init_RunLogSlicer = __esm({
|
|
54386
|
+
"../../packages/engines/src/ops/RunLogSlicer.ts"() {
|
|
54387
|
+
init_src();
|
|
54388
|
+
RECORD_HEAD = /^\[([^\]]+)\] \[([A-Z]+)\] ?/;
|
|
54389
|
+
MAX_INPUT_CHARS = 16e6;
|
|
54390
|
+
RunLogSlicerClass = class {
|
|
54391
|
+
constructor() {
|
|
54392
|
+
/**
|
|
54393
|
+
* The records belonging to one run, oldest first.
|
|
54394
|
+
*
|
|
54395
|
+
* A record is kept when it carries the run id OR falls inside the window. With no `to` — a run
|
|
54396
|
+
* whose process died before it could record one — the window stays open to the end of the file
|
|
54397
|
+
* and `maxRecords` is what bounds the answer, which is the honest shape of "we do not know when
|
|
54398
|
+
* this stopped": take the tail of what followed the start.
|
|
54399
|
+
*
|
|
54400
|
+
* With neither a run id nor a `from`, every record matches and the result is the tail of the
|
|
54401
|
+
* file. That is the degraded bundle, where there is no run history to slice by.
|
|
54402
|
+
*/
|
|
54403
|
+
this.slice = (request) => {
|
|
54404
|
+
Affirm_default(request, "Missing required parameter request");
|
|
54405
|
+
Affirm_default(request.maxRecords > 0, "Log slice maxRecords must be greater than zero");
|
|
54406
|
+
const records = this._parse(this._bound(request.text));
|
|
54407
|
+
const matched = records.filter((record) => this._matches(record, request));
|
|
54408
|
+
const kept = matched.slice(-request.maxRecords);
|
|
54409
|
+
return {
|
|
54410
|
+
text: kept.map((record) => record.text).join("\n"),
|
|
54411
|
+
kept: kept.length,
|
|
54412
|
+
scanned: records.length,
|
|
54413
|
+
dropped: matched.length - kept.length,
|
|
54414
|
+
errors: kept.filter((record) => record.level === "ERROR").map((record) => record.text)
|
|
54415
|
+
};
|
|
54416
|
+
};
|
|
54417
|
+
/** The tail of the text, so a file larger than the cap is read from its most recent end. */
|
|
54418
|
+
this._bound = (text) => {
|
|
54419
|
+
if (!text) return "";
|
|
54420
|
+
return text.length <= MAX_INPUT_CHARS ? text : text.slice(-MAX_INPUT_CHARS);
|
|
54421
|
+
};
|
|
54422
|
+
/**
|
|
54423
|
+
* Lines folded back into records. A continuation line belongs to the record above it; lines
|
|
54424
|
+
* before the first head — the tail cut above can produce them — are dropped, since they belong
|
|
54425
|
+
* to a record whose timestamp was cut off with it.
|
|
54426
|
+
*/
|
|
54427
|
+
this._parse = (text) => {
|
|
54428
|
+
const records = [];
|
|
54429
|
+
for (const line of text.split("\n")) {
|
|
54430
|
+
const head = RECORD_HEAD.exec(line);
|
|
54431
|
+
if (!head) {
|
|
54432
|
+
if (records.length) records[records.length - 1].text += `
|
|
54433
|
+
${line}`;
|
|
54434
|
+
continue;
|
|
54435
|
+
}
|
|
54436
|
+
const at3 = new Date(head[1]);
|
|
54437
|
+
records.push({ at: isNaN(at3.getTime()) ? null : at3, level: head[2], text: line });
|
|
54438
|
+
}
|
|
54439
|
+
return records;
|
|
54440
|
+
};
|
|
54441
|
+
this._matches = (record, request) => {
|
|
54442
|
+
if (request.runId && record.text.includes(request.runId))
|
|
54443
|
+
return true;
|
|
54444
|
+
if (!request.from)
|
|
54445
|
+
return true;
|
|
54446
|
+
if (!record.at)
|
|
54447
|
+
return false;
|
|
54448
|
+
return record.at >= request.from && (!request.to || record.at <= request.to);
|
|
54449
|
+
};
|
|
54450
|
+
}
|
|
54451
|
+
};
|
|
54452
|
+
RunLogSlicer = new RunLogSlicerClass();
|
|
54453
|
+
RunLogSlicer_default = RunLogSlicer;
|
|
54454
|
+
}
|
|
54455
|
+
});
|
|
54456
|
+
|
|
54457
|
+
// ../../packages/engines/src/ops/BugReportService.ts
|
|
54458
|
+
var import_fs20, import_os2, import_path18, BUG_REPORT_SCHEMA_VERSION, MAX_RUN_LOG_RECORDS, MAX_TAIL_LOG_RECORDS, REPRO_RECORDS, SEED_LENGTH, CONFIG_EXTENSIONS, BEHAVIOURAL_VARIABLES, BugReportServiceClass, BugReportService;
|
|
54459
|
+
var init_BugReportService = __esm({
|
|
54460
|
+
"../../packages/engines/src/ops/BugReportService.ts"() {
|
|
54461
|
+
import_fs20 = __toESM(require("fs"), 1);
|
|
54462
|
+
import_os2 = __toESM(require("os"), 1);
|
|
54463
|
+
import_path18 = __toESM(require("path"), 1);
|
|
54464
|
+
init_src4();
|
|
54465
|
+
init_src3();
|
|
54466
|
+
init_src();
|
|
54467
|
+
init_src6();
|
|
54468
|
+
init_src2();
|
|
54469
|
+
init_LocalUsageDB();
|
|
54470
|
+
init_UsageManager();
|
|
54471
|
+
init_BugReportBundle();
|
|
54472
|
+
init_BugReportRenderer();
|
|
54473
|
+
init_OpsService();
|
|
54474
|
+
init_RunLogSlicer();
|
|
54475
|
+
BUG_REPORT_SCHEMA_VERSION = 1;
|
|
54476
|
+
MAX_RUN_LOG_RECORDS = 5e3;
|
|
54477
|
+
MAX_TAIL_LOG_RECORDS = 500;
|
|
54478
|
+
REPRO_RECORDS = 100;
|
|
54479
|
+
SEED_LENGTH = 8;
|
|
54480
|
+
CONFIG_EXTENSIONS = [".json"];
|
|
54481
|
+
BEHAVIOURAL_VARIABLES = [
|
|
54482
|
+
"NODE_ENV",
|
|
54483
|
+
"REMORA_RUNTIME_CONTEXT",
|
|
54484
|
+
"REMORA_DEBUG_MODE",
|
|
54485
|
+
"REMORA_LOG_FORMAT",
|
|
54486
|
+
"REMORA_MAX_WORKERS",
|
|
54487
|
+
"REMORA_STATE_ENABLED",
|
|
54488
|
+
"REMORA_STATE_INTERVAL_MS",
|
|
54489
|
+
"REMORA_MASK_IN_DEV"
|
|
54490
|
+
];
|
|
54491
|
+
BugReportServiceClass = class {
|
|
54492
|
+
constructor() {
|
|
54493
|
+
/**
|
|
54494
|
+
* Collect a bundle and write it as a .zip, returning where it landed and what went into it.
|
|
54495
|
+
*
|
|
54496
|
+
* @param request which run, which project, where to write. Every field has a default.
|
|
54497
|
+
*/
|
|
54498
|
+
this.collect = (request = {}) => {
|
|
54499
|
+
const bundle = new BugReportBundle_default();
|
|
54500
|
+
const bundleId = Helper_default.uuid();
|
|
54501
|
+
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54502
|
+
const projectPath = request.projectPath ?? Environment_default._localPath ?? "";
|
|
54503
|
+
const inspection = this._inspect(projectPath);
|
|
54504
|
+
const run = this._findRun(request.runId, bundle);
|
|
54505
|
+
const environment = this._environment(projectPath, inspection);
|
|
54506
|
+
const repro = this._repro(bundleId, run, inspection);
|
|
54507
|
+
bundle.collect("environment", () => bundle.add("environment", "environment.json", JSON.stringify(environment, null, 4)));
|
|
54508
|
+
this._collectRun(bundle, run);
|
|
54509
|
+
this._collectConfig(bundle, projectPath);
|
|
54510
|
+
const log = this._collectLogs(bundle, run);
|
|
54511
|
+
bundle.skip("profile", "Producer data profiles need a re-run: use --capture (not yet available).");
|
|
54512
|
+
bundle.skip("data", "No customer data is collected at this redaction level.");
|
|
54513
|
+
bundle.collect("report", () => bundle.add("report", "report.md", BugReportRenderer_default.report({
|
|
54514
|
+
bundleId,
|
|
54515
|
+
createdAt,
|
|
54516
|
+
remoraVersion: Constants_default.cliVersion,
|
|
54517
|
+
run,
|
|
54518
|
+
environment,
|
|
54519
|
+
log,
|
|
54520
|
+
repro,
|
|
54521
|
+
sections: bundle.sections()
|
|
54522
|
+
})));
|
|
54523
|
+
const manifest = this._manifest({ bundleId, createdAt, run, environment, bundle });
|
|
54524
|
+
const outputPath = import_path18.default.resolve(request.outputPath || this._defaultOutputPath(run, environment, createdAt));
|
|
54525
|
+
Affirm_default(import_fs20.default.existsSync(import_path18.default.dirname(outputPath)), `Missing output directory: ${import_path18.default.dirname(outputPath)}`);
|
|
54526
|
+
return { path: outputPath, bytes: bundle.write(outputPath, manifest), manifest };
|
|
54527
|
+
};
|
|
54528
|
+
/**
|
|
54529
|
+
* What is on disk, read leniently.
|
|
54530
|
+
*
|
|
54531
|
+
* `inspect()` and not the loaded `Environment`, even in a process that has one: the bundle has
|
|
54532
|
+
* to describe the configuration as it is NOW, and a loaded environment cannot report the
|
|
54533
|
+
* resources that failed to load — which are routinely the whole of the bug.
|
|
54534
|
+
*/
|
|
54535
|
+
this._inspect = (projectPath) => {
|
|
54536
|
+
try {
|
|
54537
|
+
return Environment_default.inspect(projectPath);
|
|
54538
|
+
} catch (error) {
|
|
54539
|
+
Logger_default.warn(`Bug report could not read the project at "${projectPath}": ${Helper_default.asError(error).message}`);
|
|
54540
|
+
return null;
|
|
54541
|
+
}
|
|
54542
|
+
};
|
|
54543
|
+
/**
|
|
54544
|
+
* The run to report on, from the local run history.
|
|
54545
|
+
*
|
|
54546
|
+
* Absence is an expected outcome, not a failure: `UsageManager.recordsLocally()` is false
|
|
54547
|
+
* whenever `MONGO_URI` is set, so a customer on a database has no local history at all. The
|
|
54548
|
+
* bundle degrades to configuration, environment and the tail of the log, and says so.
|
|
54549
|
+
*/
|
|
54550
|
+
this._findRun = (runId, bundle) => {
|
|
54551
|
+
let found = null;
|
|
54552
|
+
bundle.collect("run", () => {
|
|
54553
|
+
found = runId ? LocalUsageDB_default.list(1, (usage) => usage._id === runId)[0] : LocalUsageDB_default.list(1)[0];
|
|
54554
|
+
if (found) return;
|
|
54555
|
+
bundle.skip("run", runId ? `No run "${runId}" in the local run history.` : this._noHistoryReason());
|
|
54556
|
+
});
|
|
54557
|
+
return found;
|
|
54558
|
+
};
|
|
54559
|
+
this._noHistoryReason = () => UsageManager_default.recordsLocally() ? "No runs recorded yet in ./remora/.temp/usage." : "MONGO_URI is set, so runs are recorded in the database rather than in the local file this reads.";
|
|
54560
|
+
this._collectRun = (bundle, run) => {
|
|
54561
|
+
if (!run) return;
|
|
54562
|
+
bundle.collect("run", () => bundle.add("run", "run.json", JSON.stringify(run, null, 4)));
|
|
54563
|
+
};
|
|
54564
|
+
/**
|
|
54565
|
+
* The project configuration, as authored, under `config/`.
|
|
54566
|
+
*
|
|
54567
|
+
* Not rewritten on the way out: a remora configuration NAMES its credentials (`{PG_PASSWORD}`)
|
|
54568
|
+
* rather than holding them, so what a reader finds in `config/` is what runs — which is the
|
|
54569
|
+
* whole point of shipping it. A literal password in a source file is a mistake to fix in the
|
|
54570
|
+
* project, and redacting it here would only hide it from whoever could.
|
|
54571
|
+
*
|
|
54572
|
+
* Built from `ConfigPackager` so the exclusions are the deployment archive's own — one place
|
|
54573
|
+
* decides that `.temp`, logs and `.env` never travel, and the `.env` refusal there is what
|
|
54574
|
+
* keeps the file that DOES hold values out of this — and then filtered down to the config
|
|
54575
|
+
* files themselves, because an allow-list is the only safe rule for anything else in there.
|
|
54576
|
+
*/
|
|
54577
|
+
this._collectConfig = (bundle, projectPath) => {
|
|
54578
|
+
bundle.collect("config", () => {
|
|
54579
|
+
const configDir = import_path18.default.join(projectPath, "remora");
|
|
54580
|
+
if (!import_fs20.default.existsSync(configDir))
|
|
54581
|
+
return bundle.skip("config", `No configuration directory at ${import_path18.default.resolve(configDir)} \u2014 run the command from the project root.`);
|
|
54582
|
+
const entries = ConfigPackager_default.build(configDir).getEntries().filter((entry) => !entry.isDirectory);
|
|
54583
|
+
const carried = entries.filter((entry) => CONFIG_EXTENSIONS.includes(import_path18.default.extname(entry.entryName).toLowerCase()));
|
|
54584
|
+
for (const entry of carried)
|
|
54585
|
+
bundle.add("config", `config/${entry.entryName}`, entry.getData());
|
|
54586
|
+
const skipped = entries.length - carried.length;
|
|
54587
|
+
if (skipped > 0)
|
|
54588
|
+
bundle.skip("config", `${skipped} non-JSON file(s) in the project directory were left out: only configuration travels.`);
|
|
54589
|
+
});
|
|
54590
|
+
};
|
|
54591
|
+
/**
|
|
54592
|
+
* This run's slice of the log file, redacted.
|
|
54593
|
+
*
|
|
54594
|
+
* Sliced by run id AND by time window, because only the orchestrator's own lines carry the id;
|
|
54595
|
+
* a driver or parser line is placed by its timestamp alone. With no run to bound it, the tail
|
|
54596
|
+
* is packaged and labelled as the tail.
|
|
54597
|
+
*/
|
|
54598
|
+
this._collectLogs = (bundle, run) => {
|
|
54599
|
+
let slice = null;
|
|
54600
|
+
bundle.collect("logs", () => {
|
|
54601
|
+
const logPath = Logger_default.filePath();
|
|
54602
|
+
if (!import_fs20.default.existsSync(logPath))
|
|
54603
|
+
return bundle.skip("logs", `No log file at ${import_path18.default.resolve(logPath)}. File logging turns itself on at the first error; a run that never errored has none.`);
|
|
54604
|
+
slice = RunLogSlicer_default.slice({
|
|
54605
|
+
text: import_fs20.default.readFileSync(logPath, "utf8"),
|
|
54606
|
+
runId: run?._id,
|
|
54607
|
+
from: run?.startedAt ? new Date(run.startedAt) : void 0,
|
|
54608
|
+
to: run?.finishedAt ? new Date(run.finishedAt) : void 0,
|
|
54609
|
+
maxRecords: run ? MAX_RUN_LOG_RECORDS : MAX_TAIL_LOG_RECORDS
|
|
54610
|
+
});
|
|
54611
|
+
if (!slice.kept)
|
|
54612
|
+
return bundle.skip("logs", `The log file holds ${slice.scanned} record(s), none of them from this run.`);
|
|
54613
|
+
bundle.add("logs", "logs/remora.log", SecretRedactor_default.redactText(slice.text));
|
|
54614
|
+
if (slice.dropped)
|
|
54615
|
+
bundle.skip("logs", `${slice.dropped} older matching record(s) dropped: the ${slice.kept} most recent are here.`);
|
|
54616
|
+
else if (!run)
|
|
54617
|
+
bundle.skip("logs", `No run to slice by, so this is the last ${slice.kept} record(s) of the file.`);
|
|
54618
|
+
});
|
|
54619
|
+
return slice;
|
|
54620
|
+
};
|
|
54621
|
+
/** Versions, machine, Node runtime, environment-variable presence and the config as read. */
|
|
54622
|
+
this._environment = (projectPath, inspection) => {
|
|
54623
|
+
return {
|
|
54624
|
+
remora: OpsService_default.status().worker,
|
|
54625
|
+
platform: this._platform(),
|
|
54626
|
+
node: this._node(),
|
|
54627
|
+
variables: this._variables(),
|
|
54628
|
+
config: this._config(projectPath, inspection)
|
|
54629
|
+
};
|
|
54630
|
+
};
|
|
54631
|
+
/**
|
|
54632
|
+
* The machine. Every field is a fact about the host, none of them about the data on it.
|
|
54633
|
+
*
|
|
54634
|
+
* Each one answers a question the other evidence cannot: the CPU model separates "slow on a
|
|
54635
|
+
* small instance" from "slow", the host name ties a run to a container an operator is looking
|
|
54636
|
+
* at, and the time zone is where a datetime that read correctly and wrote wrong comes from.
|
|
54637
|
+
*/
|
|
54638
|
+
this._platform = () => ({
|
|
54639
|
+
hostname: this._quietly(() => import_os2.default.hostname()),
|
|
54640
|
+
platform: process.platform,
|
|
54641
|
+
arch: process.arch,
|
|
54642
|
+
osType: this._quietly(() => import_os2.default.type()),
|
|
54643
|
+
release: import_os2.default.release(),
|
|
54644
|
+
osVersion: this._quietly(() => import_os2.default.version()),
|
|
54645
|
+
cpuCount: import_os2.default.cpus()?.length ?? 0,
|
|
54646
|
+
cpuModel: import_os2.default.cpus()?.[0]?.model ?? null,
|
|
54647
|
+
totalMemoryMB: Math.round(import_os2.default.totalmem() / (1024 * 1024)),
|
|
54648
|
+
freeMemoryMB: Math.round(import_os2.default.freemem() / (1024 * 1024)),
|
|
54649
|
+
heapLimitMB: Math.round(Runtime_default.getHeap().heapSizeMB),
|
|
54650
|
+
timezone: this._quietly(() => Intl.DateTimeFormat().resolvedOptions().timeZone)
|
|
54651
|
+
});
|
|
54652
|
+
/** The Node runtime, and the handful of variables whose VALUE explains what remora did. */
|
|
54653
|
+
this._node = () => {
|
|
54654
|
+
const settings = {};
|
|
54655
|
+
for (const name of BEHAVIOURAL_VARIABLES)
|
|
54656
|
+
if (process.env[name]) settings[name] = process.env[name];
|
|
54657
|
+
return {
|
|
54658
|
+
version: process.version,
|
|
54659
|
+
v8: process.versions?.v8 ?? null,
|
|
54660
|
+
openssl: process.versions?.openssl ?? null,
|
|
54661
|
+
zlib: process.versions?.zlib ?? null,
|
|
54662
|
+
icu: process.versions?.icu ?? null,
|
|
54663
|
+
settings
|
|
54664
|
+
};
|
|
54665
|
+
};
|
|
54666
|
+
/**
|
|
54667
|
+
* A host lookup that cannot fail the report. `os.version()` and the ICU time zone are the two
|
|
54668
|
+
* that vary by platform and build, and a missing one is worth less than the bundle it would
|
|
54669
|
+
* otherwise take down.
|
|
54670
|
+
*/
|
|
54671
|
+
this._quietly = (read) => {
|
|
54672
|
+
try {
|
|
54673
|
+
return read() ?? null;
|
|
54674
|
+
} catch {
|
|
54675
|
+
return null;
|
|
54676
|
+
}
|
|
54677
|
+
};
|
|
54678
|
+
/**
|
|
54679
|
+
* Which variables are SET here, never what they hold. "Is `REMORA_MAX_WORKERS` set on that
|
|
54680
|
+
* machine" is most of what anyone needs, and it is answerable without a secret in a ticket.
|
|
54681
|
+
*/
|
|
54682
|
+
this._variables = () => {
|
|
54683
|
+
const present = {};
|
|
54684
|
+
for (const name of ENVIRONMENT_VARIABLES)
|
|
54685
|
+
present[name] = !!process.env[name];
|
|
54686
|
+
return present;
|
|
54687
|
+
};
|
|
54688
|
+
this._config = (projectPath, inspection) => {
|
|
54689
|
+
const env2 = inspection?.env;
|
|
54690
|
+
return {
|
|
54691
|
+
path: import_path18.default.resolve(projectPath || "."),
|
|
54692
|
+
project: inspection?.projectName ?? null,
|
|
54693
|
+
signature: this._signature(projectPath),
|
|
54694
|
+
loaded: !!inspection,
|
|
54695
|
+
counts: {
|
|
54696
|
+
sources: env2?.sources?.length ?? 0,
|
|
54697
|
+
producers: env2?.producers?.length ?? 0,
|
|
54698
|
+
consumers: env2?.consumers?.length ?? 0,
|
|
54699
|
+
codeSets: env2?.codeSets?.length ?? 0,
|
|
54700
|
+
schemas: env2?.schemas?.length ?? 0
|
|
54701
|
+
},
|
|
54702
|
+
problems: inspection?.problems ?? [],
|
|
54703
|
+
sources: (env2?.sources ?? []).map((source) => ({
|
|
54704
|
+
name: source.name,
|
|
54705
|
+
engine: source.engine,
|
|
54706
|
+
authMethod: source.authentication?.method ?? null
|
|
54707
|
+
})),
|
|
54708
|
+
producers: (env2?.producers ?? []).map((producer) => ({
|
|
54709
|
+
name: producer.name,
|
|
54710
|
+
source: producer.source,
|
|
54711
|
+
dimensionCount: producer.dimensions?.length ?? 0
|
|
54712
|
+
})),
|
|
54713
|
+
consumers: (env2?.consumers ?? []).map((consumer) => ({
|
|
54714
|
+
name: consumer.name,
|
|
54715
|
+
producers: (consumer.producers ?? []).map((x2) => x2.name),
|
|
54716
|
+
outputs: (consumer.outputs ?? []).map((output) => output.trigger ? `${output.format} (${output.trigger.type} ${output.trigger.value ?? ""})`.trim() : `${output.format}`)
|
|
54717
|
+
}))
|
|
54718
|
+
};
|
|
54719
|
+
};
|
|
54720
|
+
this._signature = (projectPath) => {
|
|
54721
|
+
try {
|
|
54722
|
+
return Environment_default.configHash(projectPath)?.slice(0, 12) ?? null;
|
|
54723
|
+
} catch {
|
|
54724
|
+
return null;
|
|
54725
|
+
}
|
|
54726
|
+
};
|
|
54727
|
+
/**
|
|
54728
|
+
* The repro the bundle ships with.
|
|
54729
|
+
*
|
|
54730
|
+
* A post-mortem has no recorded input, so the seed comes from the bundle id: it is unrelated to
|
|
54731
|
+
* the failing run and the manifest says so. A runnable wrong-data case still beats no case —
|
|
54732
|
+
* it reproduces everything about the pipeline's shape, which is where most bugs live.
|
|
54733
|
+
*/
|
|
54734
|
+
this._repro = (bundleId, run, inspection) => ({
|
|
54735
|
+
consumer: run?.consumer ?? inspection?.env?.consumers?.[0]?.name ?? null,
|
|
54736
|
+
seed: bundleId.replace(/-/g, "").slice(0, SEED_LENGTH),
|
|
54737
|
+
records: REPRO_RECORDS,
|
|
54738
|
+
syntheticData: true
|
|
54739
|
+
});
|
|
54740
|
+
this._manifest = (args) => ({
|
|
54741
|
+
schemaVersion: BUG_REPORT_SCHEMA_VERSION,
|
|
54742
|
+
bundleId: args.bundleId,
|
|
54743
|
+
createdAt: args.createdAt,
|
|
54744
|
+
redaction: "no-data",
|
|
54745
|
+
remoraVersion: Constants_default.cliVersion,
|
|
54746
|
+
runId: args.run?._id ?? null,
|
|
54747
|
+
consumer: args.run?.consumer ?? null,
|
|
54748
|
+
project: args.environment.config.project,
|
|
54749
|
+
configLoaded: args.environment.config.loaded,
|
|
54750
|
+
sections: args.bundle.sections(),
|
|
54751
|
+
files: args.bundle.files()
|
|
54752
|
+
});
|
|
54753
|
+
/** `remora-report-<consumer>-<runId>.zip`, or the project and a timestamp when there is no run. */
|
|
54754
|
+
this._defaultOutputPath = (run, environment, createdAt) => {
|
|
54755
|
+
const subject = this._slug(run?.consumer ?? environment.config.project ?? "project");
|
|
54756
|
+
const discriminator = run?._id ?? createdAt.replace(/[:.]/g, "-");
|
|
54757
|
+
return `./remora-report-${subject}-${this._slug(discriminator)}.zip`;
|
|
54758
|
+
};
|
|
54759
|
+
/** A name safe on every file system this runs on, and readable afterwards. */
|
|
54760
|
+
this._slug = (value) => value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 60) || "report";
|
|
54761
|
+
}
|
|
54762
|
+
};
|
|
54763
|
+
BugReportService = new BugReportServiceClass();
|
|
54764
|
+
}
|
|
54765
|
+
});
|
|
54766
|
+
|
|
53373
54767
|
// ../../packages/engines/src/ops/OpsHeartbeat.ts
|
|
53374
54768
|
var DEFAULT_INTERVAL_MS, MIN_INTERVAL_MS, MAX_INTERVAL_MS, OpsHeartbeatClass, OpsHeartbeat;
|
|
53375
54769
|
var init_OpsHeartbeat = __esm({
|
|
@@ -53441,440 +54835,11 @@ var init_OpsHeartbeat = __esm({
|
|
|
53441
54835
|
}
|
|
53442
54836
|
});
|
|
53443
54837
|
|
|
53444
|
-
// ../../packages/engines/src/usage/LocalUsageDB.ts
|
|
53445
|
-
var import_fs17, import_path16, COLUMNS, CLI_DB_FOLDER, CLI_DB_FILE, CLI_DB_PATH, MAX_USAGE_ROWS, MAX_LISTED_ROWS, EXECUTED_BY_SCHEMA, EXECUTOR_OPERATION_SCHEMA, STATS_SCHEMA, COLUMN_SCHEMAS, LocalUsageDBClass, LocalUsageDB, LocalUsageDB_default;
|
|
53446
|
-
var init_LocalUsageDB = __esm({
|
|
53447
|
-
"../../packages/engines/src/usage/LocalUsageDB.ts"() {
|
|
53448
|
-
init_src();
|
|
53449
|
-
import_fs17 = require("fs");
|
|
53450
|
-
import_path16 = require("path");
|
|
53451
|
-
COLUMNS = [
|
|
53452
|
-
"_id",
|
|
53453
|
-
"consumer",
|
|
53454
|
-
"executedBy",
|
|
53455
|
-
"startedAt",
|
|
53456
|
-
"finishedAt",
|
|
53457
|
-
"itemsCount",
|
|
53458
|
-
"invokedBy",
|
|
53459
|
-
"status",
|
|
53460
|
-
"error",
|
|
53461
|
-
"stats",
|
|
53462
|
-
// Appended last on purpose: rows written before this column existed still decode positionally.
|
|
53463
|
-
"engineVersion"
|
|
53464
|
-
];
|
|
53465
|
-
CLI_DB_FOLDER = "./remora/.temp/usage";
|
|
53466
|
-
CLI_DB_FILE = "remora_usage.csv";
|
|
53467
|
-
CLI_DB_PATH = (0, import_path16.join)(CLI_DB_FOLDER, CLI_DB_FILE);
|
|
53468
|
-
MAX_USAGE_ROWS = 1e4;
|
|
53469
|
-
MAX_LISTED_ROWS = 500;
|
|
53470
|
-
EXECUTED_BY_SCHEMA = {
|
|
53471
|
-
_id: "string",
|
|
53472
|
-
name: "string",
|
|
53473
|
-
type: "string"
|
|
53474
|
-
};
|
|
53475
|
-
EXECUTOR_OPERATION_SCHEMA = {
|
|
53476
|
-
elapsedMS: ["number"],
|
|
53477
|
-
min: "number",
|
|
53478
|
-
max: "number",
|
|
53479
|
-
avg: "number"
|
|
53480
|
-
};
|
|
53481
|
-
STATS_SCHEMA = {
|
|
53482
|
-
executionId: "string",
|
|
53483
|
-
resultUri: "string",
|
|
53484
|
-
elapsedMS: "number",
|
|
53485
|
-
inputCount: "number",
|
|
53486
|
-
outputCount: "number",
|
|
53487
|
-
cycles: "number",
|
|
53488
|
-
workerCount: "number",
|
|
53489
|
-
operations: {
|
|
53490
|
-
"*": EXECUTOR_OPERATION_SCHEMA
|
|
53491
|
-
}
|
|
53492
|
-
};
|
|
53493
|
-
COLUMN_SCHEMAS = {
|
|
53494
|
-
_id: "string",
|
|
53495
|
-
consumer: "string",
|
|
53496
|
-
executedBy: EXECUTED_BY_SCHEMA,
|
|
53497
|
-
startedAt: "date",
|
|
53498
|
-
finishedAt: "date",
|
|
53499
|
-
itemsCount: "number",
|
|
53500
|
-
invokedBy: "string",
|
|
53501
|
-
status: "string",
|
|
53502
|
-
error: "string",
|
|
53503
|
-
stats: STATS_SCHEMA,
|
|
53504
|
-
engineVersion: "string"
|
|
53505
|
-
};
|
|
53506
|
-
LocalUsageDBClass = class {
|
|
53507
|
-
constructor() {
|
|
53508
|
-
this._handleError = (action, error) => {
|
|
53509
|
-
console.error(`LocalUsageDB ${action} failed:`, error);
|
|
53510
|
-
};
|
|
53511
|
-
this._splitSerialized = (value, separator) => {
|
|
53512
|
-
if (!value)
|
|
53513
|
-
return [];
|
|
53514
|
-
const parts = [];
|
|
53515
|
-
let current = "";
|
|
53516
|
-
let quoteOpen = false;
|
|
53517
|
-
let curlyDepth = 0;
|
|
53518
|
-
let squareDepth = 0;
|
|
53519
|
-
for (let index = 0; index < value.length; index++) {
|
|
53520
|
-
const char = value[index];
|
|
53521
|
-
const next = value[index + 1];
|
|
53522
|
-
if (char === '"') {
|
|
53523
|
-
current += char;
|
|
53524
|
-
if (quoteOpen && next === '"') {
|
|
53525
|
-
current += next;
|
|
53526
|
-
index++;
|
|
53527
|
-
continue;
|
|
53528
|
-
}
|
|
53529
|
-
quoteOpen = !quoteOpen;
|
|
53530
|
-
continue;
|
|
53531
|
-
}
|
|
53532
|
-
if (!quoteOpen) {
|
|
53533
|
-
if (char === "{")
|
|
53534
|
-
curlyDepth++;
|
|
53535
|
-
else if (char === "}")
|
|
53536
|
-
curlyDepth--;
|
|
53537
|
-
else if (char === "[")
|
|
53538
|
-
squareDepth++;
|
|
53539
|
-
else if (char === "]")
|
|
53540
|
-
squareDepth--;
|
|
53541
|
-
else if (char === separator && curlyDepth === 0 && squareDepth === 0) {
|
|
53542
|
-
parts.push(current);
|
|
53543
|
-
current = "";
|
|
53544
|
-
continue;
|
|
53545
|
-
}
|
|
53546
|
-
}
|
|
53547
|
-
current += char;
|
|
53548
|
-
}
|
|
53549
|
-
parts.push(current);
|
|
53550
|
-
return parts;
|
|
53551
|
-
};
|
|
53552
|
-
this._splitKeyValue = (value) => {
|
|
53553
|
-
let quoteOpen = false;
|
|
53554
|
-
let curlyDepth = 0;
|
|
53555
|
-
let squareDepth = 0;
|
|
53556
|
-
for (let index = 0; index < value.length; index++) {
|
|
53557
|
-
const char = value[index];
|
|
53558
|
-
const next = value[index + 1];
|
|
53559
|
-
if (char === '"') {
|
|
53560
|
-
if (quoteOpen && next === '"') {
|
|
53561
|
-
index++;
|
|
53562
|
-
continue;
|
|
53563
|
-
}
|
|
53564
|
-
quoteOpen = !quoteOpen;
|
|
53565
|
-
continue;
|
|
53566
|
-
}
|
|
53567
|
-
if (!quoteOpen) {
|
|
53568
|
-
if (char === "{")
|
|
53569
|
-
curlyDepth++;
|
|
53570
|
-
else if (char === "}")
|
|
53571
|
-
curlyDepth--;
|
|
53572
|
-
else if (char === "[")
|
|
53573
|
-
squareDepth++;
|
|
53574
|
-
else if (char === "]")
|
|
53575
|
-
squareDepth--;
|
|
53576
|
-
else if (char === ":" && curlyDepth === 0 && squareDepth === 0)
|
|
53577
|
-
return [value.slice(0, index), value.slice(index + 1)];
|
|
53578
|
-
}
|
|
53579
|
-
}
|
|
53580
|
-
return [value, ""];
|
|
53581
|
-
};
|
|
53582
|
-
this._decodeString = (value) => {
|
|
53583
|
-
if (!value)
|
|
53584
|
-
return "";
|
|
53585
|
-
if (value.startsWith('"') && value.endsWith('"'))
|
|
53586
|
-
return value.slice(1, -1).replaceAll('""', '"');
|
|
53587
|
-
return value;
|
|
53588
|
-
};
|
|
53589
|
-
this._decodeValue = (value, schema) => {
|
|
53590
|
-
if (!value)
|
|
53591
|
-
return null;
|
|
53592
|
-
if (value.startsWith("{") && value.endsWith("}")) {
|
|
53593
|
-
const objectSchema = schema && !Array.isArray(schema) && typeof schema === "object" ? schema : {};
|
|
53594
|
-
const dynamicSchema = objectSchema["*"];
|
|
53595
|
-
const serializedEntries = this._splitSerialized(value.slice(1, -1), ";");
|
|
53596
|
-
const decodedObject = {};
|
|
53597
|
-
serializedEntries.forEach((entry) => {
|
|
53598
|
-
const [rawKey, rawValue] = this._splitKeyValue(entry);
|
|
53599
|
-
const key = this._decodeString(rawKey);
|
|
53600
|
-
decodedObject[key] = this._decodeValue(rawValue, objectSchema[key] ?? dynamicSchema);
|
|
53601
|
-
});
|
|
53602
|
-
return decodedObject;
|
|
53603
|
-
}
|
|
53604
|
-
if (value.startsWith("[") && value.endsWith("]")) {
|
|
53605
|
-
const serializedItems = this._splitSerialized(value.slice(1, -1), ";");
|
|
53606
|
-
const itemSchema = Array.isArray(schema) ? schema[0] : void 0;
|
|
53607
|
-
return serializedItems.map((item) => this._decodeValue(item, itemSchema));
|
|
53608
|
-
}
|
|
53609
|
-
const decodedString = this._decodeString(value);
|
|
53610
|
-
if (schema === "number")
|
|
53611
|
-
return Number(decodedString);
|
|
53612
|
-
if (schema === "date")
|
|
53613
|
-
return decodedString ? new Date(decodedString) : null;
|
|
53614
|
-
return decodedString;
|
|
53615
|
-
};
|
|
53616
|
-
this._encodeValue = (value) => {
|
|
53617
|
-
if (value === null || value === void 0)
|
|
53618
|
-
return "";
|
|
53619
|
-
else if (value instanceof Date)
|
|
53620
|
-
return `"${value.toISOString().replaceAll('"', '""')}"`;
|
|
53621
|
-
else if (Array.isArray(value))
|
|
53622
|
-
return `[${value.map((item) => this._encodeValue(item)).join(";")}]`;
|
|
53623
|
-
else if (typeof value === "object")
|
|
53624
|
-
return `{${Object.keys(value).map((key) => `${this._encodeValue(key)}:${this._encodeValue(value[key])}`).join(";")}}`;
|
|
53625
|
-
else
|
|
53626
|
-
return `"${(value ?? "").toString().replaceAll('"', '""')}"`;
|
|
53627
|
-
};
|
|
53628
|
-
this._toLine = (usage) => {
|
|
53629
|
-
return COLUMNS.map((column) => this._encodeValue(usage[column])).join(",");
|
|
53630
|
-
};
|
|
53631
|
-
this._fromLine = (line) => {
|
|
53632
|
-
const values = this._splitSerialized(line, ",");
|
|
53633
|
-
const usage = {};
|
|
53634
|
-
COLUMNS.forEach((column, index) => {
|
|
53635
|
-
usage[column] = this._decodeValue(values[index] ?? "", COLUMN_SCHEMAS[column]);
|
|
53636
|
-
});
|
|
53637
|
-
return usage;
|
|
53638
|
-
};
|
|
53639
|
-
this._getRows = () => {
|
|
53640
|
-
const lines = (0, import_fs17.readFileSync)(CLI_DB_PATH, "utf8").split("\n").filter(Boolean);
|
|
53641
|
-
lines.shift();
|
|
53642
|
-
return lines;
|
|
53643
|
-
};
|
|
53644
|
-
this._writeRows = (rows) => {
|
|
53645
|
-
const trimmedRows = rows.slice(-MAX_USAGE_ROWS);
|
|
53646
|
-
(0, import_fs17.writeFileSync)(CLI_DB_PATH, `${[COLUMNS.join(","), ...trimmedRows].join("\n")}
|
|
53647
|
-
`);
|
|
53648
|
-
};
|
|
53649
|
-
this.init = () => {
|
|
53650
|
-
try {
|
|
53651
|
-
if (!(0, import_fs17.existsSync)(CLI_DB_FOLDER))
|
|
53652
|
-
(0, import_fs17.mkdirSync)(CLI_DB_FOLDER, { recursive: true });
|
|
53653
|
-
if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
|
|
53654
|
-
(0, import_fs17.writeFileSync)(CLI_DB_PATH, `${COLUMNS.join(",")}
|
|
53655
|
-
`);
|
|
53656
|
-
this._writeRows(this._getRows());
|
|
53657
|
-
} catch (error) {
|
|
53658
|
-
this._handleError("init", error);
|
|
53659
|
-
}
|
|
53660
|
-
};
|
|
53661
|
-
this.insert = (usage) => {
|
|
53662
|
-
try {
|
|
53663
|
-
this.init();
|
|
53664
|
-
this._writeRows([...this._getRows(), this._toLine(usage)]);
|
|
53665
|
-
} catch (error) {
|
|
53666
|
-
this._handleError("insert", error);
|
|
53667
|
-
}
|
|
53668
|
-
};
|
|
53669
|
-
/**
|
|
53670
|
-
* The recorded runs, newest first — the newest `limit` of the rows `keep` accepts.
|
|
53671
|
-
*
|
|
53672
|
-
* Decoded from the end and stopped as soon as that many are held: a reader wants the recent
|
|
53673
|
-
* executions, and decoding ten thousand rows to show ten of them is work nobody asked for.
|
|
53674
|
-
*
|
|
53675
|
-
* `keep` is applied BEFORE the limit, which is the only way "the newest N runs of this kind" can
|
|
53676
|
-
* mean it: filtering afterwards lets the rows a caller does not want displace the ones it does,
|
|
53677
|
-
* so one sample recorded here would silently push a real run out of a history someone is reading.
|
|
53678
|
-
*
|
|
53679
|
-
* Never throws. A missing file reads as no history, and so does a half-written one — this store is
|
|
53680
|
-
* appended to while it is being read, so failing on that would make it unreadable during a run.
|
|
53681
|
-
*/
|
|
53682
|
-
this.list = (limit = MAX_LISTED_ROWS, keep) => {
|
|
53683
|
-
try {
|
|
53684
|
-
if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
|
|
53685
|
-
return [];
|
|
53686
|
-
const rows = this._getRows();
|
|
53687
|
-
const wanted = Math.max(1, limit);
|
|
53688
|
-
const found = [];
|
|
53689
|
-
for (let index = rows.length - 1; index >= 0 && found.length < wanted; index--) {
|
|
53690
|
-
const usage = this._fromLine(rows[index]);
|
|
53691
|
-
if (!keep || keep(usage)) found.push(usage);
|
|
53692
|
-
}
|
|
53693
|
-
return found;
|
|
53694
|
-
} catch (error) {
|
|
53695
|
-
this._handleError("list", error);
|
|
53696
|
-
return [];
|
|
53697
|
-
}
|
|
53698
|
-
};
|
|
53699
|
-
/**
|
|
53700
|
-
* A token that changes whenever the store does, cheap enough to poll: a preview watching for a
|
|
53701
|
-
* finished run must not decode the whole file once a second to notice one.
|
|
53702
|
-
*/
|
|
53703
|
-
this.signature = () => {
|
|
53704
|
-
try {
|
|
53705
|
-
if (!(0, import_fs17.existsSync)(CLI_DB_PATH))
|
|
53706
|
-
return "";
|
|
53707
|
-
const stats = (0, import_fs17.statSync)(CLI_DB_PATH);
|
|
53708
|
-
return `${stats.mtimeMs}:${stats.size}`;
|
|
53709
|
-
} catch (error) {
|
|
53710
|
-
this._handleError("signature", error);
|
|
53711
|
-
return "";
|
|
53712
|
-
}
|
|
53713
|
-
};
|
|
53714
|
-
this.update = (update) => {
|
|
53715
|
-
try {
|
|
53716
|
-
Affirm_default(update?._id, "Invalid usage update _id");
|
|
53717
|
-
this.init();
|
|
53718
|
-
const updatedLines = this._getRows().map((line) => {
|
|
53719
|
-
const current = this._fromLine(line);
|
|
53720
|
-
if (current._id !== update._id) return line;
|
|
53721
|
-
return this._toLine({ ...current, ...update });
|
|
53722
|
-
});
|
|
53723
|
-
this._writeRows(updatedLines);
|
|
53724
|
-
} catch (error) {
|
|
53725
|
-
this._handleError("update", error);
|
|
53726
|
-
}
|
|
53727
|
-
};
|
|
53728
|
-
}
|
|
53729
|
-
};
|
|
53730
|
-
LocalUsageDB = new LocalUsageDBClass();
|
|
53731
|
-
LocalUsageDB_default = LocalUsageDB;
|
|
53732
|
-
}
|
|
53733
|
-
});
|
|
53734
|
-
|
|
53735
|
-
// ../../packages/engines/src/usage/UsageManager.ts
|
|
53736
|
-
var MAX_RECORDED_OPERATIONS, UsageManagerClass, UsageManager, UsageManager_default;
|
|
53737
|
-
var init_UsageManager = __esm({
|
|
53738
|
-
"../../packages/engines/src/usage/UsageManager.ts"() {
|
|
53739
|
-
init_src();
|
|
53740
|
-
init_src3();
|
|
53741
|
-
init_src();
|
|
53742
|
-
init_src4();
|
|
53743
|
-
init_src9();
|
|
53744
|
-
init_src6();
|
|
53745
|
-
init_src6();
|
|
53746
|
-
init_src2();
|
|
53747
|
-
init_RunRegistry();
|
|
53748
|
-
init_LocalUsageDB();
|
|
53749
|
-
MAX_RECORDED_OPERATIONS = 40;
|
|
53750
|
-
UsageManagerClass = class {
|
|
53751
|
-
constructor() {
|
|
53752
|
-
this._shouldSkipDb = () => Helper_default.isDev() || !process.env.MONGO_URI;
|
|
53753
|
-
this._shouldLogCLI = () => process.env.REMORA_RUNTIME_CONTEXT === "cli";
|
|
53754
|
-
/**
|
|
53755
|
-
* Whether this runtime keeps its execution records in the local file rather than in the usage
|
|
53756
|
-
* collection. Public because a READER has to make the same decision the writer just made — two
|
|
53757
|
-
* answers to "where are the runs" is one answer too many.
|
|
53758
|
-
*/
|
|
53759
|
-
this.recordsLocally = () => this._shouldSkipDb() && this._shouldLogCLI();
|
|
53760
|
-
this.startUsage = (consumer, details) => {
|
|
53761
|
-
const { user, invokedBy } = details;
|
|
53762
|
-
const newUsage = {
|
|
53763
|
-
_id: Helper_default.uuid(),
|
|
53764
|
-
consumer: consumer.name,
|
|
53765
|
-
engineVersion: Constants_default.cliVersion,
|
|
53766
|
-
startedAt: DSTE_default.now(),
|
|
53767
|
-
executedBy: user,
|
|
53768
|
-
itemsCount: -1,
|
|
53769
|
-
status: "started",
|
|
53770
|
-
invokedBy: invokedBy ?? "UNKNOWN",
|
|
53771
|
-
_signature: "",
|
|
53772
|
-
stats: null
|
|
53773
|
-
};
|
|
53774
|
-
RunRegistry_default.start({
|
|
53775
|
-
executionId: newUsage._id,
|
|
53776
|
-
consumer: newUsage.consumer,
|
|
53777
|
-
invokedBy: newUsage.invokedBy,
|
|
53778
|
-
userType: user?.type ?? "UNKNOWN",
|
|
53779
|
-
startedAt: newUsage.startedAt
|
|
53780
|
-
});
|
|
53781
|
-
if (this._shouldSkipDb()) {
|
|
53782
|
-
if (this._shouldLogCLI())
|
|
53783
|
-
LocalUsageDB_default.insert(newUsage);
|
|
53784
|
-
return { usageId: newUsage._id, usage: Promise.resolve(newUsage) };
|
|
53785
|
-
}
|
|
53786
|
-
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, newUsage._id, newUsage);
|
|
53787
|
-
return { usageId: newUsage._id, usage: updateRes };
|
|
53788
|
-
};
|
|
53789
|
-
this.endUsage = (usageId, itemsCount, stats) => {
|
|
53790
|
-
const update = {
|
|
53791
|
-
itemsCount,
|
|
53792
|
-
status: "success",
|
|
53793
|
-
finishedAt: DSTE_default.now(),
|
|
53794
|
-
stats
|
|
53795
|
-
};
|
|
53796
|
-
this._emitRunRecord(usageId, "success", stats, null);
|
|
53797
|
-
if (this._shouldSkipDb()) {
|
|
53798
|
-
if (this._shouldLogCLI())
|
|
53799
|
-
LocalUsageDB_default.update({ ...update, _id: usageId });
|
|
53800
|
-
return { usageId: null, usage: Promise.resolve(update) };
|
|
53801
|
-
}
|
|
53802
|
-
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, usageId, update);
|
|
53803
|
-
return { usageId, usage: updateRes };
|
|
53804
|
-
};
|
|
53805
|
-
this.failUsage = (usageId, error) => {
|
|
53806
|
-
const update = {
|
|
53807
|
-
status: "failed",
|
|
53808
|
-
error,
|
|
53809
|
-
finishedAt: DSTE_default.now()
|
|
53810
|
-
};
|
|
53811
|
-
this._emitRunRecord(usageId, "failed", null, error);
|
|
53812
|
-
if (this._shouldSkipDb()) {
|
|
53813
|
-
if (this._shouldLogCLI())
|
|
53814
|
-
LocalUsageDB_default.update({ ...update, _id: usageId });
|
|
53815
|
-
return { usageId: null, usage: Promise.resolve(update) };
|
|
53816
|
-
}
|
|
53817
|
-
const updateRes = DatabaseEngine_default.upsert(Settings_default.db.collections.usage, usageId, update);
|
|
53818
|
-
return { usageId, usage: updateRes };
|
|
53819
|
-
};
|
|
53820
|
-
/**
|
|
53821
|
-
* Close out the in-flight entry and write one structured record for the finished run.
|
|
53822
|
-
*
|
|
53823
|
-
* This is the cheapest useful thing in the whole ops surface: the executor already hands us the
|
|
53824
|
-
* complete result, so a single hook makes every run queryable in the log pipeline without a
|
|
53825
|
-
* database. Never throws — reporting on a run must not be able to fail the run.
|
|
53826
|
-
*/
|
|
53827
|
-
this._emitRunRecord = (usageId, status, stats, error) => {
|
|
53828
|
-
Affirm_default(usageId, "Missing usage id");
|
|
53829
|
-
try {
|
|
53830
|
-
const run = RunRegistry_default.finish(usageId);
|
|
53831
|
-
const record = {
|
|
53832
|
-
executionId: usageId,
|
|
53833
|
-
consumer: run?.consumer ?? null,
|
|
53834
|
-
engineVersion: Constants_default.cliVersion,
|
|
53835
|
-
project: Environment_default._env ? Environment_default.getProjectName() : null,
|
|
53836
|
-
invokedBy: run?.invokedBy ?? null,
|
|
53837
|
-
userType: run?.userType ?? null,
|
|
53838
|
-
status,
|
|
53839
|
-
startedAt: run?.startedAt?.toISOString() ?? null,
|
|
53840
|
-
elapsedMS: stats?.elapsedMS ?? (run ? DSTE_default.now().getTime() - run.startedAt.getTime() : -1),
|
|
53841
|
-
inputCount: stats?.inputCount ?? -1,
|
|
53842
|
-
outputCount: stats?.outputCount ?? -1,
|
|
53843
|
-
cycles: stats?.cycles ?? -1,
|
|
53844
|
-
workerCount: stats?.workerCount ?? -1,
|
|
53845
|
-
operations: this._summarizeOperations(stats),
|
|
53846
|
-
error: error ?? null
|
|
53847
|
-
};
|
|
53848
|
-
StateEmitter_default.emit("run", record);
|
|
53849
|
-
} catch {
|
|
53850
|
-
}
|
|
53851
|
-
};
|
|
53852
|
-
/**
|
|
53853
|
-
* Keep only min/max/avg per stage. The raw `elapsedMS` array holds one entry per worker chunk
|
|
53854
|
-
* and can run to thousands of numbers on a large run — useless in a log line and unbounded.
|
|
53855
|
-
*/
|
|
53856
|
-
this._summarizeOperations = (stats) => {
|
|
53857
|
-
const operations = {};
|
|
53858
|
-
if (!stats?.operations)
|
|
53859
|
-
return operations;
|
|
53860
|
-
for (const name of Object.keys(stats.operations).slice(0, MAX_RECORDED_OPERATIONS)) {
|
|
53861
|
-
const operation2 = stats.operations[name];
|
|
53862
|
-
operations[name] = { min: operation2.min, max: operation2.max, avg: operation2.avg };
|
|
53863
|
-
}
|
|
53864
|
-
return operations;
|
|
53865
|
-
};
|
|
53866
|
-
}
|
|
53867
|
-
};
|
|
53868
|
-
UsageManager = new UsageManagerClass();
|
|
53869
|
-
UsageManager_default = UsageManager;
|
|
53870
|
-
}
|
|
53871
|
-
});
|
|
53872
|
-
|
|
53873
54838
|
// ../../packages/engines/src/ops/RunOverlay.ts
|
|
53874
|
-
var
|
|
54839
|
+
var import_crypto8, MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay;
|
|
53875
54840
|
var init_RunOverlay = __esm({
|
|
53876
54841
|
"../../packages/engines/src/ops/RunOverlay.ts"() {
|
|
53877
|
-
|
|
54842
|
+
import_crypto8 = __toESM(require("crypto"), 1);
|
|
53878
54843
|
init_src();
|
|
53879
54844
|
init_LineageEngine();
|
|
53880
54845
|
init_LocalUsageDB();
|
|
@@ -53928,7 +54893,7 @@ var init_RunOverlay = __esm({
|
|
|
53928
54893
|
* finish action writes runs of its own, so a narrower token would miss a transition the graph
|
|
53929
54894
|
* exists to show.
|
|
53930
54895
|
*/
|
|
53931
|
-
this._token = (overlay) =>
|
|
54896
|
+
this._token = (overlay) => import_crypto8.default.createHash("sha256").update(JSON.stringify(overlay)).digest("hex");
|
|
53932
54897
|
this._run = (stat) => {
|
|
53933
54898
|
const startedAt = this._iso(stat.startedAt);
|
|
53934
54899
|
return {
|
|
@@ -54210,12 +55175,12 @@ var init_LookupRegistry = __esm({
|
|
|
54210
55175
|
});
|
|
54211
55176
|
|
|
54212
55177
|
// ../../packages/engines/src/transform/LookupKey.ts
|
|
54213
|
-
var
|
|
55178
|
+
var import_crypto9, DELIMITER, idFor, LookupKey, LookupKey_default;
|
|
54214
55179
|
var init_LookupKey = __esm({
|
|
54215
55180
|
"../../packages/engines/src/transform/LookupKey.ts"() {
|
|
54216
|
-
|
|
55181
|
+
import_crypto9 = __toESM(require("crypto"), 1);
|
|
54217
55182
|
DELIMITER = String.fromCharCode(0);
|
|
54218
|
-
idFor = (ref) =>
|
|
55183
|
+
idFor = (ref) => import_crypto9.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
|
|
54219
55184
|
LookupKey = { idFor };
|
|
54220
55185
|
LookupKey_default = LookupKey;
|
|
54221
55186
|
}
|
|
@@ -54845,6 +55810,9 @@ var init_src10 = __esm({
|
|
|
54845
55810
|
init_RequestExecutor();
|
|
54846
55811
|
init_RequestValidator();
|
|
54847
55812
|
init_lineage();
|
|
55813
|
+
init_BugReportService();
|
|
55814
|
+
init_BugReportRenderer();
|
|
55815
|
+
init_RunLogSlicer();
|
|
54848
55816
|
init_OpsService();
|
|
54849
55817
|
init_OpsHeartbeat();
|
|
54850
55818
|
init_RunOverlay();
|
|
@@ -54960,10 +55928,11 @@ var init_OutputExecutor = __esm({
|
|
|
54960
55928
|
const driver = await DriverFactory_default.instantiateDestination(destination);
|
|
54961
55929
|
const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
|
|
54962
55930
|
Logger_default.info(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName}, ${splitFiles.length} file(s))`);
|
|
55931
|
+
const header = this._composeHeader(output, fields);
|
|
54963
55932
|
for (const { path: startingPath, label } of splitFiles) {
|
|
54964
55933
|
const destinationPath = this.getCompletedPath(destinationName, label);
|
|
54965
55934
|
if (output.format === internalFormat) {
|
|
54966
|
-
results.push(await driver.move(startingPath, destinationPath));
|
|
55935
|
+
results.push(await driver.move(startingPath, destinationPath, header));
|
|
54967
55936
|
} else {
|
|
54968
55937
|
switch (output.format) {
|
|
54969
55938
|
case "CSV":
|
|
@@ -54971,16 +55940,17 @@ var init_OutputExecutor = __esm({
|
|
|
54971
55940
|
await driver.transformAndMove(startingPath, (line) => {
|
|
54972
55941
|
const parsed = JSON.parse(line);
|
|
54973
55942
|
return Object.keys(parsed).map((x2) => `"${parsed[x2]}"`).join(",");
|
|
54974
|
-
}, destinationPath)
|
|
55943
|
+
}, destinationPath, header)
|
|
54975
55944
|
);
|
|
54976
55945
|
break;
|
|
54977
55946
|
case "TXT": {
|
|
54978
|
-
const delimiter = output
|
|
55947
|
+
const delimiter = this._getDelimiter(output);
|
|
54979
55948
|
results.push(
|
|
54980
55949
|
await driver.transformAndMove(
|
|
54981
55950
|
startingPath,
|
|
54982
55951
|
(line) => CSVParser_default.stringifyRow(CSVParser_default.parseRow(line, ","), delimiter),
|
|
54983
|
-
destinationPath
|
|
55952
|
+
destinationPath,
|
|
55953
|
+
header
|
|
54984
55954
|
)
|
|
54985
55955
|
);
|
|
54986
55956
|
break;
|
|
@@ -55019,6 +55989,25 @@ var init_OutputExecutor = __esm({
|
|
|
55019
55989
|
resultList[0].res = resultList.every((res) => res.res);
|
|
55020
55990
|
return resultList[0];
|
|
55021
55991
|
};
|
|
55992
|
+
/**
|
|
55993
|
+
* The header line of a positional export: the visible field names, in column order, quoted and
|
|
55994
|
+
* delimited exactly as the rows under them — so the header parses with the same reader the data
|
|
55995
|
+
* does. Null for a JSON/API output, where a record already carries its own field names, and for
|
|
55996
|
+
* an output that turns the header off.
|
|
55997
|
+
*
|
|
55998
|
+
* The names are the exported ones (`finalKey`, i.e. the alias where one is set), since a reader
|
|
55999
|
+
* of the file has nothing else to match a column against.
|
|
56000
|
+
*/
|
|
56001
|
+
this._composeHeader = (output, fields) => {
|
|
56002
|
+
if (output.header === false) return null;
|
|
56003
|
+
if (output.format !== "CSV" && output.format !== "TXT") return null;
|
|
56004
|
+
const names = fields.filter((x2) => !x2.cField.hidden).map((x2) => x2.finalKey);
|
|
56005
|
+
return CSVParser_default.stringifyRow(names, this._getDelimiter(output));
|
|
56006
|
+
};
|
|
56007
|
+
/** The delimiter the exported file is written with. Only TXT configures it; CSV is a comma. */
|
|
56008
|
+
this._getDelimiter = (output) => {
|
|
56009
|
+
return output.format === "TXT" ? output.delimiter ?? " " : ",";
|
|
56010
|
+
};
|
|
55022
56011
|
this._getExtension = (output) => {
|
|
55023
56012
|
return output.format === "CSV" ? "csv" : output.format === "JSON" ? "jsonl" : "txt";
|
|
55024
56013
|
};
|
|
@@ -55039,13 +56028,13 @@ var init_OutputExecutor = __esm({
|
|
|
55039
56028
|
});
|
|
55040
56029
|
|
|
55041
56030
|
// ../../packages/executors/src/ConsumerExecutor.ts
|
|
55042
|
-
var
|
|
56031
|
+
var import_path19, import_fs21, import_promises16, import_crypto10, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
|
|
55043
56032
|
var init_ConsumerExecutor = __esm({
|
|
55044
56033
|
"../../packages/executors/src/ConsumerExecutor.ts"() {
|
|
55045
|
-
|
|
55046
|
-
|
|
56034
|
+
import_path19 = __toESM(require("path"));
|
|
56035
|
+
import_fs21 = __toESM(require("fs"));
|
|
55047
56036
|
import_promises16 = __toESM(require("fs/promises"));
|
|
55048
|
-
|
|
56037
|
+
import_crypto10 = __toESM(require("crypto"));
|
|
55049
56038
|
init_src10();
|
|
55050
56039
|
init_src10();
|
|
55051
56040
|
init_src10();
|
|
@@ -55059,11 +56048,11 @@ var init_ConsumerExecutor = __esm({
|
|
|
55059
56048
|
ConsumerExecutorClass = class {
|
|
55060
56049
|
constructor() {
|
|
55061
56050
|
this._ensurePath = (pathUri) => {
|
|
55062
|
-
const dir =
|
|
55063
|
-
if (!
|
|
55064
|
-
|
|
55065
|
-
if (!
|
|
55066
|
-
|
|
56051
|
+
const dir = import_path19.default.dirname(pathUri);
|
|
56052
|
+
if (!import_fs21.default.existsSync(dir))
|
|
56053
|
+
import_fs21.default.mkdirSync(dir, { recursive: true });
|
|
56054
|
+
if (!import_fs21.default.existsSync(pathUri))
|
|
56055
|
+
import_fs21.default.writeFileSync(pathUri, "");
|
|
55067
56056
|
};
|
|
55068
56057
|
/**
|
|
55069
56058
|
* Shapes one parsed producer record into one consumer record, following a plan compiled once per
|
|
@@ -55184,8 +56173,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
55184
56173
|
if (lineCountFields.length === 0)
|
|
55185
56174
|
return 0;
|
|
55186
56175
|
const tempWorkPath = datasetPath + "_tmp";
|
|
55187
|
-
const reader =
|
|
55188
|
-
const writer =
|
|
56176
|
+
const reader = import_fs21.default.createReadStream(datasetPath);
|
|
56177
|
+
const writer = import_fs21.default.createWriteStream(tempWorkPath);
|
|
55189
56178
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
55190
56179
|
let workerIndex = 0;
|
|
55191
56180
|
let currentWorker = executorResults[workerIndex] ?? null;
|
|
@@ -55274,14 +56263,14 @@ var init_ConsumerExecutor = __esm({
|
|
|
55274
56263
|
}
|
|
55275
56264
|
};
|
|
55276
56265
|
this.processDistinct = async (datasetPath) => {
|
|
55277
|
-
const reader =
|
|
56266
|
+
const reader = import_fs21.default.createReadStream(datasetPath);
|
|
55278
56267
|
const tempWorkPath = datasetPath + "_tmp";
|
|
55279
|
-
const writer =
|
|
56268
|
+
const writer = import_fs21.default.createWriteStream(tempWorkPath);
|
|
55280
56269
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
55281
56270
|
let newLineCount = 0;
|
|
55282
56271
|
const seen = /* @__PURE__ */ new Set();
|
|
55283
56272
|
for await (const line of readLines(reader)) {
|
|
55284
|
-
const hash =
|
|
56273
|
+
const hash = import_crypto10.default.createHash("sha1").update(line).digest("base64");
|
|
55285
56274
|
if (!seen.has(hash)) {
|
|
55286
56275
|
seen.add(hash);
|
|
55287
56276
|
if (!writer.write(line + "\n"))
|
|
@@ -55305,7 +56294,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55305
56294
|
return newLineCount;
|
|
55306
56295
|
};
|
|
55307
56296
|
this.processDistinctOn = async (consumer, datasetPath) => {
|
|
55308
|
-
const reader =
|
|
56297
|
+
const reader = import_fs21.default.createReadStream(datasetPath);
|
|
55309
56298
|
const { distinctOn } = consumer.options;
|
|
55310
56299
|
const { keys, resolution } = distinctOn;
|
|
55311
56300
|
const { strategy, orderBy, direction = "asc" } = resolution;
|
|
@@ -55326,7 +56315,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55326
56315
|
}
|
|
55327
56316
|
}
|
|
55328
56317
|
const tempWorkPath = datasetPath + "_tmp";
|
|
55329
|
-
const writer =
|
|
56318
|
+
const writer = import_fs21.default.createWriteStream(tempWorkPath);
|
|
55330
56319
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
55331
56320
|
for (const { line } of winners.values()) {
|
|
55332
56321
|
if (!writer.write(line + "\n"))
|
|
@@ -55356,7 +56345,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55356
56345
|
if (!pivotValues) {
|
|
55357
56346
|
pivotValues = [];
|
|
55358
56347
|
const discoverySet = /* @__PURE__ */ new Set();
|
|
55359
|
-
const discoverReader =
|
|
56348
|
+
const discoverReader = import_fs21.default.createReadStream(datasetPath);
|
|
55360
56349
|
for await (const line of readLines(discoverReader)) {
|
|
55361
56350
|
const record = this._parseLine(line, internalRecordFormat, internalFields);
|
|
55362
56351
|
const val = String(record[pivotColumn] ?? "");
|
|
@@ -55373,7 +56362,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55373
56362
|
}
|
|
55374
56363
|
}
|
|
55375
56364
|
const groups = /* @__PURE__ */ new Map();
|
|
55376
|
-
const reader =
|
|
56365
|
+
const reader = import_fs21.default.createReadStream(datasetPath);
|
|
55377
56366
|
for await (const line of readLines(reader)) {
|
|
55378
56367
|
const record = this._parseLine(line, internalRecordFormat, internalFields);
|
|
55379
56368
|
const compositeKey = rowKeys.map((k7) => String(record[k7] ?? "")).join("|");
|
|
@@ -55400,7 +56389,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55400
56389
|
];
|
|
55401
56390
|
const outputPlan = OutputExecutor_default.compileOutput(consumer, pivotedFields);
|
|
55402
56391
|
const tempWorkPath = datasetPath + "_tmp";
|
|
55403
|
-
const writer =
|
|
56392
|
+
const writer = import_fs21.default.createWriteStream(tempWorkPath);
|
|
55404
56393
|
const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
|
|
55405
56394
|
let outputCount = 0;
|
|
55406
56395
|
for (const { rowRecord, cells } of groups.values()) {
|
|
@@ -55493,7 +56482,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55493
56482
|
for (const fieldKey of uniqueFieldKeys) {
|
|
55494
56483
|
fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
|
|
55495
56484
|
}
|
|
55496
|
-
const reader =
|
|
56485
|
+
const reader = import_fs21.default.createReadStream(datasetPath);
|
|
55497
56486
|
for await (const line of readLines(reader)) {
|
|
55498
56487
|
rowCount++;
|
|
55499
56488
|
if (checkDuplicateRows) {
|
|
@@ -55541,7 +56530,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
55541
56530
|
});
|
|
55542
56531
|
|
|
55543
56532
|
// ../../packages/executors/src/ProducerExecutor.ts
|
|
55544
|
-
var
|
|
56533
|
+
var import_path20, ProducerExecutorClass, ProducerExecutor, ProducerExecutor_default;
|
|
55545
56534
|
var init_ProducerExecutor = __esm({
|
|
55546
56535
|
"../../packages/executors/src/ProducerExecutor.ts"() {
|
|
55547
56536
|
init_src();
|
|
@@ -55549,7 +56538,7 @@ var init_ProducerExecutor = __esm({
|
|
|
55549
56538
|
init_src4();
|
|
55550
56539
|
init_src10();
|
|
55551
56540
|
init_src10();
|
|
55552
|
-
|
|
56541
|
+
import_path20 = __toESM(require("path"));
|
|
55553
56542
|
init_src2();
|
|
55554
56543
|
ProducerExecutorClass = class {
|
|
55555
56544
|
constructor() {
|
|
@@ -55584,7 +56573,7 @@ var init_ProducerExecutor = __esm({
|
|
|
55584
56573
|
this.compileLine = (dimensions, chunk) => {
|
|
55585
56574
|
Affirm_default(dimensions, "Invalid producer dimensions");
|
|
55586
56575
|
Affirm_default(chunk, "Invalid chunk");
|
|
55587
|
-
return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename:
|
|
56576
|
+
return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename: import_path20.default.basename(chunk.fileUri) };
|
|
55588
56577
|
};
|
|
55589
56578
|
/**
|
|
55590
56579
|
* The record(s) one source line produces. Almost always exactly one — a producer with a group
|
|
@@ -55640,10 +56629,10 @@ var init_ExecutorPerformance = __esm({
|
|
|
55640
56629
|
});
|
|
55641
56630
|
|
|
55642
56631
|
// ../../packages/executors/src/ExecutorWriter.ts
|
|
55643
|
-
var
|
|
56632
|
+
var fs26, ExecutorWriter, ExecutorWriter_default;
|
|
55644
56633
|
var init_ExecutorWriter = __esm({
|
|
55645
56634
|
"../../packages/executors/src/ExecutorWriter.ts"() {
|
|
55646
|
-
|
|
56635
|
+
fs26 = __toESM(require("fs"));
|
|
55647
56636
|
init_src();
|
|
55648
56637
|
init_src3();
|
|
55649
56638
|
init_src10();
|
|
@@ -55695,7 +56684,7 @@ var init_ExecutorWriter = __esm({
|
|
|
55695
56684
|
await this._route(pass, fields);
|
|
55696
56685
|
if (pass.buckets.size === 0)
|
|
55697
56686
|
return [{ path: sourcePath, label: this.UNSPLIT_LABEL }];
|
|
55698
|
-
await
|
|
56687
|
+
await fs26.promises.unlink(sourcePath);
|
|
55699
56688
|
return this._listFiles(pass);
|
|
55700
56689
|
};
|
|
55701
56690
|
/**
|
|
@@ -55703,7 +56692,7 @@ var init_ExecutorWriter = __esm({
|
|
|
55703
56692
|
*/
|
|
55704
56693
|
this._route = async (pass, fields) => {
|
|
55705
56694
|
const internalFormat = OutputExecutor_default._getInternalRecordFormat(pass.consumer);
|
|
55706
|
-
const readStream =
|
|
56695
|
+
const readStream = fs26.createReadStream(pass.sourcePath);
|
|
55707
56696
|
try {
|
|
55708
56697
|
for await (const line of readLines(readStream)) {
|
|
55709
56698
|
const bucket = this._bucketFor(line, pass, internalFormat, fields);
|
|
@@ -55807,7 +56796,7 @@ var init_ExecutorWriter = __esm({
|
|
|
55807
56796
|
}
|
|
55808
56797
|
if (pass.openStreams.length >= Constants_default.defaults.MAX_OPEN_SPLIT_STREAMS)
|
|
55809
56798
|
await this._closeStream(pass.openStreams.shift());
|
|
55810
|
-
bucket.stream =
|
|
56799
|
+
bucket.stream = fs26.createWriteStream(this.getCompletedPath(pass.sourcePath, this._composeLabel(bucket, pass)), { flags: "a" });
|
|
55811
56800
|
pass.openStreams.push(bucket);
|
|
55812
56801
|
return bucket.stream;
|
|
55813
56802
|
};
|
|
@@ -55914,19 +56903,19 @@ var init_ExecutorProgress2 = __esm({
|
|
|
55914
56903
|
});
|
|
55915
56904
|
|
|
55916
56905
|
// ../../packages/executors/src/OrchestratorHelper.ts
|
|
55917
|
-
var
|
|
56906
|
+
var import_os3, import_path21, OrchestratorHelper, OrchestratorHelper_default;
|
|
55918
56907
|
var init_OrchestratorHelper = __esm({
|
|
55919
56908
|
"../../packages/executors/src/OrchestratorHelper.ts"() {
|
|
55920
56909
|
init_src();
|
|
55921
56910
|
init_src6();
|
|
55922
|
-
|
|
56911
|
+
import_os3 = __toESM(require("os"));
|
|
55923
56912
|
init_src4();
|
|
55924
|
-
|
|
56913
|
+
import_path21 = __toESM(require("path"));
|
|
55925
56914
|
init_src3();
|
|
55926
56915
|
OrchestratorHelper = {
|
|
55927
56916
|
getMemoryUsage: () => {
|
|
55928
56917
|
const processMemory = process.memoryUsage();
|
|
55929
|
-
const freeSystemMemory =
|
|
56918
|
+
const freeSystemMemory = import_os3.default.freemem();
|
|
55930
56919
|
return {
|
|
55931
56920
|
/**
|
|
55932
56921
|
* resident set size (heap + code + stack)
|
|
@@ -56013,14 +57002,14 @@ var init_OrchestratorHelper = __esm({
|
|
|
56013
57002
|
getPhysicalWorkerPath: () => {
|
|
56014
57003
|
const currentDir = __dirname;
|
|
56015
57004
|
if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
|
|
56016
|
-
return
|
|
57005
|
+
return import_path21.default.resolve("./.build/workers");
|
|
56017
57006
|
const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
|
|
56018
57007
|
if (forcedPath && forcedPath.length > 0)
|
|
56019
|
-
return
|
|
57008
|
+
return import_path21.default.join(__dirname, forcedPath);
|
|
56020
57009
|
if (!currentDir.includes(".build")) {
|
|
56021
|
-
return
|
|
57010
|
+
return import_path21.default.join(__dirname, "../workers");
|
|
56022
57011
|
} else {
|
|
56023
|
-
return
|
|
57012
|
+
return import_path21.default.resolve("./.build/workers");
|
|
56024
57013
|
}
|
|
56025
57014
|
},
|
|
56026
57015
|
/**
|
|
@@ -56032,8 +57021,8 @@ var init_OrchestratorHelper = __esm({
|
|
|
56032
57021
|
* fix (a 4GB box previously got `cpus - 1` workers, each permitted 2000MB).
|
|
56033
57022
|
*/
|
|
56034
57023
|
getParallelWorkerCount: () => {
|
|
56035
|
-
const cpuBoundWorkers = Math.max(1, Math.floor(
|
|
56036
|
-
const totalMemoryMB = Math.floor(
|
|
57024
|
+
const cpuBoundWorkers = Math.max(1, Math.floor(import_os3.default.cpus().length * 0.7));
|
|
57025
|
+
const totalMemoryMB = Math.floor(import_os3.default.totalmem() / (1024 * 1024));
|
|
56037
57026
|
const reservedMemoryMB = Constants_default.defaults.MIN_RUNTIME_HEAP_MB * 2;
|
|
56038
57027
|
const availableMemoryForWorkersMB = Math.max(Constants_default.defaults.MIN_RUNTIME_HEAP_MB, totalMemoryMB - reservedMemoryMB);
|
|
56039
57028
|
const memoryBoundWorkers = Math.max(1, Math.floor(availableMemoryForWorkersMB / Constants_default.defaults.MIN_RUNTIME_HEAP_MB));
|
|
@@ -56085,12 +57074,12 @@ var init_OrchestratorHelper = __esm({
|
|
|
56085
57074
|
});
|
|
56086
57075
|
|
|
56087
57076
|
// ../../packages/executors/src/LookupResolver.ts
|
|
56088
|
-
var
|
|
57077
|
+
var import_fs23, import_promises17, import_path22, LookupResolverClass, LookupResolver, LookupResolver_default;
|
|
56089
57078
|
var init_LookupResolver = __esm({
|
|
56090
57079
|
"../../packages/executors/src/LookupResolver.ts"() {
|
|
56091
|
-
|
|
57080
|
+
import_fs23 = __toESM(require("fs"));
|
|
56092
57081
|
import_promises17 = __toESM(require("fs/promises"));
|
|
56093
|
-
|
|
57082
|
+
import_path22 = __toESM(require("path"));
|
|
56094
57083
|
init_src();
|
|
56095
57084
|
init_src3();
|
|
56096
57085
|
init_src4();
|
|
@@ -56130,7 +57119,7 @@ var init_LookupResolver = __esm({
|
|
|
56130
57119
|
}
|
|
56131
57120
|
ExecutorScope_default.ensurePath(outPath);
|
|
56132
57121
|
await import_promises17.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
|
|
56133
|
-
manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path:
|
|
57122
|
+
manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path22.default.resolve(outPath) });
|
|
56134
57123
|
}
|
|
56135
57124
|
for (const capturePath of captureByConsumer.values())
|
|
56136
57125
|
await import_promises17.default.rm(capturePath, { force: true });
|
|
@@ -56159,7 +57148,7 @@ var init_LookupResolver = __esm({
|
|
|
56159
57148
|
const dict = {};
|
|
56160
57149
|
let rowsRead = 0;
|
|
56161
57150
|
let warnedDup = false;
|
|
56162
|
-
for await (const line of readLines(
|
|
57151
|
+
for await (const line of readLines(import_fs23.default.createReadStream(capturePath))) {
|
|
56163
57152
|
if (!line.length) continue;
|
|
56164
57153
|
rowsRead++;
|
|
56165
57154
|
Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
|
|
@@ -56179,8 +57168,8 @@ var init_LookupResolver = __esm({
|
|
|
56179
57168
|
return JSON.parse(line);
|
|
56180
57169
|
};
|
|
56181
57170
|
this.getCapturePath = (scope, consumerName) => {
|
|
56182
|
-
const dir =
|
|
56183
|
-
return
|
|
57171
|
+
const dir = import_path22.default.dirname(ExecutorScope_default.getLookupPath(scope, "x"));
|
|
57172
|
+
return import_path22.default.join(dir, `${consumerName}.capture`);
|
|
56184
57173
|
};
|
|
56185
57174
|
this.dedupe = (refs) => {
|
|
56186
57175
|
const byId = /* @__PURE__ */ new Map();
|
|
@@ -56202,12 +57191,12 @@ var init_LookupResolver = __esm({
|
|
|
56202
57191
|
});
|
|
56203
57192
|
|
|
56204
57193
|
// ../../packages/executors/src/ConsumerInputResolver.ts
|
|
56205
|
-
var
|
|
57194
|
+
var import_fs24, import_promises18, import_path23, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
|
|
56206
57195
|
var init_ConsumerInputResolver = __esm({
|
|
56207
57196
|
"../../packages/executors/src/ConsumerInputResolver.ts"() {
|
|
56208
|
-
|
|
57197
|
+
import_fs24 = __toESM(require("fs"));
|
|
56209
57198
|
import_promises18 = __toESM(require("fs/promises"));
|
|
56210
|
-
|
|
57199
|
+
import_path23 = __toESM(require("path"));
|
|
56211
57200
|
init_src();
|
|
56212
57201
|
init_src3();
|
|
56213
57202
|
init_src4();
|
|
@@ -56230,7 +57219,7 @@ var init_ConsumerInputResolver = __esm({
|
|
|
56230
57219
|
Affirm_default(stack.length <= Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH, `consumer-as-input nesting exceeds MAX_CONSUMER_INPUT_DEPTH (${Constants_default.defaults.MAX_CONSUMER_INPUT_DEPTH}): ${stack.join(" -> ")}`);
|
|
56231
57220
|
for (const name of upstreamNames) {
|
|
56232
57221
|
const cached = this.cache.get(name);
|
|
56233
|
-
if (!cached || !
|
|
57222
|
+
if (!cached || !import_fs24.default.existsSync(cached.path)) {
|
|
56234
57223
|
Affirm_default(!stack.includes(name), `consumer-as-input cycle detected: ${[...stack, name].join(" -> ")}`);
|
|
56235
57224
|
const upstream = Environment_default.getConsumer(name);
|
|
56236
57225
|
Affirm_default(upstream, `consumer-as-input references unknown consumer "${name}"`);
|
|
@@ -56238,7 +57227,7 @@ var init_ConsumerInputResolver = __esm({
|
|
|
56238
57227
|
ExecutorScope_default.ensurePath(capturePath);
|
|
56239
57228
|
await this.runUpstreamConsumer(upstream, capturePath, stack, request);
|
|
56240
57229
|
const { size } = await import_promises18.default.stat(capturePath);
|
|
56241
|
-
this.cache.set(name, { consumer: name, path:
|
|
57230
|
+
this.cache.set(name, { consumer: name, path: import_path23.default.resolve(capturePath), fileSize: size });
|
|
56242
57231
|
}
|
|
56243
57232
|
manifest.set(name, this.cache.get(name));
|
|
56244
57233
|
}
|
|
@@ -56300,12 +57289,12 @@ var ExecutorOrchestrator_exports = {};
|
|
|
56300
57289
|
__export(ExecutorOrchestrator_exports, {
|
|
56301
57290
|
default: () => ExecutorOrchestrator_default
|
|
56302
57291
|
});
|
|
56303
|
-
var
|
|
57292
|
+
var import_fs25, import_promises19, import_path24, import_workerpool, import_promises20, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
|
|
56304
57293
|
var init_ExecutorOrchestrator = __esm({
|
|
56305
57294
|
"../../packages/executors/src/ExecutorOrchestrator.ts"() {
|
|
56306
|
-
|
|
57295
|
+
import_fs25 = __toESM(require("fs"));
|
|
56307
57296
|
import_promises19 = __toESM(require("fs/promises"));
|
|
56308
|
-
|
|
57297
|
+
import_path24 = __toESM(require("path"));
|
|
56309
57298
|
import_workerpool = __toESM(require("workerpool"));
|
|
56310
57299
|
init_src();
|
|
56311
57300
|
init_src10();
|
|
@@ -56360,7 +57349,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56360
57349
|
};
|
|
56361
57350
|
const workerPath = OrchestratorHelper_default.getPhysicalWorkerPath();
|
|
56362
57351
|
Logger_default.info(`Initializing worker pool from ${workerPath} (workers: ${maxWorkers}, heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
|
|
56363
|
-
return import_workerpool.default.pool(
|
|
57352
|
+
return import_workerpool.default.pool(import_path24.default.join(workerPath, "ExecutorWorker.js"), options2);
|
|
56364
57353
|
};
|
|
56365
57354
|
this.launch = async (request) => {
|
|
56366
57355
|
Affirm_default(request, "Invalid options");
|
|
@@ -56564,7 +57553,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56564
57553
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": source returned no files, skipping`);
|
|
56565
57554
|
continue;
|
|
56566
57555
|
}
|
|
56567
|
-
if (!
|
|
57556
|
+
if (!import_fs25.default.existsSync(response.files[0].fullUri)) {
|
|
56568
57557
|
if (!cProd.isOptional)
|
|
56569
57558
|
throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
|
|
56570
57559
|
Logger_default.info(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
|
|
@@ -56726,7 +57715,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56726
57715
|
* Returns a single chunk for small files where parallelism overhead isn't worth it.
|
|
56727
57716
|
*/
|
|
56728
57717
|
this.scopeWork = (fileUri, numChunks) => {
|
|
56729
|
-
const fileSize =
|
|
57718
|
+
const fileSize = import_fs25.default.statSync(fileUri).size;
|
|
56730
57719
|
if (fileSize === 0) return [];
|
|
56731
57720
|
if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
|
|
56732
57721
|
return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
@@ -56736,7 +57725,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56736
57725
|
const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
|
|
56737
57726
|
if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
|
|
56738
57727
|
const targetChunkSize = Math.floor(fileSize / effectiveChunks);
|
|
56739
|
-
const fd =
|
|
57728
|
+
const fd = import_fs25.default.openSync(fileUri, "r");
|
|
56740
57729
|
try {
|
|
56741
57730
|
const offsets = [];
|
|
56742
57731
|
let currentStart = 0;
|
|
@@ -56754,7 +57743,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56754
57743
|
}
|
|
56755
57744
|
return offsets;
|
|
56756
57745
|
} finally {
|
|
56757
|
-
|
|
57746
|
+
import_fs25.default.closeSync(fd);
|
|
56758
57747
|
}
|
|
56759
57748
|
};
|
|
56760
57749
|
/**
|
|
@@ -56822,7 +57811,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56822
57811
|
let currentPos = position;
|
|
56823
57812
|
while (currentPos < fileSize) {
|
|
56824
57813
|
const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
|
|
56825
|
-
const bytesRead =
|
|
57814
|
+
const bytesRead = import_fs25.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
|
|
56826
57815
|
if (bytesRead === 0) break;
|
|
56827
57816
|
for (let i6 = 0; i6 < bytesRead; i6++) {
|
|
56828
57817
|
if (buffer[i6] === 10) {
|
|
@@ -56841,16 +57830,16 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56841
57830
|
* format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
|
|
56842
57831
|
*/
|
|
56843
57832
|
this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
|
|
56844
|
-
await import_promises19.default.mkdir(
|
|
57833
|
+
await import_promises19.default.mkdir(import_path24.default.dirname(targetPath), { recursive: true });
|
|
56845
57834
|
const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
|
|
56846
57835
|
if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
|
|
56847
57836
|
await import_promises19.default.copyFile(sourcePath, targetPath);
|
|
56848
57837
|
return;
|
|
56849
57838
|
}
|
|
56850
57839
|
const visibleFields = fields.filter((field) => !field.cField.hidden);
|
|
56851
|
-
const output =
|
|
57840
|
+
const output = import_fs25.default.createWriteStream(targetPath);
|
|
56852
57841
|
try {
|
|
56853
|
-
for await (const line of readLines(
|
|
57842
|
+
for await (const line of readLines(import_fs25.default.createReadStream(sourcePath))) {
|
|
56854
57843
|
if (!line.length) continue;
|
|
56855
57844
|
const record = LineParser_default._internalParseCSV(line, visibleFields, ",");
|
|
56856
57845
|
if (!output.write(JSON.stringify(record) + "\n"))
|
|
@@ -56938,7 +57927,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56938
57927
|
const decodedFiles = [];
|
|
56939
57928
|
let decodedCount = 0;
|
|
56940
57929
|
for (const file of response.files) {
|
|
56941
|
-
const inferredType = fileType ??
|
|
57930
|
+
const inferredType = fileType ?? import_path24.default.extname(file.fullUri).replace(".", "").toUpperCase();
|
|
56942
57931
|
const decodedPath = `${file.fullUri}.decoded.csv`;
|
|
56943
57932
|
if (inferredType === "XLS" || inferredType === "XLSX") {
|
|
56944
57933
|
const xlsCsvStream = await XLSParser_default.getStreamXls(file.fullUri, {
|
|
@@ -56948,7 +57937,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
56948
57937
|
});
|
|
56949
57938
|
await (0, import_promises20.pipeline)(
|
|
56950
57939
|
xlsCsvStream,
|
|
56951
|
-
|
|
57940
|
+
import_fs25.default.createWriteStream(decodedPath)
|
|
56952
57941
|
);
|
|
56953
57942
|
const fileStats = await import_promises19.default.stat(decodedPath);
|
|
56954
57943
|
decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
|
|
@@ -57008,8 +57997,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
57008
57997
|
const perf = performance.now();
|
|
57009
57998
|
for (const workerResult of executorResults) {
|
|
57010
57999
|
await (0, import_promises20.pipeline)(
|
|
57011
|
-
|
|
57012
|
-
|
|
58000
|
+
import_fs25.default.createReadStream(workerResult.resultUri),
|
|
58001
|
+
import_fs25.default.createWriteStream(mainPath, { flags: "a" })
|
|
57013
58002
|
);
|
|
57014
58003
|
await import_promises19.default.unlink(workerResult.resultUri);
|
|
57015
58004
|
}
|
|
@@ -57047,7 +58036,7 @@ init_src();
|
|
|
57047
58036
|
init_ConsumerExecutor();
|
|
57048
58037
|
|
|
57049
58038
|
// ../../packages/executors/src/Executor.ts
|
|
57050
|
-
var
|
|
58039
|
+
var import_fs22 = __toESM(require("fs"));
|
|
57051
58040
|
init_ProducerExecutor();
|
|
57052
58041
|
init_ConsumerExecutor();
|
|
57053
58042
|
|
|
@@ -57337,18 +58326,18 @@ var Executor = class {
|
|
|
57337
58326
|
if (!lookups?.length) return;
|
|
57338
58327
|
for (const entry of lookups) {
|
|
57339
58328
|
if (LookupRegistry_default.has(entry.id)) continue;
|
|
57340
|
-
const table = JSON.parse(
|
|
58329
|
+
const table = JSON.parse(import_fs22.default.readFileSync(entry.path, "utf-8"));
|
|
57341
58330
|
LookupRegistry_default.set(entry.id, table);
|
|
57342
58331
|
Logger_default.log(`[lookup] Hydrated table "${entry.consumer}" (${entry.key}->${entry.value}) from ${entry.path}`);
|
|
57343
58332
|
}
|
|
57344
58333
|
};
|
|
57345
58334
|
this.openReadStream = (chunk) => {
|
|
57346
58335
|
const { end, fileUri, start } = chunk;
|
|
57347
|
-
return
|
|
58336
|
+
return import_fs22.default.createReadStream(fileUri, { start, end: end - 1 });
|
|
57348
58337
|
};
|
|
57349
58338
|
this.openWriteStream = (scope, workerId) => {
|
|
57350
58339
|
const workerPath = ExecutorScope_default.getWorkerPath(scope, workerId);
|
|
57351
|
-
return new BatchWriter_default(
|
|
58340
|
+
return new BatchWriter_default(import_fs22.default.createWriteStream(workerPath));
|
|
57352
58341
|
};
|
|
57353
58342
|
// The first line of the first chunk is data unless the file type carries a header there.
|
|
57354
58343
|
this.shouldProcessFirstLine = (producer) => {
|
|
@@ -57454,9 +58443,9 @@ init_OutputExecutor();
|
|
|
57454
58443
|
init_ProducerExecutor();
|
|
57455
58444
|
|
|
57456
58445
|
// ../../packages/executors/src/SampleRunner.ts
|
|
57457
|
-
var
|
|
58446
|
+
var import_fs26 = __toESM(require("fs"));
|
|
57458
58447
|
var import_promises21 = __toESM(require("fs/promises"));
|
|
57459
|
-
var
|
|
58448
|
+
var import_path25 = __toESM(require("path"));
|
|
57460
58449
|
init_src();
|
|
57461
58450
|
init_src3();
|
|
57462
58451
|
init_src4();
|
|
@@ -57501,7 +58490,7 @@ var SampleRunnerClass = class {
|
|
|
57501
58490
|
*/
|
|
57502
58491
|
this._runConsumer = async (consumer, size, details) => {
|
|
57503
58492
|
this._assertStageable(consumer);
|
|
57504
|
-
const capturePath =
|
|
58493
|
+
const capturePath = import_path25.default.resolve(import_path25.default.join(ExecutorScope_default.getBasePath(), `sample_${consumer.name}_${Helper_default.uuid()}.jsonl`));
|
|
57505
58494
|
ExecutorScope_default.ensurePath(capturePath);
|
|
57506
58495
|
try {
|
|
57507
58496
|
const result = await ExecutorOrchestrator_default.launch({
|
|
@@ -57591,9 +58580,9 @@ var SampleRunnerClass = class {
|
|
|
57591
58580
|
};
|
|
57592
58581
|
/** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
|
|
57593
58582
|
this._readCapture = async (capturePath, size) => {
|
|
57594
|
-
if (!
|
|
58583
|
+
if (!import_fs26.default.existsSync(capturePath)) return [];
|
|
57595
58584
|
const records = [];
|
|
57596
|
-
for await (const line of readLines(
|
|
58585
|
+
for await (const line of readLines(import_fs26.default.createReadStream(capturePath))) {
|
|
57597
58586
|
if (!line.length) continue;
|
|
57598
58587
|
records.push(JSON.parse(line));
|
|
57599
58588
|
if (records.length >= size) break;
|