@forzalabs/remora 2.0.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/index.js +979 -515
- package/package.json +3 -2
- package/workers/ExecutorWorker.js +899 -468
package/index.js
CHANGED
|
@@ -1190,8 +1190,8 @@ var require_levels = __commonJS({
|
|
|
1190
1190
|
var require_align = __commonJS({
|
|
1191
1191
|
"../../node_modules/logform/align.js"(exports2, module2) {
|
|
1192
1192
|
"use strict";
|
|
1193
|
-
var
|
|
1194
|
-
module2.exports =
|
|
1193
|
+
var format4 = require_format();
|
|
1194
|
+
module2.exports = format4((info) => {
|
|
1195
1195
|
info.message = ` ${info.message}`;
|
|
1196
1196
|
return info;
|
|
1197
1197
|
});
|
|
@@ -1202,9 +1202,9 @@ var require_align = __commonJS({
|
|
|
1202
1202
|
var require_errors = __commonJS({
|
|
1203
1203
|
"../../node_modules/logform/errors.js"(exports2, module2) {
|
|
1204
1204
|
"use strict";
|
|
1205
|
-
var
|
|
1205
|
+
var format4 = require_format();
|
|
1206
1206
|
var { LEVEL, MESSAGE } = require_triple_beam();
|
|
1207
|
-
module2.exports =
|
|
1207
|
+
module2.exports = format4((einfo, { stack, cause }) => {
|
|
1208
1208
|
if (einfo instanceof Error) {
|
|
1209
1209
|
const info = Object.assign({}, einfo, {
|
|
1210
1210
|
level: einfo.level,
|
|
@@ -1340,7 +1340,7 @@ var require_cli2 = __commonJS({
|
|
|
1340
1340
|
var require_combine = __commonJS({
|
|
1341
1341
|
"../../node_modules/logform/combine.js"(exports2, module2) {
|
|
1342
1342
|
"use strict";
|
|
1343
|
-
var
|
|
1343
|
+
var format4 = require_format();
|
|
1344
1344
|
function cascade(formats) {
|
|
1345
1345
|
if (!formats.every(isValidFormat)) {
|
|
1346
1346
|
return;
|
|
@@ -1367,7 +1367,7 @@ var require_combine = __commonJS({
|
|
|
1367
1367
|
return true;
|
|
1368
1368
|
}
|
|
1369
1369
|
module2.exports = (...formats) => {
|
|
1370
|
-
const combinedFormat =
|
|
1370
|
+
const combinedFormat = format4(cascade(formats));
|
|
1371
1371
|
const instance = combinedFormat();
|
|
1372
1372
|
instance.Format = combinedFormat.Format;
|
|
1373
1373
|
return instance;
|
|
@@ -1976,7 +1976,7 @@ ${originalIndentation}`;
|
|
|
1976
1976
|
var require_json = __commonJS({
|
|
1977
1977
|
"../../node_modules/logform/json.js"(exports2, module2) {
|
|
1978
1978
|
"use strict";
|
|
1979
|
-
var
|
|
1979
|
+
var format4 = require_format();
|
|
1980
1980
|
var { MESSAGE } = require_triple_beam();
|
|
1981
1981
|
var stringify = require_safe_stable_stringify();
|
|
1982
1982
|
function replacer(key, value) {
|
|
@@ -1984,7 +1984,7 @@ var require_json = __commonJS({
|
|
|
1984
1984
|
return value.toString();
|
|
1985
1985
|
return value;
|
|
1986
1986
|
}
|
|
1987
|
-
module2.exports =
|
|
1987
|
+
module2.exports = format4((info, opts) => {
|
|
1988
1988
|
const jsonStringify = stringify.configure(opts);
|
|
1989
1989
|
info[MESSAGE] = jsonStringify(info, opts.replacer || replacer, opts.space);
|
|
1990
1990
|
return info;
|
|
@@ -1996,8 +1996,8 @@ var require_json = __commonJS({
|
|
|
1996
1996
|
var require_label = __commonJS({
|
|
1997
1997
|
"../../node_modules/logform/label.js"(exports2, module2) {
|
|
1998
1998
|
"use strict";
|
|
1999
|
-
var
|
|
2000
|
-
module2.exports =
|
|
1999
|
+
var format4 = require_format();
|
|
2000
|
+
module2.exports = format4((info, opts) => {
|
|
2001
2001
|
if (opts.message) {
|
|
2002
2002
|
info.message = `[${opts.label}] ${info.message}`;
|
|
2003
2003
|
return info;
|
|
@@ -2012,10 +2012,10 @@ var require_label = __commonJS({
|
|
|
2012
2012
|
var require_logstash = __commonJS({
|
|
2013
2013
|
"../../node_modules/logform/logstash.js"(exports2, module2) {
|
|
2014
2014
|
"use strict";
|
|
2015
|
-
var
|
|
2015
|
+
var format4 = require_format();
|
|
2016
2016
|
var { MESSAGE } = require_triple_beam();
|
|
2017
2017
|
var jsonStringify = require_safe_stable_stringify();
|
|
2018
|
-
module2.exports =
|
|
2018
|
+
module2.exports = format4((info) => {
|
|
2019
2019
|
const logstash = {};
|
|
2020
2020
|
if (info.message) {
|
|
2021
2021
|
logstash["@message"] = info.message;
|
|
@@ -2036,7 +2036,7 @@ var require_logstash = __commonJS({
|
|
|
2036
2036
|
var require_metadata = __commonJS({
|
|
2037
2037
|
"../../node_modules/logform/metadata.js"(exports2, module2) {
|
|
2038
2038
|
"use strict";
|
|
2039
|
-
var
|
|
2039
|
+
var format4 = require_format();
|
|
2040
2040
|
function fillExcept(info, fillExceptKeys, metadataKey) {
|
|
2041
2041
|
const savedKeys = fillExceptKeys.reduce((acc, key) => {
|
|
2042
2042
|
acc[key] = info[key];
|
|
@@ -2061,7 +2061,7 @@ var require_metadata = __commonJS({
|
|
|
2061
2061
|
}, {});
|
|
2062
2062
|
return info;
|
|
2063
2063
|
}
|
|
2064
|
-
module2.exports =
|
|
2064
|
+
module2.exports = format4((info, opts = {}) => {
|
|
2065
2065
|
let metadataKey = "metadata";
|
|
2066
2066
|
if (opts.key) {
|
|
2067
2067
|
metadataKey = opts.key;
|
|
@@ -2205,9 +2205,9 @@ var require_ms = __commonJS({
|
|
|
2205
2205
|
var require_ms2 = __commonJS({
|
|
2206
2206
|
"../../node_modules/logform/ms.js"(exports2, module2) {
|
|
2207
2207
|
"use strict";
|
|
2208
|
-
var
|
|
2208
|
+
var format4 = require_format();
|
|
2209
2209
|
var ms2 = require_ms();
|
|
2210
|
-
module2.exports =
|
|
2210
|
+
module2.exports = format4((info) => {
|
|
2211
2211
|
const curr = +/* @__PURE__ */ new Date();
|
|
2212
2212
|
exports2.diff = curr - (exports2.prevTime || curr);
|
|
2213
2213
|
exports2.prevTime = curr;
|
|
@@ -2222,9 +2222,9 @@ var require_pretty_print = __commonJS({
|
|
|
2222
2222
|
"../../node_modules/logform/pretty-print.js"(exports2, module2) {
|
|
2223
2223
|
"use strict";
|
|
2224
2224
|
var inspect = require("util").inspect;
|
|
2225
|
-
var
|
|
2225
|
+
var format4 = require_format();
|
|
2226
2226
|
var { LEVEL, MESSAGE, SPLAT } = require_triple_beam();
|
|
2227
|
-
module2.exports =
|
|
2227
|
+
module2.exports = format4((info, opts = {}) => {
|
|
2228
2228
|
const stripped = Object.assign({}, info);
|
|
2229
2229
|
delete stripped[LEVEL];
|
|
2230
2230
|
delete stripped[MESSAGE];
|
|
@@ -2258,10 +2258,10 @@ var require_printf = __commonJS({
|
|
|
2258
2258
|
var require_simple = __commonJS({
|
|
2259
2259
|
"../../node_modules/logform/simple.js"(exports2, module2) {
|
|
2260
2260
|
"use strict";
|
|
2261
|
-
var
|
|
2261
|
+
var format4 = require_format();
|
|
2262
2262
|
var { MESSAGE } = require_triple_beam();
|
|
2263
2263
|
var jsonStringify = require_safe_stable_stringify();
|
|
2264
|
-
module2.exports =
|
|
2264
|
+
module2.exports = format4((info) => {
|
|
2265
2265
|
const stringifiedRest = jsonStringify(Object.assign({}, info, {
|
|
2266
2266
|
level: void 0,
|
|
2267
2267
|
message: void 0,
|
|
@@ -2383,14 +2383,14 @@ function assign(origObj) {
|
|
|
2383
2383
|
}
|
|
2384
2384
|
return origObj;
|
|
2385
2385
|
}
|
|
2386
|
-
function parse(dateStr,
|
|
2386
|
+
function parse(dateStr, format4, i18n) {
|
|
2387
2387
|
if (i18n === void 0) {
|
|
2388
2388
|
i18n = {};
|
|
2389
2389
|
}
|
|
2390
|
-
if (typeof
|
|
2390
|
+
if (typeof format4 !== "string") {
|
|
2391
2391
|
throw new Error("Invalid format in fecha parse");
|
|
2392
2392
|
}
|
|
2393
|
-
|
|
2393
|
+
format4 = globalMasks[format4] || format4;
|
|
2394
2394
|
if (dateStr.length > 1e3) {
|
|
2395
2395
|
return null;
|
|
2396
2396
|
}
|
|
@@ -2408,7 +2408,7 @@ function parse(dateStr, format3, i18n) {
|
|
|
2408
2408
|
};
|
|
2409
2409
|
var parseInfo = [];
|
|
2410
2410
|
var literals = [];
|
|
2411
|
-
var newFormat =
|
|
2411
|
+
var newFormat = format4.replace(literal, function($0, $1) {
|
|
2412
2412
|
literals.push(regexEscape($1));
|
|
2413
2413
|
return "@@@";
|
|
2414
2414
|
});
|
|
@@ -2770,8 +2770,8 @@ var require_timestamp = __commonJS({
|
|
|
2770
2770
|
"../../node_modules/logform/timestamp.js"(exports2, module2) {
|
|
2771
2771
|
"use strict";
|
|
2772
2772
|
var fecha2 = (init_fecha(), __toCommonJS(fecha_exports));
|
|
2773
|
-
var
|
|
2774
|
-
module2.exports =
|
|
2773
|
+
var format4 = require_format();
|
|
2774
|
+
module2.exports = format4((info, opts = {}) => {
|
|
2775
2775
|
if (opts.format) {
|
|
2776
2776
|
info.timestamp = typeof opts.format === "function" ? opts.format() : fecha2.format(/* @__PURE__ */ new Date(), opts.format);
|
|
2777
2777
|
}
|
|
@@ -2791,9 +2791,9 @@ var require_uncolorize = __commonJS({
|
|
|
2791
2791
|
"../../node_modules/logform/uncolorize.js"(exports2, module2) {
|
|
2792
2792
|
"use strict";
|
|
2793
2793
|
var colors = require_safe();
|
|
2794
|
-
var
|
|
2794
|
+
var format4 = require_format();
|
|
2795
2795
|
var { MESSAGE } = require_triple_beam();
|
|
2796
|
-
module2.exports =
|
|
2796
|
+
module2.exports = format4((info, opts) => {
|
|
2797
2797
|
if (opts.level !== false) {
|
|
2798
2798
|
info.level = colors.strip(info.level);
|
|
2799
2799
|
}
|
|
@@ -2812,10 +2812,10 @@ var require_uncolorize = __commonJS({
|
|
|
2812
2812
|
var require_logform = __commonJS({
|
|
2813
2813
|
"../../node_modules/logform/index.js"(exports2) {
|
|
2814
2814
|
"use strict";
|
|
2815
|
-
var
|
|
2815
|
+
var format4 = exports2.format = require_format();
|
|
2816
2816
|
exports2.levels = require_levels();
|
|
2817
2817
|
function exposeFormat(name, requireFormat) {
|
|
2818
|
-
Object.defineProperty(
|
|
2818
|
+
Object.defineProperty(format4, name, {
|
|
2819
2819
|
get() {
|
|
2820
2820
|
return requireFormat();
|
|
2821
2821
|
},
|
|
@@ -2880,17 +2880,17 @@ var require_logform = __commonJS({
|
|
|
2880
2880
|
var require_common = __commonJS({
|
|
2881
2881
|
"../../node_modules/winston/lib/winston/common.js"(exports2) {
|
|
2882
2882
|
"use strict";
|
|
2883
|
-
var { format:
|
|
2883
|
+
var { format: format4 } = require("util");
|
|
2884
2884
|
exports2.warn = {
|
|
2885
2885
|
deprecated(prop) {
|
|
2886
2886
|
return () => {
|
|
2887
|
-
throw new Error(
|
|
2887
|
+
throw new Error(format4("{ %s } was removed in winston@3.0.0.", prop));
|
|
2888
2888
|
};
|
|
2889
2889
|
},
|
|
2890
2890
|
useFormat(prop) {
|
|
2891
2891
|
return () => {
|
|
2892
2892
|
throw new Error([
|
|
2893
|
-
|
|
2893
|
+
format4("{ %s } was removed in winston@3.0.0.", prop),
|
|
2894
2894
|
"Use a custom winston.format = winston.format(function) instead."
|
|
2895
2895
|
].join("\n"));
|
|
2896
2896
|
};
|
|
@@ -4155,14 +4155,14 @@ var require_from = __commonJS({
|
|
|
4155
4155
|
return (hint === "string" ? String : Number)(input);
|
|
4156
4156
|
}
|
|
4157
4157
|
var ERR_INVALID_ARG_TYPE = require_errors2().codes.ERR_INVALID_ARG_TYPE;
|
|
4158
|
-
function from(
|
|
4158
|
+
function from(Readable8, iterable, opts) {
|
|
4159
4159
|
var iterator;
|
|
4160
4160
|
if (iterable && typeof iterable.next === "function") {
|
|
4161
4161
|
iterator = iterable;
|
|
4162
4162
|
} else if (iterable && iterable[Symbol.asyncIterator]) iterator = iterable[Symbol.asyncIterator]();
|
|
4163
4163
|
else if (iterable && iterable[Symbol.iterator]) iterator = iterable[Symbol.iterator]();
|
|
4164
4164
|
else throw new ERR_INVALID_ARG_TYPE("iterable", ["Iterable"], iterable);
|
|
4165
|
-
var readable = new
|
|
4165
|
+
var readable = new Readable8(_objectSpread({
|
|
4166
4166
|
objectMode: true
|
|
4167
4167
|
}, opts));
|
|
4168
4168
|
var reading = false;
|
|
@@ -4202,9 +4202,9 @@ var require_from = __commonJS({
|
|
|
4202
4202
|
var require_stream_readable = __commonJS({
|
|
4203
4203
|
"../../node_modules/readable-stream/lib/_stream_readable.js"(exports2, module2) {
|
|
4204
4204
|
"use strict";
|
|
4205
|
-
module2.exports =
|
|
4205
|
+
module2.exports = Readable8;
|
|
4206
4206
|
var Duplex;
|
|
4207
|
-
|
|
4207
|
+
Readable8.ReadableState = ReadableState;
|
|
4208
4208
|
var EE = require("events").EventEmitter;
|
|
4209
4209
|
var EElistenerCount = function EElistenerCount2(emitter, type) {
|
|
4210
4210
|
return emitter.listeners(type).length;
|
|
@@ -4239,7 +4239,7 @@ var require_stream_readable = __commonJS({
|
|
|
4239
4239
|
var StringDecoder;
|
|
4240
4240
|
var createReadableStreamAsyncIterator;
|
|
4241
4241
|
var from;
|
|
4242
|
-
require_inherits()(
|
|
4242
|
+
require_inherits()(Readable8, Stream);
|
|
4243
4243
|
var errorOrDestroy = destroyImpl.errorOrDestroy;
|
|
4244
4244
|
var kProxyEvents = ["error", "close", "destroy", "pause", "resume"];
|
|
4245
4245
|
function prependListener(emitter, event, fn) {
|
|
@@ -4283,9 +4283,9 @@ var require_stream_readable = __commonJS({
|
|
|
4283
4283
|
this.encoding = options.encoding;
|
|
4284
4284
|
}
|
|
4285
4285
|
}
|
|
4286
|
-
function
|
|
4286
|
+
function Readable8(options) {
|
|
4287
4287
|
Duplex = Duplex || require_stream_duplex();
|
|
4288
|
-
if (!(this instanceof
|
|
4288
|
+
if (!(this instanceof Readable8)) return new Readable8(options);
|
|
4289
4289
|
var isDuplex = this instanceof Duplex;
|
|
4290
4290
|
this._readableState = new ReadableState(options, this, isDuplex);
|
|
4291
4291
|
this.readable = true;
|
|
@@ -4295,7 +4295,7 @@ var require_stream_readable = __commonJS({
|
|
|
4295
4295
|
}
|
|
4296
4296
|
Stream.call(this);
|
|
4297
4297
|
}
|
|
4298
|
-
Object.defineProperty(
|
|
4298
|
+
Object.defineProperty(Readable8.prototype, "destroyed", {
|
|
4299
4299
|
// making it explicit this property is not enumerable
|
|
4300
4300
|
// because otherwise some prototype manipulation in
|
|
4301
4301
|
// userland will fail
|
|
@@ -4313,12 +4313,12 @@ var require_stream_readable = __commonJS({
|
|
|
4313
4313
|
this._readableState.destroyed = value;
|
|
4314
4314
|
}
|
|
4315
4315
|
});
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4316
|
+
Readable8.prototype.destroy = destroyImpl.destroy;
|
|
4317
|
+
Readable8.prototype._undestroy = destroyImpl.undestroy;
|
|
4318
|
+
Readable8.prototype._destroy = function(err2, cb) {
|
|
4319
4319
|
cb(err2);
|
|
4320
4320
|
};
|
|
4321
|
-
|
|
4321
|
+
Readable8.prototype.push = function(chunk, encoding) {
|
|
4322
4322
|
var state2 = this._readableState;
|
|
4323
4323
|
var skipChunkCheck;
|
|
4324
4324
|
if (!state2.objectMode) {
|
|
@@ -4335,7 +4335,7 @@ var require_stream_readable = __commonJS({
|
|
|
4335
4335
|
}
|
|
4336
4336
|
return readableAddChunk(this, chunk, encoding, false, skipChunkCheck);
|
|
4337
4337
|
};
|
|
4338
|
-
|
|
4338
|
+
Readable8.prototype.unshift = function(chunk) {
|
|
4339
4339
|
return readableAddChunk(this, chunk, null, true, false);
|
|
4340
4340
|
};
|
|
4341
4341
|
function readableAddChunk(stream, chunk, encoding, addToFront, skipChunkCheck) {
|
|
@@ -4396,10 +4396,10 @@ var require_stream_readable = __commonJS({
|
|
|
4396
4396
|
}
|
|
4397
4397
|
return er2;
|
|
4398
4398
|
}
|
|
4399
|
-
|
|
4399
|
+
Readable8.prototype.isPaused = function() {
|
|
4400
4400
|
return this._readableState.flowing === false;
|
|
4401
4401
|
};
|
|
4402
|
-
|
|
4402
|
+
Readable8.prototype.setEncoding = function(enc) {
|
|
4403
4403
|
if (!StringDecoder) StringDecoder = require_string_decoder().StringDecoder;
|
|
4404
4404
|
var decoder = new StringDecoder(enc);
|
|
4405
4405
|
this._readableState.decoder = decoder;
|
|
@@ -4445,7 +4445,7 @@ var require_stream_readable = __commonJS({
|
|
|
4445
4445
|
}
|
|
4446
4446
|
return state2.length;
|
|
4447
4447
|
}
|
|
4448
|
-
|
|
4448
|
+
Readable8.prototype.read = function(n4) {
|
|
4449
4449
|
debug("read", n4);
|
|
4450
4450
|
n4 = parseInt(n4, 10);
|
|
4451
4451
|
var state2 = this._readableState;
|
|
@@ -4554,10 +4554,10 @@ var require_stream_readable = __commonJS({
|
|
|
4554
4554
|
}
|
|
4555
4555
|
state2.readingMore = false;
|
|
4556
4556
|
}
|
|
4557
|
-
|
|
4557
|
+
Readable8.prototype._read = function(n4) {
|
|
4558
4558
|
errorOrDestroy(this, new ERR_METHOD_NOT_IMPLEMENTED("_read()"));
|
|
4559
4559
|
};
|
|
4560
|
-
|
|
4560
|
+
Readable8.prototype.pipe = function(dest, pipeOpts) {
|
|
4561
4561
|
var src = this;
|
|
4562
4562
|
var state2 = this._readableState;
|
|
4563
4563
|
switch (state2.pipesCount) {
|
|
@@ -4660,7 +4660,7 @@ var require_stream_readable = __commonJS({
|
|
|
4660
4660
|
}
|
|
4661
4661
|
};
|
|
4662
4662
|
}
|
|
4663
|
-
|
|
4663
|
+
Readable8.prototype.unpipe = function(dest) {
|
|
4664
4664
|
var state2 = this._readableState;
|
|
4665
4665
|
var unpipeInfo = {
|
|
4666
4666
|
hasUnpiped: false
|
|
@@ -4694,7 +4694,7 @@ var require_stream_readable = __commonJS({
|
|
|
4694
4694
|
dest.emit("unpipe", this, unpipeInfo);
|
|
4695
4695
|
return this;
|
|
4696
4696
|
};
|
|
4697
|
-
|
|
4697
|
+
Readable8.prototype.on = function(ev, fn) {
|
|
4698
4698
|
var res = Stream.prototype.on.call(this, ev, fn);
|
|
4699
4699
|
var state2 = this._readableState;
|
|
4700
4700
|
if (ev === "data") {
|
|
@@ -4715,15 +4715,15 @@ var require_stream_readable = __commonJS({
|
|
|
4715
4715
|
}
|
|
4716
4716
|
return res;
|
|
4717
4717
|
};
|
|
4718
|
-
|
|
4719
|
-
|
|
4718
|
+
Readable8.prototype.addListener = Readable8.prototype.on;
|
|
4719
|
+
Readable8.prototype.removeListener = function(ev, fn) {
|
|
4720
4720
|
var res = Stream.prototype.removeListener.call(this, ev, fn);
|
|
4721
4721
|
if (ev === "readable") {
|
|
4722
4722
|
process.nextTick(updateReadableListening, this);
|
|
4723
4723
|
}
|
|
4724
4724
|
return res;
|
|
4725
4725
|
};
|
|
4726
|
-
|
|
4726
|
+
Readable8.prototype.removeAllListeners = function(ev) {
|
|
4727
4727
|
var res = Stream.prototype.removeAllListeners.apply(this, arguments);
|
|
4728
4728
|
if (ev === "readable" || ev === void 0) {
|
|
4729
4729
|
process.nextTick(updateReadableListening, this);
|
|
@@ -4743,7 +4743,7 @@ var require_stream_readable = __commonJS({
|
|
|
4743
4743
|
debug("readable nexttick read 0");
|
|
4744
4744
|
self2.read(0);
|
|
4745
4745
|
}
|
|
4746
|
-
|
|
4746
|
+
Readable8.prototype.resume = function() {
|
|
4747
4747
|
var state2 = this._readableState;
|
|
4748
4748
|
if (!state2.flowing) {
|
|
4749
4749
|
debug("resume");
|
|
@@ -4769,7 +4769,7 @@ var require_stream_readable = __commonJS({
|
|
|
4769
4769
|
flow(stream);
|
|
4770
4770
|
if (state2.flowing && !state2.reading) stream.read(0);
|
|
4771
4771
|
}
|
|
4772
|
-
|
|
4772
|
+
Readable8.prototype.pause = function() {
|
|
4773
4773
|
debug("call pause flowing=%j", this._readableState.flowing);
|
|
4774
4774
|
if (this._readableState.flowing !== false) {
|
|
4775
4775
|
debug("pause");
|
|
@@ -4784,7 +4784,7 @@ var require_stream_readable = __commonJS({
|
|
|
4784
4784
|
debug("flow", state2.flowing);
|
|
4785
4785
|
while (state2.flowing && stream.read() !== null) ;
|
|
4786
4786
|
}
|
|
4787
|
-
|
|
4787
|
+
Readable8.prototype.wrap = function(stream) {
|
|
4788
4788
|
var _this = this;
|
|
4789
4789
|
var state2 = this._readableState;
|
|
4790
4790
|
var paused = false;
|
|
@@ -4829,14 +4829,14 @@ var require_stream_readable = __commonJS({
|
|
|
4829
4829
|
return this;
|
|
4830
4830
|
};
|
|
4831
4831
|
if (typeof Symbol === "function") {
|
|
4832
|
-
|
|
4832
|
+
Readable8.prototype[Symbol.asyncIterator] = function() {
|
|
4833
4833
|
if (createReadableStreamAsyncIterator === void 0) {
|
|
4834
4834
|
createReadableStreamAsyncIterator = require_async_iterator();
|
|
4835
4835
|
}
|
|
4836
4836
|
return createReadableStreamAsyncIterator(this);
|
|
4837
4837
|
};
|
|
4838
4838
|
}
|
|
4839
|
-
Object.defineProperty(
|
|
4839
|
+
Object.defineProperty(Readable8.prototype, "readableHighWaterMark", {
|
|
4840
4840
|
// making it explicit this property is not enumerable
|
|
4841
4841
|
// because otherwise some prototype manipulation in
|
|
4842
4842
|
// userland will fail
|
|
@@ -4845,7 +4845,7 @@ var require_stream_readable = __commonJS({
|
|
|
4845
4845
|
return this._readableState.highWaterMark;
|
|
4846
4846
|
}
|
|
4847
4847
|
});
|
|
4848
|
-
Object.defineProperty(
|
|
4848
|
+
Object.defineProperty(Readable8.prototype, "readableBuffer", {
|
|
4849
4849
|
// making it explicit this property is not enumerable
|
|
4850
4850
|
// because otherwise some prototype manipulation in
|
|
4851
4851
|
// userland will fail
|
|
@@ -4854,7 +4854,7 @@ var require_stream_readable = __commonJS({
|
|
|
4854
4854
|
return this._readableState && this._readableState.buffer;
|
|
4855
4855
|
}
|
|
4856
4856
|
});
|
|
4857
|
-
Object.defineProperty(
|
|
4857
|
+
Object.defineProperty(Readable8.prototype, "readableFlowing", {
|
|
4858
4858
|
// making it explicit this property is not enumerable
|
|
4859
4859
|
// because otherwise some prototype manipulation in
|
|
4860
4860
|
// userland will fail
|
|
@@ -4868,8 +4868,8 @@ var require_stream_readable = __commonJS({
|
|
|
4868
4868
|
}
|
|
4869
4869
|
}
|
|
4870
4870
|
});
|
|
4871
|
-
|
|
4872
|
-
Object.defineProperty(
|
|
4871
|
+
Readable8._fromList = fromList;
|
|
4872
|
+
Object.defineProperty(Readable8.prototype, "readableLength", {
|
|
4873
4873
|
// making it explicit this property is not enumerable
|
|
4874
4874
|
// because otherwise some prototype manipulation in
|
|
4875
4875
|
// userland will fail
|
|
@@ -4915,11 +4915,11 @@ var require_stream_readable = __commonJS({
|
|
|
4915
4915
|
}
|
|
4916
4916
|
}
|
|
4917
4917
|
if (typeof Symbol === "function") {
|
|
4918
|
-
|
|
4918
|
+
Readable8.from = function(iterable, opts) {
|
|
4919
4919
|
if (from === void 0) {
|
|
4920
4920
|
from = require_from();
|
|
4921
4921
|
}
|
|
4922
|
-
return from(
|
|
4922
|
+
return from(Readable8, iterable, opts);
|
|
4923
4923
|
};
|
|
4924
4924
|
}
|
|
4925
4925
|
function indexOf(xs2, x2) {
|
|
@@ -4941,9 +4941,9 @@ var require_stream_duplex = __commonJS({
|
|
|
4941
4941
|
return keys2;
|
|
4942
4942
|
};
|
|
4943
4943
|
module2.exports = Duplex;
|
|
4944
|
-
var
|
|
4944
|
+
var Readable8 = require_stream_readable();
|
|
4945
4945
|
var Writable2 = require_stream_writable();
|
|
4946
|
-
require_inherits()(Duplex,
|
|
4946
|
+
require_inherits()(Duplex, Readable8);
|
|
4947
4947
|
{
|
|
4948
4948
|
keys = objectKeys(Writable2.prototype);
|
|
4949
4949
|
for (v3 = 0; v3 < keys.length; v3++) {
|
|
@@ -4956,7 +4956,7 @@ var require_stream_duplex = __commonJS({
|
|
|
4956
4956
|
var v3;
|
|
4957
4957
|
function Duplex(options) {
|
|
4958
4958
|
if (!(this instanceof Duplex)) return new Duplex(options);
|
|
4959
|
-
|
|
4959
|
+
Readable8.call(this, options);
|
|
4960
4960
|
Writable2.call(this, options);
|
|
4961
4961
|
this.allowHalfOpen = true;
|
|
4962
4962
|
if (options) {
|
|
@@ -6498,7 +6498,7 @@ var require_pipeline = __commonJS({
|
|
|
6498
6498
|
if (typeof streams[streams.length - 1] !== "function") return noop;
|
|
6499
6499
|
return streams.pop();
|
|
6500
6500
|
}
|
|
6501
|
-
function
|
|
6501
|
+
function pipeline6() {
|
|
6502
6502
|
for (var _len = arguments.length, streams = new Array(_len), _key = 0; _key < _len; _key++) {
|
|
6503
6503
|
streams[_key] = arguments[_key];
|
|
6504
6504
|
}
|
|
@@ -6521,7 +6521,7 @@ var require_pipeline = __commonJS({
|
|
|
6521
6521
|
});
|
|
6522
6522
|
return streams.reduce(pipe);
|
|
6523
6523
|
}
|
|
6524
|
-
module2.exports =
|
|
6524
|
+
module2.exports = pipeline6;
|
|
6525
6525
|
}
|
|
6526
6526
|
});
|
|
6527
6527
|
|
|
@@ -7759,15 +7759,15 @@ var require_index_cjs = __commonJS({
|
|
|
7759
7759
|
};
|
|
7760
7760
|
}
|
|
7761
7761
|
function wrapConversion(toModel, graph2) {
|
|
7762
|
-
const
|
|
7762
|
+
const path30 = [graph2[toModel].parent, toModel];
|
|
7763
7763
|
let fn = convert$1[graph2[toModel].parent][toModel];
|
|
7764
7764
|
let cur = graph2[toModel].parent;
|
|
7765
7765
|
while (graph2[cur].parent) {
|
|
7766
|
-
|
|
7766
|
+
path30.unshift(graph2[cur].parent);
|
|
7767
7767
|
fn = link(convert$1[graph2[cur].parent][cur], fn);
|
|
7768
7768
|
cur = graph2[cur].parent;
|
|
7769
7769
|
}
|
|
7770
|
-
fn.conversion =
|
|
7770
|
+
fn.conversion = path30;
|
|
7771
7771
|
return fn;
|
|
7772
7772
|
}
|
|
7773
7773
|
function route(fromModel) {
|
|
@@ -8397,7 +8397,7 @@ var require_node2 = __commonJS({
|
|
|
8397
8397
|
var require_tail_file = __commonJS({
|
|
8398
8398
|
"../../node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
|
|
8399
8399
|
"use strict";
|
|
8400
|
-
var
|
|
8400
|
+
var fs33 = require("fs");
|
|
8401
8401
|
var { StringDecoder } = require("string_decoder");
|
|
8402
8402
|
var { Stream } = require_readable();
|
|
8403
8403
|
function noop() {
|
|
@@ -8418,7 +8418,7 @@ var require_tail_file = __commonJS({
|
|
|
8418
8418
|
stream.emit("end");
|
|
8419
8419
|
stream.emit("close");
|
|
8420
8420
|
};
|
|
8421
|
-
|
|
8421
|
+
fs33.open(options.file, "a+", "0644", (err2, fd) => {
|
|
8422
8422
|
if (err2) {
|
|
8423
8423
|
if (!iter) {
|
|
8424
8424
|
stream.emit("error", err2);
|
|
@@ -8430,10 +8430,10 @@ var require_tail_file = __commonJS({
|
|
|
8430
8430
|
}
|
|
8431
8431
|
(function read() {
|
|
8432
8432
|
if (stream.destroyed) {
|
|
8433
|
-
|
|
8433
|
+
fs33.close(fd, noop);
|
|
8434
8434
|
return;
|
|
8435
8435
|
}
|
|
8436
|
-
return
|
|
8436
|
+
return fs33.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
|
|
8437
8437
|
if (error) {
|
|
8438
8438
|
if (!iter) {
|
|
8439
8439
|
stream.emit("error", error);
|
|
@@ -8492,8 +8492,8 @@ var require_tail_file = __commonJS({
|
|
|
8492
8492
|
var require_file = __commonJS({
|
|
8493
8493
|
"../../node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
|
|
8494
8494
|
"use strict";
|
|
8495
|
-
var
|
|
8496
|
-
var
|
|
8495
|
+
var fs33 = require("fs");
|
|
8496
|
+
var path30 = require("path");
|
|
8497
8497
|
var asyncSeries = require_series();
|
|
8498
8498
|
var zlib3 = require("zlib");
|
|
8499
8499
|
var { MESSAGE } = require_triple_beam();
|
|
@@ -8523,14 +8523,14 @@ var require_file = __commonJS({
|
|
|
8523
8523
|
this._onError = this._onError.bind(this);
|
|
8524
8524
|
if (options.filename || options.dirname) {
|
|
8525
8525
|
throwIf("filename or dirname", "stream");
|
|
8526
|
-
this._basename = this.filename = options.filename ?
|
|
8527
|
-
this.dirname = options.dirname ||
|
|
8526
|
+
this._basename = this.filename = options.filename ? path30.basename(options.filename) : "winston.log";
|
|
8527
|
+
this.dirname = options.dirname || path30.dirname(options.filename);
|
|
8528
8528
|
this.options = options.options || { flags: "a" };
|
|
8529
8529
|
} else if (options.stream) {
|
|
8530
8530
|
console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
|
|
8531
8531
|
throwIf("stream", "filename", "maxsize");
|
|
8532
8532
|
this._dest = this._stream.pipe(this._setupStream(options.stream));
|
|
8533
|
-
this.dirname =
|
|
8533
|
+
this.dirname = path30.dirname(this._dest.path);
|
|
8534
8534
|
} else {
|
|
8535
8535
|
throw new Error("Cannot log to file without filename or stream.");
|
|
8536
8536
|
}
|
|
@@ -8693,11 +8693,11 @@ var require_file = __commonJS({
|
|
|
8693
8693
|
options = {};
|
|
8694
8694
|
}
|
|
8695
8695
|
options = normalizeQuery(options);
|
|
8696
|
-
const file =
|
|
8696
|
+
const file = path30.join(this.dirname, this.filename);
|
|
8697
8697
|
let buff = "";
|
|
8698
8698
|
let results = [];
|
|
8699
8699
|
let row = 0;
|
|
8700
|
-
const stream =
|
|
8700
|
+
const stream = fs33.createReadStream(file, {
|
|
8701
8701
|
encoding: "utf8"
|
|
8702
8702
|
});
|
|
8703
8703
|
stream.on("error", (err2) => {
|
|
@@ -8798,7 +8798,7 @@ var require_file = __commonJS({
|
|
|
8798
8798
|
* TODO: Refactor me.
|
|
8799
8799
|
*/
|
|
8800
8800
|
stream(options = {}) {
|
|
8801
|
-
const file =
|
|
8801
|
+
const file = path30.join(this.dirname, this.filename);
|
|
8802
8802
|
const stream = new Stream();
|
|
8803
8803
|
const tail = {
|
|
8804
8804
|
file,
|
|
@@ -8848,8 +8848,8 @@ var require_file = __commonJS({
|
|
|
8848
8848
|
*/
|
|
8849
8849
|
stat(callback) {
|
|
8850
8850
|
const target = this._getFile();
|
|
8851
|
-
const fullpath =
|
|
8852
|
-
|
|
8851
|
+
const fullpath = path30.join(this.dirname, target);
|
|
8852
|
+
fs33.stat(fullpath, (err2, stat) => {
|
|
8853
8853
|
if (err2 && err2.code === "ENOENT") {
|
|
8854
8854
|
debug("ENOENT\xA0ok", fullpath);
|
|
8855
8855
|
this.filename = target;
|
|
@@ -8952,9 +8952,9 @@ var require_file = __commonJS({
|
|
|
8952
8952
|
* @returns {WritableStream} Stream that writes to disk for the active file.
|
|
8953
8953
|
*/
|
|
8954
8954
|
_createStream(source) {
|
|
8955
|
-
const fullpath =
|
|
8955
|
+
const fullpath = path30.join(this.dirname, this.filename);
|
|
8956
8956
|
debug("create stream start", fullpath, this.options);
|
|
8957
|
-
const dest =
|
|
8957
|
+
const dest = fs33.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
|
|
8958
8958
|
debug("file open ok", fullpath);
|
|
8959
8959
|
this.emit("open", fullpath);
|
|
8960
8960
|
source.pipe(dest);
|
|
@@ -8977,16 +8977,16 @@ var require_file = __commonJS({
|
|
|
8977
8977
|
*/
|
|
8978
8978
|
_incFile(callback) {
|
|
8979
8979
|
debug("_incFile", this.filename);
|
|
8980
|
-
const ext =
|
|
8981
|
-
const basename =
|
|
8980
|
+
const ext = path30.extname(this._basename);
|
|
8981
|
+
const basename = path30.basename(this._basename, ext);
|
|
8982
8982
|
const tasks = [];
|
|
8983
8983
|
if (this.zippedArchive) {
|
|
8984
8984
|
tasks.push(
|
|
8985
8985
|
function(cb) {
|
|
8986
8986
|
const num = this._created > 0 && !this.tailable ? this._created : "";
|
|
8987
8987
|
this._compressFile(
|
|
8988
|
-
|
|
8989
|
-
|
|
8988
|
+
path30.join(this.dirname, `${basename}${num}${ext}`),
|
|
8989
|
+
path30.join(this.dirname, `${basename}${num}${ext}.gz`),
|
|
8990
8990
|
cb
|
|
8991
8991
|
);
|
|
8992
8992
|
}.bind(this)
|
|
@@ -9011,8 +9011,8 @@ var require_file = __commonJS({
|
|
|
9011
9011
|
* @private
|
|
9012
9012
|
*/
|
|
9013
9013
|
_getFile() {
|
|
9014
|
-
const ext =
|
|
9015
|
-
const basename =
|
|
9014
|
+
const ext = path30.extname(this._basename);
|
|
9015
|
+
const basename = path30.basename(this._basename, ext);
|
|
9016
9016
|
const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
|
|
9017
9017
|
return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
|
|
9018
9018
|
}
|
|
@@ -9032,8 +9032,8 @@ var require_file = __commonJS({
|
|
|
9032
9032
|
const isOldest = oldest !== 0 ? oldest : "";
|
|
9033
9033
|
const isZipped = this.zippedArchive ? ".gz" : "";
|
|
9034
9034
|
const filePath = `${basename}${isOldest}${ext}${isZipped}`;
|
|
9035
|
-
const target =
|
|
9036
|
-
|
|
9035
|
+
const target = path30.join(this.dirname, filePath);
|
|
9036
|
+
fs33.unlink(target, callback);
|
|
9037
9037
|
}
|
|
9038
9038
|
/**
|
|
9039
9039
|
* Roll files forward based on integer, up to maxFiles. e.g. if base if
|
|
@@ -9055,20 +9055,20 @@ var require_file = __commonJS({
|
|
|
9055
9055
|
for (let x2 = this.maxFiles - 1; x2 > 1; x2--) {
|
|
9056
9056
|
tasks.push(function(i6, cb) {
|
|
9057
9057
|
let fileName = `${basename}${i6 - 1}${ext}${isZipped}`;
|
|
9058
|
-
const tmppath =
|
|
9059
|
-
|
|
9058
|
+
const tmppath = path30.join(this.dirname, fileName);
|
|
9059
|
+
fs33.exists(tmppath, (exists) => {
|
|
9060
9060
|
if (!exists) {
|
|
9061
9061
|
return cb(null);
|
|
9062
9062
|
}
|
|
9063
9063
|
fileName = `${basename}${i6}${ext}${isZipped}`;
|
|
9064
|
-
|
|
9064
|
+
fs33.rename(tmppath, path30.join(this.dirname, fileName), cb);
|
|
9065
9065
|
});
|
|
9066
9066
|
}.bind(this, x2));
|
|
9067
9067
|
}
|
|
9068
9068
|
asyncSeries(tasks, () => {
|
|
9069
|
-
|
|
9070
|
-
|
|
9071
|
-
|
|
9069
|
+
fs33.rename(
|
|
9070
|
+
path30.join(this.dirname, `${basename}${ext}${isZipped}`),
|
|
9071
|
+
path30.join(this.dirname, `${basename}1${ext}${isZipped}`),
|
|
9072
9072
|
callback
|
|
9073
9073
|
);
|
|
9074
9074
|
});
|
|
@@ -9082,22 +9082,22 @@ var require_file = __commonJS({
|
|
|
9082
9082
|
* @private
|
|
9083
9083
|
*/
|
|
9084
9084
|
_compressFile(src, dest, callback) {
|
|
9085
|
-
|
|
9085
|
+
fs33.access(src, fs33.F_OK, (err2) => {
|
|
9086
9086
|
if (err2) {
|
|
9087
9087
|
return callback();
|
|
9088
9088
|
}
|
|
9089
9089
|
var gzip = zlib3.createGzip();
|
|
9090
|
-
var inp =
|
|
9091
|
-
var out =
|
|
9090
|
+
var inp = fs33.createReadStream(src);
|
|
9091
|
+
var out = fs33.createWriteStream(dest);
|
|
9092
9092
|
out.on("finish", () => {
|
|
9093
|
-
|
|
9093
|
+
fs33.unlink(src, callback);
|
|
9094
9094
|
});
|
|
9095
9095
|
inp.pipe(gzip).pipe(out);
|
|
9096
9096
|
});
|
|
9097
9097
|
}
|
|
9098
9098
|
_createLogDirIfNotExist(dirPath) {
|
|
9099
|
-
if (!
|
|
9100
|
-
|
|
9099
|
+
if (!fs33.existsSync(dirPath)) {
|
|
9100
|
+
fs33.mkdirSync(dirPath, { recursive: true });
|
|
9101
9101
|
}
|
|
9102
9102
|
}
|
|
9103
9103
|
};
|
|
@@ -9181,9 +9181,9 @@ var require_http = __commonJS({
|
|
|
9181
9181
|
};
|
|
9182
9182
|
const auth = options.params.auth || null;
|
|
9183
9183
|
delete options.params.auth;
|
|
9184
|
-
const
|
|
9184
|
+
const path30 = options.params.path || null;
|
|
9185
9185
|
delete options.params.path;
|
|
9186
|
-
this._request(options, auth,
|
|
9186
|
+
this._request(options, auth, path30, (err2, res, body) => {
|
|
9187
9187
|
if (res && res.statusCode !== 200) {
|
|
9188
9188
|
err2 = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
|
|
9189
9189
|
}
|
|
@@ -9211,12 +9211,12 @@ var require_http = __commonJS({
|
|
|
9211
9211
|
method: "stream",
|
|
9212
9212
|
params: options
|
|
9213
9213
|
};
|
|
9214
|
-
const
|
|
9214
|
+
const path30 = options.params.path || null;
|
|
9215
9215
|
delete options.params.path;
|
|
9216
9216
|
const auth = options.params.auth || null;
|
|
9217
9217
|
delete options.params.auth;
|
|
9218
9218
|
let buff = "";
|
|
9219
|
-
const req = this._request(options, auth,
|
|
9219
|
+
const req = this._request(options, auth, path30);
|
|
9220
9220
|
stream.destroy = () => req.destroy();
|
|
9221
9221
|
req.on("data", (data) => {
|
|
9222
9222
|
data = (buff + data).split(/\n+/);
|
|
@@ -9242,14 +9242,14 @@ var require_http = __commonJS({
|
|
|
9242
9242
|
* @param {string} path - request path
|
|
9243
9243
|
* @param {function} callback - Continuation to respond to when complete.
|
|
9244
9244
|
*/
|
|
9245
|
-
_request(options, auth,
|
|
9245
|
+
_request(options, auth, path30, callback) {
|
|
9246
9246
|
options = options || {};
|
|
9247
9247
|
auth = auth || this.auth;
|
|
9248
|
-
|
|
9248
|
+
path30 = path30 || this.path || "";
|
|
9249
9249
|
if (this.batch) {
|
|
9250
|
-
this._doBatch(options, callback, auth,
|
|
9250
|
+
this._doBatch(options, callback, auth, path30);
|
|
9251
9251
|
} else {
|
|
9252
|
-
this._doRequest(options, callback, auth,
|
|
9252
|
+
this._doRequest(options, callback, auth, path30);
|
|
9253
9253
|
}
|
|
9254
9254
|
}
|
|
9255
9255
|
/**
|
|
@@ -9259,18 +9259,18 @@ var require_http = __commonJS({
|
|
|
9259
9259
|
* @param {Object?} auth - authentication options
|
|
9260
9260
|
* @param {string} path - request path
|
|
9261
9261
|
*/
|
|
9262
|
-
_doBatch(options, callback, auth,
|
|
9262
|
+
_doBatch(options, callback, auth, path30) {
|
|
9263
9263
|
this.batchOptions.push(options);
|
|
9264
9264
|
if (this.batchOptions.length === 1) {
|
|
9265
9265
|
const me3 = this;
|
|
9266
9266
|
this.batchCallback = callback;
|
|
9267
9267
|
this.batchTimeoutID = setTimeout(function() {
|
|
9268
9268
|
me3.batchTimeoutID = -1;
|
|
9269
|
-
me3._doBatchRequest(me3.batchCallback, auth,
|
|
9269
|
+
me3._doBatchRequest(me3.batchCallback, auth, path30);
|
|
9270
9270
|
}, this.batchInterval);
|
|
9271
9271
|
}
|
|
9272
9272
|
if (this.batchOptions.length === this.batchCount) {
|
|
9273
|
-
this._doBatchRequest(this.batchCallback, auth,
|
|
9273
|
+
this._doBatchRequest(this.batchCallback, auth, path30);
|
|
9274
9274
|
}
|
|
9275
9275
|
}
|
|
9276
9276
|
/**
|
|
@@ -9279,14 +9279,14 @@ var require_http = __commonJS({
|
|
|
9279
9279
|
* @param {Object?} auth - authentication options
|
|
9280
9280
|
* @param {string} path - request path
|
|
9281
9281
|
*/
|
|
9282
|
-
_doBatchRequest(callback, auth,
|
|
9282
|
+
_doBatchRequest(callback, auth, path30) {
|
|
9283
9283
|
if (this.batchTimeoutID > 0) {
|
|
9284
9284
|
clearTimeout(this.batchTimeoutID);
|
|
9285
9285
|
this.batchTimeoutID = -1;
|
|
9286
9286
|
}
|
|
9287
9287
|
const batchOptionsCopy = this.batchOptions.slice();
|
|
9288
9288
|
this.batchOptions = [];
|
|
9289
|
-
this._doRequest(batchOptionsCopy, callback, auth,
|
|
9289
|
+
this._doRequest(batchOptionsCopy, callback, auth, path30);
|
|
9290
9290
|
}
|
|
9291
9291
|
/**
|
|
9292
9292
|
* Make a request to a winstond server or any http server which can
|
|
@@ -9296,7 +9296,7 @@ var require_http = __commonJS({
|
|
|
9296
9296
|
* @param {Object?} auth - authentication options
|
|
9297
9297
|
* @param {string} path - request path
|
|
9298
9298
|
*/
|
|
9299
|
-
_doRequest(options, callback, auth,
|
|
9299
|
+
_doRequest(options, callback, auth, path30) {
|
|
9300
9300
|
const headers = Object.assign({}, this.headers);
|
|
9301
9301
|
if (auth && auth.bearer) {
|
|
9302
9302
|
headers.Authorization = `Bearer ${auth.bearer}`;
|
|
@@ -9306,7 +9306,7 @@ var require_http = __commonJS({
|
|
|
9306
9306
|
method: "POST",
|
|
9307
9307
|
host: this.host,
|
|
9308
9308
|
port: this.port,
|
|
9309
|
-
path: `/${
|
|
9309
|
+
path: `/${path30.replace(/^\//, "")}`,
|
|
9310
9310
|
headers,
|
|
9311
9311
|
auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
|
|
9312
9312
|
agent: this.agent
|
|
@@ -10275,7 +10275,7 @@ var require_logger = __commonJS({
|
|
|
10275
10275
|
*/
|
|
10276
10276
|
configure({
|
|
10277
10277
|
silent,
|
|
10278
|
-
format:
|
|
10278
|
+
format: format4,
|
|
10279
10279
|
defaultMeta,
|
|
10280
10280
|
levels,
|
|
10281
10281
|
level = "info",
|
|
@@ -10294,7 +10294,7 @@ var require_logger = __commonJS({
|
|
|
10294
10294
|
this.clear();
|
|
10295
10295
|
}
|
|
10296
10296
|
this.silent = silent;
|
|
10297
|
-
this.format =
|
|
10297
|
+
this.format = format4 || this.format || require_json()();
|
|
10298
10298
|
this.defaultMeta = defaultMeta || null;
|
|
10299
10299
|
this.levels = levels || this.levels || config.npm.levels;
|
|
10300
10300
|
this.level = level;
|
|
@@ -11116,18 +11116,32 @@ ${stack}` : base;
|
|
|
11116
11116
|
});
|
|
11117
11117
|
|
|
11118
11118
|
// ../../packages/logger/src/logger/Logger.ts
|
|
11119
|
-
var import_chalk, MAX_CAUSE_DEPTH, Logger, logger, Logger_default;
|
|
11119
|
+
var import_util, import_chalk, MAX_CAUSE_DEPTH, LOG_RECORD_VERSION, MAX_FIELD_CHARS, ANSI_PATTERN, nativeConsole, Logger, logger, Logger_default;
|
|
11120
11120
|
var init_Logger = __esm({
|
|
11121
11121
|
"../../packages/logger/src/logger/Logger.ts"() {
|
|
11122
|
+
import_util = require("util");
|
|
11123
|
+
init_src();
|
|
11122
11124
|
import_chalk = __toESM(require("chalk"), 1);
|
|
11123
11125
|
init_FileLogService();
|
|
11124
11126
|
MAX_CAUSE_DEPTH = 10;
|
|
11127
|
+
LOG_RECORD_VERSION = 1;
|
|
11128
|
+
MAX_FIELD_CHARS = 3e4;
|
|
11129
|
+
ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
|
|
11130
|
+
nativeConsole = {
|
|
11131
|
+
log: console.log.bind(console),
|
|
11132
|
+
info: console.info.bind(console),
|
|
11133
|
+
warn: console.warn.bind(console),
|
|
11134
|
+
error: console.error.bind(console)
|
|
11135
|
+
};
|
|
11125
11136
|
Logger = class {
|
|
11126
11137
|
constructor() {
|
|
11127
11138
|
// Counts lines written to the console. In-place TTY renderers (the CLI progress bar) read it to
|
|
11128
11139
|
// detect that something else has printed since their last frame, so they redraw on a new line
|
|
11129
11140
|
// instead of moving the cursor up and overwriting a log message.
|
|
11130
11141
|
this._consoleWrites = 0;
|
|
11142
|
+
// Whether console.* has been redirected into this logger. Once per process, and one-way: our
|
|
11143
|
+
// own writes use `nativeConsole`, so there is nothing to restore it for.
|
|
11144
|
+
this._consoleCaptured = false;
|
|
11131
11145
|
this.setLevel = (level) => this._level = level;
|
|
11132
11146
|
this.enableFileLogging = (folder, file) => {
|
|
11133
11147
|
this._fileLoggingFolder = folder;
|
|
@@ -11161,9 +11175,28 @@ var init_Logger = __esm({
|
|
|
11161
11175
|
this._level = "debug";
|
|
11162
11176
|
if (fileLoggingFolder && process.env.NODE_ENV !== "development") {
|
|
11163
11177
|
this.enableFileLogging(fileLoggingFolder);
|
|
11164
|
-
|
|
11178
|
+
nativeConsole.error("Enabled file logger.");
|
|
11165
11179
|
}
|
|
11166
11180
|
};
|
|
11181
|
+
/**
|
|
11182
|
+
* Routes `console.*` through this logger, so code that never heard of it still writes one line
|
|
11183
|
+
* per record. Call once from a deployed process entrypoint; a no-op under the `text` format,
|
|
11184
|
+
* where a human is reading the console and Node's own formatting is the better rendering.
|
|
11185
|
+
*
|
|
11186
|
+
* Interception rather than a sweep of the call sites: the requirement is about the STREAM, not
|
|
11187
|
+
* about our code — an express handler, a driver and a dependency all write to the same stdout,
|
|
11188
|
+
* and a rule enforced at the call sites is a rule that the next `console.error` breaks.
|
|
11189
|
+
*/
|
|
11190
|
+
this.captureConsole = () => {
|
|
11191
|
+
if (this._consoleCaptured) return;
|
|
11192
|
+
if (this.consoleFormat() !== "json") return;
|
|
11193
|
+
this._consoleCaptured = true;
|
|
11194
|
+
console.log = (...args) => this.info((0, import_util.format)(...args));
|
|
11195
|
+
console.info = console.log;
|
|
11196
|
+
console.debug = (...args) => this.log((0, import_util.format)(...args));
|
|
11197
|
+
console.warn = (...args) => this.warn((0, import_util.format)(...args));
|
|
11198
|
+
console.error = (...args) => args.length === 1 && args[0] instanceof Error ? this.error(args[0]) : this.error((0, import_util.format)(...args));
|
|
11199
|
+
};
|
|
11167
11200
|
// The file sink replaces the console only for the CLI: there a human is watching the progress bar
|
|
11168
11201
|
// and can read the log file afterwards. Anywhere else that file sits on an ephemeral container
|
|
11169
11202
|
// layer nobody reads, and stdout is the only sink reaching the log driver (ECS -> CloudWatch), so
|
|
@@ -11174,18 +11207,15 @@ var init_Logger = __esm({
|
|
|
11174
11207
|
if (myLevel !== "debug") return;
|
|
11175
11208
|
FileLogService_default.write("DEBUG", String(message));
|
|
11176
11209
|
if (this.isConsoleSuppressed()) return;
|
|
11177
|
-
|
|
11178
|
-
this._consoleWrites++;
|
|
11210
|
+
this.writeConsole("debug", message);
|
|
11179
11211
|
};
|
|
11180
11212
|
this.info = (message) => {
|
|
11181
11213
|
FileLogService_default.write("INFO", String(message));
|
|
11182
11214
|
if (this.isConsoleSuppressed()) return;
|
|
11183
|
-
|
|
11184
|
-
this._consoleWrites++;
|
|
11215
|
+
this.writeConsole("info", message);
|
|
11185
11216
|
};
|
|
11186
11217
|
this.warn = (message) => {
|
|
11187
|
-
|
|
11188
|
-
this._consoleWrites++;
|
|
11218
|
+
this.writeConsole("warn", message);
|
|
11189
11219
|
FileLogService_default.write("WARN", String(message));
|
|
11190
11220
|
};
|
|
11191
11221
|
this.flush = () => FileLogService_default.flush();
|
|
@@ -11203,11 +11233,88 @@ var init_Logger = __esm({
|
|
|
11203
11233
|
message = String(error);
|
|
11204
11234
|
}
|
|
11205
11235
|
const suppressed = this.isConsoleSuppressed();
|
|
11206
|
-
|
|
11207
|
-
this._consoleWrites++;
|
|
11236
|
+
this.writeConsole("error", message, suppressed ? void 0 : stack);
|
|
11208
11237
|
FileLogService_default.write("ERROR", message, stack);
|
|
11209
|
-
|
|
11210
|
-
|
|
11238
|
+
};
|
|
11239
|
+
/**
|
|
11240
|
+
* How console records are rendered: `text` for a human at a terminal, `json` for a log pipeline.
|
|
11241
|
+
*
|
|
11242
|
+
* Deployed processes default to `json` because there ONE RECORD MUST BE ONE LINE: the awslogs
|
|
11243
|
+
* driver turns every newline on stdout into its own CloudWatch event, so a stack trace arrived as
|
|
11244
|
+
* thirty separate events, none of them carrying the message they belonged to, interleaved with
|
|
11245
|
+
* whatever the other threads were writing. `REMORA_LOG_FORMAT` overrides the default either way.
|
|
11246
|
+
* Same rule as StateEmitter, whose records share the stream.
|
|
11247
|
+
*/
|
|
11248
|
+
this.consoleFormat = () => {
|
|
11249
|
+
const configured = process.env.REMORA_LOG_FORMAT?.toLowerCase();
|
|
11250
|
+
if (configured === "json" || configured === "text")
|
|
11251
|
+
return configured;
|
|
11252
|
+
return process.env.REMORA_RUNTIME_CONTEXT === "cli" ? "text" : "json";
|
|
11253
|
+
};
|
|
11254
|
+
/**
|
|
11255
|
+
* The one place anything of ours reaches the console.
|
|
11256
|
+
*
|
|
11257
|
+
* `stack` belongs to the record it explains: a second line in text mode, a field of the same JSON
|
|
11258
|
+
* object in json mode — never a record of its own, which is how it would lose its message.
|
|
11259
|
+
*/
|
|
11260
|
+
this.writeConsole = (level, message, stack) => {
|
|
11261
|
+
this._consoleWrites++;
|
|
11262
|
+
if (this.consoleFormat() === "json")
|
|
11263
|
+
return this.writeJson(level, message, stack);
|
|
11264
|
+
this.writeText(level, message, stack);
|
|
11265
|
+
};
|
|
11266
|
+
this.writeText = (level, message, stack) => {
|
|
11267
|
+
if (level === "debug")
|
|
11268
|
+
nativeConsole.log(import_chalk.default.cyanBright("DEBUG"), message);
|
|
11269
|
+
else if (level === "info")
|
|
11270
|
+
nativeConsole.info(message);
|
|
11271
|
+
else if (level === "warn")
|
|
11272
|
+
nativeConsole.warn(message);
|
|
11273
|
+
else
|
|
11274
|
+
nativeConsole.error(message);
|
|
11275
|
+
if (stack)
|
|
11276
|
+
nativeConsole.error(stack);
|
|
11277
|
+
};
|
|
11278
|
+
/**
|
|
11279
|
+
* One JSON object, one line, one log event. Written straight to the stream rather than through
|
|
11280
|
+
* `console`, which reformats and re-wraps what it is given.
|
|
11281
|
+
*/
|
|
11282
|
+
this.writeJson = (level, message, stack) => {
|
|
11283
|
+
const record = {
|
|
11284
|
+
remora: "log",
|
|
11285
|
+
v: LOG_RECORD_VERSION,
|
|
11286
|
+
ts: DSTE_default.now().toISOString(),
|
|
11287
|
+
level,
|
|
11288
|
+
message: this.asField(message),
|
|
11289
|
+
...stack ? { stack: this.asField(stack) } : {}
|
|
11290
|
+
};
|
|
11291
|
+
const stream = level === "warn" || level === "error" ? process.stderr : process.stdout;
|
|
11292
|
+
stream.write(`${this.stringify(record)}
|
|
11293
|
+
`);
|
|
11294
|
+
};
|
|
11295
|
+
/** A loggable value as one clamped, colour-free string. */
|
|
11296
|
+
this.asField = (value) => {
|
|
11297
|
+
const plain = this.asText(value).replace(ANSI_PATTERN, "");
|
|
11298
|
+
if (plain.length <= MAX_FIELD_CHARS)
|
|
11299
|
+
return plain;
|
|
11300
|
+
return `${plain.slice(0, MAX_FIELD_CHARS)}\u2026[+${plain.length - MAX_FIELD_CHARS} chars]`;
|
|
11301
|
+
};
|
|
11302
|
+
this.asText = (value) => {
|
|
11303
|
+
if (typeof value === "string")
|
|
11304
|
+
return value;
|
|
11305
|
+
if (value instanceof Error)
|
|
11306
|
+
return value.stack ?? value.message;
|
|
11307
|
+
if (typeof value === "object" && value !== null)
|
|
11308
|
+
return this.stringify(value);
|
|
11309
|
+
return String(value);
|
|
11310
|
+
};
|
|
11311
|
+
/** JSON.stringify that cannot throw: a circular or unserializable value still has to be logged. */
|
|
11312
|
+
this.stringify = (value) => {
|
|
11313
|
+
try {
|
|
11314
|
+
return JSON.stringify(value);
|
|
11315
|
+
} catch {
|
|
11316
|
+
return String(value);
|
|
11317
|
+
}
|
|
11211
11318
|
};
|
|
11212
11319
|
// Node's Error.stack omits `cause` — only util.inspect renders it, and we log strings rather
|
|
11213
11320
|
// than Error objects. Without unwrapping here the root reason of a wrapped error is lost on any
|
|
@@ -11527,7 +11634,7 @@ var CONSTANTS, Constants_default;
|
|
|
11527
11634
|
var init_Constants = __esm({
|
|
11528
11635
|
"../../packages/constants/src/Constants.ts"() {
|
|
11529
11636
|
CONSTANTS = {
|
|
11530
|
-
cliVersion: "2.0
|
|
11637
|
+
cliVersion: "2.1.0",
|
|
11531
11638
|
backendVersion: 1,
|
|
11532
11639
|
backendPort: 5088,
|
|
11533
11640
|
workerVersion: 2,
|
|
@@ -11584,7 +11691,12 @@ var init_Constants = __esm({
|
|
|
11584
11691
|
* whole run, so the entire work list is submitted at once and needs an upper bound. A queued
|
|
11585
11692
|
* task holds references, not copies, so this bounds the work list rather than memory.
|
|
11586
11693
|
*/
|
|
11587
|
-
MAX_WORKER_QUEUE_SIZE: 1e5
|
|
11694
|
+
MAX_WORKER_QUEUE_SIZE: 1e5,
|
|
11695
|
+
/**
|
|
11696
|
+
* Rows a sample may ask for out of the pipeline. A sample is a look at the data, never an
|
|
11697
|
+
* export of it — `remora sample -s` and the preview's Data tab are both held to this.
|
|
11698
|
+
*/
|
|
11699
|
+
MAX_SAMPLE_ROWS: 100
|
|
11588
11700
|
}
|
|
11589
11701
|
};
|
|
11590
11702
|
Constants_default = CONSTANTS;
|
|
@@ -12572,10 +12684,10 @@ var init_Validator = __esm({
|
|
|
12572
12684
|
errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(" and ")})`);
|
|
12573
12685
|
}
|
|
12574
12686
|
for (const output of consumer.outputs) {
|
|
12575
|
-
const
|
|
12576
|
-
if (
|
|
12687
|
+
const format4 = output.format.toUpperCase();
|
|
12688
|
+
if (format4 === "CSV" || format4 === "JSON" || format4 === "PARQUET" || format4 === "TXT") {
|
|
12577
12689
|
if (!output.exportDestination)
|
|
12578
|
-
errors.push(`A static file output must have an export destination set (${
|
|
12690
|
+
errors.push(`A static file output must have an export destination set (${format4})`);
|
|
12579
12691
|
else if (!Environment_default.getSource(output.exportDestination))
|
|
12580
12692
|
errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
|
|
12581
12693
|
}
|
|
@@ -12647,14 +12759,14 @@ var init_Validator = __esm({
|
|
|
12647
12759
|
const errors = [];
|
|
12648
12760
|
const positionalFormats = ["CSV", "TXT", "PARQUET"];
|
|
12649
12761
|
for (const output of (consumer.outputs ?? []).filter((x2) => x2.omitNull === true)) {
|
|
12650
|
-
const
|
|
12651
|
-
if (positionalFormats.includes(
|
|
12652
|
-
errors.push(`The ${
|
|
12762
|
+
const format4 = output.format.toUpperCase();
|
|
12763
|
+
if (positionalFormats.includes(format4)) {
|
|
12764
|
+
errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", which only applies to JSON/API output: a positional column can't be omitted from a row.`);
|
|
12653
12765
|
continue;
|
|
12654
12766
|
}
|
|
12655
12767
|
const positional = (consumer.outputs ?? []).filter((x2) => positionalFormats.includes(x2.format.toUpperCase()));
|
|
12656
12768
|
if (positional.length > 0)
|
|
12657
|
-
errors.push(`The ${
|
|
12769
|
+
errors.push(`The ${format4} output of consumer "${consumer.name}" declares "omitNull", but the consumer also writes ${positional.map((x2) => x2.format.toUpperCase()).join(" and ")}. Every output is serialized from one record, so omitting fields would break the positional one \u2014 split them into two consumers.`);
|
|
12658
12770
|
}
|
|
12659
12771
|
return errors;
|
|
12660
12772
|
};
|
|
@@ -12733,7 +12845,7 @@ var init_Validator = __esm({
|
|
|
12733
12845
|
*/
|
|
12734
12846
|
this.detectOnFinishRecursion = (consumer) => {
|
|
12735
12847
|
const errors = [];
|
|
12736
|
-
const
|
|
12848
|
+
const path30 = [];
|
|
12737
12849
|
const targetsOf = (cons) => {
|
|
12738
12850
|
const names = [];
|
|
12739
12851
|
for (const output of cons.outputs ?? []) {
|
|
@@ -12745,17 +12857,17 @@ var init_Validator = __esm({
|
|
|
12745
12857
|
return names;
|
|
12746
12858
|
};
|
|
12747
12859
|
const walk = (name) => {
|
|
12748
|
-
if (
|
|
12749
|
-
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...
|
|
12860
|
+
if (path30.some((x2) => x2.toLowerCase() === name.toLowerCase())) {
|
|
12861
|
+
errors.push(`Consumer "${consumer.name}" has a cyclic run-consumer (onSuccess/onError) chain: ${[...path30, name].join(" -> ")}`);
|
|
12750
12862
|
return true;
|
|
12751
12863
|
}
|
|
12752
12864
|
const cons = Environment_default.getConsumer(name);
|
|
12753
12865
|
if (!cons) return false;
|
|
12754
|
-
|
|
12866
|
+
path30.push(name);
|
|
12755
12867
|
for (const target of targetsOf(cons)) {
|
|
12756
12868
|
if (walk(target)) return true;
|
|
12757
12869
|
}
|
|
12758
|
-
|
|
12870
|
+
path30.pop();
|
|
12759
12871
|
return false;
|
|
12760
12872
|
};
|
|
12761
12873
|
walk(consumer.name);
|
|
@@ -14241,8 +14353,8 @@ var init_HttpClient = __esm({
|
|
|
14241
14353
|
throw new Error(err2.message ?? "An error occurred on GET.");
|
|
14242
14354
|
}
|
|
14243
14355
|
};
|
|
14244
|
-
this.getWorkerUrl = (
|
|
14245
|
-
const cleanPath =
|
|
14356
|
+
this.getWorkerUrl = (path30) => {
|
|
14357
|
+
const cleanPath = path30.startsWith("/") ? path30.substring(1) : path30;
|
|
14246
14358
|
return `${this.workerHost}/${cleanPath}`;
|
|
14247
14359
|
};
|
|
14248
14360
|
this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
|
|
@@ -14685,14 +14797,14 @@ var init_readFile = __esm({
|
|
|
14685
14797
|
import_promises3 = require("fs/promises");
|
|
14686
14798
|
filePromises = {};
|
|
14687
14799
|
fileIntercept = {};
|
|
14688
|
-
readFile2 = (
|
|
14689
|
-
if (fileIntercept[
|
|
14690
|
-
return fileIntercept[
|
|
14800
|
+
readFile2 = (path30, options) => {
|
|
14801
|
+
if (fileIntercept[path30] !== void 0) {
|
|
14802
|
+
return fileIntercept[path30];
|
|
14691
14803
|
}
|
|
14692
|
-
if (!filePromises[
|
|
14693
|
-
filePromises[
|
|
14804
|
+
if (!filePromises[path30] || options?.ignoreCache) {
|
|
14805
|
+
filePromises[path30] = (0, import_promises3.readFile)(path30, "utf8");
|
|
14694
14806
|
}
|
|
14695
|
-
return filePromises[
|
|
14807
|
+
return filePromises[path30];
|
|
14696
14808
|
};
|
|
14697
14809
|
}
|
|
14698
14810
|
});
|
|
@@ -14804,8 +14916,8 @@ var init_externalDataInterceptor = __esm({
|
|
|
14804
14916
|
getFileRecord() {
|
|
14805
14917
|
return fileIntercept;
|
|
14806
14918
|
},
|
|
14807
|
-
interceptFile(
|
|
14808
|
-
fileIntercept[
|
|
14919
|
+
interceptFile(path30, contents) {
|
|
14920
|
+
fileIntercept[path30] = Promise.resolve(contents);
|
|
14809
14921
|
},
|
|
14810
14922
|
getTokenRecord() {
|
|
14811
14923
|
return tokenIntercept;
|
|
@@ -17587,17 +17699,17 @@ var init_split_header = __esm({
|
|
|
17587
17699
|
});
|
|
17588
17700
|
|
|
17589
17701
|
// ../../node_modules/@smithy/core/dist-es/submodules/serde/value/NumericValue.js
|
|
17590
|
-
var
|
|
17702
|
+
var format3, NumericValue;
|
|
17591
17703
|
var init_NumericValue = __esm({
|
|
17592
17704
|
"../../node_modules/@smithy/core/dist-es/submodules/serde/value/NumericValue.js"() {
|
|
17593
|
-
|
|
17705
|
+
format3 = /^-?\d*(\.\d+)?$/;
|
|
17594
17706
|
NumericValue = class _NumericValue {
|
|
17595
17707
|
string;
|
|
17596
17708
|
type;
|
|
17597
17709
|
constructor(string, type) {
|
|
17598
17710
|
this.string = string;
|
|
17599
17711
|
this.type = type;
|
|
17600
|
-
if (!
|
|
17712
|
+
if (!format3.test(string)) {
|
|
17601
17713
|
throw new Error(`@smithy/core/serde - NumericValue must only contain [0-9], at most one decimal point ".", and an optional negation prefix "-".`);
|
|
17602
17714
|
}
|
|
17603
17715
|
}
|
|
@@ -17609,7 +17721,7 @@ var init_NumericValue = __esm({
|
|
|
17609
17721
|
return false;
|
|
17610
17722
|
}
|
|
17611
17723
|
const _nv = object;
|
|
17612
|
-
return _NumericValue.prototype.isPrototypeOf(object) || _nv.type === "bigDecimal" &&
|
|
17724
|
+
return _NumericValue.prototype.isPrototypeOf(object) || _nv.type === "bigDecimal" && format3.test(_nv.string);
|
|
17613
17725
|
}
|
|
17614
17726
|
};
|
|
17615
17727
|
}
|
|
@@ -17885,8 +17997,8 @@ var init_createConfigValueProvider = __esm({
|
|
|
17885
17997
|
return endpoint.url.href;
|
|
17886
17998
|
}
|
|
17887
17999
|
if ("hostname" in endpoint) {
|
|
17888
|
-
const { protocol, hostname, port, path:
|
|
17889
|
-
return `${protocol}//${hostname}${port ? ":" + port : ""}${
|
|
18000
|
+
const { protocol, hostname, port, path: path30 } = endpoint;
|
|
18001
|
+
return `${protocol}//${hostname}${port ? ":" + port : ""}${path30}`;
|
|
17890
18002
|
}
|
|
17891
18003
|
}
|
|
17892
18004
|
return endpoint;
|
|
@@ -18261,18 +18373,18 @@ var getAttrPathList;
|
|
|
18261
18373
|
var init_getAttrPathList = __esm({
|
|
18262
18374
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
|
|
18263
18375
|
init_types2();
|
|
18264
|
-
getAttrPathList = (
|
|
18265
|
-
const parts =
|
|
18376
|
+
getAttrPathList = (path30) => {
|
|
18377
|
+
const parts = path30.split(".");
|
|
18266
18378
|
const pathList = [];
|
|
18267
18379
|
for (const part of parts) {
|
|
18268
18380
|
const squareBracketIndex = part.indexOf("[");
|
|
18269
18381
|
if (squareBracketIndex !== -1) {
|
|
18270
18382
|
if (part.indexOf("]") !== part.length - 1) {
|
|
18271
|
-
throw new EndpointError(`Path: '${
|
|
18383
|
+
throw new EndpointError(`Path: '${path30}' does not end with ']'`);
|
|
18272
18384
|
}
|
|
18273
18385
|
const arrayIndex = part.slice(squareBracketIndex + 1, -1);
|
|
18274
18386
|
if (Number.isNaN(parseInt(arrayIndex))) {
|
|
18275
|
-
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${
|
|
18387
|
+
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path30}'`);
|
|
18276
18388
|
}
|
|
18277
18389
|
if (squareBracketIndex !== 0) {
|
|
18278
18390
|
pathList.push(part.slice(0, squareBracketIndex));
|
|
@@ -18293,9 +18405,9 @@ var init_getAttr = __esm({
|
|
|
18293
18405
|
"../../node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttr.js"() {
|
|
18294
18406
|
init_types2();
|
|
18295
18407
|
init_getAttrPathList();
|
|
18296
|
-
getAttr = (value,
|
|
18408
|
+
getAttr = (value, path30) => getAttrPathList(path30).reduce((acc, index) => {
|
|
18297
18409
|
if (typeof acc !== "object") {
|
|
18298
|
-
throw new EndpointError(`Index '${index}' in '${
|
|
18410
|
+
throw new EndpointError(`Index '${index}' in '${path30}' not found in '${JSON.stringify(value)}'`);
|
|
18299
18411
|
} else if (Array.isArray(acc)) {
|
|
18300
18412
|
const i6 = parseInt(index);
|
|
18301
18413
|
return acc[i6 < 0 ? acc.length + i6 : i6];
|
|
@@ -18356,8 +18468,8 @@ var init_parseURL = __esm({
|
|
|
18356
18468
|
return value;
|
|
18357
18469
|
}
|
|
18358
18470
|
if (typeof value === "object" && "hostname" in value) {
|
|
18359
|
-
const { hostname: hostname2, port, protocol: protocol2 = "", path:
|
|
18360
|
-
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${
|
|
18471
|
+
const { hostname: hostname2, port, protocol: protocol2 = "", path: path30 = "", query = {} } = value;
|
|
18472
|
+
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path30}`);
|
|
18361
18473
|
url.search = Object.entries(query).map(([k7, v3]) => `${k7}=${v3}`).join("&");
|
|
18362
18474
|
return url;
|
|
18363
18475
|
}
|
|
@@ -20701,11 +20813,11 @@ var init_HttpBindingProtocol = __esm({
|
|
|
20701
20813
|
const opTraits = translateTraits(operationSchema.traits);
|
|
20702
20814
|
if (opTraits.http) {
|
|
20703
20815
|
request.method = opTraits.http[0];
|
|
20704
|
-
const [
|
|
20816
|
+
const [path30, search] = opTraits.http[1].split("?");
|
|
20705
20817
|
if (request.path == "/") {
|
|
20706
|
-
request.path =
|
|
20818
|
+
request.path = path30;
|
|
20707
20819
|
} else {
|
|
20708
|
-
request.path +=
|
|
20820
|
+
request.path += path30;
|
|
20709
20821
|
}
|
|
20710
20822
|
const traitSearchParams = new URLSearchParams(search ?? "");
|
|
20711
20823
|
for (const [key, value] of traitSearchParams) {
|
|
@@ -21074,8 +21186,8 @@ var init_FromStringShapeDeserializer = __esm({
|
|
|
21074
21186
|
return (this.serdeContext?.base64Decoder ?? fromBase64)(data);
|
|
21075
21187
|
}
|
|
21076
21188
|
if (ns2.isTimestampSchema()) {
|
|
21077
|
-
const
|
|
21078
|
-
switch (
|
|
21189
|
+
const format4 = determineTimestampFormat(ns2, this.settings);
|
|
21190
|
+
switch (format4) {
|
|
21079
21191
|
case 5:
|
|
21080
21192
|
return _parseRfc3339DateTimeWithOffset(data);
|
|
21081
21193
|
case 6:
|
|
@@ -21197,8 +21309,8 @@ var init_ToStringShapeSerializer = __esm({
|
|
|
21197
21309
|
if (!(value instanceof Date)) {
|
|
21198
21310
|
throw new Error(`@smithy/core/protocols - received non-Date value ${value} when schema expected Date in ${ns2.getName(true)}`);
|
|
21199
21311
|
}
|
|
21200
|
-
const
|
|
21201
|
-
switch (
|
|
21312
|
+
const format4 = determineTimestampFormat(ns2, this.settings);
|
|
21313
|
+
switch (format4) {
|
|
21202
21314
|
case 5:
|
|
21203
21315
|
this.stringBuffer = value.toISOString().replace(".000Z", "Z");
|
|
21204
21316
|
break;
|
|
@@ -24418,12 +24530,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
24418
24530
|
const password = request.password ?? "";
|
|
24419
24531
|
auth = `${username}:${password}`;
|
|
24420
24532
|
}
|
|
24421
|
-
let
|
|
24533
|
+
let path30 = request.path;
|
|
24422
24534
|
if (queryString) {
|
|
24423
|
-
|
|
24535
|
+
path30 += `?${queryString}`;
|
|
24424
24536
|
}
|
|
24425
24537
|
if (request.fragment) {
|
|
24426
|
-
|
|
24538
|
+
path30 += `#${request.fragment}`;
|
|
24427
24539
|
}
|
|
24428
24540
|
let hostname = request.hostname ?? "";
|
|
24429
24541
|
if (hostname[0] === "[" && hostname.endsWith("]")) {
|
|
@@ -24435,7 +24547,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
24435
24547
|
headers: request.headers,
|
|
24436
24548
|
host: hostname,
|
|
24437
24549
|
method: request.method,
|
|
24438
|
-
path:
|
|
24550
|
+
path: path30,
|
|
24439
24551
|
port: request.port,
|
|
24440
24552
|
agent,
|
|
24441
24553
|
auth
|
|
@@ -25344,10 +25456,10 @@ ${longDate}
|
|
|
25344
25456
|
${credentialScope}
|
|
25345
25457
|
${toHex(hashedRequest)}`;
|
|
25346
25458
|
}
|
|
25347
|
-
getCanonicalPath({ path:
|
|
25459
|
+
getCanonicalPath({ path: path30 }) {
|
|
25348
25460
|
if (this.uriEscapePath) {
|
|
25349
25461
|
const normalizedPathSegments = [];
|
|
25350
|
-
for (const pathSegment of
|
|
25462
|
+
for (const pathSegment of path30.split("/")) {
|
|
25351
25463
|
if (pathSegment?.length === 0)
|
|
25352
25464
|
continue;
|
|
25353
25465
|
if (pathSegment === ".")
|
|
@@ -25358,11 +25470,11 @@ ${toHex(hashedRequest)}`;
|
|
|
25358
25470
|
normalizedPathSegments.push(pathSegment);
|
|
25359
25471
|
}
|
|
25360
25472
|
}
|
|
25361
|
-
const normalizedPath = `${
|
|
25473
|
+
const normalizedPath = `${path30?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path30?.endsWith("/") ? "/" : ""}`;
|
|
25362
25474
|
const doubleEncoded = escapeUri(normalizedPath);
|
|
25363
25475
|
return doubleEncoded.replace(/%2F/g, "/");
|
|
25364
25476
|
}
|
|
25365
|
-
return
|
|
25477
|
+
return path30;
|
|
25366
25478
|
}
|
|
25367
25479
|
validateResolvedCredentials(credentials) {
|
|
25368
25480
|
if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
|
|
@@ -26432,8 +26544,8 @@ var init_JsonShapeDeserializer = __esm({
|
|
|
26432
26544
|
return value;
|
|
26433
26545
|
}
|
|
26434
26546
|
if (ns2.isTimestampSchema() && value != null) {
|
|
26435
|
-
const
|
|
26436
|
-
switch (
|
|
26547
|
+
const format4 = determineTimestampFormat(ns2, this.settings);
|
|
26548
|
+
switch (format4) {
|
|
26437
26549
|
case 5:
|
|
26438
26550
|
return parseRfc3339DateTimeWithOffset(value);
|
|
26439
26551
|
case 6:
|
|
@@ -26657,8 +26769,8 @@ var init_JsonShapeSerializer = __esm({
|
|
|
26657
26769
|
return (this.serdeContext?.base64Encoder ?? toBase64)(value);
|
|
26658
26770
|
}
|
|
26659
26771
|
if (value instanceof Date && (ns2.isTimestampSchema() || ns2.isDocumentSchema())) {
|
|
26660
|
-
const
|
|
26661
|
-
switch (
|
|
26772
|
+
const format4 = determineTimestampFormat(ns2, this.settings);
|
|
26773
|
+
switch (format4) {
|
|
26662
26774
|
case 5:
|
|
26663
26775
|
return value.toISOString().replace(".000Z", "Z");
|
|
26664
26776
|
case 6:
|
|
@@ -27286,8 +27398,8 @@ var init_QueryShapeSerializer = __esm({
|
|
|
27286
27398
|
} else if (ns2.isTimestampSchema()) {
|
|
27287
27399
|
if (value instanceof Date) {
|
|
27288
27400
|
this.writeKey(prefix2);
|
|
27289
|
-
const
|
|
27290
|
-
switch (
|
|
27401
|
+
const format4 = determineTimestampFormat(ns2, this.settings);
|
|
27402
|
+
switch (format4) {
|
|
27291
27403
|
case 5:
|
|
27292
27404
|
this.writeValue(value.toISOString().replace(".000Z", "Z"));
|
|
27293
27405
|
break;
|
|
@@ -41722,11 +41834,13 @@ var init_src7 = __esm({
|
|
|
41722
41834
|
});
|
|
41723
41835
|
|
|
41724
41836
|
// ../../packages/drivers/src/DeltaShareDriver.ts
|
|
41725
|
-
var fs10, import_promises7, MAX_LIMIT_HINT, DeltaShareSourceDriver, DeltaShareDriver_default;
|
|
41837
|
+
var fs10, import_promises7, import_stream, import_promises8, MAX_LIMIT_HINT, PARQUET_BATCH_ROWS, DeltaShareSourceDriver, DeltaShareDriver_default;
|
|
41726
41838
|
var init_DeltaShareDriver = __esm({
|
|
41727
41839
|
"../../packages/drivers/src/DeltaShareDriver.ts"() {
|
|
41728
41840
|
fs10 = __toESM(require("fs"), 1);
|
|
41729
41841
|
import_promises7 = __toESM(require("fs/promises"), 1);
|
|
41842
|
+
import_stream = require("stream");
|
|
41843
|
+
import_promises8 = require("stream/promises");
|
|
41730
41844
|
init_src();
|
|
41731
41845
|
init_src4();
|
|
41732
41846
|
init_src2();
|
|
@@ -41734,6 +41848,7 @@ var init_DeltaShareDriver = __esm({
|
|
|
41734
41848
|
init_DeltaSharePredicate();
|
|
41735
41849
|
init_OidcTokenProvider();
|
|
41736
41850
|
MAX_LIMIT_HINT = 2147483647;
|
|
41851
|
+
PARQUET_BATCH_ROWS = 5e4;
|
|
41737
41852
|
DeltaShareSourceDriver = class {
|
|
41738
41853
|
constructor() {
|
|
41739
41854
|
this._query = "{prefix}/shares/{share}/schemas/{schema}/tables/{table}/query";
|
|
@@ -41743,6 +41858,9 @@ var init_DeltaShareDriver = __esm({
|
|
|
41743
41858
|
// Column Delta types per table, for the predicate's literals. A table's schema does not change
|
|
41744
41859
|
// under a run, and only a read with something to push ever asks for it.
|
|
41745
41860
|
this._columnTypes = /* @__PURE__ */ new Map();
|
|
41861
|
+
// Rows held as objects at once. A run leaves this at PARQUET_BATCH_ROWS; the tests lower it so a
|
|
41862
|
+
// fixture small enough to read whole still splits its row groups the way a real part file does.
|
|
41863
|
+
this._batchRows = PARQUET_BATCH_ROWS;
|
|
41746
41864
|
this.init = async (source) => {
|
|
41747
41865
|
Affirm_default(source, "Invalid source");
|
|
41748
41866
|
const { authentication } = source;
|
|
@@ -41803,12 +41921,11 @@ var init_DeltaShareDriver = __esm({
|
|
|
41803
41921
|
Affirm_default(request, "Invalid download request");
|
|
41804
41922
|
const table = this._resolveTable(request.fileKey);
|
|
41805
41923
|
const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
|
|
41806
|
-
const hyparquet = await import("hyparquet");
|
|
41807
41924
|
const lines = [];
|
|
41808
|
-
for (const deltaFile of deltaFiles)
|
|
41809
|
-
const
|
|
41810
|
-
|
|
41811
|
-
|
|
41925
|
+
for (const deltaFile of deltaFiles)
|
|
41926
|
+
for await (const batch of this._readRemoteParquetBatches(deltaFile))
|
|
41927
|
+
for (const record of batch)
|
|
41928
|
+
lines.push(JSON.stringify(record));
|
|
41812
41929
|
Logger_default.log(`Delta Share readAll: ${lines.length} record(s) from ${deltaFiles.length} file(s) of table "${table}"`);
|
|
41813
41930
|
return lines;
|
|
41814
41931
|
};
|
|
@@ -41818,22 +41935,41 @@ var init_DeltaShareDriver = __esm({
|
|
|
41818
41935
|
Affirm_default(request.options.lineFrom !== void 0 && request.options.lineTo !== void 0, "Missing read range");
|
|
41819
41936
|
const table = this._resolveTable(request.fileKey);
|
|
41820
41937
|
const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
|
|
41821
|
-
const hyparquet = await import("hyparquet");
|
|
41822
41938
|
const { options: { lineFrom, lineTo } } = request;
|
|
41939
|
+
const lines = await this._collectLinesInRange(deltaFiles, { from: lineFrom, to: lineTo });
|
|
41940
|
+
Logger_default.log(`Delta Share readLinesInRange [${lineFrom}, ${lineTo}) on table "${table}": ${lines.length} record(s)`);
|
|
41941
|
+
return lines;
|
|
41942
|
+
};
|
|
41943
|
+
/**
|
|
41944
|
+
* The records of the given range, serialized. Reading stops the moment the range is filled: a
|
|
41945
|
+
* part file holds hundreds of thousands of rows and a preview asks for a handful of them, so
|
|
41946
|
+
* decoding past `to` is both slow and the memory the batching above exists to avoid.
|
|
41947
|
+
*/
|
|
41948
|
+
this._collectLinesInRange = async (deltaFiles, range2) => {
|
|
41949
|
+
if (range2.to <= range2.from)
|
|
41950
|
+
return [];
|
|
41823
41951
|
const lines = [];
|
|
41824
41952
|
let index = 0;
|
|
41825
41953
|
for (const deltaFile of deltaFiles) {
|
|
41826
|
-
const
|
|
41827
|
-
|
|
41828
|
-
if (index >=
|
|
41829
|
-
lines
|
|
41830
|
-
index++;
|
|
41831
|
-
if (index >= lineTo) break;
|
|
41954
|
+
for await (const batch of this._readRemoteParquetBatches(deltaFile)) {
|
|
41955
|
+
index = this._pushRecordsInRange(lines, batch, index, range2);
|
|
41956
|
+
if (index >= range2.to)
|
|
41957
|
+
return lines;
|
|
41832
41958
|
}
|
|
41833
41959
|
}
|
|
41834
|
-
Logger_default.log(`Delta Share readLinesInRange [${lineFrom}, ${lineTo}) on table "${table}": ${lines.length} record(s)`);
|
|
41835
41960
|
return lines;
|
|
41836
41961
|
};
|
|
41962
|
+
// The index the batch ends on, so the next batch carries on counting where this one stopped.
|
|
41963
|
+
this._pushRecordsInRange = (lines, batch, index, range2) => {
|
|
41964
|
+
for (const record of batch) {
|
|
41965
|
+
if (index >= range2.from && index < range2.to)
|
|
41966
|
+
lines.push(JSON.stringify(record));
|
|
41967
|
+
index++;
|
|
41968
|
+
if (index >= range2.to)
|
|
41969
|
+
break;
|
|
41970
|
+
}
|
|
41971
|
+
return index;
|
|
41972
|
+
};
|
|
41837
41973
|
this.exist = async (_producer) => {
|
|
41838
41974
|
void _producer;
|
|
41839
41975
|
let nextPageToken;
|
|
@@ -41977,20 +42113,48 @@ var init_DeltaShareDriver = __esm({
|
|
|
41977
42113
|
return "(unparseable url)";
|
|
41978
42114
|
}
|
|
41979
42115
|
};
|
|
41980
|
-
|
|
42116
|
+
// Split out so the batch loop above stays one level of nesting, and so both failures name the file
|
|
42117
|
+
this._readParquetMetadata = async (file, label) => {
|
|
41981
42118
|
const hyparquet = await import("hyparquet");
|
|
41982
|
-
const { compressors: compressors2 } = await Promise.resolve().then(() => (init_src7(), src_exports));
|
|
41983
|
-
const byteLength = deltaFile.file.deltaSingleAction.add?.size ?? deltaFile.file.deltaSingleAction.remove?.size;
|
|
41984
|
-
const startedMs = Date.now();
|
|
41985
|
-
let records;
|
|
41986
42119
|
try {
|
|
41987
|
-
|
|
41988
|
-
records = await hyparquet.parquetReadObjects({ file, compressors: compressors2 });
|
|
42120
|
+
return await hyparquet.parquetMetadataAsync(file);
|
|
41989
42121
|
} catch (error) {
|
|
41990
|
-
throw new Error(`Delta Share failed reading parquet
|
|
42122
|
+
throw new Error(`Delta Share failed reading the parquet footer of ${label}`, { cause: error });
|
|
41991
42123
|
}
|
|
41992
|
-
|
|
41993
|
-
|
|
42124
|
+
};
|
|
42125
|
+
this._readParquetBatch = async (options, label) => {
|
|
42126
|
+
const hyparquet = await import("hyparquet");
|
|
42127
|
+
try {
|
|
42128
|
+
return await hyparquet.parquetReadObjects(options);
|
|
42129
|
+
} catch (error) {
|
|
42130
|
+
throw new Error(`Delta Share failed reading parquet rows [${options.rowStart}, ${options.rowEnd}) of ${label}`, { cause: error });
|
|
42131
|
+
}
|
|
42132
|
+
};
|
|
42133
|
+
// Reads straight from the pre-signed URL. Only for the bounded readers (preview, sample): every
|
|
42134
|
+
// batch refetches the row group, which is why `ready` stages the file locally instead.
|
|
42135
|
+
this._readRemoteParquetBatches = (deltaFile) => {
|
|
42136
|
+
const byteLength = deltaFile.file.deltaSingleAction.add?.size ?? deltaFile.file.deltaSingleAction.remove?.size;
|
|
42137
|
+
const label = `${this._redactUrl(deltaFile.file.url)} (${byteLength ?? "?"} bytes)`;
|
|
42138
|
+
const open = async () => {
|
|
42139
|
+
const hyparquet = await import("hyparquet");
|
|
42140
|
+
return await hyparquet.asyncBufferFromUrl({ url: deltaFile.file.url, byteLength });
|
|
42141
|
+
};
|
|
42142
|
+
return this._streamParquetBatchesFrom(open, label);
|
|
42143
|
+
};
|
|
42144
|
+
/**
|
|
42145
|
+
* Copies a part file to local disk, streamed, so nothing but the socket buffer is ever in memory.
|
|
42146
|
+
* `ready` re-reads the file once per batch and re-reading it over HTTP would refetch the same
|
|
42147
|
+
* bytes every time.
|
|
42148
|
+
*/
|
|
42149
|
+
this._downloadParquet = async (deltaFile, localPath) => {
|
|
42150
|
+
const startedMs = Date.now();
|
|
42151
|
+
const res = await this._fetch(deltaFile.file.url, { method: "GET" }, `parquet download of ${this._redactUrl(deltaFile.file.url)}`);
|
|
42152
|
+
if (!res.ok)
|
|
42153
|
+
throw new Error(`Error downloading parquet from the delta share: ${res.status} ${res.statusText} (${this._redactUrl(deltaFile.file.url)})`);
|
|
42154
|
+
Affirm_default(res.body, `Empty response body downloading parquet from ${this._redactUrl(deltaFile.file.url)}`);
|
|
42155
|
+
await (0, import_promises8.pipeline)(import_stream.Readable.fromWeb(res.body), fs10.createWriteStream(localPath));
|
|
42156
|
+
const { size } = await import_promises7.default.stat(localPath);
|
|
42157
|
+
Logger_default.log(`Delta Share staged ${this._redactUrl(deltaFile.file.url)} (${size} bytes) in ${Date.now() - startedMs}ms`);
|
|
41994
42158
|
};
|
|
41995
42159
|
this._serializeDatasetValue = (value) => {
|
|
41996
42160
|
if (value === null || value === void 0)
|
|
@@ -42004,17 +42168,41 @@ var init_DeltaShareDriver = __esm({
|
|
|
42004
42168
|
this._getDeltaFileKey = (table, index) => {
|
|
42005
42169
|
return `${this._resolveSchema()}/${table}/part-${index}.parquet`;
|
|
42006
42170
|
};
|
|
42007
|
-
|
|
42008
|
-
|
|
42009
|
-
|
|
42010
|
-
|
|
42011
|
-
|
|
42012
|
-
|
|
42013
|
-
|
|
42014
|
-
|
|
42015
|
-
|
|
42016
|
-
|
|
42017
|
-
|
|
42171
|
+
/**
|
|
42172
|
+
* Writes the batches out as one CSV dataset file and reports how many records landed. Rows are
|
|
42173
|
+
* written as they arrive rather than joined into one string first: a file's worth of lines, and
|
|
42174
|
+
* the single string joining them, were each as large as the records themselves.
|
|
42175
|
+
*
|
|
42176
|
+
* The header is taken from the first record and reused for every later one, so a file whose rows
|
|
42177
|
+
* disagree on their columns still produces a rectangular dataset.
|
|
42178
|
+
*/
|
|
42179
|
+
this._writeDatasetFile = async (localPath, batches, delimiter) => {
|
|
42180
|
+
const stream = fs10.createWriteStream(localPath);
|
|
42181
|
+
let header = null;
|
|
42182
|
+
let records = 0;
|
|
42183
|
+
try {
|
|
42184
|
+
for await (const batch of batches) {
|
|
42185
|
+
for (const record of batch) {
|
|
42186
|
+
if (!header) {
|
|
42187
|
+
header = Object.keys(record);
|
|
42188
|
+
await this._writeLine(stream, CSVParser_default.stringifyRow(header, delimiter));
|
|
42189
|
+
}
|
|
42190
|
+
await this._writeLine(stream, CSVParser_default.stringifyRow(header.map((column) => this._serializeDatasetValue(record[column])), delimiter));
|
|
42191
|
+
records++;
|
|
42192
|
+
}
|
|
42193
|
+
}
|
|
42194
|
+
} finally {
|
|
42195
|
+
await new Promise((resolve) => stream.end(resolve));
|
|
42196
|
+
}
|
|
42197
|
+
return records;
|
|
42198
|
+
};
|
|
42199
|
+
// A false from write() means the buffer is full; waiting for the drain just then keeps the queued
|
|
42200
|
+
// rows bounded instead of letting them pile up faster than the disk takes them.
|
|
42201
|
+
this._writeLine = async (stream, line) => {
|
|
42202
|
+
if (stream.write(`${line}
|
|
42203
|
+
`))
|
|
42204
|
+
return;
|
|
42205
|
+
await new Promise((resolve) => stream.once("drain", resolve));
|
|
42018
42206
|
};
|
|
42019
42207
|
/**
|
|
42020
42208
|
* Every file the share reports for this table. `jsonPredicateHints` lets the server skip the
|
|
@@ -42078,30 +42266,22 @@ var init_DeltaShareDriver = __esm({
|
|
|
42078
42266
|
const table = this._resolveTable(producer.settings.fileKey);
|
|
42079
42267
|
const hints = await this._buildHints(producer, table, request.pushdown);
|
|
42080
42268
|
const deltaFiles = await this._getAllFilesInTable(table, producer.settings.disableHistory, hints);
|
|
42081
|
-
const hyparquet = await import("hyparquet");
|
|
42082
42269
|
const delimiter = producer.settings.delimiter ?? ",";
|
|
42083
42270
|
const files = [];
|
|
42084
42271
|
let records = 0;
|
|
42085
42272
|
let skipped = 0;
|
|
42086
42273
|
for (const [index, deltaFile] of deltaFiles.entries()) {
|
|
42087
42274
|
onProgress?.(index / deltaFiles.length);
|
|
42088
|
-
const parquetRecords = await this._readParquetObjects(deltaFile);
|
|
42089
|
-
if (parquetRecords.length === 0) {
|
|
42090
|
-
skipped++;
|
|
42091
|
-
Logger_default.log(`Delta Share ready: file ${index + 1}/${deltaFiles.length} of table "${table}" holds no records, skipping`);
|
|
42092
|
-
continue;
|
|
42093
|
-
}
|
|
42094
42275
|
const sourceFileKey = this._getDeltaFileKey(table, index);
|
|
42095
42276
|
const localPath = ExecutorScope_default.getProducerPath(scope, producer, sourceFileKey);
|
|
42096
42277
|
ExecutorScope_default.ensurePath(localPath);
|
|
42097
|
-
const
|
|
42098
|
-
|
|
42099
|
-
if (written) {
|
|
42278
|
+
const written = await this._stageDeltaFile(deltaFile, localPath, delimiter);
|
|
42279
|
+
if (written > 0) {
|
|
42100
42280
|
files.push({ fullUri: localPath, fileSize: fs10.statSync(localPath).size });
|
|
42101
|
-
records +=
|
|
42281
|
+
records += written;
|
|
42102
42282
|
} else {
|
|
42103
42283
|
skipped++;
|
|
42104
|
-
Logger_default.log(`Delta Share ready:
|
|
42284
|
+
Logger_default.log(`Delta Share ready: file ${index + 1}/${deltaFiles.length} of table "${table}" (${sourceFileKey}) holds no records, discarding the staged file`);
|
|
42105
42285
|
await import_promises7.default.unlink(localPath).catch(() => {
|
|
42106
42286
|
});
|
|
42107
42287
|
}
|
|
@@ -42112,6 +42292,57 @@ var init_DeltaShareDriver = __esm({
|
|
|
42112
42292
|
Logger_default.log(`Delta Share ready: staged ${records} record(s) into ${files.length} dataset file(s) (${bytes} bytes) for producer "${producer.name}" from table "${table}"${skippedNote}`);
|
|
42113
42293
|
return { files, filteredAtSource: !!hints?.jsonPredicateHints };
|
|
42114
42294
|
};
|
|
42295
|
+
/**
|
|
42296
|
+
* One part file turned into one CSV dataset file, downloaded then converted a batch at a time so
|
|
42297
|
+
* that neither step's memory grows with the file. The download is dropped either way — it is the
|
|
42298
|
+
* dataset file the run reads from here on.
|
|
42299
|
+
*/
|
|
42300
|
+
this._stageDeltaFile = async (deltaFile, localPath, delimiter) => {
|
|
42301
|
+
const downloadPath = `${localPath}.parquet`;
|
|
42302
|
+
await this._downloadParquet(deltaFile, downloadPath);
|
|
42303
|
+
try {
|
|
42304
|
+
const open = async () => {
|
|
42305
|
+
const hyparquet = await import("hyparquet");
|
|
42306
|
+
return await hyparquet.asyncBufferFromFile(downloadPath);
|
|
42307
|
+
};
|
|
42308
|
+
return await this._writeDatasetFile(localPath, this._streamParquetBatchesFrom(open, this._redactUrl(deltaFile.file.url)), delimiter);
|
|
42309
|
+
} finally {
|
|
42310
|
+
await import_promises7.default.unlink(downloadPath).catch(() => {
|
|
42311
|
+
});
|
|
42312
|
+
}
|
|
42313
|
+
};
|
|
42314
|
+
}
|
|
42315
|
+
/**
|
|
42316
|
+
* The rows of a part file, `_batchRows` at a time and already JSON-safe.
|
|
42317
|
+
*
|
|
42318
|
+
* hyparquet decodes a whole row group's columns per read either way, but only the requested rows
|
|
42319
|
+
* are turned into objects — and the objects are what dominates — so a row group larger than a
|
|
42320
|
+
* batch is read in several passes. Every pass re-reads the group's bytes, which is why `ready`
|
|
42321
|
+
* stages the file on disk first and only the small readers below stream it from its URL.
|
|
42322
|
+
*/
|
|
42323
|
+
async *_streamParquetBatches(file, label) {
|
|
42324
|
+
const hyparquet = await import("hyparquet");
|
|
42325
|
+
const { compressors: compressors2 } = await Promise.resolve().then(() => (init_src7(), src_exports));
|
|
42326
|
+
const startedMs = Date.now();
|
|
42327
|
+
const metadata = await this._readParquetMetadata(file, label);
|
|
42328
|
+
let records = 0;
|
|
42329
|
+
let groupStart = 0;
|
|
42330
|
+
for (const rowGroup of metadata.row_groups) {
|
|
42331
|
+
const groupRows = Number(rowGroup.num_rows);
|
|
42332
|
+
const groupEnd = groupStart + groupRows;
|
|
42333
|
+
for (let rowStart = groupStart; rowStart < groupEnd; rowStart += this._batchRows) {
|
|
42334
|
+
const rowEnd = Math.min(rowStart + this._batchRows, groupEnd);
|
|
42335
|
+
const batch = await this._readParquetBatch({ file, metadata, compressors: compressors2, rowStart, rowEnd }, label);
|
|
42336
|
+
records += batch.length;
|
|
42337
|
+
yield batch.map((record) => hyparquet.toJson(record));
|
|
42338
|
+
}
|
|
42339
|
+
groupStart = groupEnd;
|
|
42340
|
+
}
|
|
42341
|
+
Logger_default.log(`Delta Share read ${records} record(s) from ${label} in ${Date.now() - startedMs}ms`);
|
|
42342
|
+
}
|
|
42343
|
+
// The buffer is opened inside the generator so nothing is fetched until the first batch is pulled
|
|
42344
|
+
async *_streamParquetBatchesFrom(open, label) {
|
|
42345
|
+
yield* this._streamParquetBatches(await open(), label);
|
|
42115
42346
|
}
|
|
42116
42347
|
};
|
|
42117
42348
|
DeltaShareDriver_default = DeltaShareSourceDriver;
|
|
@@ -42445,12 +42676,12 @@ var init_HttpApiDriver = __esm({
|
|
|
42445
42676
|
});
|
|
42446
42677
|
|
|
42447
42678
|
// ../../packages/drivers/src/DriverHelper.ts
|
|
42448
|
-
var
|
|
42679
|
+
var import_stream2, import_readline, import_promises9, import_fs9, DriverHelper, DriverHelper_default;
|
|
42449
42680
|
var init_DriverHelper = __esm({
|
|
42450
42681
|
"../../packages/drivers/src/DriverHelper.ts"() {
|
|
42451
|
-
|
|
42682
|
+
import_stream2 = require("stream");
|
|
42452
42683
|
import_readline = require("readline");
|
|
42453
|
-
|
|
42684
|
+
import_promises9 = require("stream/promises");
|
|
42454
42685
|
import_fs9 = require("fs");
|
|
42455
42686
|
init_src2();
|
|
42456
42687
|
init_src();
|
|
@@ -42467,7 +42698,7 @@ var init_DriverHelper = __esm({
|
|
|
42467
42698
|
let leftoverData = "";
|
|
42468
42699
|
let globalIndex = 0;
|
|
42469
42700
|
let lineCount = 0;
|
|
42470
|
-
const headerValidationTransform = new
|
|
42701
|
+
const headerValidationTransform = new import_stream2.Transform({
|
|
42471
42702
|
transform(chunk, encoding, callback) {
|
|
42472
42703
|
const chunkStr = leftoverData + chunk.toString();
|
|
42473
42704
|
const lines = chunkStr.split("\n");
|
|
@@ -42543,7 +42774,7 @@ var init_DriverHelper = __esm({
|
|
|
42543
42774
|
};
|
|
42544
42775
|
const writeOptions = append ? { flags: "a" } : {};
|
|
42545
42776
|
const writeStream = (0, import_fs9.createWriteStream)(destinationPath, writeOptions);
|
|
42546
|
-
await (0,
|
|
42777
|
+
await (0, import_promises9.pipeline)(stream, headerValidationTransform, writeStream);
|
|
42547
42778
|
return lineCount;
|
|
42548
42779
|
},
|
|
42549
42780
|
appendObjectsToUnifiedFile: async (options) => {
|
|
@@ -42569,32 +42800,32 @@ var init_DriverHelper = __esm({
|
|
|
42569
42800
|
},
|
|
42570
42801
|
quickReadFile: async (filePath, lineCount) => {
|
|
42571
42802
|
const fileStream = (0, import_fs9.createReadStream)(filePath);
|
|
42572
|
-
const
|
|
42803
|
+
const rl7 = (0, import_readline.createInterface)({ input: fileStream, crlfDelay: Infinity });
|
|
42573
42804
|
const lines = [];
|
|
42574
42805
|
let counter = 0;
|
|
42575
|
-
for await (const line of
|
|
42806
|
+
for await (const line of rl7) {
|
|
42576
42807
|
lines.push(line);
|
|
42577
42808
|
counter++;
|
|
42578
42809
|
if (counter >= lineCount) {
|
|
42579
42810
|
break;
|
|
42580
42811
|
}
|
|
42581
42812
|
}
|
|
42582
|
-
|
|
42813
|
+
rl7.close();
|
|
42583
42814
|
fileStream.close();
|
|
42584
42815
|
return lines;
|
|
42585
42816
|
},
|
|
42586
42817
|
quickReadStream: async (stream, lineCount) => {
|
|
42587
|
-
const
|
|
42818
|
+
const rl7 = (0, import_readline.createInterface)({ input: stream, crlfDelay: Infinity });
|
|
42588
42819
|
const lines = [];
|
|
42589
42820
|
let counter = 0;
|
|
42590
|
-
for await (const line of
|
|
42821
|
+
for await (const line of rl7) {
|
|
42591
42822
|
lines.push(line);
|
|
42592
42823
|
counter++;
|
|
42593
42824
|
if (counter >= lineCount) {
|
|
42594
42825
|
break;
|
|
42595
42826
|
}
|
|
42596
42827
|
}
|
|
42597
|
-
|
|
42828
|
+
rl7.close();
|
|
42598
42829
|
return lines;
|
|
42599
42830
|
}
|
|
42600
42831
|
};
|
|
@@ -42853,11 +43084,11 @@ var init_LocalSourceDriver = __esm({
|
|
|
42853
43084
|
});
|
|
42854
43085
|
|
|
42855
43086
|
// ../../packages/drivers/src/files/LocalDestinationDriver.ts
|
|
42856
|
-
var fs12,
|
|
43087
|
+
var fs12, import_promises10, import_readline3, import_path10, LocalDestinationDriver;
|
|
42857
43088
|
var init_LocalDestinationDriver = __esm({
|
|
42858
43089
|
"../../packages/drivers/src/files/LocalDestinationDriver.ts"() {
|
|
42859
43090
|
fs12 = __toESM(require("fs"), 1);
|
|
42860
|
-
|
|
43091
|
+
import_promises10 = __toESM(require("fs/promises"), 1);
|
|
42861
43092
|
import_readline3 = __toESM(require("readline"), 1);
|
|
42862
43093
|
import_path10 = __toESM(require("path"), 1);
|
|
42863
43094
|
init_src();
|
|
@@ -42892,8 +43123,8 @@ var init_LocalDestinationDriver = __esm({
|
|
|
42892
43123
|
this.move = async (fromPath, toName) => {
|
|
42893
43124
|
try {
|
|
42894
43125
|
const toFilePath = import_path10.default.join(this._path, toName);
|
|
42895
|
-
await
|
|
42896
|
-
await
|
|
43126
|
+
await import_promises10.default.copyFile(fromPath, toFilePath);
|
|
43127
|
+
await import_promises10.default.unlink(fromPath);
|
|
42897
43128
|
return { bucket: "", key: toFilePath, res: true };
|
|
42898
43129
|
} catch (error) {
|
|
42899
43130
|
Logger_default.error(error);
|
|
@@ -42929,14 +43160,14 @@ var init_LocalDestinationDriver = __esm({
|
|
|
42929
43160
|
});
|
|
42930
43161
|
|
|
42931
43162
|
// ../../packages/drivers/src/s3/S3SourceDriver.ts
|
|
42932
|
-
var import_client_s32,
|
|
43163
|
+
var import_client_s32, import_stream3, import_promises11, import_readline4, import_path11, import_fs10, import_xlsx2, S3SourceDriver;
|
|
42933
43164
|
var init_S3SourceDriver = __esm({
|
|
42934
43165
|
"../../packages/drivers/src/s3/S3SourceDriver.ts"() {
|
|
42935
43166
|
import_client_s32 = require("@aws-sdk/client-s3");
|
|
42936
43167
|
init_src();
|
|
42937
43168
|
init_AwsCredentials();
|
|
42938
|
-
|
|
42939
|
-
|
|
43169
|
+
import_stream3 = require("stream");
|
|
43170
|
+
import_promises11 = require("stream/promises");
|
|
42940
43171
|
import_readline4 = __toESM(require("readline"), 1);
|
|
42941
43172
|
import_path11 = __toESM(require("path"), 1);
|
|
42942
43173
|
import_fs10 = __toESM(require("fs"), 1);
|
|
@@ -43204,7 +43435,7 @@ var init_S3SourceDriver = __esm({
|
|
|
43204
43435
|
}
|
|
43205
43436
|
const totalBytes = response.ContentLength ?? 0;
|
|
43206
43437
|
let stagedBytes = 0;
|
|
43207
|
-
const meter = new
|
|
43438
|
+
const meter = new import_stream3.Transform({
|
|
43208
43439
|
transform(chunk, _encoding, done) {
|
|
43209
43440
|
stagedBytes += chunk.length;
|
|
43210
43441
|
if (totalBytes > 0)
|
|
@@ -43213,7 +43444,7 @@ var init_S3SourceDriver = __esm({
|
|
|
43213
43444
|
}
|
|
43214
43445
|
});
|
|
43215
43446
|
const writeStream = import_fs10.default.createWriteStream(localPath);
|
|
43216
|
-
await (0,
|
|
43447
|
+
await (0, import_promises11.pipeline)(response.Body, meter, writeStream);
|
|
43217
43448
|
report3(1);
|
|
43218
43449
|
};
|
|
43219
43450
|
if (fileKey.includes("%")) {
|
|
@@ -43368,7 +43599,7 @@ var init_S3DestinationDriver = __esm({
|
|
|
43368
43599
|
});
|
|
43369
43600
|
|
|
43370
43601
|
// ../../packages/drivers/src/DriverFactory.ts
|
|
43371
|
-
var DriverFactoryClass, DriverFactory, DriverFactory_default;
|
|
43602
|
+
var STAGEABLE_ENGINES, DriverFactoryClass, DriverFactory, DriverFactory_default;
|
|
43372
43603
|
var init_DriverFactory = __esm({
|
|
43373
43604
|
"../../packages/drivers/src/DriverFactory.ts"() {
|
|
43374
43605
|
init_RedshiftDriver();
|
|
@@ -43378,8 +43609,11 @@ var init_DriverFactory = __esm({
|
|
|
43378
43609
|
init_LocalDestinationDriver();
|
|
43379
43610
|
init_S3SourceDriver();
|
|
43380
43611
|
init_S3DestinationDriver();
|
|
43612
|
+
STAGEABLE_ENGINES = ["local", "aws-s3", "delta-share"];
|
|
43381
43613
|
DriverFactoryClass = class {
|
|
43382
43614
|
constructor() {
|
|
43615
|
+
/** Whether a run can stage a producer from this engine — see `STAGEABLE_ENGINES`. */
|
|
43616
|
+
this.canStage = (engine) => STAGEABLE_ENGINES.includes(engine);
|
|
43383
43617
|
this.instantiateSource = async (source) => {
|
|
43384
43618
|
switch (source.engine) {
|
|
43385
43619
|
case "aws-redshift": {
|
|
@@ -44132,6 +44366,7 @@ var init_ProducerManager = __esm({
|
|
|
44132
44366
|
init_src4();
|
|
44133
44367
|
init_src3();
|
|
44134
44368
|
init_src6();
|
|
44369
|
+
init_CryptoEngine();
|
|
44135
44370
|
ProducerManagerClass = class {
|
|
44136
44371
|
constructor() {
|
|
44137
44372
|
this.getColumns = (producer) => {
|
|
@@ -44186,6 +44421,46 @@ var init_ProducerManager = __esm({
|
|
|
44186
44421
|
return true;
|
|
44187
44422
|
return FixedWidthParser_default.matchesSelector(line, resolved.selector, resolved.range);
|
|
44188
44423
|
};
|
|
44424
|
+
/**
|
|
44425
|
+
* The post-parse work a producer implies for every record it produces: which keys carry the
|
|
44426
|
+
* source file name, and which carry a mask.
|
|
44427
|
+
*
|
|
44428
|
+
* Resolved once per chunk, never per line — `getMask` resolves a secret and validates the value,
|
|
44429
|
+
* and neither depends on the record. Empty for almost every producer, which is the case worth
|
|
44430
|
+
* keeping fast.
|
|
44431
|
+
*/
|
|
44432
|
+
this.compileFills = (dimensions) => {
|
|
44433
|
+
Affirm_default(dimensions, "Invalid producer dimensions");
|
|
44434
|
+
const fills = [];
|
|
44435
|
+
for (const dimension of dimensions) {
|
|
44436
|
+
const { prodDimension } = dimension;
|
|
44437
|
+
const mask = this.getMask(prodDimension);
|
|
44438
|
+
const sourceFilename = prodDimension.sourceFilename === true;
|
|
44439
|
+
if (!sourceFilename && !Algo_default.hasVal(mask))
|
|
44440
|
+
continue;
|
|
44441
|
+
fills.push({ key: dimension.name, mask, sourceFilename, type: prodDimension.type });
|
|
44442
|
+
}
|
|
44443
|
+
return fills;
|
|
44444
|
+
};
|
|
44445
|
+
/**
|
|
44446
|
+
* Applies the compiled fills to one parsed record, in place.
|
|
44447
|
+
*
|
|
44448
|
+
* The one place a producer dimension's mask is ever applied, on purpose: a run reaches it through
|
|
44449
|
+
* `ProducerExecutor.processLine` and a sample through `ProducerEngine`, and a masked dimension
|
|
44450
|
+
* that came back in the clear from one of the two would be a `hash` the project declared and the
|
|
44451
|
+
* screen ignored — a disclosure, not a display bug.
|
|
44452
|
+
*/
|
|
44453
|
+
this.applyFills = (record, fills, sourceFilename) => {
|
|
44454
|
+
if (!fills?.length)
|
|
44455
|
+
return record;
|
|
44456
|
+
for (const fill2 of fills) {
|
|
44457
|
+
if (fill2.sourceFilename)
|
|
44458
|
+
record[fill2.key] = sourceFilename;
|
|
44459
|
+
if (Algo_default.hasVal(fill2.mask))
|
|
44460
|
+
record[fill2.key] = CryptoEngine_default.hashValue(fill2.mask, record[fill2.key]?.toString(), fill2.type);
|
|
44461
|
+
}
|
|
44462
|
+
return record;
|
|
44463
|
+
};
|
|
44189
44464
|
}
|
|
44190
44465
|
};
|
|
44191
44466
|
ProducerManager = new ProducerManagerClass();
|
|
@@ -44222,7 +44497,7 @@ var init_TypeCaster = __esm({
|
|
|
44222
44497
|
* - Parsing dates (type 'date'/'datetime') from string with tokens: yyyy, mm, dd
|
|
44223
44498
|
* - Formatting dates when casting to string with same tokens
|
|
44224
44499
|
*/
|
|
44225
|
-
cast(value, type,
|
|
44500
|
+
cast(value, type, format4) {
|
|
44226
44501
|
if (!Algo_default.hasVal(value)) return value;
|
|
44227
44502
|
switch (type) {
|
|
44228
44503
|
case "boolean": {
|
|
@@ -44233,8 +44508,8 @@ var init_TypeCaster = __esm({
|
|
|
44233
44508
|
case "date": {
|
|
44234
44509
|
let dateValue = null;
|
|
44235
44510
|
try {
|
|
44236
|
-
if (
|
|
44237
|
-
dateValue = this.parseDate(value,
|
|
44511
|
+
if (format4 && typeof value === "string")
|
|
44512
|
+
dateValue = this.parseDate(value, format4, true).toDate();
|
|
44238
44513
|
else
|
|
44239
44514
|
dateValue = new Date(value);
|
|
44240
44515
|
return dateValue.toISOString();
|
|
@@ -44242,7 +44517,7 @@ var init_TypeCaster = __esm({
|
|
|
44242
44517
|
dateValue = new Date(value);
|
|
44243
44518
|
if (!isNaN(dateValue))
|
|
44244
44519
|
return dateValue.toISOString();
|
|
44245
|
-
throw new Error(`Error casting "${value}" to date with format "${
|
|
44520
|
+
throw new Error(`Error casting "${value}" to date with format "${format4}": ${error}`);
|
|
44246
44521
|
}
|
|
44247
44522
|
}
|
|
44248
44523
|
case "number": {
|
|
@@ -44269,13 +44544,13 @@ var init_TypeCaster = __esm({
|
|
|
44269
44544
|
* advancedFormat plugin, and then unpadded for `DDD`, while a positional ordinal date is always
|
|
44270
44545
|
* 3 digits wide; the token is therefore rendered here rather than handed to dayjs.
|
|
44271
44546
|
*/
|
|
44272
|
-
formatDate(value,
|
|
44547
|
+
formatDate(value, format4) {
|
|
44273
44548
|
const date2 = import_dayjs.default.utc(value);
|
|
44274
|
-
if (!DAY_OF_YEAR_TOKEN.test(this._stripLiterals(
|
|
44275
|
-
return date2.format(
|
|
44549
|
+
if (!DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format4)))
|
|
44550
|
+
return date2.format(format4);
|
|
44276
44551
|
const startOfYear = import_dayjs.default.utc(`${String(date2.year()).padStart(4, "0")}-01-01`, "YYYY-MM-DD", true);
|
|
44277
44552
|
const dayOfYear = date2.diff(startOfYear, "day") + 1;
|
|
44278
|
-
return date2.format(
|
|
44553
|
+
return date2.format(format4.replace(/D{3,4}/g, `[${String(dayOfYear).padStart(3, "0")}]`));
|
|
44279
44554
|
}
|
|
44280
44555
|
/**
|
|
44281
44556
|
* Parses a date string with a dayjs format, as UTC. Single entry point for every date parse, so
|
|
@@ -44284,13 +44559,13 @@ var init_TypeCaster = __esm({
|
|
|
44284
44559
|
* ever FORMAT with are rejected outright — non-strict parsing would otherwise read `2007305` as
|
|
44285
44560
|
* 2007-01-05 rather than 2007-11-01 and report no error at all.
|
|
44286
44561
|
*/
|
|
44287
|
-
parseDate(value,
|
|
44288
|
-
if (DAY_OF_YEAR_TOKEN.test(this._stripLiterals(
|
|
44289
|
-
return this._parseDayOfYear(value,
|
|
44290
|
-
this._assertParsableFormat(
|
|
44291
|
-
const parsed = import_dayjs.default.utc(value,
|
|
44562
|
+
parseDate(value, format4, strict = true) {
|
|
44563
|
+
if (DAY_OF_YEAR_TOKEN.test(this._stripLiterals(format4)))
|
|
44564
|
+
return this._parseDayOfYear(value, format4);
|
|
44565
|
+
this._assertParsableFormat(format4);
|
|
44566
|
+
const parsed = import_dayjs.default.utc(value, format4, strict);
|
|
44292
44567
|
if (!parsed.isValid())
|
|
44293
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44568
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}"`);
|
|
44294
44569
|
return parsed;
|
|
44295
44570
|
}
|
|
44296
44571
|
/**
|
|
@@ -44299,61 +44574,61 @@ var init_TypeCaster = __esm({
|
|
|
44299
44574
|
* walked token by token against the format — every token of such a format has a fixed width, so
|
|
44300
44575
|
* a plain cursor is enough — and the date is then built from the year plus the ordinal offset.
|
|
44301
44576
|
*/
|
|
44302
|
-
_parseDayOfYear(value,
|
|
44577
|
+
_parseDayOfYear(value, format4) {
|
|
44303
44578
|
const parts = {};
|
|
44304
44579
|
let cursor = 0;
|
|
44305
44580
|
let index = 0;
|
|
44306
|
-
while (index <
|
|
44307
|
-
if (
|
|
44308
|
-
const close =
|
|
44581
|
+
while (index < format4.length) {
|
|
44582
|
+
if (format4[index] === "[") {
|
|
44583
|
+
const close = format4.indexOf("]", index);
|
|
44309
44584
|
if (close < 0)
|
|
44310
|
-
throw new Error(`Unterminated "[" literal in date format "${
|
|
44311
|
-
const literal2 =
|
|
44585
|
+
throw new Error(`Unterminated "[" literal in date format "${format4}"`);
|
|
44586
|
+
const literal2 = format4.slice(index + 1, close);
|
|
44312
44587
|
if (value.slice(cursor, cursor + literal2.length) !== literal2)
|
|
44313
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44588
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": expected "${literal2}" at character ${cursor + 1}`);
|
|
44314
44589
|
cursor += literal2.length;
|
|
44315
44590
|
index = close + 1;
|
|
44316
44591
|
continue;
|
|
44317
44592
|
}
|
|
44318
|
-
const match = DATE_PARSE_TOKENS.find((x2) =>
|
|
44593
|
+
const match = DATE_PARSE_TOKENS.find((x2) => format4.startsWith(x2.token, index));
|
|
44319
44594
|
if (!match) {
|
|
44320
|
-
if (/[A-Za-z]/.test(
|
|
44321
|
-
throw new Error(`Date format "${
|
|
44322
|
-
if (value[cursor] !==
|
|
44323
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44595
|
+
if (/[A-Za-z]/.test(format4[index]))
|
|
44596
|
+
throw new Error(`Date format "${format4}" mixes a day-of-year token with "${format4[index]}", which can't be parsed alongside it. Use YYYY/YY, DDD/DDDD, HH, mm, ss and SSS only.`);
|
|
44597
|
+
if (value[cursor] !== format4[index])
|
|
44598
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": expected "${format4[index]}" at character ${cursor + 1}`);
|
|
44324
44599
|
cursor++;
|
|
44325
44600
|
index++;
|
|
44326
44601
|
continue;
|
|
44327
44602
|
}
|
|
44328
44603
|
const digits = value.slice(cursor, cursor + match.width);
|
|
44329
44604
|
if (digits.length !== match.width || !/^\d+$/.test(digits))
|
|
44330
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44605
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": expected ${match.width} digit(s) for "${match.token}" at character ${cursor + 1}`);
|
|
44331
44606
|
parts[match.token] = Number(digits);
|
|
44332
44607
|
cursor += match.width;
|
|
44333
44608
|
index += match.token.length;
|
|
44334
44609
|
}
|
|
44335
44610
|
if (cursor !== value.length)
|
|
44336
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44611
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": ${value.length - cursor} unexpected trailing character(s)`);
|
|
44337
44612
|
const year2 = Algo_default.hasVal(parts["YYYY"]) ? parts["YYYY"] : parts["YY"];
|
|
44338
44613
|
const dayOfYear = Algo_default.hasVal(parts["DDDD"]) ? parts["DDDD"] : parts["DDD"];
|
|
44339
44614
|
if (!Algo_default.hasVal(year2))
|
|
44340
|
-
throw new Error(`Date format "${
|
|
44615
|
+
throw new Error(`Date format "${format4}" declares a day of year but no year, so the date is ambiguous.`);
|
|
44341
44616
|
const startOfYear = import_dayjs.default.utc(`${String(year2).padStart(4, "0")}-01-01`, "YYYY-MM-DD", true);
|
|
44342
44617
|
if (!startOfYear.isValid())
|
|
44343
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44618
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": "${year2}" is not a valid year`);
|
|
44344
44619
|
const parsed = startOfYear.add(dayOfYear - 1, "day").add(parts["HH"] ?? 0, "hour").add(parts["mm"] ?? 0, "minute").add(parts["ss"] ?? 0, "second").add(parts["SSS"] ?? 0, "millisecond");
|
|
44345
44620
|
if (dayOfYear < 1 || parsed.year() !== startOfYear.year())
|
|
44346
|
-
throw new Error(`Cannot parse "${value}" with input format "${
|
|
44621
|
+
throw new Error(`Cannot parse "${value}" with input format "${format4}": day of year ${dayOfYear} is outside year ${year2} (which has ${startOfYear.add(1, "year").diff(startOfYear, "day")} days)`);
|
|
44347
44622
|
return parsed;
|
|
44348
44623
|
}
|
|
44349
|
-
_assertParsableFormat(
|
|
44350
|
-
const stripped = this._stripLiterals(
|
|
44624
|
+
_assertParsableFormat(format4) {
|
|
44625
|
+
const stripped = this._stripLiterals(format4);
|
|
44351
44626
|
const offending = FORMAT_ONLY_TOKENS.find((token2) => stripped.includes(token2));
|
|
44352
44627
|
if (offending)
|
|
44353
|
-
throw new Error(`Date format "${
|
|
44628
|
+
throw new Error(`Date format "${format4}" uses the token "${offending}", which dayjs can only write, never read: parsing would silently produce the wrong date. Use a parseable token, or bracket it as a literal ("[${offending}]") if it is part of the text.`);
|
|
44354
44629
|
}
|
|
44355
|
-
_stripLiterals(
|
|
44356
|
-
return String(
|
|
44630
|
+
_stripLiterals(format4) {
|
|
44631
|
+
return String(format4 ?? "").replace(/\[[^\]]*\]/g, "");
|
|
44357
44632
|
}
|
|
44358
44633
|
};
|
|
44359
44634
|
TypeCaster = new TypeCasterClass();
|
|
@@ -44600,7 +44875,8 @@ var init_ProducerEngine = __esm({
|
|
|
44600
44875
|
}
|
|
44601
44876
|
};
|
|
44602
44877
|
}
|
|
44603
|
-
const
|
|
44878
|
+
const headerLines = ParseHelper_default.hasHeaderLine(effectiveProducer.settings.fileType, effectiveProducer.settings.hasHeaderRow) ? 1 : 0;
|
|
44879
|
+
const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize + headerLines } }, effectiveSource);
|
|
44604
44880
|
rawData = fileData.data;
|
|
44605
44881
|
} finally {
|
|
44606
44882
|
if (scope)
|
|
@@ -44619,13 +44895,15 @@ var init_ProducerEngine = __esm({
|
|
|
44619
44895
|
if (!rawData || rawData.length === 0) return [];
|
|
44620
44896
|
const firstItem = rawData[0];
|
|
44621
44897
|
const isLineData = typeof firstItem === "string";
|
|
44898
|
+
const fills = discover2 ? [] : ProducerManager_default.compileFills(this._fillDimensions(producer));
|
|
44899
|
+
const sourceFilename = import_path18.default.basename(producer.settings.fileKey ?? producer.settings.sqlTable ?? "");
|
|
44622
44900
|
if (isLineData) {
|
|
44623
44901
|
const lines = rawData;
|
|
44624
44902
|
const dimensions = this.deriveDimensions(lines[0], producer, discover2);
|
|
44625
44903
|
const { fileType, hasHeaderRow } = producer.settings;
|
|
44626
44904
|
const dataLines = ParseHelper_default.hasHeaderLine(fileType, hasHeaderRow) ? lines.slice(1) : lines;
|
|
44627
44905
|
const recordSelector = ProducerManager_default.resolveRecordSelector(producer);
|
|
44628
|
-
return dataLines.filter((line) => line.trim().length > 0).filter((line) => ProducerManager_default.shouldReadLine(line, recordSelector)).flatMap((line) => LineParser_default.parseAll(line, producer, dimensions, noopTracker));
|
|
44906
|
+
return dataLines.filter((line) => line.trim().length > 0).filter((line) => ProducerManager_default.shouldReadLine(line, recordSelector)).flatMap((line) => LineParser_default.parseAll(line, producer, dimensions, noopTracker)).map((record) => ProducerManager_default.applyFills(record, fills, sourceFilename));
|
|
44629
44907
|
}
|
|
44630
44908
|
if (discover2) {
|
|
44631
44909
|
return rawData.map((item) => ({ ...item }));
|
|
@@ -44636,9 +44914,15 @@ var init_ProducerEngine = __esm({
|
|
|
44636
44914
|
const key = dim.alias ?? dim.name;
|
|
44637
44915
|
record[dim.name] = TypeCaster_default.cast(item[key], dim.type, dim.format);
|
|
44638
44916
|
}
|
|
44639
|
-
return record;
|
|
44917
|
+
return ProducerManager_default.applyFills(record, fills, sourceFilename);
|
|
44640
44918
|
});
|
|
44641
44919
|
};
|
|
44920
|
+
/**
|
|
44921
|
+
* The producer's declared dimensions in the shape `compileFills` reads them in. Declaration
|
|
44922
|
+
* order, because a fill is addressed by key and never by position — unlike the executor
|
|
44923
|
+
* dimensions of a run, whose index is reconciled against the file's own header.
|
|
44924
|
+
*/
|
|
44925
|
+
this._fillDimensions = (producer) => producer.dimensions.map((prodDimension, index) => ({ index, name: prodDimension.name, prodDimension }));
|
|
44642
44926
|
/**
|
|
44643
44927
|
* Builds the executor dimensions of a producer. There are two shapes of source: a FIXED file has
|
|
44644
44928
|
* no header, so its layout comes from the positions declared on the dimensions; every other file
|
|
@@ -44736,7 +45020,7 @@ var init_ProducerEngine = __esm({
|
|
|
44736
45020
|
});
|
|
44737
45021
|
|
|
44738
45022
|
// ../../packages/engines/src/ai/DeveloperEngine.ts
|
|
44739
|
-
var import_path19,
|
|
45023
|
+
var import_path19, import_promises12, import_dayjs2, import_customParseFormat2, DeveloperEngineClass, DeveloperEngine, DeveloperEngine_default;
|
|
44740
45024
|
var init_DeveloperEngine = __esm({
|
|
44741
45025
|
"../../packages/engines/src/ai/DeveloperEngine.ts"() {
|
|
44742
45026
|
init_src();
|
|
@@ -44744,7 +45028,7 @@ var init_DeveloperEngine = __esm({
|
|
|
44744
45028
|
init_src4();
|
|
44745
45029
|
init_src8();
|
|
44746
45030
|
import_path19 = __toESM(require("path"), 1);
|
|
44747
|
-
|
|
45031
|
+
import_promises12 = __toESM(require("fs/promises"), 1);
|
|
44748
45032
|
import_dayjs2 = __toESM(require("dayjs"), 1);
|
|
44749
45033
|
import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
|
|
44750
45034
|
import_dayjs2.default.extend(import_customParseFormat2.default);
|
|
@@ -44774,8 +45058,8 @@ var init_DeveloperEngine = __esm({
|
|
|
44774
45058
|
};
|
|
44775
45059
|
mappedProducer["$schema"] = producer["$schema"];
|
|
44776
45060
|
const producerPath = Environment_default.resolveResourceFilePath("producers", producer.name);
|
|
44777
|
-
await
|
|
44778
|
-
await
|
|
45061
|
+
await import_promises12.default.mkdir(import_path19.default.dirname(producerPath), { recursive: true });
|
|
45062
|
+
await import_promises12.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
|
|
44779
45063
|
return { producer: mappedProducer, fields: typeDefinitions };
|
|
44780
45064
|
};
|
|
44781
45065
|
// Discover-only precondition: sampling real data requires a configured, readable file.
|
|
@@ -47106,9 +47390,10 @@ var init_MockDataFactory = __esm({
|
|
|
47106
47390
|
if (type === "number") {
|
|
47107
47391
|
const first = bounds?.min ?? 1;
|
|
47108
47392
|
const last = first + poolSize - 1;
|
|
47393
|
+
const ceiling = bounds?.max;
|
|
47109
47394
|
Affirm_default(
|
|
47110
|
-
|
|
47111
|
-
`Key "${norm}" has a synth range of ${first}-${
|
|
47395
|
+
ceiling === void 0 || ceiling >= last,
|
|
47396
|
+
`Key "${norm}" has a synth range of ${first}-${ceiling}, which holds fewer than the ${poolSize} distinct values it has to generate; widen it or generate fewer records`
|
|
47112
47397
|
);
|
|
47113
47398
|
for (let i6 = 0; i6 < poolSize; i6++) pool.push(first + i6);
|
|
47114
47399
|
return pool;
|
|
@@ -47311,10 +47596,10 @@ var init_MockDataFactory = __esm({
|
|
|
47311
47596
|
});
|
|
47312
47597
|
|
|
47313
47598
|
// ../../packages/engines/src/mock/SyntheticInputEngine.ts
|
|
47314
|
-
var
|
|
47599
|
+
var import_promises13, import_os, import_path20, import_dayjs4, import_customParseFormat3, import_utc3, POOL_CAP, SyntheticInputEngineClass, SyntheticInputEngine, SyntheticInputEngine_default;
|
|
47315
47600
|
var init_SyntheticInputEngine = __esm({
|
|
47316
47601
|
"../../packages/engines/src/mock/SyntheticInputEngine.ts"() {
|
|
47317
|
-
|
|
47602
|
+
import_promises13 = __toESM(require("fs/promises"), 1);
|
|
47318
47603
|
import_os = __toESM(require("os"), 1);
|
|
47319
47604
|
import_path20 = __toESM(require("path"), 1);
|
|
47320
47605
|
import_dayjs4 = __toESM(require("dayjs"), 1);
|
|
@@ -47340,7 +47625,7 @@ var init_SyntheticInputEngine = __esm({
|
|
|
47340
47625
|
for (const cs2 of [...BUILTIN_CODE_SETS, ...codeSets]) codeSetMap.set(cs2.name.toLowerCase(), cs2);
|
|
47341
47626
|
const producerMap = new Map(producers.map((p5) => [p5.name, p5]));
|
|
47342
47627
|
const refTargets = this._collectReferences(producers, producerMap);
|
|
47343
|
-
const dir = await
|
|
47628
|
+
const dir = await import_promises13.default.mkdtemp(import_path20.default.join(import_os.default.tmpdir(), "remora-synth-"));
|
|
47344
47629
|
const poolSize = Math.min(Math.max(opts.records, 1), POOL_CAP);
|
|
47345
47630
|
const registry = new MockKeyRegistry();
|
|
47346
47631
|
const specs = producers.flatMap((p5) => this._specs(p5, codeSetMap, producerMap, refTargets));
|
|
@@ -47357,7 +47642,7 @@ var init_SyntheticInputEngine = __esm({
|
|
|
47357
47642
|
return { inputs, dir };
|
|
47358
47643
|
};
|
|
47359
47644
|
this.cleanup = async (bundle) => {
|
|
47360
|
-
if (bundle?.dir) await
|
|
47645
|
+
if (bundle?.dir) await import_promises13.default.rm(bundle.dir, { recursive: true, force: true });
|
|
47361
47646
|
};
|
|
47362
47647
|
this._assertOpts = (opts) => {
|
|
47363
47648
|
Affirm_default(opts, "Invalid options");
|
|
@@ -47407,8 +47692,8 @@ var init_SyntheticInputEngine = __esm({
|
|
|
47407
47692
|
lines.push(JSON.stringify(row) + "\n");
|
|
47408
47693
|
}
|
|
47409
47694
|
const filePath = import_path20.default.join(dir, `${producer.name}.jsonl`);
|
|
47410
|
-
await
|
|
47411
|
-
const { size } = await
|
|
47695
|
+
await import_promises13.default.writeFile(filePath, lines.join(""), "utf-8");
|
|
47696
|
+
const { size } = await import_promises13.default.stat(filePath);
|
|
47412
47697
|
return { path: filePath, fileSize: size };
|
|
47413
47698
|
};
|
|
47414
47699
|
// Distinct structured code sets referenced by these specs -> their records (deduped by name).
|
|
@@ -47519,7 +47804,7 @@ var init_SyntheticInputEngine = __esm({
|
|
|
47519
47804
|
// True when a date bound names a day rather than an instant — neither the value nor the dimension's
|
|
47520
47805
|
// format carries a time. Both have to agree: the value alone can't tell `"20240131"` from a compact
|
|
47521
47806
|
// `YYYYMMDDHHmmss` instant, and the format alone can't tell that an ISO value overrode it.
|
|
47522
|
-
this._isDayGranular = (value,
|
|
47807
|
+
this._isDayGranular = (value, format4) => !/[T ]\d{1,2}:/.test(value) && !/[HhmsSAa]/.test(format4 ?? "");
|
|
47523
47808
|
/**
|
|
47524
47809
|
* One bound in the units the generator works in: the number itself, a count of characters for a
|
|
47525
47810
|
* `string`, or epoch milliseconds for a `datetime`. A date bound is read with the dimension's own
|
|
@@ -48449,23 +48734,23 @@ var init_Diagnostics = __esm({
|
|
|
48449
48734
|
return [.../* @__PURE__ */ new Set([...inputs, ...chained])];
|
|
48450
48735
|
};
|
|
48451
48736
|
this._walk = (start, next, onCycle) => {
|
|
48452
|
-
const
|
|
48737
|
+
const path30 = [];
|
|
48453
48738
|
const onPath = /* @__PURE__ */ new Set();
|
|
48454
48739
|
const stack = [{ name: start, done: false }];
|
|
48455
48740
|
const visited = /* @__PURE__ */ new Set();
|
|
48456
48741
|
while (stack.length > 0) {
|
|
48457
48742
|
const frame = stack.pop();
|
|
48458
48743
|
if (frame.done) {
|
|
48459
|
-
onPath.delete(
|
|
48744
|
+
onPath.delete(path30.pop());
|
|
48460
48745
|
continue;
|
|
48461
48746
|
}
|
|
48462
48747
|
if (onPath.has(frame.name)) {
|
|
48463
|
-
|
|
48748
|
+
path30.slice(path30.indexOf(frame.name)).forEach((name) => onCycle.add(name));
|
|
48464
48749
|
continue;
|
|
48465
48750
|
}
|
|
48466
48751
|
if (visited.has(frame.name)) continue;
|
|
48467
48752
|
visited.add(frame.name);
|
|
48468
|
-
|
|
48753
|
+
path30.push(frame.name);
|
|
48469
48754
|
onPath.add(frame.name);
|
|
48470
48755
|
stack.push({ name: frame.name, done: true });
|
|
48471
48756
|
const successors = next.get(frame.name) ?? [];
|
|
@@ -48809,7 +49094,7 @@ var init_LineageRenderer = __esm({
|
|
|
48809
49094
|
});
|
|
48810
49095
|
|
|
48811
49096
|
// ../../packages/engines/src/lineage/ResourceView.ts
|
|
48812
|
-
var VISIBLE_AUTH_KEYS, MAX_TABLE_ROWS, MAX_RAW_BYTES, MAX_CELL_CHARS, ResourceViewClass, ResourceView, ResourceView_default;
|
|
49097
|
+
var VISIBLE_AUTH_KEYS, MAX_TABLE_ROWS, MAX_RAW_BYTES, MAX_CELL_CHARS, MAX_NOTE_CHARS, ResourceViewClass, ResourceView, ResourceView_default;
|
|
48813
49098
|
var init_ResourceView = __esm({
|
|
48814
49099
|
"../../packages/engines/src/lineage/ResourceView.ts"() {
|
|
48815
49100
|
init_src();
|
|
@@ -48848,6 +49133,7 @@ var init_ResourceView = __esm({
|
|
|
48848
49133
|
MAX_TABLE_ROWS = 500;
|
|
48849
49134
|
MAX_RAW_BYTES = 256 * 1024;
|
|
48850
49135
|
MAX_CELL_CHARS = 200;
|
|
49136
|
+
MAX_NOTE_CHARS = 240;
|
|
48851
49137
|
ResourceViewClass = class {
|
|
48852
49138
|
constructor() {
|
|
48853
49139
|
/**
|
|
@@ -49019,7 +49305,7 @@ var init_ResourceView = __esm({
|
|
|
49019
49305
|
{ key: "classification", label: "Classification" }
|
|
49020
49306
|
];
|
|
49021
49307
|
const rows = dimensions.map((dimension) => [
|
|
49022
|
-
{ value: dimension.name },
|
|
49308
|
+
{ value: dimension.name, note: this._note(dimension.description) },
|
|
49023
49309
|
{ value: this._text(dimension.type) },
|
|
49024
49310
|
positioned ? { value: this._position(dimension.position) } : { value: this._text(dimension.alias), muted: !dimension.alias },
|
|
49025
49311
|
{ value: !!dimension.pk, muted: !dimension.pk },
|
|
@@ -49031,7 +49317,7 @@ var init_ResourceView = __esm({
|
|
|
49031
49317
|
};
|
|
49032
49318
|
this._measuresTable = (producer) => {
|
|
49033
49319
|
const rows = (producer.measures ?? []).map((measure) => [
|
|
49034
|
-
{ value: measure.name },
|
|
49320
|
+
{ value: measure.name, note: this._note(measure.description) },
|
|
49035
49321
|
{ value: measure.sql }
|
|
49036
49322
|
]);
|
|
49037
49323
|
return this._table("Measures", [
|
|
@@ -49097,7 +49383,7 @@ var init_ResourceView = __esm({
|
|
|
49097
49383
|
};
|
|
49098
49384
|
this._consumerFieldsTable = (env2, consumer) => {
|
|
49099
49385
|
const rows = (consumer.fields ?? []).map((field) => [
|
|
49100
|
-
{ value: field.key },
|
|
49386
|
+
{ value: field.key, note: this._note(field.description) },
|
|
49101
49387
|
{ value: this._text(field.alias), muted: !field.alias },
|
|
49102
49388
|
{ value: this._text(field.from), link: field.from ? this._inputLink(env2, field.from) : void 0, muted: !field.from },
|
|
49103
49389
|
{ value: this._text(field.transform ? Object.keys(field.transform) : null), muted: !field.transform },
|
|
@@ -49143,8 +49429,8 @@ var init_ResourceView = __esm({
|
|
|
49143
49429
|
if (!source || origin.path.length < 2) return { value: "declared here", muted: true };
|
|
49144
49430
|
return { value: `${source.resource}.${source.field}`, link: source.node };
|
|
49145
49431
|
};
|
|
49146
|
-
this._pathText = (
|
|
49147
|
-
return
|
|
49432
|
+
this._pathText = (path30) => {
|
|
49433
|
+
return path30.map((step) => `${step.resource}.${step.field}`).join(" \u2192 ");
|
|
49148
49434
|
};
|
|
49149
49435
|
this._consumerOutputsTable = (env2, consumer) => {
|
|
49150
49436
|
const declared = new Set((env2.sources ?? []).map((x2) => x2.name));
|
|
@@ -49186,7 +49472,7 @@ var init_ResourceView = __esm({
|
|
|
49186
49472
|
const required = new Set(schema.required ?? []);
|
|
49187
49473
|
const properties = Object.entries(schema.properties ?? {});
|
|
49188
49474
|
const rows = properties.map(([property, definition]) => [
|
|
49189
|
-
{ value: property },
|
|
49475
|
+
{ value: property, note: this._note(this._schemaKeyword(definition, "description")) },
|
|
49190
49476
|
{ value: this._schemaType(definition) },
|
|
49191
49477
|
{ value: required.has(property), muted: !required.has(property) },
|
|
49192
49478
|
{ value: this._text(this._schemaKeyword(definition, "format")), muted: !this._schemaKeyword(definition, "format") }
|
|
@@ -49292,6 +49578,15 @@ var init_ResourceView = __esm({
|
|
|
49292
49578
|
const minutes = Math.floor(elapsedMS / 6e4);
|
|
49293
49579
|
return `${minutes}m ${Math.round(elapsedMS % 6e4 / 1e3)}s`;
|
|
49294
49580
|
};
|
|
49581
|
+
/**
|
|
49582
|
+
* A description as the secondary line of a cell. Blank prose is the same as none, so a field
|
|
49583
|
+
* documented with a stray space does not cost the row a line.
|
|
49584
|
+
*/
|
|
49585
|
+
this._note = (description) => {
|
|
49586
|
+
const text = (description ?? "").trim();
|
|
49587
|
+
if (!text) return void 0;
|
|
49588
|
+
return text.length > MAX_NOTE_CHARS ? `${text.slice(0, MAX_NOTE_CHARS)}\u2026` : text;
|
|
49589
|
+
};
|
|
49295
49590
|
/** Anything that is not already a scalar becomes readable text; absent becomes `null`. */
|
|
49296
49591
|
this._text = (value) => {
|
|
49297
49592
|
if (value === void 0 || value === null) return null;
|
|
@@ -49608,7 +49903,7 @@ var init_OpsHeartbeat = __esm({
|
|
|
49608
49903
|
"../../packages/engines/src/ops/OpsHeartbeat.ts"() {
|
|
49609
49904
|
init_src2();
|
|
49610
49905
|
init_OpsService();
|
|
49611
|
-
DEFAULT_INTERVAL_MS =
|
|
49906
|
+
DEFAULT_INTERVAL_MS = 3e5;
|
|
49612
49907
|
MIN_INTERVAL_MS = 5e3;
|
|
49613
49908
|
MAX_INTERVAL_MS = 36e5;
|
|
49614
49909
|
OpsHeartbeatClass = class {
|
|
@@ -49648,11 +49943,24 @@ var init_OpsHeartbeat = __esm({
|
|
|
49648
49943
|
Logger_default.error(error);
|
|
49649
49944
|
}
|
|
49650
49945
|
};
|
|
49946
|
+
/**
|
|
49947
|
+
* Resolve the configured interval, in milliseconds. A value that is unusable or out of range is
|
|
49948
|
+
* reported rather than silently replaced: an operator who set an interval and got a different
|
|
49949
|
+
* one has no other way to find out.
|
|
49950
|
+
*/
|
|
49651
49951
|
this.resolveInterval = () => {
|
|
49652
|
-
const
|
|
49653
|
-
if (!
|
|
49952
|
+
const raw = process.env.REMORA_STATE_INTERVAL_MS;
|
|
49953
|
+
if (!raw)
|
|
49954
|
+
return DEFAULT_INTERVAL_MS;
|
|
49955
|
+
const configured = Number(raw);
|
|
49956
|
+
if (!Number.isFinite(configured) || configured <= 0) {
|
|
49957
|
+
Logger_default.warn(`REMORA_STATE_INTERVAL_MS is not a positive number of milliseconds ("${raw}"); using ${DEFAULT_INTERVAL_MS}ms.`);
|
|
49654
49958
|
return DEFAULT_INTERVAL_MS;
|
|
49655
|
-
|
|
49959
|
+
}
|
|
49960
|
+
const clamped = Math.min(Math.max(configured, MIN_INTERVAL_MS), MAX_INTERVAL_MS);
|
|
49961
|
+
if (clamped !== configured)
|
|
49962
|
+
Logger_default.warn(`REMORA_STATE_INTERVAL_MS of ${configured}ms is outside ${MIN_INTERVAL_MS}ms..${MAX_INTERVAL_MS}ms; using ${clamped}ms.`);
|
|
49963
|
+
return clamped;
|
|
49656
49964
|
};
|
|
49657
49965
|
}
|
|
49658
49966
|
};
|
|
@@ -49886,17 +50194,30 @@ var init_LocalUsageDB = __esm({
|
|
|
49886
50194
|
}
|
|
49887
50195
|
};
|
|
49888
50196
|
/**
|
|
49889
|
-
* The recorded runs, newest first
|
|
50197
|
+
* The recorded runs, newest first — the newest `limit` of the rows `keep` accepts.
|
|
50198
|
+
*
|
|
50199
|
+
* Decoded from the end and stopped as soon as that many are held: a reader wants the recent
|
|
49890
50200
|
* executions, and decoding ten thousand rows to show ten of them is work nobody asked for.
|
|
49891
50201
|
*
|
|
50202
|
+
* `keep` is applied BEFORE the limit, which is the only way "the newest N runs of this kind" can
|
|
50203
|
+
* mean it: filtering afterwards lets the rows a caller does not want displace the ones it does,
|
|
50204
|
+
* so one sample recorded here would silently push a real run out of a history someone is reading.
|
|
50205
|
+
*
|
|
49892
50206
|
* Never throws. A missing file reads as no history, and so does a half-written one — this store is
|
|
49893
50207
|
* appended to while it is being read, so failing on that would make it unreadable during a run.
|
|
49894
50208
|
*/
|
|
49895
|
-
this.list = (limit = MAX_LISTED_ROWS) => {
|
|
50209
|
+
this.list = (limit = MAX_LISTED_ROWS, keep) => {
|
|
49896
50210
|
try {
|
|
49897
50211
|
if (!(0, import_fs16.existsSync)(CLI_DB_PATH))
|
|
49898
50212
|
return [];
|
|
49899
|
-
|
|
50213
|
+
const rows = this._getRows();
|
|
50214
|
+
const wanted = Math.max(1, limit);
|
|
50215
|
+
const found = [];
|
|
50216
|
+
for (let index = rows.length - 1; index >= 0 && found.length < wanted; index--) {
|
|
50217
|
+
const usage = this._fromLine(rows[index]);
|
|
50218
|
+
if (!keep || keep(usage)) found.push(usage);
|
|
50219
|
+
}
|
|
50220
|
+
return found;
|
|
49900
50221
|
} catch (error) {
|
|
49901
50222
|
this._handleError("list", error);
|
|
49902
50223
|
return [];
|
|
@@ -50077,9 +50398,10 @@ var init_UsageManager = __esm({
|
|
|
50077
50398
|
});
|
|
50078
50399
|
|
|
50079
50400
|
// ../../packages/engines/src/ops/RunOverlay.ts
|
|
50080
|
-
var MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay, RunOverlay_default;
|
|
50401
|
+
var import_crypto5, MAX_RUNS, MAX_OPERATIONS, RunOverlayClass, RunOverlay, RunOverlay_default;
|
|
50081
50402
|
var init_RunOverlay = __esm({
|
|
50082
50403
|
"../../packages/engines/src/ops/RunOverlay.ts"() {
|
|
50404
|
+
import_crypto5 = __toESM(require("crypto"), 1);
|
|
50083
50405
|
init_src();
|
|
50084
50406
|
init_LineageEngine();
|
|
50085
50407
|
init_LocalUsageDB();
|
|
@@ -50088,6 +50410,8 @@ var init_RunOverlay = __esm({
|
|
|
50088
50410
|
MAX_OPERATIONS = 40;
|
|
50089
50411
|
RunOverlayClass = class {
|
|
50090
50412
|
constructor() {
|
|
50413
|
+
/** The last store token read, and the overlay token it resolved to. */
|
|
50414
|
+
this._signed = { store: null, overlay: "" };
|
|
50091
50415
|
/**
|
|
50092
50416
|
* The recent executions and their aggregates. ONE read of the store answers both, so a page
|
|
50093
50417
|
* showing a graph, a run list and a timing breakdown costs one file read rather than three.
|
|
@@ -50098,20 +50422,40 @@ var init_RunOverlay = __esm({
|
|
|
50098
50422
|
this.read = (limit = MAX_RUNS) => {
|
|
50099
50423
|
Affirm_default(limit > 0, "Run overlay limit must be greater than zero");
|
|
50100
50424
|
if (!UsageManager_default.recordsLocally()) return { runs: [], metrics: {} };
|
|
50101
|
-
const stats = LocalUsageDB_default.list(limit
|
|
50425
|
+
const stats = LocalUsageDB_default.list(limit, (stat) => !!stat?.consumer && stat.invokedBy !== "SAMPLE");
|
|
50102
50426
|
return { runs: stats.map((stat) => this._run(stat)), metrics: this._metrics(stats) };
|
|
50103
50427
|
};
|
|
50104
50428
|
/**
|
|
50105
|
-
* A token that changes when the
|
|
50429
|
+
* A token that changes when the OVERLAY does. Empty when there is nothing to read.
|
|
50106
50430
|
*
|
|
50107
50431
|
* A run finishing changes no configuration file, so the config hash cannot notice it — this is
|
|
50108
|
-
* the second half of what the preview polls
|
|
50109
|
-
*
|
|
50432
|
+
* the second half of what the preview polls.
|
|
50433
|
+
*
|
|
50434
|
+
* It signs the painted runs rather than the store, because the two do not move together: a
|
|
50435
|
+
* sample is recorded like any other execution and then filtered out of `read`, so the store's
|
|
50436
|
+
* own token changes while the picture does not. Signing the store there announced a project
|
|
50437
|
+
* change for every sample — and a page told its resource changed drops the sample it is showing,
|
|
50438
|
+
* which is exactly the rows someone had just asked for.
|
|
50439
|
+
*
|
|
50440
|
+
* Still cheap to poll: the store's mtime/size gates the read, so the rows are decoded only after
|
|
50441
|
+
* something has actually been written, never once a second.
|
|
50110
50442
|
*/
|
|
50111
50443
|
this.signature = () => {
|
|
50112
50444
|
if (!UsageManager_default.recordsLocally()) return "";
|
|
50113
|
-
|
|
50445
|
+
const store = LocalUsageDB_default.signature();
|
|
50446
|
+
if (store === this._signed.store) return this._signed.overlay;
|
|
50447
|
+
const overlay = this._token(this.read());
|
|
50448
|
+
this._signed = { store, overlay };
|
|
50449
|
+
return overlay;
|
|
50114
50450
|
};
|
|
50451
|
+
/**
|
|
50452
|
+
* A token over the whole overlay, runs and metrics alike.
|
|
50453
|
+
*
|
|
50454
|
+
* Everything it carries, not the newest row and a count: `endUsage` updates a row in place and a
|
|
50455
|
+
* finish action writes runs of its own, so a narrower token would miss a transition the graph
|
|
50456
|
+
* exists to show.
|
|
50457
|
+
*/
|
|
50458
|
+
this._token = (overlay) => import_crypto5.default.createHash("sha256").update(JSON.stringify(overlay)).digest("hex");
|
|
50115
50459
|
this._run = (stat) => {
|
|
50116
50460
|
const startedAt = this._iso(stat.startedAt);
|
|
50117
50461
|
return {
|
|
@@ -50358,64 +50702,6 @@ var init_PushdownEngine = __esm({
|
|
|
50358
50702
|
}
|
|
50359
50703
|
});
|
|
50360
50704
|
|
|
50361
|
-
// ../../packages/engines/src/producer/SampleEngine.ts
|
|
50362
|
-
var MAX_CHAIN_DEPTH2, SampleEngineClass, SampleEngine, SampleEngine_default;
|
|
50363
|
-
var init_SampleEngine = __esm({
|
|
50364
|
-
"../../packages/engines/src/producer/SampleEngine.ts"() {
|
|
50365
|
-
init_src();
|
|
50366
|
-
init_src4();
|
|
50367
|
-
init_ProducerEngine();
|
|
50368
|
-
MAX_CHAIN_DEPTH2 = 20;
|
|
50369
|
-
SampleEngineClass = class {
|
|
50370
|
-
constructor() {
|
|
50371
|
-
/**
|
|
50372
|
-
* @param resourceName a producer or consumer name, as declared.
|
|
50373
|
-
* @throws when the name is neither, or when the source cannot be read.
|
|
50374
|
-
*/
|
|
50375
|
-
this.read = async (resourceName, size = 10) => {
|
|
50376
|
-
Affirm_default(resourceName, "Missing resource name to sample");
|
|
50377
|
-
Affirm_default(size > 0, "Sample size must be greater than 0");
|
|
50378
|
-
const producer = Environment_default.getProducer(resourceName);
|
|
50379
|
-
if (producer)
|
|
50380
|
-
return { kind: "producer", name: resourceName, records: await ProducerEngine_default.readSampleData(producer, size, false) };
|
|
50381
|
-
const consumer = Environment_default.getConsumer(resourceName);
|
|
50382
|
-
Affirm_default(consumer, `Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`);
|
|
50383
|
-
return { kind: "consumer", name: resourceName, records: await this._readConsumer(consumer, size, []) };
|
|
50384
|
-
};
|
|
50385
|
-
/**
|
|
50386
|
-
* A consumer has no data of its own until it runs, so what it shows is its FIRST input, projected
|
|
50387
|
-
* through its field list — near enough to answer "what will come out of this" without a run.
|
|
50388
|
-
*/
|
|
50389
|
-
this._readConsumer = async (consumer, size, visited) => {
|
|
50390
|
-
Affirm_default(!visited.includes(consumer.name), `Consumer "${consumer.name}" reads itself through ${visited.join(" \u2192 ")}`);
|
|
50391
|
-
Affirm_default(visited.length < MAX_CHAIN_DEPTH2, `Consumer chain from "${visited[0]}" is more than ${MAX_CHAIN_DEPTH2} deep`);
|
|
50392
|
-
const input = consumer.producers[0];
|
|
50393
|
-
Affirm_default(input, `Consumer "${consumer.name}" has no producers configured`);
|
|
50394
|
-
const producer = Environment_default.getProducer(input.name);
|
|
50395
|
-
if (!producer) {
|
|
50396
|
-
const upstream = Environment_default.getConsumer(input.name);
|
|
50397
|
-
Affirm_default(upstream, `Producer or consumer "${input.name}" not found for consumer "${consumer.name}"`);
|
|
50398
|
-
return this._readConsumer(upstream, size, [...visited, consumer.name]);
|
|
50399
|
-
}
|
|
50400
|
-
const records = await ProducerEngine_default.readSampleData(producer, size, false);
|
|
50401
|
-
if (consumer.fields.some((field) => field.key === "*")) return records;
|
|
50402
|
-
return records.map((record) => this._project(consumer, record));
|
|
50403
|
-
};
|
|
50404
|
-
this._project = (consumer, record) => {
|
|
50405
|
-
const projected = {};
|
|
50406
|
-
for (const field of consumer.fields) {
|
|
50407
|
-
if (field.key === "*") continue;
|
|
50408
|
-
projected[field.alias || field.key] = record[field.key];
|
|
50409
|
-
}
|
|
50410
|
-
return projected;
|
|
50411
|
-
};
|
|
50412
|
-
}
|
|
50413
|
-
};
|
|
50414
|
-
SampleEngine = new SampleEngineClass();
|
|
50415
|
-
SampleEngine_default = SampleEngine;
|
|
50416
|
-
}
|
|
50417
|
-
});
|
|
50418
|
-
|
|
50419
50705
|
// ../../packages/engines/src/transform/LookupRegistry.ts
|
|
50420
50706
|
var LookupRegistryClass, LookupRegistry, LookupRegistry_default;
|
|
50421
50707
|
var init_LookupRegistry = __esm({
|
|
@@ -50452,12 +50738,12 @@ var init_LookupRegistry = __esm({
|
|
|
50452
50738
|
});
|
|
50453
50739
|
|
|
50454
50740
|
// ../../packages/engines/src/transform/LookupKey.ts
|
|
50455
|
-
var
|
|
50741
|
+
var import_crypto6, DELIMITER, idFor, LookupKey, LookupKey_default;
|
|
50456
50742
|
var init_LookupKey = __esm({
|
|
50457
50743
|
"../../packages/engines/src/transform/LookupKey.ts"() {
|
|
50458
|
-
|
|
50744
|
+
import_crypto6 = __toESM(require("crypto"), 1);
|
|
50459
50745
|
DELIMITER = String.fromCharCode(0);
|
|
50460
|
-
idFor = (ref) =>
|
|
50746
|
+
idFor = (ref) => import_crypto6.default.createHash("sha1").update([ref.consumer, ref.key, ref.value].join(DELIMITER)).digest("hex").slice(0, 16);
|
|
50461
50747
|
LookupKey = { idFor };
|
|
50462
50748
|
LookupKey_default = LookupKey;
|
|
50463
50749
|
}
|
|
@@ -50484,8 +50770,8 @@ var init_TransformationEngine = __esm({
|
|
|
50484
50770
|
return result;
|
|
50485
50771
|
}
|
|
50486
50772
|
if ("cast" in transformations) {
|
|
50487
|
-
const { cast, format:
|
|
50488
|
-
const casted = TypeCaster_default.cast(value, cast,
|
|
50773
|
+
const { cast, format: format4 } = transformations;
|
|
50774
|
+
const casted = TypeCaster_default.cast(value, cast, format4);
|
|
50489
50775
|
if (cast === "number" && isNaN(casted))
|
|
50490
50776
|
throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
|
|
50491
50777
|
if (cast === "datetime" && casted instanceof Date && isNaN(casted.getTime()))
|
|
@@ -51094,7 +51380,6 @@ var init_src10 = __esm({
|
|
|
51094
51380
|
init_ProducerEngine();
|
|
51095
51381
|
init_ProducerManager();
|
|
51096
51382
|
init_PushdownEngine();
|
|
51097
|
-
init_SampleEngine();
|
|
51098
51383
|
init_TransformationEngine();
|
|
51099
51384
|
init_TypeCaster();
|
|
51100
51385
|
init_LookupRegistry();
|
|
@@ -51264,14 +51549,14 @@ var init_OutputExecutor = __esm({
|
|
|
51264
51549
|
});
|
|
51265
51550
|
|
|
51266
51551
|
// ../../packages/executors/src/ConsumerExecutor.ts
|
|
51267
|
-
var import_path23, import_fs17, import_readline6,
|
|
51552
|
+
var import_path23, import_fs17, import_readline6, import_promises14, import_crypto7, LIMIT_SCAN_BYTES, NEWLINE_BYTE, ConsumerExecutorClass, ConsumerExecutor, ConsumerExecutor_default;
|
|
51268
51553
|
var init_ConsumerExecutor = __esm({
|
|
51269
51554
|
"../../packages/executors/src/ConsumerExecutor.ts"() {
|
|
51270
51555
|
import_path23 = __toESM(require("path"));
|
|
51271
51556
|
import_fs17 = __toESM(require("fs"));
|
|
51272
51557
|
import_readline6 = __toESM(require("readline"));
|
|
51273
|
-
|
|
51274
|
-
|
|
51558
|
+
import_promises14 = __toESM(require("fs/promises"));
|
|
51559
|
+
import_crypto7 = __toESM(require("crypto"));
|
|
51275
51560
|
init_src10();
|
|
51276
51561
|
init_src10();
|
|
51277
51562
|
init_src10();
|
|
@@ -51457,8 +51742,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
51457
51742
|
reader.destroy();
|
|
51458
51743
|
});
|
|
51459
51744
|
}
|
|
51460
|
-
await
|
|
51461
|
-
await
|
|
51745
|
+
await import_promises14.default.unlink(datasetPath);
|
|
51746
|
+
await import_promises14.default.rename(tempWorkPath, datasetPath);
|
|
51462
51747
|
return processedLineCount;
|
|
51463
51748
|
};
|
|
51464
51749
|
/**
|
|
@@ -51476,7 +51761,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
51476
51761
|
this.processLimit = async (datasetPath, limit) => {
|
|
51477
51762
|
Affirm_default(datasetPath, "Invalid dataset path");
|
|
51478
51763
|
Affirm_default(limit > 0, `Invalid record limit ${limit}`);
|
|
51479
|
-
const handle = await
|
|
51764
|
+
const handle = await import_promises14.default.open(datasetPath, "r+");
|
|
51480
51765
|
try {
|
|
51481
51766
|
const buffer = Buffer.allocUnsafe(LIMIT_SCAN_BYTES);
|
|
51482
51767
|
let newlines = 0;
|
|
@@ -51510,7 +51795,7 @@ var init_ConsumerExecutor = __esm({
|
|
|
51510
51795
|
let newLineCount = 0;
|
|
51511
51796
|
const seen = /* @__PURE__ */ new Set();
|
|
51512
51797
|
for await (const line of lineReader) {
|
|
51513
|
-
const hash =
|
|
51798
|
+
const hash = import_crypto7.default.createHash("sha1").update(line).digest("base64");
|
|
51514
51799
|
if (!seen.has(hash)) {
|
|
51515
51800
|
seen.add(hash);
|
|
51516
51801
|
if (!writer.write(line + "\n"))
|
|
@@ -51530,8 +51815,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
51530
51815
|
reader.destroy();
|
|
51531
51816
|
});
|
|
51532
51817
|
}
|
|
51533
|
-
await
|
|
51534
|
-
await
|
|
51818
|
+
await import_promises14.default.unlink(datasetPath);
|
|
51819
|
+
await import_promises14.default.rename(tempWorkPath, datasetPath);
|
|
51535
51820
|
return newLineCount;
|
|
51536
51821
|
};
|
|
51537
51822
|
this.processDistinctOn = async (consumer, datasetPath) => {
|
|
@@ -51575,8 +51860,8 @@ var init_ConsumerExecutor = __esm({
|
|
|
51575
51860
|
reader.destroy();
|
|
51576
51861
|
});
|
|
51577
51862
|
}
|
|
51578
|
-
await
|
|
51579
|
-
await
|
|
51863
|
+
await import_promises14.default.unlink(datasetPath);
|
|
51864
|
+
await import_promises14.default.rename(tempWorkPath, datasetPath);
|
|
51580
51865
|
return winners.size;
|
|
51581
51866
|
};
|
|
51582
51867
|
this.processPivot = async (consumer, datasetPath) => {
|
|
@@ -51682,12 +51967,12 @@ var init_ConsumerExecutor = __esm({
|
|
|
51682
51967
|
reader.destroy();
|
|
51683
51968
|
});
|
|
51684
51969
|
}
|
|
51685
|
-
await
|
|
51686
|
-
await
|
|
51970
|
+
await import_promises14.default.unlink(datasetPath);
|
|
51971
|
+
await import_promises14.default.rename(tempWorkPath, datasetPath);
|
|
51687
51972
|
return outputCount;
|
|
51688
51973
|
};
|
|
51689
|
-
this._parseLine = (line,
|
|
51690
|
-
return
|
|
51974
|
+
this._parseLine = (line, format4, fields) => {
|
|
51975
|
+
return format4 === "CSV" ? LineParser_default._internalParseCSV(line, fields) : LineParser_default._internalParseJSON(line);
|
|
51691
51976
|
};
|
|
51692
51977
|
/**
|
|
51693
51978
|
* Determines if the new record should replace the existing record based on the resolution strategy
|
|
@@ -51786,8 +52071,6 @@ var init_ProducerExecutor = __esm({
|
|
|
51786
52071
|
init_src8();
|
|
51787
52072
|
init_src4();
|
|
51788
52073
|
init_src10();
|
|
51789
|
-
init_src();
|
|
51790
|
-
init_src10();
|
|
51791
52074
|
init_src10();
|
|
51792
52075
|
import_path24 = __toESM(require("path"));
|
|
51793
52076
|
init_src2();
|
|
@@ -51818,20 +52101,14 @@ var init_ProducerExecutor = __esm({
|
|
|
51818
52101
|
* the source file name or a mask, and what that file name is. None of it depends on the line, so
|
|
51819
52102
|
* it is compiled once per chunk instead of rebuilt per line — including the mask's own
|
|
51820
52103
|
* validation, which used to run once per dimension per line.
|
|
52104
|
+
*
|
|
52105
|
+
* The fills themselves come from `ProducerManager`, which is also where the sample path gets
|
|
52106
|
+
* them: one producer stage, reached from two places.
|
|
51821
52107
|
*/
|
|
51822
52108
|
this.compileLine = (dimensions, chunk) => {
|
|
51823
52109
|
Affirm_default(dimensions, "Invalid producer dimensions");
|
|
51824
52110
|
Affirm_default(chunk, "Invalid chunk");
|
|
51825
|
-
|
|
51826
|
-
for (const dimension of dimensions) {
|
|
51827
|
-
const { prodDimension } = dimension;
|
|
51828
|
-
const mask = ProducerManager_default.getMask(prodDimension);
|
|
51829
|
-
const sourceFilename = prodDimension.sourceFilename === true;
|
|
51830
|
-
if (!sourceFilename && !Algo_default.hasVal(mask))
|
|
51831
|
-
continue;
|
|
51832
|
-
fills.push({ key: dimension.name, mask, sourceFilename, type: prodDimension.type });
|
|
51833
|
-
}
|
|
51834
|
-
return { fills, sourceFilename: import_path24.default.basename(chunk.fileUri) };
|
|
52111
|
+
return { fills: ProducerManager_default.compileFills(dimensions), sourceFilename: import_path24.default.basename(chunk.fileUri) };
|
|
51835
52112
|
};
|
|
51836
52113
|
/**
|
|
51837
52114
|
* The record(s) one source line produces. Almost always exactly one — a producer with a group
|
|
@@ -51852,14 +52129,8 @@ var init_ProducerExecutor = __esm({
|
|
|
51852
52129
|
if (plan.fills.length === 0)
|
|
51853
52130
|
return records;
|
|
51854
52131
|
counter = performance.now();
|
|
51855
|
-
for (const record of records)
|
|
51856
|
-
|
|
51857
|
-
if (fill2.sourceFilename)
|
|
51858
|
-
record[fill2.key] = plan.sourceFilename;
|
|
51859
|
-
if (Algo_default.hasVal(fill2.mask))
|
|
51860
|
-
record[fill2.key] = CryptoEngine_default.hashValue(fill2.mask, record[fill2.key]?.toString(), fill2.type);
|
|
51861
|
-
}
|
|
51862
|
-
}
|
|
52132
|
+
for (const record of records)
|
|
52133
|
+
ProducerManager_default.applyFills(record, plan.fills, plan.sourceFilename);
|
|
51863
52134
|
tracker.measure("process-line:dimensions-filename-mask", performance.now() - counter);
|
|
51864
52135
|
return records;
|
|
51865
52136
|
};
|
|
@@ -52170,11 +52441,11 @@ var init_OrchestratorHelper = __esm({
|
|
|
52170
52441
|
});
|
|
52171
52442
|
|
|
52172
52443
|
// ../../packages/executors/src/LookupResolver.ts
|
|
52173
|
-
var import_fs18,
|
|
52444
|
+
var import_fs18, import_promises15, import_path26, import_readline8, LookupResolverClass, LookupResolver, LookupResolver_default;
|
|
52174
52445
|
var init_LookupResolver = __esm({
|
|
52175
52446
|
"../../packages/executors/src/LookupResolver.ts"() {
|
|
52176
52447
|
import_fs18 = __toESM(require("fs"));
|
|
52177
|
-
|
|
52448
|
+
import_promises15 = __toESM(require("fs/promises"));
|
|
52178
52449
|
import_path26 = __toESM(require("path"));
|
|
52179
52450
|
import_readline8 = __toESM(require("readline"));
|
|
52180
52451
|
init_src();
|
|
@@ -52215,11 +52486,11 @@ var init_LookupResolver = __esm({
|
|
|
52215
52486
|
this.cache.set(id, await this.buildDict(lookupConsumer, capturePath, ref));
|
|
52216
52487
|
}
|
|
52217
52488
|
ExecutorScope_default.ensurePath(outPath);
|
|
52218
|
-
await
|
|
52489
|
+
await import_promises15.default.writeFile(outPath, JSON.stringify(this.cache.get(id)));
|
|
52219
52490
|
manifest.push({ id, consumer: ref.consumer, key: ref.key, value: ref.value, path: import_path26.default.resolve(outPath) });
|
|
52220
52491
|
}
|
|
52221
52492
|
for (const capturePath of captureByConsumer.values())
|
|
52222
|
-
await
|
|
52493
|
+
await import_promises15.default.rm(capturePath, { force: true });
|
|
52223
52494
|
return manifest;
|
|
52224
52495
|
};
|
|
52225
52496
|
// Runs the source consumer through the real pipeline, capturing its merged output before cleanup.
|
|
@@ -52237,7 +52508,7 @@ var init_LookupResolver = __esm({
|
|
|
52237
52508
|
};
|
|
52238
52509
|
// Projects the captured dataset to a { code: value } dict, bounded by MAX_LOOKUP_ROWS.
|
|
52239
52510
|
this.buildDict = async (lookupConsumer, capturePath, ref) => {
|
|
52240
|
-
const
|
|
52511
|
+
const format4 = OutputExecutor_default._getInternalRecordFormat(lookupConsumer);
|
|
52241
52512
|
const visibleFields = ConsumerManager_default.getExpandedFields(lookupConsumer).filter((f7) => !f7.cField.hidden);
|
|
52242
52513
|
const finalKeys = visibleFields.map((f7) => f7.finalKey);
|
|
52243
52514
|
Affirm_default(finalKeys.includes(ref.key), `code_lookup key column "${ref.key}" is not in the output of consumer "${ref.consumer}"`);
|
|
@@ -52250,7 +52521,7 @@ var init_LookupResolver = __esm({
|
|
|
52250
52521
|
if (!line.length) continue;
|
|
52251
52522
|
rowsRead++;
|
|
52252
52523
|
Affirm_default(rowsRead <= Constants_default.defaults.MAX_LOOKUP_ROWS, `code_lookup table from consumer "${ref.consumer}" exceeds MAX_LOOKUP_ROWS (${Constants_default.defaults.MAX_LOOKUP_ROWS})`);
|
|
52253
|
-
const record = this.parseLine(line,
|
|
52524
|
+
const record = this.parseLine(line, format4, visibleFields);
|
|
52254
52525
|
const code = String(record[ref.key]);
|
|
52255
52526
|
if (code in dict && !warnedDup) {
|
|
52256
52527
|
Logger_default.warn(`code_lookup: duplicate key(s) in consumer "${ref.consumer}" for "${ref.key}"; last value wins`);
|
|
@@ -52261,8 +52532,8 @@ var init_LookupResolver = __esm({
|
|
|
52261
52532
|
Logger_default.log(`[lookup] Built table "${ref.consumer}" (${ref.key}->${ref.value}): ${Object.keys(dict).length} entr(ies) from ${rowsRead} row(s)`);
|
|
52262
52533
|
return dict;
|
|
52263
52534
|
};
|
|
52264
|
-
this.parseLine = (line,
|
|
52265
|
-
if (
|
|
52535
|
+
this.parseLine = (line, format4, visibleFields) => {
|
|
52536
|
+
if (format4 === "CSV") return LineParser_default._internalParseCSV(line, visibleFields, ",");
|
|
52266
52537
|
return JSON.parse(line);
|
|
52267
52538
|
};
|
|
52268
52539
|
this.getCapturePath = (scope, consumerName) => {
|
|
@@ -52289,11 +52560,11 @@ var init_LookupResolver = __esm({
|
|
|
52289
52560
|
});
|
|
52290
52561
|
|
|
52291
52562
|
// ../../packages/executors/src/ConsumerInputResolver.ts
|
|
52292
|
-
var import_fs19,
|
|
52563
|
+
var import_fs19, import_promises16, import_path27, ConsumerInputResolverClass, ConsumerInputResolver, ConsumerInputResolver_default;
|
|
52293
52564
|
var init_ConsumerInputResolver = __esm({
|
|
52294
52565
|
"../../packages/executors/src/ConsumerInputResolver.ts"() {
|
|
52295
52566
|
import_fs19 = __toESM(require("fs"));
|
|
52296
|
-
|
|
52567
|
+
import_promises16 = __toESM(require("fs/promises"));
|
|
52297
52568
|
import_path27 = __toESM(require("path"));
|
|
52298
52569
|
init_src();
|
|
52299
52570
|
init_src3();
|
|
@@ -52324,7 +52595,7 @@ var init_ConsumerInputResolver = __esm({
|
|
|
52324
52595
|
const capturePath = ExecutorScope_default.getConsumerInputPath(scope, name);
|
|
52325
52596
|
ExecutorScope_default.ensurePath(capturePath);
|
|
52326
52597
|
await this.runUpstreamConsumer(upstream, capturePath, stack, request);
|
|
52327
|
-
const { size } = await
|
|
52598
|
+
const { size } = await import_promises16.default.stat(capturePath);
|
|
52328
52599
|
this.cache.set(name, { consumer: name, path: import_path27.default.resolve(capturePath), fileSize: size });
|
|
52329
52600
|
}
|
|
52330
52601
|
manifest.set(name, this.cache.get(name));
|
|
@@ -52387,11 +52658,11 @@ var ExecutorOrchestrator_exports = {};
|
|
|
52387
52658
|
__export(ExecutorOrchestrator_exports, {
|
|
52388
52659
|
default: () => ExecutorOrchestrator_default
|
|
52389
52660
|
});
|
|
52390
|
-
var import_fs20,
|
|
52661
|
+
var import_fs20, import_promises17, import_path28, import_readline9, import_workerpool, import_promises18, ExecutorOrchestratorClass, ExecutorOrchestrator, ExecutorOrchestrator_default;
|
|
52391
52662
|
var init_ExecutorOrchestrator = __esm({
|
|
52392
52663
|
"../../packages/executors/src/ExecutorOrchestrator.ts"() {
|
|
52393
52664
|
import_fs20 = __toESM(require("fs"));
|
|
52394
|
-
|
|
52665
|
+
import_promises17 = __toESM(require("fs/promises"));
|
|
52395
52666
|
import_path28 = __toESM(require("path"));
|
|
52396
52667
|
import_readline9 = __toESM(require("readline"));
|
|
52397
52668
|
import_workerpool = __toESM(require("workerpool"));
|
|
@@ -52409,7 +52680,7 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
52409
52680
|
init_src();
|
|
52410
52681
|
init_src10();
|
|
52411
52682
|
init_ExecutorWriter();
|
|
52412
|
-
|
|
52683
|
+
import_promises18 = require("stream/promises");
|
|
52413
52684
|
init_src2();
|
|
52414
52685
|
init_ExecutorProgress2();
|
|
52415
52686
|
init_src6();
|
|
@@ -52627,7 +52898,8 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
52627
52898
|
} catch (error) {
|
|
52628
52899
|
Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
|
|
52629
52900
|
Logger_default.error(error);
|
|
52630
|
-
|
|
52901
|
+
if (!request.skipExport)
|
|
52902
|
+
await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId, this.makeOnFinishRunner(consumer, request, usageId));
|
|
52631
52903
|
Logger_default.log(`[${usageId}] Running cleanup after failure`);
|
|
52632
52904
|
await this.performCleanupOperations(scope, tracker);
|
|
52633
52905
|
UsageManager_default.failUsage(usageId, Helper_default.asError(error).message);
|
|
@@ -52925,10 +53197,10 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
52925
53197
|
* format has no header row, so each row is reprojected. Without a captureFormat it's a verbatim copy.
|
|
52926
53198
|
*/
|
|
52927
53199
|
this.captureDataset = async (consumer, fields, sourcePath, targetPath, captureFormat) => {
|
|
52928
|
-
await
|
|
53200
|
+
await import_promises17.default.mkdir(import_path28.default.dirname(targetPath), { recursive: true });
|
|
52929
53201
|
const internalFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
|
|
52930
53202
|
if (captureFormat !== "JSONL" || internalFormat !== "CSV") {
|
|
52931
|
-
await
|
|
53203
|
+
await import_promises17.default.copyFile(sourcePath, targetPath);
|
|
52932
53204
|
return;
|
|
52933
53205
|
}
|
|
52934
53206
|
const visibleFields = fields.filter((field) => !field.cField.hidden);
|
|
@@ -53031,22 +53303,22 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
53031
53303
|
startRow: prod.settings.startRow,
|
|
53032
53304
|
startColumn: prod.settings.startColumn
|
|
53033
53305
|
});
|
|
53034
|
-
await (0,
|
|
53306
|
+
await (0, import_promises18.pipeline)(
|
|
53035
53307
|
xlsCsvStream,
|
|
53036
53308
|
import_fs20.default.createWriteStream(decodedPath)
|
|
53037
53309
|
);
|
|
53038
|
-
const fileStats = await
|
|
53310
|
+
const fileStats = await import_promises17.default.stat(decodedPath);
|
|
53039
53311
|
decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
|
|
53040
53312
|
decodedCount++;
|
|
53041
53313
|
reportFile();
|
|
53042
53314
|
continue;
|
|
53043
53315
|
}
|
|
53044
53316
|
if (inferredType === "XML") {
|
|
53045
|
-
const fileContent = await
|
|
53317
|
+
const fileContent = await import_promises17.default.readFile(file.fullUri, "utf-8");
|
|
53046
53318
|
const jsonData = XMLParser_default.xmlToJson(fileContent);
|
|
53047
53319
|
const records = normalizeXmlRows(jsonData);
|
|
53048
53320
|
if (records.length === 0) {
|
|
53049
|
-
await
|
|
53321
|
+
await import_promises17.default.writeFile(decodedPath, "", "utf-8");
|
|
53050
53322
|
} else {
|
|
53051
53323
|
const columns = [];
|
|
53052
53324
|
for (const record of records) {
|
|
@@ -53062,9 +53334,9 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
53062
53334
|
const row = columns.map((column) => csvSafeValue(record[column]));
|
|
53063
53335
|
lines.push(CSVParser_default.stringifyRow(row));
|
|
53064
53336
|
}
|
|
53065
|
-
await
|
|
53337
|
+
await import_promises17.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
|
|
53066
53338
|
}
|
|
53067
|
-
const fileStats = await
|
|
53339
|
+
const fileStats = await import_promises17.default.stat(decodedPath);
|
|
53068
53340
|
decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
|
|
53069
53341
|
decodedCount++;
|
|
53070
53342
|
reportFile();
|
|
@@ -53092,20 +53364,20 @@ var init_ExecutorOrchestrator = __esm({
|
|
|
53092
53364
|
Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
|
|
53093
53365
|
const perf = performance.now();
|
|
53094
53366
|
for (const workerResult of executorResults) {
|
|
53095
|
-
await (0,
|
|
53367
|
+
await (0, import_promises18.pipeline)(
|
|
53096
53368
|
import_fs20.default.createReadStream(workerResult.resultUri),
|
|
53097
53369
|
import_fs20.default.createWriteStream(mainPath, { flags: "a" })
|
|
53098
53370
|
);
|
|
53099
|
-
await
|
|
53371
|
+
await import_promises17.default.unlink(workerResult.resultUri);
|
|
53100
53372
|
}
|
|
53101
53373
|
tracker.measure("merge-workers", performance.now() - perf);
|
|
53102
53374
|
Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
|
|
53103
53375
|
} else if (executorResults.length === 1) {
|
|
53104
53376
|
Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
|
|
53105
|
-
await
|
|
53377
|
+
await import_promises17.default.rename(executorResults[0].resultUri, mainPath);
|
|
53106
53378
|
} else {
|
|
53107
53379
|
Logger_default.log(`[${scope.id}] No worker output \u2014 writing an empty dataset to ${mainPath}`);
|
|
53108
|
-
await
|
|
53380
|
+
await import_promises17.default.writeFile(mainPath, "");
|
|
53109
53381
|
}
|
|
53110
53382
|
return mainPath;
|
|
53111
53383
|
};
|
|
@@ -53826,6 +54098,164 @@ init_src4();
|
|
|
53826
54098
|
init_OutputExecutor();
|
|
53827
54099
|
init_ProducerExecutor();
|
|
53828
54100
|
|
|
54101
|
+
// ../../packages/executors/src/SampleRunner.ts
|
|
54102
|
+
var import_fs21 = __toESM(require("fs"));
|
|
54103
|
+
var import_promises19 = __toESM(require("fs/promises"));
|
|
54104
|
+
var import_path29 = __toESM(require("path"));
|
|
54105
|
+
var import_readline10 = __toESM(require("readline"));
|
|
54106
|
+
init_src();
|
|
54107
|
+
init_src3();
|
|
54108
|
+
init_src4();
|
|
54109
|
+
init_src8();
|
|
54110
|
+
init_src10();
|
|
54111
|
+
init_src5();
|
|
54112
|
+
init_src2();
|
|
54113
|
+
init_ExecutorOrchestrator();
|
|
54114
|
+
var MAX_WALK_DEPTH = 20;
|
|
54115
|
+
var SAMPLE_USER = { user: { _id: "sample", name: "sample", type: "licence" } };
|
|
54116
|
+
var SampleRunnerClass = class {
|
|
54117
|
+
constructor() {
|
|
54118
|
+
/**
|
|
54119
|
+
* @param resourceName a producer or consumer name, as declared.
|
|
54120
|
+
* @param rows the most rows to return, capped at `MAX_SAMPLE_ROWS`.
|
|
54121
|
+
* @throws when the name is neither, when a source cannot be read, or when the pipeline itself
|
|
54122
|
+
* fails — a consumer whose transformation throws is a sample that reports the reason.
|
|
54123
|
+
*/
|
|
54124
|
+
this.read = async (resourceName, rows = 10, details) => {
|
|
54125
|
+
Affirm_default(resourceName, "Missing resource name to sample");
|
|
54126
|
+
Affirm_default(rows > 0, "Sample size must be greater than 0");
|
|
54127
|
+
const size = Math.min(rows, Constants_default.defaults.MAX_SAMPLE_ROWS);
|
|
54128
|
+
const producer = Environment_default.getProducer(resourceName);
|
|
54129
|
+
if (producer) {
|
|
54130
|
+
const records = await ProducerEngine_default.readSampleData(producer, size, false);
|
|
54131
|
+
return {
|
|
54132
|
+
kind: "producer",
|
|
54133
|
+
name: resourceName,
|
|
54134
|
+
records,
|
|
54135
|
+
note: `${records.length} row(s) read from the source now, with this producer's dimensions, types and masks applied.`
|
|
54136
|
+
};
|
|
54137
|
+
}
|
|
54138
|
+
const consumer = Environment_default.getConsumer(resourceName);
|
|
54139
|
+
Affirm_default(consumer, `Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`);
|
|
54140
|
+
return this._runConsumer(consumer, size, details);
|
|
54141
|
+
};
|
|
54142
|
+
/**
|
|
54143
|
+
* Runs the consumer for real and reads back what it produced.
|
|
54144
|
+
*
|
|
54145
|
+
* The output is captured instead of exported, so nothing reaches the consumer's destination and
|
|
54146
|
+
* no onSuccess action fires: looking at data must never be indistinguishable from publishing it.
|
|
54147
|
+
*/
|
|
54148
|
+
this._runConsumer = async (consumer, size, details) => {
|
|
54149
|
+
this._assertStageable(consumer);
|
|
54150
|
+
const capturePath = import_path29.default.resolve(import_path29.default.join(ExecutorScope_default.getBasePath(), `sample_${consumer.name}_${Helper_default.uuid()}.jsonl`));
|
|
54151
|
+
ExecutorScope_default.ensurePath(capturePath);
|
|
54152
|
+
try {
|
|
54153
|
+
const result = await ExecutorOrchestrator_default.launch({
|
|
54154
|
+
consumer,
|
|
54155
|
+
// `SAMPLE`, so the run it records is distinguishable from one the project did. The
|
|
54156
|
+
// executor treats it like any other invocation; only `RunOverlay` reads the marker.
|
|
54157
|
+
details: { ...details ?? SAMPLE_USER, invokedBy: "SAMPLE" },
|
|
54158
|
+
// The rows asked for, bounded the way `remora run -l` bounds them — pushed to the
|
|
54159
|
+
// source and stopped early per worker wherever nothing downstream can drop a record.
|
|
54160
|
+
options: { limit: size },
|
|
54161
|
+
// Nothing reaches the destination and no finish action fires; the merged dataset
|
|
54162
|
+
// comes back through the same capture the lookup and consumer-input runs use.
|
|
54163
|
+
skipExport: true,
|
|
54164
|
+
captureMainDatasetTo: capturePath,
|
|
54165
|
+
captureFormat: "JSONL"
|
|
54166
|
+
});
|
|
54167
|
+
const captured = await this._readCapture(capturePath, size);
|
|
54168
|
+
const records = captured.map((record) => this._inFieldOrder(consumer, record));
|
|
54169
|
+
Logger_default.log(`[sample] Consumer "${consumer.name}": ${records.length} row(s) from ${result.inputCount} input line(s)`);
|
|
54170
|
+
return {
|
|
54171
|
+
kind: "consumer",
|
|
54172
|
+
name: consumer.name,
|
|
54173
|
+
records,
|
|
54174
|
+
note: `${records.length} row(s) from the real pipeline \u2014 filters, transformations, masking and validation all applied, exactly as in a run. Nothing was written to the destination.`
|
|
54175
|
+
};
|
|
54176
|
+
} finally {
|
|
54177
|
+
await import_promises19.default.rm(capturePath, { force: true });
|
|
54178
|
+
}
|
|
54179
|
+
};
|
|
54180
|
+
/**
|
|
54181
|
+
* Refuses, naming the source, when a run could not read one of this consumer's producers.
|
|
54182
|
+
*
|
|
54183
|
+
* `RedshiftDriver.ready` and `HttpApiSourceDriver.ready` are unimplemented, so the pipeline
|
|
54184
|
+
* cannot reach those sources at all. Checked up front because the alternative is a run that
|
|
54185
|
+
* spins up a worker pool and then surfaces "Not implemented yet" from three layers down, which
|
|
54186
|
+
* names neither the producer nor the source that caused it.
|
|
54187
|
+
*/
|
|
54188
|
+
this._assertStageable = (consumer) => {
|
|
54189
|
+
for (const producer of this._inputProducers(consumer)) {
|
|
54190
|
+
const source = Environment_default.getSource(producer.source);
|
|
54191
|
+
Affirm_default(source, `Producer "${producer.name}" of consumer "${consumer.name}" has no source "${producer.source}"`);
|
|
54192
|
+
if (!DriverFactory_default.canStage(source.engine))
|
|
54193
|
+
throw new Error(`Cannot sample consumer "${consumer.name}": its producer "${producer.name}" reads source "${source.name}", and a "${source.engine}" source cannot be run through the pipeline yet. Sample the producer "${producer.name}" instead to see its rows.`);
|
|
54194
|
+
}
|
|
54195
|
+
};
|
|
54196
|
+
/**
|
|
54197
|
+
* Every producer a run of this consumer would stage — its own, plus those of the consumers it
|
|
54198
|
+
* reads as input or consults as a code_lookup table, since each of those is executed too.
|
|
54199
|
+
*/
|
|
54200
|
+
this._inputProducers = (consumer, visited = []) => {
|
|
54201
|
+
Affirm_default(!visited.includes(consumer.name), `Consumer "${consumer.name}" reads itself through ${visited.join(" \u2192 ")}`);
|
|
54202
|
+
Affirm_default(visited.length < MAX_WALK_DEPTH, `Consumer chain from "${visited[0]}" is more than ${MAX_WALK_DEPTH} deep`);
|
|
54203
|
+
const seen = [...visited, consumer.name];
|
|
54204
|
+
const upstream = [
|
|
54205
|
+
...consumer.producers.map((entry) => entry.name),
|
|
54206
|
+
...ConsumerManager_default.collectLookupRefs(ConsumerManager_default.getExpandedFields(consumer)).map((ref) => ref.consumer)
|
|
54207
|
+
];
|
|
54208
|
+
return upstream.flatMap((name) => {
|
|
54209
|
+
const producer = Environment_default.getProducer(name);
|
|
54210
|
+
if (producer) return [producer];
|
|
54211
|
+
const nested = Environment_default.getConsumer(name);
|
|
54212
|
+
Affirm_default(nested, `Producer or consumer "${name}" not found for consumer "${consumer.name}"`);
|
|
54213
|
+
return this._inputProducers(nested, seen);
|
|
54214
|
+
});
|
|
54215
|
+
};
|
|
54216
|
+
/**
|
|
54217
|
+
* One row's keys in the order the consumer DECLARES its fields.
|
|
54218
|
+
*
|
|
54219
|
+
* A record comes out of a worker keyed in the order the producer's dimensions were read, which is
|
|
54220
|
+
* the source's layout and not the consumer's — so a consumer declaring country, plan, is_active
|
|
54221
|
+
* over a producer that stores is_active first would be shown its own columns in someone else's
|
|
54222
|
+
* order. Invisible in an exported JSON object; the whole shape of the table here.
|
|
54223
|
+
*
|
|
54224
|
+
* Anything the record carries that the field list does not name is appended rather than dropped:
|
|
54225
|
+
* a column that turns up unexpectedly is exactly the one worth seeing.
|
|
54226
|
+
*/
|
|
54227
|
+
this._inFieldOrder = (consumer, record) => {
|
|
54228
|
+
const declared = ConsumerManager_default.getExpandedFields(consumer).filter((field) => !field.cField.hidden).map((field) => field.finalKey);
|
|
54229
|
+
const ordered = {};
|
|
54230
|
+
for (const key of declared) {
|
|
54231
|
+
if (key in record) ordered[key] = record[key];
|
|
54232
|
+
}
|
|
54233
|
+
for (const key of Object.keys(record)) {
|
|
54234
|
+
if (!(key in ordered)) ordered[key] = record[key];
|
|
54235
|
+
}
|
|
54236
|
+
return ordered;
|
|
54237
|
+
};
|
|
54238
|
+
/** The captured dataset, which `captureFormat: 'JSONL'` guarantees is one JSON object per line. */
|
|
54239
|
+
this._readCapture = async (capturePath, size) => {
|
|
54240
|
+
if (!import_fs21.default.existsSync(capturePath)) return [];
|
|
54241
|
+
const records = [];
|
|
54242
|
+
const lines = import_readline10.default.createInterface({ input: import_fs21.default.createReadStream(capturePath), crlfDelay: Infinity });
|
|
54243
|
+
try {
|
|
54244
|
+
for await (const line of lines) {
|
|
54245
|
+
if (!line.length) continue;
|
|
54246
|
+
records.push(JSON.parse(line));
|
|
54247
|
+
if (records.length >= size) break;
|
|
54248
|
+
}
|
|
54249
|
+
} finally {
|
|
54250
|
+
lines.close();
|
|
54251
|
+
}
|
|
54252
|
+
return records;
|
|
54253
|
+
};
|
|
54254
|
+
}
|
|
54255
|
+
};
|
|
54256
|
+
var SampleRunner = new SampleRunnerClass();
|
|
54257
|
+
var SampleRunner_default = SampleRunner;
|
|
54258
|
+
|
|
53829
54259
|
// src/actions/run.ts
|
|
53830
54260
|
init_src4();
|
|
53831
54261
|
init_src2();
|
|
@@ -53966,10 +54396,10 @@ init_src5();
|
|
|
53966
54396
|
init_src2();
|
|
53967
54397
|
var FORMATS = ["mermaid", "dot", "json"];
|
|
53968
54398
|
var graph = async (options) => {
|
|
53969
|
-
const
|
|
54399
|
+
const format4 = options?.format ?? "mermaid";
|
|
53970
54400
|
try {
|
|
53971
|
-
if (!FORMATS.includes(
|
|
53972
|
-
console.error(import_chalk9.default.red(`Unknown format "${
|
|
54401
|
+
if (!FORMATS.includes(format4)) {
|
|
54402
|
+
console.error(import_chalk9.default.red(`Unknown format "${format4}". Use one of: ${FORMATS.join(", ")}.`));
|
|
53973
54403
|
await Logger_default.flush();
|
|
53974
54404
|
process.exit(1);
|
|
53975
54405
|
}
|
|
@@ -53979,10 +54409,10 @@ var graph = async (options) => {
|
|
|
53979
54409
|
generatedBy: `remora ${Constants_default.cliVersion}`,
|
|
53980
54410
|
loadDiagnostics: Diagnostics_default.fromLoadProblems(problems),
|
|
53981
54411
|
// The json format IS the viewer's contract, so every resource view ships with it.
|
|
53982
|
-
resources:
|
|
54412
|
+
resources: format4 === "json" ? "inline" : "lazy"
|
|
53983
54413
|
});
|
|
53984
|
-
process.stdout.write(render(
|
|
53985
|
-
if (
|
|
54414
|
+
process.stdout.write(render(format4, snapshot) + "\n");
|
|
54415
|
+
if (format4 !== "json")
|
|
53986
54416
|
reportDiagnostics(snapshot.diagnostics);
|
|
53987
54417
|
} catch (err2) {
|
|
53988
54418
|
const myErr = Helper_default.asError(err2);
|
|
@@ -53992,9 +54422,9 @@ var graph = async (options) => {
|
|
|
53992
54422
|
process.exit(1);
|
|
53993
54423
|
}
|
|
53994
54424
|
};
|
|
53995
|
-
var render = (
|
|
53996
|
-
if (
|
|
53997
|
-
if (
|
|
54425
|
+
var render = (format4, snapshot) => {
|
|
54426
|
+
if (format4 === "json") return JSON.stringify(snapshot, null, 4);
|
|
54427
|
+
if (format4 === "dot") return LineageRenderer_default.toDot(snapshot.graph);
|
|
53998
54428
|
return LineageRenderer_default.toMermaid(snapshot.graph);
|
|
53999
54429
|
};
|
|
54000
54430
|
var reportDiagnostics = (diagnostics) => {
|
|
@@ -54018,6 +54448,7 @@ init_src5();
|
|
|
54018
54448
|
init_src2();
|
|
54019
54449
|
var WATCH_INTERVAL_MS = 1e3;
|
|
54020
54450
|
var SAMPLEABLE_KINDS = ["producer", "consumer"];
|
|
54451
|
+
var PIPELINE_SAMPLE_KINDS = ["consumer"];
|
|
54021
54452
|
var EDITABLE_KINDS = {
|
|
54022
54453
|
source: "source",
|
|
54023
54454
|
producer: "producer",
|
|
@@ -54054,11 +54485,15 @@ var PreviewProject = class {
|
|
|
54054
54485
|
this._files = [];
|
|
54055
54486
|
this._hash = null;
|
|
54056
54487
|
/**
|
|
54057
|
-
*
|
|
54488
|
+
* What this page is currently executing, or null — a run, or a consumer sample.
|
|
54489
|
+
*
|
|
54490
|
+
* One at a time, deliberately: `remora run` runs consumers sequentially, and two concurrent
|
|
54491
|
+
* executions would each spin up a worker pool in this one process. Refusing the second with the
|
|
54492
|
+
* name of the first is a better answer than thrashing the machine.
|
|
54058
54493
|
*
|
|
54059
|
-
*
|
|
54060
|
-
*
|
|
54061
|
-
*
|
|
54494
|
+
* A sample takes the same lock because it IS an execution: bounded at the input and exporting
|
|
54495
|
+
* nothing, but a worker pool either way. That also means a sample and a run cannot overlap, which
|
|
54496
|
+
* is what stops a click on one panel from slowing the run someone is watching on another.
|
|
54062
54497
|
*/
|
|
54063
54498
|
this._running = null;
|
|
54064
54499
|
/** @returns true when the project changed and the snapshot was rebuilt. */
|
|
@@ -54104,18 +54539,47 @@ var PreviewProject = class {
|
|
|
54104
54539
|
]
|
|
54105
54540
|
};
|
|
54106
54541
|
},
|
|
54107
|
-
sample: async (nodeId, rows) =>
|
|
54108
|
-
if (!this._has(nodeId)) return null;
|
|
54109
|
-
const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
|
|
54110
|
-
if (!SAMPLEABLE_KINDS.includes(kind)) return null;
|
|
54111
|
-
const { records } = await SampleEngine_default.read(name, rows);
|
|
54112
|
-
return ResourceView_default.buildSampleTable(records);
|
|
54113
|
-
},
|
|
54542
|
+
sample: async (nodeId, rows) => this._sample(nodeId, rows),
|
|
54114
54543
|
run: async (nodeId) => this._run(nodeId),
|
|
54115
54544
|
file: (nodeId) => this._file(nodeId),
|
|
54116
54545
|
save: async (nodeId, content) => this._save(nodeId, content),
|
|
54117
54546
|
logs: (lines) => this._logs(lines)
|
|
54118
54547
|
});
|
|
54548
|
+
/**
|
|
54549
|
+
* The rows a resource actually holds, as a table.
|
|
54550
|
+
*
|
|
54551
|
+
* A producer answers from a bounded read of its source. A consumer answers by RUNNING — every
|
|
54552
|
+
* filter, transformation, mask, union, distinct, pivot and validation it declares, over a
|
|
54553
|
+
* bounded slice of each producer's input, exporting nothing and recording nothing. Which of the
|
|
54554
|
+
* two happened, and over how much input, is the note `SampleRunner` returns; it becomes the
|
|
54555
|
+
* table's title, so the panel states what it is showing instead of leaving it to be assumed.
|
|
54556
|
+
*
|
|
54557
|
+
* Awaited, unlike a run: a sample is bounded, and its ANSWER is the rows — there is nothing for
|
|
54558
|
+
* the run overlay to show, because a sample deliberately leaves no trace on it.
|
|
54559
|
+
*/
|
|
54560
|
+
this._sample = async (nodeId, rows) => {
|
|
54561
|
+
if (!this._has(nodeId)) return null;
|
|
54562
|
+
const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
|
|
54563
|
+
if (!SAMPLEABLE_KINDS.includes(kind)) return null;
|
|
54564
|
+
const pipeline6 = PIPELINE_SAMPLE_KINDS.includes(kind);
|
|
54565
|
+
if (pipeline6 && this._running)
|
|
54566
|
+
throw new Error(`"${this._running}" is still running; one execution at a time`);
|
|
54567
|
+
const blocking = pipeline6 ? this.snapshot.diagnostics.find((x2) => x2.node === nodeId && x2.severity === "error") : null;
|
|
54568
|
+
if (blocking) throw new Error(`Refusing to sample "${name}": ${blocking.message}`);
|
|
54569
|
+
if (pipeline6) this._running = name;
|
|
54570
|
+
try {
|
|
54571
|
+
const licence = LicenceManager_default.validate(ProcessENVManager_default.getEnvVariable("REMORA_LICENCE_KEY"));
|
|
54572
|
+
const { records, note } = await SampleRunner_default.read(name, rows, {
|
|
54573
|
+
// A sample from the preview is a sample from the CLI. It records no usage, so this
|
|
54574
|
+
// only ever names the run in the log.
|
|
54575
|
+
invokedBy: "CLI",
|
|
54576
|
+
user: { _id: licence.customer, name: licence.customer, type: "licence" }
|
|
54577
|
+
});
|
|
54578
|
+
return ResourceView_default.buildSampleTable(records, note);
|
|
54579
|
+
} finally {
|
|
54580
|
+
if (pipeline6) this._running = null;
|
|
54581
|
+
}
|
|
54582
|
+
};
|
|
54119
54583
|
/**
|
|
54120
54584
|
* Executes a consumer, and answers as soon as it has STARTED.
|
|
54121
54585
|
*
|
|
@@ -54127,7 +54591,7 @@ var PreviewProject = class {
|
|
|
54127
54591
|
if (!this._has(nodeId)) return { started: false, message: `No resource "${nodeId}" in this project` };
|
|
54128
54592
|
const { kind, name } = LineageEngine_default.parseNodeId(nodeId);
|
|
54129
54593
|
if (kind !== "consumer") return { started: false, message: `Only a consumer can be run; "${name}" is a ${kind}` };
|
|
54130
|
-
if (this._running) return { started: false, message: `"${this._running}" is still running; one
|
|
54594
|
+
if (this._running) return { started: false, message: `"${this._running}" is still running; one execution at a time` };
|
|
54131
54595
|
const blocking = this.snapshot.diagnostics.find((x2) => x2.node === nodeId && x2.severity === "error");
|
|
54132
54596
|
if (blocking) return { started: false, message: `Refusing to run "${name}": ${blocking.message}` };
|
|
54133
54597
|
const consumer = Environment_default.getConsumer(name);
|
|
@@ -54251,22 +54715,22 @@ var stopOnSignal = (server, watch) => {
|
|
|
54251
54715
|
|
|
54252
54716
|
// src/actions/create_producer.ts
|
|
54253
54717
|
var import_chalk11 = __toESM(require("chalk"));
|
|
54254
|
-
var
|
|
54255
|
-
var
|
|
54718
|
+
var import_fs22 = __toESM(require("fs"));
|
|
54719
|
+
var import_path30 = __toESM(require("path"));
|
|
54256
54720
|
init_src2();
|
|
54257
54721
|
var create_producer = async (name) => {
|
|
54258
54722
|
try {
|
|
54259
|
-
if (!
|
|
54723
|
+
if (!import_fs22.default.existsSync("./remora/producers")) {
|
|
54260
54724
|
throw new Error(import_chalk11.default.red("Missing directory: ") + import_chalk11.default.yellow("./remora/producers"));
|
|
54261
54725
|
}
|
|
54262
|
-
const defaultProducerTemplate =
|
|
54263
|
-
|
|
54726
|
+
const defaultProducerTemplate = import_fs22.default.readFileSync(
|
|
54727
|
+
import_path30.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
|
|
54264
54728
|
"utf-8"
|
|
54265
54729
|
);
|
|
54266
54730
|
const defaultProducer = JSON.parse(defaultProducerTemplate);
|
|
54267
54731
|
defaultProducer.name = name;
|
|
54268
|
-
const producerPath =
|
|
54269
|
-
|
|
54732
|
+
const producerPath = import_path30.default.join("remora/producers", `${name}.json`);
|
|
54733
|
+
import_fs22.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
|
|
54270
54734
|
console.log(import_chalk11.default.green(`\u2705 Created producer config at ${producerPath}`));
|
|
54271
54735
|
console.log(import_chalk11.default.blue("Remember to:"));
|
|
54272
54736
|
console.log(import_chalk11.default.blue("1. Set the correct source name"));
|
|
@@ -54281,26 +54745,26 @@ var create_producer = async (name) => {
|
|
|
54281
54745
|
|
|
54282
54746
|
// src/actions/create_consumer.ts
|
|
54283
54747
|
var import_chalk12 = __toESM(require("chalk"));
|
|
54284
|
-
var
|
|
54285
|
-
var
|
|
54748
|
+
var import_fs23 = __toESM(require("fs"));
|
|
54749
|
+
var import_path31 = __toESM(require("path"));
|
|
54286
54750
|
init_src2();
|
|
54287
54751
|
var create_consumer = async (name, producerName) => {
|
|
54288
54752
|
try {
|
|
54289
|
-
if (!
|
|
54753
|
+
if (!import_fs23.default.existsSync("./remora/consumers")) {
|
|
54290
54754
|
throw new Error(import_chalk12.default.red("Missing directory: ") + import_chalk12.default.yellow("./remora/consumers"));
|
|
54291
54755
|
}
|
|
54292
|
-
const defaultConsumerTemplate =
|
|
54293
|
-
|
|
54756
|
+
const defaultConsumerTemplate = import_fs23.default.readFileSync(
|
|
54757
|
+
import_path31.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
|
|
54294
54758
|
"utf-8"
|
|
54295
54759
|
);
|
|
54296
54760
|
const defaultConsumer = JSON.parse(defaultConsumerTemplate);
|
|
54297
54761
|
defaultConsumer.name = name;
|
|
54298
54762
|
if (producerName) {
|
|
54299
|
-
const producerPath =
|
|
54300
|
-
if (!
|
|
54763
|
+
const producerPath = import_path31.default.join("remora/producers", `${producerName}.json`);
|
|
54764
|
+
if (!import_fs23.default.existsSync(producerPath)) {
|
|
54301
54765
|
throw new Error(import_chalk12.default.red("Producer not found: ") + import_chalk12.default.yellow(producerPath));
|
|
54302
54766
|
}
|
|
54303
|
-
const producerConfig = JSON.parse(
|
|
54767
|
+
const producerConfig = JSON.parse(import_fs23.default.readFileSync(producerPath, "utf-8"));
|
|
54304
54768
|
defaultConsumer.producers = [{ name: producerName }];
|
|
54305
54769
|
defaultConsumer.fields = producerConfig.dimensions.map((dim) => ({
|
|
54306
54770
|
key: dim.name,
|
|
@@ -54319,8 +54783,8 @@ var create_consumer = async (name, producerName) => {
|
|
|
54319
54783
|
defaultConsumer.filters = void 0;
|
|
54320
54784
|
defaultConsumer.metadata = void 0;
|
|
54321
54785
|
}
|
|
54322
|
-
const consumerPath =
|
|
54323
|
-
|
|
54786
|
+
const consumerPath = import_path31.default.join("remora/consumers", `${name}.json`);
|
|
54787
|
+
import_fs23.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
|
|
54324
54788
|
console.log(import_chalk12.default.green(`\u2705 Created consumer config at ${consumerPath}`));
|
|
54325
54789
|
if (!producerName) {
|
|
54326
54790
|
console.log(import_chalk12.default.blue("Remember to:"));
|
|
@@ -54348,8 +54812,8 @@ init_src5();
|
|
|
54348
54812
|
var import_chalk13 = __toESM(require("chalk"));
|
|
54349
54813
|
var import_ora5 = __toESM(require("ora"));
|
|
54350
54814
|
init_src10();
|
|
54351
|
-
var
|
|
54352
|
-
var
|
|
54815
|
+
var import_fs24 = __toESM(require("fs"));
|
|
54816
|
+
var import_path32 = __toESM(require("path"));
|
|
54353
54817
|
init_src4();
|
|
54354
54818
|
init_src10();
|
|
54355
54819
|
init_src2();
|
|
@@ -54378,14 +54842,14 @@ var automap = async (producerName, schemaNames) => {
|
|
|
54378
54842
|
const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
|
|
54379
54843
|
for (const producer2 of mapResult.producers) {
|
|
54380
54844
|
const producerPath = Environment_default.resolveResourceFilePath("producers", producer2.name);
|
|
54381
|
-
|
|
54382
|
-
|
|
54845
|
+
import_fs24.default.mkdirSync(import_path32.default.dirname(producerPath), { recursive: true });
|
|
54846
|
+
import_fs24.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
|
|
54383
54847
|
console.log(import_chalk13.default.blue(`Created producer: ${producer2.name}`));
|
|
54384
54848
|
}
|
|
54385
54849
|
for (const consumer of mapResult.consumers) {
|
|
54386
54850
|
const consumerPath = Environment_default.resolveResourceFilePath("consumers", consumer.name);
|
|
54387
|
-
|
|
54388
|
-
|
|
54851
|
+
import_fs24.default.mkdirSync(import_path32.default.dirname(consumerPath), { recursive: true });
|
|
54852
|
+
import_fs24.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
|
|
54389
54853
|
console.log(import_chalk13.default.blue(`Created consumer: ${consumer.name}`));
|
|
54390
54854
|
}
|
|
54391
54855
|
spinner.succeed("Producer has been successfully mapped");
|
|
@@ -54405,7 +54869,6 @@ var automap = async (producerName, schemaNames) => {
|
|
|
54405
54869
|
// src/actions/sample.ts
|
|
54406
54870
|
var import_chalk14 = __toESM(require("chalk"));
|
|
54407
54871
|
var import_ora6 = __toESM(require("ora"));
|
|
54408
|
-
init_src10();
|
|
54409
54872
|
init_src5();
|
|
54410
54873
|
init_src2();
|
|
54411
54874
|
var sample = async (resourceName, sampleSize = 10) => {
|
|
@@ -54413,8 +54876,9 @@ var sample = async (resourceName, sampleSize = 10) => {
|
|
|
54413
54876
|
compile();
|
|
54414
54877
|
console.log();
|
|
54415
54878
|
const spinner = (0, import_ora6.default)(import_chalk14.default.blue(`Sampling "${resourceName}"...`)).start();
|
|
54416
|
-
const { kind, records } = await
|
|
54879
|
+
const { kind, records, note } = await SampleRunner_default.read(resourceName, sampleSize);
|
|
54417
54880
|
spinner.succeed(import_chalk14.default.green(`Sample data retrieved from ${kind} "${resourceName}"`));
|
|
54881
|
+
console.log(import_chalk14.default.gray(` ${note}`));
|
|
54418
54882
|
if (records.length === 0) {
|
|
54419
54883
|
console.log(import_chalk14.default.yellow("No data found in the dataset."));
|
|
54420
54884
|
return;
|