@naturalcycles/nodejs-lib 13.45.3 → 14.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/generate-build-info.js +15 -18
- package/dist/bin/json2env.js +6 -9
- package/dist/bin/kpy.js +6 -9
- package/dist/bin/secrets-decrypt.d.ts +1 -1
- package/dist/bin/secrets-decrypt.js +9 -12
- package/dist/bin/secrets-encrypt.d.ts +1 -1
- package/dist/bin/secrets-encrypt.js +9 -12
- package/dist/bin/secrets-gen-key.js +8 -11
- package/dist/bin/slack-this.js +6 -9
- package/dist/buffer/buffer.util.js +1 -4
- package/dist/colors/colors.d.ts +26 -27
- package/dist/colors/colors.js +29 -33
- package/dist/csv/csvReader.js +4 -8
- package/dist/csv/csvWriter.js +7 -12
- package/dist/csv/transformToCSV.d.ts +2 -2
- package/dist/csv/transformToCSV.js +5 -8
- package/dist/diff/tableDiff.js +4 -7
- package/dist/fs/fs2.d.ts +1 -1
- package/dist/fs/fs2.js +88 -94
- package/dist/fs/json2env.js +18 -28
- package/dist/fs/kpy.js +28 -33
- package/dist/index.d.ts +71 -71
- package/dist/index.js +73 -78
- package/dist/infra/process.util.js +23 -28
- package/dist/jwt/jwt.service.js +17 -21
- package/dist/log/log.util.js +4 -7
- package/dist/script/runScript.js +11 -15
- package/dist/secret/secrets-decrypt.util.js +18 -22
- package/dist/secret/secrets-encrypt.util.js +17 -21
- package/dist/security/crypto.util.js +24 -34
- package/dist/security/hash.util.js +19 -37
- package/dist/security/id.util.js +11 -18
- package/dist/security/nanoid.js +15 -20
- package/dist/security/secret.util.js +20 -30
- package/dist/slack/slack.service.d.ts +1 -1
- package/dist/slack/slack.service.js +13 -16
- package/dist/slack/slack.service.model.d.ts +1 -1
- package/dist/slack/slack.service.model.js +1 -2
- package/dist/stream/ndjson/ndjson.model.js +10 -16
- package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonMap.js +11 -14
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformJsonParse.js +4 -9
- package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +5 -8
- package/dist/stream/pipeline/pipeline.js +13 -17
- package/dist/stream/progressLogger.d.ts +1 -1
- package/dist/stream/progressLogger.js +32 -28
- package/dist/stream/readable/readableCreate.d.ts +1 -1
- package/dist/stream/readable/readableCreate.js +5 -9
- package/dist/stream/readable/readableForEach.d.ts +2 -2
- package/dist/stream/readable/readableForEach.js +7 -11
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +4 -7
- package/dist/stream/readable/readableToArray.d.ts +1 -1
- package/dist/stream/readable/readableToArray.js +1 -4
- package/dist/stream/sizeStack.js +9 -12
- package/dist/stream/stream.model.js +1 -2
- package/dist/stream/stream.util.js +1 -4
- package/dist/stream/transform/transformChunk.d.ts +1 -1
- package/dist/stream/transform/transformChunk.js +3 -6
- package/dist/stream/transform/transformFilter.d.ts +2 -2
- package/dist/stream/transform/transformFilter.js +6 -10
- package/dist/stream/transform/transformLimit.d.ts +2 -2
- package/dist/stream/transform/transformLimit.js +6 -11
- package/dist/stream/transform/transformLogProgress.d.ts +2 -2
- package/dist/stream/transform/transformLogProgress.js +5 -8
- package/dist/stream/transform/transformMap.d.ts +1 -1
- package/dist/stream/transform/transformMap.js +18 -22
- package/dist/stream/transform/transformMapSimple.d.ts +1 -1
- package/dist/stream/transform/transformMapSimple.js +6 -9
- package/dist/stream/transform/transformMapSync.d.ts +3 -3
- package/dist/stream/transform/transformMapSync.js +14 -19
- package/dist/stream/transform/transformNoOp.d.ts +1 -1
- package/dist/stream/transform/transformNoOp.js +3 -6
- package/dist/stream/transform/transformOffset.d.ts +1 -1
- package/dist/stream/transform/transformOffset.js +5 -8
- package/dist/stream/transform/transformSplit.d.ts +1 -1
- package/dist/stream/transform/transformSplit.js +5 -9
- package/dist/stream/transform/transformTap.d.ts +1 -1
- package/dist/stream/transform/transformTap.js +3 -6
- package/dist/stream/transform/transformTee.d.ts +1 -1
- package/dist/stream/transform/transformTee.js +7 -10
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +9 -12
- package/dist/stream/transform/transformToArray.d.ts +1 -1
- package/dist/stream/transform/transformToArray.js +3 -6
- package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
- package/dist/stream/transform/worker/baseWorkerClass.js +2 -5
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
- package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
- package/dist/stream/transform/worker/workerClassProxy.js +6 -4
- package/dist/stream/writable/writableForEach.d.ts +2 -2
- package/dist/stream/writable/writableForEach.js +6 -10
- package/dist/stream/writable/writableFork.d.ts +1 -1
- package/dist/stream/writable/writableFork.js +6 -9
- package/dist/stream/writable/writableLimit.d.ts +1 -1
- package/dist/stream/writable/writableLimit.js +3 -6
- package/dist/stream/writable/writablePushToArray.d.ts +1 -1
- package/dist/stream/writable/writablePushToArray.js +3 -6
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/stream/writable/writableVoid.js +3 -6
- package/dist/string/inspect.js +8 -13
- package/dist/util/buildInfo.util.js +12 -15
- package/dist/util/env.util.js +5 -9
- package/dist/util/exec2.js +17 -21
- package/dist/util/git2.js +17 -20
- package/dist/util/lruMemoCache.js +4 -7
- package/dist/util/zip.util.js +14 -25
- package/dist/validation/ajv/ajv.util.d.ts +3 -3
- package/dist/validation/ajv/ajv.util.js +6 -10
- package/dist/validation/ajv/ajvSchema.d.ts +2 -2
- package/dist/validation/ajv/ajvSchema.js +17 -18
- package/dist/validation/ajv/ajvValidationError.js +2 -6
- package/dist/validation/ajv/getAjv.d.ts +1 -1
- package/dist/validation/ajv/getAjv.js +9 -9
- package/dist/validation/joi/joi.extensions.d.ts +2 -2
- package/dist/validation/joi/joi.extensions.js +6 -10
- package/dist/validation/joi/joi.model.js +1 -2
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.shared.schemas.js +61 -74
- package/dist/validation/joi/joi.validation.error.js +2 -6
- package/dist/validation/joi/joi.validation.util.d.ts +1 -1
- package/dist/validation/joi/joi.validation.util.js +10 -17
- package/dist/validation/joi/number.extensions.js +1 -4
- package/dist/validation/joi/string.extensions.js +4 -7
- package/dist/yargs.util.js +5 -8
- package/package.json +5 -4
- package/src/bin/generate-build-info.ts +3 -3
- package/src/bin/json2env.ts +2 -2
- package/src/bin/kpy.ts +2 -2
- package/src/bin/secrets-decrypt.ts +7 -8
- package/src/bin/secrets-encrypt.ts +7 -8
- package/src/bin/secrets-gen-key.ts +2 -2
- package/src/bin/slack-this.ts +2 -2
- package/src/csv/transformToCSV.ts +3 -3
- package/src/fs/fs2.ts +4 -4
- package/src/fs/json2env.ts +2 -2
- package/src/fs/kpy.ts +2 -2
- package/src/index.ts +71 -71
- package/src/jwt/jwt.service.ts +2 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +4 -3
- package/src/secret/secrets-decrypt.util.ts +3 -3
- package/src/secret/secrets-encrypt.util.ts +3 -3
- package/src/security/crypto.util.ts +1 -1
- package/src/security/id.util.ts +1 -1
- package/src/security/secret.util.ts +2 -2
- package/src/slack/slack.service.model.ts +1 -1
- package/src/slack/slack.service.ts +3 -3
- package/src/stream/ndjson/ndjson.model.ts +1 -1
- package/src/stream/ndjson/ndjsonMap.ts +2 -2
- package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +1 -1
- package/src/stream/progressLogger.ts +3 -3
- package/src/stream/readable/readableCreate.ts +1 -1
- package/src/stream/readable/readableForEach.ts +4 -4
- package/src/stream/readable/readableFromArray.ts +1 -1
- package/src/stream/readable/readableToArray.ts +1 -1
- package/src/stream/sizeStack.ts +2 -2
- package/src/stream/transform/transformChunk.ts +1 -1
- package/src/stream/transform/transformFilter.ts +3 -3
- package/src/stream/transform/transformLimit.ts +3 -3
- package/src/stream/transform/transformLogProgress.ts +3 -3
- package/src/stream/transform/transformMap.ts +5 -5
- package/src/stream/transform/transformMapSimple.ts +1 -1
- package/src/stream/transform/transformMapSync.ts +5 -5
- package/src/stream/transform/transformNoOp.ts +1 -1
- package/src/stream/transform/transformOffset.ts +2 -2
- package/src/stream/transform/transformSplit.ts +1 -1
- package/src/stream/transform/transformTap.ts +1 -1
- package/src/stream/transform/transformTee.ts +3 -3
- package/src/stream/transform/transformThrottle.ts +1 -1
- package/src/stream/transform/transformToArray.ts +1 -1
- package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
- package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
- package/src/stream/transform/worker/workerClassProxy.js +6 -4
- package/src/stream/writable/writableForEach.ts +3 -3
- package/src/stream/writable/writableFork.ts +3 -3
- package/src/stream/writable/writableLimit.ts +1 -1
- package/src/stream/writable/writablePushToArray.ts +1 -1
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/util/buildInfo.util.ts +2 -2
- package/src/util/exec2.ts +1 -1
- package/src/util/git2.ts +2 -2
- package/src/validation/ajv/ajv.util.ts +4 -4
- package/src/validation/ajv/ajvSchema.ts +4 -5
- package/src/validation/ajv/getAjv.ts +7 -3
- package/src/validation/joi/joi.extensions.ts +4 -4
- package/src/validation/joi/joi.shared.schemas.ts +3 -3
- package/src/validation/joi/joi.validation.util.ts +2 -2
- package/src/yargs.util.ts +1 -0
|
@@ -1,14 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const colors_1 = require("../../colors/colors");
|
|
6
|
-
class NDJsonStats {
|
|
7
|
-
constructor() {
|
|
8
|
-
this.tookMillis = 0;
|
|
9
|
-
this.rows = 0;
|
|
10
|
-
this.sizeBytes = 0;
|
|
11
|
-
}
|
|
1
|
+
import { _hb, _ms } from '@naturalcycles/js-lib';
|
|
2
|
+
import { boldWhite, dimWhite } from '../../colors/colors.js';
|
|
3
|
+
export class NDJsonStats {
|
|
12
4
|
static create(o = {}) {
|
|
13
5
|
return Object.assign(new NDJsonStats(), o);
|
|
14
6
|
}
|
|
@@ -19,6 +11,9 @@ class NDJsonStats {
|
|
|
19
11
|
// eslint-disable-next-line unicorn/no-array-reduce
|
|
20
12
|
return stats.reduce((statsTotal, stats) => statsTotal.add(stats), new NDJsonStats());
|
|
21
13
|
}
|
|
14
|
+
tookMillis = 0;
|
|
15
|
+
rows = 0;
|
|
16
|
+
sizeBytes = 0;
|
|
22
17
|
get rpsTotal() {
|
|
23
18
|
return Math.round(this.rows / ((this.tookMillis || 1) / 1000));
|
|
24
19
|
}
|
|
@@ -40,11 +35,10 @@ class NDJsonStats {
|
|
|
40
35
|
}
|
|
41
36
|
toPretty(name) {
|
|
42
37
|
return [
|
|
43
|
-
`Processed ${name ?
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
38
|
+
`Processed ${name ? boldWhite(name) + ': ' : ''}${dimWhite(this.rows)} rows, ${dimWhite(_hb(this.sizeBytes))} in ${dimWhite(_ms(this.tookMillis))}`,
|
|
39
|
+
dimWhite(this.rpsTotal + ' rows/sec'),
|
|
40
|
+
dimWhite(_hb(this.avgBytesPerRow) + '/row'),
|
|
41
|
+
dimWhite(_hb(this.bpsTotal) + '/sec'),
|
|
47
42
|
].join(', ');
|
|
48
43
|
}
|
|
49
44
|
}
|
|
50
|
-
exports.NDJsonStats = NDJsonStats;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformLogProgressOptions, TransformMapOptions } from '
|
|
2
|
+
import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
|
|
3
3
|
export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT>, TransformLogProgressOptions<IN> {
|
|
4
4
|
inputFilePath: string;
|
|
5
5
|
outputFilePath: string;
|
|
@@ -1,31 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.ndjsonMap = ndjsonMap;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const __1 = require("../..");
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline, fs2, transformLimit, transformLogProgress, transformMap } from '../../index.js';
|
|
6
3
|
/**
|
|
7
4
|
* Unzips input file automatically, if it ends with `.gz`.
|
|
8
5
|
* Zips output file automatically, if it ends with `.gz`.
|
|
9
6
|
*/
|
|
10
|
-
async function ndjsonMap(mapper, opt) {
|
|
7
|
+
export async function ndjsonMap(mapper, opt) {
|
|
11
8
|
const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;
|
|
12
9
|
console.log({
|
|
13
10
|
inputFilePath,
|
|
14
11
|
outputFilePath,
|
|
15
12
|
});
|
|
16
|
-
const readable =
|
|
13
|
+
const readable = fs2
|
|
17
14
|
.createReadStreamAsNDJSON(inputFilePath)
|
|
18
15
|
.take(limitInput || Number.POSITIVE_INFINITY);
|
|
19
|
-
await
|
|
16
|
+
await _pipeline([
|
|
20
17
|
readable,
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
transformLogProgress({ metric: 'read', ...opt }),
|
|
19
|
+
transformMap(mapper, {
|
|
23
20
|
flattenArrayOutput: true,
|
|
24
|
-
errorMode:
|
|
21
|
+
errorMode: ErrorMode.SUPPRESS,
|
|
25
22
|
...opt,
|
|
26
23
|
}),
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
...
|
|
24
|
+
transformLimit({ limit: limitOutput, sourceReadable: readable }),
|
|
25
|
+
transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
|
|
26
|
+
...fs2.createWriteStreamAsNDJSON(outputFilePath),
|
|
30
27
|
]);
|
|
31
28
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformLogProgressOptions, TransformMapOptions } from '
|
|
2
|
+
import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
|
|
3
3
|
export interface NDJSONStreamForEachOptions<IN = any> extends TransformMapOptions<IN, void>, TransformLogProgressOptions<IN> {
|
|
4
4
|
inputFilePath: string;
|
|
5
5
|
}
|
|
@@ -1,20 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.ndjsonStreamForEach = ndjsonStreamForEach;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const __1 = require("../..");
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline, fs2, transformLogProgress, transformMap, writableVoid } from '../../index.js';
|
|
6
3
|
/**
|
|
7
4
|
* Convenience function to `forEach` through an ndjson file.
|
|
8
5
|
*/
|
|
9
|
-
async function ndjsonStreamForEach(mapper, opt) {
|
|
10
|
-
await
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
errorMode:
|
|
6
|
+
export async function ndjsonStreamForEach(mapper, opt) {
|
|
7
|
+
await _pipeline([
|
|
8
|
+
fs2.createReadStreamAsNDJSON(opt.inputFilePath),
|
|
9
|
+
transformMap(mapper, {
|
|
10
|
+
errorMode: ErrorMode.THROW_AGGREGATED,
|
|
14
11
|
...opt,
|
|
15
12
|
predicate: () => true, // to log progress properly
|
|
16
13
|
}),
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
transformLogProgress(opt),
|
|
15
|
+
writableVoid(),
|
|
19
16
|
]);
|
|
20
17
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Reviver } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformTyped } from '../stream.model';
|
|
2
|
+
import type { TransformTyped } from '../stream.model.js';
|
|
3
3
|
export interface TransformJsonParseOptions {
|
|
4
4
|
/**
|
|
5
5
|
* If true - will throw an error on JSON.parse / stringify error
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.bufferReviver = void 0;
|
|
4
|
-
exports.transformJsonParse = transformJsonParse;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
6
2
|
/**
|
|
7
3
|
* Transforms chunks of JSON strings/Buffers (objectMode=false) into parsed objects (readableObjectMode=true).
|
|
8
4
|
*
|
|
@@ -17,9 +13,9 @@ const node_stream_1 = require("node:stream");
|
|
|
17
13
|
* consumeYourStream...
|
|
18
14
|
* [)
|
|
19
15
|
*/
|
|
20
|
-
function transformJsonParse(opt = {}) {
|
|
16
|
+
export function transformJsonParse(opt = {}) {
|
|
21
17
|
const { strict = true, reviver } = opt;
|
|
22
|
-
return new
|
|
18
|
+
return new Transform({
|
|
23
19
|
writableObjectMode: false,
|
|
24
20
|
readableObjectMode: true,
|
|
25
21
|
// highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
|
|
@@ -43,10 +39,9 @@ function transformJsonParse(opt = {}) {
|
|
|
43
39
|
});
|
|
44
40
|
}
|
|
45
41
|
// Based on: https://stackoverflow.com/a/34557997/4919972
|
|
46
|
-
const bufferReviver = (_k, v) => {
|
|
42
|
+
export const bufferReviver = (_k, v) => {
|
|
47
43
|
if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
|
|
48
44
|
return Buffer.from(v.data);
|
|
49
45
|
}
|
|
50
46
|
return v;
|
|
51
47
|
};
|
|
52
|
-
exports.bufferReviver = bufferReviver;
|
|
@@ -1,21 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.transformToNDJson = transformToNDJson;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { _sortObjectDeep } from '@naturalcycles/js-lib';
|
|
6
3
|
/**
|
|
7
4
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
8
5
|
*/
|
|
9
|
-
function transformToNDJson(opt = {}) {
|
|
6
|
+
export function transformToNDJson(opt = {}) {
|
|
10
7
|
const { strict = true, separator = '\n', sortObjects = false } = opt;
|
|
11
|
-
return new
|
|
8
|
+
return new Transform({
|
|
12
9
|
writableObjectMode: true,
|
|
13
10
|
readableObjectMode: false,
|
|
14
11
|
readableHighWaterMark: 64 * 1024,
|
|
15
12
|
transform(chunk, _, cb) {
|
|
16
13
|
try {
|
|
17
14
|
if (sortObjects) {
|
|
18
|
-
chunk =
|
|
15
|
+
chunk = _sortObjectDeep(chunk);
|
|
19
16
|
}
|
|
20
17
|
cb(null, JSON.stringify(chunk) + separator);
|
|
21
18
|
}
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports._pipelineToArray = _pipelineToArray;
|
|
6
|
-
const node_stream_1 = require("node:stream");
|
|
7
|
-
const promises_1 = require("node:stream/promises");
|
|
8
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
9
|
-
const writablePushToArray_1 = require("../writable/writablePushToArray");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { pipeline } from 'node:stream/promises';
|
|
3
|
+
import { _last, pDefer } from '@naturalcycles/js-lib';
|
|
4
|
+
import { writablePushToArray } from '../writable/writablePushToArray.js';
|
|
10
5
|
/**
|
|
11
6
|
* Promisified `stream.pipeline`.
|
|
12
7
|
*
|
|
13
8
|
* Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
|
|
14
9
|
* without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
|
|
15
10
|
*/
|
|
16
|
-
async function _pipeline(streams, opt = {}) {
|
|
11
|
+
export async function _pipeline(streams, opt = {}) {
|
|
17
12
|
const first = streams[0];
|
|
18
13
|
const rest = streams.slice(1);
|
|
19
14
|
if (opt.allowClose) {
|
|
@@ -35,9 +30,9 @@ async function _pipeline(streams, opt = {}) {
|
|
|
35
30
|
// 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
|
|
36
31
|
// 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
|
|
37
32
|
// inevitable in this flow). Know a better way to close the stream? Tell me!
|
|
38
|
-
const streamDone =
|
|
33
|
+
const streamDone = pDefer();
|
|
39
34
|
const sourceReadable = first;
|
|
40
|
-
const last =
|
|
35
|
+
const last = _last(streams);
|
|
41
36
|
const lastFinal = last._final?.bind(last) || ((cb) => cb());
|
|
42
37
|
last._final = cb => {
|
|
43
38
|
lastFinal(() => {
|
|
@@ -56,7 +51,7 @@ async function _pipeline(streams, opt = {}) {
|
|
|
56
51
|
});
|
|
57
52
|
}
|
|
58
53
|
try {
|
|
59
|
-
return await
|
|
54
|
+
return await pipeline([first, ...rest], opt);
|
|
60
55
|
}
|
|
61
56
|
catch (err) {
|
|
62
57
|
if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
@@ -75,11 +70,12 @@ async function _pipeline(streams, opt = {}) {
|
|
|
75
70
|
* Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
|
|
76
71
|
* and return.
|
|
77
72
|
*/
|
|
78
|
-
async function _pipelineToArray(streams, opt = {}) {
|
|
73
|
+
export async function _pipelineToArray(streams, opt = {}) {
|
|
79
74
|
const a = [];
|
|
80
|
-
await _pipeline([...streams,
|
|
75
|
+
await _pipeline([...streams, writablePushToArray(a)], opt);
|
|
81
76
|
return a;
|
|
82
77
|
}
|
|
83
|
-
class AbortableTransform extends
|
|
78
|
+
export class AbortableTransform extends Transform {
|
|
79
|
+
sourceReadable;
|
|
80
|
+
streamDone;
|
|
84
81
|
}
|
|
85
|
-
exports.AbortableTransform = AbortableTransform;
|
|
@@ -1,17 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports.progressReadableMapper = progressReadableMapper;
|
|
6
|
-
const node_util_1 = require("node:util");
|
|
7
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
8
|
-
const colors_1 = require("../colors/colors");
|
|
9
|
-
const sizeStack_1 = require("./sizeStack");
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { _mb, _since, localTime, SimpleMovingAverage } from '@naturalcycles/js-lib';
|
|
3
|
+
import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
|
|
4
|
+
import { SizeStack } from './sizeStack.js';
|
|
10
5
|
const inspectOpt = {
|
|
11
|
-
colors:
|
|
6
|
+
colors: hasColors,
|
|
12
7
|
breakLength: 300,
|
|
13
8
|
};
|
|
14
|
-
class ProgressLogger {
|
|
9
|
+
export class ProgressLogger {
|
|
15
10
|
constructor(cfg = {}) {
|
|
16
11
|
this.cfg = {
|
|
17
12
|
metric: 'progress',
|
|
@@ -29,16 +24,26 @@ class ProgressLogger {
|
|
|
29
24
|
this.start();
|
|
30
25
|
this.logStats(); // initial
|
|
31
26
|
}
|
|
27
|
+
cfg;
|
|
28
|
+
started;
|
|
29
|
+
lastSecondStarted;
|
|
30
|
+
sma;
|
|
31
|
+
logEvery10;
|
|
32
|
+
processedLastSecond;
|
|
33
|
+
progress;
|
|
34
|
+
peakRSS;
|
|
35
|
+
sizes;
|
|
36
|
+
sizesZipped;
|
|
32
37
|
start() {
|
|
33
38
|
this.started = Date.now();
|
|
34
39
|
this.lastSecondStarted = Date.now();
|
|
35
|
-
this.sma = new
|
|
40
|
+
this.sma = new SimpleMovingAverage(10);
|
|
36
41
|
this.processedLastSecond = 0;
|
|
37
42
|
this.progress = 0;
|
|
38
43
|
this.peakRSS = 0;
|
|
39
|
-
this.sizes = this.cfg.logSizes ? new
|
|
44
|
+
this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
|
|
40
45
|
this.sizesZipped = this.cfg.logZippedSizes
|
|
41
|
-
? new
|
|
46
|
+
? new SizeStack('json.gz', this.cfg.logSizesBuffer)
|
|
42
47
|
: undefined;
|
|
43
48
|
}
|
|
44
49
|
log(chunk) {
|
|
@@ -46,7 +51,7 @@ class ProgressLogger {
|
|
|
46
51
|
this.processedLastSecond++;
|
|
47
52
|
if (this.sizes) {
|
|
48
53
|
// Check it, cause gzipping might be delayed here..
|
|
49
|
-
void
|
|
54
|
+
void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
|
|
50
55
|
}
|
|
51
56
|
if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
|
|
52
57
|
this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
|
|
@@ -78,22 +83,22 @@ class ProgressLogger {
|
|
|
78
83
|
if (extra)
|
|
79
84
|
Object.assign(o, extra(chunk, this.progress));
|
|
80
85
|
if (logHeapUsed)
|
|
81
|
-
o.heapUsed =
|
|
86
|
+
o.heapUsed = _mb(mem.heapUsed);
|
|
82
87
|
if (logHeapTotal)
|
|
83
|
-
o.heapTotal =
|
|
88
|
+
o.heapTotal = _mb(mem.heapTotal);
|
|
84
89
|
if (logRss)
|
|
85
|
-
o.rss =
|
|
90
|
+
o.rss = _mb(mem.rss);
|
|
86
91
|
if (logPeakRss)
|
|
87
|
-
o.peakRSS =
|
|
92
|
+
o.peakRSS = _mb(this.peakRSS);
|
|
88
93
|
if (rssMinusHeap)
|
|
89
|
-
o.rssMinusHeap =
|
|
94
|
+
o.rssMinusHeap = _mb(mem.rss - mem.heapTotal);
|
|
90
95
|
if (external)
|
|
91
|
-
o.external =
|
|
96
|
+
o.external = _mb(mem.external);
|
|
92
97
|
if (arrayBuffers)
|
|
93
|
-
o.arrayBuffers =
|
|
98
|
+
o.arrayBuffers = _mb(mem.arrayBuffers || 0);
|
|
94
99
|
if (logRPS)
|
|
95
100
|
Object.assign(o, { rps10, rpsTotal });
|
|
96
|
-
logger.log(
|
|
101
|
+
logger.log(inspect(o, inspectOpt));
|
|
97
102
|
if (this.sizes?.items.length) {
|
|
98
103
|
logger.log(this.sizes.getStats());
|
|
99
104
|
if (this.sizesZipped?.items.length) {
|
|
@@ -105,10 +110,10 @@ class ProgressLogger {
|
|
|
105
110
|
if (perHour > 900) {
|
|
106
111
|
perHour = Math.round(perHour / 1000) + 'K';
|
|
107
112
|
}
|
|
108
|
-
logger.log(`${
|
|
113
|
+
logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`);
|
|
109
114
|
}
|
|
110
115
|
else if (final) {
|
|
111
|
-
logger.log(`${
|
|
116
|
+
logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);
|
|
112
117
|
try {
|
|
113
118
|
this.cfg.onProgressDone?.(o);
|
|
114
119
|
}
|
|
@@ -118,11 +123,10 @@ class ProgressLogger {
|
|
|
118
123
|
}
|
|
119
124
|
}
|
|
120
125
|
}
|
|
121
|
-
exports.ProgressLogger = ProgressLogger;
|
|
122
126
|
/**
|
|
123
127
|
* Create new ProgressLogger.
|
|
124
128
|
*/
|
|
125
|
-
function progressLogger(cfg = {}) {
|
|
129
|
+
export function progressLogger(cfg = {}) {
|
|
126
130
|
return new ProgressLogger(cfg);
|
|
127
131
|
}
|
|
128
132
|
/**
|
|
@@ -130,7 +134,7 @@ function progressLogger(cfg = {}) {
|
|
|
130
134
|
*
|
|
131
135
|
* @experimental
|
|
132
136
|
*/
|
|
133
|
-
function progressReadableMapper(cfg = {}) {
|
|
137
|
+
export function progressReadableMapper(cfg = {}) {
|
|
134
138
|
const progress = new ProgressLogger(cfg);
|
|
135
139
|
return chunk => {
|
|
136
140
|
progress.log(chunk);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ReadableOptions } from 'node:stream';
|
|
2
|
-
import type { ReadableTyped } from '../stream.model';
|
|
2
|
+
import type { ReadableTyped } from '../stream.model.js';
|
|
3
3
|
/**
|
|
4
4
|
* Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
|
|
5
5
|
* Push `null` to it to complete (similar to RxJS `.complete()`).
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.readableCreate = readableCreate;
|
|
4
|
-
exports.readableFrom = readableFrom;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
6
2
|
/**
|
|
7
3
|
* Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
|
|
8
4
|
* Push `null` to it to complete (similar to RxJS `.complete()`).
|
|
@@ -17,8 +13,8 @@ const node_stream_1 = require("node:stream");
|
|
|
17
13
|
*
|
|
18
14
|
* @deprecated because of the caution above
|
|
19
15
|
*/
|
|
20
|
-
function readableCreate(items = [], opt) {
|
|
21
|
-
const readable = new
|
|
16
|
+
export function readableCreate(items = [], opt) {
|
|
17
|
+
const readable = new Readable({
|
|
22
18
|
objectMode: true,
|
|
23
19
|
...opt,
|
|
24
20
|
read() { }, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
|
|
@@ -31,6 +27,6 @@ function readableCreate(items = [], opt) {
|
|
|
31
27
|
/**
|
|
32
28
|
* Convenience type-safe wrapper around Readable.from() that infers the Type of input.
|
|
33
29
|
*/
|
|
34
|
-
function readableFrom(iterable, opt) {
|
|
35
|
-
return
|
|
30
|
+
export function readableFrom(iterable, opt) {
|
|
31
|
+
return Readable.from(iterable, opt);
|
|
36
32
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AbortableAsyncMapper, Mapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { ReadableTyped } from '../../index';
|
|
3
|
-
import type { TransformMapOptions } from '../transform/transformMap';
|
|
2
|
+
import type { ReadableTyped } from '../../index.js';
|
|
3
|
+
import type { TransformMapOptions } from '../transform/transformMap.js';
|
|
4
4
|
/**
|
|
5
5
|
* Convenience function to do `.forEach` over a Readable.
|
|
6
6
|
* Typed! (unlike default Readable).
|
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.readableForEachSync = readableForEachSync;
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
-
const index_1 = require("../../index");
|
|
7
|
-
const transformMap_1 = require("../transform/transformMap");
|
|
1
|
+
import { _passNothingPredicate } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline } from '../../index.js';
|
|
3
|
+
import { transformMap } from '../transform/transformMap.js';
|
|
8
4
|
/**
|
|
9
5
|
* Convenience function to do `.forEach` over a Readable.
|
|
10
6
|
* Typed! (unlike default Readable).
|
|
@@ -13,10 +9,10 @@ const transformMap_1 = require("../transform/transformMap");
|
|
|
13
9
|
*
|
|
14
10
|
* @experimental
|
|
15
11
|
*/
|
|
16
|
-
async function readableForEach(readable, mapper, opt = {}) {
|
|
17
|
-
await
|
|
12
|
+
export async function readableForEach(readable, mapper, opt = {}) {
|
|
13
|
+
await _pipeline([
|
|
18
14
|
readable,
|
|
19
|
-
|
|
15
|
+
transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
|
|
20
16
|
]);
|
|
21
17
|
}
|
|
22
18
|
/**
|
|
@@ -25,7 +21,7 @@ async function readableForEach(readable, mapper, opt = {}) {
|
|
|
25
21
|
*
|
|
26
22
|
* @experimental
|
|
27
23
|
*/
|
|
28
|
-
async function readableForEachSync(readable, mapper) {
|
|
24
|
+
export async function readableForEachSync(readable, mapper) {
|
|
29
25
|
// async iteration
|
|
30
26
|
let index = 0;
|
|
31
27
|
for await (const item of readable) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ReadableOptions } from 'node:stream';
|
|
2
2
|
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
3
|
-
import type { ReadableTyped } from '../stream.model';
|
|
3
|
+
import type { ReadableTyped } from '../stream.model.js';
|
|
4
4
|
/**
|
|
5
5
|
* Create Readable from Array.
|
|
6
6
|
* Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
|
|
@@ -1,17 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.readableFromArray = readableFromArray;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib';
|
|
6
3
|
/**
|
|
7
4
|
* Create Readable from Array.
|
|
8
5
|
* Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
|
|
9
6
|
*
|
|
10
7
|
* For simple cases use Readable.from(...) (Node.js 12+)
|
|
11
8
|
*/
|
|
12
|
-
function readableFromArray(items, mapper =
|
|
9
|
+
export function readableFromArray(items, mapper = _passthroughMapper, opt) {
|
|
13
10
|
let i = -1;
|
|
14
|
-
return new
|
|
11
|
+
return new Readable({
|
|
15
12
|
objectMode: true,
|
|
16
13
|
...opt,
|
|
17
14
|
async read() {
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.readableToArray = readableToArray;
|
|
4
1
|
/**
|
|
5
2
|
* Convenience function to read the whole Readable stream into Array (in-memory)
|
|
6
3
|
* and return that array.
|
|
@@ -8,7 +5,7 @@ exports.readableToArray = readableToArray;
|
|
|
8
5
|
* Native `await readable.toArray()` can be used instead.
|
|
9
6
|
* This helper is kept for type-safery support.
|
|
10
7
|
*/
|
|
11
|
-
async function readableToArray(readable) {
|
|
8
|
+
export async function readableToArray(readable) {
|
|
12
9
|
return await readable.toArray();
|
|
13
10
|
// const a: T[] = []
|
|
14
11
|
//
|
package/dist/stream/sizeStack.js
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const zip_util_1 = require("../util/zip.util");
|
|
7
|
-
class SizeStack extends js_lib_1.NumberStack {
|
|
1
|
+
import { _hb, NumberStack } from '@naturalcycles/js-lib';
|
|
2
|
+
import { yellow } from '../colors/colors.js';
|
|
3
|
+
import { gzipBuffer } from '../util/zip.util.js';
|
|
4
|
+
export class SizeStack extends NumberStack {
|
|
5
|
+
name;
|
|
8
6
|
constructor(name, size) {
|
|
9
7
|
super(size);
|
|
10
8
|
this.name = name;
|
|
11
|
-
this.total = 0;
|
|
12
9
|
}
|
|
10
|
+
total = 0;
|
|
13
11
|
push(item) {
|
|
14
12
|
this.total += item;
|
|
15
13
|
return super.push(item);
|
|
@@ -19,13 +17,13 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
19
17
|
return [
|
|
20
18
|
' ' + this.name,
|
|
21
19
|
'avg',
|
|
22
|
-
|
|
20
|
+
yellow(_hb(this.avg())),
|
|
23
21
|
// 'p50',
|
|
24
22
|
// yellow(_hb(pcs[50])),
|
|
25
23
|
// 'p90',
|
|
26
24
|
// yellow(_hb(pcs[90])),
|
|
27
25
|
'total',
|
|
28
|
-
|
|
26
|
+
yellow(_hb(this.total)),
|
|
29
27
|
].join(' ');
|
|
30
28
|
}
|
|
31
29
|
static async countItem(item, logger, sizes, sizesZipped) {
|
|
@@ -36,7 +34,7 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
36
34
|
const buf = Buffer.from(JSON.stringify(item));
|
|
37
35
|
sizes.push(buf.byteLength);
|
|
38
36
|
if (sizesZipped) {
|
|
39
|
-
const { byteLength } = await
|
|
37
|
+
const { byteLength } = await gzipBuffer(buf);
|
|
40
38
|
sizesZipped.push(byteLength);
|
|
41
39
|
}
|
|
42
40
|
}
|
|
@@ -45,4 +43,3 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
45
43
|
}
|
|
46
44
|
}
|
|
47
45
|
}
|
|
48
|
-
exports.SizeStack = SizeStack;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.pipelineClose = pipelineClose;
|
|
4
|
-
function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
|
|
1
|
+
export function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
|
|
5
2
|
readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
|
|
6
3
|
if (!sourceReadable) {
|
|
7
4
|
logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
|
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.transformChunk = transformChunk;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
5
2
|
/**
|
|
6
3
|
* Similar to RxJS bufferCount(),
|
|
7
4
|
* allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
|
|
8
5
|
* Last chunk will contain the remaining items, possibly less than chunkSize.
|
|
9
6
|
*/
|
|
10
|
-
function transformChunk(opt) {
|
|
7
|
+
export function transformChunk(opt) {
|
|
11
8
|
const { chunkSize } = opt;
|
|
12
9
|
let buf = [];
|
|
13
|
-
return new
|
|
10
|
+
return new Transform({
|
|
14
11
|
objectMode: true,
|
|
15
12
|
...opt,
|
|
16
13
|
transform(chunk, _, cb) {
|