@naturalcycles/nodejs-lib 13.45.4 → 14.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/generate-build-info.js +15 -18
- package/dist/bin/json2env.js +6 -9
- package/dist/bin/kpy.js +6 -9
- package/dist/bin/secrets-decrypt.d.ts +1 -1
- package/dist/bin/secrets-decrypt.js +9 -12
- package/dist/bin/secrets-encrypt.d.ts +1 -1
- package/dist/bin/secrets-encrypt.js +9 -12
- package/dist/bin/secrets-gen-key.js +8 -11
- package/dist/bin/slack-this.js +6 -9
- package/dist/buffer/buffer.util.js +1 -4
- package/dist/colors/colors.d.ts +26 -27
- package/dist/colors/colors.js +29 -33
- package/dist/csv/csvReader.js +4 -8
- package/dist/csv/csvWriter.js +6 -12
- package/dist/csv/transformToCSV.d.ts +2 -2
- package/dist/csv/transformToCSV.js +5 -8
- package/dist/diff/tableDiff.js +4 -7
- package/dist/fs/fs2.d.ts +1 -1
- package/dist/fs/fs2.js +82 -86
- package/dist/fs/json2env.js +18 -28
- package/dist/fs/kpy.js +28 -33
- package/dist/index.d.ts +71 -71
- package/dist/index.js +73 -78
- package/dist/infra/process.util.js +22 -28
- package/dist/jwt/jwt.service.js +16 -21
- package/dist/log/log.util.js +4 -7
- package/dist/script/runScript.js +11 -15
- package/dist/secret/secrets-decrypt.util.js +18 -22
- package/dist/secret/secrets-encrypt.util.js +17 -21
- package/dist/security/crypto.util.js +24 -34
- package/dist/security/hash.util.js +19 -37
- package/dist/security/id.util.js +11 -18
- package/dist/security/nanoid.js +15 -20
- package/dist/security/secret.util.js +20 -30
- package/dist/slack/slack.service.d.ts +1 -1
- package/dist/slack/slack.service.js +11 -16
- package/dist/slack/slack.service.model.d.ts +1 -1
- package/dist/slack/slack.service.model.js +1 -2
- package/dist/stream/ndjson/ndjson.model.js +7 -11
- package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonMap.js +11 -14
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformJsonParse.js +4 -9
- package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +5 -8
- package/dist/stream/pipeline/pipeline.js +11 -17
- package/dist/stream/progressLogger.d.ts +1 -1
- package/dist/stream/progressLogger.js +22 -28
- package/dist/stream/readable/readableCreate.d.ts +1 -1
- package/dist/stream/readable/readableCreate.js +5 -9
- package/dist/stream/readable/readableForEach.d.ts +2 -2
- package/dist/stream/readable/readableForEach.js +7 -11
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +4 -7
- package/dist/stream/readable/readableToArray.d.ts +1 -1
- package/dist/stream/readable/readableToArray.js +1 -4
- package/dist/stream/sizeStack.js +7 -11
- package/dist/stream/stream.model.js +1 -2
- package/dist/stream/stream.util.js +1 -4
- package/dist/stream/transform/transformChunk.d.ts +1 -1
- package/dist/stream/transform/transformChunk.js +3 -6
- package/dist/stream/transform/transformFilter.d.ts +2 -2
- package/dist/stream/transform/transformFilter.js +6 -10
- package/dist/stream/transform/transformLimit.d.ts +2 -2
- package/dist/stream/transform/transformLimit.js +6 -11
- package/dist/stream/transform/transformLogProgress.d.ts +2 -2
- package/dist/stream/transform/transformLogProgress.js +5 -8
- package/dist/stream/transform/transformMap.d.ts +1 -1
- package/dist/stream/transform/transformMap.js +18 -22
- package/dist/stream/transform/transformMapSimple.d.ts +1 -1
- package/dist/stream/transform/transformMapSimple.js +6 -9
- package/dist/stream/transform/transformMapSync.d.ts +3 -3
- package/dist/stream/transform/transformMapSync.js +14 -19
- package/dist/stream/transform/transformNoOp.d.ts +1 -1
- package/dist/stream/transform/transformNoOp.js +3 -6
- package/dist/stream/transform/transformOffset.d.ts +1 -1
- package/dist/stream/transform/transformOffset.js +5 -8
- package/dist/stream/transform/transformSplit.d.ts +1 -1
- package/dist/stream/transform/transformSplit.js +5 -9
- package/dist/stream/transform/transformTap.d.ts +1 -1
- package/dist/stream/transform/transformTap.js +3 -6
- package/dist/stream/transform/transformTee.d.ts +1 -1
- package/dist/stream/transform/transformTee.js +7 -10
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +9 -12
- package/dist/stream/transform/transformToArray.d.ts +1 -1
- package/dist/stream/transform/transformToArray.js +3 -6
- package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
- package/dist/stream/transform/worker/baseWorkerClass.js +1 -5
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
- package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
- package/dist/stream/transform/worker/workerClassProxy.js +6 -4
- package/dist/stream/writable/writableForEach.d.ts +2 -2
- package/dist/stream/writable/writableForEach.js +6 -10
- package/dist/stream/writable/writableFork.d.ts +1 -1
- package/dist/stream/writable/writableFork.js +6 -9
- package/dist/stream/writable/writableLimit.d.ts +1 -1
- package/dist/stream/writable/writableLimit.js +3 -6
- package/dist/stream/writable/writablePushToArray.d.ts +1 -1
- package/dist/stream/writable/writablePushToArray.js +3 -6
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/stream/writable/writableVoid.js +3 -6
- package/dist/string/inspect.js +8 -13
- package/dist/util/buildInfo.util.js +12 -15
- package/dist/util/env.util.js +5 -9
- package/dist/util/exec2.js +17 -21
- package/dist/util/git2.js +17 -20
- package/dist/util/lruMemoCache.js +3 -7
- package/dist/util/zip.util.js +14 -25
- package/dist/validation/ajv/ajv.util.d.ts +3 -3
- package/dist/validation/ajv/ajv.util.js +6 -10
- package/dist/validation/ajv/ajvSchema.d.ts +2 -2
- package/dist/validation/ajv/ajvSchema.js +14 -18
- package/dist/validation/ajv/ajvValidationError.js +2 -6
- package/dist/validation/ajv/getAjv.d.ts +1 -1
- package/dist/validation/ajv/getAjv.js +9 -9
- package/dist/validation/joi/joi.extensions.d.ts +2 -2
- package/dist/validation/joi/joi.extensions.js +6 -10
- package/dist/validation/joi/joi.model.js +1 -2
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.shared.schemas.js +61 -74
- package/dist/validation/joi/joi.validation.error.js +2 -6
- package/dist/validation/joi/joi.validation.util.d.ts +1 -1
- package/dist/validation/joi/joi.validation.util.js +10 -17
- package/dist/validation/joi/number.extensions.js +1 -4
- package/dist/validation/joi/string.extensions.js +4 -7
- package/dist/yargs.util.js +5 -8
- package/package.json +5 -4
- package/src/bin/generate-build-info.ts +3 -3
- package/src/bin/json2env.ts +2 -2
- package/src/bin/kpy.ts +2 -2
- package/src/bin/secrets-decrypt.ts +7 -8
- package/src/bin/secrets-encrypt.ts +7 -8
- package/src/bin/secrets-gen-key.ts +2 -2
- package/src/bin/slack-this.ts +2 -2
- package/src/csv/transformToCSV.ts +3 -3
- package/src/fs/fs2.ts +4 -4
- package/src/fs/json2env.ts +2 -2
- package/src/fs/kpy.ts +2 -2
- package/src/index.ts +71 -71
- package/src/jwt/jwt.service.ts +2 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +4 -3
- package/src/secret/secrets-decrypt.util.ts +3 -3
- package/src/secret/secrets-encrypt.util.ts +3 -3
- package/src/security/crypto.util.ts +1 -1
- package/src/security/id.util.ts +1 -1
- package/src/security/secret.util.ts +2 -2
- package/src/slack/slack.service.model.ts +1 -1
- package/src/slack/slack.service.ts +3 -3
- package/src/stream/ndjson/ndjson.model.ts +1 -1
- package/src/stream/ndjson/ndjsonMap.ts +2 -2
- package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +1 -1
- package/src/stream/progressLogger.ts +3 -3
- package/src/stream/readable/readableCreate.ts +1 -1
- package/src/stream/readable/readableForEach.ts +4 -4
- package/src/stream/readable/readableFromArray.ts +1 -1
- package/src/stream/readable/readableToArray.ts +1 -1
- package/src/stream/sizeStack.ts +2 -2
- package/src/stream/transform/transformChunk.ts +1 -1
- package/src/stream/transform/transformFilter.ts +3 -3
- package/src/stream/transform/transformLimit.ts +3 -3
- package/src/stream/transform/transformLogProgress.ts +3 -3
- package/src/stream/transform/transformMap.ts +5 -5
- package/src/stream/transform/transformMapSimple.ts +1 -1
- package/src/stream/transform/transformMapSync.ts +5 -5
- package/src/stream/transform/transformNoOp.ts +1 -1
- package/src/stream/transform/transformOffset.ts +2 -2
- package/src/stream/transform/transformSplit.ts +1 -1
- package/src/stream/transform/transformTap.ts +1 -1
- package/src/stream/transform/transformTee.ts +3 -3
- package/src/stream/transform/transformThrottle.ts +1 -1
- package/src/stream/transform/transformToArray.ts +1 -1
- package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
- package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
- package/src/stream/transform/worker/workerClassProxy.js +6 -4
- package/src/stream/writable/writableForEach.ts +3 -3
- package/src/stream/writable/writableFork.ts +3 -3
- package/src/stream/writable/writableLimit.ts +1 -1
- package/src/stream/writable/writablePushToArray.ts +1 -1
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/util/buildInfo.util.ts +2 -2
- package/src/util/exec2.ts +1 -1
- package/src/util/git2.ts +2 -2
- package/src/validation/ajv/ajv.util.ts +4 -4
- package/src/validation/ajv/ajvSchema.ts +4 -5
- package/src/validation/ajv/getAjv.ts +7 -3
- package/src/validation/joi/joi.extensions.ts +4 -4
- package/src/validation/joi/joi.shared.schemas.ts +3 -3
- package/src/validation/joi/joi.validation.util.ts +2 -2
- package/src/yargs.util.ts +1 -0
|
@@ -1,31 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.ndjsonMap = ndjsonMap;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const __1 = require("../..");
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline, fs2, transformLimit, transformLogProgress, transformMap } from '../../index.js';
|
|
6
3
|
/**
|
|
7
4
|
* Unzips input file automatically, if it ends with `.gz`.
|
|
8
5
|
* Zips output file automatically, if it ends with `.gz`.
|
|
9
6
|
*/
|
|
10
|
-
async function ndjsonMap(mapper, opt) {
|
|
7
|
+
export async function ndjsonMap(mapper, opt) {
|
|
11
8
|
const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;
|
|
12
9
|
console.log({
|
|
13
10
|
inputFilePath,
|
|
14
11
|
outputFilePath,
|
|
15
12
|
});
|
|
16
|
-
const readable =
|
|
13
|
+
const readable = fs2
|
|
17
14
|
.createReadStreamAsNDJSON(inputFilePath)
|
|
18
15
|
.take(limitInput || Number.POSITIVE_INFINITY);
|
|
19
|
-
await
|
|
16
|
+
await _pipeline([
|
|
20
17
|
readable,
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
transformLogProgress({ metric: 'read', ...opt }),
|
|
19
|
+
transformMap(mapper, {
|
|
23
20
|
flattenArrayOutput: true,
|
|
24
|
-
errorMode:
|
|
21
|
+
errorMode: ErrorMode.SUPPRESS,
|
|
25
22
|
...opt,
|
|
26
23
|
}),
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
...
|
|
24
|
+
transformLimit({ limit: limitOutput, sourceReadable: readable }),
|
|
25
|
+
transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
|
|
26
|
+
...fs2.createWriteStreamAsNDJSON(outputFilePath),
|
|
30
27
|
]);
|
|
31
28
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformLogProgressOptions, TransformMapOptions } from '
|
|
2
|
+
import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
|
|
3
3
|
export interface NDJSONStreamForEachOptions<IN = any> extends TransformMapOptions<IN, void>, TransformLogProgressOptions<IN> {
|
|
4
4
|
inputFilePath: string;
|
|
5
5
|
}
|
|
@@ -1,20 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.ndjsonStreamForEach = ndjsonStreamForEach;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const __1 = require("../..");
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline, fs2, transformLogProgress, transformMap, writableVoid } from '../../index.js';
|
|
6
3
|
/**
|
|
7
4
|
* Convenience function to `forEach` through an ndjson file.
|
|
8
5
|
*/
|
|
9
|
-
async function ndjsonStreamForEach(mapper, opt) {
|
|
10
|
-
await
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
errorMode:
|
|
6
|
+
export async function ndjsonStreamForEach(mapper, opt) {
|
|
7
|
+
await _pipeline([
|
|
8
|
+
fs2.createReadStreamAsNDJSON(opt.inputFilePath),
|
|
9
|
+
transformMap(mapper, {
|
|
10
|
+
errorMode: ErrorMode.THROW_AGGREGATED,
|
|
14
11
|
...opt,
|
|
15
12
|
predicate: () => true, // to log progress properly
|
|
16
13
|
}),
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
transformLogProgress(opt),
|
|
15
|
+
writableVoid(),
|
|
19
16
|
]);
|
|
20
17
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Reviver } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformTyped } from '../stream.model';
|
|
2
|
+
import type { TransformTyped } from '../stream.model.js';
|
|
3
3
|
export interface TransformJsonParseOptions {
|
|
4
4
|
/**
|
|
5
5
|
* If true - will throw an error on JSON.parse / stringify error
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.bufferReviver = void 0;
|
|
4
|
-
exports.transformJsonParse = transformJsonParse;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
6
2
|
/**
|
|
7
3
|
* Transforms chunks of JSON strings/Buffers (objectMode=false) into parsed objects (readableObjectMode=true).
|
|
8
4
|
*
|
|
@@ -17,9 +13,9 @@ const node_stream_1 = require("node:stream");
|
|
|
17
13
|
* consumeYourStream...
|
|
18
14
|
* [)
|
|
19
15
|
*/
|
|
20
|
-
function transformJsonParse(opt = {}) {
|
|
16
|
+
export function transformJsonParse(opt = {}) {
|
|
21
17
|
const { strict = true, reviver } = opt;
|
|
22
|
-
return new
|
|
18
|
+
return new Transform({
|
|
23
19
|
writableObjectMode: false,
|
|
24
20
|
readableObjectMode: true,
|
|
25
21
|
// highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
|
|
@@ -43,10 +39,9 @@ function transformJsonParse(opt = {}) {
|
|
|
43
39
|
});
|
|
44
40
|
}
|
|
45
41
|
// Based on: https://stackoverflow.com/a/34557997/4919972
|
|
46
|
-
const bufferReviver = (_k, v) => {
|
|
42
|
+
export const bufferReviver = (_k, v) => {
|
|
47
43
|
if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
|
|
48
44
|
return Buffer.from(v.data);
|
|
49
45
|
}
|
|
50
46
|
return v;
|
|
51
47
|
};
|
|
52
|
-
exports.bufferReviver = bufferReviver;
|
|
@@ -1,21 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.transformToNDJson = transformToNDJson;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { _sortObjectDeep } from '@naturalcycles/js-lib';
|
|
6
3
|
/**
|
|
7
4
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
8
5
|
*/
|
|
9
|
-
function transformToNDJson(opt = {}) {
|
|
6
|
+
export function transformToNDJson(opt = {}) {
|
|
10
7
|
const { strict = true, separator = '\n', sortObjects = false } = opt;
|
|
11
|
-
return new
|
|
8
|
+
return new Transform({
|
|
12
9
|
writableObjectMode: true,
|
|
13
10
|
readableObjectMode: false,
|
|
14
11
|
readableHighWaterMark: 64 * 1024,
|
|
15
12
|
transform(chunk, _, cb) {
|
|
16
13
|
try {
|
|
17
14
|
if (sortObjects) {
|
|
18
|
-
chunk =
|
|
15
|
+
chunk = _sortObjectDeep(chunk);
|
|
19
16
|
}
|
|
20
17
|
cb(null, JSON.stringify(chunk) + separator);
|
|
21
18
|
}
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports._pipelineToArray = _pipelineToArray;
|
|
6
|
-
const node_stream_1 = require("node:stream");
|
|
7
|
-
const promises_1 = require("node:stream/promises");
|
|
8
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
9
|
-
const writablePushToArray_1 = require("../writable/writablePushToArray");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { pipeline } from 'node:stream/promises';
|
|
3
|
+
import { _last, pDefer } from '@naturalcycles/js-lib';
|
|
4
|
+
import { writablePushToArray } from '../writable/writablePushToArray.js';
|
|
10
5
|
/**
|
|
11
6
|
* Promisified `stream.pipeline`.
|
|
12
7
|
*
|
|
13
8
|
* Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
|
|
14
9
|
* without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
|
|
15
10
|
*/
|
|
16
|
-
async function _pipeline(streams, opt = {}) {
|
|
11
|
+
export async function _pipeline(streams, opt = {}) {
|
|
17
12
|
const first = streams[0];
|
|
18
13
|
const rest = streams.slice(1);
|
|
19
14
|
if (opt.allowClose) {
|
|
@@ -35,9 +30,9 @@ async function _pipeline(streams, opt = {}) {
|
|
|
35
30
|
// 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
|
|
36
31
|
// 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
|
|
37
32
|
// inevitable in this flow). Know a better way to close the stream? Tell me!
|
|
38
|
-
const streamDone =
|
|
33
|
+
const streamDone = pDefer();
|
|
39
34
|
const sourceReadable = first;
|
|
40
|
-
const last =
|
|
35
|
+
const last = _last(streams);
|
|
41
36
|
const lastFinal = last._final?.bind(last) || ((cb) => cb());
|
|
42
37
|
last._final = cb => {
|
|
43
38
|
lastFinal(() => {
|
|
@@ -56,7 +51,7 @@ async function _pipeline(streams, opt = {}) {
|
|
|
56
51
|
});
|
|
57
52
|
}
|
|
58
53
|
try {
|
|
59
|
-
return await
|
|
54
|
+
return await pipeline([first, ...rest], opt);
|
|
60
55
|
}
|
|
61
56
|
catch (err) {
|
|
62
57
|
if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
@@ -75,13 +70,12 @@ async function _pipeline(streams, opt = {}) {
|
|
|
75
70
|
* Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
|
|
76
71
|
* and return.
|
|
77
72
|
*/
|
|
78
|
-
async function _pipelineToArray(streams, opt = {}) {
|
|
73
|
+
export async function _pipelineToArray(streams, opt = {}) {
|
|
79
74
|
const a = [];
|
|
80
|
-
await _pipeline([...streams,
|
|
75
|
+
await _pipeline([...streams, writablePushToArray(a)], opt);
|
|
81
76
|
return a;
|
|
82
77
|
}
|
|
83
|
-
class AbortableTransform extends
|
|
78
|
+
export class AbortableTransform extends Transform {
|
|
84
79
|
sourceReadable;
|
|
85
80
|
streamDone;
|
|
86
81
|
}
|
|
87
|
-
exports.AbortableTransform = AbortableTransform;
|
|
@@ -1,17 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports.progressReadableMapper = progressReadableMapper;
|
|
6
|
-
const node_util_1 = require("node:util");
|
|
7
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
8
|
-
const colors_1 = require("../colors/colors");
|
|
9
|
-
const sizeStack_1 = require("./sizeStack");
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
import { _mb, _since, localTime, SimpleMovingAverage } from '@naturalcycles/js-lib';
|
|
3
|
+
import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
|
|
4
|
+
import { SizeStack } from './sizeStack.js';
|
|
10
5
|
const inspectOpt = {
|
|
11
|
-
colors:
|
|
6
|
+
colors: hasColors,
|
|
12
7
|
breakLength: 300,
|
|
13
8
|
};
|
|
14
|
-
class ProgressLogger {
|
|
9
|
+
export class ProgressLogger {
|
|
15
10
|
constructor(cfg = {}) {
|
|
16
11
|
this.cfg = {
|
|
17
12
|
metric: 'progress',
|
|
@@ -42,13 +37,13 @@ class ProgressLogger {
|
|
|
42
37
|
start() {
|
|
43
38
|
this.started = Date.now();
|
|
44
39
|
this.lastSecondStarted = Date.now();
|
|
45
|
-
this.sma = new
|
|
40
|
+
this.sma = new SimpleMovingAverage(10);
|
|
46
41
|
this.processedLastSecond = 0;
|
|
47
42
|
this.progress = 0;
|
|
48
43
|
this.peakRSS = 0;
|
|
49
|
-
this.sizes = this.cfg.logSizes ? new
|
|
44
|
+
this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
|
|
50
45
|
this.sizesZipped = this.cfg.logZippedSizes
|
|
51
|
-
? new
|
|
46
|
+
? new SizeStack('json.gz', this.cfg.logSizesBuffer)
|
|
52
47
|
: undefined;
|
|
53
48
|
}
|
|
54
49
|
log(chunk) {
|
|
@@ -56,7 +51,7 @@ class ProgressLogger {
|
|
|
56
51
|
this.processedLastSecond++;
|
|
57
52
|
if (this.sizes) {
|
|
58
53
|
// Check it, cause gzipping might be delayed here..
|
|
59
|
-
void
|
|
54
|
+
void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
|
|
60
55
|
}
|
|
61
56
|
if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
|
|
62
57
|
this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
|
|
@@ -88,22 +83,22 @@ class ProgressLogger {
|
|
|
88
83
|
if (extra)
|
|
89
84
|
Object.assign(o, extra(chunk, this.progress));
|
|
90
85
|
if (logHeapUsed)
|
|
91
|
-
o.heapUsed =
|
|
86
|
+
o.heapUsed = _mb(mem.heapUsed);
|
|
92
87
|
if (logHeapTotal)
|
|
93
|
-
o.heapTotal =
|
|
88
|
+
o.heapTotal = _mb(mem.heapTotal);
|
|
94
89
|
if (logRss)
|
|
95
|
-
o.rss =
|
|
90
|
+
o.rss = _mb(mem.rss);
|
|
96
91
|
if (logPeakRss)
|
|
97
|
-
o.peakRSS =
|
|
92
|
+
o.peakRSS = _mb(this.peakRSS);
|
|
98
93
|
if (rssMinusHeap)
|
|
99
|
-
o.rssMinusHeap =
|
|
94
|
+
o.rssMinusHeap = _mb(mem.rss - mem.heapTotal);
|
|
100
95
|
if (external)
|
|
101
|
-
o.external =
|
|
96
|
+
o.external = _mb(mem.external);
|
|
102
97
|
if (arrayBuffers)
|
|
103
|
-
o.arrayBuffers =
|
|
98
|
+
o.arrayBuffers = _mb(mem.arrayBuffers || 0);
|
|
104
99
|
if (logRPS)
|
|
105
100
|
Object.assign(o, { rps10, rpsTotal });
|
|
106
|
-
logger.log(
|
|
101
|
+
logger.log(inspect(o, inspectOpt));
|
|
107
102
|
if (this.sizes?.items.length) {
|
|
108
103
|
logger.log(this.sizes.getStats());
|
|
109
104
|
if (this.sizesZipped?.items.length) {
|
|
@@ -115,10 +110,10 @@ class ProgressLogger {
|
|
|
115
110
|
if (perHour > 900) {
|
|
116
111
|
perHour = Math.round(perHour / 1000) + 'K';
|
|
117
112
|
}
|
|
118
|
-
logger.log(`${
|
|
113
|
+
logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`);
|
|
119
114
|
}
|
|
120
115
|
else if (final) {
|
|
121
|
-
logger.log(`${
|
|
116
|
+
logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);
|
|
122
117
|
try {
|
|
123
118
|
this.cfg.onProgressDone?.(o);
|
|
124
119
|
}
|
|
@@ -128,11 +123,10 @@ class ProgressLogger {
|
|
|
128
123
|
}
|
|
129
124
|
}
|
|
130
125
|
}
|
|
131
|
-
exports.ProgressLogger = ProgressLogger;
|
|
132
126
|
/**
|
|
133
127
|
* Create new ProgressLogger.
|
|
134
128
|
*/
|
|
135
|
-
function progressLogger(cfg = {}) {
|
|
129
|
+
export function progressLogger(cfg = {}) {
|
|
136
130
|
return new ProgressLogger(cfg);
|
|
137
131
|
}
|
|
138
132
|
/**
|
|
@@ -140,7 +134,7 @@ function progressLogger(cfg = {}) {
|
|
|
140
134
|
*
|
|
141
135
|
* @experimental
|
|
142
136
|
*/
|
|
143
|
-
function progressReadableMapper(cfg = {}) {
|
|
137
|
+
export function progressReadableMapper(cfg = {}) {
|
|
144
138
|
const progress = new ProgressLogger(cfg);
|
|
145
139
|
return chunk => {
|
|
146
140
|
progress.log(chunk);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ReadableOptions } from 'node:stream';
|
|
2
|
-
import type { ReadableTyped } from '../stream.model';
|
|
2
|
+
import type { ReadableTyped } from '../stream.model.js';
|
|
3
3
|
/**
|
|
4
4
|
* Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
|
|
5
5
|
* Push `null` to it to complete (similar to RxJS `.complete()`).
|
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.readableCreate = readableCreate;
|
|
4
|
-
exports.readableFrom = readableFrom;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
6
2
|
/**
|
|
7
3
|
* Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
|
|
8
4
|
* Push `null` to it to complete (similar to RxJS `.complete()`).
|
|
@@ -17,8 +13,8 @@ const node_stream_1 = require("node:stream");
|
|
|
17
13
|
*
|
|
18
14
|
* @deprecated because of the caution above
|
|
19
15
|
*/
|
|
20
|
-
function readableCreate(items = [], opt) {
|
|
21
|
-
const readable = new
|
|
16
|
+
export function readableCreate(items = [], opt) {
|
|
17
|
+
const readable = new Readable({
|
|
22
18
|
objectMode: true,
|
|
23
19
|
...opt,
|
|
24
20
|
read() { }, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
|
|
@@ -31,6 +27,6 @@ function readableCreate(items = [], opt) {
|
|
|
31
27
|
/**
|
|
32
28
|
* Convenience type-safe wrapper around Readable.from() that infers the Type of input.
|
|
33
29
|
*/
|
|
34
|
-
function readableFrom(iterable, opt) {
|
|
35
|
-
return
|
|
30
|
+
export function readableFrom(iterable, opt) {
|
|
31
|
+
return Readable.from(iterable, opt);
|
|
36
32
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AbortableAsyncMapper, Mapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { ReadableTyped } from '../../index';
|
|
3
|
-
import type { TransformMapOptions } from '../transform/transformMap';
|
|
2
|
+
import type { ReadableTyped } from '../../index.js';
|
|
3
|
+
import type { TransformMapOptions } from '../transform/transformMap.js';
|
|
4
4
|
/**
|
|
5
5
|
* Convenience function to do `.forEach` over a Readable.
|
|
6
6
|
* Typed! (unlike default Readable).
|
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.readableForEachSync = readableForEachSync;
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
-
const index_1 = require("../../index");
|
|
7
|
-
const transformMap_1 = require("../transform/transformMap");
|
|
1
|
+
import { _passNothingPredicate } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _pipeline } from '../../index.js';
|
|
3
|
+
import { transformMap } from '../transform/transformMap.js';
|
|
8
4
|
/**
|
|
9
5
|
* Convenience function to do `.forEach` over a Readable.
|
|
10
6
|
* Typed! (unlike default Readable).
|
|
@@ -13,10 +9,10 @@ const transformMap_1 = require("../transform/transformMap");
|
|
|
13
9
|
*
|
|
14
10
|
* @experimental
|
|
15
11
|
*/
|
|
16
|
-
async function readableForEach(readable, mapper, opt = {}) {
|
|
17
|
-
await
|
|
12
|
+
export async function readableForEach(readable, mapper, opt = {}) {
|
|
13
|
+
await _pipeline([
|
|
18
14
|
readable,
|
|
19
|
-
|
|
15
|
+
transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
|
|
20
16
|
]);
|
|
21
17
|
}
|
|
22
18
|
/**
|
|
@@ -25,7 +21,7 @@ async function readableForEach(readable, mapper, opt = {}) {
|
|
|
25
21
|
*
|
|
26
22
|
* @experimental
|
|
27
23
|
*/
|
|
28
|
-
async function readableForEachSync(readable, mapper) {
|
|
24
|
+
export async function readableForEachSync(readable, mapper) {
|
|
29
25
|
// async iteration
|
|
30
26
|
let index = 0;
|
|
31
27
|
for await (const item of readable) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ReadableOptions } from 'node:stream';
|
|
2
2
|
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
3
|
-
import type { ReadableTyped } from '../stream.model';
|
|
3
|
+
import type { ReadableTyped } from '../stream.model.js';
|
|
4
4
|
/**
|
|
5
5
|
* Create Readable from Array.
|
|
6
6
|
* Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
|
|
@@ -1,17 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.readableFromArray = readableFromArray;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib';
|
|
6
3
|
/**
|
|
7
4
|
* Create Readable from Array.
|
|
8
5
|
* Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
|
|
9
6
|
*
|
|
10
7
|
* For simple cases use Readable.from(...) (Node.js 12+)
|
|
11
8
|
*/
|
|
12
|
-
function readableFromArray(items, mapper =
|
|
9
|
+
export function readableFromArray(items, mapper = _passthroughMapper, opt) {
|
|
13
10
|
let i = -1;
|
|
14
|
-
return new
|
|
11
|
+
return new Readable({
|
|
15
12
|
objectMode: true,
|
|
16
13
|
...opt,
|
|
17
14
|
async read() {
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.readableToArray = readableToArray;
|
|
4
1
|
/**
|
|
5
2
|
* Convenience function to read the whole Readable stream into Array (in-memory)
|
|
6
3
|
* and return that array.
|
|
@@ -8,7 +5,7 @@ exports.readableToArray = readableToArray;
|
|
|
8
5
|
* Native `await readable.toArray()` can be used instead.
|
|
9
6
|
* This helper is kept for type-safery support.
|
|
10
7
|
*/
|
|
11
|
-
async function readableToArray(readable) {
|
|
8
|
+
export async function readableToArray(readable) {
|
|
12
9
|
return await readable.toArray();
|
|
13
10
|
// const a: T[] = []
|
|
14
11
|
//
|
package/dist/stream/sizeStack.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const colors_1 = require("../colors/colors");
|
|
6
|
-
const zip_util_1 = require("../util/zip.util");
|
|
7
|
-
class SizeStack extends js_lib_1.NumberStack {
|
|
1
|
+
import { _hb, NumberStack } from '@naturalcycles/js-lib';
|
|
2
|
+
import { yellow } from '../colors/colors.js';
|
|
3
|
+
import { gzipBuffer } from '../util/zip.util.js';
|
|
4
|
+
export class SizeStack extends NumberStack {
|
|
8
5
|
name;
|
|
9
6
|
constructor(name, size) {
|
|
10
7
|
super(size);
|
|
@@ -20,13 +17,13 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
20
17
|
return [
|
|
21
18
|
' ' + this.name,
|
|
22
19
|
'avg',
|
|
23
|
-
|
|
20
|
+
yellow(_hb(this.avg())),
|
|
24
21
|
// 'p50',
|
|
25
22
|
// yellow(_hb(pcs[50])),
|
|
26
23
|
// 'p90',
|
|
27
24
|
// yellow(_hb(pcs[90])),
|
|
28
25
|
'total',
|
|
29
|
-
|
|
26
|
+
yellow(_hb(this.total)),
|
|
30
27
|
].join(' ');
|
|
31
28
|
}
|
|
32
29
|
static async countItem(item, logger, sizes, sizesZipped) {
|
|
@@ -37,7 +34,7 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
37
34
|
const buf = Buffer.from(JSON.stringify(item));
|
|
38
35
|
sizes.push(buf.byteLength);
|
|
39
36
|
if (sizesZipped) {
|
|
40
|
-
const { byteLength } = await
|
|
37
|
+
const { byteLength } = await gzipBuffer(buf);
|
|
41
38
|
sizesZipped.push(byteLength);
|
|
42
39
|
}
|
|
43
40
|
}
|
|
@@ -46,4 +43,3 @@ class SizeStack extends js_lib_1.NumberStack {
|
|
|
46
43
|
}
|
|
47
44
|
}
|
|
48
45
|
}
|
|
49
|
-
exports.SizeStack = SizeStack;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.pipelineClose = pipelineClose;
|
|
4
|
-
function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
|
|
1
|
+
export function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
|
|
5
2
|
readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
|
|
6
3
|
if (!sourceReadable) {
|
|
7
4
|
logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
|
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.transformChunk = transformChunk;
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
5
2
|
/**
|
|
6
3
|
* Similar to RxJS bufferCount(),
|
|
7
4
|
* allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
|
|
8
5
|
* Last chunk will contain the remaining items, possibly less than chunkSize.
|
|
9
6
|
*/
|
|
10
|
-
function transformChunk(opt) {
|
|
7
|
+
export function transformChunk(opt) {
|
|
11
8
|
const { chunkSize } = opt;
|
|
12
9
|
let buf = [];
|
|
13
|
-
return new
|
|
10
|
+
return new Transform({
|
|
14
11
|
objectMode: true,
|
|
15
12
|
...opt,
|
|
16
13
|
transform(chunk, _, cb) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { TransformOptions, TransformTyped } from '../stream.model';
|
|
3
|
-
import type { TransformMapOptions } from './transformMap';
|
|
2
|
+
import type { TransformOptions, TransformTyped } from '../stream.model.js';
|
|
3
|
+
import type { TransformMapOptions } from './transformMap.js';
|
|
4
4
|
/**
|
|
5
5
|
* Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
|
|
6
6
|
*/
|
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.transformFilter = transformFilter;
|
|
4
|
-
exports.transformFilterSync = transformFilterSync;
|
|
5
|
-
const node_stream_1 = require("node:stream");
|
|
6
|
-
const transformMap_1 = require("./transformMap");
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
|
+
import { transformMap } from './transformMap.js';
|
|
7
3
|
/**
|
|
8
4
|
* Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
|
|
9
5
|
*/
|
|
10
|
-
function transformFilter(predicate, opt = {}) {
|
|
11
|
-
return
|
|
6
|
+
export function transformFilter(predicate, opt = {}) {
|
|
7
|
+
return transformMap(v => v, {
|
|
12
8
|
predicate,
|
|
13
9
|
...opt,
|
|
14
10
|
});
|
|
@@ -16,9 +12,9 @@ function transformFilter(predicate, opt = {}) {
|
|
|
16
12
|
/**
|
|
17
13
|
* Sync version of `transformFilter`
|
|
18
14
|
*/
|
|
19
|
-
function transformFilterSync(predicate, opt = {}) {
|
|
15
|
+
export function transformFilterSync(predicate, opt = {}) {
|
|
20
16
|
let index = 0;
|
|
21
|
-
return new
|
|
17
|
+
return new Transform({
|
|
22
18
|
objectMode: true,
|
|
23
19
|
...opt,
|
|
24
20
|
transform(chunk, _, cb) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Readable } from 'node:stream';
|
|
2
2
|
import type { CommonLogger } from '@naturalcycles/js-lib';
|
|
3
|
-
import { AbortableTransform } from '../../index';
|
|
4
|
-
import type { TransformOptions, TransformTyped } from '../stream.model';
|
|
3
|
+
import { AbortableTransform } from '../../index.js';
|
|
4
|
+
import type { TransformOptions, TransformTyped } from '../stream.model.js';
|
|
5
5
|
export interface TransformLimitOptions extends TransformOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Nullish value (e.g 0 or undefined) would mean "no limit"
|