@naturalcycles/nodejs-lib 13.45.4 → 14.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/dist/bin/generate-build-info.js +15 -18
  2. package/dist/bin/json2env.js +6 -9
  3. package/dist/bin/kpy.js +6 -9
  4. package/dist/bin/secrets-decrypt.d.ts +1 -1
  5. package/dist/bin/secrets-decrypt.js +9 -12
  6. package/dist/bin/secrets-encrypt.d.ts +1 -1
  7. package/dist/bin/secrets-encrypt.js +9 -12
  8. package/dist/bin/secrets-gen-key.js +8 -11
  9. package/dist/bin/slack-this.js +6 -9
  10. package/dist/buffer/buffer.util.js +1 -4
  11. package/dist/colors/colors.d.ts +26 -27
  12. package/dist/colors/colors.js +29 -33
  13. package/dist/csv/csvReader.js +4 -8
  14. package/dist/csv/csvWriter.js +6 -12
  15. package/dist/csv/transformToCSV.d.ts +2 -2
  16. package/dist/csv/transformToCSV.js +5 -8
  17. package/dist/diff/tableDiff.js +4 -7
  18. package/dist/fs/fs2.d.ts +1 -1
  19. package/dist/fs/fs2.js +82 -86
  20. package/dist/fs/json2env.js +18 -28
  21. package/dist/fs/kpy.js +28 -33
  22. package/dist/index.d.ts +71 -71
  23. package/dist/index.js +73 -78
  24. package/dist/infra/process.util.js +22 -28
  25. package/dist/jwt/jwt.service.js +16 -21
  26. package/dist/log/log.util.js +4 -7
  27. package/dist/script/runScript.js +11 -15
  28. package/dist/secret/secrets-decrypt.util.js +18 -22
  29. package/dist/secret/secrets-encrypt.util.js +17 -21
  30. package/dist/security/crypto.util.js +24 -34
  31. package/dist/security/hash.util.js +19 -37
  32. package/dist/security/id.util.js +11 -18
  33. package/dist/security/nanoid.js +15 -20
  34. package/dist/security/secret.util.js +20 -30
  35. package/dist/slack/slack.service.d.ts +1 -1
  36. package/dist/slack/slack.service.js +11 -16
  37. package/dist/slack/slack.service.model.d.ts +1 -1
  38. package/dist/slack/slack.service.model.js +1 -2
  39. package/dist/stream/ndjson/ndjson.model.js +7 -11
  40. package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
  41. package/dist/stream/ndjson/ndjsonMap.js +11 -14
  42. package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
  43. package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
  44. package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
  45. package/dist/stream/ndjson/transformJsonParse.js +4 -9
  46. package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
  47. package/dist/stream/ndjson/transformToNDJson.js +5 -8
  48. package/dist/stream/pipeline/pipeline.js +11 -17
  49. package/dist/stream/progressLogger.d.ts +1 -1
  50. package/dist/stream/progressLogger.js +22 -28
  51. package/dist/stream/readable/readableCreate.d.ts +1 -1
  52. package/dist/stream/readable/readableCreate.js +5 -9
  53. package/dist/stream/readable/readableForEach.d.ts +2 -2
  54. package/dist/stream/readable/readableForEach.js +7 -11
  55. package/dist/stream/readable/readableFromArray.d.ts +1 -1
  56. package/dist/stream/readable/readableFromArray.js +4 -7
  57. package/dist/stream/readable/readableToArray.d.ts +1 -1
  58. package/dist/stream/readable/readableToArray.js +1 -4
  59. package/dist/stream/sizeStack.js +7 -11
  60. package/dist/stream/stream.model.js +1 -2
  61. package/dist/stream/stream.util.js +1 -4
  62. package/dist/stream/transform/transformChunk.d.ts +1 -1
  63. package/dist/stream/transform/transformChunk.js +3 -6
  64. package/dist/stream/transform/transformFilter.d.ts +2 -2
  65. package/dist/stream/transform/transformFilter.js +6 -10
  66. package/dist/stream/transform/transformLimit.d.ts +2 -2
  67. package/dist/stream/transform/transformLimit.js +6 -11
  68. package/dist/stream/transform/transformLogProgress.d.ts +2 -2
  69. package/dist/stream/transform/transformLogProgress.js +5 -8
  70. package/dist/stream/transform/transformMap.d.ts +1 -1
  71. package/dist/stream/transform/transformMap.js +18 -22
  72. package/dist/stream/transform/transformMapSimple.d.ts +1 -1
  73. package/dist/stream/transform/transformMapSimple.js +6 -9
  74. package/dist/stream/transform/transformMapSync.d.ts +3 -3
  75. package/dist/stream/transform/transformMapSync.js +14 -19
  76. package/dist/stream/transform/transformNoOp.d.ts +1 -1
  77. package/dist/stream/transform/transformNoOp.js +3 -6
  78. package/dist/stream/transform/transformOffset.d.ts +1 -1
  79. package/dist/stream/transform/transformOffset.js +5 -8
  80. package/dist/stream/transform/transformSplit.d.ts +1 -1
  81. package/dist/stream/transform/transformSplit.js +5 -9
  82. package/dist/stream/transform/transformTap.d.ts +1 -1
  83. package/dist/stream/transform/transformTap.js +3 -6
  84. package/dist/stream/transform/transformTee.d.ts +1 -1
  85. package/dist/stream/transform/transformTee.js +7 -10
  86. package/dist/stream/transform/transformThrottle.d.ts +1 -1
  87. package/dist/stream/transform/transformThrottle.js +9 -12
  88. package/dist/stream/transform/transformToArray.d.ts +1 -1
  89. package/dist/stream/transform/transformToArray.js +3 -6
  90. package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
  91. package/dist/stream/transform/worker/baseWorkerClass.js +1 -5
  92. package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
  93. package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
  94. package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
  95. package/dist/stream/transform/worker/workerClassProxy.js +6 -4
  96. package/dist/stream/writable/writableForEach.d.ts +2 -2
  97. package/dist/stream/writable/writableForEach.js +6 -10
  98. package/dist/stream/writable/writableFork.d.ts +1 -1
  99. package/dist/stream/writable/writableFork.js +6 -9
  100. package/dist/stream/writable/writableLimit.d.ts +1 -1
  101. package/dist/stream/writable/writableLimit.js +3 -6
  102. package/dist/stream/writable/writablePushToArray.d.ts +1 -1
  103. package/dist/stream/writable/writablePushToArray.js +3 -6
  104. package/dist/stream/writable/writableVoid.d.ts +1 -1
  105. package/dist/stream/writable/writableVoid.js +3 -6
  106. package/dist/string/inspect.js +8 -13
  107. package/dist/util/buildInfo.util.js +12 -15
  108. package/dist/util/env.util.js +5 -9
  109. package/dist/util/exec2.js +17 -21
  110. package/dist/util/git2.js +17 -20
  111. package/dist/util/lruMemoCache.js +3 -7
  112. package/dist/util/zip.util.js +14 -25
  113. package/dist/validation/ajv/ajv.util.d.ts +3 -3
  114. package/dist/validation/ajv/ajv.util.js +6 -10
  115. package/dist/validation/ajv/ajvSchema.d.ts +2 -2
  116. package/dist/validation/ajv/ajvSchema.js +14 -18
  117. package/dist/validation/ajv/ajvValidationError.js +2 -6
  118. package/dist/validation/ajv/getAjv.d.ts +1 -1
  119. package/dist/validation/ajv/getAjv.js +9 -9
  120. package/dist/validation/joi/joi.extensions.d.ts +2 -2
  121. package/dist/validation/joi/joi.extensions.js +6 -10
  122. package/dist/validation/joi/joi.model.js +1 -2
  123. package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
  124. package/dist/validation/joi/joi.shared.schemas.js +61 -74
  125. package/dist/validation/joi/joi.validation.error.js +2 -6
  126. package/dist/validation/joi/joi.validation.util.d.ts +1 -1
  127. package/dist/validation/joi/joi.validation.util.js +10 -17
  128. package/dist/validation/joi/number.extensions.js +1 -4
  129. package/dist/validation/joi/string.extensions.js +4 -7
  130. package/dist/yargs.util.js +5 -8
  131. package/package.json +5 -4
  132. package/src/bin/generate-build-info.ts +3 -3
  133. package/src/bin/json2env.ts +2 -2
  134. package/src/bin/kpy.ts +2 -2
  135. package/src/bin/secrets-decrypt.ts +7 -8
  136. package/src/bin/secrets-encrypt.ts +7 -8
  137. package/src/bin/secrets-gen-key.ts +2 -2
  138. package/src/bin/slack-this.ts +2 -2
  139. package/src/csv/transformToCSV.ts +3 -3
  140. package/src/fs/fs2.ts +4 -4
  141. package/src/fs/json2env.ts +2 -2
  142. package/src/fs/kpy.ts +2 -2
  143. package/src/index.ts +71 -71
  144. package/src/jwt/jwt.service.ts +2 -2
  145. package/src/log/log.util.ts +1 -1
  146. package/src/script/runScript.ts +4 -3
  147. package/src/secret/secrets-decrypt.util.ts +3 -3
  148. package/src/secret/secrets-encrypt.util.ts +3 -3
  149. package/src/security/crypto.util.ts +1 -1
  150. package/src/security/id.util.ts +1 -1
  151. package/src/security/secret.util.ts +2 -2
  152. package/src/slack/slack.service.model.ts +1 -1
  153. package/src/slack/slack.service.ts +3 -3
  154. package/src/stream/ndjson/ndjson.model.ts +1 -1
  155. package/src/stream/ndjson/ndjsonMap.ts +2 -2
  156. package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
  157. package/src/stream/ndjson/transformJsonParse.ts +1 -1
  158. package/src/stream/ndjson/transformToNDJson.ts +1 -1
  159. package/src/stream/pipeline/pipeline.ts +1 -1
  160. package/src/stream/progressLogger.ts +3 -3
  161. package/src/stream/readable/readableCreate.ts +1 -1
  162. package/src/stream/readable/readableForEach.ts +4 -4
  163. package/src/stream/readable/readableFromArray.ts +1 -1
  164. package/src/stream/readable/readableToArray.ts +1 -1
  165. package/src/stream/sizeStack.ts +2 -2
  166. package/src/stream/transform/transformChunk.ts +1 -1
  167. package/src/stream/transform/transformFilter.ts +3 -3
  168. package/src/stream/transform/transformLimit.ts +3 -3
  169. package/src/stream/transform/transformLogProgress.ts +3 -3
  170. package/src/stream/transform/transformMap.ts +5 -5
  171. package/src/stream/transform/transformMapSimple.ts +1 -1
  172. package/src/stream/transform/transformMapSync.ts +5 -5
  173. package/src/stream/transform/transformNoOp.ts +1 -1
  174. package/src/stream/transform/transformOffset.ts +2 -2
  175. package/src/stream/transform/transformSplit.ts +1 -1
  176. package/src/stream/transform/transformTap.ts +1 -1
  177. package/src/stream/transform/transformTee.ts +3 -3
  178. package/src/stream/transform/transformThrottle.ts +1 -1
  179. package/src/stream/transform/transformToArray.ts +1 -1
  180. package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
  181. package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
  182. package/src/stream/transform/worker/workerClassProxy.js +6 -4
  183. package/src/stream/writable/writableForEach.ts +3 -3
  184. package/src/stream/writable/writableFork.ts +3 -3
  185. package/src/stream/writable/writableLimit.ts +1 -1
  186. package/src/stream/writable/writablePushToArray.ts +1 -1
  187. package/src/stream/writable/writableVoid.ts +1 -1
  188. package/src/util/buildInfo.util.ts +2 -2
  189. package/src/util/exec2.ts +1 -1
  190. package/src/util/git2.ts +2 -2
  191. package/src/validation/ajv/ajv.util.ts +4 -4
  192. package/src/validation/ajv/ajvSchema.ts +4 -5
  193. package/src/validation/ajv/getAjv.ts +7 -3
  194. package/src/validation/joi/joi.extensions.ts +4 -4
  195. package/src/validation/joi/joi.shared.schemas.ts +3 -3
  196. package/src/validation/joi/joi.validation.util.ts +2 -2
  197. package/src/yargs.util.ts +1 -0
@@ -1,31 +1,28 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ndjsonMap = ndjsonMap;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const __1 = require("../..");
1
+ import { ErrorMode } from '@naturalcycles/js-lib';
2
+ import { _pipeline, fs2, transformLimit, transformLogProgress, transformMap } from '../../index.js';
6
3
  /**
7
4
  * Unzips input file automatically, if it ends with `.gz`.
8
5
  * Zips output file automatically, if it ends with `.gz`.
9
6
  */
10
- async function ndjsonMap(mapper, opt) {
7
+ export async function ndjsonMap(mapper, opt) {
11
8
  const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;
12
9
  console.log({
13
10
  inputFilePath,
14
11
  outputFilePath,
15
12
  });
16
- const readable = __1.fs2
13
+ const readable = fs2
17
14
  .createReadStreamAsNDJSON(inputFilePath)
18
15
  .take(limitInput || Number.POSITIVE_INFINITY);
19
- await (0, __1._pipeline)([
16
+ await _pipeline([
20
17
  readable,
21
- (0, __1.transformLogProgress)({ metric: 'read', ...opt }),
22
- (0, __1.transformMap)(mapper, {
18
+ transformLogProgress({ metric: 'read', ...opt }),
19
+ transformMap(mapper, {
23
20
  flattenArrayOutput: true,
24
- errorMode: js_lib_1.ErrorMode.SUPPRESS,
21
+ errorMode: ErrorMode.SUPPRESS,
25
22
  ...opt,
26
23
  }),
27
- (0, __1.transformLimit)({ limit: limitOutput, sourceReadable: readable }),
28
- (0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }),
29
- ...__1.fs2.createWriteStreamAsNDJSON(outputFilePath),
24
+ transformLimit({ limit: limitOutput, sourceReadable: readable }),
25
+ transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
26
+ ...fs2.createWriteStreamAsNDJSON(outputFilePath),
30
27
  ]);
31
28
  }
@@ -1,5 +1,5 @@
1
1
  import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
2
- import type { TransformLogProgressOptions, TransformMapOptions } from '../..';
2
+ import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
3
3
  export interface NDJSONStreamForEachOptions<IN = any> extends TransformMapOptions<IN, void>, TransformLogProgressOptions<IN> {
4
4
  inputFilePath: string;
5
5
  }
@@ -1,20 +1,17 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ndjsonStreamForEach = ndjsonStreamForEach;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const __1 = require("../..");
1
+ import { ErrorMode } from '@naturalcycles/js-lib';
2
+ import { _pipeline, fs2, transformLogProgress, transformMap, writableVoid } from '../../index.js';
6
3
  /**
7
4
  * Convenience function to `forEach` through an ndjson file.
8
5
  */
9
- async function ndjsonStreamForEach(mapper, opt) {
10
- await (0, __1._pipeline)([
11
- __1.fs2.createReadStreamAsNDJSON(opt.inputFilePath),
12
- (0, __1.transformMap)(mapper, {
13
- errorMode: js_lib_1.ErrorMode.THROW_AGGREGATED,
6
+ export async function ndjsonStreamForEach(mapper, opt) {
7
+ await _pipeline([
8
+ fs2.createReadStreamAsNDJSON(opt.inputFilePath),
9
+ transformMap(mapper, {
10
+ errorMode: ErrorMode.THROW_AGGREGATED,
14
11
  ...opt,
15
12
  predicate: () => true, // to log progress properly
16
13
  }),
17
- (0, __1.transformLogProgress)(opt),
18
- (0, __1.writableVoid)(),
14
+ transformLogProgress(opt),
15
+ writableVoid(),
19
16
  ]);
20
17
  }
@@ -1,5 +1,5 @@
1
1
  import type { Reviver } from '@naturalcycles/js-lib';
2
- import type { TransformTyped } from '../stream.model';
2
+ import type { TransformTyped } from '../stream.model.js';
3
3
  export interface TransformJsonParseOptions {
4
4
  /**
5
5
  * If true - will throw an error on JSON.parse / stringify error
@@ -1,8 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.bufferReviver = void 0;
4
- exports.transformJsonParse = transformJsonParse;
5
- const node_stream_1 = require("node:stream");
1
+ import { Transform } from 'node:stream';
6
2
  /**
7
3
  * Transforms chunks of JSON strings/Buffers (objectMode=false) into parsed objects (readableObjectMode=true).
8
4
  *
@@ -17,9 +13,9 @@ const node_stream_1 = require("node:stream");
17
13
  * consumeYourStream...
18
14
  * [)
19
15
  */
20
- function transformJsonParse(opt = {}) {
16
+ export function transformJsonParse(opt = {}) {
21
17
  const { strict = true, reviver } = opt;
22
- return new node_stream_1.Transform({
18
+ return new Transform({
23
19
  writableObjectMode: false,
24
20
  readableObjectMode: true,
25
21
  // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
@@ -43,10 +39,9 @@ function transformJsonParse(opt = {}) {
43
39
  });
44
40
  }
45
41
  // Based on: https://stackoverflow.com/a/34557997/4919972
46
- const bufferReviver = (_k, v) => {
42
+ export const bufferReviver = (_k, v) => {
47
43
  if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
48
44
  return Buffer.from(v.data);
49
45
  }
50
46
  return v;
51
47
  };
52
- exports.bufferReviver = bufferReviver;
@@ -1,4 +1,4 @@
1
- import type { TransformTyped } from '../stream.model';
1
+ import type { TransformTyped } from '../stream.model.js';
2
2
  export interface TransformToNDJsonOptions {
3
3
  /**
4
4
  * If true - will throw an error on JSON.parse / stringify error
@@ -1,21 +1,18 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.transformToNDJson = transformToNDJson;
4
- const node_stream_1 = require("node:stream");
5
- const js_lib_1 = require("@naturalcycles/js-lib");
1
+ import { Transform } from 'node:stream';
2
+ import { _sortObjectDeep } from '@naturalcycles/js-lib';
6
3
  /**
7
4
  * Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
8
5
  */
9
- function transformToNDJson(opt = {}) {
6
+ export function transformToNDJson(opt = {}) {
10
7
  const { strict = true, separator = '\n', sortObjects = false } = opt;
11
- return new node_stream_1.Transform({
8
+ return new Transform({
12
9
  writableObjectMode: true,
13
10
  readableObjectMode: false,
14
11
  readableHighWaterMark: 64 * 1024,
15
12
  transform(chunk, _, cb) {
16
13
  try {
17
14
  if (sortObjects) {
18
- chunk = (0, js_lib_1._sortObjectDeep)(chunk);
15
+ chunk = _sortObjectDeep(chunk);
19
16
  }
20
17
  cb(null, JSON.stringify(chunk) + separator);
21
18
  }
@@ -1,19 +1,14 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AbortableTransform = void 0;
4
- exports._pipeline = _pipeline;
5
- exports._pipelineToArray = _pipelineToArray;
6
- const node_stream_1 = require("node:stream");
7
- const promises_1 = require("node:stream/promises");
8
- const js_lib_1 = require("@naturalcycles/js-lib");
9
- const writablePushToArray_1 = require("../writable/writablePushToArray");
1
+ import { Transform } from 'node:stream';
2
+ import { pipeline } from 'node:stream/promises';
3
+ import { _last, pDefer } from '@naturalcycles/js-lib';
4
+ import { writablePushToArray } from '../writable/writablePushToArray.js';
10
5
  /**
11
6
  * Promisified `stream.pipeline`.
12
7
  *
13
8
  * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
14
9
  * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
15
10
  */
16
- async function _pipeline(streams, opt = {}) {
11
+ export async function _pipeline(streams, opt = {}) {
17
12
  const first = streams[0];
18
13
  const rest = streams.slice(1);
19
14
  if (opt.allowClose) {
@@ -35,9 +30,9 @@ async function _pipeline(streams, opt = {}) {
35
30
  // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
36
31
  // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
37
32
  // inevitable in this flow). Know a better way to close the stream? Tell me!
38
- const streamDone = (0, js_lib_1.pDefer)();
33
+ const streamDone = pDefer();
39
34
  const sourceReadable = first;
40
- const last = (0, js_lib_1._last)(streams);
35
+ const last = _last(streams);
41
36
  const lastFinal = last._final?.bind(last) || ((cb) => cb());
42
37
  last._final = cb => {
43
38
  lastFinal(() => {
@@ -56,7 +51,7 @@ async function _pipeline(streams, opt = {}) {
56
51
  });
57
52
  }
58
53
  try {
59
- return await (0, promises_1.pipeline)([first, ...rest], opt);
54
+ return await pipeline([first, ...rest], opt);
60
55
  }
61
56
  catch (err) {
62
57
  if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
@@ -75,13 +70,12 @@ async function _pipeline(streams, opt = {}) {
75
70
  * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
76
71
  * and return.
77
72
  */
78
- async function _pipelineToArray(streams, opt = {}) {
73
+ export async function _pipelineToArray(streams, opt = {}) {
79
74
  const a = [];
80
- await _pipeline([...streams, (0, writablePushToArray_1.writablePushToArray)(a)], opt);
75
+ await _pipeline([...streams, writablePushToArray(a)], opt);
81
76
  return a;
82
77
  }
83
- class AbortableTransform extends node_stream_1.Transform {
78
+ export class AbortableTransform extends Transform {
84
79
  sourceReadable;
85
80
  streamDone;
86
81
  }
87
- exports.AbortableTransform = AbortableTransform;
@@ -1,5 +1,5 @@
1
1
  import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
2
- import type { ReadableMapper } from './stream.model';
2
+ import type { ReadableMapper } from './stream.model.js';
3
3
  export interface ProgressLoggerCfg<T = any> {
4
4
  /**
5
5
  * Progress metric
@@ -1,17 +1,12 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ProgressLogger = void 0;
4
- exports.progressLogger = progressLogger;
5
- exports.progressReadableMapper = progressReadableMapper;
6
- const node_util_1 = require("node:util");
7
- const js_lib_1 = require("@naturalcycles/js-lib");
8
- const colors_1 = require("../colors/colors");
9
- const sizeStack_1 = require("./sizeStack");
1
+ import { inspect } from 'node:util';
2
+ import { _mb, _since, localTime, SimpleMovingAverage } from '@naturalcycles/js-lib';
3
+ import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
4
+ import { SizeStack } from './sizeStack.js';
10
5
  const inspectOpt = {
11
- colors: colors_1.hasColors,
6
+ colors: hasColors,
12
7
  breakLength: 300,
13
8
  };
14
- class ProgressLogger {
9
+ export class ProgressLogger {
15
10
  constructor(cfg = {}) {
16
11
  this.cfg = {
17
12
  metric: 'progress',
@@ -42,13 +37,13 @@ class ProgressLogger {
42
37
  start() {
43
38
  this.started = Date.now();
44
39
  this.lastSecondStarted = Date.now();
45
- this.sma = new js_lib_1.SimpleMovingAverage(10);
40
+ this.sma = new SimpleMovingAverage(10);
46
41
  this.processedLastSecond = 0;
47
42
  this.progress = 0;
48
43
  this.peakRSS = 0;
49
- this.sizes = this.cfg.logSizes ? new sizeStack_1.SizeStack('json', this.cfg.logSizesBuffer) : undefined;
44
+ this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
50
45
  this.sizesZipped = this.cfg.logZippedSizes
51
- ? new sizeStack_1.SizeStack('json.gz', this.cfg.logSizesBuffer)
46
+ ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
52
47
  : undefined;
53
48
  }
54
49
  log(chunk) {
@@ -56,7 +51,7 @@ class ProgressLogger {
56
51
  this.processedLastSecond++;
57
52
  if (this.sizes) {
58
53
  // Check it, cause gzipping might be delayed here..
59
- void sizeStack_1.SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
54
+ void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
60
55
  }
61
56
  if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
62
57
  this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
@@ -88,22 +83,22 @@ class ProgressLogger {
88
83
  if (extra)
89
84
  Object.assign(o, extra(chunk, this.progress));
90
85
  if (logHeapUsed)
91
- o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
86
+ o.heapUsed = _mb(mem.heapUsed);
92
87
  if (logHeapTotal)
93
- o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
88
+ o.heapTotal = _mb(mem.heapTotal);
94
89
  if (logRss)
95
- o.rss = (0, js_lib_1._mb)(mem.rss);
90
+ o.rss = _mb(mem.rss);
96
91
  if (logPeakRss)
97
- o.peakRSS = (0, js_lib_1._mb)(this.peakRSS);
92
+ o.peakRSS = _mb(this.peakRSS);
98
93
  if (rssMinusHeap)
99
- o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
94
+ o.rssMinusHeap = _mb(mem.rss - mem.heapTotal);
100
95
  if (external)
101
- o.external = (0, js_lib_1._mb)(mem.external);
96
+ o.external = _mb(mem.external);
102
97
  if (arrayBuffers)
103
- o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
98
+ o.arrayBuffers = _mb(mem.arrayBuffers || 0);
104
99
  if (logRPS)
105
100
  Object.assign(o, { rps10, rpsTotal });
106
- logger.log((0, node_util_1.inspect)(o, inspectOpt));
101
+ logger.log(inspect(o, inspectOpt));
107
102
  if (this.sizes?.items.length) {
108
103
  logger.log(this.sizes.getStats());
109
104
  if (this.sizesZipped?.items.length) {
@@ -115,10 +110,10 @@ class ProgressLogger {
115
110
  if (perHour > 900) {
116
111
  perHour = Math.round(perHour / 1000) + 'K';
117
112
  }
118
- logger.log(`${(0, colors_1.dimGrey)(js_lib_1.localTime.now().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
113
+ logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`);
119
114
  }
120
115
  else if (final) {
121
- logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
116
+ logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);
122
117
  try {
123
118
  this.cfg.onProgressDone?.(o);
124
119
  }
@@ -128,11 +123,10 @@ class ProgressLogger {
128
123
  }
129
124
  }
130
125
  }
131
- exports.ProgressLogger = ProgressLogger;
132
126
  /**
133
127
  * Create new ProgressLogger.
134
128
  */
135
- function progressLogger(cfg = {}) {
129
+ export function progressLogger(cfg = {}) {
136
130
  return new ProgressLogger(cfg);
137
131
  }
138
132
  /**
@@ -140,7 +134,7 @@ function progressLogger(cfg = {}) {
140
134
  *
141
135
  * @experimental
142
136
  */
143
- function progressReadableMapper(cfg = {}) {
137
+ export function progressReadableMapper(cfg = {}) {
144
138
  const progress = new ProgressLogger(cfg);
145
139
  return chunk => {
146
140
  progress.log(chunk);
@@ -1,5 +1,5 @@
1
1
  import type { ReadableOptions } from 'node:stream';
2
- import type { ReadableTyped } from '../stream.model';
2
+ import type { ReadableTyped } from '../stream.model.js';
3
3
  /**
4
4
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
5
5
  * Push `null` to it to complete (similar to RxJS `.complete()`).
@@ -1,8 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableCreate = readableCreate;
4
- exports.readableFrom = readableFrom;
5
- const node_stream_1 = require("node:stream");
1
+ import { Readable } from 'node:stream';
6
2
  /**
7
3
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
8
4
  * Push `null` to it to complete (similar to RxJS `.complete()`).
@@ -17,8 +13,8 @@ const node_stream_1 = require("node:stream");
17
13
  *
18
14
  * @deprecated because of the caution above
19
15
  */
20
- function readableCreate(items = [], opt) {
21
- const readable = new node_stream_1.Readable({
16
+ export function readableCreate(items = [], opt) {
17
+ const readable = new Readable({
22
18
  objectMode: true,
23
19
  ...opt,
24
20
  read() { }, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
@@ -31,6 +27,6 @@ function readableCreate(items = [], opt) {
31
27
  /**
32
28
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
33
29
  */
34
- function readableFrom(iterable, opt) {
35
- return node_stream_1.Readable.from(iterable, opt);
30
+ export function readableFrom(iterable, opt) {
31
+ return Readable.from(iterable, opt);
36
32
  }
@@ -1,6 +1,6 @@
1
1
  import type { AbortableAsyncMapper, Mapper } from '@naturalcycles/js-lib';
2
- import type { ReadableTyped } from '../../index';
3
- import type { TransformMapOptions } from '../transform/transformMap';
2
+ import type { ReadableTyped } from '../../index.js';
3
+ import type { TransformMapOptions } from '../transform/transformMap.js';
4
4
  /**
5
5
  * Convenience function to do `.forEach` over a Readable.
6
6
  * Typed! (unlike default Readable).
@@ -1,10 +1,6 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableForEach = readableForEach;
4
- exports.readableForEachSync = readableForEachSync;
5
- const js_lib_1 = require("@naturalcycles/js-lib");
6
- const index_1 = require("../../index");
7
- const transformMap_1 = require("../transform/transformMap");
1
+ import { _passNothingPredicate } from '@naturalcycles/js-lib';
2
+ import { _pipeline } from '../../index.js';
3
+ import { transformMap } from '../transform/transformMap.js';
8
4
  /**
9
5
  * Convenience function to do `.forEach` over a Readable.
10
6
  * Typed! (unlike default Readable).
@@ -13,10 +9,10 @@ const transformMap_1 = require("../transform/transformMap");
13
9
  *
14
10
  * @experimental
15
11
  */
16
- async function readableForEach(readable, mapper, opt = {}) {
17
- await (0, index_1._pipeline)([
12
+ export async function readableForEach(readable, mapper, opt = {}) {
13
+ await _pipeline([
18
14
  readable,
19
- (0, transformMap_1.transformMap)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate }),
15
+ transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
20
16
  ]);
21
17
  }
22
18
  /**
@@ -25,7 +21,7 @@ async function readableForEach(readable, mapper, opt = {}) {
25
21
  *
26
22
  * @experimental
27
23
  */
28
- async function readableForEachSync(readable, mapper) {
24
+ export async function readableForEachSync(readable, mapper) {
29
25
  // async iteration
30
26
  let index = 0;
31
27
  for await (const item of readable) {
@@ -1,6 +1,6 @@
1
1
  import type { ReadableOptions } from 'node:stream';
2
2
  import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
3
- import type { ReadableTyped } from '../stream.model';
3
+ import type { ReadableTyped } from '../stream.model.js';
4
4
  /**
5
5
  * Create Readable from Array.
6
6
  * Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
@@ -1,17 +1,14 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableFromArray = readableFromArray;
4
- const node_stream_1 = require("node:stream");
5
- const js_lib_1 = require("@naturalcycles/js-lib");
1
+ import { Readable } from 'node:stream';
2
+ import { _passthroughMapper } from '@naturalcycles/js-lib';
6
3
  /**
7
4
  * Create Readable from Array.
8
5
  * Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
9
6
  *
10
7
  * For simple cases use Readable.from(...) (Node.js 12+)
11
8
  */
12
- function readableFromArray(items, mapper = js_lib_1._passthroughMapper, opt) {
9
+ export function readableFromArray(items, mapper = _passthroughMapper, opt) {
13
10
  let i = -1;
14
- return new node_stream_1.Readable({
11
+ return new Readable({
15
12
  objectMode: true,
16
13
  ...opt,
17
14
  async read() {
@@ -1,4 +1,4 @@
1
- import type { ReadableTyped } from '../stream.model';
1
+ import type { ReadableTyped } from '../stream.model.js';
2
2
  /**
3
3
  * Convenience function to read the whole Readable stream into Array (in-memory)
4
4
  * and return that array.
@@ -1,6 +1,3 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableToArray = readableToArray;
4
1
  /**
5
2
  * Convenience function to read the whole Readable stream into Array (in-memory)
6
3
  * and return that array.
@@ -8,7 +5,7 @@ exports.readableToArray = readableToArray;
8
5
  * Native `await readable.toArray()` can be used instead.
9
6
  * This helper is kept for type-safery support.
10
7
  */
11
- async function readableToArray(readable) {
8
+ export async function readableToArray(readable) {
12
9
  return await readable.toArray();
13
10
  // const a: T[] = []
14
11
  //
@@ -1,10 +1,7 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SizeStack = void 0;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const colors_1 = require("../colors/colors");
6
- const zip_util_1 = require("../util/zip.util");
7
- class SizeStack extends js_lib_1.NumberStack {
1
+ import { _hb, NumberStack } from '@naturalcycles/js-lib';
2
+ import { yellow } from '../colors/colors.js';
3
+ import { gzipBuffer } from '../util/zip.util.js';
4
+ export class SizeStack extends NumberStack {
8
5
  name;
9
6
  constructor(name, size) {
10
7
  super(size);
@@ -20,13 +17,13 @@ class SizeStack extends js_lib_1.NumberStack {
20
17
  return [
21
18
  ' ' + this.name,
22
19
  'avg',
23
- (0, colors_1.yellow)((0, js_lib_1._hb)(this.avg())),
20
+ yellow(_hb(this.avg())),
24
21
  // 'p50',
25
22
  // yellow(_hb(pcs[50])),
26
23
  // 'p90',
27
24
  // yellow(_hb(pcs[90])),
28
25
  'total',
29
- (0, colors_1.yellow)((0, js_lib_1._hb)(this.total)),
26
+ yellow(_hb(this.total)),
30
27
  ].join(' ');
31
28
  }
32
29
  static async countItem(item, logger, sizes, sizesZipped) {
@@ -37,7 +34,7 @@ class SizeStack extends js_lib_1.NumberStack {
37
34
  const buf = Buffer.from(JSON.stringify(item));
38
35
  sizes.push(buf.byteLength);
39
36
  if (sizesZipped) {
40
- const { byteLength } = await (0, zip_util_1.gzipBuffer)(buf);
37
+ const { byteLength } = await gzipBuffer(buf);
41
38
  sizesZipped.push(byteLength);
42
39
  }
43
40
  }
@@ -46,4 +43,3 @@ class SizeStack extends js_lib_1.NumberStack {
46
43
  }
47
44
  }
48
45
  }
49
- exports.SizeStack = SizeStack;
@@ -1,2 +1 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
@@ -1,7 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.pipelineClose = pipelineClose;
4
- function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
1
+ export function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
5
2
  readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
6
3
  if (!sourceReadable) {
7
4
  logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
@@ -1,4 +1,4 @@
1
- import type { TransformOptions, TransformTyped } from '../stream.model';
1
+ import type { TransformOptions, TransformTyped } from '../stream.model.js';
2
2
  export interface TransformChunkOptions extends TransformOptions {
3
3
  /**
4
4
  * How many items to include in each chunk.
@@ -1,16 +1,13 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.transformChunk = transformChunk;
4
- const node_stream_1 = require("node:stream");
1
+ import { Transform } from 'node:stream';
5
2
  /**
6
3
  * Similar to RxJS bufferCount(),
7
4
  * allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
8
5
  * Last chunk will contain the remaining items, possibly less than chunkSize.
9
6
  */
10
- function transformChunk(opt) {
7
+ export function transformChunk(opt) {
11
8
  const { chunkSize } = opt;
12
9
  let buf = [];
13
- return new node_stream_1.Transform({
10
+ return new Transform({
14
11
  objectMode: true,
15
12
  ...opt,
16
13
  transform(chunk, _, cb) {
@@ -1,6 +1,6 @@
1
1
  import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib';
2
- import type { TransformOptions, TransformTyped } from '../stream.model';
3
- import type { TransformMapOptions } from './transformMap';
2
+ import type { TransformOptions, TransformTyped } from '../stream.model.js';
3
+ import type { TransformMapOptions } from './transformMap.js';
4
4
  /**
5
5
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
6
6
  */
@@ -1,14 +1,10 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.transformFilter = transformFilter;
4
- exports.transformFilterSync = transformFilterSync;
5
- const node_stream_1 = require("node:stream");
6
- const transformMap_1 = require("./transformMap");
1
+ import { Transform } from 'node:stream';
2
+ import { transformMap } from './transformMap.js';
7
3
  /**
8
4
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
9
5
  */
10
- function transformFilter(predicate, opt = {}) {
11
- return (0, transformMap_1.transformMap)(v => v, {
6
+ export function transformFilter(predicate, opt = {}) {
7
+ return transformMap(v => v, {
12
8
  predicate,
13
9
  ...opt,
14
10
  });
@@ -16,9 +12,9 @@ function transformFilter(predicate, opt = {}) {
16
12
  /**
17
13
  * Sync version of `transformFilter`
18
14
  */
19
- function transformFilterSync(predicate, opt = {}) {
15
+ export function transformFilterSync(predicate, opt = {}) {
20
16
  let index = 0;
21
- return new node_stream_1.Transform({
17
+ return new Transform({
22
18
  objectMode: true,
23
19
  ...opt,
24
20
  transform(chunk, _, cb) {
@@ -1,7 +1,7 @@
1
1
  import type { Readable } from 'node:stream';
2
2
  import type { CommonLogger } from '@naturalcycles/js-lib';
3
- import { AbortableTransform } from '../../index';
4
- import type { TransformOptions, TransformTyped } from '../stream.model';
3
+ import { AbortableTransform } from '../../index.js';
4
+ import type { TransformOptions, TransformTyped } from '../stream.model.js';
5
5
  export interface TransformLimitOptions extends TransformOptions {
6
6
  /**
7
7
  * Nullish value (e.g 0 or undefined) would mean "no limit"