@naturalcycles/nodejs-lib 13.45.3 → 14.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/dist/bin/generate-build-info.js +15 -18
  2. package/dist/bin/json2env.js +6 -9
  3. package/dist/bin/kpy.js +6 -9
  4. package/dist/bin/secrets-decrypt.d.ts +1 -1
  5. package/dist/bin/secrets-decrypt.js +9 -12
  6. package/dist/bin/secrets-encrypt.d.ts +1 -1
  7. package/dist/bin/secrets-encrypt.js +9 -12
  8. package/dist/bin/secrets-gen-key.js +8 -11
  9. package/dist/bin/slack-this.js +6 -9
  10. package/dist/buffer/buffer.util.js +1 -4
  11. package/dist/colors/colors.d.ts +26 -27
  12. package/dist/colors/colors.js +29 -33
  13. package/dist/csv/csvReader.js +4 -8
  14. package/dist/csv/csvWriter.js +7 -12
  15. package/dist/csv/transformToCSV.d.ts +2 -2
  16. package/dist/csv/transformToCSV.js +5 -8
  17. package/dist/diff/tableDiff.js +4 -7
  18. package/dist/fs/fs2.d.ts +1 -1
  19. package/dist/fs/fs2.js +88 -94
  20. package/dist/fs/json2env.js +18 -28
  21. package/dist/fs/kpy.js +28 -33
  22. package/dist/index.d.ts +71 -71
  23. package/dist/index.js +73 -78
  24. package/dist/infra/process.util.js +23 -28
  25. package/dist/jwt/jwt.service.js +17 -21
  26. package/dist/log/log.util.js +4 -7
  27. package/dist/script/runScript.js +11 -15
  28. package/dist/secret/secrets-decrypt.util.js +18 -22
  29. package/dist/secret/secrets-encrypt.util.js +17 -21
  30. package/dist/security/crypto.util.js +24 -34
  31. package/dist/security/hash.util.js +19 -37
  32. package/dist/security/id.util.js +11 -18
  33. package/dist/security/nanoid.js +15 -20
  34. package/dist/security/secret.util.js +20 -30
  35. package/dist/slack/slack.service.d.ts +1 -1
  36. package/dist/slack/slack.service.js +13 -16
  37. package/dist/slack/slack.service.model.d.ts +1 -1
  38. package/dist/slack/slack.service.model.js +1 -2
  39. package/dist/stream/ndjson/ndjson.model.js +10 -16
  40. package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
  41. package/dist/stream/ndjson/ndjsonMap.js +11 -14
  42. package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
  43. package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
  44. package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
  45. package/dist/stream/ndjson/transformJsonParse.js +4 -9
  46. package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
  47. package/dist/stream/ndjson/transformToNDJson.js +5 -8
  48. package/dist/stream/pipeline/pipeline.js +13 -17
  49. package/dist/stream/progressLogger.d.ts +1 -1
  50. package/dist/stream/progressLogger.js +32 -28
  51. package/dist/stream/readable/readableCreate.d.ts +1 -1
  52. package/dist/stream/readable/readableCreate.js +5 -9
  53. package/dist/stream/readable/readableForEach.d.ts +2 -2
  54. package/dist/stream/readable/readableForEach.js +7 -11
  55. package/dist/stream/readable/readableFromArray.d.ts +1 -1
  56. package/dist/stream/readable/readableFromArray.js +4 -7
  57. package/dist/stream/readable/readableToArray.d.ts +1 -1
  58. package/dist/stream/readable/readableToArray.js +1 -4
  59. package/dist/stream/sizeStack.js +9 -12
  60. package/dist/stream/stream.model.js +1 -2
  61. package/dist/stream/stream.util.js +1 -4
  62. package/dist/stream/transform/transformChunk.d.ts +1 -1
  63. package/dist/stream/transform/transformChunk.js +3 -6
  64. package/dist/stream/transform/transformFilter.d.ts +2 -2
  65. package/dist/stream/transform/transformFilter.js +6 -10
  66. package/dist/stream/transform/transformLimit.d.ts +2 -2
  67. package/dist/stream/transform/transformLimit.js +6 -11
  68. package/dist/stream/transform/transformLogProgress.d.ts +2 -2
  69. package/dist/stream/transform/transformLogProgress.js +5 -8
  70. package/dist/stream/transform/transformMap.d.ts +1 -1
  71. package/dist/stream/transform/transformMap.js +18 -22
  72. package/dist/stream/transform/transformMapSimple.d.ts +1 -1
  73. package/dist/stream/transform/transformMapSimple.js +6 -9
  74. package/dist/stream/transform/transformMapSync.d.ts +3 -3
  75. package/dist/stream/transform/transformMapSync.js +14 -19
  76. package/dist/stream/transform/transformNoOp.d.ts +1 -1
  77. package/dist/stream/transform/transformNoOp.js +3 -6
  78. package/dist/stream/transform/transformOffset.d.ts +1 -1
  79. package/dist/stream/transform/transformOffset.js +5 -8
  80. package/dist/stream/transform/transformSplit.d.ts +1 -1
  81. package/dist/stream/transform/transformSplit.js +5 -9
  82. package/dist/stream/transform/transformTap.d.ts +1 -1
  83. package/dist/stream/transform/transformTap.js +3 -6
  84. package/dist/stream/transform/transformTee.d.ts +1 -1
  85. package/dist/stream/transform/transformTee.js +7 -10
  86. package/dist/stream/transform/transformThrottle.d.ts +1 -1
  87. package/dist/stream/transform/transformThrottle.js +9 -12
  88. package/dist/stream/transform/transformToArray.d.ts +1 -1
  89. package/dist/stream/transform/transformToArray.js +3 -6
  90. package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
  91. package/dist/stream/transform/worker/baseWorkerClass.js +2 -5
  92. package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
  93. package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
  94. package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
  95. package/dist/stream/transform/worker/workerClassProxy.js +6 -4
  96. package/dist/stream/writable/writableForEach.d.ts +2 -2
  97. package/dist/stream/writable/writableForEach.js +6 -10
  98. package/dist/stream/writable/writableFork.d.ts +1 -1
  99. package/dist/stream/writable/writableFork.js +6 -9
  100. package/dist/stream/writable/writableLimit.d.ts +1 -1
  101. package/dist/stream/writable/writableLimit.js +3 -6
  102. package/dist/stream/writable/writablePushToArray.d.ts +1 -1
  103. package/dist/stream/writable/writablePushToArray.js +3 -6
  104. package/dist/stream/writable/writableVoid.d.ts +1 -1
  105. package/dist/stream/writable/writableVoid.js +3 -6
  106. package/dist/string/inspect.js +8 -13
  107. package/dist/util/buildInfo.util.js +12 -15
  108. package/dist/util/env.util.js +5 -9
  109. package/dist/util/exec2.js +17 -21
  110. package/dist/util/git2.js +17 -20
  111. package/dist/util/lruMemoCache.js +4 -7
  112. package/dist/util/zip.util.js +14 -25
  113. package/dist/validation/ajv/ajv.util.d.ts +3 -3
  114. package/dist/validation/ajv/ajv.util.js +6 -10
  115. package/dist/validation/ajv/ajvSchema.d.ts +2 -2
  116. package/dist/validation/ajv/ajvSchema.js +17 -18
  117. package/dist/validation/ajv/ajvValidationError.js +2 -6
  118. package/dist/validation/ajv/getAjv.d.ts +1 -1
  119. package/dist/validation/ajv/getAjv.js +9 -9
  120. package/dist/validation/joi/joi.extensions.d.ts +2 -2
  121. package/dist/validation/joi/joi.extensions.js +6 -10
  122. package/dist/validation/joi/joi.model.js +1 -2
  123. package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
  124. package/dist/validation/joi/joi.shared.schemas.js +61 -74
  125. package/dist/validation/joi/joi.validation.error.js +2 -6
  126. package/dist/validation/joi/joi.validation.util.d.ts +1 -1
  127. package/dist/validation/joi/joi.validation.util.js +10 -17
  128. package/dist/validation/joi/number.extensions.js +1 -4
  129. package/dist/validation/joi/string.extensions.js +4 -7
  130. package/dist/yargs.util.js +5 -8
  131. package/package.json +5 -4
  132. package/src/bin/generate-build-info.ts +3 -3
  133. package/src/bin/json2env.ts +2 -2
  134. package/src/bin/kpy.ts +2 -2
  135. package/src/bin/secrets-decrypt.ts +7 -8
  136. package/src/bin/secrets-encrypt.ts +7 -8
  137. package/src/bin/secrets-gen-key.ts +2 -2
  138. package/src/bin/slack-this.ts +2 -2
  139. package/src/csv/transformToCSV.ts +3 -3
  140. package/src/fs/fs2.ts +4 -4
  141. package/src/fs/json2env.ts +2 -2
  142. package/src/fs/kpy.ts +2 -2
  143. package/src/index.ts +71 -71
  144. package/src/jwt/jwt.service.ts +2 -2
  145. package/src/log/log.util.ts +1 -1
  146. package/src/script/runScript.ts +4 -3
  147. package/src/secret/secrets-decrypt.util.ts +3 -3
  148. package/src/secret/secrets-encrypt.util.ts +3 -3
  149. package/src/security/crypto.util.ts +1 -1
  150. package/src/security/id.util.ts +1 -1
  151. package/src/security/secret.util.ts +2 -2
  152. package/src/slack/slack.service.model.ts +1 -1
  153. package/src/slack/slack.service.ts +3 -3
  154. package/src/stream/ndjson/ndjson.model.ts +1 -1
  155. package/src/stream/ndjson/ndjsonMap.ts +2 -2
  156. package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
  157. package/src/stream/ndjson/transformJsonParse.ts +1 -1
  158. package/src/stream/ndjson/transformToNDJson.ts +1 -1
  159. package/src/stream/pipeline/pipeline.ts +1 -1
  160. package/src/stream/progressLogger.ts +3 -3
  161. package/src/stream/readable/readableCreate.ts +1 -1
  162. package/src/stream/readable/readableForEach.ts +4 -4
  163. package/src/stream/readable/readableFromArray.ts +1 -1
  164. package/src/stream/readable/readableToArray.ts +1 -1
  165. package/src/stream/sizeStack.ts +2 -2
  166. package/src/stream/transform/transformChunk.ts +1 -1
  167. package/src/stream/transform/transformFilter.ts +3 -3
  168. package/src/stream/transform/transformLimit.ts +3 -3
  169. package/src/stream/transform/transformLogProgress.ts +3 -3
  170. package/src/stream/transform/transformMap.ts +5 -5
  171. package/src/stream/transform/transformMapSimple.ts +1 -1
  172. package/src/stream/transform/transformMapSync.ts +5 -5
  173. package/src/stream/transform/transformNoOp.ts +1 -1
  174. package/src/stream/transform/transformOffset.ts +2 -2
  175. package/src/stream/transform/transformSplit.ts +1 -1
  176. package/src/stream/transform/transformTap.ts +1 -1
  177. package/src/stream/transform/transformTee.ts +3 -3
  178. package/src/stream/transform/transformThrottle.ts +1 -1
  179. package/src/stream/transform/transformToArray.ts +1 -1
  180. package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
  181. package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
  182. package/src/stream/transform/worker/workerClassProxy.js +6 -4
  183. package/src/stream/writable/writableForEach.ts +3 -3
  184. package/src/stream/writable/writableFork.ts +3 -3
  185. package/src/stream/writable/writableLimit.ts +1 -1
  186. package/src/stream/writable/writablePushToArray.ts +1 -1
  187. package/src/stream/writable/writableVoid.ts +1 -1
  188. package/src/util/buildInfo.util.ts +2 -2
  189. package/src/util/exec2.ts +1 -1
  190. package/src/util/git2.ts +2 -2
  191. package/src/validation/ajv/ajv.util.ts +4 -4
  192. package/src/validation/ajv/ajvSchema.ts +4 -5
  193. package/src/validation/ajv/getAjv.ts +7 -3
  194. package/src/validation/joi/joi.extensions.ts +4 -4
  195. package/src/validation/joi/joi.shared.schemas.ts +3 -3
  196. package/src/validation/joi/joi.validation.util.ts +2 -2
  197. package/src/yargs.util.ts +1 -0
@@ -1,14 +1,6 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.NDJsonStats = void 0;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const colors_1 = require("../../colors/colors");
6
- class NDJsonStats {
7
- constructor() {
8
- this.tookMillis = 0;
9
- this.rows = 0;
10
- this.sizeBytes = 0;
11
- }
1
+ import { _hb, _ms } from '@naturalcycles/js-lib';
2
+ import { boldWhite, dimWhite } from '../../colors/colors.js';
3
+ export class NDJsonStats {
12
4
  static create(o = {}) {
13
5
  return Object.assign(new NDJsonStats(), o);
14
6
  }
@@ -19,6 +11,9 @@ class NDJsonStats {
19
11
  // eslint-disable-next-line unicorn/no-array-reduce
20
12
  return stats.reduce((statsTotal, stats) => statsTotal.add(stats), new NDJsonStats());
21
13
  }
14
+ tookMillis = 0;
15
+ rows = 0;
16
+ sizeBytes = 0;
22
17
  get rpsTotal() {
23
18
  return Math.round(this.rows / ((this.tookMillis || 1) / 1000));
24
19
  }
@@ -40,11 +35,10 @@ class NDJsonStats {
40
35
  }
41
36
  toPretty(name) {
42
37
  return [
43
- `Processed ${name ? (0, colors_1.boldWhite)(name) + ': ' : ''}${(0, colors_1.dimWhite)(this.rows)} rows, ${(0, colors_1.dimWhite)((0, js_lib_1._hb)(this.sizeBytes))} in ${(0, colors_1.dimWhite)((0, js_lib_1._ms)(this.tookMillis))}`,
44
- (0, colors_1.dimWhite)(this.rpsTotal + ' rows/sec'),
45
- (0, colors_1.dimWhite)((0, js_lib_1._hb)(this.avgBytesPerRow) + '/row'),
46
- (0, colors_1.dimWhite)((0, js_lib_1._hb)(this.bpsTotal) + '/sec'),
38
+ `Processed ${name ? boldWhite(name) + ': ' : ''}${dimWhite(this.rows)} rows, ${dimWhite(_hb(this.sizeBytes))} in ${dimWhite(_ms(this.tookMillis))}`,
39
+ dimWhite(this.rpsTotal + ' rows/sec'),
40
+ dimWhite(_hb(this.avgBytesPerRow) + '/row'),
41
+ dimWhite(_hb(this.bpsTotal) + '/sec'),
47
42
  ].join(', ');
48
43
  }
49
44
  }
50
- exports.NDJsonStats = NDJsonStats;
@@ -1,5 +1,5 @@
1
1
  import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
2
- import type { TransformLogProgressOptions, TransformMapOptions } from '../..';
2
+ import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
3
3
  export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT>, TransformLogProgressOptions<IN> {
4
4
  inputFilePath: string;
5
5
  outputFilePath: string;
@@ -1,31 +1,28 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ndjsonMap = ndjsonMap;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const __1 = require("../..");
1
+ import { ErrorMode } from '@naturalcycles/js-lib';
2
+ import { _pipeline, fs2, transformLimit, transformLogProgress, transformMap } from '../../index.js';
6
3
  /**
7
4
  * Unzips input file automatically, if it ends with `.gz`.
8
5
  * Zips output file automatically, if it ends with `.gz`.
9
6
  */
10
- async function ndjsonMap(mapper, opt) {
7
+ export async function ndjsonMap(mapper, opt) {
11
8
  const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;
12
9
  console.log({
13
10
  inputFilePath,
14
11
  outputFilePath,
15
12
  });
16
- const readable = __1.fs2
13
+ const readable = fs2
17
14
  .createReadStreamAsNDJSON(inputFilePath)
18
15
  .take(limitInput || Number.POSITIVE_INFINITY);
19
- await (0, __1._pipeline)([
16
+ await _pipeline([
20
17
  readable,
21
- (0, __1.transformLogProgress)({ metric: 'read', ...opt }),
22
- (0, __1.transformMap)(mapper, {
18
+ transformLogProgress({ metric: 'read', ...opt }),
19
+ transformMap(mapper, {
23
20
  flattenArrayOutput: true,
24
- errorMode: js_lib_1.ErrorMode.SUPPRESS,
21
+ errorMode: ErrorMode.SUPPRESS,
25
22
  ...opt,
26
23
  }),
27
- (0, __1.transformLimit)({ limit: limitOutput, sourceReadable: readable }),
28
- (0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }),
29
- ...__1.fs2.createWriteStreamAsNDJSON(outputFilePath),
24
+ transformLimit({ limit: limitOutput, sourceReadable: readable }),
25
+ transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
26
+ ...fs2.createWriteStreamAsNDJSON(outputFilePath),
30
27
  ]);
31
28
  }
@@ -1,5 +1,5 @@
1
1
  import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
2
- import type { TransformLogProgressOptions, TransformMapOptions } from '../..';
2
+ import type { TransformLogProgressOptions, TransformMapOptions } from '../../index.js';
3
3
  export interface NDJSONStreamForEachOptions<IN = any> extends TransformMapOptions<IN, void>, TransformLogProgressOptions<IN> {
4
4
  inputFilePath: string;
5
5
  }
@@ -1,20 +1,17 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ndjsonStreamForEach = ndjsonStreamForEach;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const __1 = require("../..");
1
+ import { ErrorMode } from '@naturalcycles/js-lib';
2
+ import { _pipeline, fs2, transformLogProgress, transformMap, writableVoid } from '../../index.js';
6
3
  /**
7
4
  * Convenience function to `forEach` through an ndjson file.
8
5
  */
9
- async function ndjsonStreamForEach(mapper, opt) {
10
- await (0, __1._pipeline)([
11
- __1.fs2.createReadStreamAsNDJSON(opt.inputFilePath),
12
- (0, __1.transformMap)(mapper, {
13
- errorMode: js_lib_1.ErrorMode.THROW_AGGREGATED,
6
+ export async function ndjsonStreamForEach(mapper, opt) {
7
+ await _pipeline([
8
+ fs2.createReadStreamAsNDJSON(opt.inputFilePath),
9
+ transformMap(mapper, {
10
+ errorMode: ErrorMode.THROW_AGGREGATED,
14
11
  ...opt,
15
12
  predicate: () => true, // to log progress properly
16
13
  }),
17
- (0, __1.transformLogProgress)(opt),
18
- (0, __1.writableVoid)(),
14
+ transformLogProgress(opt),
15
+ writableVoid(),
19
16
  ]);
20
17
  }
@@ -1,5 +1,5 @@
1
1
  import type { Reviver } from '@naturalcycles/js-lib';
2
- import type { TransformTyped } from '../stream.model';
2
+ import type { TransformTyped } from '../stream.model.js';
3
3
  export interface TransformJsonParseOptions {
4
4
  /**
5
5
  * If true - will throw an error on JSON.parse / stringify error
@@ -1,8 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.bufferReviver = void 0;
4
- exports.transformJsonParse = transformJsonParse;
5
- const node_stream_1 = require("node:stream");
1
+ import { Transform } from 'node:stream';
6
2
  /**
7
3
  * Transforms chunks of JSON strings/Buffers (objectMode=false) into parsed objects (readableObjectMode=true).
8
4
  *
@@ -17,9 +13,9 @@ const node_stream_1 = require("node:stream");
17
13
  * consumeYourStream...
18
14
  * [)
19
15
  */
20
- function transformJsonParse(opt = {}) {
16
+ export function transformJsonParse(opt = {}) {
21
17
  const { strict = true, reviver } = opt;
22
- return new node_stream_1.Transform({
18
+ return new Transform({
23
19
  writableObjectMode: false,
24
20
  readableObjectMode: true,
25
21
  // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
@@ -43,10 +39,9 @@ function transformJsonParse(opt = {}) {
43
39
  });
44
40
  }
45
41
  // Based on: https://stackoverflow.com/a/34557997/4919972
46
- const bufferReviver = (_k, v) => {
42
+ export const bufferReviver = (_k, v) => {
47
43
  if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
48
44
  return Buffer.from(v.data);
49
45
  }
50
46
  return v;
51
47
  };
52
- exports.bufferReviver = bufferReviver;
@@ -1,4 +1,4 @@
1
- import type { TransformTyped } from '../stream.model';
1
+ import type { TransformTyped } from '../stream.model.js';
2
2
  export interface TransformToNDJsonOptions {
3
3
  /**
4
4
  * If true - will throw an error on JSON.parse / stringify error
@@ -1,21 +1,18 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.transformToNDJson = transformToNDJson;
4
- const node_stream_1 = require("node:stream");
5
- const js_lib_1 = require("@naturalcycles/js-lib");
1
+ import { Transform } from 'node:stream';
2
+ import { _sortObjectDeep } from '@naturalcycles/js-lib';
6
3
  /**
7
4
  * Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
8
5
  */
9
- function transformToNDJson(opt = {}) {
6
+ export function transformToNDJson(opt = {}) {
10
7
  const { strict = true, separator = '\n', sortObjects = false } = opt;
11
- return new node_stream_1.Transform({
8
+ return new Transform({
12
9
  writableObjectMode: true,
13
10
  readableObjectMode: false,
14
11
  readableHighWaterMark: 64 * 1024,
15
12
  transform(chunk, _, cb) {
16
13
  try {
17
14
  if (sortObjects) {
18
- chunk = (0, js_lib_1._sortObjectDeep)(chunk);
15
+ chunk = _sortObjectDeep(chunk);
19
16
  }
20
17
  cb(null, JSON.stringify(chunk) + separator);
21
18
  }
@@ -1,19 +1,14 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AbortableTransform = void 0;
4
- exports._pipeline = _pipeline;
5
- exports._pipelineToArray = _pipelineToArray;
6
- const node_stream_1 = require("node:stream");
7
- const promises_1 = require("node:stream/promises");
8
- const js_lib_1 = require("@naturalcycles/js-lib");
9
- const writablePushToArray_1 = require("../writable/writablePushToArray");
1
+ import { Transform } from 'node:stream';
2
+ import { pipeline } from 'node:stream/promises';
3
+ import { _last, pDefer } from '@naturalcycles/js-lib';
4
+ import { writablePushToArray } from '../writable/writablePushToArray.js';
10
5
  /**
11
6
  * Promisified `stream.pipeline`.
12
7
  *
13
8
  * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
14
9
  * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
15
10
  */
16
- async function _pipeline(streams, opt = {}) {
11
+ export async function _pipeline(streams, opt = {}) {
17
12
  const first = streams[0];
18
13
  const rest = streams.slice(1);
19
14
  if (opt.allowClose) {
@@ -35,9 +30,9 @@ async function _pipeline(streams, opt = {}) {
35
30
  // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
36
31
  // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
37
32
  // inevitable in this flow). Know a better way to close the stream? Tell me!
38
- const streamDone = (0, js_lib_1.pDefer)();
33
+ const streamDone = pDefer();
39
34
  const sourceReadable = first;
40
- const last = (0, js_lib_1._last)(streams);
35
+ const last = _last(streams);
41
36
  const lastFinal = last._final?.bind(last) || ((cb) => cb());
42
37
  last._final = cb => {
43
38
  lastFinal(() => {
@@ -56,7 +51,7 @@ async function _pipeline(streams, opt = {}) {
56
51
  });
57
52
  }
58
53
  try {
59
- return await (0, promises_1.pipeline)([first, ...rest], opt);
54
+ return await pipeline([first, ...rest], opt);
60
55
  }
61
56
  catch (err) {
62
57
  if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
@@ -75,11 +70,12 @@ async function _pipeline(streams, opt = {}) {
75
70
  * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
76
71
  * and return.
77
72
  */
78
- async function _pipelineToArray(streams, opt = {}) {
73
+ export async function _pipelineToArray(streams, opt = {}) {
79
74
  const a = [];
80
- await _pipeline([...streams, (0, writablePushToArray_1.writablePushToArray)(a)], opt);
75
+ await _pipeline([...streams, writablePushToArray(a)], opt);
81
76
  return a;
82
77
  }
83
- class AbortableTransform extends node_stream_1.Transform {
78
+ export class AbortableTransform extends Transform {
79
+ sourceReadable;
80
+ streamDone;
84
81
  }
85
- exports.AbortableTransform = AbortableTransform;
@@ -1,5 +1,5 @@
1
1
  import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
2
- import type { ReadableMapper } from './stream.model';
2
+ import type { ReadableMapper } from './stream.model.js';
3
3
  export interface ProgressLoggerCfg<T = any> {
4
4
  /**
5
5
  * Progress metric
@@ -1,17 +1,12 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ProgressLogger = void 0;
4
- exports.progressLogger = progressLogger;
5
- exports.progressReadableMapper = progressReadableMapper;
6
- const node_util_1 = require("node:util");
7
- const js_lib_1 = require("@naturalcycles/js-lib");
8
- const colors_1 = require("../colors/colors");
9
- const sizeStack_1 = require("./sizeStack");
1
+ import { inspect } from 'node:util';
2
+ import { _mb, _since, localTime, SimpleMovingAverage } from '@naturalcycles/js-lib';
3
+ import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
4
+ import { SizeStack } from './sizeStack.js';
10
5
  const inspectOpt = {
11
- colors: colors_1.hasColors,
6
+ colors: hasColors,
12
7
  breakLength: 300,
13
8
  };
14
- class ProgressLogger {
9
+ export class ProgressLogger {
15
10
  constructor(cfg = {}) {
16
11
  this.cfg = {
17
12
  metric: 'progress',
@@ -29,16 +24,26 @@ class ProgressLogger {
29
24
  this.start();
30
25
  this.logStats(); // initial
31
26
  }
27
+ cfg;
28
+ started;
29
+ lastSecondStarted;
30
+ sma;
31
+ logEvery10;
32
+ processedLastSecond;
33
+ progress;
34
+ peakRSS;
35
+ sizes;
36
+ sizesZipped;
32
37
  start() {
33
38
  this.started = Date.now();
34
39
  this.lastSecondStarted = Date.now();
35
- this.sma = new js_lib_1.SimpleMovingAverage(10);
40
+ this.sma = new SimpleMovingAverage(10);
36
41
  this.processedLastSecond = 0;
37
42
  this.progress = 0;
38
43
  this.peakRSS = 0;
39
- this.sizes = this.cfg.logSizes ? new sizeStack_1.SizeStack('json', this.cfg.logSizesBuffer) : undefined;
44
+ this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
40
45
  this.sizesZipped = this.cfg.logZippedSizes
41
- ? new sizeStack_1.SizeStack('json.gz', this.cfg.logSizesBuffer)
46
+ ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
42
47
  : undefined;
43
48
  }
44
49
  log(chunk) {
@@ -46,7 +51,7 @@ class ProgressLogger {
46
51
  this.processedLastSecond++;
47
52
  if (this.sizes) {
48
53
  // Check it, cause gzipping might be delayed here..
49
- void sizeStack_1.SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
54
+ void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
50
55
  }
51
56
  if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
52
57
  this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
@@ -78,22 +83,22 @@ class ProgressLogger {
78
83
  if (extra)
79
84
  Object.assign(o, extra(chunk, this.progress));
80
85
  if (logHeapUsed)
81
- o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
86
+ o.heapUsed = _mb(mem.heapUsed);
82
87
  if (logHeapTotal)
83
- o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
88
+ o.heapTotal = _mb(mem.heapTotal);
84
89
  if (logRss)
85
- o.rss = (0, js_lib_1._mb)(mem.rss);
90
+ o.rss = _mb(mem.rss);
86
91
  if (logPeakRss)
87
- o.peakRSS = (0, js_lib_1._mb)(this.peakRSS);
92
+ o.peakRSS = _mb(this.peakRSS);
88
93
  if (rssMinusHeap)
89
- o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
94
+ o.rssMinusHeap = _mb(mem.rss - mem.heapTotal);
90
95
  if (external)
91
- o.external = (0, js_lib_1._mb)(mem.external);
96
+ o.external = _mb(mem.external);
92
97
  if (arrayBuffers)
93
- o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
98
+ o.arrayBuffers = _mb(mem.arrayBuffers || 0);
94
99
  if (logRPS)
95
100
  Object.assign(o, { rps10, rpsTotal });
96
- logger.log((0, node_util_1.inspect)(o, inspectOpt));
101
+ logger.log(inspect(o, inspectOpt));
97
102
  if (this.sizes?.items.length) {
98
103
  logger.log(this.sizes.getStats());
99
104
  if (this.sizesZipped?.items.length) {
@@ -105,10 +110,10 @@ class ProgressLogger {
105
110
  if (perHour > 900) {
106
111
  perHour = Math.round(perHour / 1000) + 'K';
107
112
  }
108
- logger.log(`${(0, colors_1.dimGrey)(js_lib_1.localTime.now().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
113
+ logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(batchedProgress)} rows, ~${yellow(perHour)}/hour`);
109
114
  }
110
115
  else if (final) {
111
- logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(this.started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
116
+ logger.log(`${boldWhite(metric)} took ${yellow(_since(this.started))} to process ${yellow(batchedProgress)} rows with total RPS of ${yellow(rpsTotal)}`);
112
117
  try {
113
118
  this.cfg.onProgressDone?.(o);
114
119
  }
@@ -118,11 +123,10 @@ class ProgressLogger {
118
123
  }
119
124
  }
120
125
  }
121
- exports.ProgressLogger = ProgressLogger;
122
126
  /**
123
127
  * Create new ProgressLogger.
124
128
  */
125
- function progressLogger(cfg = {}) {
129
+ export function progressLogger(cfg = {}) {
126
130
  return new ProgressLogger(cfg);
127
131
  }
128
132
  /**
@@ -130,7 +134,7 @@ function progressLogger(cfg = {}) {
130
134
  *
131
135
  * @experimental
132
136
  */
133
- function progressReadableMapper(cfg = {}) {
137
+ export function progressReadableMapper(cfg = {}) {
134
138
  const progress = new ProgressLogger(cfg);
135
139
  return chunk => {
136
140
  progress.log(chunk);
@@ -1,5 +1,5 @@
1
1
  import type { ReadableOptions } from 'node:stream';
2
- import type { ReadableTyped } from '../stream.model';
2
+ import type { ReadableTyped } from '../stream.model.js';
3
3
  /**
4
4
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
5
5
  * Push `null` to it to complete (similar to RxJS `.complete()`).
@@ -1,8 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableCreate = readableCreate;
4
- exports.readableFrom = readableFrom;
5
- const node_stream_1 = require("node:stream");
1
+ import { Readable } from 'node:stream';
6
2
  /**
7
3
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
8
4
  * Push `null` to it to complete (similar to RxJS `.complete()`).
@@ -17,8 +13,8 @@ const node_stream_1 = require("node:stream");
17
13
  *
18
14
  * @deprecated because of the caution above
19
15
  */
20
- function readableCreate(items = [], opt) {
21
- const readable = new node_stream_1.Readable({
16
+ export function readableCreate(items = [], opt) {
17
+ const readable = new Readable({
22
18
  objectMode: true,
23
19
  ...opt,
24
20
  read() { }, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
@@ -31,6 +27,6 @@ function readableCreate(items = [], opt) {
31
27
  /**
32
28
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
33
29
  */
34
- function readableFrom(iterable, opt) {
35
- return node_stream_1.Readable.from(iterable, opt);
30
+ export function readableFrom(iterable, opt) {
31
+ return Readable.from(iterable, opt);
36
32
  }
@@ -1,6 +1,6 @@
1
1
  import type { AbortableAsyncMapper, Mapper } from '@naturalcycles/js-lib';
2
- import type { ReadableTyped } from '../../index';
3
- import type { TransformMapOptions } from '../transform/transformMap';
2
+ import type { ReadableTyped } from '../../index.js';
3
+ import type { TransformMapOptions } from '../transform/transformMap.js';
4
4
  /**
5
5
  * Convenience function to do `.forEach` over a Readable.
6
6
  * Typed! (unlike default Readable).
@@ -1,10 +1,6 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableForEach = readableForEach;
4
- exports.readableForEachSync = readableForEachSync;
5
- const js_lib_1 = require("@naturalcycles/js-lib");
6
- const index_1 = require("../../index");
7
- const transformMap_1 = require("../transform/transformMap");
1
+ import { _passNothingPredicate } from '@naturalcycles/js-lib';
2
+ import { _pipeline } from '../../index.js';
3
+ import { transformMap } from '../transform/transformMap.js';
8
4
  /**
9
5
  * Convenience function to do `.forEach` over a Readable.
10
6
  * Typed! (unlike default Readable).
@@ -13,10 +9,10 @@ const transformMap_1 = require("../transform/transformMap");
13
9
  *
14
10
  * @experimental
15
11
  */
16
- async function readableForEach(readable, mapper, opt = {}) {
17
- await (0, index_1._pipeline)([
12
+ export async function readableForEach(readable, mapper, opt = {}) {
13
+ await _pipeline([
18
14
  readable,
19
- (0, transformMap_1.transformMap)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate }),
15
+ transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
20
16
  ]);
21
17
  }
22
18
  /**
@@ -25,7 +21,7 @@ async function readableForEach(readable, mapper, opt = {}) {
25
21
  *
26
22
  * @experimental
27
23
  */
28
- async function readableForEachSync(readable, mapper) {
24
+ export async function readableForEachSync(readable, mapper) {
29
25
  // async iteration
30
26
  let index = 0;
31
27
  for await (const item of readable) {
@@ -1,6 +1,6 @@
1
1
  import type { ReadableOptions } from 'node:stream';
2
2
  import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
3
- import type { ReadableTyped } from '../stream.model';
3
+ import type { ReadableTyped } from '../stream.model.js';
4
4
  /**
5
5
  * Create Readable from Array.
6
6
  * Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
@@ -1,17 +1,14 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableFromArray = readableFromArray;
4
- const node_stream_1 = require("node:stream");
5
- const js_lib_1 = require("@naturalcycles/js-lib");
1
+ import { Readable } from 'node:stream';
2
+ import { _passthroughMapper } from '@naturalcycles/js-lib';
6
3
  /**
7
4
  * Create Readable from Array.
8
5
  * Supports a `mapper` function (async) that you can use to e.g create a timer-emitting-readable.
9
6
  *
10
7
  * For simple cases use Readable.from(...) (Node.js 12+)
11
8
  */
12
- function readableFromArray(items, mapper = js_lib_1._passthroughMapper, opt) {
9
+ export function readableFromArray(items, mapper = _passthroughMapper, opt) {
13
10
  let i = -1;
14
- return new node_stream_1.Readable({
11
+ return new Readable({
15
12
  objectMode: true,
16
13
  ...opt,
17
14
  async read() {
@@ -1,4 +1,4 @@
1
- import type { ReadableTyped } from '../stream.model';
1
+ import type { ReadableTyped } from '../stream.model.js';
2
2
  /**
3
3
  * Convenience function to read the whole Readable stream into Array (in-memory)
4
4
  * and return that array.
@@ -1,6 +1,3 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.readableToArray = readableToArray;
4
1
  /**
5
2
  * Convenience function to read the whole Readable stream into Array (in-memory)
6
3
  * and return that array.
@@ -8,7 +5,7 @@ exports.readableToArray = readableToArray;
8
5
  * Native `await readable.toArray()` can be used instead.
9
6
  * This helper is kept for type-safery support.
10
7
  */
11
- async function readableToArray(readable) {
8
+ export async function readableToArray(readable) {
12
9
  return await readable.toArray();
13
10
  // const a: T[] = []
14
11
  //
@@ -1,15 +1,13 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SizeStack = void 0;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const colors_1 = require("../colors/colors");
6
- const zip_util_1 = require("../util/zip.util");
7
- class SizeStack extends js_lib_1.NumberStack {
1
+ import { _hb, NumberStack } from '@naturalcycles/js-lib';
2
+ import { yellow } from '../colors/colors.js';
3
+ import { gzipBuffer } from '../util/zip.util.js';
4
+ export class SizeStack extends NumberStack {
5
+ name;
8
6
  constructor(name, size) {
9
7
  super(size);
10
8
  this.name = name;
11
- this.total = 0;
12
9
  }
10
+ total = 0;
13
11
  push(item) {
14
12
  this.total += item;
15
13
  return super.push(item);
@@ -19,13 +17,13 @@ class SizeStack extends js_lib_1.NumberStack {
19
17
  return [
20
18
  ' ' + this.name,
21
19
  'avg',
22
- (0, colors_1.yellow)((0, js_lib_1._hb)(this.avg())),
20
+ yellow(_hb(this.avg())),
23
21
  // 'p50',
24
22
  // yellow(_hb(pcs[50])),
25
23
  // 'p90',
26
24
  // yellow(_hb(pcs[90])),
27
25
  'total',
28
- (0, colors_1.yellow)((0, js_lib_1._hb)(this.total)),
26
+ yellow(_hb(this.total)),
29
27
  ].join(' ');
30
28
  }
31
29
  static async countItem(item, logger, sizes, sizesZipped) {
@@ -36,7 +34,7 @@ class SizeStack extends js_lib_1.NumberStack {
36
34
  const buf = Buffer.from(JSON.stringify(item));
37
35
  sizes.push(buf.byteLength);
38
36
  if (sizesZipped) {
39
- const { byteLength } = await (0, zip_util_1.gzipBuffer)(buf);
37
+ const { byteLength } = await gzipBuffer(buf);
40
38
  sizesZipped.push(byteLength);
41
39
  }
42
40
  }
@@ -45,4 +43,3 @@ class SizeStack extends js_lib_1.NumberStack {
45
43
  }
46
44
  }
47
45
  }
48
- exports.SizeStack = SizeStack;
@@ -1,2 +1 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
@@ -1,7 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.pipelineClose = pipelineClose;
4
- function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
1
+ export function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
5
2
  readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
6
3
  if (!sourceReadable) {
7
4
  logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
@@ -1,4 +1,4 @@
1
- import type { TransformOptions, TransformTyped } from '../stream.model';
1
+ import type { TransformOptions, TransformTyped } from '../stream.model.js';
2
2
  export interface TransformChunkOptions extends TransformOptions {
3
3
  /**
4
4
  * How many items to include in each chunk.
@@ -1,16 +1,13 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.transformChunk = transformChunk;
4
- const node_stream_1 = require("node:stream");
1
+ import { Transform } from 'node:stream';
5
2
  /**
6
3
  * Similar to RxJS bufferCount(),
7
4
  * allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
8
5
  * Last chunk will contain the remaining items, possibly less than chunkSize.
9
6
  */
10
- function transformChunk(opt) {
7
+ export function transformChunk(opt) {
11
8
  const { chunkSize } = opt;
12
9
  let buf = [];
13
- return new node_stream_1.Transform({
10
+ return new Transform({
14
11
  objectMode: true,
15
12
  ...opt,
16
13
  transform(chunk, _, cb) {