@naturalcycles/nodejs-lib 15.1.0 → 15.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/bin/slack-this.js +1 -1
  2. package/dist/{util → cache}/lruMemoCache.d.ts +1 -1
  3. package/dist/csv/csvReader.d.ts +1 -1
  4. package/dist/csv/csvReader.js +1 -1
  5. package/dist/csv/csvWriter.d.ts +1 -1
  6. package/dist/csv/csvWriter.js +1 -1
  7. package/dist/csv/transformToCSV.d.ts +1 -1
  8. package/dist/diff/tableDiff.d.ts +1 -1
  9. package/dist/diff/tableDiff.js +1 -1
  10. package/dist/{util/exec2.d.ts → exec2.d.ts} +2 -2
  11. package/dist/{util/exec2.js → exec2.js} +4 -3
  12. package/dist/fs/fs2.d.ts +0 -10
  13. package/dist/fs/fs2.js +1 -85
  14. package/dist/fs/json2env.d.ts +1 -1
  15. package/dist/fs/kpy.js +1 -1
  16. package/dist/fs/yaml2.d.ts +11 -0
  17. package/dist/fs/yaml2.js +29 -0
  18. package/dist/index.d.ts +0 -4
  19. package/dist/index.js +0 -4
  20. package/dist/jwt/jwt.service.d.ts +2 -1
  21. package/dist/jwt/jwt.service.js +1 -1
  22. package/dist/log/log.util.d.ts +1 -1
  23. package/dist/log/log.util.js +1 -1
  24. package/dist/script/runScript.d.ts +1 -2
  25. package/dist/script/runScript.js +6 -5
  26. package/dist/secret/secrets-decrypt.util.js +1 -1
  27. package/dist/secret/secrets-encrypt.util.js +1 -1
  28. package/dist/security/crypto.util.d.ts +1 -1
  29. package/dist/security/crypto.util.js +1 -1
  30. package/dist/security/hash.util.d.ts +1 -1
  31. package/dist/security/secret.util.d.ts +1 -1
  32. package/dist/security/secret.util.js +2 -1
  33. package/dist/slack/slack.service.d.ts +2 -1
  34. package/dist/slack/slack.service.js +5 -1
  35. package/dist/slack/slack.service.model.d.ts +2 -1
  36. package/dist/stream/index.d.ts +2 -0
  37. package/dist/stream/index.js +2 -0
  38. package/dist/stream/ndjson/createReadStreamAsNDJSON.d.ts +19 -0
  39. package/dist/stream/ndjson/createReadStreamAsNDJSON.js +38 -0
  40. package/dist/stream/ndjson/createWriteStreamAsNDJSON.d.ts +11 -0
  41. package/dist/stream/ndjson/createWriteStreamAsNDJSON.js +27 -0
  42. package/dist/stream/ndjson/ndjson.model.js +2 -1
  43. package/dist/stream/ndjson/ndjsonMap.d.ts +2 -2
  44. package/dist/stream/ndjson/ndjsonMap.js +4 -6
  45. package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +2 -2
  46. package/dist/stream/ndjson/ndjsonStreamForEach.js +3 -3
  47. package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
  48. package/dist/stream/ndjson/transformToNDJson.js +1 -1
  49. package/dist/stream/pipeline/pipeline.d.ts +1 -1
  50. package/dist/stream/pipeline/pipeline.js +2 -1
  51. package/dist/stream/progressLogger.d.ts +2 -1
  52. package/dist/stream/progressLogger.js +3 -1
  53. package/dist/stream/readable/readableForEach.d.ts +1 -1
  54. package/dist/stream/readable/readableForEach.js +1 -1
  55. package/dist/stream/readable/readableFromArray.d.ts +1 -1
  56. package/dist/stream/readable/readableFromArray.js +1 -1
  57. package/dist/stream/sizeStack.d.ts +2 -2
  58. package/dist/stream/sizeStack.js +3 -2
  59. package/dist/stream/stream.model.d.ts +1 -1
  60. package/dist/stream/stream.util.d.ts +1 -1
  61. package/dist/stream/transform/transformFilter.d.ts +1 -1
  62. package/dist/stream/transform/transformLimit.d.ts +1 -1
  63. package/dist/stream/transform/transformMap.d.ts +3 -2
  64. package/dist/stream/transform/transformMap.js +6 -1
  65. package/dist/stream/transform/transformMapSimple.d.ts +3 -2
  66. package/dist/stream/transform/transformMapSimple.js +1 -1
  67. package/dist/stream/transform/transformMapSync.d.ts +4 -2
  68. package/dist/stream/transform/transformMapSync.js +2 -1
  69. package/dist/stream/transform/transformTap.d.ts +2 -1
  70. package/dist/stream/transform/transformThrottle.d.ts +1 -1
  71. package/dist/stream/transform/transformThrottle.js +2 -1
  72. package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
  73. package/dist/stream/transform/worker/transformMultiThreaded.js +2 -1
  74. package/dist/stream/writable/writableForEach.d.ts +1 -1
  75. package/dist/stream/writable/writableForEach.js +1 -1
  76. package/dist/stream/writable/writableVoid.d.ts +1 -1
  77. package/dist/string/inspect.d.ts +1 -1
  78. package/dist/string/inspect.js +1 -1
  79. package/dist/util/buildInfo.util.d.ts +2 -1
  80. package/dist/util/buildInfo.util.js +2 -1
  81. package/dist/util/env.util.d.ts +1 -2
  82. package/dist/util/env.util.js +0 -1
  83. package/dist/util/git2.d.ts +1 -1
  84. package/dist/util/git2.js +2 -3
  85. package/dist/validation/ajv/ajv.util.d.ts +1 -1
  86. package/dist/validation/ajv/ajvSchema.d.ts +1 -1
  87. package/dist/validation/ajv/ajvSchema.js +4 -1
  88. package/dist/validation/ajv/ajvValidationError.d.ts +2 -2
  89. package/dist/validation/ajv/ajvValidationError.js +1 -1
  90. package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
  91. package/dist/validation/joi/joi.validation.error.d.ts +2 -2
  92. package/dist/validation/joi/joi.validation.error.js +1 -1
  93. package/dist/validation/joi/joi.validation.util.js +2 -1
  94. package/dist/validation/joi/string.extensions.d.ts +1 -1
  95. package/dist/validation/joi/string.extensions.js +1 -1
  96. package/dist/{util → zip}/zip.util.js +0 -1
  97. package/package.json +12 -5
  98. package/src/bin/generate-build-info.ts +1 -1
  99. package/src/bin/slack-this.ts +1 -1
  100. package/src/{util → cache}/lruMemoCache.ts +1 -1
  101. package/src/csv/csvReader.ts +2 -2
  102. package/src/csv/csvWriter.ts +2 -2
  103. package/src/csv/transformToCSV.ts +1 -1
  104. package/src/diff/tableDiff.ts +2 -2
  105. package/src/{util/exec2.ts → exec2.ts} +9 -8
  106. package/src/fs/fs2.ts +1 -101
  107. package/src/fs/json2env.ts +1 -1
  108. package/src/fs/kpy.ts +2 -2
  109. package/src/fs/yaml2.ts +37 -0
  110. package/src/index.ts +0 -4
  111. package/src/jwt/jwt.service.ts +3 -2
  112. package/src/log/log.util.ts +1 -1
  113. package/src/script/runScript.ts +10 -6
  114. package/src/secret/secrets-decrypt.util.ts +1 -1
  115. package/src/secret/secrets-encrypt.util.ts +1 -1
  116. package/src/security/crypto.util.ts +1 -2
  117. package/src/security/hash.util.ts +1 -1
  118. package/src/security/secret.util.ts +3 -2
  119. package/src/slack/slack.service.model.ts +2 -1
  120. package/src/slack/slack.service.ts +10 -2
  121. package/src/stream/index.ts +2 -0
  122. package/src/stream/ndjson/createReadStreamAsNDJSON.ts +46 -0
  123. package/src/stream/ndjson/createWriteStreamAsNDJSON.ts +30 -0
  124. package/src/stream/ndjson/ndjson.model.ts +2 -1
  125. package/src/stream/ndjson/ndjsonMap.ts +12 -8
  126. package/src/stream/ndjson/ndjsonStreamForEach.ts +8 -5
  127. package/src/stream/ndjson/transformJsonParse.ts +1 -1
  128. package/src/stream/ndjson/transformToNDJson.ts +1 -1
  129. package/src/stream/pipeline/pipeline.ts +4 -2
  130. package/src/stream/progressLogger.ts +5 -2
  131. package/src/stream/readable/readableForEach.ts +2 -2
  132. package/src/stream/readable/readableFromArray.ts +2 -2
  133. package/src/stream/sizeStack.ts +4 -3
  134. package/src/stream/stream.model.ts +1 -1
  135. package/src/stream/stream.util.ts +1 -1
  136. package/src/stream/transform/transformFilter.ts +1 -1
  137. package/src/stream/transform/transformLimit.ts +1 -1
  138. package/src/stream/transform/transformMap.ts +12 -15
  139. package/src/stream/transform/transformMapSimple.ts +3 -2
  140. package/src/stream/transform/transformMapSync.ts +4 -7
  141. package/src/stream/transform/transformTap.ts +2 -1
  142. package/src/stream/transform/transformThrottle.ts +4 -3
  143. package/src/stream/transform/worker/transformMultiThreaded.ts +4 -2
  144. package/src/stream/writable/writableForEach.ts +2 -2
  145. package/src/stream/writable/writableVoid.ts +1 -1
  146. package/src/string/inspect.ts +2 -2
  147. package/src/util/buildInfo.util.ts +4 -2
  148. package/src/util/env.util.ts +1 -2
  149. package/src/util/git2.ts +3 -4
  150. package/src/validation/ajv/ajv.util.ts +1 -1
  151. package/src/validation/ajv/ajvSchema.ts +5 -7
  152. package/src/validation/ajv/ajvValidationError.ts +2 -2
  153. package/src/validation/joi/joi.shared.schemas.ts +7 -7
  154. package/src/validation/joi/joi.validation.error.ts +2 -2
  155. package/src/validation/joi/joi.validation.util.ts +2 -1
  156. package/src/validation/joi/string.extensions.ts +2 -2
  157. package/src/{util → zip}/zip.util.ts +0 -2
  158. package/dist/fs/index.d.ts +0 -3
  159. package/dist/fs/index.js +0 -3
  160. package/dist/jwt/index.d.ts +0 -1
  161. package/dist/jwt/index.js +0 -1
  162. package/dist/yargs/index.d.ts +0 -1
  163. package/dist/yargs/index.js +0 -1
  164. package/src/fs/index.ts +0 -3
  165. package/src/jwt/index.ts +0 -1
  166. package/src/yargs/index.ts +0 -1
  167. /package/dist/{util → cache}/lruMemoCache.js +0 -0
  168. /package/dist/{util → zip}/zip.util.d.ts +0 -0
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  import { SlackService } from '../index.js';
3
3
  import { runScript } from '../script/runScript.js';
4
- import { _yargs } from '../yargs/index.js';
4
+ import { _yargs } from '../yargs/yargs.util.js';
5
5
  runScript(async () => {
6
6
  const { channel, msg, username, emoji, webhook: webhookUrl, } = _yargs().options({
7
7
  channel: {
@@ -1,4 +1,4 @@
1
- import type { MemoCache } from '@naturalcycles/js-lib';
1
+ import type { MemoCache } from '@naturalcycles/js-lib/decorators';
2
2
  import { LRUCache } from 'lru-cache';
3
3
  export type LRUMemoCacheOptions<KEY, VALUE> = Partial<LRUCache.Options<KEY, VALUE, any>>;
4
4
  /**
@@ -1,4 +1,4 @@
1
- import type { AnyObject } from '@naturalcycles/js-lib';
1
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
2
  export interface CSVReaderConfig {
3
3
  /**
4
4
  * Default: comma
@@ -1,5 +1,5 @@
1
1
  // Inspired by: https://gist.github.com/Jezternz/c8e9fafc2c114e079829974e3764db75
2
- import { _assert } from '@naturalcycles/js-lib';
2
+ import { _assert } from '@naturalcycles/js-lib/error';
3
3
  // export class CSVReader {
4
4
  // constructor (cfg: CSVReaderConfig) {
5
5
  // this.cfg = {
@@ -1,4 +1,4 @@
1
- import type { AnyObject } from '@naturalcycles/js-lib';
1
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
2
  export interface CSVWriterConfig {
3
3
  /**
4
4
  * Default: comma
@@ -1,5 +1,5 @@
1
1
  // Inspired by: https://github.com/ryu1kn/csv-writer/
2
- import { _assert } from '@naturalcycles/js-lib';
2
+ import { _assert } from '@naturalcycles/js-lib/error';
3
3
  export class CSVWriter {
4
4
  constructor(cfg) {
5
5
  this.cfg = {
@@ -1,4 +1,4 @@
1
- import type { AnyObject } from '@naturalcycles/js-lib';
1
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
2
  import type { TransformTyped } from '../stream/stream.model.js';
3
3
  import type { CSVWriterConfig } from './csvWriter.js';
4
4
  export interface TransformToCSVOptions extends CSVWriterConfig {
@@ -1,4 +1,4 @@
1
- import type { AnyObject } from '@naturalcycles/js-lib';
1
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
2
  export interface TableDiffOptions {
3
3
  /**
4
4
  * @default false
@@ -1,4 +1,4 @@
1
- import { _truncate } from '@naturalcycles/js-lib';
1
+ import { _truncate } from '@naturalcycles/js-lib/string';
2
2
  /**
3
3
  * Compares 2 objects, logs their differences via `console.table`.
4
4
  *
@@ -1,5 +1,5 @@
1
- import { type AnyObject, type NumberOfMilliseconds } from '@naturalcycles/js-lib';
2
- import { AppError } from '@naturalcycles/js-lib';
1
+ import { AppError } from '@naturalcycles/js-lib/error';
2
+ import type { AnyObject, NumberOfMilliseconds } from '@naturalcycles/js-lib/types';
3
3
  /**
4
4
  * Set of utility functions to work with Spawn / Exec.
5
5
  *
@@ -1,7 +1,8 @@
1
1
  import { execSync, spawn, spawnSync } from 'node:child_process';
2
- import { _substringAfterLast, } from '@naturalcycles/js-lib';
3
- import { _since, AppError } from '@naturalcycles/js-lib';
4
- import { dimGrey, dimRed, hasColors, white } from '../colors/colors.js';
2
+ import { _since } from '@naturalcycles/js-lib/datetime';
3
+ import { AppError } from '@naturalcycles/js-lib/error';
4
+ import { _substringAfterLast } from '@naturalcycles/js-lib/string';
5
+ import { dimGrey, dimRed, hasColors, white } from './colors/colors.js';
5
6
  /**
6
7
  * Set of utility functions to work with Spawn / Exec.
7
8
  *
package/dist/fs/fs2.d.ts CHANGED
@@ -1,7 +1,5 @@
1
1
  import type { RmOptions } from 'node:fs';
2
2
  import fs from 'node:fs';
3
- import type { DumpOptions } from 'js-yaml';
4
- import type { ReadableTyped, TransformTyped } from '../stream/stream.model.js';
5
3
  /**
6
4
  * fs2 conveniently groups filesystem functions together.
7
5
  * Supposed to be almost a drop-in replacement for these things together:
@@ -23,20 +21,14 @@ declare class FS2 {
23
21
  readBufferAsync(filePath: string): Promise<Buffer>;
24
22
  readJson<T = unknown>(filePath: string): T;
25
23
  readJsonAsync<T = unknown>(filePath: string): Promise<T>;
26
- readYaml<T = unknown>(filePath: string): T;
27
- readYamlAsync<T = unknown>(filePath: string): Promise<T>;
28
24
  writeFile(filePath: string, data: string | Buffer): void;
29
25
  writeFileAsync(filePath: string, data: string | Buffer): Promise<void>;
30
26
  writeJson(filePath: string, data: any, opt?: JsonOptions): void;
31
27
  writeJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
32
- writeYaml(filePath: string, data: any, opt?: DumpOptions): void;
33
- writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
34
28
  appendFile(filePath: string, data: string | Buffer): void;
35
29
  appendFileAsync(filePath: string, data: string | Buffer): Promise<void>;
36
30
  outputJson(filePath: string, data: any, opt?: JsonOptions): void;
37
31
  outputJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
38
- outputYaml(filePath: string, data: any, opt?: DumpOptions): void;
39
- outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
40
32
  outputFile(filePath: string, data: string | Buffer): void;
41
33
  outputFileAsync(filePath: string, data: string | Buffer): Promise<void>;
42
34
  pathExists(filePath: string): boolean;
@@ -80,8 +72,6 @@ declare class FS2 {
80
72
  readdirAsync: typeof fs.promises.readdir;
81
73
  createWriteStream: typeof fs.createWriteStream;
82
74
  createReadStream: typeof fs.createReadStream;
83
- createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;
84
- createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];
85
75
  }
86
76
  export declare const fs2: FS2;
87
77
  export interface JsonOptions {
package/dist/fs/fs2.js CHANGED
@@ -16,11 +16,7 @@ Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
16
16
  import fs from 'node:fs';
17
17
  import fsp from 'node:fs/promises';
18
18
  import path from 'node:path';
19
- import { createGzip, createUnzip } from 'node:zlib';
20
- import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib';
21
- import yaml from 'js-yaml';
22
- import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js';
23
- import { transformSplitOnNewline } from '../stream/transform/transformSplit.js';
19
+ import { _jsonParse } from '@naturalcycles/js-lib/string';
24
20
  /**
25
21
  * fs2 conveniently groups filesystem functions together.
26
22
  * Supposed to be almost a drop-in replacement for these things together:
@@ -60,12 +56,6 @@ class FS2 {
60
56
  // eslint-disable-next-line @typescript-eslint/return-await
61
57
  return _jsonParse(str);
62
58
  }
63
- readYaml(filePath) {
64
- return yaml.load(fs.readFileSync(filePath, 'utf8'));
65
- }
66
- async readYamlAsync(filePath) {
67
- return yaml.load(await fsp.readFile(filePath, 'utf8'));
68
- }
69
59
  writeFile(filePath, data) {
70
60
  fs.writeFileSync(filePath, data);
71
61
  }
@@ -80,14 +70,6 @@ class FS2 {
80
70
  const str = stringify(data, opt);
81
71
  await fsp.writeFile(filePath, str);
82
72
  }
83
- writeYaml(filePath, data, opt) {
84
- const str = yaml.dump(data, opt);
85
- fs.writeFileSync(filePath, str);
86
- }
87
- async writeYamlAsync(filePath, data, opt) {
88
- const str = yaml.dump(data, opt);
89
- await fsp.writeFile(filePath, str);
90
- }
91
73
  appendFile(filePath, data) {
92
74
  fs.appendFileSync(filePath, data);
93
75
  }
@@ -102,14 +84,6 @@ class FS2 {
102
84
  const str = stringify(data, opt);
103
85
  await this.outputFileAsync(filePath, str);
104
86
  }
105
- outputYaml(filePath, data, opt) {
106
- const str = yaml.dump(data, opt);
107
- this.outputFile(filePath, str);
108
- }
109
- async outputYamlAsync(filePath, data, opt) {
110
- const str = yaml.dump(data, opt);
111
- await this.outputFileAsync(filePath, str);
112
- }
113
87
  outputFile(filePath, data) {
114
88
  const dirPath = path.dirname(filePath);
115
89
  if (!fs.existsSync(dirPath)) {
@@ -289,64 +263,6 @@ class FS2 {
289
263
  readdirAsync = fsp.readdir;
290
264
  createWriteStream = fs.createWriteStream;
291
265
  createReadStream = fs.createReadStream;
292
- /*
293
- Returns a Readable of [already parsed] NDJSON objects.
294
-
295
- Replaces a list of operations:
296
- - requireFileToExist(inputPath)
297
- - fs.createReadStream
298
- - createUnzip (only if path ends with '.gz')
299
- - transformSplitOnNewline
300
- - transformJsonParse
301
-
302
- To add a Limit or Offset: just add .take() or .drop(), example:
303
-
304
- _pipeline([
305
- fs2.createReadStreamAsNDJSON().take(100),
306
- transformX(),
307
- ])
308
- */
309
- createReadStreamAsNDJSON(inputPath) {
310
- this.requireFileToExist(inputPath);
311
- let stream = fs
312
- .createReadStream(inputPath, {
313
- highWaterMark: 64 * 1024, // no observed speedup
314
- })
315
- .on('error', err => stream.emit('error', err));
316
- if (inputPath.endsWith('.gz')) {
317
- stream = stream.pipe(createUnzip({
318
- chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
319
- }));
320
- }
321
- return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
322
- // For some crazy reason .map is much faster than transformJsonParse!
323
- // ~5000 vs ~4000 rps !!!
324
- // .on('error', err => stream.emit('error', err))
325
- // .pipe(transformJsonParse<ROW>())
326
- }
327
- /*
328
- Returns an array of Transforms, so that you can ...destructure them at
329
- the end of the _pipeline.
330
-
331
- Replaces a list of operations:
332
- - transformToNDJson
333
- - createGzip (only if path ends with '.gz')
334
- - fs.createWriteStream
335
- */
336
- createWriteStreamAsNDJSON(outputPath) {
337
- this.ensureFile(outputPath);
338
- return [
339
- transformToNDJson(),
340
- outputPath.endsWith('.gz')
341
- ? createGzip({
342
- // chunkSize: 64 * 1024, // no observed speedup
343
- })
344
- : undefined,
345
- fs.createWriteStream(outputPath, {
346
- // highWaterMark: 64 * 1024, // no observed speedup
347
- }),
348
- ].filter(_isTruthy);
349
- }
350
266
  }
351
267
  export const fs2 = new FS2();
352
268
  function stringify(data, opt) {
@@ -1,4 +1,4 @@
1
- import type { AnyObject } from '@naturalcycles/js-lib';
1
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
2
  export interface Json2EnvOptions {
3
3
  jsonPath: string;
4
4
  prefix?: string;
package/dist/fs/kpy.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import path from 'node:path';
2
- import { _since, localTime } from '@naturalcycles/js-lib';
2
+ import { _since, localTime } from '@naturalcycles/js-lib/datetime';
3
3
  import { glob, globSync } from 'tinyglobby';
4
4
  import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js';
5
5
  import { fs2 } from './fs2.js';
@@ -0,0 +1,11 @@
1
+ import type { DumpOptions } from 'js-yaml';
2
+ declare class Yaml2 {
3
+ readYaml<T = unknown>(filePath: string): T;
4
+ readYamlAsync<T = unknown>(filePath: string): Promise<T>;
5
+ writeYaml(filePath: string, data: any, opt?: DumpOptions): void;
6
+ writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
7
+ outputYaml(filePath: string, data: any, opt?: DumpOptions): void;
8
+ outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
9
+ }
10
+ export declare const yaml2: Yaml2;
11
+ export {};
@@ -0,0 +1,29 @@
1
+ import fs from 'node:fs';
2
+ import fsp from 'node:fs/promises';
3
+ import yaml from 'js-yaml';
4
+ import { fs2 } from './fs2.js';
5
+ class Yaml2 {
6
+ readYaml(filePath) {
7
+ return yaml.load(fs.readFileSync(filePath, 'utf8'));
8
+ }
9
+ async readYamlAsync(filePath) {
10
+ return yaml.load(await fsp.readFile(filePath, 'utf8'));
11
+ }
12
+ writeYaml(filePath, data, opt) {
13
+ const str = yaml.dump(data, opt);
14
+ fs.writeFileSync(filePath, str);
15
+ }
16
+ async writeYamlAsync(filePath, data, opt) {
17
+ const str = yaml.dump(data, opt);
18
+ await fsp.writeFile(filePath, str);
19
+ }
20
+ outputYaml(filePath, data, opt) {
21
+ const str = yaml.dump(data, opt);
22
+ fs2.outputFile(filePath, str);
23
+ }
24
+ async outputYamlAsync(filePath, data, opt) {
25
+ const str = yaml.dump(data, opt);
26
+ await fs2.outputFileAsync(filePath, str);
27
+ }
28
+ }
29
+ export const yaml2 = new Yaml2();
package/dist/index.d.ts CHANGED
@@ -1,5 +1,4 @@
1
1
  export * from './buffer/buffer.util.js';
2
- export * from './colors/colors.js';
3
2
  export * from './diff/tableDiff.js';
4
3
  export * from './infra/process.util.js';
5
4
  export * from './log/log.util.js';
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js';
14
13
  export * from './string/inspect.js';
15
14
  export * from './util/buildInfo.util.js';
16
15
  export * from './util/env.util.js';
17
- export * from './util/exec2.js';
18
16
  export * from './util/git2.js';
19
- export * from './util/lruMemoCache.js';
20
- export * from './util/zip.util.js';
package/dist/index.js CHANGED
@@ -1,5 +1,4 @@
1
1
  export * from './buffer/buffer.util.js';
2
- export * from './colors/colors.js';
3
2
  export * from './diff/tableDiff.js';
4
3
  export * from './infra/process.util.js';
5
4
  export * from './log/log.util.js';
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js';
14
13
  export * from './string/inspect.js';
15
14
  export * from './util/buildInfo.util.js';
16
15
  export * from './util/env.util.js';
17
- export * from './util/exec2.js';
18
16
  export * from './util/git2.js';
19
- export * from './util/lruMemoCache.js';
20
- export * from './util/zip.util.js';
@@ -1,4 +1,5 @@
1
- import type { AnyObject, ErrorData, JWTString } from '@naturalcycles/js-lib';
1
+ import type { ErrorData } from '@naturalcycles/js-lib/error';
2
+ import type { AnyObject, JWTString } from '@naturalcycles/js-lib/types';
2
3
  import type { AnySchema } from 'joi';
3
4
  import type { Algorithm, JwtHeader, SignOptions, VerifyOptions } from 'jsonwebtoken';
4
5
  import jsonwebtoken from 'jsonwebtoken';
@@ -1,4 +1,4 @@
1
- import { _assert, _errorDataAppend } from '@naturalcycles/js-lib';
1
+ import { _assert, _errorDataAppend } from '@naturalcycles/js-lib/error';
2
2
  import jsonwebtoken from 'jsonwebtoken';
3
3
  import { anyObjectSchema } from '../validation/joi/joi.shared.schemas.js';
4
4
  import { validate } from '../validation/joi/joi.validation.util.js';
@@ -1,4 +1,4 @@
1
1
  /**
2
2
  * CommonLogger that logs to process.stdout directly (bypassing console.log).
3
3
  */
4
- export declare const stdoutLogger: import("@naturalcycles/js-lib").CommonLogger;
4
+ export declare const stdoutLogger: import("@naturalcycles/js-lib/log").CommonLogger;
@@ -1,4 +1,4 @@
1
- import { commonLoggerCreate } from '@naturalcycles/js-lib';
1
+ import { commonLoggerCreate } from '@naturalcycles/js-lib/log';
2
2
  import { _inspect } from '../string/inspect.js';
3
3
  /**
4
4
  * CommonLogger that logs to process.stdout directly (bypassing console.log).
@@ -1,5 +1,4 @@
1
- import 'dotenv/config';
2
- import type { CommonLogger } from '@naturalcycles/js-lib';
1
+ import type { CommonLogger } from '@naturalcycles/js-lib/log';
3
2
  export interface RunScriptOptions {
4
3
  /**
5
4
  * @default false
@@ -1,7 +1,6 @@
1
- import 'dotenv/config';
2
1
  import os from 'node:os';
3
- import { pDelay, setGlobalStringifyFunction } from '@naturalcycles/js-lib';
4
- import { dimGrey } from '../colors/colors.js';
2
+ import { pDelay } from '@naturalcycles/js-lib/promise';
3
+ import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string';
5
4
  import { inspectStringifyFn } from '../string/inspect.js';
6
5
  const { DEBUG_RUN_SCRIPT } = process.env;
7
6
  /**
@@ -23,7 +22,6 @@ const { DEBUG_RUN_SCRIPT } = process.env;
23
22
  * Set env DEBUG_RUN_SCRIPT for extra debugging.
24
23
  */
25
24
  export function runScript(fn, opt = {}) {
26
- checkAndlogEnvironment();
27
25
  setGlobalStringifyFunction(inspectStringifyFn);
28
26
  const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt;
29
27
  if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
@@ -42,6 +40,8 @@ export function runScript(fn, opt = {}) {
42
40
  const timeout = setTimeout(() => { }, 10000000);
43
41
  void (async () => {
44
42
  try {
43
+ await import('dotenv/config');
44
+ await checkAndlogEnvironment();
45
45
  await fn();
46
46
  await pDelay(); // to ensure all async operations are completed
47
47
  if (DEBUG_RUN_SCRIPT)
@@ -62,7 +62,8 @@ export function runScript(fn, opt = {}) {
62
62
  }
63
63
  })();
64
64
  }
65
- function checkAndlogEnvironment() {
65
+ async function checkAndlogEnvironment() {
66
+ const { dimGrey } = await import('../colors/colors.js');
66
67
  const { platform, arch, versions: { node }, env: { CPU_LIMIT, NODE_OPTIONS, TZ }, } = process;
67
68
  const cpuLimit = Number(CPU_LIMIT) || undefined;
68
69
  const availableParallelism = os.availableParallelism?.();
@@ -1,6 +1,6 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
- import { _assert } from '@naturalcycles/js-lib';
3
+ import { _assert } from '@naturalcycles/js-lib/error';
4
4
  import { globSync } from 'tinyglobby';
5
5
  import { dimGrey, yellow } from '../colors/colors.js';
6
6
  import { fs2 } from '../fs/fs2.js';
@@ -1,6 +1,6 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
- import { _assert } from '@naturalcycles/js-lib';
3
+ import { _assert } from '@naturalcycles/js-lib/error';
4
4
  import { globSync } from 'tinyglobby';
5
5
  import { dimGrey, yellow } from '../colors/colors.js';
6
6
  import { fs2 } from '../fs/fs2.js';
@@ -1,4 +1,4 @@
1
- import type { Base64String, StringMap } from '@naturalcycles/js-lib';
1
+ import { type Base64String, type StringMap } from '@naturalcycles/js-lib/types';
2
2
  /**
3
3
  * Using aes-256-cbc.
4
4
  */
@@ -1,5 +1,5 @@
1
1
  import crypto from 'node:crypto';
2
- import { _stringMapEntries } from '@naturalcycles/js-lib';
2
+ import { _stringMapEntries } from '@naturalcycles/js-lib/types';
3
3
  import { md5AsBuffer, sha256AsBuffer } from './hash.util.js';
4
4
  const algorithm = 'aes-256-cbc';
5
5
  /**
@@ -1,5 +1,5 @@
1
1
  import type { BinaryToTextEncoding } from 'node:crypto';
2
- import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib';
2
+ import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types';
3
3
  export declare function md5(s: string | Buffer, outputEncoding?: BinaryToTextEncoding): string;
4
4
  export declare function md5AsBuffer(s: string | Buffer): Buffer;
5
5
  export declare function sha256(s: string | Buffer, outputEncoding?: BinaryToTextEncoding): string;
@@ -1,4 +1,4 @@
1
- import type { Base64String, StringMap } from '@naturalcycles/js-lib';
1
+ import type { Base64String, StringMap } from '@naturalcycles/js-lib/types';
2
2
  /**
3
3
  * Loads plaintext secrets from process.env, removes them, stores locally.
4
4
  * Make sure to call this function early on server startup, so secrets are removed from process.env
@@ -1,5 +1,6 @@
1
1
  import { existsSync, readFileSync } from 'node:fs';
2
- import { _assert, _jsonParseIfPossible } from '@naturalcycles/js-lib';
2
+ import { _assert } from '@naturalcycles/js-lib/error';
3
+ import { _jsonParseIfPossible } from '@naturalcycles/js-lib/string';
3
4
  import { decryptObject, decryptRandomIVBuffer } from './crypto.util.js';
4
5
  let loaded = false;
5
6
  const secretMap = {};
@@ -1,4 +1,5 @@
1
- import type { AnyObject, CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib';
1
+ import { type CommonLogger, type CommonLogLevel } from '@naturalcycles/js-lib/log';
2
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
3
  import type { SlackAttachmentField, SlackMessage, SlackServiceCfg } from './slack.service.model.js';
3
4
  /**
4
5
  * Has 2 main methods:
@@ -1,4 +1,8 @@
1
- import { _omit, commonLoggerMinLevel, getFetcher, localTime, PQueue } from '@naturalcycles/js-lib';
1
+ import { localTime } from '@naturalcycles/js-lib/datetime';
2
+ import { getFetcher } from '@naturalcycles/js-lib/http';
3
+ import { commonLoggerMinLevel, } from '@naturalcycles/js-lib/log';
4
+ import { _omit } from '@naturalcycles/js-lib/object';
5
+ import { PQueue } from '@naturalcycles/js-lib/promise';
2
6
  import { _inspect } from '../index.js';
3
7
  const GAE = !!process.env['GAE_INSTANCE'];
4
8
  const DEFAULTS = {
@@ -1,4 +1,5 @@
1
- import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
1
+ import type { CommonLogger } from '@naturalcycles/js-lib/log';
2
+ import type { AnyObject } from '@naturalcycles/js-lib/types';
2
3
  import type { InspectAnyOptions } from '../index.js';
3
4
  /**
4
5
  * Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
@@ -1,3 +1,5 @@
1
+ export * from './ndjson/createReadStreamAsNDJSON.js';
2
+ export * from './ndjson/createWriteStreamAsNDJSON.js';
1
3
  export * from './ndjson/ndjson.model.js';
2
4
  export * from './ndjson/ndjsonMap.js';
3
5
  export * from './ndjson/ndjsonStreamForEach.js';
@@ -1,3 +1,5 @@
1
+ export * from './ndjson/createReadStreamAsNDJSON.js';
2
+ export * from './ndjson/createWriteStreamAsNDJSON.js';
1
3
  export * from './ndjson/ndjson.model.js';
2
4
  export * from './ndjson/ndjsonMap.js';
3
5
  export * from './ndjson/ndjsonStreamForEach.js';
@@ -0,0 +1,19 @@
1
+ import type { ReadableTyped } from '../stream.model.js';
2
+ /**
3
+ Returns a Readable of [already parsed] NDJSON objects.
4
+
5
+ Replaces a list of operations:
6
+ - requireFileToExist(inputPath)
7
+ - fs.createReadStream
8
+ - createUnzip (only if path ends with '.gz')
9
+ - transformSplitOnNewline
10
+ - transformJsonParse
11
+
12
+ To add a Limit or Offset: just add .take() or .drop(), example:
13
+
14
+ _pipeline([
15
+ fs2.createReadStreamAsNDJSON().take(100),
16
+ transformX(),
17
+ ])
18
+ */
19
+ export declare function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;
@@ -0,0 +1,38 @@
1
+ import { createUnzip } from 'node:zlib';
2
+ import { fs2 } from '../../fs/fs2.js';
3
+ import { transformSplitOnNewline } from '../transform/transformSplit.js';
4
+ /**
5
+ Returns a Readable of [already parsed] NDJSON objects.
6
+
7
+ Replaces a list of operations:
8
+ - requireFileToExist(inputPath)
9
+ - fs.createReadStream
10
+ - createUnzip (only if path ends with '.gz')
11
+ - transformSplitOnNewline
12
+ - transformJsonParse
13
+
14
+ To add a Limit or Offset: just add .take() or .drop(), example:
15
+
16
+ _pipeline([
17
+ fs2.createReadStreamAsNDJSON().take(100),
18
+ transformX(),
19
+ ])
20
+ */
21
+ export function createReadStreamAsNDJSON(inputPath) {
22
+ fs2.requireFileToExist(inputPath);
23
+ let stream = fs2
24
+ .createReadStream(inputPath, {
25
+ highWaterMark: 64 * 1024, // no observed speedup
26
+ })
27
+ .on('error', err => stream.emit('error', err));
28
+ if (inputPath.endsWith('.gz')) {
29
+ stream = stream.pipe(createUnzip({
30
+ chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
31
+ }));
32
+ }
33
+ return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
34
+ // For some crazy reason .map is much faster than transformJsonParse!
35
+ // ~5000 vs ~4000 rps !!!
36
+ // .on('error', err => stream.emit('error', err))
37
+ // .pipe(transformJsonParse<ROW>())
38
+ }
@@ -0,0 +1,11 @@
1
+ import type { TransformTyped } from '../stream.model.js';
2
+ /**
3
+ Returns an array of Transforms, so that you can ...destructure them at
4
+ the end of the _pipeline.
5
+
6
+ Replaces a list of operations:
7
+ - transformToNDJson
8
+ - createGzip (only if path ends with '.gz')
9
+ - fs.createWriteStream
10
+ */
11
+ export declare function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];
@@ -0,0 +1,27 @@
1
+ import { createGzip } from 'node:zlib';
2
+ import { _isTruthy } from '@naturalcycles/js-lib';
3
+ import { fs2 } from '../../fs/fs2.js';
4
+ import { transformToNDJson } from './transformToNDJson.js';
5
+ /**
6
+ Returns an array of Transforms, so that you can ...destructure them at
7
+ the end of the _pipeline.
8
+
9
+ Replaces a list of operations:
10
+ - transformToNDJson
11
+ - createGzip (only if path ends with '.gz')
12
+ - fs.createWriteStream
13
+ */
14
+ export function createWriteStreamAsNDJSON(outputPath) {
15
+ fs2.ensureFile(outputPath);
16
+ return [
17
+ transformToNDJson(),
18
+ outputPath.endsWith('.gz')
19
+ ? createGzip({
20
+ // chunkSize: 64 * 1024, // no observed speedup
21
+ })
22
+ : undefined,
23
+ fs2.createWriteStream(outputPath, {
24
+ // highWaterMark: 64 * 1024, // no observed speedup
25
+ }),
26
+ ].filter(_isTruthy);
27
+ }
@@ -1,4 +1,5 @@
1
- import { _hb, _ms } from '@naturalcycles/js-lib';
1
+ import { _hb } from '@naturalcycles/js-lib';
2
+ import { _ms } from '@naturalcycles/js-lib/datetime';
2
3
  import { boldWhite, dimWhite } from '../../colors/colors.js';
3
4
  export class NDJsonStats {
4
5
  static create(o = {}) {
@@ -1,5 +1,5 @@
1
- import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
2
- import type { TransformLogProgressOptions, TransformMapOptions } from '../index.js';
1
+ import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types';
2
+ import { type TransformLogProgressOptions, type TransformMapOptions } from '../index.js';
3
3
  export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT>, TransformLogProgressOptions<IN> {
4
4
  inputFilePath: string;
5
5
  outputFilePath: string;