@naturalcycles/nodejs-lib 15.1.0 → 15.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/bin/slack-this.js +1 -1
  2. package/dist/{util → cache}/lruMemoCache.d.ts +1 -1
  3. package/dist/csv/csvReader.d.ts +1 -1
  4. package/dist/csv/csvReader.js +1 -1
  5. package/dist/csv/csvWriter.d.ts +1 -1
  6. package/dist/csv/csvWriter.js +1 -1
  7. package/dist/csv/transformToCSV.d.ts +1 -1
  8. package/dist/diff/tableDiff.d.ts +1 -1
  9. package/dist/diff/tableDiff.js +1 -1
  10. package/dist/{util/exec2.d.ts → exec2.d.ts} +2 -2
  11. package/dist/{util/exec2.js → exec2.js} +4 -3
  12. package/dist/fs/fs2.d.ts +0 -10
  13. package/dist/fs/fs2.js +1 -85
  14. package/dist/fs/json2env.d.ts +1 -1
  15. package/dist/fs/kpy.js +1 -1
  16. package/dist/fs/yaml2.d.ts +11 -0
  17. package/dist/fs/yaml2.js +29 -0
  18. package/dist/index.d.ts +0 -4
  19. package/dist/index.js +0 -4
  20. package/dist/jwt/jwt.service.d.ts +2 -1
  21. package/dist/jwt/jwt.service.js +1 -1
  22. package/dist/log/log.util.d.ts +1 -1
  23. package/dist/log/log.util.js +1 -1
  24. package/dist/script/runScript.d.ts +1 -2
  25. package/dist/script/runScript.js +6 -5
  26. package/dist/secret/secrets-decrypt.util.js +1 -1
  27. package/dist/secret/secrets-encrypt.util.js +1 -1
  28. package/dist/security/crypto.util.d.ts +1 -1
  29. package/dist/security/crypto.util.js +1 -1
  30. package/dist/security/hash.util.d.ts +1 -1
  31. package/dist/security/secret.util.d.ts +1 -1
  32. package/dist/security/secret.util.js +2 -1
  33. package/dist/slack/slack.service.d.ts +2 -1
  34. package/dist/slack/slack.service.js +5 -1
  35. package/dist/slack/slack.service.model.d.ts +2 -1
  36. package/dist/stream/index.d.ts +2 -0
  37. package/dist/stream/index.js +2 -0
  38. package/dist/stream/ndjson/createReadStreamAsNDJSON.d.ts +19 -0
  39. package/dist/stream/ndjson/createReadStreamAsNDJSON.js +38 -0
  40. package/dist/stream/ndjson/createWriteStreamAsNDJSON.d.ts +11 -0
  41. package/dist/stream/ndjson/createWriteStreamAsNDJSON.js +27 -0
  42. package/dist/stream/ndjson/ndjson.model.js +2 -1
  43. package/dist/stream/ndjson/ndjsonMap.d.ts +2 -2
  44. package/dist/stream/ndjson/ndjsonMap.js +4 -6
  45. package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +2 -2
  46. package/dist/stream/ndjson/ndjsonStreamForEach.js +3 -3
  47. package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
  48. package/dist/stream/ndjson/transformToNDJson.js +1 -1
  49. package/dist/stream/pipeline/pipeline.d.ts +1 -1
  50. package/dist/stream/pipeline/pipeline.js +2 -1
  51. package/dist/stream/progressLogger.d.ts +2 -1
  52. package/dist/stream/progressLogger.js +3 -1
  53. package/dist/stream/readable/readableForEach.d.ts +1 -1
  54. package/dist/stream/readable/readableForEach.js +1 -1
  55. package/dist/stream/readable/readableFromArray.d.ts +1 -1
  56. package/dist/stream/readable/readableFromArray.js +1 -1
  57. package/dist/stream/sizeStack.d.ts +2 -2
  58. package/dist/stream/sizeStack.js +3 -2
  59. package/dist/stream/stream.model.d.ts +1 -1
  60. package/dist/stream/stream.util.d.ts +1 -1
  61. package/dist/stream/transform/transformFilter.d.ts +1 -1
  62. package/dist/stream/transform/transformLimit.d.ts +1 -1
  63. package/dist/stream/transform/transformMap.d.ts +3 -2
  64. package/dist/stream/transform/transformMap.js +6 -1
  65. package/dist/stream/transform/transformMapSimple.d.ts +3 -2
  66. package/dist/stream/transform/transformMapSimple.js +1 -1
  67. package/dist/stream/transform/transformMapSync.d.ts +4 -2
  68. package/dist/stream/transform/transformMapSync.js +2 -1
  69. package/dist/stream/transform/transformTap.d.ts +2 -1
  70. package/dist/stream/transform/transformThrottle.d.ts +1 -1
  71. package/dist/stream/transform/transformThrottle.js +2 -1
  72. package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
  73. package/dist/stream/transform/worker/transformMultiThreaded.js +2 -1
  74. package/dist/stream/writable/writableForEach.d.ts +1 -1
  75. package/dist/stream/writable/writableForEach.js +1 -1
  76. package/dist/stream/writable/writableVoid.d.ts +1 -1
  77. package/dist/string/inspect.d.ts +1 -1
  78. package/dist/string/inspect.js +1 -1
  79. package/dist/util/buildInfo.util.d.ts +2 -1
  80. package/dist/util/buildInfo.util.js +2 -1
  81. package/dist/util/env.util.d.ts +1 -2
  82. package/dist/util/env.util.js +0 -1
  83. package/dist/util/git2.d.ts +1 -1
  84. package/dist/util/git2.js +2 -3
  85. package/dist/validation/ajv/ajv.util.d.ts +1 -1
  86. package/dist/validation/ajv/ajvSchema.d.ts +1 -1
  87. package/dist/validation/ajv/ajvSchema.js +4 -1
  88. package/dist/validation/ajv/ajvValidationError.d.ts +2 -2
  89. package/dist/validation/ajv/ajvValidationError.js +1 -1
  90. package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
  91. package/dist/validation/joi/joi.validation.error.d.ts +2 -2
  92. package/dist/validation/joi/joi.validation.error.js +1 -1
  93. package/dist/validation/joi/joi.validation.util.js +2 -1
  94. package/dist/validation/joi/string.extensions.d.ts +1 -1
  95. package/dist/validation/joi/string.extensions.js +1 -1
  96. package/dist/{util → zip}/zip.util.js +0 -1
  97. package/package.json +12 -5
  98. package/src/bin/generate-build-info.ts +1 -1
  99. package/src/bin/slack-this.ts +1 -1
  100. package/src/{util → cache}/lruMemoCache.ts +1 -1
  101. package/src/csv/csvReader.ts +2 -2
  102. package/src/csv/csvWriter.ts +2 -2
  103. package/src/csv/transformToCSV.ts +1 -1
  104. package/src/diff/tableDiff.ts +2 -2
  105. package/src/{util/exec2.ts → exec2.ts} +9 -8
  106. package/src/fs/fs2.ts +1 -101
  107. package/src/fs/json2env.ts +1 -1
  108. package/src/fs/kpy.ts +2 -2
  109. package/src/fs/yaml2.ts +37 -0
  110. package/src/index.ts +0 -4
  111. package/src/jwt/jwt.service.ts +3 -2
  112. package/src/log/log.util.ts +1 -1
  113. package/src/script/runScript.ts +10 -6
  114. package/src/secret/secrets-decrypt.util.ts +1 -1
  115. package/src/secret/secrets-encrypt.util.ts +1 -1
  116. package/src/security/crypto.util.ts +1 -2
  117. package/src/security/hash.util.ts +1 -1
  118. package/src/security/secret.util.ts +3 -2
  119. package/src/slack/slack.service.model.ts +2 -1
  120. package/src/slack/slack.service.ts +10 -2
  121. package/src/stream/index.ts +2 -0
  122. package/src/stream/ndjson/createReadStreamAsNDJSON.ts +46 -0
  123. package/src/stream/ndjson/createWriteStreamAsNDJSON.ts +30 -0
  124. package/src/stream/ndjson/ndjson.model.ts +2 -1
  125. package/src/stream/ndjson/ndjsonMap.ts +12 -8
  126. package/src/stream/ndjson/ndjsonStreamForEach.ts +8 -5
  127. package/src/stream/ndjson/transformJsonParse.ts +1 -1
  128. package/src/stream/ndjson/transformToNDJson.ts +1 -1
  129. package/src/stream/pipeline/pipeline.ts +4 -2
  130. package/src/stream/progressLogger.ts +5 -2
  131. package/src/stream/readable/readableForEach.ts +2 -2
  132. package/src/stream/readable/readableFromArray.ts +2 -2
  133. package/src/stream/sizeStack.ts +4 -3
  134. package/src/stream/stream.model.ts +1 -1
  135. package/src/stream/stream.util.ts +1 -1
  136. package/src/stream/transform/transformFilter.ts +1 -1
  137. package/src/stream/transform/transformLimit.ts +1 -1
  138. package/src/stream/transform/transformMap.ts +12 -15
  139. package/src/stream/transform/transformMapSimple.ts +3 -2
  140. package/src/stream/transform/transformMapSync.ts +4 -7
  141. package/src/stream/transform/transformTap.ts +2 -1
  142. package/src/stream/transform/transformThrottle.ts +4 -3
  143. package/src/stream/transform/worker/transformMultiThreaded.ts +4 -2
  144. package/src/stream/writable/writableForEach.ts +2 -2
  145. package/src/stream/writable/writableVoid.ts +1 -1
  146. package/src/string/inspect.ts +2 -2
  147. package/src/util/buildInfo.util.ts +4 -2
  148. package/src/util/env.util.ts +1 -2
  149. package/src/util/git2.ts +3 -4
  150. package/src/validation/ajv/ajv.util.ts +1 -1
  151. package/src/validation/ajv/ajvSchema.ts +5 -7
  152. package/src/validation/ajv/ajvValidationError.ts +2 -2
  153. package/src/validation/joi/joi.shared.schemas.ts +7 -7
  154. package/src/validation/joi/joi.validation.error.ts +2 -2
  155. package/src/validation/joi/joi.validation.util.ts +2 -1
  156. package/src/validation/joi/string.extensions.ts +2 -2
  157. package/src/{util → zip}/zip.util.ts +0 -2
  158. package/dist/fs/index.d.ts +0 -3
  159. package/dist/fs/index.js +0 -3
  160. package/dist/jwt/index.d.ts +0 -1
  161. package/dist/jwt/index.js +0 -1
  162. package/dist/yargs/index.d.ts +0 -1
  163. package/dist/yargs/index.js +0 -1
  164. package/src/fs/index.ts +0 -3
  165. package/src/jwt/index.ts +0 -1
  166. package/src/yargs/index.ts +0 -1
  167. /package/dist/{util → cache}/lruMemoCache.js +0 -0
  168. /package/dist/{util → zip}/zip.util.d.ts +0 -0
@@ -1,12 +1,13 @@
1
1
  import { execSync, spawn, spawnSync } from 'node:child_process'
2
- import {
3
- _substringAfterLast,
4
- type AnyObject,
5
- type NumberOfMilliseconds,
6
- type UnixTimestampMillis,
7
- } from '@naturalcycles/js-lib'
8
- import { _since, AppError } from '@naturalcycles/js-lib'
9
- import { dimGrey, dimRed, hasColors, white } from '../colors/colors.js'
2
+ import { _since } from '@naturalcycles/js-lib/datetime'
3
+ import { AppError } from '@naturalcycles/js-lib/error'
4
+ import { _substringAfterLast } from '@naturalcycles/js-lib/string'
5
+ import type {
6
+ AnyObject,
7
+ NumberOfMilliseconds,
8
+ UnixTimestampMillis,
9
+ } from '@naturalcycles/js-lib/types'
10
+ import { dimGrey, dimRed, hasColors, white } from './colors/colors.js'
10
11
 
11
12
  /**
12
13
  * Set of utility functions to work with Spawn / Exec.
package/src/fs/fs2.ts CHANGED
@@ -18,13 +18,7 @@ import type { RmOptions, Stats } from 'node:fs'
18
18
  import fs from 'node:fs'
19
19
  import fsp from 'node:fs/promises'
20
20
  import path from 'node:path'
21
- import { createGzip, createUnzip } from 'node:zlib'
22
- import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib'
23
- import type { DumpOptions } from 'js-yaml'
24
- import yaml from 'js-yaml'
25
- import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js'
26
- import type { ReadableTyped, TransformTyped } from '../stream/stream.model.js'
27
- import { transformSplitOnNewline } from '../stream/transform/transformSplit.js'
21
+ import { _jsonParse } from '@naturalcycles/js-lib/string'
28
22
 
29
23
  /**
30
24
  * fs2 conveniently groups filesystem functions together.
@@ -72,14 +66,6 @@ class FS2 {
72
66
  return _jsonParse(str)
73
67
  }
74
68
 
75
- readYaml<T = unknown>(filePath: string): T {
76
- return yaml.load(fs.readFileSync(filePath, 'utf8')) as T
77
- }
78
-
79
- async readYamlAsync<T = unknown>(filePath: string): Promise<T> {
80
- return yaml.load(await fsp.readFile(filePath, 'utf8')) as T
81
- }
82
-
83
69
  writeFile(filePath: string, data: string | Buffer): void {
84
70
  fs.writeFileSync(filePath, data)
85
71
  }
@@ -98,16 +84,6 @@ class FS2 {
98
84
  await fsp.writeFile(filePath, str)
99
85
  }
100
86
 
101
- writeYaml(filePath: string, data: any, opt?: DumpOptions): void {
102
- const str = yaml.dump(data, opt)
103
- fs.writeFileSync(filePath, str)
104
- }
105
-
106
- async writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
107
- const str = yaml.dump(data, opt)
108
- await fsp.writeFile(filePath, str)
109
- }
110
-
111
87
  appendFile(filePath: string, data: string | Buffer): void {
112
88
  fs.appendFileSync(filePath, data)
113
89
  }
@@ -126,16 +102,6 @@ class FS2 {
126
102
  await this.outputFileAsync(filePath, str)
127
103
  }
128
104
 
129
- outputYaml(filePath: string, data: any, opt?: DumpOptions): void {
130
- const str = yaml.dump(data, opt)
131
- this.outputFile(filePath, str)
132
- }
133
-
134
- async outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
135
- const str = yaml.dump(data, opt)
136
- await this.outputFileAsync(filePath, str)
137
- }
138
-
139
105
  outputFile(filePath: string, data: string | Buffer): void {
140
106
  const dirPath = path.dirname(filePath)
141
107
  if (!fs.existsSync(dirPath)) {
@@ -336,72 +302,6 @@ class FS2 {
336
302
  readdirAsync = fsp.readdir
337
303
  createWriteStream = fs.createWriteStream
338
304
  createReadStream = fs.createReadStream
339
-
340
- /*
341
- Returns a Readable of [already parsed] NDJSON objects.
342
-
343
- Replaces a list of operations:
344
- - requireFileToExist(inputPath)
345
- - fs.createReadStream
346
- - createUnzip (only if path ends with '.gz')
347
- - transformSplitOnNewline
348
- - transformJsonParse
349
-
350
- To add a Limit or Offset: just add .take() or .drop(), example:
351
-
352
- _pipeline([
353
- fs2.createReadStreamAsNDJSON().take(100),
354
- transformX(),
355
- ])
356
- */
357
- createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
358
- this.requireFileToExist(inputPath)
359
-
360
- let stream: ReadableTyped<ROW> = fs
361
- .createReadStream(inputPath, {
362
- highWaterMark: 64 * 1024, // no observed speedup
363
- })
364
- .on('error', err => stream.emit('error', err))
365
-
366
- if (inputPath.endsWith('.gz')) {
367
- stream = stream.pipe(
368
- createUnzip({
369
- chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
370
- }),
371
- )
372
- }
373
-
374
- return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
375
- // For some crazy reason .map is much faster than transformJsonParse!
376
- // ~5000 vs ~4000 rps !!!
377
- // .on('error', err => stream.emit('error', err))
378
- // .pipe(transformJsonParse<ROW>())
379
- }
380
-
381
- /*
382
- Returns an array of Transforms, so that you can ...destructure them at
383
- the end of the _pipeline.
384
-
385
- Replaces a list of operations:
386
- - transformToNDJson
387
- - createGzip (only if path ends with '.gz')
388
- - fs.createWriteStream
389
- */
390
- createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
391
- this.ensureFile(outputPath)
392
-
393
- return [
394
- transformToNDJson(),
395
- outputPath.endsWith('.gz')
396
- ? createGzip({
397
- // chunkSize: 64 * 1024, // no observed speedup
398
- })
399
- : undefined,
400
- fs.createWriteStream(outputPath, {
401
- // highWaterMark: 64 * 1024, // no observed speedup
402
- }),
403
- ].filter(_isTruthy) as TransformTyped<any, any>[]
404
- }
405
305
  }
406
306
 
407
307
  export const fs2 = new FS2()
@@ -1,5 +1,5 @@
1
1
  import fs from 'node:fs'
2
- import type { AnyObject } from '@naturalcycles/js-lib'
2
+ import type { AnyObject } from '@naturalcycles/js-lib/types'
3
3
  import { dimGrey } from '../colors/colors.js'
4
4
  import { fs2 } from './fs2.js'
5
5
 
package/src/fs/kpy.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import path from 'node:path'
2
- import type { UnixTimestampMillis } from '@naturalcycles/js-lib'
3
- import { _since, localTime } from '@naturalcycles/js-lib'
2
+ import { _since, localTime } from '@naturalcycles/js-lib/datetime'
3
+ import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
4
4
  import { glob, globSync } from 'tinyglobby'
5
5
  import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js'
6
6
  import { fs2 } from './fs2.js'
@@ -0,0 +1,37 @@
1
+ import fs from 'node:fs'
2
+ import fsp from 'node:fs/promises'
3
+ import type { DumpOptions } from 'js-yaml'
4
+ import yaml from 'js-yaml'
5
+ import { fs2 } from './fs2.js'
6
+
7
+ class Yaml2 {
8
+ readYaml<T = unknown>(filePath: string): T {
9
+ return yaml.load(fs.readFileSync(filePath, 'utf8')) as T
10
+ }
11
+
12
+ async readYamlAsync<T = unknown>(filePath: string): Promise<T> {
13
+ return yaml.load(await fsp.readFile(filePath, 'utf8')) as T
14
+ }
15
+
16
+ writeYaml(filePath: string, data: any, opt?: DumpOptions): void {
17
+ const str = yaml.dump(data, opt)
18
+ fs.writeFileSync(filePath, str)
19
+ }
20
+
21
+ async writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
22
+ const str = yaml.dump(data, opt)
23
+ await fsp.writeFile(filePath, str)
24
+ }
25
+
26
+ outputYaml(filePath: string, data: any, opt?: DumpOptions): void {
27
+ const str = yaml.dump(data, opt)
28
+ fs2.outputFile(filePath, str)
29
+ }
30
+
31
+ async outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
32
+ const str = yaml.dump(data, opt)
33
+ await fs2.outputFileAsync(filePath, str)
34
+ }
35
+ }
36
+
37
+ export const yaml2 = new Yaml2()
package/src/index.ts CHANGED
@@ -1,5 +1,4 @@
1
1
  export * from './buffer/buffer.util.js'
2
- export * from './colors/colors.js'
3
2
  export * from './diff/tableDiff.js'
4
3
  export * from './infra/process.util.js'
5
4
  export * from './log/log.util.js'
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js'
14
13
  export * from './string/inspect.js'
15
14
  export * from './util/buildInfo.util.js'
16
15
  export * from './util/env.util.js'
17
- export * from './util/exec2.js'
18
16
  export * from './util/git2.js'
19
- export * from './util/lruMemoCache.js'
20
- export * from './util/zip.util.js'
@@ -1,5 +1,6 @@
1
- import type { AnyObject, ErrorData, JWTString } from '@naturalcycles/js-lib'
2
- import { _assert, _errorDataAppend } from '@naturalcycles/js-lib'
1
+ import type { ErrorData } from '@naturalcycles/js-lib/error'
2
+ import { _assert, _errorDataAppend } from '@naturalcycles/js-lib/error'
3
+ import type { AnyObject, JWTString } from '@naturalcycles/js-lib/types'
3
4
  import type { AnySchema } from 'joi'
4
5
  import type { Algorithm, JwtHeader, SignOptions, VerifyOptions } from 'jsonwebtoken'
5
6
  import jsonwebtoken from 'jsonwebtoken'
@@ -1,4 +1,4 @@
1
- import { commonLoggerCreate } from '@naturalcycles/js-lib'
1
+ import { commonLoggerCreate } from '@naturalcycles/js-lib/log'
2
2
  import { _inspect } from '../string/inspect.js'
3
3
 
4
4
  /**
@@ -1,8 +1,8 @@
1
- import 'dotenv/config'
2
1
  import os from 'node:os'
3
- import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
4
- import { pDelay, setGlobalStringifyFunction } from '@naturalcycles/js-lib'
5
- import { dimGrey } from '../colors/colors.js'
2
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
+ import { pDelay } from '@naturalcycles/js-lib/promise'
4
+ import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string'
5
+ import type { AnyObject } from '@naturalcycles/js-lib/types'
6
6
  import { inspectStringifyFn } from '../string/inspect.js'
7
7
 
8
8
  export interface RunScriptOptions {
@@ -46,7 +46,6 @@ const { DEBUG_RUN_SCRIPT } = process.env
46
46
  * Set env DEBUG_RUN_SCRIPT for extra debugging.
47
47
  */
48
48
  export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {}): void {
49
- checkAndlogEnvironment()
50
49
  setGlobalStringifyFunction(inspectStringifyFn)
51
50
 
52
51
  const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt
@@ -70,6 +69,9 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
70
69
 
71
70
  void (async () => {
72
71
  try {
72
+ await import('dotenv/config')
73
+ await checkAndlogEnvironment()
74
+
73
75
  await fn()
74
76
 
75
77
  await pDelay() // to ensure all async operations are completed
@@ -91,7 +93,9 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
91
93
  })()
92
94
  }
93
95
 
94
- function checkAndlogEnvironment(): void {
96
+ async function checkAndlogEnvironment(): Promise<void> {
97
+ const { dimGrey } = await import('../colors/colors.js')
98
+
95
99
  const {
96
100
  platform,
97
101
  arch,
@@ -1,6 +1,6 @@
1
1
  import fs from 'node:fs'
2
2
  import path from 'node:path'
3
- import { _assert } from '@naturalcycles/js-lib'
3
+ import { _assert } from '@naturalcycles/js-lib/error'
4
4
  import { globSync } from 'tinyglobby'
5
5
  import { dimGrey, yellow } from '../colors/colors.js'
6
6
  import { fs2 } from '../fs/fs2.js'
@@ -1,6 +1,6 @@
1
1
  import fs from 'node:fs'
2
2
  import path from 'node:path'
3
- import { _assert } from '@naturalcycles/js-lib'
3
+ import { _assert } from '@naturalcycles/js-lib/error'
4
4
  import { globSync } from 'tinyglobby'
5
5
  import { dimGrey, yellow } from '../colors/colors.js'
6
6
  import { fs2 } from '../fs/fs2.js'
@@ -1,6 +1,5 @@
1
1
  import crypto from 'node:crypto'
2
- import type { Base64String, StringMap } from '@naturalcycles/js-lib'
3
- import { _stringMapEntries } from '@naturalcycles/js-lib'
2
+ import { _stringMapEntries, type Base64String, type StringMap } from '@naturalcycles/js-lib/types'
4
3
  import { md5AsBuffer, sha256AsBuffer } from './hash.util.js'
5
4
 
6
5
  const algorithm = 'aes-256-cbc'
@@ -1,6 +1,6 @@
1
1
  import type { BinaryToTextEncoding } from 'node:crypto'
2
2
  import { hash as cryptoHash } from 'node:crypto'
3
- import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib'
3
+ import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types'
4
4
 
5
5
  export function md5(s: string | Buffer, outputEncoding: BinaryToTextEncoding = 'hex'): string {
6
6
  return hash(s, 'md5', outputEncoding)
@@ -1,6 +1,7 @@
1
1
  import { existsSync, readFileSync } from 'node:fs'
2
- import type { Base64String, StringMap } from '@naturalcycles/js-lib'
3
- import { _assert, _jsonParseIfPossible } from '@naturalcycles/js-lib'
2
+ import { _assert } from '@naturalcycles/js-lib/error'
3
+ import { _jsonParseIfPossible } from '@naturalcycles/js-lib/string'
4
+ import type { Base64String, StringMap } from '@naturalcycles/js-lib/types'
4
5
  import { decryptObject, decryptRandomIVBuffer } from './crypto.util.js'
5
6
 
6
7
  let loaded = false
@@ -1,4 +1,5 @@
1
- import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
1
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
2
+ import type { AnyObject } from '@naturalcycles/js-lib/types'
2
3
  import type { InspectAnyOptions } from '../index.js'
3
4
 
4
5
  /**
@@ -1,5 +1,13 @@
1
- import type { AnyObject, CommonLogger, CommonLogLevel, Fetcher } from '@naturalcycles/js-lib'
2
- import { _omit, commonLoggerMinLevel, getFetcher, localTime, PQueue } from '@naturalcycles/js-lib'
1
+ import { localTime } from '@naturalcycles/js-lib/datetime'
2
+ import { type Fetcher, getFetcher } from '@naturalcycles/js-lib/http'
3
+ import {
4
+ type CommonLogger,
5
+ commonLoggerMinLevel,
6
+ type CommonLogLevel,
7
+ } from '@naturalcycles/js-lib/log'
8
+ import { _omit } from '@naturalcycles/js-lib/object'
9
+ import { PQueue } from '@naturalcycles/js-lib/promise'
10
+ import type { AnyObject } from '@naturalcycles/js-lib/types'
3
11
  import type { InspectAnyOptions } from '../index.js'
4
12
  import { _inspect } from '../index.js'
5
13
  import type {
@@ -1,3 +1,5 @@
1
+ export * from './ndjson/createReadStreamAsNDJSON.js'
2
+ export * from './ndjson/createWriteStreamAsNDJSON.js'
1
3
  export * from './ndjson/ndjson.model.js'
2
4
  export * from './ndjson/ndjsonMap.js'
3
5
  export * from './ndjson/ndjsonStreamForEach.js'
@@ -0,0 +1,46 @@
1
+ import { createUnzip } from 'node:zlib'
2
+ import { fs2 } from '../../fs/fs2.js'
3
+ import type { ReadableTyped } from '../stream.model.js'
4
+ import { transformSplitOnNewline } from '../transform/transformSplit.js'
5
+
6
+ /**
7
+ Returns a Readable of [already parsed] NDJSON objects.
8
+
9
+ Replaces a list of operations:
10
+ - requireFileToExist(inputPath)
11
+ - fs.createReadStream
12
+ - createUnzip (only if path ends with '.gz')
13
+ - transformSplitOnNewline
14
+ - transformJsonParse
15
+
16
+ To add a Limit or Offset: just add .take() or .drop(), example:
17
+
18
+ _pipeline([
19
+ fs2.createReadStreamAsNDJSON().take(100),
20
+ transformX(),
21
+ ])
22
+ */
23
+
24
+ export function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
25
+ fs2.requireFileToExist(inputPath)
26
+
27
+ let stream: ReadableTyped<ROW> = fs2
28
+ .createReadStream(inputPath, {
29
+ highWaterMark: 64 * 1024, // no observed speedup
30
+ })
31
+ .on('error', err => stream.emit('error', err))
32
+
33
+ if (inputPath.endsWith('.gz')) {
34
+ stream = stream.pipe(
35
+ createUnzip({
36
+ chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
37
+ }),
38
+ )
39
+ }
40
+
41
+ return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
42
+ // For some crazy reason .map is much faster than transformJsonParse!
43
+ // ~5000 vs ~4000 rps !!!
44
+ // .on('error', err => stream.emit('error', err))
45
+ // .pipe(transformJsonParse<ROW>())
46
+ }
@@ -0,0 +1,30 @@
1
+ import { createGzip } from 'node:zlib'
2
+ import { _isTruthy } from '@naturalcycles/js-lib'
3
+ import { fs2 } from '../../fs/fs2.js'
4
+ import type { TransformTyped } from '../stream.model.js'
5
+ import { transformToNDJson } from './transformToNDJson.js'
6
+
7
+ /**
8
+ Returns an array of Transforms, so that you can ...destructure them at
9
+ the end of the _pipeline.
10
+
11
+ Replaces a list of operations:
12
+ - transformToNDJson
13
+ - createGzip (only if path ends with '.gz')
14
+ - fs.createWriteStream
15
+ */
16
+ export function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
17
+ fs2.ensureFile(outputPath)
18
+
19
+ return [
20
+ transformToNDJson(),
21
+ outputPath.endsWith('.gz')
22
+ ? createGzip({
23
+ // chunkSize: 64 * 1024, // no observed speedup
24
+ })
25
+ : undefined,
26
+ fs2.createWriteStream(outputPath, {
27
+ // highWaterMark: 64 * 1024, // no observed speedup
28
+ }),
29
+ ].filter(_isTruthy) as TransformTyped<any, any>[]
30
+ }
@@ -1,4 +1,5 @@
1
- import { _hb, _ms } from '@naturalcycles/js-lib'
1
+ import { _hb } from '@naturalcycles/js-lib'
2
+ import { _ms } from '@naturalcycles/js-lib/datetime'
2
3
  import { boldWhite, dimWhite } from '../../colors/colors.js'
3
4
 
4
5
  export class NDJsonStats {
@@ -1,7 +1,11 @@
1
- import type { AbortableAsyncMapper } from '@naturalcycles/js-lib'
2
- import { ErrorMode } from '@naturalcycles/js-lib'
3
- import { fs2 } from '../../fs/index.js'
4
- import type { TransformLogProgressOptions, TransformMapOptions } from '../index.js'
1
+ import { ErrorMode } from '@naturalcycles/js-lib/error'
2
+ import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
3
+ import {
4
+ createReadStreamAsNDJSON,
5
+ createWriteStreamAsNDJSON,
6
+ type TransformLogProgressOptions,
7
+ type TransformMapOptions,
8
+ } from '../index.js'
5
9
  import { _pipeline, transformLimit, transformLogProgress, transformMap } from '../index.js'
6
10
 
7
11
  export interface NDJSONMapOptions<IN = any, OUT = IN>
@@ -41,9 +45,9 @@ export async function ndjsonMap<IN = any, OUT = any>(
41
45
  outputFilePath,
42
46
  })
43
47
 
44
- const readable = fs2
45
- .createReadStreamAsNDJSON(inputFilePath)
46
- .take(limitInput || Number.POSITIVE_INFINITY)
48
+ const readable = createReadStreamAsNDJSON(inputFilePath).take(
49
+ limitInput || Number.POSITIVE_INFINITY,
50
+ )
47
51
 
48
52
  await _pipeline([
49
53
  readable,
@@ -55,6 +59,6 @@ export async function ndjsonMap<IN = any, OUT = any>(
55
59
  }),
56
60
  transformLimit({ limit: limitOutput, sourceReadable: readable }),
57
61
  transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
58
- ...fs2.createWriteStreamAsNDJSON(outputFilePath),
62
+ ...createWriteStreamAsNDJSON(outputFilePath),
59
63
  ])
60
64
  }
@@ -1,7 +1,10 @@
1
- import type { AbortableAsyncMapper } from '@naturalcycles/js-lib'
2
- import { ErrorMode } from '@naturalcycles/js-lib'
3
- import { fs2 } from '../../fs/index.js'
4
- import type { TransformLogProgressOptions, TransformMapOptions } from '../index.js'
1
+ import { ErrorMode } from '@naturalcycles/js-lib/error'
2
+ import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
3
+ import {
4
+ createReadStreamAsNDJSON,
5
+ type TransformLogProgressOptions,
6
+ type TransformMapOptions,
7
+ } from '../index.js'
5
8
  import { _pipeline, transformLogProgress, transformMap, writableVoid } from '../index.js'
6
9
 
7
10
  export interface NDJSONStreamForEachOptions<IN = any>
@@ -18,7 +21,7 @@ export async function ndjsonStreamForEach<T>(
18
21
  opt: NDJSONStreamForEachOptions<T>,
19
22
  ): Promise<void> {
20
23
  await _pipeline([
21
- fs2.createReadStreamAsNDJSON(opt.inputFilePath),
24
+ createReadStreamAsNDJSON(opt.inputFilePath),
22
25
  transformMap<T, any>(mapper, {
23
26
  errorMode: ErrorMode.THROW_AGGREGATED,
24
27
  ...opt,
@@ -1,5 +1,5 @@
1
1
  import { Transform } from 'node:stream'
2
- import type { Reviver } from '@naturalcycles/js-lib'
2
+ import type { Reviver } from '@naturalcycles/js-lib/types'
3
3
  import type { TransformTyped } from '../stream.model.js'
4
4
 
5
5
  export interface TransformJsonParseOptions {
@@ -1,5 +1,5 @@
1
1
  import { Transform } from 'node:stream'
2
- import { _sortObjectDeep } from '@naturalcycles/js-lib'
2
+ import { _sortObjectDeep } from '@naturalcycles/js-lib/object'
3
3
  import type { TransformTyped } from '../stream.model.js'
4
4
 
5
5
  export interface TransformToNDJsonOptions {
@@ -1,8 +1,10 @@
1
1
  import type { Readable, Writable } from 'node:stream'
2
2
  import { Transform } from 'node:stream'
3
3
  import { pipeline } from 'node:stream/promises'
4
- import type { AnyFunction, DeferredPromise } from '@naturalcycles/js-lib'
5
- import { _last, pDefer } from '@naturalcycles/js-lib'
4
+ import { _last } from '@naturalcycles/js-lib'
5
+ import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
6
+ import { pDefer } from '@naturalcycles/js-lib/promise'
7
+ import type { AnyFunction } from '@naturalcycles/js-lib/types'
6
8
  import { writablePushToArray } from '../writable/writablePushToArray.js'
7
9
 
8
10
  type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream
@@ -1,7 +1,10 @@
1
1
  import type { InspectOptions } from 'node:util'
2
2
  import { inspect } from 'node:util'
3
- import type { AnyObject, CommonLogger, UnixTimestampMillis } from '@naturalcycles/js-lib'
4
- import { _mb, _since, localTime, SimpleMovingAverage } from '@naturalcycles/js-lib'
3
+ import { _mb } from '@naturalcycles/js-lib'
4
+ import { _since, localTime } from '@naturalcycles/js-lib/datetime'
5
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
6
+ import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
7
+ import type { AnyObject, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
5
8
  import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js'
6
9
  import { SizeStack } from './sizeStack.js'
7
10
  import type { ReadableMapper } from './stream.model.js'
@@ -1,5 +1,5 @@
1
- import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib'
2
- import { _passNothingPredicate } from '@naturalcycles/js-lib'
1
+ import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
2
+ import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
3
3
  import type { ReadableTyped } from '../index.js'
4
4
  import { _pipeline } from '../index.js'
5
5
  import type { TransformMapOptions } from '../transform/transformMap.js'
@@ -1,7 +1,7 @@
1
1
  import type { ReadableOptions } from 'node:stream'
2
2
  import { Readable } from 'node:stream'
3
- import type { AbortableAsyncMapper } from '@naturalcycles/js-lib'
4
- import { _passthroughMapper } from '@naturalcycles/js-lib'
3
+ import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
4
+ import { _passthroughMapper } from '@naturalcycles/js-lib/types'
5
5
  import type { ReadableTyped } from '../stream.model.js'
6
6
 
7
7
  /**
@@ -1,7 +1,8 @@
1
- import type { CommonLogger } from '@naturalcycles/js-lib'
2
- import { _hb, NumberStack } from '@naturalcycles/js-lib'
1
+ import { _hb } from '@naturalcycles/js-lib'
2
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
+ import { NumberStack } from '@naturalcycles/js-lib/math'
3
4
  import { yellow } from '../colors/colors.js'
4
- import { gzipBuffer } from '../util/zip.util.js'
5
+ import { gzipBuffer } from '../zip/zip.util.js'
5
6
 
6
7
  export class SizeStack extends NumberStack {
7
8
  constructor(
@@ -1,5 +1,5 @@
1
1
  import type { Readable, Transform, Writable } from 'node:stream'
2
- import type { Promisable } from '@naturalcycles/js-lib'
2
+ import type { Promisable } from '@naturalcycles/js-lib/types'
3
3
 
4
4
  export interface ReadableSignalOptions {
5
5
  /** allows destroying the stream if the signal is aborted. */
@@ -1,5 +1,5 @@
1
1
  import type { Readable } from 'node:stream'
2
- import type { CommonLogger } from '@naturalcycles/js-lib'
2
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
3
 
4
4
  export function pipelineClose(
5
5
  name: string,
@@ -1,5 +1,5 @@
1
1
  import { Transform } from 'node:stream'
2
- import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib'
2
+ import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib/types'
3
3
  import type { TransformOptions, TransformTyped } from '../stream.model.js'
4
4
  import type { TransformMapOptions } from './transformMap.js'
5
5
  import { transformMap } from './transformMap.js'
@@ -1,5 +1,5 @@
1
1
  import type { Readable } from 'node:stream'
2
- import type { CommonLogger } from '@naturalcycles/js-lib'
2
+ import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
3
  import { AbortableTransform, transformNoOp } from '../index.js'
4
4
  import type { TransformOptions, TransformTyped } from '../stream.model.js'
5
5
  import { pipelineClose } from '../stream.util.js'