@naturalcycles/nodejs-lib 15.1.0 → 15.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/slack-this.js +1 -1
- package/dist/{util → cache}/lruMemoCache.d.ts +1 -1
- package/dist/csv/csvReader.d.ts +1 -1
- package/dist/csv/csvReader.js +1 -1
- package/dist/csv/csvWriter.d.ts +1 -1
- package/dist/csv/csvWriter.js +1 -1
- package/dist/csv/transformToCSV.d.ts +1 -1
- package/dist/diff/tableDiff.d.ts +1 -1
- package/dist/diff/tableDiff.js +1 -1
- package/dist/{util/exec2.d.ts → exec2.d.ts} +2 -2
- package/dist/{util/exec2.js → exec2.js} +4 -3
- package/dist/fs/fs2.d.ts +0 -10
- package/dist/fs/fs2.js +1 -85
- package/dist/fs/json2env.d.ts +1 -1
- package/dist/fs/kpy.js +1 -1
- package/dist/fs/yaml2.d.ts +11 -0
- package/dist/fs/yaml2.js +29 -0
- package/dist/index.d.ts +0 -4
- package/dist/index.js +0 -4
- package/dist/jwt/jwt.service.d.ts +2 -1
- package/dist/jwt/jwt.service.js +1 -1
- package/dist/log/log.util.d.ts +1 -1
- package/dist/log/log.util.js +1 -1
- package/dist/script/runScript.d.ts +1 -2
- package/dist/script/runScript.js +6 -5
- package/dist/secret/secrets-decrypt.util.js +1 -1
- package/dist/secret/secrets-encrypt.util.js +1 -1
- package/dist/security/crypto.util.d.ts +1 -1
- package/dist/security/crypto.util.js +1 -1
- package/dist/security/hash.util.d.ts +1 -1
- package/dist/security/secret.util.d.ts +1 -1
- package/dist/security/secret.util.js +2 -1
- package/dist/slack/slack.service.d.ts +2 -1
- package/dist/slack/slack.service.js +5 -1
- package/dist/slack/slack.service.model.d.ts +2 -1
- package/dist/stream/index.d.ts +2 -0
- package/dist/stream/index.js +2 -0
- package/dist/stream/ndjson/createReadStreamAsNDJSON.d.ts +19 -0
- package/dist/stream/ndjson/createReadStreamAsNDJSON.js +38 -0
- package/dist/stream/ndjson/createWriteStreamAsNDJSON.d.ts +11 -0
- package/dist/stream/ndjson/createWriteStreamAsNDJSON.js +27 -0
- package/dist/stream/ndjson/ndjson.model.js +2 -1
- package/dist/stream/ndjson/ndjsonMap.d.ts +2 -2
- package/dist/stream/ndjson/ndjsonMap.js +4 -6
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +2 -2
- package/dist/stream/ndjson/ndjsonStreamForEach.js +3 -3
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +1 -1
- package/dist/stream/pipeline/pipeline.d.ts +1 -1
- package/dist/stream/pipeline/pipeline.js +2 -1
- package/dist/stream/progressLogger.d.ts +2 -1
- package/dist/stream/progressLogger.js +3 -1
- package/dist/stream/readable/readableForEach.d.ts +1 -1
- package/dist/stream/readable/readableForEach.js +1 -1
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +1 -1
- package/dist/stream/sizeStack.d.ts +2 -2
- package/dist/stream/sizeStack.js +3 -2
- package/dist/stream/stream.model.d.ts +1 -1
- package/dist/stream/stream.util.d.ts +1 -1
- package/dist/stream/transform/transformFilter.d.ts +1 -1
- package/dist/stream/transform/transformLimit.d.ts +1 -1
- package/dist/stream/transform/transformMap.d.ts +3 -2
- package/dist/stream/transform/transformMap.js +6 -1
- package/dist/stream/transform/transformMapSimple.d.ts +3 -2
- package/dist/stream/transform/transformMapSimple.js +1 -1
- package/dist/stream/transform/transformMapSync.d.ts +4 -2
- package/dist/stream/transform/transformMapSync.js +2 -1
- package/dist/stream/transform/transformTap.d.ts +2 -1
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +2 -1
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +2 -1
- package/dist/stream/writable/writableForEach.d.ts +1 -1
- package/dist/stream/writable/writableForEach.js +1 -1
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/string/inspect.d.ts +1 -1
- package/dist/string/inspect.js +1 -1
- package/dist/util/buildInfo.util.d.ts +2 -1
- package/dist/util/buildInfo.util.js +2 -1
- package/dist/util/env.util.d.ts +1 -2
- package/dist/util/env.util.js +0 -1
- package/dist/util/git2.d.ts +1 -1
- package/dist/util/git2.js +2 -3
- package/dist/validation/ajv/ajv.util.d.ts +1 -1
- package/dist/validation/ajv/ajvSchema.d.ts +1 -1
- package/dist/validation/ajv/ajvSchema.js +4 -1
- package/dist/validation/ajv/ajvValidationError.d.ts +2 -2
- package/dist/validation/ajv/ajvValidationError.js +1 -1
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.validation.error.d.ts +2 -2
- package/dist/validation/joi/joi.validation.error.js +1 -1
- package/dist/validation/joi/joi.validation.util.js +2 -1
- package/dist/validation/joi/string.extensions.d.ts +1 -1
- package/dist/validation/joi/string.extensions.js +1 -1
- package/dist/{util → zip}/zip.util.js +0 -1
- package/package.json +12 -5
- package/src/bin/generate-build-info.ts +1 -1
- package/src/bin/slack-this.ts +1 -1
- package/src/{util → cache}/lruMemoCache.ts +1 -1
- package/src/csv/csvReader.ts +2 -2
- package/src/csv/csvWriter.ts +2 -2
- package/src/csv/transformToCSV.ts +1 -1
- package/src/diff/tableDiff.ts +2 -2
- package/src/{util/exec2.ts → exec2.ts} +9 -8
- package/src/fs/fs2.ts +1 -101
- package/src/fs/json2env.ts +1 -1
- package/src/fs/kpy.ts +2 -2
- package/src/fs/yaml2.ts +37 -0
- package/src/index.ts +0 -4
- package/src/jwt/jwt.service.ts +3 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +10 -6
- package/src/secret/secrets-decrypt.util.ts +1 -1
- package/src/secret/secrets-encrypt.util.ts +1 -1
- package/src/security/crypto.util.ts +1 -2
- package/src/security/hash.util.ts +1 -1
- package/src/security/secret.util.ts +3 -2
- package/src/slack/slack.service.model.ts +2 -1
- package/src/slack/slack.service.ts +10 -2
- package/src/stream/index.ts +2 -0
- package/src/stream/ndjson/createReadStreamAsNDJSON.ts +46 -0
- package/src/stream/ndjson/createWriteStreamAsNDJSON.ts +30 -0
- package/src/stream/ndjson/ndjson.model.ts +2 -1
- package/src/stream/ndjson/ndjsonMap.ts +12 -8
- package/src/stream/ndjson/ndjsonStreamForEach.ts +8 -5
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +4 -2
- package/src/stream/progressLogger.ts +5 -2
- package/src/stream/readable/readableForEach.ts +2 -2
- package/src/stream/readable/readableFromArray.ts +2 -2
- package/src/stream/sizeStack.ts +4 -3
- package/src/stream/stream.model.ts +1 -1
- package/src/stream/stream.util.ts +1 -1
- package/src/stream/transform/transformFilter.ts +1 -1
- package/src/stream/transform/transformLimit.ts +1 -1
- package/src/stream/transform/transformMap.ts +12 -15
- package/src/stream/transform/transformMapSimple.ts +3 -2
- package/src/stream/transform/transformMapSync.ts +4 -7
- package/src/stream/transform/transformTap.ts +2 -1
- package/src/stream/transform/transformThrottle.ts +4 -3
- package/src/stream/transform/worker/transformMultiThreaded.ts +4 -2
- package/src/stream/writable/writableForEach.ts +2 -2
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/string/inspect.ts +2 -2
- package/src/util/buildInfo.util.ts +4 -2
- package/src/util/env.util.ts +1 -2
- package/src/util/git2.ts +3 -4
- package/src/validation/ajv/ajv.util.ts +1 -1
- package/src/validation/ajv/ajvSchema.ts +5 -7
- package/src/validation/ajv/ajvValidationError.ts +2 -2
- package/src/validation/joi/joi.shared.schemas.ts +7 -7
- package/src/validation/joi/joi.validation.error.ts +2 -2
- package/src/validation/joi/joi.validation.util.ts +2 -1
- package/src/validation/joi/string.extensions.ts +2 -2
- package/src/{util → zip}/zip.util.ts +0 -2
- package/dist/fs/index.d.ts +0 -3
- package/dist/fs/index.js +0 -3
- package/dist/jwt/index.d.ts +0 -1
- package/dist/jwt/index.js +0 -1
- package/dist/yargs/index.d.ts +0 -1
- package/dist/yargs/index.js +0 -1
- package/src/fs/index.ts +0 -3
- package/src/jwt/index.ts +0 -1
- package/src/yargs/index.ts +0 -1
- /package/dist/{util → cache}/lruMemoCache.js +0 -0
- /package/dist/{util → zip}/zip.util.d.ts +0 -0
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { execSync, spawn, spawnSync } from 'node:child_process'
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
2
|
+
import { _since } from '@naturalcycles/js-lib/datetime'
|
|
3
|
+
import { AppError } from '@naturalcycles/js-lib/error'
|
|
4
|
+
import { _substringAfterLast } from '@naturalcycles/js-lib/string'
|
|
5
|
+
import type {
|
|
6
|
+
AnyObject,
|
|
7
|
+
NumberOfMilliseconds,
|
|
8
|
+
UnixTimestampMillis,
|
|
9
|
+
} from '@naturalcycles/js-lib/types'
|
|
10
|
+
import { dimGrey, dimRed, hasColors, white } from './colors/colors.js'
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Set of utility functions to work with Spawn / Exec.
|
package/src/fs/fs2.ts
CHANGED
|
@@ -18,13 +18,7 @@ import type { RmOptions, Stats } from 'node:fs'
|
|
|
18
18
|
import fs from 'node:fs'
|
|
19
19
|
import fsp from 'node:fs/promises'
|
|
20
20
|
import path from 'node:path'
|
|
21
|
-
import {
|
|
22
|
-
import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib'
|
|
23
|
-
import type { DumpOptions } from 'js-yaml'
|
|
24
|
-
import yaml from 'js-yaml'
|
|
25
|
-
import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js'
|
|
26
|
-
import type { ReadableTyped, TransformTyped } from '../stream/stream.model.js'
|
|
27
|
-
import { transformSplitOnNewline } from '../stream/transform/transformSplit.js'
|
|
21
|
+
import { _jsonParse } from '@naturalcycles/js-lib/string'
|
|
28
22
|
|
|
29
23
|
/**
|
|
30
24
|
* fs2 conveniently groups filesystem functions together.
|
|
@@ -72,14 +66,6 @@ class FS2 {
|
|
|
72
66
|
return _jsonParse(str)
|
|
73
67
|
}
|
|
74
68
|
|
|
75
|
-
readYaml<T = unknown>(filePath: string): T {
|
|
76
|
-
return yaml.load(fs.readFileSync(filePath, 'utf8')) as T
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
async readYamlAsync<T = unknown>(filePath: string): Promise<T> {
|
|
80
|
-
return yaml.load(await fsp.readFile(filePath, 'utf8')) as T
|
|
81
|
-
}
|
|
82
|
-
|
|
83
69
|
writeFile(filePath: string, data: string | Buffer): void {
|
|
84
70
|
fs.writeFileSync(filePath, data)
|
|
85
71
|
}
|
|
@@ -98,16 +84,6 @@ class FS2 {
|
|
|
98
84
|
await fsp.writeFile(filePath, str)
|
|
99
85
|
}
|
|
100
86
|
|
|
101
|
-
writeYaml(filePath: string, data: any, opt?: DumpOptions): void {
|
|
102
|
-
const str = yaml.dump(data, opt)
|
|
103
|
-
fs.writeFileSync(filePath, str)
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
async writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
|
|
107
|
-
const str = yaml.dump(data, opt)
|
|
108
|
-
await fsp.writeFile(filePath, str)
|
|
109
|
-
}
|
|
110
|
-
|
|
111
87
|
appendFile(filePath: string, data: string | Buffer): void {
|
|
112
88
|
fs.appendFileSync(filePath, data)
|
|
113
89
|
}
|
|
@@ -126,16 +102,6 @@ class FS2 {
|
|
|
126
102
|
await this.outputFileAsync(filePath, str)
|
|
127
103
|
}
|
|
128
104
|
|
|
129
|
-
outputYaml(filePath: string, data: any, opt?: DumpOptions): void {
|
|
130
|
-
const str = yaml.dump(data, opt)
|
|
131
|
-
this.outputFile(filePath, str)
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
async outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
|
|
135
|
-
const str = yaml.dump(data, opt)
|
|
136
|
-
await this.outputFileAsync(filePath, str)
|
|
137
|
-
}
|
|
138
|
-
|
|
139
105
|
outputFile(filePath: string, data: string | Buffer): void {
|
|
140
106
|
const dirPath = path.dirname(filePath)
|
|
141
107
|
if (!fs.existsSync(dirPath)) {
|
|
@@ -336,72 +302,6 @@ class FS2 {
|
|
|
336
302
|
readdirAsync = fsp.readdir
|
|
337
303
|
createWriteStream = fs.createWriteStream
|
|
338
304
|
createReadStream = fs.createReadStream
|
|
339
|
-
|
|
340
|
-
/*
|
|
341
|
-
Returns a Readable of [already parsed] NDJSON objects.
|
|
342
|
-
|
|
343
|
-
Replaces a list of operations:
|
|
344
|
-
- requireFileToExist(inputPath)
|
|
345
|
-
- fs.createReadStream
|
|
346
|
-
- createUnzip (only if path ends with '.gz')
|
|
347
|
-
- transformSplitOnNewline
|
|
348
|
-
- transformJsonParse
|
|
349
|
-
|
|
350
|
-
To add a Limit or Offset: just add .take() or .drop(), example:
|
|
351
|
-
|
|
352
|
-
_pipeline([
|
|
353
|
-
fs2.createReadStreamAsNDJSON().take(100),
|
|
354
|
-
transformX(),
|
|
355
|
-
])
|
|
356
|
-
*/
|
|
357
|
-
createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
|
|
358
|
-
this.requireFileToExist(inputPath)
|
|
359
|
-
|
|
360
|
-
let stream: ReadableTyped<ROW> = fs
|
|
361
|
-
.createReadStream(inputPath, {
|
|
362
|
-
highWaterMark: 64 * 1024, // no observed speedup
|
|
363
|
-
})
|
|
364
|
-
.on('error', err => stream.emit('error', err))
|
|
365
|
-
|
|
366
|
-
if (inputPath.endsWith('.gz')) {
|
|
367
|
-
stream = stream.pipe(
|
|
368
|
-
createUnzip({
|
|
369
|
-
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
370
|
-
}),
|
|
371
|
-
)
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
|
|
375
|
-
// For some crazy reason .map is much faster than transformJsonParse!
|
|
376
|
-
// ~5000 vs ~4000 rps !!!
|
|
377
|
-
// .on('error', err => stream.emit('error', err))
|
|
378
|
-
// .pipe(transformJsonParse<ROW>())
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/*
|
|
382
|
-
Returns an array of Transforms, so that you can ...destructure them at
|
|
383
|
-
the end of the _pipeline.
|
|
384
|
-
|
|
385
|
-
Replaces a list of operations:
|
|
386
|
-
- transformToNDJson
|
|
387
|
-
- createGzip (only if path ends with '.gz')
|
|
388
|
-
- fs.createWriteStream
|
|
389
|
-
*/
|
|
390
|
-
createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
|
|
391
|
-
this.ensureFile(outputPath)
|
|
392
|
-
|
|
393
|
-
return [
|
|
394
|
-
transformToNDJson(),
|
|
395
|
-
outputPath.endsWith('.gz')
|
|
396
|
-
? createGzip({
|
|
397
|
-
// chunkSize: 64 * 1024, // no observed speedup
|
|
398
|
-
})
|
|
399
|
-
: undefined,
|
|
400
|
-
fs.createWriteStream(outputPath, {
|
|
401
|
-
// highWaterMark: 64 * 1024, // no observed speedup
|
|
402
|
-
}),
|
|
403
|
-
].filter(_isTruthy) as TransformTyped<any, any>[]
|
|
404
|
-
}
|
|
405
305
|
}
|
|
406
306
|
|
|
407
307
|
export const fs2 = new FS2()
|
package/src/fs/json2env.ts
CHANGED
package/src/fs/kpy.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path'
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
3
|
+
import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
|
|
4
4
|
import { glob, globSync } from 'tinyglobby'
|
|
5
5
|
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js'
|
|
6
6
|
import { fs2 } from './fs2.js'
|
package/src/fs/yaml2.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import fsp from 'node:fs/promises'
|
|
3
|
+
import type { DumpOptions } from 'js-yaml'
|
|
4
|
+
import yaml from 'js-yaml'
|
|
5
|
+
import { fs2 } from './fs2.js'
|
|
6
|
+
|
|
7
|
+
class Yaml2 {
|
|
8
|
+
readYaml<T = unknown>(filePath: string): T {
|
|
9
|
+
return yaml.load(fs.readFileSync(filePath, 'utf8')) as T
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async readYamlAsync<T = unknown>(filePath: string): Promise<T> {
|
|
13
|
+
return yaml.load(await fsp.readFile(filePath, 'utf8')) as T
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
writeYaml(filePath: string, data: any, opt?: DumpOptions): void {
|
|
17
|
+
const str = yaml.dump(data, opt)
|
|
18
|
+
fs.writeFileSync(filePath, str)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
|
|
22
|
+
const str = yaml.dump(data, opt)
|
|
23
|
+
await fsp.writeFile(filePath, str)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
outputYaml(filePath: string, data: any, opt?: DumpOptions): void {
|
|
27
|
+
const str = yaml.dump(data, opt)
|
|
28
|
+
fs2.outputFile(filePath, str)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
|
|
32
|
+
const str = yaml.dump(data, opt)
|
|
33
|
+
await fs2.outputFileAsync(filePath, str)
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export const yaml2 = new Yaml2()
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export * from './buffer/buffer.util.js'
|
|
2
|
-
export * from './colors/colors.js'
|
|
3
2
|
export * from './diff/tableDiff.js'
|
|
4
3
|
export * from './infra/process.util.js'
|
|
5
4
|
export * from './log/log.util.js'
|
|
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js'
|
|
|
14
13
|
export * from './string/inspect.js'
|
|
15
14
|
export * from './util/buildInfo.util.js'
|
|
16
15
|
export * from './util/env.util.js'
|
|
17
|
-
export * from './util/exec2.js'
|
|
18
16
|
export * from './util/git2.js'
|
|
19
|
-
export * from './util/lruMemoCache.js'
|
|
20
|
-
export * from './util/zip.util.js'
|
package/src/jwt/jwt.service.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import { _assert, _errorDataAppend } from '@naturalcycles/js-lib'
|
|
1
|
+
import type { ErrorData } from '@naturalcycles/js-lib/error'
|
|
2
|
+
import { _assert, _errorDataAppend } from '@naturalcycles/js-lib/error'
|
|
3
|
+
import type { AnyObject, JWTString } from '@naturalcycles/js-lib/types'
|
|
3
4
|
import type { AnySchema } from 'joi'
|
|
4
5
|
import type { Algorithm, JwtHeader, SignOptions, VerifyOptions } from 'jsonwebtoken'
|
|
5
6
|
import jsonwebtoken from 'jsonwebtoken'
|
package/src/log/log.util.ts
CHANGED
package/src/script/runScript.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import 'dotenv/config'
|
|
2
1
|
import os from 'node:os'
|
|
3
|
-
import type {
|
|
4
|
-
import { pDelay
|
|
5
|
-
import {
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
3
|
+
import { pDelay } from '@naturalcycles/js-lib/promise'
|
|
4
|
+
import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string'
|
|
5
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types'
|
|
6
6
|
import { inspectStringifyFn } from '../string/inspect.js'
|
|
7
7
|
|
|
8
8
|
export interface RunScriptOptions {
|
|
@@ -46,7 +46,6 @@ const { DEBUG_RUN_SCRIPT } = process.env
|
|
|
46
46
|
* Set env DEBUG_RUN_SCRIPT for extra debugging.
|
|
47
47
|
*/
|
|
48
48
|
export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {}): void {
|
|
49
|
-
checkAndlogEnvironment()
|
|
50
49
|
setGlobalStringifyFunction(inspectStringifyFn)
|
|
51
50
|
|
|
52
51
|
const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt
|
|
@@ -70,6 +69,9 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
70
69
|
|
|
71
70
|
void (async () => {
|
|
72
71
|
try {
|
|
72
|
+
await import('dotenv/config')
|
|
73
|
+
await checkAndlogEnvironment()
|
|
74
|
+
|
|
73
75
|
await fn()
|
|
74
76
|
|
|
75
77
|
await pDelay() // to ensure all async operations are completed
|
|
@@ -91,7 +93,9 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
91
93
|
})()
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
function checkAndlogEnvironment(): void {
|
|
96
|
+
async function checkAndlogEnvironment(): Promise<void> {
|
|
97
|
+
const { dimGrey } = await import('../colors/colors.js')
|
|
98
|
+
|
|
95
99
|
const {
|
|
96
100
|
platform,
|
|
97
101
|
arch,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
|
-
import { _assert } from '@naturalcycles/js-lib'
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
4
4
|
import { globSync } from 'tinyglobby'
|
|
5
5
|
import { dimGrey, yellow } from '../colors/colors.js'
|
|
6
6
|
import { fs2 } from '../fs/fs2.js'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
|
-
import { _assert } from '@naturalcycles/js-lib'
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
4
4
|
import { globSync } from 'tinyglobby'
|
|
5
5
|
import { dimGrey, yellow } from '../colors/colors.js'
|
|
6
6
|
import { fs2 } from '../fs/fs2.js'
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import crypto from 'node:crypto'
|
|
2
|
-
import type
|
|
3
|
-
import { _stringMapEntries } from '@naturalcycles/js-lib'
|
|
2
|
+
import { _stringMapEntries, type Base64String, type StringMap } from '@naturalcycles/js-lib/types'
|
|
4
3
|
import { md5AsBuffer, sha256AsBuffer } from './hash.util.js'
|
|
5
4
|
|
|
6
5
|
const algorithm = 'aes-256-cbc'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BinaryToTextEncoding } from 'node:crypto'
|
|
2
2
|
import { hash as cryptoHash } from 'node:crypto'
|
|
3
|
-
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib'
|
|
3
|
+
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types'
|
|
4
4
|
|
|
5
5
|
export function md5(s: string | Buffer, outputEncoding: BinaryToTextEncoding = 'hex'): string {
|
|
6
6
|
return hash(s, 'md5', outputEncoding)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from 'node:fs'
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
3
|
+
import { _jsonParseIfPossible } from '@naturalcycles/js-lib/string'
|
|
4
|
+
import type { Base64String, StringMap } from '@naturalcycles/js-lib/types'
|
|
4
5
|
import { decryptObject, decryptRandomIVBuffer } from './crypto.util.js'
|
|
5
6
|
|
|
6
7
|
let loaded = false
|
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
2
|
+
import { type Fetcher, getFetcher } from '@naturalcycles/js-lib/http'
|
|
3
|
+
import {
|
|
4
|
+
type CommonLogger,
|
|
5
|
+
commonLoggerMinLevel,
|
|
6
|
+
type CommonLogLevel,
|
|
7
|
+
} from '@naturalcycles/js-lib/log'
|
|
8
|
+
import { _omit } from '@naturalcycles/js-lib/object'
|
|
9
|
+
import { PQueue } from '@naturalcycles/js-lib/promise'
|
|
10
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types'
|
|
3
11
|
import type { InspectAnyOptions } from '../index.js'
|
|
4
12
|
import { _inspect } from '../index.js'
|
|
5
13
|
import type {
|
package/src/stream/index.ts
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createUnzip } from 'node:zlib'
|
|
2
|
+
import { fs2 } from '../../fs/fs2.js'
|
|
3
|
+
import type { ReadableTyped } from '../stream.model.js'
|
|
4
|
+
import { transformSplitOnNewline } from '../transform/transformSplit.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
Returns a Readable of [already parsed] NDJSON objects.
|
|
8
|
+
|
|
9
|
+
Replaces a list of operations:
|
|
10
|
+
- requireFileToExist(inputPath)
|
|
11
|
+
- fs.createReadStream
|
|
12
|
+
- createUnzip (only if path ends with '.gz')
|
|
13
|
+
- transformSplitOnNewline
|
|
14
|
+
- transformJsonParse
|
|
15
|
+
|
|
16
|
+
To add a Limit or Offset: just add .take() or .drop(), example:
|
|
17
|
+
|
|
18
|
+
_pipeline([
|
|
19
|
+
fs2.createReadStreamAsNDJSON().take(100),
|
|
20
|
+
transformX(),
|
|
21
|
+
])
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
|
|
25
|
+
fs2.requireFileToExist(inputPath)
|
|
26
|
+
|
|
27
|
+
let stream: ReadableTyped<ROW> = fs2
|
|
28
|
+
.createReadStream(inputPath, {
|
|
29
|
+
highWaterMark: 64 * 1024, // no observed speedup
|
|
30
|
+
})
|
|
31
|
+
.on('error', err => stream.emit('error', err))
|
|
32
|
+
|
|
33
|
+
if (inputPath.endsWith('.gz')) {
|
|
34
|
+
stream = stream.pipe(
|
|
35
|
+
createUnzip({
|
|
36
|
+
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
37
|
+
}),
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
|
|
42
|
+
// For some crazy reason .map is much faster than transformJsonParse!
|
|
43
|
+
// ~5000 vs ~4000 rps !!!
|
|
44
|
+
// .on('error', err => stream.emit('error', err))
|
|
45
|
+
// .pipe(transformJsonParse<ROW>())
|
|
46
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { createGzip } from 'node:zlib'
|
|
2
|
+
import { _isTruthy } from '@naturalcycles/js-lib'
|
|
3
|
+
import { fs2 } from '../../fs/fs2.js'
|
|
4
|
+
import type { TransformTyped } from '../stream.model.js'
|
|
5
|
+
import { transformToNDJson } from './transformToNDJson.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
Returns an array of Transforms, so that you can ...destructure them at
|
|
9
|
+
the end of the _pipeline.
|
|
10
|
+
|
|
11
|
+
Replaces a list of operations:
|
|
12
|
+
- transformToNDJson
|
|
13
|
+
- createGzip (only if path ends with '.gz')
|
|
14
|
+
- fs.createWriteStream
|
|
15
|
+
*/
|
|
16
|
+
export function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
|
|
17
|
+
fs2.ensureFile(outputPath)
|
|
18
|
+
|
|
19
|
+
return [
|
|
20
|
+
transformToNDJson(),
|
|
21
|
+
outputPath.endsWith('.gz')
|
|
22
|
+
? createGzip({
|
|
23
|
+
// chunkSize: 64 * 1024, // no observed speedup
|
|
24
|
+
})
|
|
25
|
+
: undefined,
|
|
26
|
+
fs2.createWriteStream(outputPath, {
|
|
27
|
+
// highWaterMark: 64 * 1024, // no observed speedup
|
|
28
|
+
}),
|
|
29
|
+
].filter(_isTruthy) as TransformTyped<any, any>[]
|
|
30
|
+
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error'
|
|
2
|
+
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
|
|
3
|
+
import {
|
|
4
|
+
createReadStreamAsNDJSON,
|
|
5
|
+
createWriteStreamAsNDJSON,
|
|
6
|
+
type TransformLogProgressOptions,
|
|
7
|
+
type TransformMapOptions,
|
|
8
|
+
} from '../index.js'
|
|
5
9
|
import { _pipeline, transformLimit, transformLogProgress, transformMap } from '../index.js'
|
|
6
10
|
|
|
7
11
|
export interface NDJSONMapOptions<IN = any, OUT = IN>
|
|
@@ -41,9 +45,9 @@ export async function ndjsonMap<IN = any, OUT = any>(
|
|
|
41
45
|
outputFilePath,
|
|
42
46
|
})
|
|
43
47
|
|
|
44
|
-
const readable =
|
|
45
|
-
.
|
|
46
|
-
|
|
48
|
+
const readable = createReadStreamAsNDJSON(inputFilePath).take(
|
|
49
|
+
limitInput || Number.POSITIVE_INFINITY,
|
|
50
|
+
)
|
|
47
51
|
|
|
48
52
|
await _pipeline([
|
|
49
53
|
readable,
|
|
@@ -55,6 +59,6 @@ export async function ndjsonMap<IN = any, OUT = any>(
|
|
|
55
59
|
}),
|
|
56
60
|
transformLimit({ limit: limitOutput, sourceReadable: readable }),
|
|
57
61
|
transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
|
|
58
|
-
...
|
|
62
|
+
...createWriteStreamAsNDJSON(outputFilePath),
|
|
59
63
|
])
|
|
60
64
|
}
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error'
|
|
2
|
+
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
|
|
3
|
+
import {
|
|
4
|
+
createReadStreamAsNDJSON,
|
|
5
|
+
type TransformLogProgressOptions,
|
|
6
|
+
type TransformMapOptions,
|
|
7
|
+
} from '../index.js'
|
|
5
8
|
import { _pipeline, transformLogProgress, transformMap, writableVoid } from '../index.js'
|
|
6
9
|
|
|
7
10
|
export interface NDJSONStreamForEachOptions<IN = any>
|
|
@@ -18,7 +21,7 @@ export async function ndjsonStreamForEach<T>(
|
|
|
18
21
|
opt: NDJSONStreamForEachOptions<T>,
|
|
19
22
|
): Promise<void> {
|
|
20
23
|
await _pipeline([
|
|
21
|
-
|
|
24
|
+
createReadStreamAsNDJSON(opt.inputFilePath),
|
|
22
25
|
transformMap<T, any>(mapper, {
|
|
23
26
|
errorMode: ErrorMode.THROW_AGGREGATED,
|
|
24
27
|
...opt,
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { Readable, Writable } from 'node:stream'
|
|
2
2
|
import { Transform } from 'node:stream'
|
|
3
3
|
import { pipeline } from 'node:stream/promises'
|
|
4
|
-
import
|
|
5
|
-
import {
|
|
4
|
+
import { _last } from '@naturalcycles/js-lib'
|
|
5
|
+
import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
|
|
6
|
+
import { pDefer } from '@naturalcycles/js-lib/promise'
|
|
7
|
+
import type { AnyFunction } from '@naturalcycles/js-lib/types'
|
|
6
8
|
import { writablePushToArray } from '../writable/writablePushToArray.js'
|
|
7
9
|
|
|
8
10
|
type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import type { InspectOptions } from 'node:util'
|
|
2
2
|
import { inspect } from 'node:util'
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
3
|
+
import { _mb } from '@naturalcycles/js-lib'
|
|
4
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
5
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
6
|
+
import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
|
|
7
|
+
import type { AnyObject, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
|
|
5
8
|
import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js'
|
|
6
9
|
import { SizeStack } from './sizeStack.js'
|
|
7
10
|
import type { ReadableMapper } from './stream.model.js'
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib'
|
|
2
|
-
import { _passNothingPredicate } from '@naturalcycles/js-lib'
|
|
1
|
+
import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
|
|
2
|
+
import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
|
|
3
3
|
import type { ReadableTyped } from '../index.js'
|
|
4
4
|
import { _pipeline } from '../index.js'
|
|
5
5
|
import type { TransformMapOptions } from '../transform/transformMap.js'
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ReadableOptions } from 'node:stream'
|
|
2
2
|
import { Readable } from 'node:stream'
|
|
3
|
-
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib'
|
|
4
|
-
import { _passthroughMapper } from '@naturalcycles/js-lib'
|
|
3
|
+
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
|
|
4
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types'
|
|
5
5
|
import type { ReadableTyped } from '../stream.model.js'
|
|
6
6
|
|
|
7
7
|
/**
|
package/src/stream/sizeStack.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { _hb } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
3
|
+
import { NumberStack } from '@naturalcycles/js-lib/math'
|
|
3
4
|
import { yellow } from '../colors/colors.js'
|
|
4
|
-
import { gzipBuffer } from '../
|
|
5
|
+
import { gzipBuffer } from '../zip/zip.util.js'
|
|
5
6
|
|
|
6
7
|
export class SizeStack extends NumberStack {
|
|
7
8
|
constructor(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Readable, Transform, Writable } from 'node:stream'
|
|
2
|
-
import type { Promisable } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { Promisable } from '@naturalcycles/js-lib/types'
|
|
3
3
|
|
|
4
4
|
export interface ReadableSignalOptions {
|
|
5
5
|
/** allows destroying the stream if the signal is aborted. */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
|
-
import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib/types'
|
|
3
3
|
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
4
4
|
import type { TransformMapOptions } from './transformMap.js'
|
|
5
5
|
import { transformMap } from './transformMap.js'
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Readable } from 'node:stream'
|
|
2
|
-
import type { CommonLogger } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
3
3
|
import { AbortableTransform, transformNoOp } from '../index.js'
|
|
4
4
|
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
5
5
|
import { pipelineClose } from '../stream.util.js'
|