@naturalcycles/nodejs-lib 15.1.0 → 15.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/slack-this.js +1 -1
- package/dist/{util → cache}/lruMemoCache.d.ts +1 -1
- package/dist/csv/csvReader.d.ts +1 -1
- package/dist/csv/csvReader.js +1 -1
- package/dist/csv/csvWriter.d.ts +1 -1
- package/dist/csv/csvWriter.js +1 -1
- package/dist/csv/transformToCSV.d.ts +1 -1
- package/dist/diff/tableDiff.d.ts +1 -1
- package/dist/diff/tableDiff.js +1 -1
- package/dist/{util/exec2.d.ts → exec2.d.ts} +2 -2
- package/dist/{util/exec2.js → exec2.js} +4 -3
- package/dist/fs/fs2.d.ts +0 -10
- package/dist/fs/fs2.js +1 -85
- package/dist/fs/json2env.d.ts +1 -1
- package/dist/fs/kpy.js +1 -1
- package/dist/fs/yaml2.d.ts +11 -0
- package/dist/fs/yaml2.js +29 -0
- package/dist/index.d.ts +0 -4
- package/dist/index.js +0 -4
- package/dist/jwt/jwt.service.d.ts +2 -1
- package/dist/jwt/jwt.service.js +1 -1
- package/dist/log/log.util.d.ts +1 -1
- package/dist/log/log.util.js +1 -1
- package/dist/script/runScript.d.ts +1 -2
- package/dist/script/runScript.js +6 -5
- package/dist/secret/secrets-decrypt.util.js +1 -1
- package/dist/secret/secrets-encrypt.util.js +1 -1
- package/dist/security/crypto.util.d.ts +1 -1
- package/dist/security/crypto.util.js +1 -1
- package/dist/security/hash.util.d.ts +1 -1
- package/dist/security/secret.util.d.ts +1 -1
- package/dist/security/secret.util.js +2 -1
- package/dist/slack/slack.service.d.ts +2 -1
- package/dist/slack/slack.service.js +5 -1
- package/dist/slack/slack.service.model.d.ts +2 -1
- package/dist/stream/index.d.ts +2 -0
- package/dist/stream/index.js +2 -0
- package/dist/stream/ndjson/createReadStreamAsNDJSON.d.ts +19 -0
- package/dist/stream/ndjson/createReadStreamAsNDJSON.js +38 -0
- package/dist/stream/ndjson/createWriteStreamAsNDJSON.d.ts +11 -0
- package/dist/stream/ndjson/createWriteStreamAsNDJSON.js +27 -0
- package/dist/stream/ndjson/ndjson.model.js +2 -1
- package/dist/stream/ndjson/ndjsonMap.d.ts +2 -2
- package/dist/stream/ndjson/ndjsonMap.js +4 -6
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +2 -2
- package/dist/stream/ndjson/ndjsonStreamForEach.js +3 -3
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +1 -1
- package/dist/stream/pipeline/pipeline.d.ts +1 -1
- package/dist/stream/pipeline/pipeline.js +2 -1
- package/dist/stream/progressLogger.d.ts +2 -1
- package/dist/stream/progressLogger.js +3 -1
- package/dist/stream/readable/readableForEach.d.ts +1 -1
- package/dist/stream/readable/readableForEach.js +1 -1
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +1 -1
- package/dist/stream/sizeStack.d.ts +2 -2
- package/dist/stream/sizeStack.js +3 -2
- package/dist/stream/stream.model.d.ts +1 -1
- package/dist/stream/stream.util.d.ts +1 -1
- package/dist/stream/transform/transformFilter.d.ts +1 -1
- package/dist/stream/transform/transformLimit.d.ts +1 -1
- package/dist/stream/transform/transformMap.d.ts +3 -2
- package/dist/stream/transform/transformMap.js +6 -1
- package/dist/stream/transform/transformMapSimple.d.ts +3 -2
- package/dist/stream/transform/transformMapSimple.js +1 -1
- package/dist/stream/transform/transformMapSync.d.ts +4 -2
- package/dist/stream/transform/transformMapSync.js +2 -1
- package/dist/stream/transform/transformTap.d.ts +2 -1
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +2 -1
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +2 -1
- package/dist/stream/writable/writableForEach.d.ts +1 -1
- package/dist/stream/writable/writableForEach.js +1 -1
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/string/inspect.d.ts +1 -1
- package/dist/string/inspect.js +1 -1
- package/dist/util/buildInfo.util.d.ts +2 -1
- package/dist/util/buildInfo.util.js +2 -1
- package/dist/util/env.util.d.ts +1 -2
- package/dist/util/env.util.js +0 -1
- package/dist/util/git2.d.ts +1 -1
- package/dist/util/git2.js +2 -3
- package/dist/validation/ajv/ajv.util.d.ts +1 -1
- package/dist/validation/ajv/ajvSchema.d.ts +1 -1
- package/dist/validation/ajv/ajvSchema.js +4 -1
- package/dist/validation/ajv/ajvValidationError.d.ts +2 -2
- package/dist/validation/ajv/ajvValidationError.js +1 -1
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.validation.error.d.ts +2 -2
- package/dist/validation/joi/joi.validation.error.js +1 -1
- package/dist/validation/joi/joi.validation.util.js +2 -1
- package/dist/validation/joi/string.extensions.d.ts +1 -1
- package/dist/validation/joi/string.extensions.js +1 -1
- package/dist/{util → zip}/zip.util.js +0 -1
- package/package.json +12 -5
- package/src/bin/generate-build-info.ts +1 -1
- package/src/bin/slack-this.ts +1 -1
- package/src/{util → cache}/lruMemoCache.ts +1 -1
- package/src/csv/csvReader.ts +2 -2
- package/src/csv/csvWriter.ts +2 -2
- package/src/csv/transformToCSV.ts +1 -1
- package/src/diff/tableDiff.ts +2 -2
- package/src/{util/exec2.ts → exec2.ts} +9 -8
- package/src/fs/fs2.ts +1 -101
- package/src/fs/json2env.ts +1 -1
- package/src/fs/kpy.ts +2 -2
- package/src/fs/yaml2.ts +37 -0
- package/src/index.ts +0 -4
- package/src/jwt/jwt.service.ts +3 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +10 -6
- package/src/secret/secrets-decrypt.util.ts +1 -1
- package/src/secret/secrets-encrypt.util.ts +1 -1
- package/src/security/crypto.util.ts +1 -2
- package/src/security/hash.util.ts +1 -1
- package/src/security/secret.util.ts +3 -2
- package/src/slack/slack.service.model.ts +2 -1
- package/src/slack/slack.service.ts +10 -2
- package/src/stream/index.ts +2 -0
- package/src/stream/ndjson/createReadStreamAsNDJSON.ts +46 -0
- package/src/stream/ndjson/createWriteStreamAsNDJSON.ts +30 -0
- package/src/stream/ndjson/ndjson.model.ts +2 -1
- package/src/stream/ndjson/ndjsonMap.ts +12 -8
- package/src/stream/ndjson/ndjsonStreamForEach.ts +8 -5
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +4 -2
- package/src/stream/progressLogger.ts +5 -2
- package/src/stream/readable/readableForEach.ts +2 -2
- package/src/stream/readable/readableFromArray.ts +2 -2
- package/src/stream/sizeStack.ts +4 -3
- package/src/stream/stream.model.ts +1 -1
- package/src/stream/stream.util.ts +1 -1
- package/src/stream/transform/transformFilter.ts +1 -1
- package/src/stream/transform/transformLimit.ts +1 -1
- package/src/stream/transform/transformMap.ts +12 -15
- package/src/stream/transform/transformMapSimple.ts +3 -2
- package/src/stream/transform/transformMapSync.ts +4 -7
- package/src/stream/transform/transformTap.ts +2 -1
- package/src/stream/transform/transformThrottle.ts +4 -3
- package/src/stream/transform/worker/transformMultiThreaded.ts +4 -2
- package/src/stream/writable/writableForEach.ts +2 -2
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/string/inspect.ts +2 -2
- package/src/util/buildInfo.util.ts +4 -2
- package/src/util/env.util.ts +1 -2
- package/src/util/git2.ts +3 -4
- package/src/validation/ajv/ajv.util.ts +1 -1
- package/src/validation/ajv/ajvSchema.ts +5 -7
- package/src/validation/ajv/ajvValidationError.ts +2 -2
- package/src/validation/joi/joi.shared.schemas.ts +7 -7
- package/src/validation/joi/joi.validation.error.ts +2 -2
- package/src/validation/joi/joi.validation.util.ts +2 -1
- package/src/validation/joi/string.extensions.ts +2 -2
- package/src/{util → zip}/zip.util.ts +0 -2
- package/dist/fs/index.d.ts +0 -3
- package/dist/fs/index.js +0 -3
- package/dist/jwt/index.d.ts +0 -1
- package/dist/jwt/index.js +0 -1
- package/dist/yargs/index.d.ts +0 -1
- package/dist/yargs/index.js +0 -1
- package/src/fs/index.ts +0 -3
- package/src/jwt/index.ts +0 -1
- package/src/yargs/index.ts +0 -1
- /package/dist/{util → cache}/lruMemoCache.js +0 -0
- /package/dist/{util → zip}/zip.util.d.ts +0 -0
package/dist/bin/slack-this.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { SlackService } from '../index.js';
|
|
3
3
|
import { runScript } from '../script/runScript.js';
|
|
4
|
-
import { _yargs } from '../yargs/
|
|
4
|
+
import { _yargs } from '../yargs/yargs.util.js';
|
|
5
5
|
runScript(async () => {
|
|
6
6
|
const { channel, msg, username, emoji, webhook: webhookUrl, } = _yargs().options({
|
|
7
7
|
channel: {
|
package/dist/csv/csvReader.d.ts
CHANGED
package/dist/csv/csvReader.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Inspired by: https://gist.github.com/Jezternz/c8e9fafc2c114e079829974e3764db75
|
|
2
|
-
import { _assert } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
3
3
|
// export class CSVReader {
|
|
4
4
|
// constructor (cfg: CSVReaderConfig) {
|
|
5
5
|
// this.cfg = {
|
package/dist/csv/csvWriter.d.ts
CHANGED
package/dist/csv/csvWriter.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AnyObject } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { TransformTyped } from '../stream/stream.model.js';
|
|
3
3
|
import type { CSVWriterConfig } from './csvWriter.js';
|
|
4
4
|
export interface TransformToCSVOptions extends CSVWriterConfig {
|
package/dist/diff/tableDiff.d.ts
CHANGED
package/dist/diff/tableDiff.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { AppError } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { AnyObject, NumberOfMilliseconds } from '@naturalcycles/js-lib/types';
|
|
3
3
|
/**
|
|
4
4
|
* Set of utility functions to work with Spawn / Exec.
|
|
5
5
|
*
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { execSync, spawn, spawnSync } from 'node:child_process';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { _since } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { AppError } from '@naturalcycles/js-lib/error';
|
|
4
|
+
import { _substringAfterLast } from '@naturalcycles/js-lib/string';
|
|
5
|
+
import { dimGrey, dimRed, hasColors, white } from './colors/colors.js';
|
|
5
6
|
/**
|
|
6
7
|
* Set of utility functions to work with Spawn / Exec.
|
|
7
8
|
*
|
package/dist/fs/fs2.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { RmOptions } from 'node:fs';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
-
import type { DumpOptions } from 'js-yaml';
|
|
4
|
-
import type { ReadableTyped, TransformTyped } from '../stream/stream.model.js';
|
|
5
3
|
/**
|
|
6
4
|
* fs2 conveniently groups filesystem functions together.
|
|
7
5
|
* Supposed to be almost a drop-in replacement for these things together:
|
|
@@ -23,20 +21,14 @@ declare class FS2 {
|
|
|
23
21
|
readBufferAsync(filePath: string): Promise<Buffer>;
|
|
24
22
|
readJson<T = unknown>(filePath: string): T;
|
|
25
23
|
readJsonAsync<T = unknown>(filePath: string): Promise<T>;
|
|
26
|
-
readYaml<T = unknown>(filePath: string): T;
|
|
27
|
-
readYamlAsync<T = unknown>(filePath: string): Promise<T>;
|
|
28
24
|
writeFile(filePath: string, data: string | Buffer): void;
|
|
29
25
|
writeFileAsync(filePath: string, data: string | Buffer): Promise<void>;
|
|
30
26
|
writeJson(filePath: string, data: any, opt?: JsonOptions): void;
|
|
31
27
|
writeJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
|
|
32
|
-
writeYaml(filePath: string, data: any, opt?: DumpOptions): void;
|
|
33
|
-
writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
|
|
34
28
|
appendFile(filePath: string, data: string | Buffer): void;
|
|
35
29
|
appendFileAsync(filePath: string, data: string | Buffer): Promise<void>;
|
|
36
30
|
outputJson(filePath: string, data: any, opt?: JsonOptions): void;
|
|
37
31
|
outputJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
|
|
38
|
-
outputYaml(filePath: string, data: any, opt?: DumpOptions): void;
|
|
39
|
-
outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
|
|
40
32
|
outputFile(filePath: string, data: string | Buffer): void;
|
|
41
33
|
outputFileAsync(filePath: string, data: string | Buffer): Promise<void>;
|
|
42
34
|
pathExists(filePath: string): boolean;
|
|
@@ -80,8 +72,6 @@ declare class FS2 {
|
|
|
80
72
|
readdirAsync: typeof fs.promises.readdir;
|
|
81
73
|
createWriteStream: typeof fs.createWriteStream;
|
|
82
74
|
createReadStream: typeof fs.createReadStream;
|
|
83
|
-
createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;
|
|
84
|
-
createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];
|
|
85
75
|
}
|
|
86
76
|
export declare const fs2: FS2;
|
|
87
77
|
export interface JsonOptions {
|
package/dist/fs/fs2.js
CHANGED
|
@@ -16,11 +16,7 @@ Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
|
|
|
16
16
|
import fs from 'node:fs';
|
|
17
17
|
import fsp from 'node:fs/promises';
|
|
18
18
|
import path from 'node:path';
|
|
19
|
-
import {
|
|
20
|
-
import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib';
|
|
21
|
-
import yaml from 'js-yaml';
|
|
22
|
-
import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js';
|
|
23
|
-
import { transformSplitOnNewline } from '../stream/transform/transformSplit.js';
|
|
19
|
+
import { _jsonParse } from '@naturalcycles/js-lib/string';
|
|
24
20
|
/**
|
|
25
21
|
* fs2 conveniently groups filesystem functions together.
|
|
26
22
|
* Supposed to be almost a drop-in replacement for these things together:
|
|
@@ -60,12 +56,6 @@ class FS2 {
|
|
|
60
56
|
// eslint-disable-next-line @typescript-eslint/return-await
|
|
61
57
|
return _jsonParse(str);
|
|
62
58
|
}
|
|
63
|
-
readYaml(filePath) {
|
|
64
|
-
return yaml.load(fs.readFileSync(filePath, 'utf8'));
|
|
65
|
-
}
|
|
66
|
-
async readYamlAsync(filePath) {
|
|
67
|
-
return yaml.load(await fsp.readFile(filePath, 'utf8'));
|
|
68
|
-
}
|
|
69
59
|
writeFile(filePath, data) {
|
|
70
60
|
fs.writeFileSync(filePath, data);
|
|
71
61
|
}
|
|
@@ -80,14 +70,6 @@ class FS2 {
|
|
|
80
70
|
const str = stringify(data, opt);
|
|
81
71
|
await fsp.writeFile(filePath, str);
|
|
82
72
|
}
|
|
83
|
-
writeYaml(filePath, data, opt) {
|
|
84
|
-
const str = yaml.dump(data, opt);
|
|
85
|
-
fs.writeFileSync(filePath, str);
|
|
86
|
-
}
|
|
87
|
-
async writeYamlAsync(filePath, data, opt) {
|
|
88
|
-
const str = yaml.dump(data, opt);
|
|
89
|
-
await fsp.writeFile(filePath, str);
|
|
90
|
-
}
|
|
91
73
|
appendFile(filePath, data) {
|
|
92
74
|
fs.appendFileSync(filePath, data);
|
|
93
75
|
}
|
|
@@ -102,14 +84,6 @@ class FS2 {
|
|
|
102
84
|
const str = stringify(data, opt);
|
|
103
85
|
await this.outputFileAsync(filePath, str);
|
|
104
86
|
}
|
|
105
|
-
outputYaml(filePath, data, opt) {
|
|
106
|
-
const str = yaml.dump(data, opt);
|
|
107
|
-
this.outputFile(filePath, str);
|
|
108
|
-
}
|
|
109
|
-
async outputYamlAsync(filePath, data, opt) {
|
|
110
|
-
const str = yaml.dump(data, opt);
|
|
111
|
-
await this.outputFileAsync(filePath, str);
|
|
112
|
-
}
|
|
113
87
|
outputFile(filePath, data) {
|
|
114
88
|
const dirPath = path.dirname(filePath);
|
|
115
89
|
if (!fs.existsSync(dirPath)) {
|
|
@@ -289,64 +263,6 @@ class FS2 {
|
|
|
289
263
|
readdirAsync = fsp.readdir;
|
|
290
264
|
createWriteStream = fs.createWriteStream;
|
|
291
265
|
createReadStream = fs.createReadStream;
|
|
292
|
-
/*
|
|
293
|
-
Returns a Readable of [already parsed] NDJSON objects.
|
|
294
|
-
|
|
295
|
-
Replaces a list of operations:
|
|
296
|
-
- requireFileToExist(inputPath)
|
|
297
|
-
- fs.createReadStream
|
|
298
|
-
- createUnzip (only if path ends with '.gz')
|
|
299
|
-
- transformSplitOnNewline
|
|
300
|
-
- transformJsonParse
|
|
301
|
-
|
|
302
|
-
To add a Limit or Offset: just add .take() or .drop(), example:
|
|
303
|
-
|
|
304
|
-
_pipeline([
|
|
305
|
-
fs2.createReadStreamAsNDJSON().take(100),
|
|
306
|
-
transformX(),
|
|
307
|
-
])
|
|
308
|
-
*/
|
|
309
|
-
createReadStreamAsNDJSON(inputPath) {
|
|
310
|
-
this.requireFileToExist(inputPath);
|
|
311
|
-
let stream = fs
|
|
312
|
-
.createReadStream(inputPath, {
|
|
313
|
-
highWaterMark: 64 * 1024, // no observed speedup
|
|
314
|
-
})
|
|
315
|
-
.on('error', err => stream.emit('error', err));
|
|
316
|
-
if (inputPath.endsWith('.gz')) {
|
|
317
|
-
stream = stream.pipe(createUnzip({
|
|
318
|
-
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
319
|
-
}));
|
|
320
|
-
}
|
|
321
|
-
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
|
|
322
|
-
// For some crazy reason .map is much faster than transformJsonParse!
|
|
323
|
-
// ~5000 vs ~4000 rps !!!
|
|
324
|
-
// .on('error', err => stream.emit('error', err))
|
|
325
|
-
// .pipe(transformJsonParse<ROW>())
|
|
326
|
-
}
|
|
327
|
-
/*
|
|
328
|
-
Returns an array of Transforms, so that you can ...destructure them at
|
|
329
|
-
the end of the _pipeline.
|
|
330
|
-
|
|
331
|
-
Replaces a list of operations:
|
|
332
|
-
- transformToNDJson
|
|
333
|
-
- createGzip (only if path ends with '.gz')
|
|
334
|
-
- fs.createWriteStream
|
|
335
|
-
*/
|
|
336
|
-
createWriteStreamAsNDJSON(outputPath) {
|
|
337
|
-
this.ensureFile(outputPath);
|
|
338
|
-
return [
|
|
339
|
-
transformToNDJson(),
|
|
340
|
-
outputPath.endsWith('.gz')
|
|
341
|
-
? createGzip({
|
|
342
|
-
// chunkSize: 64 * 1024, // no observed speedup
|
|
343
|
-
})
|
|
344
|
-
: undefined,
|
|
345
|
-
fs.createWriteStream(outputPath, {
|
|
346
|
-
// highWaterMark: 64 * 1024, // no observed speedup
|
|
347
|
-
}),
|
|
348
|
-
].filter(_isTruthy);
|
|
349
|
-
}
|
|
350
266
|
}
|
|
351
267
|
export const fs2 = new FS2();
|
|
352
268
|
function stringify(data, opt) {
|
package/dist/fs/json2env.d.ts
CHANGED
package/dist/fs/kpy.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import { _since, localTime } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
|
|
3
3
|
import { glob, globSync } from 'tinyglobby';
|
|
4
4
|
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js';
|
|
5
5
|
import { fs2 } from './fs2.js';
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DumpOptions } from 'js-yaml';
|
|
2
|
+
declare class Yaml2 {
|
|
3
|
+
readYaml<T = unknown>(filePath: string): T;
|
|
4
|
+
readYamlAsync<T = unknown>(filePath: string): Promise<T>;
|
|
5
|
+
writeYaml(filePath: string, data: any, opt?: DumpOptions): void;
|
|
6
|
+
writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
|
|
7
|
+
outputYaml(filePath: string, data: any, opt?: DumpOptions): void;
|
|
8
|
+
outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
export declare const yaml2: Yaml2;
|
|
11
|
+
export {};
|
package/dist/fs/yaml2.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import fsp from 'node:fs/promises';
|
|
3
|
+
import yaml from 'js-yaml';
|
|
4
|
+
import { fs2 } from './fs2.js';
|
|
5
|
+
class Yaml2 {
|
|
6
|
+
readYaml(filePath) {
|
|
7
|
+
return yaml.load(fs.readFileSync(filePath, 'utf8'));
|
|
8
|
+
}
|
|
9
|
+
async readYamlAsync(filePath) {
|
|
10
|
+
return yaml.load(await fsp.readFile(filePath, 'utf8'));
|
|
11
|
+
}
|
|
12
|
+
writeYaml(filePath, data, opt) {
|
|
13
|
+
const str = yaml.dump(data, opt);
|
|
14
|
+
fs.writeFileSync(filePath, str);
|
|
15
|
+
}
|
|
16
|
+
async writeYamlAsync(filePath, data, opt) {
|
|
17
|
+
const str = yaml.dump(data, opt);
|
|
18
|
+
await fsp.writeFile(filePath, str);
|
|
19
|
+
}
|
|
20
|
+
outputYaml(filePath, data, opt) {
|
|
21
|
+
const str = yaml.dump(data, opt);
|
|
22
|
+
fs2.outputFile(filePath, str);
|
|
23
|
+
}
|
|
24
|
+
async outputYamlAsync(filePath, data, opt) {
|
|
25
|
+
const str = yaml.dump(data, opt);
|
|
26
|
+
await fs2.outputFileAsync(filePath, str);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export const yaml2 = new Yaml2();
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export * from './buffer/buffer.util.js';
|
|
2
|
-
export * from './colors/colors.js';
|
|
3
2
|
export * from './diff/tableDiff.js';
|
|
4
3
|
export * from './infra/process.util.js';
|
|
5
4
|
export * from './log/log.util.js';
|
|
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js';
|
|
|
14
13
|
export * from './string/inspect.js';
|
|
15
14
|
export * from './util/buildInfo.util.js';
|
|
16
15
|
export * from './util/env.util.js';
|
|
17
|
-
export * from './util/exec2.js';
|
|
18
16
|
export * from './util/git2.js';
|
|
19
|
-
export * from './util/lruMemoCache.js';
|
|
20
|
-
export * from './util/zip.util.js';
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export * from './buffer/buffer.util.js';
|
|
2
|
-
export * from './colors/colors.js';
|
|
3
2
|
export * from './diff/tableDiff.js';
|
|
4
3
|
export * from './infra/process.util.js';
|
|
5
4
|
export * from './log/log.util.js';
|
|
@@ -14,7 +13,4 @@ export * from './slack/slack.service.model.js';
|
|
|
14
13
|
export * from './string/inspect.js';
|
|
15
14
|
export * from './util/buildInfo.util.js';
|
|
16
15
|
export * from './util/env.util.js';
|
|
17
|
-
export * from './util/exec2.js';
|
|
18
16
|
export * from './util/git2.js';
|
|
19
|
-
export * from './util/lruMemoCache.js';
|
|
20
|
-
export * from './util/zip.util.js';
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ErrorData } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { AnyObject, JWTString } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { AnySchema } from 'joi';
|
|
3
4
|
import type { Algorithm, JwtHeader, SignOptions, VerifyOptions } from 'jsonwebtoken';
|
|
4
5
|
import jsonwebtoken from 'jsonwebtoken';
|
package/dist/jwt/jwt.service.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { _assert, _errorDataAppend } from '@naturalcycles/js-lib';
|
|
1
|
+
import { _assert, _errorDataAppend } from '@naturalcycles/js-lib/error';
|
|
2
2
|
import jsonwebtoken from 'jsonwebtoken';
|
|
3
3
|
import { anyObjectSchema } from '../validation/joi/joi.shared.schemas.js';
|
|
4
4
|
import { validate } from '../validation/joi/joi.validation.util.js';
|
package/dist/log/log.util.d.ts
CHANGED
package/dist/log/log.util.js
CHANGED
package/dist/script/runScript.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import 'dotenv/config';
|
|
2
1
|
import os from 'node:os';
|
|
3
|
-
import { pDelay
|
|
4
|
-
import {
|
|
2
|
+
import { pDelay } from '@naturalcycles/js-lib/promise';
|
|
3
|
+
import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string';
|
|
5
4
|
import { inspectStringifyFn } from '../string/inspect.js';
|
|
6
5
|
const { DEBUG_RUN_SCRIPT } = process.env;
|
|
7
6
|
/**
|
|
@@ -23,7 +22,6 @@ const { DEBUG_RUN_SCRIPT } = process.env;
|
|
|
23
22
|
* Set env DEBUG_RUN_SCRIPT for extra debugging.
|
|
24
23
|
*/
|
|
25
24
|
export function runScript(fn, opt = {}) {
|
|
26
|
-
checkAndlogEnvironment();
|
|
27
25
|
setGlobalStringifyFunction(inspectStringifyFn);
|
|
28
26
|
const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt;
|
|
29
27
|
if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
|
|
@@ -42,6 +40,8 @@ export function runScript(fn, opt = {}) {
|
|
|
42
40
|
const timeout = setTimeout(() => { }, 10000000);
|
|
43
41
|
void (async () => {
|
|
44
42
|
try {
|
|
43
|
+
await import('dotenv/config');
|
|
44
|
+
await checkAndlogEnvironment();
|
|
45
45
|
await fn();
|
|
46
46
|
await pDelay(); // to ensure all async operations are completed
|
|
47
47
|
if (DEBUG_RUN_SCRIPT)
|
|
@@ -62,7 +62,8 @@ export function runScript(fn, opt = {}) {
|
|
|
62
62
|
}
|
|
63
63
|
})();
|
|
64
64
|
}
|
|
65
|
-
function checkAndlogEnvironment() {
|
|
65
|
+
async function checkAndlogEnvironment() {
|
|
66
|
+
const { dimGrey } = await import('../colors/colors.js');
|
|
66
67
|
const { platform, arch, versions: { node }, env: { CPU_LIMIT, NODE_OPTIONS, TZ }, } = process;
|
|
67
68
|
const cpuLimit = Number(CPU_LIMIT) || undefined;
|
|
68
69
|
const availableParallelism = os.availableParallelism?.();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { _assert } from '@naturalcycles/js-lib';
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
4
4
|
import { globSync } from 'tinyglobby';
|
|
5
5
|
import { dimGrey, yellow } from '../colors/colors.js';
|
|
6
6
|
import { fs2 } from '../fs/fs2.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { _assert } from '@naturalcycles/js-lib';
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
4
4
|
import { globSync } from 'tinyglobby';
|
|
5
5
|
import { dimGrey, yellow } from '../colors/colors.js';
|
|
6
6
|
import { fs2 } from '../fs/fs2.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { BinaryToTextEncoding } from 'node:crypto';
|
|
2
|
-
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types';
|
|
3
3
|
export declare function md5(s: string | Buffer, outputEncoding?: BinaryToTextEncoding): string;
|
|
4
4
|
export declare function md5AsBuffer(s: string | Buffer): Buffer;
|
|
5
5
|
export declare function sha256(s: string | Buffer, outputEncoding?: BinaryToTextEncoding): string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Base64String, StringMap } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { Base64String, StringMap } from '@naturalcycles/js-lib/types';
|
|
2
2
|
/**
|
|
3
3
|
* Loads plaintext secrets from process.env, removes them, stores locally.
|
|
4
4
|
* Make sure to call this function early on server startup, so secrets are removed from process.env
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { _assert
|
|
2
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
3
|
+
import { _jsonParseIfPossible } from '@naturalcycles/js-lib/string';
|
|
3
4
|
import { decryptObject, decryptRandomIVBuffer } from './crypto.util.js';
|
|
4
5
|
let loaded = false;
|
|
5
6
|
const secretMap = {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type CommonLogger, type CommonLogLevel } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { SlackAttachmentField, SlackMessage, SlackServiceCfg } from './slack.service.model.js';
|
|
3
4
|
/**
|
|
4
5
|
* Has 2 main methods:
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
2
|
+
import { getFetcher } from '@naturalcycles/js-lib/http';
|
|
3
|
+
import { commonLoggerMinLevel, } from '@naturalcycles/js-lib/log';
|
|
4
|
+
import { _omit } from '@naturalcycles/js-lib/object';
|
|
5
|
+
import { PQueue } from '@naturalcycles/js-lib/promise';
|
|
2
6
|
import { _inspect } from '../index.js';
|
|
3
7
|
const GAE = !!process.env['GAE_INSTANCE'];
|
|
4
8
|
const DEFAULTS = {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { InspectAnyOptions } from '../index.js';
|
|
3
4
|
/**
|
|
4
5
|
* Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
|
package/dist/stream/index.d.ts
CHANGED
package/dist/stream/index.js
CHANGED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ReadableTyped } from '../stream.model.js';
|
|
2
|
+
/**
|
|
3
|
+
Returns a Readable of [already parsed] NDJSON objects.
|
|
4
|
+
|
|
5
|
+
Replaces a list of operations:
|
|
6
|
+
- requireFileToExist(inputPath)
|
|
7
|
+
- fs.createReadStream
|
|
8
|
+
- createUnzip (only if path ends with '.gz')
|
|
9
|
+
- transformSplitOnNewline
|
|
10
|
+
- transformJsonParse
|
|
11
|
+
|
|
12
|
+
To add a Limit or Offset: just add .take() or .drop(), example:
|
|
13
|
+
|
|
14
|
+
_pipeline([
|
|
15
|
+
fs2.createReadStreamAsNDJSON().take(100),
|
|
16
|
+
transformX(),
|
|
17
|
+
])
|
|
18
|
+
*/
|
|
19
|
+
export declare function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createUnzip } from 'node:zlib';
|
|
2
|
+
import { fs2 } from '../../fs/fs2.js';
|
|
3
|
+
import { transformSplitOnNewline } from '../transform/transformSplit.js';
|
|
4
|
+
/**
|
|
5
|
+
Returns a Readable of [already parsed] NDJSON objects.
|
|
6
|
+
|
|
7
|
+
Replaces a list of operations:
|
|
8
|
+
- requireFileToExist(inputPath)
|
|
9
|
+
- fs.createReadStream
|
|
10
|
+
- createUnzip (only if path ends with '.gz')
|
|
11
|
+
- transformSplitOnNewline
|
|
12
|
+
- transformJsonParse
|
|
13
|
+
|
|
14
|
+
To add a Limit or Offset: just add .take() or .drop(), example:
|
|
15
|
+
|
|
16
|
+
_pipeline([
|
|
17
|
+
fs2.createReadStreamAsNDJSON().take(100),
|
|
18
|
+
transformX(),
|
|
19
|
+
])
|
|
20
|
+
*/
|
|
21
|
+
export function createReadStreamAsNDJSON(inputPath) {
|
|
22
|
+
fs2.requireFileToExist(inputPath);
|
|
23
|
+
let stream = fs2
|
|
24
|
+
.createReadStream(inputPath, {
|
|
25
|
+
highWaterMark: 64 * 1024, // no observed speedup
|
|
26
|
+
})
|
|
27
|
+
.on('error', err => stream.emit('error', err));
|
|
28
|
+
if (inputPath.endsWith('.gz')) {
|
|
29
|
+
stream = stream.pipe(createUnzip({
|
|
30
|
+
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
31
|
+
}));
|
|
32
|
+
}
|
|
33
|
+
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
|
|
34
|
+
// For some crazy reason .map is much faster than transformJsonParse!
|
|
35
|
+
// ~5000 vs ~4000 rps !!!
|
|
36
|
+
// .on('error', err => stream.emit('error', err))
|
|
37
|
+
// .pipe(transformJsonParse<ROW>())
|
|
38
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { TransformTyped } from '../stream.model.js';
|
|
2
|
+
/**
|
|
3
|
+
Returns an array of Transforms, so that you can ...destructure them at
|
|
4
|
+
the end of the _pipeline.
|
|
5
|
+
|
|
6
|
+
Replaces a list of operations:
|
|
7
|
+
- transformToNDJson
|
|
8
|
+
- createGzip (only if path ends with '.gz')
|
|
9
|
+
- fs.createWriteStream
|
|
10
|
+
*/
|
|
11
|
+
export declare function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createGzip } from 'node:zlib';
|
|
2
|
+
import { _isTruthy } from '@naturalcycles/js-lib';
|
|
3
|
+
import { fs2 } from '../../fs/fs2.js';
|
|
4
|
+
import { transformToNDJson } from './transformToNDJson.js';
|
|
5
|
+
/**
|
|
6
|
+
Returns an array of Transforms, so that you can ...destructure them at
|
|
7
|
+
the end of the _pipeline.
|
|
8
|
+
|
|
9
|
+
Replaces a list of operations:
|
|
10
|
+
- transformToNDJson
|
|
11
|
+
- createGzip (only if path ends with '.gz')
|
|
12
|
+
- fs.createWriteStream
|
|
13
|
+
*/
|
|
14
|
+
export function createWriteStreamAsNDJSON(outputPath) {
|
|
15
|
+
fs2.ensureFile(outputPath);
|
|
16
|
+
return [
|
|
17
|
+
transformToNDJson(),
|
|
18
|
+
outputPath.endsWith('.gz')
|
|
19
|
+
? createGzip({
|
|
20
|
+
// chunkSize: 64 * 1024, // no observed speedup
|
|
21
|
+
})
|
|
22
|
+
: undefined,
|
|
23
|
+
fs2.createWriteStream(outputPath, {
|
|
24
|
+
// highWaterMark: 64 * 1024, // no observed speedup
|
|
25
|
+
}),
|
|
26
|
+
].filter(_isTruthy);
|
|
27
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { _hb
|
|
1
|
+
import { _hb } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _ms } from '@naturalcycles/js-lib/datetime';
|
|
2
3
|
import { boldWhite, dimWhite } from '../../colors/colors.js';
|
|
3
4
|
export class NDJsonStats {
|
|
4
5
|
static create(o = {}) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import type
|
|
1
|
+
import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types';
|
|
2
|
+
import { type TransformLogProgressOptions, type TransformMapOptions } from '../index.js';
|
|
3
3
|
export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT>, TransformLogProgressOptions<IN> {
|
|
4
4
|
inputFilePath: string;
|
|
5
5
|
outputFilePath: string;
|