@naturalcycles/nodejs-lib 13.45.3 → 14.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/generate-build-info.js +15 -18
- package/dist/bin/json2env.js +6 -9
- package/dist/bin/kpy.js +6 -9
- package/dist/bin/secrets-decrypt.d.ts +1 -1
- package/dist/bin/secrets-decrypt.js +9 -12
- package/dist/bin/secrets-encrypt.d.ts +1 -1
- package/dist/bin/secrets-encrypt.js +9 -12
- package/dist/bin/secrets-gen-key.js +8 -11
- package/dist/bin/slack-this.js +6 -9
- package/dist/buffer/buffer.util.js +1 -4
- package/dist/colors/colors.d.ts +26 -27
- package/dist/colors/colors.js +29 -33
- package/dist/csv/csvReader.js +4 -8
- package/dist/csv/csvWriter.js +7 -12
- package/dist/csv/transformToCSV.d.ts +2 -2
- package/dist/csv/transformToCSV.js +5 -8
- package/dist/diff/tableDiff.js +4 -7
- package/dist/fs/fs2.d.ts +1 -1
- package/dist/fs/fs2.js +88 -94
- package/dist/fs/json2env.js +18 -28
- package/dist/fs/kpy.js +28 -33
- package/dist/index.d.ts +71 -71
- package/dist/index.js +73 -78
- package/dist/infra/process.util.js +23 -28
- package/dist/jwt/jwt.service.js +17 -21
- package/dist/log/log.util.js +4 -7
- package/dist/script/runScript.js +11 -15
- package/dist/secret/secrets-decrypt.util.js +18 -22
- package/dist/secret/secrets-encrypt.util.js +17 -21
- package/dist/security/crypto.util.js +24 -34
- package/dist/security/hash.util.js +19 -37
- package/dist/security/id.util.js +11 -18
- package/dist/security/nanoid.js +15 -20
- package/dist/security/secret.util.js +20 -30
- package/dist/slack/slack.service.d.ts +1 -1
- package/dist/slack/slack.service.js +13 -16
- package/dist/slack/slack.service.model.d.ts +1 -1
- package/dist/slack/slack.service.model.js +1 -2
- package/dist/stream/ndjson/ndjson.model.js +10 -16
- package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonMap.js +11 -14
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformJsonParse.js +4 -9
- package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +5 -8
- package/dist/stream/pipeline/pipeline.js +13 -17
- package/dist/stream/progressLogger.d.ts +1 -1
- package/dist/stream/progressLogger.js +32 -28
- package/dist/stream/readable/readableCreate.d.ts +1 -1
- package/dist/stream/readable/readableCreate.js +5 -9
- package/dist/stream/readable/readableForEach.d.ts +2 -2
- package/dist/stream/readable/readableForEach.js +7 -11
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +4 -7
- package/dist/stream/readable/readableToArray.d.ts +1 -1
- package/dist/stream/readable/readableToArray.js +1 -4
- package/dist/stream/sizeStack.js +9 -12
- package/dist/stream/stream.model.js +1 -2
- package/dist/stream/stream.util.js +1 -4
- package/dist/stream/transform/transformChunk.d.ts +1 -1
- package/dist/stream/transform/transformChunk.js +3 -6
- package/dist/stream/transform/transformFilter.d.ts +2 -2
- package/dist/stream/transform/transformFilter.js +6 -10
- package/dist/stream/transform/transformLimit.d.ts +2 -2
- package/dist/stream/transform/transformLimit.js +6 -11
- package/dist/stream/transform/transformLogProgress.d.ts +2 -2
- package/dist/stream/transform/transformLogProgress.js +5 -8
- package/dist/stream/transform/transformMap.d.ts +1 -1
- package/dist/stream/transform/transformMap.js +18 -22
- package/dist/stream/transform/transformMapSimple.d.ts +1 -1
- package/dist/stream/transform/transformMapSimple.js +6 -9
- package/dist/stream/transform/transformMapSync.d.ts +3 -3
- package/dist/stream/transform/transformMapSync.js +14 -19
- package/dist/stream/transform/transformNoOp.d.ts +1 -1
- package/dist/stream/transform/transformNoOp.js +3 -6
- package/dist/stream/transform/transformOffset.d.ts +1 -1
- package/dist/stream/transform/transformOffset.js +5 -8
- package/dist/stream/transform/transformSplit.d.ts +1 -1
- package/dist/stream/transform/transformSplit.js +5 -9
- package/dist/stream/transform/transformTap.d.ts +1 -1
- package/dist/stream/transform/transformTap.js +3 -6
- package/dist/stream/transform/transformTee.d.ts +1 -1
- package/dist/stream/transform/transformTee.js +7 -10
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +9 -12
- package/dist/stream/transform/transformToArray.d.ts +1 -1
- package/dist/stream/transform/transformToArray.js +3 -6
- package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
- package/dist/stream/transform/worker/baseWorkerClass.js +2 -5
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
- package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
- package/dist/stream/transform/worker/workerClassProxy.js +6 -4
- package/dist/stream/writable/writableForEach.d.ts +2 -2
- package/dist/stream/writable/writableForEach.js +6 -10
- package/dist/stream/writable/writableFork.d.ts +1 -1
- package/dist/stream/writable/writableFork.js +6 -9
- package/dist/stream/writable/writableLimit.d.ts +1 -1
- package/dist/stream/writable/writableLimit.js +3 -6
- package/dist/stream/writable/writablePushToArray.d.ts +1 -1
- package/dist/stream/writable/writablePushToArray.js +3 -6
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/stream/writable/writableVoid.js +3 -6
- package/dist/string/inspect.js +8 -13
- package/dist/util/buildInfo.util.js +12 -15
- package/dist/util/env.util.js +5 -9
- package/dist/util/exec2.js +17 -21
- package/dist/util/git2.js +17 -20
- package/dist/util/lruMemoCache.js +4 -7
- package/dist/util/zip.util.js +14 -25
- package/dist/validation/ajv/ajv.util.d.ts +3 -3
- package/dist/validation/ajv/ajv.util.js +6 -10
- package/dist/validation/ajv/ajvSchema.d.ts +2 -2
- package/dist/validation/ajv/ajvSchema.js +17 -18
- package/dist/validation/ajv/ajvValidationError.js +2 -6
- package/dist/validation/ajv/getAjv.d.ts +1 -1
- package/dist/validation/ajv/getAjv.js +9 -9
- package/dist/validation/joi/joi.extensions.d.ts +2 -2
- package/dist/validation/joi/joi.extensions.js +6 -10
- package/dist/validation/joi/joi.model.js +1 -2
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.shared.schemas.js +61 -74
- package/dist/validation/joi/joi.validation.error.js +2 -6
- package/dist/validation/joi/joi.validation.util.d.ts +1 -1
- package/dist/validation/joi/joi.validation.util.js +10 -17
- package/dist/validation/joi/number.extensions.js +1 -4
- package/dist/validation/joi/string.extensions.js +4 -7
- package/dist/yargs.util.js +5 -8
- package/package.json +5 -4
- package/src/bin/generate-build-info.ts +3 -3
- package/src/bin/json2env.ts +2 -2
- package/src/bin/kpy.ts +2 -2
- package/src/bin/secrets-decrypt.ts +7 -8
- package/src/bin/secrets-encrypt.ts +7 -8
- package/src/bin/secrets-gen-key.ts +2 -2
- package/src/bin/slack-this.ts +2 -2
- package/src/csv/transformToCSV.ts +3 -3
- package/src/fs/fs2.ts +4 -4
- package/src/fs/json2env.ts +2 -2
- package/src/fs/kpy.ts +2 -2
- package/src/index.ts +71 -71
- package/src/jwt/jwt.service.ts +2 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +4 -3
- package/src/secret/secrets-decrypt.util.ts +3 -3
- package/src/secret/secrets-encrypt.util.ts +3 -3
- package/src/security/crypto.util.ts +1 -1
- package/src/security/id.util.ts +1 -1
- package/src/security/secret.util.ts +2 -2
- package/src/slack/slack.service.model.ts +1 -1
- package/src/slack/slack.service.ts +3 -3
- package/src/stream/ndjson/ndjson.model.ts +1 -1
- package/src/stream/ndjson/ndjsonMap.ts +2 -2
- package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +1 -1
- package/src/stream/progressLogger.ts +3 -3
- package/src/stream/readable/readableCreate.ts +1 -1
- package/src/stream/readable/readableForEach.ts +4 -4
- package/src/stream/readable/readableFromArray.ts +1 -1
- package/src/stream/readable/readableToArray.ts +1 -1
- package/src/stream/sizeStack.ts +2 -2
- package/src/stream/transform/transformChunk.ts +1 -1
- package/src/stream/transform/transformFilter.ts +3 -3
- package/src/stream/transform/transformLimit.ts +3 -3
- package/src/stream/transform/transformLogProgress.ts +3 -3
- package/src/stream/transform/transformMap.ts +5 -5
- package/src/stream/transform/transformMapSimple.ts +1 -1
- package/src/stream/transform/transformMapSync.ts +5 -5
- package/src/stream/transform/transformNoOp.ts +1 -1
- package/src/stream/transform/transformOffset.ts +2 -2
- package/src/stream/transform/transformSplit.ts +1 -1
- package/src/stream/transform/transformTap.ts +1 -1
- package/src/stream/transform/transformTee.ts +3 -3
- package/src/stream/transform/transformThrottle.ts +1 -1
- package/src/stream/transform/transformToArray.ts +1 -1
- package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
- package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
- package/src/stream/transform/worker/workerClassProxy.js +6 -4
- package/src/stream/writable/writableForEach.ts +3 -3
- package/src/stream/writable/writableFork.ts +3 -3
- package/src/stream/writable/writableLimit.ts +1 -1
- package/src/stream/writable/writablePushToArray.ts +1 -1
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/util/buildInfo.util.ts +2 -2
- package/src/util/exec2.ts +1 -1
- package/src/util/git2.ts +2 -2
- package/src/validation/ajv/ajv.util.ts +4 -4
- package/src/validation/ajv/ajvSchema.ts +4 -5
- package/src/validation/ajv/getAjv.ts +7 -3
- package/src/validation/joi/joi.extensions.ts +4 -4
- package/src/validation/joi/joi.shared.schemas.ts +3 -3
- package/src/validation/joi/joi.validation.util.ts +2 -2
- package/src/yargs.util.ts +1 -0
package/dist/fs/fs2.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/*
|
|
3
2
|
|
|
4
3
|
Why?
|
|
@@ -14,18 +13,15 @@ That's why function names are slightly renamed, to avoid conflict.
|
|
|
14
13
|
Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
|
|
15
14
|
|
|
16
15
|
*/
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const transformToNDJson_1 = require("../stream/ndjson/transformToNDJson");
|
|
27
|
-
const transformSplit_1 = require("../stream/transform/transformSplit");
|
|
28
|
-
const env_util_1 = require("../util/env.util");
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import fsp from 'node:fs/promises';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { createGzip, createUnzip } from 'node:zlib';
|
|
20
|
+
import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib';
|
|
21
|
+
import yaml from 'js-yaml';
|
|
22
|
+
import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js';
|
|
23
|
+
import { transformSplitOnNewline } from '../stream/transform/transformSplit.js';
|
|
24
|
+
import { requireFileToExist } from '../util/env.util.js';
|
|
29
25
|
/**
|
|
30
26
|
* fs2 conveniently groups filesystem functions together.
|
|
31
27
|
* Supposed to be almost a drop-in replacement for these things together:
|
|
@@ -35,86 +31,69 @@ const env_util_1 = require("../util/env.util");
|
|
|
35
31
|
* 3. fs-extra
|
|
36
32
|
*/
|
|
37
33
|
class FS2 {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
// - sync function has postfix in the name, e.g readText
|
|
42
|
-
// Re-export the whole fs/fsp, for the edge cases where they are needed
|
|
43
|
-
this.fs = node_fs_1.default;
|
|
44
|
-
this.fsp = promises_1.default;
|
|
45
|
-
// Re-export existing fs/fsp functions
|
|
46
|
-
// rm/rmAsync are replaced with removePath/removePathAsync
|
|
47
|
-
this.lstat = node_fs_1.default.lstatSync;
|
|
48
|
-
this.lstatAsync = promises_1.default.lstat;
|
|
49
|
-
this.stat = node_fs_1.default.statSync;
|
|
50
|
-
this.statAsync = promises_1.default.stat;
|
|
51
|
-
this.mkdir = node_fs_1.default.mkdirSync;
|
|
52
|
-
this.mkdirAsync = promises_1.default.mkdir;
|
|
53
|
-
this.readdir = node_fs_1.default.readdirSync;
|
|
54
|
-
this.readdirAsync = promises_1.default.readdir;
|
|
55
|
-
this.createWriteStream = node_fs_1.default.createWriteStream;
|
|
56
|
-
this.createReadStream = node_fs_1.default.createReadStream;
|
|
57
|
-
}
|
|
34
|
+
// Naming convention is:
|
|
35
|
+
// - async function has Async in the name, e.g readTextAsync
|
|
36
|
+
// - sync function has postfix in the name, e.g readText
|
|
58
37
|
/**
|
|
59
38
|
* Convenience wrapper that defaults to utf-8 string output.
|
|
60
39
|
*/
|
|
61
40
|
readText(filePath) {
|
|
62
|
-
return
|
|
41
|
+
return fs.readFileSync(filePath, 'utf8');
|
|
63
42
|
}
|
|
64
43
|
/**
|
|
65
44
|
* Convenience wrapper that defaults to utf-8 string output.
|
|
66
45
|
*/
|
|
67
46
|
async readTextAsync(filePath) {
|
|
68
|
-
return await
|
|
47
|
+
return await fsp.readFile(filePath, 'utf8');
|
|
69
48
|
}
|
|
70
49
|
readBuffer(filePath) {
|
|
71
|
-
return
|
|
50
|
+
return fs.readFileSync(filePath);
|
|
72
51
|
}
|
|
73
52
|
async readBufferAsync(filePath) {
|
|
74
|
-
return await
|
|
53
|
+
return await fsp.readFile(filePath);
|
|
75
54
|
}
|
|
76
55
|
readJson(filePath) {
|
|
77
|
-
const str =
|
|
78
|
-
return
|
|
56
|
+
const str = fs.readFileSync(filePath, 'utf8');
|
|
57
|
+
return _jsonParse(str);
|
|
79
58
|
}
|
|
80
59
|
async readJsonAsync(filePath) {
|
|
81
|
-
const str = await
|
|
60
|
+
const str = await fsp.readFile(filePath, 'utf8');
|
|
82
61
|
// eslint-disable-next-line @typescript-eslint/return-await
|
|
83
|
-
return
|
|
62
|
+
return _jsonParse(str);
|
|
84
63
|
}
|
|
85
64
|
readYaml(filePath) {
|
|
86
|
-
return
|
|
65
|
+
return yaml.load(fs.readFileSync(filePath, 'utf8'));
|
|
87
66
|
}
|
|
88
67
|
async readYamlAsync(filePath) {
|
|
89
|
-
return
|
|
68
|
+
return yaml.load(await fsp.readFile(filePath, 'utf8'));
|
|
90
69
|
}
|
|
91
70
|
writeFile(filePath, data) {
|
|
92
|
-
|
|
71
|
+
fs.writeFileSync(filePath, data);
|
|
93
72
|
}
|
|
94
73
|
async writeFileAsync(filePath, data) {
|
|
95
|
-
await
|
|
74
|
+
await fsp.writeFile(filePath, data);
|
|
96
75
|
}
|
|
97
76
|
writeJson(filePath, data, opt) {
|
|
98
77
|
const str = stringify(data, opt);
|
|
99
|
-
|
|
78
|
+
fs.writeFileSync(filePath, str);
|
|
100
79
|
}
|
|
101
80
|
async writeJsonAsync(filePath, data, opt) {
|
|
102
81
|
const str = stringify(data, opt);
|
|
103
|
-
await
|
|
82
|
+
await fsp.writeFile(filePath, str);
|
|
104
83
|
}
|
|
105
84
|
writeYaml(filePath, data, opt) {
|
|
106
|
-
const str =
|
|
107
|
-
|
|
85
|
+
const str = yaml.dump(data, opt);
|
|
86
|
+
fs.writeFileSync(filePath, str);
|
|
108
87
|
}
|
|
109
88
|
async writeYamlAsync(filePath, data, opt) {
|
|
110
|
-
const str =
|
|
111
|
-
await
|
|
89
|
+
const str = yaml.dump(data, opt);
|
|
90
|
+
await fsp.writeFile(filePath, str);
|
|
112
91
|
}
|
|
113
92
|
appendFile(filePath, data) {
|
|
114
|
-
|
|
93
|
+
fs.appendFileSync(filePath, data);
|
|
115
94
|
}
|
|
116
95
|
async appendFileAsync(filePath, data) {
|
|
117
|
-
await
|
|
96
|
+
await fsp.appendFile(filePath, data);
|
|
118
97
|
}
|
|
119
98
|
outputJson(filePath, data, opt) {
|
|
120
99
|
const str = stringify(data, opt);
|
|
@@ -125,33 +104,33 @@ class FS2 {
|
|
|
125
104
|
await this.outputFileAsync(filePath, str);
|
|
126
105
|
}
|
|
127
106
|
outputYaml(filePath, data, opt) {
|
|
128
|
-
const str =
|
|
107
|
+
const str = yaml.dump(data, opt);
|
|
129
108
|
this.outputFile(filePath, str);
|
|
130
109
|
}
|
|
131
110
|
async outputYamlAsync(filePath, data, opt) {
|
|
132
|
-
const str =
|
|
111
|
+
const str = yaml.dump(data, opt);
|
|
133
112
|
await this.outputFileAsync(filePath, str);
|
|
134
113
|
}
|
|
135
114
|
outputFile(filePath, data) {
|
|
136
|
-
const dirPath =
|
|
137
|
-
if (!
|
|
115
|
+
const dirPath = path.dirname(filePath);
|
|
116
|
+
if (!fs.existsSync(dirPath)) {
|
|
138
117
|
this.ensureDir(dirPath);
|
|
139
118
|
}
|
|
140
|
-
|
|
119
|
+
fs.writeFileSync(filePath, data);
|
|
141
120
|
}
|
|
142
121
|
async outputFileAsync(filePath, data) {
|
|
143
|
-
const dirPath =
|
|
122
|
+
const dirPath = path.dirname(filePath);
|
|
144
123
|
if (!(await this.pathExistsAsync(dirPath))) {
|
|
145
124
|
await this.ensureDirAsync(dirPath);
|
|
146
125
|
}
|
|
147
|
-
await
|
|
126
|
+
await fsp.writeFile(filePath, data);
|
|
148
127
|
}
|
|
149
128
|
pathExists(filePath) {
|
|
150
|
-
return
|
|
129
|
+
return fs.existsSync(filePath);
|
|
151
130
|
}
|
|
152
131
|
async pathExistsAsync(filePath) {
|
|
153
132
|
try {
|
|
154
|
-
await
|
|
133
|
+
await fsp.access(filePath);
|
|
155
134
|
return true;
|
|
156
135
|
}
|
|
157
136
|
catch {
|
|
@@ -159,13 +138,13 @@ class FS2 {
|
|
|
159
138
|
}
|
|
160
139
|
}
|
|
161
140
|
ensureDir(dirPath) {
|
|
162
|
-
|
|
141
|
+
fs.mkdirSync(dirPath, {
|
|
163
142
|
mode: 0o777,
|
|
164
143
|
recursive: true,
|
|
165
144
|
});
|
|
166
145
|
}
|
|
167
146
|
async ensureDirAsync(dirPath) {
|
|
168
|
-
await
|
|
147
|
+
await fsp.mkdir(dirPath, {
|
|
169
148
|
mode: 0o777,
|
|
170
149
|
recursive: true,
|
|
171
150
|
});
|
|
@@ -173,17 +152,17 @@ class FS2 {
|
|
|
173
152
|
ensureFile(filePath) {
|
|
174
153
|
let stats;
|
|
175
154
|
try {
|
|
176
|
-
stats =
|
|
155
|
+
stats = fs.statSync(filePath);
|
|
177
156
|
}
|
|
178
157
|
catch { }
|
|
179
158
|
if (stats?.isFile())
|
|
180
159
|
return;
|
|
181
|
-
const dir =
|
|
160
|
+
const dir = path.dirname(filePath);
|
|
182
161
|
try {
|
|
183
|
-
if (!
|
|
162
|
+
if (!fs.statSync(dir).isDirectory()) {
|
|
184
163
|
// parent is not a directory
|
|
185
164
|
// This is just to cause an internal ENOTDIR error to be thrown
|
|
186
|
-
|
|
165
|
+
fs.readdirSync(dir);
|
|
187
166
|
}
|
|
188
167
|
}
|
|
189
168
|
catch (err) {
|
|
@@ -194,22 +173,22 @@ class FS2 {
|
|
|
194
173
|
}
|
|
195
174
|
throw err;
|
|
196
175
|
}
|
|
197
|
-
|
|
176
|
+
fs.writeFileSync(filePath, '');
|
|
198
177
|
}
|
|
199
178
|
async ensureFileAsync(filePath) {
|
|
200
179
|
let stats;
|
|
201
180
|
try {
|
|
202
|
-
stats = await
|
|
181
|
+
stats = await fsp.stat(filePath);
|
|
203
182
|
}
|
|
204
183
|
catch { }
|
|
205
184
|
if (stats?.isFile())
|
|
206
185
|
return;
|
|
207
|
-
const dir =
|
|
186
|
+
const dir = path.dirname(filePath);
|
|
208
187
|
try {
|
|
209
|
-
if (!(await
|
|
188
|
+
if (!(await fsp.stat(dir)).isDirectory()) {
|
|
210
189
|
// parent is not a directory
|
|
211
190
|
// This is just to cause an internal ENOTDIR error to be thrown
|
|
212
|
-
await
|
|
191
|
+
await fsp.readdir(dir);
|
|
213
192
|
}
|
|
214
193
|
}
|
|
215
194
|
catch (err) {
|
|
@@ -218,40 +197,40 @@ class FS2 {
|
|
|
218
197
|
return await this.ensureDirAsync(dir);
|
|
219
198
|
throw err;
|
|
220
199
|
}
|
|
221
|
-
await
|
|
200
|
+
await fsp.writeFile(filePath, '');
|
|
222
201
|
}
|
|
223
202
|
removePath(fileOrDirPath, opt) {
|
|
224
|
-
|
|
203
|
+
fs.rmSync(fileOrDirPath, { recursive: true, force: true, ...opt });
|
|
225
204
|
}
|
|
226
205
|
async removePathAsync(fileOrDirPath, opt) {
|
|
227
|
-
await
|
|
206
|
+
await fsp.rm(fileOrDirPath, { recursive: true, force: true, ...opt });
|
|
228
207
|
}
|
|
229
208
|
emptyDir(dirPath) {
|
|
230
209
|
let items;
|
|
231
210
|
try {
|
|
232
|
-
items =
|
|
211
|
+
items = fs.readdirSync(dirPath);
|
|
233
212
|
}
|
|
234
213
|
catch {
|
|
235
214
|
this.ensureDir(dirPath);
|
|
236
215
|
return;
|
|
237
216
|
}
|
|
238
|
-
items.forEach(item => this.removePath(
|
|
217
|
+
items.forEach(item => this.removePath(path.join(dirPath, item)));
|
|
239
218
|
}
|
|
240
219
|
async emptyDirAsync(dirPath) {
|
|
241
220
|
let items;
|
|
242
221
|
try {
|
|
243
|
-
items = await
|
|
222
|
+
items = await fsp.readdir(dirPath);
|
|
244
223
|
}
|
|
245
224
|
catch {
|
|
246
225
|
return await this.ensureDirAsync(dirPath);
|
|
247
226
|
}
|
|
248
|
-
await Promise.all(items.map(item => this.removePathAsync(
|
|
227
|
+
await Promise.all(items.map(item => this.removePathAsync(path.join(dirPath, item))));
|
|
249
228
|
}
|
|
250
229
|
/**
|
|
251
230
|
* Cautious, underlying Node function is currently Experimental.
|
|
252
231
|
*/
|
|
253
232
|
copyPath(src, dest, opt) {
|
|
254
|
-
|
|
233
|
+
fs.cpSync(src, dest, {
|
|
255
234
|
recursive: true,
|
|
256
235
|
...opt,
|
|
257
236
|
});
|
|
@@ -260,16 +239,16 @@ class FS2 {
|
|
|
260
239
|
* Cautious, underlying Node function is currently Experimental.
|
|
261
240
|
*/
|
|
262
241
|
async copyPathAsync(src, dest, opt) {
|
|
263
|
-
await
|
|
242
|
+
await fsp.cp(src, dest, {
|
|
264
243
|
recursive: true,
|
|
265
244
|
...opt,
|
|
266
245
|
});
|
|
267
246
|
}
|
|
268
247
|
renamePath(src, dest) {
|
|
269
|
-
|
|
248
|
+
fs.renameSync(src, dest);
|
|
270
249
|
}
|
|
271
250
|
async renamePathAsync(src, dest) {
|
|
272
|
-
await
|
|
251
|
+
await fsp.rename(src, dest);
|
|
273
252
|
}
|
|
274
253
|
movePath(src, dest, opt) {
|
|
275
254
|
this.copyPath(src, dest, opt);
|
|
@@ -285,12 +264,27 @@ class FS2 {
|
|
|
285
264
|
* Doesn't throw, returns false instead.
|
|
286
265
|
*/
|
|
287
266
|
isDirectory(filePath) {
|
|
288
|
-
return (
|
|
267
|
+
return (fs2
|
|
289
268
|
.stat(filePath, {
|
|
290
269
|
throwIfNoEntry: false,
|
|
291
270
|
})
|
|
292
271
|
?.isDirectory() || false);
|
|
293
272
|
}
|
|
273
|
+
// Re-export the whole fs/fsp, for the edge cases where they are needed
|
|
274
|
+
fs = fs;
|
|
275
|
+
fsp = fsp;
|
|
276
|
+
// Re-export existing fs/fsp functions
|
|
277
|
+
// rm/rmAsync are replaced with removePath/removePathAsync
|
|
278
|
+
lstat = fs.lstatSync;
|
|
279
|
+
lstatAsync = fsp.lstat;
|
|
280
|
+
stat = fs.statSync;
|
|
281
|
+
statAsync = fsp.stat;
|
|
282
|
+
mkdir = fs.mkdirSync;
|
|
283
|
+
mkdirAsync = fsp.mkdir;
|
|
284
|
+
readdir = fs.readdirSync;
|
|
285
|
+
readdirAsync = fsp.readdir;
|
|
286
|
+
createWriteStream = fs.createWriteStream;
|
|
287
|
+
createReadStream = fs.createReadStream;
|
|
294
288
|
/*
|
|
295
289
|
Returns a Readable of [already parsed] NDJSON objects.
|
|
296
290
|
|
|
@@ -309,18 +303,18 @@ class FS2 {
|
|
|
309
303
|
])
|
|
310
304
|
*/
|
|
311
305
|
createReadStreamAsNDJSON(inputPath) {
|
|
312
|
-
|
|
313
|
-
let stream =
|
|
306
|
+
requireFileToExist(inputPath);
|
|
307
|
+
let stream = fs
|
|
314
308
|
.createReadStream(inputPath, {
|
|
315
309
|
highWaterMark: 64 * 1024, // no observed speedup
|
|
316
310
|
})
|
|
317
311
|
.on('error', err => stream.emit('error', err));
|
|
318
312
|
if (inputPath.endsWith('.gz')) {
|
|
319
|
-
stream = stream.pipe(
|
|
313
|
+
stream = stream.pipe(createUnzip({
|
|
320
314
|
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
321
315
|
}));
|
|
322
316
|
}
|
|
323
|
-
return stream.pipe(
|
|
317
|
+
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
|
|
324
318
|
// For some crazy reason .map is much faster than transformJsonParse!
|
|
325
319
|
// ~5000 vs ~4000 rps !!!
|
|
326
320
|
// .on('error', err => stream.emit('error', err))
|
|
@@ -338,19 +332,19 @@ class FS2 {
|
|
|
338
332
|
createWriteStreamAsNDJSON(outputPath) {
|
|
339
333
|
this.ensureFile(outputPath);
|
|
340
334
|
return [
|
|
341
|
-
|
|
335
|
+
transformToNDJson(),
|
|
342
336
|
outputPath.endsWith('.gz')
|
|
343
|
-
?
|
|
337
|
+
? createGzip({
|
|
344
338
|
// chunkSize: 64 * 1024, // no observed speedup
|
|
345
339
|
})
|
|
346
340
|
: undefined,
|
|
347
|
-
|
|
341
|
+
fs.createWriteStream(outputPath, {
|
|
348
342
|
// highWaterMark: 64 * 1024, // no observed speedup
|
|
349
343
|
}),
|
|
350
|
-
].filter(
|
|
344
|
+
].filter(_isTruthy);
|
|
351
345
|
}
|
|
352
346
|
}
|
|
353
|
-
|
|
347
|
+
export const fs2 = new FS2();
|
|
354
348
|
function stringify(data, opt) {
|
|
355
349
|
// If pretty-printing is enabled (spaces) - also add a newline at the end (to match our prettier config)
|
|
356
350
|
return JSON.stringify(data, null, opt?.spaces) + (opt?.spaces ? '\n' : '');
|
package/dist/fs/json2env.js
CHANGED
|
@@ -1,28 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.appendToBashEnv = appendToBashEnv;
|
|
5
|
-
exports.appendToGithubEnv = appendToGithubEnv;
|
|
6
|
-
exports.appendToGithubOutput = appendToGithubOutput;
|
|
7
|
-
exports.appendToGithubSummary = appendToGithubSummary;
|
|
8
|
-
exports.objectToShellExport = objectToShellExport;
|
|
9
|
-
exports.objectToGithubActionsEnv = objectToGithubActionsEnv;
|
|
10
|
-
const tslib_1 = require("tslib");
|
|
11
|
-
const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
|
|
12
|
-
const colors_1 = require("../colors/colors");
|
|
13
|
-
const fs2_1 = require("./fs2");
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { dimGrey } from '../colors/colors.js';
|
|
3
|
+
import { fs2 } from './fs2.js';
|
|
14
4
|
const JSON2ENV_OPT_DEF = {
|
|
15
5
|
saveEnvFile: true,
|
|
16
6
|
bashEnv: true,
|
|
17
7
|
githubEnv: true,
|
|
18
8
|
fail: true,
|
|
19
9
|
};
|
|
20
|
-
function json2env(opt) {
|
|
10
|
+
export function json2env(opt) {
|
|
21
11
|
const { jsonPath, prefix, saveEnvFile, bashEnv, githubEnv, fail, debug, silent } = {
|
|
22
12
|
...JSON2ENV_OPT_DEF,
|
|
23
13
|
...opt,
|
|
24
14
|
};
|
|
25
|
-
if (!
|
|
15
|
+
if (!fs2.pathExists(jsonPath)) {
|
|
26
16
|
if (fail) {
|
|
27
17
|
throw new Error(`Path doesn't exist: ${jsonPath}`);
|
|
28
18
|
}
|
|
@@ -35,16 +25,16 @@ function json2env(opt) {
|
|
|
35
25
|
return;
|
|
36
26
|
}
|
|
37
27
|
// read file
|
|
38
|
-
const json =
|
|
28
|
+
const json = fs2.readJson(jsonPath);
|
|
39
29
|
if (debug) {
|
|
40
30
|
console.log(json);
|
|
41
31
|
}
|
|
42
32
|
if (saveEnvFile) {
|
|
43
33
|
const shPath = `${jsonPath}.sh`;
|
|
44
34
|
const exportStr = objectToShellExport(json, prefix);
|
|
45
|
-
|
|
35
|
+
fs2.writeFile(shPath, exportStr);
|
|
46
36
|
if (!silent) {
|
|
47
|
-
console.log(`json2env created ${
|
|
37
|
+
console.log(`json2env created ${dimGrey(shPath)}:`);
|
|
48
38
|
console.log(exportStr);
|
|
49
39
|
}
|
|
50
40
|
}
|
|
@@ -56,38 +46,38 @@ function json2env(opt) {
|
|
|
56
46
|
appendToGithubOutput(json, prefix);
|
|
57
47
|
}
|
|
58
48
|
}
|
|
59
|
-
function appendToBashEnv(obj, prefix = '') {
|
|
49
|
+
export function appendToBashEnv(obj, prefix = '') {
|
|
60
50
|
const { BASH_ENV } = process.env;
|
|
61
51
|
if (BASH_ENV) {
|
|
62
52
|
const data = objectToShellExport(obj, prefix);
|
|
63
|
-
|
|
53
|
+
fs.appendFileSync(BASH_ENV, data);
|
|
64
54
|
console.log(`BASH_ENV appended:\n${data}`);
|
|
65
55
|
}
|
|
66
56
|
}
|
|
67
|
-
function appendToGithubEnv(obj, prefix = '') {
|
|
57
|
+
export function appendToGithubEnv(obj, prefix = '') {
|
|
68
58
|
const { GITHUB_ENV } = process.env;
|
|
69
59
|
if (GITHUB_ENV) {
|
|
70
60
|
const data = objectToGithubActionsEnv(obj, prefix);
|
|
71
|
-
|
|
61
|
+
fs.appendFileSync(GITHUB_ENV, data);
|
|
72
62
|
console.log(`GITHUB_ENV appended:\n${data}`);
|
|
73
63
|
}
|
|
74
64
|
}
|
|
75
|
-
function appendToGithubOutput(obj, prefix = '') {
|
|
65
|
+
export function appendToGithubOutput(obj, prefix = '') {
|
|
76
66
|
const { GITHUB_OUTPUT } = process.env;
|
|
77
67
|
if (GITHUB_OUTPUT) {
|
|
78
68
|
const data = objectToGithubActionsEnv(obj, prefix);
|
|
79
|
-
|
|
69
|
+
fs.appendFileSync(GITHUB_OUTPUT, data);
|
|
80
70
|
console.log(`GITHUB_OUTPUT appended:\n${data}`);
|
|
81
71
|
}
|
|
82
72
|
}
|
|
83
73
|
/**
|
|
84
74
|
* https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
|
|
85
75
|
*/
|
|
86
|
-
function appendToGithubSummary(...lines) {
|
|
76
|
+
export function appendToGithubSummary(...lines) {
|
|
87
77
|
const { GITHUB_STEP_SUMMARY } = process.env;
|
|
88
78
|
if (GITHUB_STEP_SUMMARY) {
|
|
89
79
|
const str = lines.join('\n') + '\n';
|
|
90
|
-
|
|
80
|
+
fs.appendFileSync(GITHUB_STEP_SUMMARY, str);
|
|
91
81
|
console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`);
|
|
92
82
|
}
|
|
93
83
|
}
|
|
@@ -105,7 +95,7 @@ function appendToGithubSummary(...lines) {
|
|
|
105
95
|
* Quotes are important, otherwise it'll break on e.g space character in the value.
|
|
106
96
|
* Includes trailing newline for composability.
|
|
107
97
|
*/
|
|
108
|
-
function objectToShellExport(obj, prefix = '') {
|
|
98
|
+
export function objectToShellExport(obj, prefix = '') {
|
|
109
99
|
if (!Object.keys(obj).length)
|
|
110
100
|
return '';
|
|
111
101
|
return (Object.entries(obj)
|
|
@@ -133,7 +123,7 @@ function objectToShellExport(obj, prefix = '') {
|
|
|
133
123
|
*
|
|
134
124
|
* UPD: Quoted values behave inconsistently, so we're trying to NOT quote now, and-see-what-happens.
|
|
135
125
|
*/
|
|
136
|
-
function objectToGithubActionsEnv(obj, prefix = '') {
|
|
126
|
+
export function objectToGithubActionsEnv(obj, prefix = '') {
|
|
137
127
|
if (!Object.keys(obj).length)
|
|
138
128
|
return '';
|
|
139
129
|
return (Object.entries(obj)
|
package/dist/fs/kpy.js
CHANGED
|
@@ -1,64 +1,59 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
8
|
-
const colors_1 = require("../colors/colors");
|
|
9
|
-
const index_1 = require("../index");
|
|
10
|
-
async function kpy(opt) {
|
|
11
|
-
const started = js_lib_1.localTime.nowUnixMillis();
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib';
|
|
3
|
+
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js';
|
|
4
|
+
import { fastGlob, fs2 } from '../index.js';
|
|
5
|
+
export async function kpy(opt) {
|
|
6
|
+
const started = localTime.nowUnixMillis();
|
|
12
7
|
kpyPrepare(opt);
|
|
13
|
-
const filenames = await
|
|
8
|
+
const filenames = await fastGlob(opt.inputPatterns, {
|
|
14
9
|
cwd: opt.baseDir,
|
|
15
10
|
dot: opt.dotfiles,
|
|
16
11
|
});
|
|
17
12
|
kpyLogFilenames(opt, filenames);
|
|
18
13
|
const overwrite = !opt.noOverwrite;
|
|
19
14
|
await Promise.all(filenames.map(async (filename) => {
|
|
20
|
-
const basename =
|
|
21
|
-
const srcFilename =
|
|
22
|
-
const destFilename =
|
|
15
|
+
const basename = path.basename(filename);
|
|
16
|
+
const srcFilename = path.resolve(opt.baseDir, filename);
|
|
17
|
+
const destFilename = path.resolve(opt.outputDir, opt.flat ? basename : filename);
|
|
23
18
|
if (!opt.dry) {
|
|
24
19
|
if (opt.move) {
|
|
25
|
-
await
|
|
20
|
+
await fs2.movePathAsync(srcFilename, destFilename, {
|
|
26
21
|
force: overwrite,
|
|
27
22
|
});
|
|
28
23
|
}
|
|
29
24
|
else {
|
|
30
|
-
await
|
|
25
|
+
await fs2.copyPathAsync(srcFilename, destFilename, { force: overwrite });
|
|
31
26
|
}
|
|
32
27
|
}
|
|
33
28
|
if (opt.verbose) {
|
|
34
|
-
console.log(
|
|
29
|
+
console.log(grey(` ${filename}`));
|
|
35
30
|
}
|
|
36
31
|
}));
|
|
37
32
|
kpyLogResult(opt, filenames, started);
|
|
38
33
|
}
|
|
39
|
-
function kpySync(opt) {
|
|
40
|
-
const started =
|
|
34
|
+
export function kpySync(opt) {
|
|
35
|
+
const started = localTime.nowUnixMillis();
|
|
41
36
|
kpyPrepare(opt);
|
|
42
|
-
const filenames =
|
|
37
|
+
const filenames = fastGlob.sync(opt.inputPatterns, {
|
|
43
38
|
cwd: opt.baseDir,
|
|
44
39
|
dot: opt.dotfiles,
|
|
45
40
|
});
|
|
46
41
|
kpyLogFilenames(opt, filenames);
|
|
47
42
|
const overwrite = !opt.noOverwrite;
|
|
48
43
|
filenames.forEach(filename => {
|
|
49
|
-
const basename =
|
|
50
|
-
const srcFilename =
|
|
51
|
-
const destFilename =
|
|
44
|
+
const basename = path.basename(filename);
|
|
45
|
+
const srcFilename = path.resolve(opt.baseDir, filename);
|
|
46
|
+
const destFilename = path.resolve(opt.outputDir, opt.flat ? basename : filename);
|
|
52
47
|
if (!opt.dry) {
|
|
53
48
|
if (opt.move) {
|
|
54
|
-
|
|
49
|
+
fs2.movePath(srcFilename, destFilename, { force: overwrite });
|
|
55
50
|
}
|
|
56
51
|
else {
|
|
57
|
-
|
|
52
|
+
fs2.copyPath(srcFilename, destFilename, { force: overwrite });
|
|
58
53
|
}
|
|
59
54
|
}
|
|
60
55
|
if (opt.verbose) {
|
|
61
|
-
console.log(
|
|
56
|
+
console.log(grey(` ${filename}`));
|
|
62
57
|
}
|
|
63
58
|
});
|
|
64
59
|
kpyLogResult(opt, filenames, started);
|
|
@@ -70,17 +65,17 @@ function kpyPrepare(opt) {
|
|
|
70
65
|
// default to cwd
|
|
71
66
|
opt.baseDir ||= '.';
|
|
72
67
|
opt.outputDir ||= '.';
|
|
73
|
-
if (!
|
|
74
|
-
console.log(`kpy: baseDir doesn't exist: ${
|
|
68
|
+
if (!fs2.pathExists(opt.baseDir)) {
|
|
69
|
+
console.log(`kpy: baseDir doesn't exist: ${boldWhite(opt.baseDir)}`);
|
|
75
70
|
return;
|
|
76
71
|
}
|
|
77
|
-
|
|
72
|
+
fs2.ensureDir(opt.outputDir);
|
|
78
73
|
// Expand directories (ex-globby feature), experimental!
|
|
79
74
|
const extraPatterns = [];
|
|
80
75
|
for (const pattern of opt.inputPatterns) {
|
|
81
76
|
if (pattern.includes('*'))
|
|
82
77
|
continue;
|
|
83
|
-
if (
|
|
78
|
+
if (fs2.isDirectory(path.resolve(opt.baseDir, pattern))) {
|
|
84
79
|
extraPatterns.push(`${pattern}/**`);
|
|
85
80
|
}
|
|
86
81
|
}
|
|
@@ -93,10 +88,10 @@ function kpyLogFilenames(opt, filenames) {
|
|
|
93
88
|
if (opt.silent)
|
|
94
89
|
return;
|
|
95
90
|
// console.log({filenames})
|
|
96
|
-
console.log(`Will ${opt.move ? 'move' : 'copy'} ${
|
|
91
|
+
console.log(`Will ${opt.move ? 'move' : 'copy'} ${yellow(filenames.length)} files from ${dimGrey(opt.baseDir)} to ${dimGrey(opt.outputDir)} (${dimGrey(opt.inputPatterns.join(' '))})`);
|
|
97
92
|
}
|
|
98
93
|
function kpyLogResult(opt, filenames, started) {
|
|
99
94
|
if (opt.silent || filenames.length === 0)
|
|
100
95
|
return;
|
|
101
|
-
console.log(`${opt.move ? 'Moved' : 'Copied'} ${
|
|
96
|
+
console.log(`${opt.move ? 'Moved' : 'Copied'} ${yellow(filenames.length)} files to ${dimGrey(opt.outputDir)} ${dimGrey(_since(started))}`);
|
|
102
97
|
}
|