@naturalcycles/nodejs-lib 13.45.4 → 14.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/generate-build-info.js +15 -18
- package/dist/bin/json2env.js +6 -9
- package/dist/bin/kpy.js +6 -9
- package/dist/bin/secrets-decrypt.d.ts +1 -1
- package/dist/bin/secrets-decrypt.js +9 -12
- package/dist/bin/secrets-encrypt.d.ts +1 -1
- package/dist/bin/secrets-encrypt.js +9 -12
- package/dist/bin/secrets-gen-key.js +8 -11
- package/dist/bin/slack-this.js +6 -9
- package/dist/buffer/buffer.util.js +1 -4
- package/dist/colors/colors.d.ts +26 -27
- package/dist/colors/colors.js +29 -33
- package/dist/csv/csvReader.js +4 -8
- package/dist/csv/csvWriter.js +6 -12
- package/dist/csv/transformToCSV.d.ts +2 -2
- package/dist/csv/transformToCSV.js +5 -8
- package/dist/diff/tableDiff.js +4 -7
- package/dist/fs/fs2.d.ts +1 -1
- package/dist/fs/fs2.js +82 -86
- package/dist/fs/json2env.js +18 -28
- package/dist/fs/kpy.js +28 -33
- package/dist/index.d.ts +71 -71
- package/dist/index.js +73 -78
- package/dist/infra/process.util.js +22 -28
- package/dist/jwt/jwt.service.js +16 -21
- package/dist/log/log.util.js +4 -7
- package/dist/script/runScript.js +11 -15
- package/dist/secret/secrets-decrypt.util.js +18 -22
- package/dist/secret/secrets-encrypt.util.js +17 -21
- package/dist/security/crypto.util.js +24 -34
- package/dist/security/hash.util.js +19 -37
- package/dist/security/id.util.js +11 -18
- package/dist/security/nanoid.js +15 -20
- package/dist/security/secret.util.js +20 -30
- package/dist/slack/slack.service.d.ts +1 -1
- package/dist/slack/slack.service.js +11 -16
- package/dist/slack/slack.service.model.d.ts +1 -1
- package/dist/slack/slack.service.model.js +1 -2
- package/dist/stream/ndjson/ndjson.model.js +7 -11
- package/dist/stream/ndjson/ndjsonMap.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonMap.js +11 -14
- package/dist/stream/ndjson/ndjsonStreamForEach.d.ts +1 -1
- package/dist/stream/ndjson/ndjsonStreamForEach.js +9 -12
- package/dist/stream/ndjson/transformJsonParse.d.ts +1 -1
- package/dist/stream/ndjson/transformJsonParse.js +4 -9
- package/dist/stream/ndjson/transformToNDJson.d.ts +1 -1
- package/dist/stream/ndjson/transformToNDJson.js +5 -8
- package/dist/stream/pipeline/pipeline.js +11 -17
- package/dist/stream/progressLogger.d.ts +1 -1
- package/dist/stream/progressLogger.js +22 -28
- package/dist/stream/readable/readableCreate.d.ts +1 -1
- package/dist/stream/readable/readableCreate.js +5 -9
- package/dist/stream/readable/readableForEach.d.ts +2 -2
- package/dist/stream/readable/readableForEach.js +7 -11
- package/dist/stream/readable/readableFromArray.d.ts +1 -1
- package/dist/stream/readable/readableFromArray.js +4 -7
- package/dist/stream/readable/readableToArray.d.ts +1 -1
- package/dist/stream/readable/readableToArray.js +1 -4
- package/dist/stream/sizeStack.js +7 -11
- package/dist/stream/stream.model.js +1 -2
- package/dist/stream/stream.util.js +1 -4
- package/dist/stream/transform/transformChunk.d.ts +1 -1
- package/dist/stream/transform/transformChunk.js +3 -6
- package/dist/stream/transform/transformFilter.d.ts +2 -2
- package/dist/stream/transform/transformFilter.js +6 -10
- package/dist/stream/transform/transformLimit.d.ts +2 -2
- package/dist/stream/transform/transformLimit.js +6 -11
- package/dist/stream/transform/transformLogProgress.d.ts +2 -2
- package/dist/stream/transform/transformLogProgress.js +5 -8
- package/dist/stream/transform/transformMap.d.ts +1 -1
- package/dist/stream/transform/transformMap.js +18 -22
- package/dist/stream/transform/transformMapSimple.d.ts +1 -1
- package/dist/stream/transform/transformMapSimple.js +6 -9
- package/dist/stream/transform/transformMapSync.d.ts +3 -3
- package/dist/stream/transform/transformMapSync.js +14 -19
- package/dist/stream/transform/transformNoOp.d.ts +1 -1
- package/dist/stream/transform/transformNoOp.js +3 -6
- package/dist/stream/transform/transformOffset.d.ts +1 -1
- package/dist/stream/transform/transformOffset.js +5 -8
- package/dist/stream/transform/transformSplit.d.ts +1 -1
- package/dist/stream/transform/transformSplit.js +5 -9
- package/dist/stream/transform/transformTap.d.ts +1 -1
- package/dist/stream/transform/transformTap.js +3 -6
- package/dist/stream/transform/transformTee.d.ts +1 -1
- package/dist/stream/transform/transformTee.js +7 -10
- package/dist/stream/transform/transformThrottle.d.ts +1 -1
- package/dist/stream/transform/transformThrottle.js +9 -12
- package/dist/stream/transform/transformToArray.d.ts +1 -1
- package/dist/stream/transform/transformToArray.js +3 -6
- package/dist/stream/transform/worker/baseWorkerClass.d.ts +1 -1
- package/dist/stream/transform/worker/baseWorkerClass.js +1 -5
- package/dist/stream/transform/worker/transformMultiThreaded.d.ts +1 -1
- package/dist/stream/transform/worker/transformMultiThreaded.js +12 -11
- package/dist/stream/transform/worker/transformMultiThreaded.model.js +1 -2
- package/dist/stream/transform/worker/workerClassProxy.js +6 -4
- package/dist/stream/writable/writableForEach.d.ts +2 -2
- package/dist/stream/writable/writableForEach.js +6 -10
- package/dist/stream/writable/writableFork.d.ts +1 -1
- package/dist/stream/writable/writableFork.js +6 -9
- package/dist/stream/writable/writableLimit.d.ts +1 -1
- package/dist/stream/writable/writableLimit.js +3 -6
- package/dist/stream/writable/writablePushToArray.d.ts +1 -1
- package/dist/stream/writable/writablePushToArray.js +3 -6
- package/dist/stream/writable/writableVoid.d.ts +1 -1
- package/dist/stream/writable/writableVoid.js +3 -6
- package/dist/string/inspect.js +8 -13
- package/dist/util/buildInfo.util.js +12 -15
- package/dist/util/env.util.js +5 -9
- package/dist/util/exec2.js +17 -21
- package/dist/util/git2.js +17 -20
- package/dist/util/lruMemoCache.js +3 -7
- package/dist/util/zip.util.js +14 -25
- package/dist/validation/ajv/ajv.util.d.ts +3 -3
- package/dist/validation/ajv/ajv.util.js +6 -10
- package/dist/validation/ajv/ajvSchema.d.ts +2 -2
- package/dist/validation/ajv/ajvSchema.js +14 -18
- package/dist/validation/ajv/ajvValidationError.js +2 -6
- package/dist/validation/ajv/getAjv.d.ts +1 -1
- package/dist/validation/ajv/getAjv.js +9 -9
- package/dist/validation/joi/joi.extensions.d.ts +2 -2
- package/dist/validation/joi/joi.extensions.js +6 -10
- package/dist/validation/joi/joi.model.js +1 -2
- package/dist/validation/joi/joi.shared.schemas.d.ts +2 -2
- package/dist/validation/joi/joi.shared.schemas.js +61 -74
- package/dist/validation/joi/joi.validation.error.js +2 -6
- package/dist/validation/joi/joi.validation.util.d.ts +1 -1
- package/dist/validation/joi/joi.validation.util.js +10 -17
- package/dist/validation/joi/number.extensions.js +1 -4
- package/dist/validation/joi/string.extensions.js +4 -7
- package/dist/yargs.util.js +5 -8
- package/package.json +6 -5
- package/src/bin/generate-build-info.ts +3 -3
- package/src/bin/json2env.ts +2 -2
- package/src/bin/kpy.ts +2 -2
- package/src/bin/secrets-decrypt.ts +7 -8
- package/src/bin/secrets-encrypt.ts +7 -8
- package/src/bin/secrets-gen-key.ts +2 -2
- package/src/bin/slack-this.ts +2 -2
- package/src/csv/transformToCSV.ts +3 -3
- package/src/fs/fs2.ts +4 -4
- package/src/fs/json2env.ts +2 -2
- package/src/fs/kpy.ts +2 -2
- package/src/index.ts +71 -71
- package/src/jwt/jwt.service.ts +2 -2
- package/src/log/log.util.ts +1 -1
- package/src/script/runScript.ts +4 -3
- package/src/secret/secrets-decrypt.util.ts +3 -3
- package/src/secret/secrets-encrypt.util.ts +3 -3
- package/src/security/crypto.util.ts +1 -1
- package/src/security/id.util.ts +1 -1
- package/src/security/secret.util.ts +2 -2
- package/src/slack/slack.service.model.ts +1 -1
- package/src/slack/slack.service.ts +3 -3
- package/src/stream/ndjson/ndjson.model.ts +1 -1
- package/src/stream/ndjson/ndjsonMap.ts +2 -2
- package/src/stream/ndjson/ndjsonStreamForEach.ts +2 -2
- package/src/stream/ndjson/transformJsonParse.ts +1 -1
- package/src/stream/ndjson/transformToNDJson.ts +1 -1
- package/src/stream/pipeline/pipeline.ts +1 -1
- package/src/stream/progressLogger.ts +3 -3
- package/src/stream/readable/readableCreate.ts +1 -1
- package/src/stream/readable/readableForEach.ts +4 -4
- package/src/stream/readable/readableFromArray.ts +1 -1
- package/src/stream/readable/readableToArray.ts +1 -1
- package/src/stream/sizeStack.ts +2 -2
- package/src/stream/transform/transformChunk.ts +1 -1
- package/src/stream/transform/transformFilter.ts +3 -3
- package/src/stream/transform/transformLimit.ts +3 -3
- package/src/stream/transform/transformLogProgress.ts +3 -3
- package/src/stream/transform/transformMap.ts +5 -5
- package/src/stream/transform/transformMapSimple.ts +1 -1
- package/src/stream/transform/transformMapSync.ts +5 -5
- package/src/stream/transform/transformNoOp.ts +1 -1
- package/src/stream/transform/transformOffset.ts +2 -2
- package/src/stream/transform/transformSplit.ts +1 -1
- package/src/stream/transform/transformTap.ts +1 -1
- package/src/stream/transform/transformTee.ts +3 -3
- package/src/stream/transform/transformThrottle.ts +1 -1
- package/src/stream/transform/transformToArray.ts +1 -1
- package/src/stream/transform/worker/baseWorkerClass.ts +1 -1
- package/src/stream/transform/worker/transformMultiThreaded.ts +8 -3
- package/src/stream/transform/worker/workerClassProxy.js +6 -4
- package/src/stream/writable/writableForEach.ts +3 -3
- package/src/stream/writable/writableFork.ts +3 -3
- package/src/stream/writable/writableLimit.ts +1 -1
- package/src/stream/writable/writablePushToArray.ts +1 -1
- package/src/stream/writable/writableVoid.ts +1 -1
- package/src/util/buildInfo.util.ts +2 -2
- package/src/util/exec2.ts +1 -1
- package/src/util/git2.ts +2 -2
- package/src/validation/ajv/ajv.util.ts +4 -4
- package/src/validation/ajv/ajvSchema.ts +4 -5
- package/src/validation/ajv/getAjv.ts +7 -3
- package/src/validation/joi/joi.extensions.ts +4 -4
- package/src/validation/joi/joi.shared.schemas.ts +3 -3
- package/src/validation/joi/joi.validation.util.ts +2 -2
- package/src/yargs.util.ts +1 -0
package/dist/fs/fs2.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/*
|
|
3
2
|
|
|
4
3
|
Why?
|
|
@@ -14,18 +13,15 @@ That's why function names are slightly renamed, to avoid conflict.
|
|
|
14
13
|
Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
|
|
15
14
|
|
|
16
15
|
*/
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const transformToNDJson_1 = require("../stream/ndjson/transformToNDJson");
|
|
27
|
-
const transformSplit_1 = require("../stream/transform/transformSplit");
|
|
28
|
-
const env_util_1 = require("../util/env.util");
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import fsp from 'node:fs/promises';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { createGzip, createUnzip } from 'node:zlib';
|
|
20
|
+
import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib';
|
|
21
|
+
import yaml from 'js-yaml';
|
|
22
|
+
import { transformToNDJson } from '../stream/ndjson/transformToNDJson.js';
|
|
23
|
+
import { transformSplitOnNewline } from '../stream/transform/transformSplit.js';
|
|
24
|
+
import { requireFileToExist } from '../util/env.util.js';
|
|
29
25
|
/**
|
|
30
26
|
* fs2 conveniently groups filesystem functions together.
|
|
31
27
|
* Supposed to be almost a drop-in replacement for these things together:
|
|
@@ -42,62 +38,62 @@ class FS2 {
|
|
|
42
38
|
* Convenience wrapper that defaults to utf-8 string output.
|
|
43
39
|
*/
|
|
44
40
|
readText(filePath) {
|
|
45
|
-
return
|
|
41
|
+
return fs.readFileSync(filePath, 'utf8');
|
|
46
42
|
}
|
|
47
43
|
/**
|
|
48
44
|
* Convenience wrapper that defaults to utf-8 string output.
|
|
49
45
|
*/
|
|
50
46
|
async readTextAsync(filePath) {
|
|
51
|
-
return await
|
|
47
|
+
return await fsp.readFile(filePath, 'utf8');
|
|
52
48
|
}
|
|
53
49
|
readBuffer(filePath) {
|
|
54
|
-
return
|
|
50
|
+
return fs.readFileSync(filePath);
|
|
55
51
|
}
|
|
56
52
|
async readBufferAsync(filePath) {
|
|
57
|
-
return await
|
|
53
|
+
return await fsp.readFile(filePath);
|
|
58
54
|
}
|
|
59
55
|
readJson(filePath) {
|
|
60
|
-
const str =
|
|
61
|
-
return
|
|
56
|
+
const str = fs.readFileSync(filePath, 'utf8');
|
|
57
|
+
return _jsonParse(str);
|
|
62
58
|
}
|
|
63
59
|
async readJsonAsync(filePath) {
|
|
64
|
-
const str = await
|
|
60
|
+
const str = await fsp.readFile(filePath, 'utf8');
|
|
65
61
|
// eslint-disable-next-line @typescript-eslint/return-await
|
|
66
|
-
return
|
|
62
|
+
return _jsonParse(str);
|
|
67
63
|
}
|
|
68
64
|
readYaml(filePath) {
|
|
69
|
-
return
|
|
65
|
+
return yaml.load(fs.readFileSync(filePath, 'utf8'));
|
|
70
66
|
}
|
|
71
67
|
async readYamlAsync(filePath) {
|
|
72
|
-
return
|
|
68
|
+
return yaml.load(await fsp.readFile(filePath, 'utf8'));
|
|
73
69
|
}
|
|
74
70
|
writeFile(filePath, data) {
|
|
75
|
-
|
|
71
|
+
fs.writeFileSync(filePath, data);
|
|
76
72
|
}
|
|
77
73
|
async writeFileAsync(filePath, data) {
|
|
78
|
-
await
|
|
74
|
+
await fsp.writeFile(filePath, data);
|
|
79
75
|
}
|
|
80
76
|
writeJson(filePath, data, opt) {
|
|
81
77
|
const str = stringify(data, opt);
|
|
82
|
-
|
|
78
|
+
fs.writeFileSync(filePath, str);
|
|
83
79
|
}
|
|
84
80
|
async writeJsonAsync(filePath, data, opt) {
|
|
85
81
|
const str = stringify(data, opt);
|
|
86
|
-
await
|
|
82
|
+
await fsp.writeFile(filePath, str);
|
|
87
83
|
}
|
|
88
84
|
writeYaml(filePath, data, opt) {
|
|
89
|
-
const str =
|
|
90
|
-
|
|
85
|
+
const str = yaml.dump(data, opt);
|
|
86
|
+
fs.writeFileSync(filePath, str);
|
|
91
87
|
}
|
|
92
88
|
async writeYamlAsync(filePath, data, opt) {
|
|
93
|
-
const str =
|
|
94
|
-
await
|
|
89
|
+
const str = yaml.dump(data, opt);
|
|
90
|
+
await fsp.writeFile(filePath, str);
|
|
95
91
|
}
|
|
96
92
|
appendFile(filePath, data) {
|
|
97
|
-
|
|
93
|
+
fs.appendFileSync(filePath, data);
|
|
98
94
|
}
|
|
99
95
|
async appendFileAsync(filePath, data) {
|
|
100
|
-
await
|
|
96
|
+
await fsp.appendFile(filePath, data);
|
|
101
97
|
}
|
|
102
98
|
outputJson(filePath, data, opt) {
|
|
103
99
|
const str = stringify(data, opt);
|
|
@@ -108,33 +104,33 @@ class FS2 {
|
|
|
108
104
|
await this.outputFileAsync(filePath, str);
|
|
109
105
|
}
|
|
110
106
|
outputYaml(filePath, data, opt) {
|
|
111
|
-
const str =
|
|
107
|
+
const str = yaml.dump(data, opt);
|
|
112
108
|
this.outputFile(filePath, str);
|
|
113
109
|
}
|
|
114
110
|
async outputYamlAsync(filePath, data, opt) {
|
|
115
|
-
const str =
|
|
111
|
+
const str = yaml.dump(data, opt);
|
|
116
112
|
await this.outputFileAsync(filePath, str);
|
|
117
113
|
}
|
|
118
114
|
outputFile(filePath, data) {
|
|
119
|
-
const dirPath =
|
|
120
|
-
if (!
|
|
115
|
+
const dirPath = path.dirname(filePath);
|
|
116
|
+
if (!fs.existsSync(dirPath)) {
|
|
121
117
|
this.ensureDir(dirPath);
|
|
122
118
|
}
|
|
123
|
-
|
|
119
|
+
fs.writeFileSync(filePath, data);
|
|
124
120
|
}
|
|
125
121
|
async outputFileAsync(filePath, data) {
|
|
126
|
-
const dirPath =
|
|
122
|
+
const dirPath = path.dirname(filePath);
|
|
127
123
|
if (!(await this.pathExistsAsync(dirPath))) {
|
|
128
124
|
await this.ensureDirAsync(dirPath);
|
|
129
125
|
}
|
|
130
|
-
await
|
|
126
|
+
await fsp.writeFile(filePath, data);
|
|
131
127
|
}
|
|
132
128
|
pathExists(filePath) {
|
|
133
|
-
return
|
|
129
|
+
return fs.existsSync(filePath);
|
|
134
130
|
}
|
|
135
131
|
async pathExistsAsync(filePath) {
|
|
136
132
|
try {
|
|
137
|
-
await
|
|
133
|
+
await fsp.access(filePath);
|
|
138
134
|
return true;
|
|
139
135
|
}
|
|
140
136
|
catch {
|
|
@@ -142,13 +138,13 @@ class FS2 {
|
|
|
142
138
|
}
|
|
143
139
|
}
|
|
144
140
|
ensureDir(dirPath) {
|
|
145
|
-
|
|
141
|
+
fs.mkdirSync(dirPath, {
|
|
146
142
|
mode: 0o777,
|
|
147
143
|
recursive: true,
|
|
148
144
|
});
|
|
149
145
|
}
|
|
150
146
|
async ensureDirAsync(dirPath) {
|
|
151
|
-
await
|
|
147
|
+
await fsp.mkdir(dirPath, {
|
|
152
148
|
mode: 0o777,
|
|
153
149
|
recursive: true,
|
|
154
150
|
});
|
|
@@ -156,17 +152,17 @@ class FS2 {
|
|
|
156
152
|
ensureFile(filePath) {
|
|
157
153
|
let stats;
|
|
158
154
|
try {
|
|
159
|
-
stats =
|
|
155
|
+
stats = fs.statSync(filePath);
|
|
160
156
|
}
|
|
161
157
|
catch { }
|
|
162
158
|
if (stats?.isFile())
|
|
163
159
|
return;
|
|
164
|
-
const dir =
|
|
160
|
+
const dir = path.dirname(filePath);
|
|
165
161
|
try {
|
|
166
|
-
if (!
|
|
162
|
+
if (!fs.statSync(dir).isDirectory()) {
|
|
167
163
|
// parent is not a directory
|
|
168
164
|
// This is just to cause an internal ENOTDIR error to be thrown
|
|
169
|
-
|
|
165
|
+
fs.readdirSync(dir);
|
|
170
166
|
}
|
|
171
167
|
}
|
|
172
168
|
catch (err) {
|
|
@@ -177,22 +173,22 @@ class FS2 {
|
|
|
177
173
|
}
|
|
178
174
|
throw err;
|
|
179
175
|
}
|
|
180
|
-
|
|
176
|
+
fs.writeFileSync(filePath, '');
|
|
181
177
|
}
|
|
182
178
|
async ensureFileAsync(filePath) {
|
|
183
179
|
let stats;
|
|
184
180
|
try {
|
|
185
|
-
stats = await
|
|
181
|
+
stats = await fsp.stat(filePath);
|
|
186
182
|
}
|
|
187
183
|
catch { }
|
|
188
184
|
if (stats?.isFile())
|
|
189
185
|
return;
|
|
190
|
-
const dir =
|
|
186
|
+
const dir = path.dirname(filePath);
|
|
191
187
|
try {
|
|
192
|
-
if (!(await
|
|
188
|
+
if (!(await fsp.stat(dir)).isDirectory()) {
|
|
193
189
|
// parent is not a directory
|
|
194
190
|
// This is just to cause an internal ENOTDIR error to be thrown
|
|
195
|
-
await
|
|
191
|
+
await fsp.readdir(dir);
|
|
196
192
|
}
|
|
197
193
|
}
|
|
198
194
|
catch (err) {
|
|
@@ -201,40 +197,40 @@ class FS2 {
|
|
|
201
197
|
return await this.ensureDirAsync(dir);
|
|
202
198
|
throw err;
|
|
203
199
|
}
|
|
204
|
-
await
|
|
200
|
+
await fsp.writeFile(filePath, '');
|
|
205
201
|
}
|
|
206
202
|
removePath(fileOrDirPath, opt) {
|
|
207
|
-
|
|
203
|
+
fs.rmSync(fileOrDirPath, { recursive: true, force: true, ...opt });
|
|
208
204
|
}
|
|
209
205
|
async removePathAsync(fileOrDirPath, opt) {
|
|
210
|
-
await
|
|
206
|
+
await fsp.rm(fileOrDirPath, { recursive: true, force: true, ...opt });
|
|
211
207
|
}
|
|
212
208
|
emptyDir(dirPath) {
|
|
213
209
|
let items;
|
|
214
210
|
try {
|
|
215
|
-
items =
|
|
211
|
+
items = fs.readdirSync(dirPath);
|
|
216
212
|
}
|
|
217
213
|
catch {
|
|
218
214
|
this.ensureDir(dirPath);
|
|
219
215
|
return;
|
|
220
216
|
}
|
|
221
|
-
items.forEach(item => this.removePath(
|
|
217
|
+
items.forEach(item => this.removePath(path.join(dirPath, item)));
|
|
222
218
|
}
|
|
223
219
|
async emptyDirAsync(dirPath) {
|
|
224
220
|
let items;
|
|
225
221
|
try {
|
|
226
|
-
items = await
|
|
222
|
+
items = await fsp.readdir(dirPath);
|
|
227
223
|
}
|
|
228
224
|
catch {
|
|
229
225
|
return await this.ensureDirAsync(dirPath);
|
|
230
226
|
}
|
|
231
|
-
await Promise.all(items.map(item => this.removePathAsync(
|
|
227
|
+
await Promise.all(items.map(item => this.removePathAsync(path.join(dirPath, item))));
|
|
232
228
|
}
|
|
233
229
|
/**
|
|
234
230
|
* Cautious, underlying Node function is currently Experimental.
|
|
235
231
|
*/
|
|
236
232
|
copyPath(src, dest, opt) {
|
|
237
|
-
|
|
233
|
+
fs.cpSync(src, dest, {
|
|
238
234
|
recursive: true,
|
|
239
235
|
...opt,
|
|
240
236
|
});
|
|
@@ -243,16 +239,16 @@ class FS2 {
|
|
|
243
239
|
* Cautious, underlying Node function is currently Experimental.
|
|
244
240
|
*/
|
|
245
241
|
async copyPathAsync(src, dest, opt) {
|
|
246
|
-
await
|
|
242
|
+
await fsp.cp(src, dest, {
|
|
247
243
|
recursive: true,
|
|
248
244
|
...opt,
|
|
249
245
|
});
|
|
250
246
|
}
|
|
251
247
|
renamePath(src, dest) {
|
|
252
|
-
|
|
248
|
+
fs.renameSync(src, dest);
|
|
253
249
|
}
|
|
254
250
|
async renamePathAsync(src, dest) {
|
|
255
|
-
await
|
|
251
|
+
await fsp.rename(src, dest);
|
|
256
252
|
}
|
|
257
253
|
movePath(src, dest, opt) {
|
|
258
254
|
this.copyPath(src, dest, opt);
|
|
@@ -268,27 +264,27 @@ class FS2 {
|
|
|
268
264
|
* Doesn't throw, returns false instead.
|
|
269
265
|
*/
|
|
270
266
|
isDirectory(filePath) {
|
|
271
|
-
return (
|
|
267
|
+
return (fs2
|
|
272
268
|
.stat(filePath, {
|
|
273
269
|
throwIfNoEntry: false,
|
|
274
270
|
})
|
|
275
271
|
?.isDirectory() || false);
|
|
276
272
|
}
|
|
277
273
|
// Re-export the whole fs/fsp, for the edge cases where they are needed
|
|
278
|
-
fs =
|
|
279
|
-
fsp =
|
|
274
|
+
fs = fs;
|
|
275
|
+
fsp = fsp;
|
|
280
276
|
// Re-export existing fs/fsp functions
|
|
281
277
|
// rm/rmAsync are replaced with removePath/removePathAsync
|
|
282
|
-
lstat =
|
|
283
|
-
lstatAsync =
|
|
284
|
-
stat =
|
|
285
|
-
statAsync =
|
|
286
|
-
mkdir =
|
|
287
|
-
mkdirAsync =
|
|
288
|
-
readdir =
|
|
289
|
-
readdirAsync =
|
|
290
|
-
createWriteStream =
|
|
291
|
-
createReadStream =
|
|
278
|
+
lstat = fs.lstatSync;
|
|
279
|
+
lstatAsync = fsp.lstat;
|
|
280
|
+
stat = fs.statSync;
|
|
281
|
+
statAsync = fsp.stat;
|
|
282
|
+
mkdir = fs.mkdirSync;
|
|
283
|
+
mkdirAsync = fsp.mkdir;
|
|
284
|
+
readdir = fs.readdirSync;
|
|
285
|
+
readdirAsync = fsp.readdir;
|
|
286
|
+
createWriteStream = fs.createWriteStream;
|
|
287
|
+
createReadStream = fs.createReadStream;
|
|
292
288
|
/*
|
|
293
289
|
Returns a Readable of [already parsed] NDJSON objects.
|
|
294
290
|
|
|
@@ -307,18 +303,18 @@ class FS2 {
|
|
|
307
303
|
])
|
|
308
304
|
*/
|
|
309
305
|
createReadStreamAsNDJSON(inputPath) {
|
|
310
|
-
|
|
311
|
-
let stream =
|
|
306
|
+
requireFileToExist(inputPath);
|
|
307
|
+
let stream = fs
|
|
312
308
|
.createReadStream(inputPath, {
|
|
313
309
|
highWaterMark: 64 * 1024, // no observed speedup
|
|
314
310
|
})
|
|
315
311
|
.on('error', err => stream.emit('error', err));
|
|
316
312
|
if (inputPath.endsWith('.gz')) {
|
|
317
|
-
stream = stream.pipe(
|
|
313
|
+
stream = stream.pipe(createUnzip({
|
|
318
314
|
chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
|
|
319
315
|
}));
|
|
320
316
|
}
|
|
321
|
-
return stream.pipe(
|
|
317
|
+
return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
|
|
322
318
|
// For some crazy reason .map is much faster than transformJsonParse!
|
|
323
319
|
// ~5000 vs ~4000 rps !!!
|
|
324
320
|
// .on('error', err => stream.emit('error', err))
|
|
@@ -336,19 +332,19 @@ class FS2 {
|
|
|
336
332
|
createWriteStreamAsNDJSON(outputPath) {
|
|
337
333
|
this.ensureFile(outputPath);
|
|
338
334
|
return [
|
|
339
|
-
|
|
335
|
+
transformToNDJson(),
|
|
340
336
|
outputPath.endsWith('.gz')
|
|
341
|
-
?
|
|
337
|
+
? createGzip({
|
|
342
338
|
// chunkSize: 64 * 1024, // no observed speedup
|
|
343
339
|
})
|
|
344
340
|
: undefined,
|
|
345
|
-
|
|
341
|
+
fs.createWriteStream(outputPath, {
|
|
346
342
|
// highWaterMark: 64 * 1024, // no observed speedup
|
|
347
343
|
}),
|
|
348
|
-
].filter(
|
|
344
|
+
].filter(_isTruthy);
|
|
349
345
|
}
|
|
350
346
|
}
|
|
351
|
-
|
|
347
|
+
export const fs2 = new FS2();
|
|
352
348
|
function stringify(data, opt) {
|
|
353
349
|
// If pretty-printing is enabled (spaces) - also add a newline at the end (to match our prettier config)
|
|
354
350
|
return JSON.stringify(data, null, opt?.spaces) + (opt?.spaces ? '\n' : '');
|
package/dist/fs/json2env.js
CHANGED
|
@@ -1,28 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.appendToBashEnv = appendToBashEnv;
|
|
5
|
-
exports.appendToGithubEnv = appendToGithubEnv;
|
|
6
|
-
exports.appendToGithubOutput = appendToGithubOutput;
|
|
7
|
-
exports.appendToGithubSummary = appendToGithubSummary;
|
|
8
|
-
exports.objectToShellExport = objectToShellExport;
|
|
9
|
-
exports.objectToGithubActionsEnv = objectToGithubActionsEnv;
|
|
10
|
-
const tslib_1 = require("tslib");
|
|
11
|
-
const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
|
|
12
|
-
const colors_1 = require("../colors/colors");
|
|
13
|
-
const fs2_1 = require("./fs2");
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { dimGrey } from '../colors/colors.js';
|
|
3
|
+
import { fs2 } from './fs2.js';
|
|
14
4
|
const JSON2ENV_OPT_DEF = {
|
|
15
5
|
saveEnvFile: true,
|
|
16
6
|
bashEnv: true,
|
|
17
7
|
githubEnv: true,
|
|
18
8
|
fail: true,
|
|
19
9
|
};
|
|
20
|
-
function json2env(opt) {
|
|
10
|
+
export function json2env(opt) {
|
|
21
11
|
const { jsonPath, prefix, saveEnvFile, bashEnv, githubEnv, fail, debug, silent } = {
|
|
22
12
|
...JSON2ENV_OPT_DEF,
|
|
23
13
|
...opt,
|
|
24
14
|
};
|
|
25
|
-
if (!
|
|
15
|
+
if (!fs2.pathExists(jsonPath)) {
|
|
26
16
|
if (fail) {
|
|
27
17
|
throw new Error(`Path doesn't exist: ${jsonPath}`);
|
|
28
18
|
}
|
|
@@ -35,16 +25,16 @@ function json2env(opt) {
|
|
|
35
25
|
return;
|
|
36
26
|
}
|
|
37
27
|
// read file
|
|
38
|
-
const json =
|
|
28
|
+
const json = fs2.readJson(jsonPath);
|
|
39
29
|
if (debug) {
|
|
40
30
|
console.log(json);
|
|
41
31
|
}
|
|
42
32
|
if (saveEnvFile) {
|
|
43
33
|
const shPath = `${jsonPath}.sh`;
|
|
44
34
|
const exportStr = objectToShellExport(json, prefix);
|
|
45
|
-
|
|
35
|
+
fs2.writeFile(shPath, exportStr);
|
|
46
36
|
if (!silent) {
|
|
47
|
-
console.log(`json2env created ${
|
|
37
|
+
console.log(`json2env created ${dimGrey(shPath)}:`);
|
|
48
38
|
console.log(exportStr);
|
|
49
39
|
}
|
|
50
40
|
}
|
|
@@ -56,38 +46,38 @@ function json2env(opt) {
|
|
|
56
46
|
appendToGithubOutput(json, prefix);
|
|
57
47
|
}
|
|
58
48
|
}
|
|
59
|
-
function appendToBashEnv(obj, prefix = '') {
|
|
49
|
+
export function appendToBashEnv(obj, prefix = '') {
|
|
60
50
|
const { BASH_ENV } = process.env;
|
|
61
51
|
if (BASH_ENV) {
|
|
62
52
|
const data = objectToShellExport(obj, prefix);
|
|
63
|
-
|
|
53
|
+
fs.appendFileSync(BASH_ENV, data);
|
|
64
54
|
console.log(`BASH_ENV appended:\n${data}`);
|
|
65
55
|
}
|
|
66
56
|
}
|
|
67
|
-
function appendToGithubEnv(obj, prefix = '') {
|
|
57
|
+
export function appendToGithubEnv(obj, prefix = '') {
|
|
68
58
|
const { GITHUB_ENV } = process.env;
|
|
69
59
|
if (GITHUB_ENV) {
|
|
70
60
|
const data = objectToGithubActionsEnv(obj, prefix);
|
|
71
|
-
|
|
61
|
+
fs.appendFileSync(GITHUB_ENV, data);
|
|
72
62
|
console.log(`GITHUB_ENV appended:\n${data}`);
|
|
73
63
|
}
|
|
74
64
|
}
|
|
75
|
-
function appendToGithubOutput(obj, prefix = '') {
|
|
65
|
+
export function appendToGithubOutput(obj, prefix = '') {
|
|
76
66
|
const { GITHUB_OUTPUT } = process.env;
|
|
77
67
|
if (GITHUB_OUTPUT) {
|
|
78
68
|
const data = objectToGithubActionsEnv(obj, prefix);
|
|
79
|
-
|
|
69
|
+
fs.appendFileSync(GITHUB_OUTPUT, data);
|
|
80
70
|
console.log(`GITHUB_OUTPUT appended:\n${data}`);
|
|
81
71
|
}
|
|
82
72
|
}
|
|
83
73
|
/**
|
|
84
74
|
* https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary
|
|
85
75
|
*/
|
|
86
|
-
function appendToGithubSummary(...lines) {
|
|
76
|
+
export function appendToGithubSummary(...lines) {
|
|
87
77
|
const { GITHUB_STEP_SUMMARY } = process.env;
|
|
88
78
|
if (GITHUB_STEP_SUMMARY) {
|
|
89
79
|
const str = lines.join('\n') + '\n';
|
|
90
|
-
|
|
80
|
+
fs.appendFileSync(GITHUB_STEP_SUMMARY, str);
|
|
91
81
|
console.log(`GITHUB_STEP_SUMMARY appended:\n${str}`);
|
|
92
82
|
}
|
|
93
83
|
}
|
|
@@ -105,7 +95,7 @@ function appendToGithubSummary(...lines) {
|
|
|
105
95
|
* Quotes are important, otherwise it'll break on e.g space character in the value.
|
|
106
96
|
* Includes trailing newline for composability.
|
|
107
97
|
*/
|
|
108
|
-
function objectToShellExport(obj, prefix = '') {
|
|
98
|
+
export function objectToShellExport(obj, prefix = '') {
|
|
109
99
|
if (!Object.keys(obj).length)
|
|
110
100
|
return '';
|
|
111
101
|
return (Object.entries(obj)
|
|
@@ -133,7 +123,7 @@ function objectToShellExport(obj, prefix = '') {
|
|
|
133
123
|
*
|
|
134
124
|
* UPD: Quoted values behave inconsistently, so we're trying to NOT quote now, and-see-what-happens.
|
|
135
125
|
*/
|
|
136
|
-
function objectToGithubActionsEnv(obj, prefix = '') {
|
|
126
|
+
export function objectToGithubActionsEnv(obj, prefix = '') {
|
|
137
127
|
if (!Object.keys(obj).length)
|
|
138
128
|
return '';
|
|
139
129
|
return (Object.entries(obj)
|
package/dist/fs/kpy.js
CHANGED
|
@@ -1,64 +1,59 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
8
|
-
const colors_1 = require("../colors/colors");
|
|
9
|
-
const index_1 = require("../index");
|
|
10
|
-
async function kpy(opt) {
|
|
11
|
-
const started = js_lib_1.localTime.nowUnixMillis();
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib';
|
|
3
|
+
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js';
|
|
4
|
+
import { fastGlob, fs2 } from '../index.js';
|
|
5
|
+
export async function kpy(opt) {
|
|
6
|
+
const started = localTime.nowUnixMillis();
|
|
12
7
|
kpyPrepare(opt);
|
|
13
|
-
const filenames = await
|
|
8
|
+
const filenames = await fastGlob(opt.inputPatterns, {
|
|
14
9
|
cwd: opt.baseDir,
|
|
15
10
|
dot: opt.dotfiles,
|
|
16
11
|
});
|
|
17
12
|
kpyLogFilenames(opt, filenames);
|
|
18
13
|
const overwrite = !opt.noOverwrite;
|
|
19
14
|
await Promise.all(filenames.map(async (filename) => {
|
|
20
|
-
const basename =
|
|
21
|
-
const srcFilename =
|
|
22
|
-
const destFilename =
|
|
15
|
+
const basename = path.basename(filename);
|
|
16
|
+
const srcFilename = path.resolve(opt.baseDir, filename);
|
|
17
|
+
const destFilename = path.resolve(opt.outputDir, opt.flat ? basename : filename);
|
|
23
18
|
if (!opt.dry) {
|
|
24
19
|
if (opt.move) {
|
|
25
|
-
await
|
|
20
|
+
await fs2.movePathAsync(srcFilename, destFilename, {
|
|
26
21
|
force: overwrite,
|
|
27
22
|
});
|
|
28
23
|
}
|
|
29
24
|
else {
|
|
30
|
-
await
|
|
25
|
+
await fs2.copyPathAsync(srcFilename, destFilename, { force: overwrite });
|
|
31
26
|
}
|
|
32
27
|
}
|
|
33
28
|
if (opt.verbose) {
|
|
34
|
-
console.log(
|
|
29
|
+
console.log(grey(` ${filename}`));
|
|
35
30
|
}
|
|
36
31
|
}));
|
|
37
32
|
kpyLogResult(opt, filenames, started);
|
|
38
33
|
}
|
|
39
|
-
function kpySync(opt) {
|
|
40
|
-
const started =
|
|
34
|
+
export function kpySync(opt) {
|
|
35
|
+
const started = localTime.nowUnixMillis();
|
|
41
36
|
kpyPrepare(opt);
|
|
42
|
-
const filenames =
|
|
37
|
+
const filenames = fastGlob.sync(opt.inputPatterns, {
|
|
43
38
|
cwd: opt.baseDir,
|
|
44
39
|
dot: opt.dotfiles,
|
|
45
40
|
});
|
|
46
41
|
kpyLogFilenames(opt, filenames);
|
|
47
42
|
const overwrite = !opt.noOverwrite;
|
|
48
43
|
filenames.forEach(filename => {
|
|
49
|
-
const basename =
|
|
50
|
-
const srcFilename =
|
|
51
|
-
const destFilename =
|
|
44
|
+
const basename = path.basename(filename);
|
|
45
|
+
const srcFilename = path.resolve(opt.baseDir, filename);
|
|
46
|
+
const destFilename = path.resolve(opt.outputDir, opt.flat ? basename : filename);
|
|
52
47
|
if (!opt.dry) {
|
|
53
48
|
if (opt.move) {
|
|
54
|
-
|
|
49
|
+
fs2.movePath(srcFilename, destFilename, { force: overwrite });
|
|
55
50
|
}
|
|
56
51
|
else {
|
|
57
|
-
|
|
52
|
+
fs2.copyPath(srcFilename, destFilename, { force: overwrite });
|
|
58
53
|
}
|
|
59
54
|
}
|
|
60
55
|
if (opt.verbose) {
|
|
61
|
-
console.log(
|
|
56
|
+
console.log(grey(` ${filename}`));
|
|
62
57
|
}
|
|
63
58
|
});
|
|
64
59
|
kpyLogResult(opt, filenames, started);
|
|
@@ -70,17 +65,17 @@ function kpyPrepare(opt) {
|
|
|
70
65
|
// default to cwd
|
|
71
66
|
opt.baseDir ||= '.';
|
|
72
67
|
opt.outputDir ||= '.';
|
|
73
|
-
if (!
|
|
74
|
-
console.log(`kpy: baseDir doesn't exist: ${
|
|
68
|
+
if (!fs2.pathExists(opt.baseDir)) {
|
|
69
|
+
console.log(`kpy: baseDir doesn't exist: ${boldWhite(opt.baseDir)}`);
|
|
75
70
|
return;
|
|
76
71
|
}
|
|
77
|
-
|
|
72
|
+
fs2.ensureDir(opt.outputDir);
|
|
78
73
|
// Expand directories (ex-globby feature), experimental!
|
|
79
74
|
const extraPatterns = [];
|
|
80
75
|
for (const pattern of opt.inputPatterns) {
|
|
81
76
|
if (pattern.includes('*'))
|
|
82
77
|
continue;
|
|
83
|
-
if (
|
|
78
|
+
if (fs2.isDirectory(path.resolve(opt.baseDir, pattern))) {
|
|
84
79
|
extraPatterns.push(`${pattern}/**`);
|
|
85
80
|
}
|
|
86
81
|
}
|
|
@@ -93,10 +88,10 @@ function kpyLogFilenames(opt, filenames) {
|
|
|
93
88
|
if (opt.silent)
|
|
94
89
|
return;
|
|
95
90
|
// console.log({filenames})
|
|
96
|
-
console.log(`Will ${opt.move ? 'move' : 'copy'} ${
|
|
91
|
+
console.log(`Will ${opt.move ? 'move' : 'copy'} ${yellow(filenames.length)} files from ${dimGrey(opt.baseDir)} to ${dimGrey(opt.outputDir)} (${dimGrey(opt.inputPatterns.join(' '))})`);
|
|
97
92
|
}
|
|
98
93
|
function kpyLogResult(opt, filenames, started) {
|
|
99
94
|
if (opt.silent || filenames.length === 0)
|
|
100
95
|
return;
|
|
101
|
-
console.log(`${opt.move ? 'Moved' : 'Copied'} ${
|
|
96
|
+
console.log(`${opt.move ? 'Moved' : 'Copied'} ${yellow(filenames.length)} files to ${dimGrey(opt.outputDir)} ${dimGrey(_since(started))}`);
|
|
102
97
|
}
|