@naturalcycles/nodejs-lib 15.130.0 → 15.131.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fs/fs2.d.ts +9 -9
- package/dist/fs/fs2.js +7 -9
- package/dist/secret/secrets-decrypt.util.js +1 -1
- package/dist/secret/secrets-encrypt.util.js +5 -8
- package/dist/security/crypto.util.d.ts +6 -6
- package/dist/security/hash.util.d.ts +12 -12
- package/dist/security/hash.util.js +12 -4
- package/dist/stream/index.d.ts +0 -1
- package/dist/stream/index.js +0 -1
- package/dist/stream/pipeline.d.ts +0 -5
- package/dist/stream/pipeline.js +0 -8
- package/dist/validation/timezones.js +2 -2
- package/dist/zip/zip2.d.ts +25 -25
- package/dist/zip/zip2.js +2 -2
- package/dist/zip/zipWriter.d.ts +3 -3
- package/dist/zip/zipWriter.js +1 -1
- package/package.json +3 -3
- package/src/fs/fs2.ts +13 -16
- package/src/secret/secrets-decrypt.util.ts +1 -1
- package/src/secret/secrets-encrypt.util.ts +5 -8
- package/src/security/crypto.util.ts +19 -7
- package/src/security/hash.util.ts +25 -16
- package/src/stream/index.ts +0 -1
- package/src/stream/pipeline.ts +0 -10
- package/src/validation/timezones.ts +2 -2
- package/src/zip/zip2.ts +30 -28
- package/src/zip/zipWriter.ts +9 -5
- package/dist/stream/transform/transformWarmup.d.ts +0 -24
- package/dist/stream/transform/transformWarmup.js +0 -79
- package/src/stream/transform/transformWarmup.ts +0 -105
package/dist/fs/fs2.d.ts
CHANGED
|
@@ -18,24 +18,24 @@ declare class FS2 {
|
|
|
18
18
|
* Convenience wrapper that defaults to utf-8 string output.
|
|
19
19
|
*/
|
|
20
20
|
readTextAsync(filePath: string): Promise<string>;
|
|
21
|
-
readBuffer(filePath: string): Buffer
|
|
22
|
-
readBufferAsync(filePath: string): Promise<Buffer
|
|
21
|
+
readBuffer(filePath: string): Buffer<ArrayBuffer>;
|
|
22
|
+
readBufferAsync(filePath: string): Promise<Buffer<ArrayBuffer>>;
|
|
23
23
|
readJson<T = unknown>(filePath: string): T;
|
|
24
24
|
readJsonAsync<T = unknown>(filePath: string): Promise<T>;
|
|
25
|
-
writeFile(filePath: string, data: string |
|
|
26
|
-
writeFileAsync(filePath: string, data: string |
|
|
25
|
+
writeFile(filePath: string, data: string | Uint8Array, opt?: WriteFileOptions): void;
|
|
26
|
+
writeFileAsync(filePath: string, data: string | Uint8Array, opt?: WriteFileOptions): Promise<void>;
|
|
27
27
|
writePrettyJson(filePath: string, data: any): void;
|
|
28
28
|
writeJson(filePath: string, data: any, opt?: JsonOptions): void;
|
|
29
29
|
writePrettyJsonAsync(filePath: string, data: any): Promise<void>;
|
|
30
30
|
writeJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
|
|
31
|
-
appendFile(filePath: string, data: string |
|
|
32
|
-
appendFileAsync(filePath: string, data: string |
|
|
31
|
+
appendFile(filePath: string, data: string | Uint8Array): void;
|
|
32
|
+
appendFileAsync(filePath: string, data: string | Uint8Array): Promise<void>;
|
|
33
33
|
outputPrettyJson(filePath: string, data: any): void;
|
|
34
34
|
outputJson(filePath: string, data: any, opt?: JsonOptions): void;
|
|
35
|
-
|
|
35
|
+
outputPrettyJsonAsync(filePath: string, data: any): Promise<void>;
|
|
36
36
|
outputJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void>;
|
|
37
|
-
outputFile(filePath: string, data: string |
|
|
38
|
-
outputFileAsync(filePath: string, data: string |
|
|
37
|
+
outputFile(filePath: string, data: string | Uint8Array): void;
|
|
38
|
+
outputFileAsync(filePath: string, data: string | Uint8Array): Promise<void>;
|
|
39
39
|
pathExists(filePath: string): boolean;
|
|
40
40
|
pathExistsAsync(filePath: string): Promise<boolean>;
|
|
41
41
|
requireFileToExist(filePath: string): void;
|
package/dist/fs/fs2.js
CHANGED
|
@@ -88,7 +88,7 @@ class FS2 {
|
|
|
88
88
|
const str = stringify(data, opt);
|
|
89
89
|
this.outputFile(filePath, str);
|
|
90
90
|
}
|
|
91
|
-
async
|
|
91
|
+
async outputPrettyJsonAsync(filePath, data) {
|
|
92
92
|
await this.outputJsonAsync(filePath, data, prettyJsonOptions);
|
|
93
93
|
}
|
|
94
94
|
async outputJsonAsync(filePath, data, opt) {
|
|
@@ -156,11 +156,9 @@ class FS2 {
|
|
|
156
156
|
}
|
|
157
157
|
catch (err) {
|
|
158
158
|
// If the stat call above failed because the directory doesn't exist, create it
|
|
159
|
-
if (err?.code
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
}
|
|
163
|
-
throw err;
|
|
159
|
+
if (err?.code !== 'ENOENT')
|
|
160
|
+
throw err;
|
|
161
|
+
this.ensureDir(dir);
|
|
164
162
|
}
|
|
165
163
|
fs.writeFileSync(filePath, '');
|
|
166
164
|
}
|
|
@@ -182,9 +180,9 @@ class FS2 {
|
|
|
182
180
|
}
|
|
183
181
|
catch (err) {
|
|
184
182
|
// If the stat call above failed because the directory doesn't exist, create it
|
|
185
|
-
if (err?.code
|
|
186
|
-
|
|
187
|
-
|
|
183
|
+
if (err?.code !== 'ENOENT')
|
|
184
|
+
throw err;
|
|
185
|
+
await this.ensureDirAsync(dir);
|
|
188
186
|
}
|
|
189
187
|
await fsp.writeFile(filePath, '');
|
|
190
188
|
}
|
|
@@ -24,7 +24,7 @@ export function secretsDecrypt(dir, file, encKeyBuffer, del = false, jsonMode =
|
|
|
24
24
|
_assert(!filename.endsWith('.plain.json'), `${path.basename(filename)} MUST NOT end with '.plain.json'`);
|
|
25
25
|
plainFilename = filename.replace('.json', '.plain.json');
|
|
26
26
|
const json = decryptObject(fs2.readJson(filename), encKeyBuffer);
|
|
27
|
-
fs2.
|
|
27
|
+
fs2.writePrettyJson(plainFilename, json);
|
|
28
28
|
}
|
|
29
29
|
else {
|
|
30
30
|
const enc = fs.readFileSync(filename);
|
|
@@ -10,20 +10,17 @@ import { encryptObject, encryptRandomIVBuffer } from '../security/crypto.util.js
|
|
|
10
10
|
* Using provided encKey.
|
|
11
11
|
*/
|
|
12
12
|
export function secretsEncrypt(pattern, file, encKeyBuffer, del = false, jsonMode = false) {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
`!**/*.enc`, // excluding already encoded
|
|
18
|
-
];
|
|
19
|
-
const filenames = globSync(patterns);
|
|
13
|
+
// Already encrypted files are filtered out: a `!**/*.enc` pattern only excludes them under the cwd
|
|
14
|
+
const filenames = file
|
|
15
|
+
? globSync([file])
|
|
16
|
+
: globSync(pattern).filter(filename => !filename.endsWith('.enc'));
|
|
20
17
|
let encFilename;
|
|
21
18
|
filenames.forEach(filename => {
|
|
22
19
|
if (jsonMode) {
|
|
23
20
|
_assert(filename.endsWith('.plain.json'), `${path.basename(filename)} MUST end with '.plain.json'`);
|
|
24
21
|
encFilename = filename.replace('.plain', '');
|
|
25
22
|
const json = encryptObject(fs2.readJson(filename), encKeyBuffer);
|
|
26
|
-
fs2.
|
|
23
|
+
fs2.writePrettyJson(encFilename, json);
|
|
27
24
|
}
|
|
28
25
|
else {
|
|
29
26
|
const plain = fs.readFileSync(filename);
|
|
@@ -2,35 +2,35 @@ import type { Base64String, StringMap } from '@naturalcycles/js-lib/types';
|
|
|
2
2
|
/**
|
|
3
3
|
* Using aes-256-cbc.
|
|
4
4
|
*/
|
|
5
|
-
export declare function encryptRandomIVBuffer(input:
|
|
5
|
+
export declare function encryptRandomIVBuffer(input: Uint8Array, secretKeyBuffer: Uint8Array): Buffer<ArrayBuffer>;
|
|
6
6
|
/**
|
|
7
7
|
* Using aes-256-cbc.
|
|
8
8
|
*/
|
|
9
|
-
export declare function decryptRandomIVBuffer(input:
|
|
9
|
+
export declare function decryptRandomIVBuffer(input: Uint8Array, secretKeyBuffer: Uint8Array): Buffer<ArrayBuffer>;
|
|
10
10
|
/**
|
|
11
11
|
* Decrypts all object values (base64 strings).
|
|
12
12
|
* Returns object with decrypted values (utf8 strings).
|
|
13
13
|
*/
|
|
14
|
-
export declare function decryptObject(obj: StringMap<Base64String>, secretKeyBuffer:
|
|
14
|
+
export declare function decryptObject(obj: StringMap<Base64String>, secretKeyBuffer: Uint8Array): StringMap;
|
|
15
15
|
/**
|
|
16
16
|
* Encrypts all object values (utf8 strings).
|
|
17
17
|
* Returns object with encrypted values (base64 strings).
|
|
18
18
|
*/
|
|
19
|
-
export declare function encryptObject(obj: StringMap, secretKeyBuffer:
|
|
19
|
+
export declare function encryptObject(obj: StringMap, secretKeyBuffer: Uint8Array): StringMap<Base64String>;
|
|
20
20
|
/**
|
|
21
21
|
* Using aes-256-cbc.
|
|
22
22
|
*
|
|
23
23
|
* Input is base64 string.
|
|
24
24
|
* Output is utf8 string.
|
|
25
25
|
*/
|
|
26
|
-
export declare function decryptString(str: Base64String, secretKeyBuffer:
|
|
26
|
+
export declare function decryptString(str: Base64String, secretKeyBuffer: Uint8Array): string;
|
|
27
27
|
/**
|
|
28
28
|
* Using aes-256-cbc.
|
|
29
29
|
*
|
|
30
30
|
* Input is utf8 string.
|
|
31
31
|
* Output is base64 string.
|
|
32
32
|
*/
|
|
33
|
-
export declare function encryptString(str: string, secretKeyBuffer:
|
|
33
|
+
export declare function encryptString(str: string, secretKeyBuffer: Uint8Array): Base64String;
|
|
34
34
|
/**
|
|
35
35
|
* Wraps `crypto.timingSafeEqual` and allows it to be used with String inputs:
|
|
36
36
|
*
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import type { BinaryToTextEncoding } from 'node:crypto';
|
|
2
2
|
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types';
|
|
3
|
-
export declare function md5(s: string |
|
|
4
|
-
export declare function md5AsBuffer(s: string |
|
|
5
|
-
export declare function sha256(s: string |
|
|
6
|
-
export declare function sha256AsBuffer(s: string |
|
|
7
|
-
export declare function hash(s: string |
|
|
8
|
-
export declare function hashAsBuffer(s: string |
|
|
9
|
-
export declare function base64(s: string |
|
|
10
|
-
export declare function base64Url(s: string |
|
|
3
|
+
export declare function md5(s: string | Uint8Array, outputEncoding?: BinaryToTextEncoding): string;
|
|
4
|
+
export declare function md5AsBuffer(s: string | Uint8Array): Buffer<ArrayBuffer>;
|
|
5
|
+
export declare function sha256(s: string | Uint8Array, outputEncoding?: BinaryToTextEncoding): string;
|
|
6
|
+
export declare function sha256AsBuffer(s: string | Uint8Array): Buffer<ArrayBuffer>;
|
|
7
|
+
export declare function hash(s: string | Uint8Array, algorithm: string, outputEncoding?: BinaryToTextEncoding): string;
|
|
8
|
+
export declare function hashAsBuffer(s: string | Uint8Array, algorithm: string): Buffer<ArrayBuffer>;
|
|
9
|
+
export declare function base64(s: string | Uint8Array): Base64String;
|
|
10
|
+
export declare function base64Url(s: string | Uint8Array): Base64UrlString;
|
|
11
11
|
export declare function base64ToString(strBase64: Base64String): string;
|
|
12
12
|
export declare function base64UrlToString(strBase64Url: Base64UrlString): string;
|
|
13
|
-
export declare function base64ToBuffer(strBase64: Base64String): Buffer
|
|
14
|
-
export declare function base64UrlToBuffer(strBase64Url: Base64UrlString): Buffer
|
|
13
|
+
export declare function base64ToBuffer(strBase64: Base64String): Buffer<ArrayBuffer>;
|
|
14
|
+
export declare function base64UrlToBuffer(strBase64Url: Base64UrlString): Buffer<ArrayBuffer>;
|
|
15
15
|
export declare function stringToBase64(s: string): Base64String;
|
|
16
16
|
export declare function stringToBase64Url(s: string): Base64UrlString;
|
|
17
|
-
export declare function bufferToBase64(b:
|
|
18
|
-
export declare function bufferToBase64Url(b:
|
|
17
|
+
export declare function bufferToBase64(b: Uint8Array): Base64String;
|
|
18
|
+
export declare function bufferToBase64Url(b: Uint8Array): Base64UrlString;
|
|
@@ -18,10 +18,14 @@ export function hashAsBuffer(s, algorithm) {
|
|
|
18
18
|
return cryptoHash(algorithm, s, 'buffer');
|
|
19
19
|
}
|
|
20
20
|
export function base64(s) {
|
|
21
|
-
|
|
21
|
+
if (typeof s === 'string')
|
|
22
|
+
return Buffer.from(s).toString('base64');
|
|
23
|
+
return bufferToBase64(s);
|
|
22
24
|
}
|
|
23
25
|
export function base64Url(s) {
|
|
24
|
-
|
|
26
|
+
if (typeof s === 'string')
|
|
27
|
+
return Buffer.from(s).toString('base64url');
|
|
28
|
+
return bufferToBase64Url(s);
|
|
25
29
|
}
|
|
26
30
|
export function base64ToString(strBase64) {
|
|
27
31
|
return Buffer.from(strBase64, 'base64').toString('utf8');
|
|
@@ -41,9 +45,13 @@ export function stringToBase64(s) {
|
|
|
41
45
|
export function stringToBase64Url(s) {
|
|
42
46
|
return Buffer.from(s, 'utf8').toString('base64url');
|
|
43
47
|
}
|
|
48
|
+
// Buffer#toString is faster than Uint8Array#toBase64 on small inputs (~1.3-1.7x up to ~512 bytes,
|
|
49
|
+
// measured on Node 26), hence the Buffer fast-path. On larger inputs they're on par.
|
|
44
50
|
export function bufferToBase64(b) {
|
|
45
|
-
return b.toString('base64');
|
|
51
|
+
return Buffer.isBuffer(b) ? b.toString('base64') : b.toBase64();
|
|
46
52
|
}
|
|
47
53
|
export function bufferToBase64Url(b) {
|
|
48
|
-
return
|
|
54
|
+
return Buffer.isBuffer(b)
|
|
55
|
+
? b.toString('base64url')
|
|
56
|
+
: b.toBase64({ alphabet: 'base64url', omitPadding: true });
|
|
49
57
|
}
|
package/dist/stream/index.d.ts
CHANGED
|
@@ -24,7 +24,6 @@ export * from './transform/transformSplit.js';
|
|
|
24
24
|
export * from './transform/transformTap.js';
|
|
25
25
|
export * from './transform/transformThrottle.js';
|
|
26
26
|
export * from './transform/transformThrottleByRSS.js';
|
|
27
|
-
export * from './transform/transformWarmup.js';
|
|
28
27
|
export * from './transform/worker/baseWorkerClass.js';
|
|
29
28
|
export * from './transform/worker/transformMultiThreaded.js';
|
|
30
29
|
export type * from './transform/worker/transformMultiThreaded.model.js';
|
package/dist/stream/index.js
CHANGED
|
@@ -23,7 +23,6 @@ export * from './transform/transformSplit.js';
|
|
|
23
23
|
export * from './transform/transformTap.js';
|
|
24
24
|
export * from './transform/transformThrottle.js';
|
|
25
25
|
export * from './transform/transformThrottleByRSS.js';
|
|
26
|
-
export * from './transform/transformWarmup.js';
|
|
27
26
|
export * from './transform/worker/baseWorkerClass.js';
|
|
28
27
|
export * from './transform/worker/transformMultiThreaded.js';
|
|
29
28
|
export * from './writable/writableChunk.js';
|
|
@@ -10,7 +10,6 @@ import type { TransformMapSyncOptions } from './transform/transformMapSync.js';
|
|
|
10
10
|
import type { TransformOffsetOptions } from './transform/transformOffset.js';
|
|
11
11
|
import type { TransformThrottleOptions } from './transform/transformThrottle.js';
|
|
12
12
|
import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js';
|
|
13
|
-
import type { TransformWarmupOptions } from './transform/transformWarmup.js';
|
|
14
13
|
export declare class Pipeline<T = unknown> {
|
|
15
14
|
private readonly source;
|
|
16
15
|
private transforms;
|
|
@@ -72,10 +71,6 @@ export declare class Pipeline<T = unknown> {
|
|
|
72
71
|
tapSync(fn: IndexedMapper<T, void>, opt?: TransformOptions): this;
|
|
73
72
|
throttle(opt: TransformThrottleOptions): this;
|
|
74
73
|
throttleByRSS(opt: TransformThrottleByRSSOptions): this;
|
|
75
|
-
/**
|
|
76
|
-
* @experimental to be removed after transformMap2 is stable
|
|
77
|
-
*/
|
|
78
|
-
warmup(opt: TransformWarmupOptions): this;
|
|
79
74
|
transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO>;
|
|
80
75
|
/**
|
|
81
76
|
* Helper method to add multiple transforms at once.
|
package/dist/stream/pipeline.js
CHANGED
|
@@ -25,7 +25,6 @@ import { transformSplitOnNewline } from './transform/transformSplit.js';
|
|
|
25
25
|
import { transformTap, transformTapSync } from './transform/transformTap.js';
|
|
26
26
|
import { transformThrottle } from './transform/transformThrottle.js';
|
|
27
27
|
import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js';
|
|
28
|
-
import { transformWarmup } from './transform/transformWarmup.js';
|
|
29
28
|
import { writablePushToArray } from './writable/writablePushToArray.js';
|
|
30
29
|
import { writableVoid } from './writable/writableVoid.js';
|
|
31
30
|
export class Pipeline {
|
|
@@ -180,13 +179,6 @@ export class Pipeline {
|
|
|
180
179
|
this.transforms.push(transformThrottleByRSS(opt));
|
|
181
180
|
return this;
|
|
182
181
|
}
|
|
183
|
-
/**
|
|
184
|
-
* @experimental to be removed after transformMap2 is stable
|
|
185
|
-
*/
|
|
186
|
-
warmup(opt) {
|
|
187
|
-
this.transforms.push(transformWarmup(opt));
|
|
188
|
-
return this;
|
|
189
|
-
}
|
|
190
182
|
transform(transform) {
|
|
191
183
|
this.transforms.push(transform);
|
|
192
184
|
return this;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { _uniq } from '@naturalcycles/js-lib/array';
|
|
2
|
+
import { Intl2 } from '@naturalcycles/js-lib/intl';
|
|
2
3
|
// List extracted from Wikipedia (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
|
|
3
4
|
// with the "Factory" value removed as its not working with our toolings
|
|
4
5
|
const TIMEZONES_FROM_WIKI = [
|
|
@@ -600,8 +601,7 @@ const TIMEZONES_FROM_WIKI = [
|
|
|
600
601
|
'WET',
|
|
601
602
|
'Zulu',
|
|
602
603
|
];
|
|
603
|
-
|
|
604
|
-
const TIMEZONES_FROM_JS = Intl.supportedValuesOf('timeZone');
|
|
604
|
+
const TIMEZONES_FROM_JS = Intl2.getSupportedTimezones();
|
|
605
605
|
/**
|
|
606
606
|
* A complicated merge of timezones from the underlying Javascript engine
|
|
607
607
|
* and a list exported from Wikipedia.
|
package/dist/zip/zip2.d.ts
CHANGED
|
@@ -4,59 +4,59 @@ import type { ZipEntry } from './zipReader.js';
|
|
|
4
4
|
import { ZipWriter } from './zipWriter.js';
|
|
5
5
|
import type { ZipFileEntry, ZipPathsOptions } from './zipWriter.js';
|
|
6
6
|
declare class Zip2 {
|
|
7
|
-
decompressZstdOrInflateToString(buf:
|
|
8
|
-
decompressZstdOrInflateToStringSync(buf:
|
|
7
|
+
decompressZstdOrInflateToString(buf: Uint8Array): Promise<string>;
|
|
8
|
+
decompressZstdOrInflateToStringSync(buf: Uint8Array): string;
|
|
9
9
|
/**
|
|
10
10
|
* Detects if Buffer is zstd-compressed.
|
|
11
11
|
* Otherwise attempts to Inflate.
|
|
12
12
|
*/
|
|
13
|
-
decompressZstdOrInflate(buf:
|
|
14
|
-
decompressZstdOrInflateSync(buf:
|
|
13
|
+
decompressZstdOrInflate(buf: Uint8Array): Promise<Buffer<ArrayBuffer>>;
|
|
14
|
+
decompressZstdOrInflateSync(buf: Uint8Array): Buffer<ArrayBuffer>;
|
|
15
15
|
/**
|
|
16
16
|
* deflateBuffer uses `deflate`.
|
|
17
17
|
* It's 9 bytes shorter than `gzip`.
|
|
18
18
|
*/
|
|
19
|
-
deflate(input: string |
|
|
19
|
+
deflate(input: string | Uint8Array, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
|
|
20
20
|
/**
|
|
21
21
|
* deflateSync uses `deflate`.
|
|
22
22
|
* It's 9 bytes shorter than `gzip`.
|
|
23
23
|
*/
|
|
24
|
-
deflateSync(input: string |
|
|
25
|
-
inflate(buf:
|
|
26
|
-
inflateSync(buf:
|
|
27
|
-
inflateToString(buf:
|
|
28
|
-
inflateToStringSync(buf:
|
|
24
|
+
deflateSync(input: string | Uint8Array, options?: ZlibOptions): Buffer<ArrayBuffer>;
|
|
25
|
+
inflate(buf: Uint8Array, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
|
|
26
|
+
inflateSync(buf: Uint8Array, options?: ZlibOptions): Buffer<ArrayBuffer>;
|
|
27
|
+
inflateToString(buf: Uint8Array, options?: ZlibOptions): Promise<string>;
|
|
28
|
+
inflateToStringSync(buf: Uint8Array, options?: ZlibOptions): string;
|
|
29
29
|
/**
|
|
30
30
|
* gzipBuffer uses `gzip`
|
|
31
31
|
* It's 9 bytes longer than `deflate`.
|
|
32
32
|
*/
|
|
33
|
-
gzip(input: string |
|
|
33
|
+
gzip(input: string | Uint8Array, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
|
|
34
34
|
/**
|
|
35
35
|
* gzipBuffer uses `gzip`
|
|
36
36
|
* It's 9 bytes longer than `deflate`.
|
|
37
37
|
*/
|
|
38
|
-
gzipSync(input: string |
|
|
39
|
-
gunzip(buf:
|
|
40
|
-
gunzipSync(buf:
|
|
41
|
-
gunzipToString(buf:
|
|
42
|
-
gunzipToStringSync(buf:
|
|
43
|
-
zstdCompress(input:
|
|
38
|
+
gzipSync(input: string | Uint8Array, options?: ZlibOptions): Buffer<ArrayBuffer>;
|
|
39
|
+
gunzip(buf: Uint8Array, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
|
|
40
|
+
gunzipSync(buf: Uint8Array, options?: ZlibOptions): Buffer<ArrayBuffer>;
|
|
41
|
+
gunzipToString(buf: Uint8Array, options?: ZlibOptions): Promise<string>;
|
|
42
|
+
gunzipToStringSync(buf: Uint8Array, options?: ZlibOptions): string;
|
|
43
|
+
zstdCompress(input: Uint8Array | string, level?: Integer, // defaults to 3
|
|
44
44
|
options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
|
|
45
|
-
zstdCompressSync(input:
|
|
45
|
+
zstdCompressSync(input: Uint8Array | string, level?: Integer, // defaults to 3
|
|
46
46
|
options?: ZstdOptions): Buffer<ArrayBuffer>;
|
|
47
47
|
zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
|
|
48
|
-
zstdDecompressToString(input:
|
|
48
|
+
zstdDecompressToString(input: Uint8Array, options?: ZstdOptions): Promise<string>;
|
|
49
49
|
/**
|
|
50
50
|
* Warning! It leaks memory severely. Prefer sync.
|
|
51
51
|
*/
|
|
52
|
-
zstdDecompress(input:
|
|
52
|
+
zstdDecompress(input: Uint8Array, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
|
|
53
53
|
/**
|
|
54
54
|
* Warning! It leaks memory severely. Prefer sync.
|
|
55
55
|
*/
|
|
56
|
-
zstdDecompressToStringSync(input:
|
|
57
|
-
zstdDecompressSync(input:
|
|
58
|
-
isZstdBuffer(input:
|
|
59
|
-
isGzipBuffer(input:
|
|
56
|
+
zstdDecompressToStringSync(input: Uint8Array, options?: ZstdOptions): string;
|
|
57
|
+
zstdDecompressSync(input: Uint8Array, options?: ZstdOptions): Buffer<ArrayBuffer>;
|
|
58
|
+
isZstdBuffer(input: Uint8Array): boolean;
|
|
59
|
+
isGzipBuffer(input: Uint8Array): boolean;
|
|
60
60
|
/**
|
|
61
61
|
* Open a zip file and extract all of its entries into `destDir`, streaming each
|
|
62
62
|
* file to disk. Directories are created as needed.
|
|
@@ -90,7 +90,7 @@ declare class Zip2 {
|
|
|
90
90
|
*
|
|
91
91
|
* For large archives or streamed inputs prefer {@link createZip} / {@link ZipWriter}.
|
|
92
92
|
*/
|
|
93
|
-
createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer
|
|
93
|
+
createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer<ArrayBuffer>>;
|
|
94
94
|
}
|
|
95
95
|
export declare const zip2: Zip2;
|
|
96
96
|
export {};
|
package/dist/zip/zip2.js
CHANGED
|
@@ -122,7 +122,8 @@ class Zip2 {
|
|
|
122
122
|
return zlib.zstdDecompressSync(input, options);
|
|
123
123
|
}
|
|
124
124
|
isZstdBuffer(input) {
|
|
125
|
-
|
|
125
|
+
// zstd magic number 0xFD2FB528, stored little-endian
|
|
126
|
+
return input[0] === 0x28 && input[1] === 0xb5 && input[2] === 0x2f && input[3] === 0xfd;
|
|
126
127
|
}
|
|
127
128
|
isGzipBuffer(input) {
|
|
128
129
|
return input[0] === 0x1f && input[1] === 0x8b;
|
|
@@ -177,4 +178,3 @@ class Zip2 {
|
|
|
177
178
|
}
|
|
178
179
|
}
|
|
179
180
|
export const zip2 = new Zip2();
|
|
180
|
-
const ZSTD_MAGIC_NUMBER = 0xfd2fb528;
|
package/dist/zip/zipWriter.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
|
6
6
|
*
|
|
7
7
|
* Each input path may be a file (added directly) or a directory (walked
|
|
8
8
|
* recursively; all nested files are added). By default an entry's name is its
|
|
9
|
-
* path relative to its input's parent directory, so `
|
|
9
|
+
* path relative to its input's parent directory, so `zipPaths(['./photos'], 'out.zip')`
|
|
10
10
|
* stores entries under `photos/...`. Override the base via {@link ZipPathsOptions.baseDir}.
|
|
11
11
|
*
|
|
12
12
|
* Files discovered by walking a directory are added in deterministic (sorted)
|
|
@@ -25,7 +25,7 @@ export declare function zipPaths(paths: string[], outputZipFilePath: string, opt
|
|
|
25
25
|
*
|
|
26
26
|
* For large archives or streamed inputs prefer {@link createZip} / {@link ZipWriter}.
|
|
27
27
|
*/
|
|
28
|
-
export declare function createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer
|
|
28
|
+
export declare function createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer<ArrayBuffer>>;
|
|
29
29
|
/**
|
|
30
30
|
* Options for {@link zipPaths}.
|
|
31
31
|
*/
|
|
@@ -69,7 +69,7 @@ export declare class ZipWriter implements AsyncDisposable {
|
|
|
69
69
|
* Add an in-memory buffer as a file entry.
|
|
70
70
|
* The CRC-32 and sizes are computed up front and written inline (no data descriptor).
|
|
71
71
|
*/
|
|
72
|
-
addBuffer(data:
|
|
72
|
+
addBuffer(data: Uint8Array, fileName: string, opt?: ZipWriterEntryOptions): Promise<void>;
|
|
73
73
|
/**
|
|
74
74
|
* Add a file from disk, streaming its contents. The entry's mtime and mode
|
|
75
75
|
* default to the file's own (override via `opt`). If `fileName` is omitted,
|
package/dist/zip/zipWriter.js
CHANGED
|
@@ -38,7 +38,7 @@ const deflateRawAsync = promisify(zlib.deflateRaw.bind(zlib));
|
|
|
38
38
|
*
|
|
39
39
|
* Each input path may be a file (added directly) or a directory (walked
|
|
40
40
|
* recursively; all nested files are added). By default an entry's name is its
|
|
41
|
-
* path relative to its input's parent directory, so `
|
|
41
|
+
* path relative to its input's parent directory, so `zipPaths(['./photos'], 'out.zip')`
|
|
42
42
|
* stores entries under `photos/...`. Override the base via {@link ZipPathsOptions.baseDir}.
|
|
43
43
|
*
|
|
44
44
|
* Files discovered by walking a directory are added in deterministic (sorted)
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/nodejs-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "15.
|
|
4
|
+
"version": "15.131.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@naturalcycles/js-lib": "^15",
|
|
7
7
|
"@standard-schema/spec": "^1",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"yaml": "^2"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@naturalcycles/dev-lib": "
|
|
17
|
+
"@naturalcycles/dev-lib": "20.66.0",
|
|
18
18
|
"typescript": "^7"
|
|
19
19
|
},
|
|
20
20
|
"exports": {
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"directory": "packages/nodejs-lib"
|
|
66
66
|
},
|
|
67
67
|
"engines": {
|
|
68
|
-
"node": ">=
|
|
68
|
+
"node": ">=26.12.1"
|
|
69
69
|
},
|
|
70
70
|
"description": "Standard library for Node.js",
|
|
71
71
|
"author": "Natural Cycles Team",
|
package/src/fs/fs2.ts
CHANGED
|
@@ -47,11 +47,11 @@ class FS2 {
|
|
|
47
47
|
return await fsp.readFile(filePath, 'utf8')
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
readBuffer(filePath: string): Buffer {
|
|
50
|
+
readBuffer(filePath: string): Buffer<ArrayBuffer> {
|
|
51
51
|
return fs.readFileSync(filePath)
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
async readBufferAsync(filePath: string): Promise<Buffer
|
|
54
|
+
async readBufferAsync(filePath: string): Promise<Buffer<ArrayBuffer>> {
|
|
55
55
|
return await fsp.readFile(filePath)
|
|
56
56
|
}
|
|
57
57
|
|
|
@@ -65,13 +65,13 @@ class FS2 {
|
|
|
65
65
|
return _jsonParse(str)
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
writeFile(filePath: string, data: string |
|
|
68
|
+
writeFile(filePath: string, data: string | Uint8Array, opt?: WriteFileOptions): void {
|
|
69
69
|
fs.writeFileSync(filePath, data, opt)
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
async writeFileAsync(
|
|
73
73
|
filePath: string,
|
|
74
|
-
data: string |
|
|
74
|
+
data: string | Uint8Array,
|
|
75
75
|
opt?: WriteFileOptions,
|
|
76
76
|
): Promise<void> {
|
|
77
77
|
await fsp.writeFile(filePath, data, opt)
|
|
@@ -95,11 +95,11 @@ class FS2 {
|
|
|
95
95
|
await fsp.writeFile(filePath, str)
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
appendFile(filePath: string, data: string |
|
|
98
|
+
appendFile(filePath: string, data: string | Uint8Array): void {
|
|
99
99
|
fs.appendFileSync(filePath, data)
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
-
async appendFileAsync(filePath: string, data: string |
|
|
102
|
+
async appendFileAsync(filePath: string, data: string | Uint8Array): Promise<void> {
|
|
103
103
|
await fsp.appendFile(filePath, data)
|
|
104
104
|
}
|
|
105
105
|
|
|
@@ -112,7 +112,7 @@ class FS2 {
|
|
|
112
112
|
this.outputFile(filePath, str)
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
async
|
|
115
|
+
async outputPrettyJsonAsync(filePath: string, data: any): Promise<void> {
|
|
116
116
|
await this.outputJsonAsync(filePath, data, prettyJsonOptions)
|
|
117
117
|
}
|
|
118
118
|
|
|
@@ -121,7 +121,7 @@ class FS2 {
|
|
|
121
121
|
await this.outputFileAsync(filePath, str)
|
|
122
122
|
}
|
|
123
123
|
|
|
124
|
-
outputFile(filePath: string, data: string |
|
|
124
|
+
outputFile(filePath: string, data: string | Uint8Array): void {
|
|
125
125
|
const dirPath = path.dirname(filePath)
|
|
126
126
|
if (!fs.existsSync(dirPath)) {
|
|
127
127
|
this.ensureDir(dirPath)
|
|
@@ -130,7 +130,7 @@ class FS2 {
|
|
|
130
130
|
fs.writeFileSync(filePath, data)
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
-
async outputFileAsync(filePath: string, data: string |
|
|
133
|
+
async outputFileAsync(filePath: string, data: string | Uint8Array): Promise<void> {
|
|
134
134
|
const dirPath = path.dirname(filePath)
|
|
135
135
|
if (!(await this.pathExistsAsync(dirPath))) {
|
|
136
136
|
await this.ensureDirAsync(dirPath)
|
|
@@ -188,11 +188,8 @@ class FS2 {
|
|
|
188
188
|
}
|
|
189
189
|
} catch (err) {
|
|
190
190
|
// If the stat call above failed because the directory doesn't exist, create it
|
|
191
|
-
if ((err as any)?.code
|
|
192
|
-
|
|
193
|
-
return
|
|
194
|
-
}
|
|
195
|
-
throw err
|
|
191
|
+
if ((err as any)?.code !== 'ENOENT') throw err
|
|
192
|
+
this.ensureDir(dir)
|
|
196
193
|
}
|
|
197
194
|
|
|
198
195
|
fs.writeFileSync(filePath, '')
|
|
@@ -214,8 +211,8 @@ class FS2 {
|
|
|
214
211
|
}
|
|
215
212
|
} catch (err) {
|
|
216
213
|
// If the stat call above failed because the directory doesn't exist, create it
|
|
217
|
-
if ((err as any)?.code
|
|
218
|
-
|
|
214
|
+
if ((err as any)?.code !== 'ENOENT') throw err
|
|
215
|
+
await this.ensureDirAsync(dir)
|
|
219
216
|
}
|
|
220
217
|
|
|
221
218
|
await fsp.writeFile(filePath, '')
|
|
@@ -49,7 +49,7 @@ export function secretsDecrypt(
|
|
|
49
49
|
|
|
50
50
|
const json = decryptObject(fs2.readJson(filename), encKeyBuffer)
|
|
51
51
|
|
|
52
|
-
fs2.
|
|
52
|
+
fs2.writePrettyJson(plainFilename, json)
|
|
53
53
|
} else {
|
|
54
54
|
const enc = fs.readFileSync(filename)
|
|
55
55
|
const plain = decryptRandomIVBuffer(enc, encKeyBuffer)
|
|
@@ -25,13 +25,10 @@ export function secretsEncrypt(
|
|
|
25
25
|
del = false,
|
|
26
26
|
jsonMode = false,
|
|
27
27
|
): void {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
`!**/*.enc`, // excluding already encoded
|
|
33
|
-
]
|
|
34
|
-
const filenames = globSync(patterns)
|
|
28
|
+
// Already encrypted files are filtered out: a `!**/*.enc` pattern only excludes them under the cwd
|
|
29
|
+
const filenames = file
|
|
30
|
+
? globSync([file])
|
|
31
|
+
: globSync(pattern).filter(filename => !filename.endsWith('.enc'))
|
|
35
32
|
let encFilename: string
|
|
36
33
|
|
|
37
34
|
filenames.forEach(filename => {
|
|
@@ -44,7 +41,7 @@ export function secretsEncrypt(
|
|
|
44
41
|
|
|
45
42
|
const json = encryptObject(fs2.readJson(filename), encKeyBuffer)
|
|
46
43
|
|
|
47
|
-
fs2.
|
|
44
|
+
fs2.writePrettyJson(encFilename, json)
|
|
48
45
|
} else {
|
|
49
46
|
const plain = fs.readFileSync(filename)
|
|
50
47
|
const enc = encryptRandomIVBuffer(plain, encKeyBuffer)
|
|
@@ -8,7 +8,10 @@ const algorithm = 'aes-256-cbc'
|
|
|
8
8
|
/**
|
|
9
9
|
* Using aes-256-cbc.
|
|
10
10
|
*/
|
|
11
|
-
export function encryptRandomIVBuffer(
|
|
11
|
+
export function encryptRandomIVBuffer(
|
|
12
|
+
input: Uint8Array,
|
|
13
|
+
secretKeyBuffer: Uint8Array,
|
|
14
|
+
): Buffer<ArrayBuffer> {
|
|
12
15
|
// sha256 to match aes-256 key length
|
|
13
16
|
const key = sha256AsBuffer(secretKeyBuffer)
|
|
14
17
|
|
|
@@ -22,7 +25,10 @@ export function encryptRandomIVBuffer(input: Buffer, secretKeyBuffer: Buffer): B
|
|
|
22
25
|
/**
|
|
23
26
|
* Using aes-256-cbc.
|
|
24
27
|
*/
|
|
25
|
-
export function decryptRandomIVBuffer(
|
|
28
|
+
export function decryptRandomIVBuffer(
|
|
29
|
+
input: Uint8Array,
|
|
30
|
+
secretKeyBuffer: Uint8Array,
|
|
31
|
+
): Buffer<ArrayBuffer> {
|
|
26
32
|
// sha256 to match aes-256 key length
|
|
27
33
|
const key = sha256AsBuffer(secretKeyBuffer)
|
|
28
34
|
|
|
@@ -39,7 +45,10 @@ export function decryptRandomIVBuffer(input: Buffer, secretKeyBuffer: Buffer): B
|
|
|
39
45
|
* Decrypts all object values (base64 strings).
|
|
40
46
|
* Returns object with decrypted values (utf8 strings).
|
|
41
47
|
*/
|
|
42
|
-
export function decryptObject(
|
|
48
|
+
export function decryptObject(
|
|
49
|
+
obj: StringMap<Base64String>,
|
|
50
|
+
secretKeyBuffer: Uint8Array,
|
|
51
|
+
): StringMap {
|
|
43
52
|
const { key, iv } = getCryptoParams(secretKeyBuffer)
|
|
44
53
|
|
|
45
54
|
const r: StringMap = {}
|
|
@@ -54,7 +63,10 @@ export function decryptObject(obj: StringMap<Base64String>, secretKeyBuffer: Buf
|
|
|
54
63
|
* Encrypts all object values (utf8 strings).
|
|
55
64
|
* Returns object with encrypted values (base64 strings).
|
|
56
65
|
*/
|
|
57
|
-
export function encryptObject(
|
|
66
|
+
export function encryptObject(
|
|
67
|
+
obj: StringMap,
|
|
68
|
+
secretKeyBuffer: Uint8Array,
|
|
69
|
+
): StringMap<Base64String> {
|
|
58
70
|
const { key, iv } = getCryptoParams(secretKeyBuffer)
|
|
59
71
|
|
|
60
72
|
const r: StringMap = {}
|
|
@@ -71,7 +83,7 @@ export function encryptObject(obj: StringMap, secretKeyBuffer: Buffer): StringMa
|
|
|
71
83
|
* Input is base64 string.
|
|
72
84
|
* Output is utf8 string.
|
|
73
85
|
*/
|
|
74
|
-
export function decryptString(str: Base64String, secretKeyBuffer:
|
|
86
|
+
export function decryptString(str: Base64String, secretKeyBuffer: Uint8Array): string {
|
|
75
87
|
const { key, iv } = getCryptoParams(secretKeyBuffer)
|
|
76
88
|
const decipher = crypto.createDecipheriv(algorithm, key, iv)
|
|
77
89
|
return decipher.update(str, 'base64', 'utf8') + decipher.final('utf8')
|
|
@@ -83,13 +95,13 @@ export function decryptString(str: Base64String, secretKeyBuffer: Buffer): strin
|
|
|
83
95
|
* Input is utf8 string.
|
|
84
96
|
* Output is base64 string.
|
|
85
97
|
*/
|
|
86
|
-
export function encryptString(str: string, secretKeyBuffer:
|
|
98
|
+
export function encryptString(str: string, secretKeyBuffer: Uint8Array): Base64String {
|
|
87
99
|
const { key, iv } = getCryptoParams(secretKeyBuffer)
|
|
88
100
|
const cipher = crypto.createCipheriv(algorithm, key, iv)
|
|
89
101
|
return cipher.update(str, 'utf8', 'base64') + cipher.final('base64')
|
|
90
102
|
}
|
|
91
103
|
|
|
92
|
-
function getCryptoParams(secretKeyBuffer:
|
|
104
|
+
function getCryptoParams(secretKeyBuffer: Uint8Array): { key: Buffer; iv: Buffer } {
|
|
93
105
|
const key = sha256AsBuffer(secretKeyBuffer)
|
|
94
106
|
const iv = md5AsBuffer(Buffer.concat([secretKeyBuffer, key]))
|
|
95
107
|
return { key, iv }
|
|
@@ -2,40 +2,45 @@ import type { BinaryToTextEncoding } from 'node:crypto'
|
|
|
2
2
|
import { hash as cryptoHash } from 'node:crypto'
|
|
3
3
|
import type { Base64String, Base64UrlString } from '@naturalcycles/js-lib/types'
|
|
4
4
|
|
|
5
|
-
export function md5(s: string |
|
|
5
|
+
export function md5(s: string | Uint8Array, outputEncoding: BinaryToTextEncoding = 'hex'): string {
|
|
6
6
|
return hash(s, 'md5', outputEncoding)
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
export function md5AsBuffer(s: string |
|
|
9
|
+
export function md5AsBuffer(s: string | Uint8Array): Buffer<ArrayBuffer> {
|
|
10
10
|
return hashAsBuffer(s, 'md5')
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
export function sha256(
|
|
13
|
+
export function sha256(
|
|
14
|
+
s: string | Uint8Array,
|
|
15
|
+
outputEncoding: BinaryToTextEncoding = 'hex',
|
|
16
|
+
): string {
|
|
14
17
|
return hash(s, 'sha256', outputEncoding)
|
|
15
18
|
}
|
|
16
19
|
|
|
17
|
-
export function sha256AsBuffer(s: string |
|
|
20
|
+
export function sha256AsBuffer(s: string | Uint8Array): Buffer<ArrayBuffer> {
|
|
18
21
|
return hashAsBuffer(s, 'sha256')
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
export function hash(
|
|
22
|
-
s: string |
|
|
25
|
+
s: string | Uint8Array,
|
|
23
26
|
algorithm: string,
|
|
24
27
|
outputEncoding: BinaryToTextEncoding = 'hex',
|
|
25
28
|
): string {
|
|
26
29
|
return cryptoHash(algorithm, s, outputEncoding)
|
|
27
30
|
}
|
|
28
31
|
|
|
29
|
-
export function hashAsBuffer(s: string |
|
|
32
|
+
export function hashAsBuffer(s: string | Uint8Array, algorithm: string): Buffer<ArrayBuffer> {
|
|
30
33
|
return cryptoHash(algorithm, s, 'buffer')
|
|
31
34
|
}
|
|
32
35
|
|
|
33
|
-
export function base64(s: string |
|
|
34
|
-
|
|
36
|
+
export function base64(s: string | Uint8Array): Base64String {
|
|
37
|
+
if (typeof s === 'string') return Buffer.from(s).toString('base64')
|
|
38
|
+
return bufferToBase64(s)
|
|
35
39
|
}
|
|
36
40
|
|
|
37
|
-
export function base64Url(s: string |
|
|
38
|
-
|
|
41
|
+
export function base64Url(s: string | Uint8Array): Base64UrlString {
|
|
42
|
+
if (typeof s === 'string') return Buffer.from(s).toString('base64url')
|
|
43
|
+
return bufferToBase64Url(s)
|
|
39
44
|
}
|
|
40
45
|
|
|
41
46
|
export function base64ToString(strBase64: Base64String): string {
|
|
@@ -46,11 +51,11 @@ export function base64UrlToString(strBase64Url: Base64UrlString): string {
|
|
|
46
51
|
return Buffer.from(strBase64Url, 'base64url').toString('utf8')
|
|
47
52
|
}
|
|
48
53
|
|
|
49
|
-
export function base64ToBuffer(strBase64: Base64String): Buffer {
|
|
54
|
+
export function base64ToBuffer(strBase64: Base64String): Buffer<ArrayBuffer> {
|
|
50
55
|
return Buffer.from(strBase64, 'base64')
|
|
51
56
|
}
|
|
52
57
|
|
|
53
|
-
export function base64UrlToBuffer(strBase64Url: Base64UrlString): Buffer {
|
|
58
|
+
export function base64UrlToBuffer(strBase64Url: Base64UrlString): Buffer<ArrayBuffer> {
|
|
54
59
|
return Buffer.from(strBase64Url, 'base64url')
|
|
55
60
|
}
|
|
56
61
|
|
|
@@ -62,10 +67,14 @@ export function stringToBase64Url(s: string): Base64UrlString {
|
|
|
62
67
|
return Buffer.from(s, 'utf8').toString('base64url')
|
|
63
68
|
}
|
|
64
69
|
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
// Buffer#toString is faster than Uint8Array#toBase64 on small inputs (~1.3-1.7x up to ~512 bytes,
|
|
71
|
+
// measured on Node 26), hence the Buffer fast-path. On larger inputs they're on par.
|
|
72
|
+
export function bufferToBase64(b: Uint8Array): Base64String {
|
|
73
|
+
return Buffer.isBuffer(b) ? b.toString('base64') : b.toBase64()
|
|
67
74
|
}
|
|
68
75
|
|
|
69
|
-
export function bufferToBase64Url(b:
|
|
70
|
-
return
|
|
76
|
+
export function bufferToBase64Url(b: Uint8Array): Base64UrlString {
|
|
77
|
+
return Buffer.isBuffer(b)
|
|
78
|
+
? b.toString('base64url')
|
|
79
|
+
: b.toBase64({ alphabet: 'base64url', omitPadding: true })
|
|
71
80
|
}
|
package/src/stream/index.ts
CHANGED
|
@@ -24,7 +24,6 @@ export * from './transform/transformSplit.js'
|
|
|
24
24
|
export * from './transform/transformTap.js'
|
|
25
25
|
export * from './transform/transformThrottle.js'
|
|
26
26
|
export * from './transform/transformThrottleByRSS.js'
|
|
27
|
-
export * from './transform/transformWarmup.js'
|
|
28
27
|
export * from './transform/worker/baseWorkerClass.js'
|
|
29
28
|
export * from './transform/worker/transformMultiThreaded.js'
|
|
30
29
|
export type * from './transform/worker/transformMultiThreaded.model.js'
|
package/src/stream/pipeline.ts
CHANGED
|
@@ -54,8 +54,6 @@ import { transformThrottle } from './transform/transformThrottle.js'
|
|
|
54
54
|
import type { TransformThrottleOptions } from './transform/transformThrottle.js'
|
|
55
55
|
import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js'
|
|
56
56
|
import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js'
|
|
57
|
-
import { transformWarmup } from './transform/transformWarmup.js'
|
|
58
|
-
import type { TransformWarmupOptions } from './transform/transformWarmup.js'
|
|
59
57
|
import { writablePushToArray } from './writable/writablePushToArray.js'
|
|
60
58
|
import { writableVoid } from './writable/writableVoid.js'
|
|
61
59
|
|
|
@@ -257,14 +255,6 @@ export class Pipeline<T = unknown> {
|
|
|
257
255
|
return this
|
|
258
256
|
}
|
|
259
257
|
|
|
260
|
-
/**
|
|
261
|
-
* @experimental to be removed after transformMap2 is stable
|
|
262
|
-
*/
|
|
263
|
-
warmup(opt: TransformWarmupOptions): this {
|
|
264
|
-
this.transforms.push(transformWarmup(opt))
|
|
265
|
-
return this
|
|
266
|
-
}
|
|
267
|
-
|
|
268
258
|
transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO> {
|
|
269
259
|
this.transforms.push(transform)
|
|
270
260
|
return this as any
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { _uniq } from '@naturalcycles/js-lib/array'
|
|
2
|
+
import { Intl2 } from '@naturalcycles/js-lib/intl'
|
|
2
3
|
import type { IANATimezone } from '@naturalcycles/js-lib/types'
|
|
3
4
|
|
|
4
5
|
// List extracted from Wikipedia (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
|
|
@@ -603,8 +604,7 @@ const TIMEZONES_FROM_WIKI = [
|
|
|
603
604
|
'Zulu',
|
|
604
605
|
] as IANATimezone[]
|
|
605
606
|
|
|
606
|
-
|
|
607
|
-
const TIMEZONES_FROM_JS = Intl.supportedValuesOf('timeZone') as IANATimezone[]
|
|
607
|
+
const TIMEZONES_FROM_JS = Intl2.getSupportedTimezones()
|
|
608
608
|
|
|
609
609
|
/**
|
|
610
610
|
* A complicated merge of timezones from the underlying Javascript engine
|
package/src/zip/zip2.ts
CHANGED
|
@@ -16,11 +16,11 @@ const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
|
|
|
16
16
|
const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
|
|
17
17
|
|
|
18
18
|
class Zip2 {
|
|
19
|
-
async decompressZstdOrInflateToString(buf:
|
|
19
|
+
async decompressZstdOrInflateToString(buf: Uint8Array): Promise<string> {
|
|
20
20
|
return (await this.decompressZstdOrInflate(buf)).toString()
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
decompressZstdOrInflateToStringSync(buf:
|
|
23
|
+
decompressZstdOrInflateToStringSync(buf: Uint8Array): string {
|
|
24
24
|
return this.decompressZstdOrInflateSync(buf).toString()
|
|
25
25
|
}
|
|
26
26
|
|
|
@@ -28,14 +28,14 @@ class Zip2 {
|
|
|
28
28
|
* Detects if Buffer is zstd-compressed.
|
|
29
29
|
* Otherwise attempts to Inflate.
|
|
30
30
|
*/
|
|
31
|
-
async decompressZstdOrInflate(buf:
|
|
31
|
+
async decompressZstdOrInflate(buf: Uint8Array): Promise<Buffer<ArrayBuffer>> {
|
|
32
32
|
if (this.isZstdBuffer(buf)) {
|
|
33
33
|
return await zstdDecompressAsync(buf)
|
|
34
34
|
}
|
|
35
35
|
return await inflateAsync(buf)
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
decompressZstdOrInflateSync(buf:
|
|
38
|
+
decompressZstdOrInflateSync(buf: Uint8Array): Buffer<ArrayBuffer> {
|
|
39
39
|
if (this.isZstdBuffer(buf)) {
|
|
40
40
|
return zlib.zstdDecompressSync(buf)
|
|
41
41
|
}
|
|
@@ -46,7 +46,10 @@ class Zip2 {
|
|
|
46
46
|
* deflateBuffer uses `deflate`.
|
|
47
47
|
* It's 9 bytes shorter than `gzip`.
|
|
48
48
|
*/
|
|
49
|
-
async deflate(
|
|
49
|
+
async deflate(
|
|
50
|
+
input: string | Uint8Array,
|
|
51
|
+
options: ZlibOptions = {},
|
|
52
|
+
): Promise<Buffer<ArrayBuffer>> {
|
|
50
53
|
return await deflateAsync(input, options)
|
|
51
54
|
}
|
|
52
55
|
|
|
@@ -54,23 +57,23 @@ class Zip2 {
|
|
|
54
57
|
* deflateSync uses `deflate`.
|
|
55
58
|
* It's 9 bytes shorter than `gzip`.
|
|
56
59
|
*/
|
|
57
|
-
deflateSync(input: string |
|
|
60
|
+
deflateSync(input: string | Uint8Array, options?: ZlibOptions): Buffer<ArrayBuffer> {
|
|
58
61
|
return zlib.deflateSync(input, options)
|
|
59
62
|
}
|
|
60
63
|
|
|
61
|
-
async inflate(buf:
|
|
64
|
+
async inflate(buf: Uint8Array, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
|
|
62
65
|
return await inflateAsync(buf, options)
|
|
63
66
|
}
|
|
64
67
|
|
|
65
|
-
inflateSync(buf:
|
|
68
|
+
inflateSync(buf: Uint8Array, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
|
|
66
69
|
return zlib.inflateSync(buf, options)
|
|
67
70
|
}
|
|
68
71
|
|
|
69
|
-
async inflateToString(buf:
|
|
72
|
+
async inflateToString(buf: Uint8Array, options?: ZlibOptions): Promise<string> {
|
|
70
73
|
return (await this.inflate(buf, options)).toString()
|
|
71
74
|
}
|
|
72
75
|
|
|
73
|
-
inflateToStringSync(buf:
|
|
76
|
+
inflateToStringSync(buf: Uint8Array, options?: ZlibOptions): string {
|
|
74
77
|
return zlib.inflateSync(buf, options).toString()
|
|
75
78
|
}
|
|
76
79
|
|
|
@@ -78,7 +81,7 @@ class Zip2 {
|
|
|
78
81
|
* gzipBuffer uses `gzip`
|
|
79
82
|
* It's 9 bytes longer than `deflate`.
|
|
80
83
|
*/
|
|
81
|
-
async gzip(input: string |
|
|
84
|
+
async gzip(input: string | Uint8Array, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
|
|
82
85
|
return await gzipAsync(input, options)
|
|
83
86
|
}
|
|
84
87
|
|
|
@@ -86,28 +89,28 @@ class Zip2 {
|
|
|
86
89
|
* gzipBuffer uses `gzip`
|
|
87
90
|
* It's 9 bytes longer than `deflate`.
|
|
88
91
|
*/
|
|
89
|
-
gzipSync(input: string |
|
|
92
|
+
gzipSync(input: string | Uint8Array, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
|
|
90
93
|
return zlib.gzipSync(input, options)
|
|
91
94
|
}
|
|
92
95
|
|
|
93
|
-
async gunzip(buf:
|
|
96
|
+
async gunzip(buf: Uint8Array, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
|
|
94
97
|
return await gunzipAsync(buf, options)
|
|
95
98
|
}
|
|
96
99
|
|
|
97
|
-
gunzipSync(buf:
|
|
100
|
+
gunzipSync(buf: Uint8Array, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
|
|
98
101
|
return zlib.gunzipSync(buf, options)
|
|
99
102
|
}
|
|
100
103
|
|
|
101
|
-
async gunzipToString(buf:
|
|
104
|
+
async gunzipToString(buf: Uint8Array, options?: ZlibOptions): Promise<string> {
|
|
102
105
|
return (await this.gunzip(buf, options)).toString()
|
|
103
106
|
}
|
|
104
107
|
|
|
105
|
-
gunzipToStringSync(buf:
|
|
108
|
+
gunzipToStringSync(buf: Uint8Array, options?: ZlibOptions): string {
|
|
106
109
|
return zlib.gunzipSync(buf, options).toString()
|
|
107
110
|
}
|
|
108
111
|
|
|
109
112
|
async zstdCompress(
|
|
110
|
-
input:
|
|
113
|
+
input: Uint8Array | string,
|
|
111
114
|
level?: Integer, // defaults to 3
|
|
112
115
|
options: ZstdOptions = {},
|
|
113
116
|
): Promise<Buffer<ArrayBuffer>> {
|
|
@@ -115,7 +118,7 @@ class Zip2 {
|
|
|
115
118
|
}
|
|
116
119
|
|
|
117
120
|
zstdCompressSync(
|
|
118
|
-
input:
|
|
121
|
+
input: Uint8Array | string,
|
|
119
122
|
level?: Integer, // defaults to 3
|
|
120
123
|
options: ZstdOptions = {},
|
|
121
124
|
): Buffer<ArrayBuffer> {
|
|
@@ -134,33 +137,34 @@ class Zip2 {
|
|
|
134
137
|
}
|
|
135
138
|
}
|
|
136
139
|
|
|
137
|
-
async zstdDecompressToString(input:
|
|
140
|
+
async zstdDecompressToString(input: Uint8Array, options: ZstdOptions = {}): Promise<string> {
|
|
138
141
|
return (await zstdDecompressAsync(input, options)).toString()
|
|
139
142
|
}
|
|
140
143
|
|
|
141
144
|
/**
|
|
142
145
|
* Warning! It leaks memory severely. Prefer sync.
|
|
143
146
|
*/
|
|
144
|
-
async zstdDecompress(input:
|
|
147
|
+
async zstdDecompress(input: Uint8Array, options: ZstdOptions = {}): Promise<Buffer<ArrayBuffer>> {
|
|
145
148
|
return await zstdDecompressAsync(input, options)
|
|
146
149
|
}
|
|
147
150
|
|
|
148
151
|
/**
|
|
149
152
|
* Warning! It leaks memory severely. Prefer sync.
|
|
150
153
|
*/
|
|
151
|
-
zstdDecompressToStringSync(input:
|
|
154
|
+
zstdDecompressToStringSync(input: Uint8Array, options: ZstdOptions = {}): string {
|
|
152
155
|
return zlib.zstdDecompressSync(input, options).toString()
|
|
153
156
|
}
|
|
154
157
|
|
|
155
|
-
zstdDecompressSync(input:
|
|
158
|
+
zstdDecompressSync(input: Uint8Array, options: ZstdOptions = {}): Buffer<ArrayBuffer> {
|
|
156
159
|
return zlib.zstdDecompressSync(input, options)
|
|
157
160
|
}
|
|
158
161
|
|
|
159
|
-
isZstdBuffer(input:
|
|
160
|
-
|
|
162
|
+
isZstdBuffer(input: Uint8Array): boolean {
|
|
163
|
+
// zstd magic number 0xFD2FB528, stored little-endian
|
|
164
|
+
return input[0] === 0x28 && input[1] === 0xb5 && input[2] === 0x2f && input[3] === 0xfd
|
|
161
165
|
}
|
|
162
166
|
|
|
163
|
-
isGzipBuffer(input:
|
|
167
|
+
isGzipBuffer(input: Uint8Array): boolean {
|
|
164
168
|
return input[0] === 0x1f && input[1] === 0x8b
|
|
165
169
|
}
|
|
166
170
|
|
|
@@ -211,11 +215,9 @@ class Zip2 {
|
|
|
211
215
|
*
|
|
212
216
|
* For large archives or streamed inputs prefer {@link createZip} / {@link ZipWriter}.
|
|
213
217
|
*/
|
|
214
|
-
async createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer
|
|
218
|
+
async createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer<ArrayBuffer>> {
|
|
215
219
|
return await createZipBuffer(entries)
|
|
216
220
|
}
|
|
217
221
|
}
|
|
218
222
|
|
|
219
223
|
export const zip2 = new Zip2()
|
|
220
|
-
|
|
221
|
-
const ZSTD_MAGIC_NUMBER = 0xfd2fb528
|
package/src/zip/zipWriter.ts
CHANGED
|
@@ -62,7 +62,7 @@ const deflateRawAsync = promisify(zlib.deflateRaw.bind(zlib))
|
|
|
62
62
|
*
|
|
63
63
|
* Each input path may be a file (added directly) or a directory (walked
|
|
64
64
|
* recursively; all nested files are added). By default an entry's name is its
|
|
65
|
-
* path relative to its input's parent directory, so `
|
|
65
|
+
* path relative to its input's parent directory, so `zipPaths(['./photos'], 'out.zip')`
|
|
66
66
|
* stores entries under `photos/...`. Override the base via {@link ZipPathsOptions.baseDir}.
|
|
67
67
|
*
|
|
68
68
|
* Files discovered by walking a directory are added in deterministic (sorted)
|
|
@@ -105,7 +105,7 @@ export async function zipPaths(
|
|
|
105
105
|
*
|
|
106
106
|
* For large archives or streamed inputs prefer {@link createZip} / {@link ZipWriter}.
|
|
107
107
|
*/
|
|
108
|
-
export async function createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer
|
|
108
|
+
export async function createZipBuffer(entries: ZipFileEntry[]): Promise<Buffer<ArrayBuffer>> {
|
|
109
109
|
const chunks: Buffer[] = []
|
|
110
110
|
const sink = new Writable({
|
|
111
111
|
write(chunk: Buffer, _enc, cb) {
|
|
@@ -176,7 +176,11 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
176
176
|
* Add an in-memory buffer as a file entry.
|
|
177
177
|
* The CRC-32 and sizes are computed up front and written inline (no data descriptor).
|
|
178
178
|
*/
|
|
179
|
-
async addBuffer(
|
|
179
|
+
async addBuffer(
|
|
180
|
+
data: Uint8Array,
|
|
181
|
+
fileName: string,
|
|
182
|
+
opt: ZipWriterEntryOptions = {},
|
|
183
|
+
): Promise<void> {
|
|
180
184
|
this.assertWritable()
|
|
181
185
|
const entry = this.createEntry(fileName, false, opt)
|
|
182
186
|
entry.crc32 = zlib.crc32(data)
|
|
@@ -325,7 +329,7 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
325
329
|
}
|
|
326
330
|
|
|
327
331
|
/** Write an entry whose CRC and sizes are already known: header, name, data, no descriptor. */
|
|
328
|
-
private async writeKnownEntry(entry: WriteEntry, data:
|
|
332
|
+
private async writeKnownEntry(entry: WriteEntry, data: Uint8Array): Promise<void> {
|
|
329
333
|
entry.relativeOffsetOfLocalHeader = this.offset
|
|
330
334
|
await this.write(buildLocalFileHeader(entry))
|
|
331
335
|
await this.write(entry.nameBuf)
|
|
@@ -389,7 +393,7 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
389
393
|
}
|
|
390
394
|
|
|
391
395
|
/** Write a buffer to the output, tracking the byte offset and respecting backpressure. */
|
|
392
|
-
private async write(buf:
|
|
396
|
+
private async write(buf: Uint8Array): Promise<void> {
|
|
393
397
|
if (this.streamError) throw this.streamError
|
|
394
398
|
if (buf.length === 0) return
|
|
395
399
|
this.offset += buf.length
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import type { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib/types';
|
|
2
|
-
import type { TransformOptions, TransformTyped } from '../stream.model.js';
|
|
3
|
-
export interface TransformWarmupOptions extends TransformOptions {
|
|
4
|
-
/**
|
|
5
|
-
* Target concurrency after warmup completes.
|
|
6
|
-
*/
|
|
7
|
-
concurrency: PositiveInteger;
|
|
8
|
-
/**
|
|
9
|
-
* Time in seconds to gradually increase concurrency from 1 to `concurrency`.
|
|
10
|
-
* Set to 0 to disable warmup (pass-through mode from the start).
|
|
11
|
-
*/
|
|
12
|
-
warmupSeconds: NumberOfSeconds;
|
|
13
|
-
}
|
|
14
|
-
/**
|
|
15
|
-
* Transform that gradually increases concurrency from 1 to the configured maximum
|
|
16
|
-
* over a warmup period. Useful for scenarios where you want to avoid overwhelming
|
|
17
|
-
* a system at startup (e.g., database connections, API rate limits).
|
|
18
|
-
*
|
|
19
|
-
* During warmup: limits concurrent items based on elapsed time.
|
|
20
|
-
* After warmup: passes items through immediately with zero overhead.
|
|
21
|
-
*
|
|
22
|
-
* @experimental
|
|
23
|
-
*/
|
|
24
|
-
export declare function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T>;
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import { Transform } from 'node:stream';
|
|
2
|
-
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
3
|
-
import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
|
|
4
|
-
/**
|
|
5
|
-
* Transform that gradually increases concurrency from 1 to the configured maximum
|
|
6
|
-
* over a warmup period. Useful for scenarios where you want to avoid overwhelming
|
|
7
|
-
* a system at startup (e.g., database connections, API rate limits).
|
|
8
|
-
*
|
|
9
|
-
* During warmup: limits concurrent items based on elapsed time.
|
|
10
|
-
* After warmup: passes items through immediately with zero overhead.
|
|
11
|
-
*
|
|
12
|
-
* @experimental
|
|
13
|
-
*/
|
|
14
|
-
export function transformWarmup(opt) {
|
|
15
|
-
const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt;
|
|
16
|
-
const warmupMs = warmupSeconds * 1000;
|
|
17
|
-
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
|
|
18
|
-
let startTime = 0;
|
|
19
|
-
let warmupComplete = warmupSeconds <= 0 || concurrency <= 1;
|
|
20
|
-
let inFlight = 0;
|
|
21
|
-
const waiters = [];
|
|
22
|
-
return new Transform({
|
|
23
|
-
objectMode,
|
|
24
|
-
highWaterMark,
|
|
25
|
-
async transform(item, _, cb) {
|
|
26
|
-
// Initialize start time on first item
|
|
27
|
-
if (startTime === 0) {
|
|
28
|
-
startTime = Date.now();
|
|
29
|
-
}
|
|
30
|
-
// Fast-path: after warmup, just pass through with zero overhead
|
|
31
|
-
if (warmupComplete) {
|
|
32
|
-
cb(null, item);
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
const currentConcurrency = getCurrentConcurrency();
|
|
36
|
-
if (inFlight < currentConcurrency) {
|
|
37
|
-
// Have room, proceed immediately
|
|
38
|
-
inFlight++;
|
|
39
|
-
logger.debug(`inFlight++ ${inFlight}/${currentConcurrency}, waiters ${waiters.length}`);
|
|
40
|
-
}
|
|
41
|
-
else {
|
|
42
|
-
// Wait for a slot
|
|
43
|
-
const waiter = pDefer();
|
|
44
|
-
waiters.push(waiter);
|
|
45
|
-
logger.debug(`inFlight ${inFlight}/${currentConcurrency}, waiters++ ${waiters.length}`);
|
|
46
|
-
await waiter;
|
|
47
|
-
logger.debug(`waiter resolved, inFlight ${inFlight}/${getCurrentConcurrency()}`);
|
|
48
|
-
}
|
|
49
|
-
// Push the item
|
|
50
|
-
cb(null, item);
|
|
51
|
-
// Release slot on next microtask - essential for concurrency control.
|
|
52
|
-
// Without this, the slot would be freed immediately and items would
|
|
53
|
-
// flow through without any limiting effect.
|
|
54
|
-
queueMicrotask(release);
|
|
55
|
-
},
|
|
56
|
-
});
|
|
57
|
-
function getCurrentConcurrency() {
|
|
58
|
-
if (warmupComplete)
|
|
59
|
-
return concurrency;
|
|
60
|
-
const elapsed = Date.now() - startTime;
|
|
61
|
-
if (elapsed >= warmupMs) {
|
|
62
|
-
warmupComplete = true;
|
|
63
|
-
logger.debug('warmup complete');
|
|
64
|
-
return concurrency;
|
|
65
|
-
}
|
|
66
|
-
// Linear interpolation from 1 to concurrency
|
|
67
|
-
const progress = elapsed / warmupMs;
|
|
68
|
-
return Math.max(1, Math.floor(1 + (concurrency - 1) * progress));
|
|
69
|
-
}
|
|
70
|
-
function release() {
|
|
71
|
-
inFlight--;
|
|
72
|
-
// Wake up waiters based on current concurrency (may have increased)
|
|
73
|
-
const currentConcurrency = getCurrentConcurrency();
|
|
74
|
-
while (waiters.length && inFlight < currentConcurrency) {
|
|
75
|
-
inFlight++;
|
|
76
|
-
waiters.shift().resolve();
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import { Transform } from 'node:stream'
|
|
2
|
-
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
3
|
-
import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
|
|
4
|
-
import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
|
|
5
|
-
import type { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib/types'
|
|
6
|
-
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
7
|
-
|
|
8
|
-
export interface TransformWarmupOptions extends TransformOptions {
|
|
9
|
-
/**
|
|
10
|
-
* Target concurrency after warmup completes.
|
|
11
|
-
*/
|
|
12
|
-
concurrency: PositiveInteger
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Time in seconds to gradually increase concurrency from 1 to `concurrency`.
|
|
16
|
-
* Set to 0 to disable warmup (pass-through mode from the start).
|
|
17
|
-
*/
|
|
18
|
-
warmupSeconds: NumberOfSeconds
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Transform that gradually increases concurrency from 1 to the configured maximum
|
|
23
|
-
* over a warmup period. Useful for scenarios where you want to avoid overwhelming
|
|
24
|
-
* a system at startup (e.g., database connections, API rate limits).
|
|
25
|
-
*
|
|
26
|
-
* During warmup: limits concurrent items based on elapsed time.
|
|
27
|
-
* After warmup: passes items through immediately with zero overhead.
|
|
28
|
-
*
|
|
29
|
-
* @experimental
|
|
30
|
-
*/
|
|
31
|
-
export function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T> {
|
|
32
|
-
const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt
|
|
33
|
-
const warmupMs = warmupSeconds * 1000
|
|
34
|
-
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
|
|
35
|
-
|
|
36
|
-
let startTime = 0
|
|
37
|
-
let warmupComplete = warmupSeconds <= 0 || concurrency <= 1
|
|
38
|
-
let inFlight = 0
|
|
39
|
-
const waiters: DeferredPromise[] = []
|
|
40
|
-
|
|
41
|
-
return new Transform({
|
|
42
|
-
objectMode,
|
|
43
|
-
highWaterMark,
|
|
44
|
-
async transform(item: T, _, cb) {
|
|
45
|
-
// Initialize start time on first item
|
|
46
|
-
if (startTime === 0) {
|
|
47
|
-
startTime = Date.now()
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// Fast-path: after warmup, just pass through with zero overhead
|
|
51
|
-
if (warmupComplete) {
|
|
52
|
-
cb(null, item)
|
|
53
|
-
return
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const currentConcurrency = getCurrentConcurrency()
|
|
57
|
-
|
|
58
|
-
if (inFlight < currentConcurrency) {
|
|
59
|
-
// Have room, proceed immediately
|
|
60
|
-
inFlight++
|
|
61
|
-
logger.debug(`inFlight++ ${inFlight}/${currentConcurrency}, waiters ${waiters.length}`)
|
|
62
|
-
} else {
|
|
63
|
-
// Wait for a slot
|
|
64
|
-
const waiter = pDefer()
|
|
65
|
-
waiters.push(waiter)
|
|
66
|
-
logger.debug(`inFlight ${inFlight}/${currentConcurrency}, waiters++ ${waiters.length}`)
|
|
67
|
-
await waiter
|
|
68
|
-
logger.debug(`waiter resolved, inFlight ${inFlight}/${getCurrentConcurrency()}`)
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// Push the item
|
|
72
|
-
cb(null, item)
|
|
73
|
-
|
|
74
|
-
// Release slot on next microtask - essential for concurrency control.
|
|
75
|
-
// Without this, the slot would be freed immediately and items would
|
|
76
|
-
// flow through without any limiting effect.
|
|
77
|
-
queueMicrotask(release)
|
|
78
|
-
},
|
|
79
|
-
})
|
|
80
|
-
|
|
81
|
-
function getCurrentConcurrency(): number {
|
|
82
|
-
if (warmupComplete) return concurrency
|
|
83
|
-
|
|
84
|
-
const elapsed = Date.now() - startTime
|
|
85
|
-
if (elapsed >= warmupMs) {
|
|
86
|
-
warmupComplete = true
|
|
87
|
-
logger.debug('warmup complete')
|
|
88
|
-
return concurrency
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Linear interpolation from 1 to concurrency
|
|
92
|
-
const progress = elapsed / warmupMs
|
|
93
|
-
return Math.max(1, Math.floor(1 + (concurrency - 1) * progress))
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function release(): void {
|
|
97
|
-
inFlight--
|
|
98
|
-
// Wake up waiters based on current concurrency (may have increased)
|
|
99
|
-
const currentConcurrency = getCurrentConcurrency()
|
|
100
|
-
while (waiters.length && inFlight < currentConcurrency) {
|
|
101
|
-
inFlight++
|
|
102
|
-
waiters.shift()!.resolve()
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
}
|