@naturalcycles/nodejs-lib 15.104.0 → 15.105.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/dist/stream/pipeline.js +3 -3
  2. package/dist/stream/progressLogger.d.ts +0 -22
  3. package/dist/stream/progressLogger.js +0 -17
  4. package/dist/zip/zip2.d.ts +51 -0
  5. package/dist/zip/zip2.js +123 -0
  6. package/package.json +2 -2
  7. package/src/stream/pipeline.ts +3 -3
  8. package/src/stream/progressLogger.ts +0 -43
  9. package/src/zip/zip2.ts +159 -0
  10. package/dist/stream/sizeStack.d.ts +0 -10
  11. package/dist/stream/sizeStack.js +0 -46
  12. package/dist/zip/zip.util.d.ts +0 -43
  13. package/dist/zip/zip.util.js +0 -108
  14. package/src/stream/sizeStack.ts +0 -61
  15. package/src/zip/zip.util.ts +0 -160
package/dist/stream/pipeline.js CHANGED
@@ -4,7 +4,7 @@ import { createGzip, createUnzip, createZstdCompress, createZstdDecompress } fro
4
4
  import { createAbortableSignal } from '@naturalcycles/js-lib';
5
5
  import { _passthroughPredicate } from '@naturalcycles/js-lib/types';
6
6
  import { fs2 } from '../fs/fs2.js';
7
- import { zstdLevelToOptions } from '../zip/zip.util.js';
7
+ import { zip2 } from '../zip/zip2.js';
8
8
  import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
9
9
  import { transformJsonParse } from './ndjson/transformJsonParse.js';
10
10
  import { transformToNDJson } from './ndjson/transformToNDJson.js';
@@ -262,7 +262,7 @@ export class Pipeline {
262
262
  }
263
263
  zstdCompress(level, // defaults to 3
264
264
  opt) {
265
- this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)));
265
+ this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level, opt)));
266
266
  this.objectMode = false;
267
267
  return this;
268
268
  }
@@ -302,7 +302,7 @@ export class Pipeline {
302
302
  }));
303
303
  }
304
304
  else if (outputFilePath.endsWith('.zst')) {
305
- this.transforms.push(createZstdCompress(zstdLevelToOptions(level)));
305
+ this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level)));
306
306
  }
307
307
  this.destination = fs2.createWriteStream(outputFilePath, {});
308
308
  await this.run();
package/dist/stream/progressLogger.d.ts CHANGED
@@ -90,26 +90,6 @@ export interface ProgressLoggerCfg<T = any> {
90
90
  * Defaults to 1.
91
91
  */
92
92
  chunkSize?: PositiveInteger;
93
- /**
94
- * Experimental logging of item (shunk) sizes, when json-stringified.
95
- *
96
- * Defaults to false.
97
- *
98
- * @experimental
99
- */
100
- logSizes?: boolean;
101
- /**
102
- * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
103
- * Defaults to 100_000.
104
- * Cannot be Infinity.
105
- */
106
- logSizesBuffer?: number;
107
- /**
108
- * Works in addition to `logSizes`. Adds "zipped sizes".
109
- *
110
- * @experimental
111
- */
112
- logZippedSizes?: boolean;
113
93
  }
114
94
  export interface ProgressLogItem extends AnyObject {
115
95
  heapUsed?: number;
@@ -138,8 +118,6 @@ export declare class ProgressLogger<T> implements Disposable {
138
118
  private processedLastSecond;
139
119
  private progress;
140
120
  private peakRSS;
141
- private sizes?;
142
- private sizesZipped?;
143
121
  private start;
144
122
  log(chunk?: T): void;
145
123
  done(): void;
package/dist/stream/progressLogger.js CHANGED
@@ -3,7 +3,6 @@ import { _hc, _mb } from '@naturalcycles/js-lib';
3
3
  import { _since, localTime } from '@naturalcycles/js-lib/datetime';
4
4
  import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
5
5
  import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
6
- import { SizeStack } from './sizeStack.js';
7
6
  const inspectOpt = {
8
7
  colors: hasColors,
9
8
  breakLength: 300,
@@ -34,8 +33,6 @@ export class ProgressLogger {
34
33
  processedLastSecond;
35
34
  progress;
36
35
  peakRSS;
37
- sizes;
38
- sizesZipped;
39
36
  start() {
40
37
  this.started = Date.now();
41
38
  this.lastSecondStarted = Date.now();
@@ -43,18 +40,10 @@ export class ProgressLogger {
43
40
  this.processedLastSecond = 0;
44
41
  this.progress = 0;
45
42
  this.peakRSS = 0;
46
- this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
47
- this.sizesZipped = this.cfg.logZippedSizes
48
- ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
49
- : undefined;
50
43
  }
51
44
  log(chunk) {
52
45
  this.progress++;
53
46
  this.processedLastSecond++;
54
- if (this.sizes) {
55
- // Check it, cause gzipping might be delayed here..
56
- void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
57
- }
58
47
  if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
59
48
  this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
60
49
  }
@@ -101,12 +90,6 @@ export class ProgressLogger {
101
90
  if (logRPS)
102
91
  Object.assign(o, { rps10, rpsTotal });
103
92
  logger.log(inspect(o, inspectOpt));
104
- if (this.sizes?.items.length) {
105
- logger.log(this.sizes.getStats());
106
- if (this.sizesZipped?.items.length) {
107
- logger.log(this.sizesZipped.getStats());
108
- }
109
- }
110
93
  if (tenx) {
111
94
  const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started));
112
95
  logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(_hc(batchedProgress))} rows, ~${yellow(perHour)}/hour`);
package/dist/zip/zip2.d.ts ADDED
@@ -0,0 +1,51 @@
1
+ import type { ZlibOptions, ZstdOptions } from 'node:zlib';
2
+ import type { Integer } from '@naturalcycles/js-lib/types';
3
+ declare class Zip2 {
4
+ decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
5
+ decompressZstdOrInflateToStringSync(buf: Buffer): string;
6
+ /**
7
+ * Detects if Buffer is zstd-compressed.
8
+ * Otherwise attempts to Inflate.
9
+ */
10
+ decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>>;
11
+ decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer>;
12
+ /**
13
+ * deflateBuffer uses `deflate`.
14
+ * It's 9 bytes shorter than `gzip`.
15
+ */
16
+ deflate(input: string | Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
17
+ /**
18
+ * deflateSync uses `deflate`.
19
+ * It's 9 bytes shorter than `gzip`.
20
+ */
21
+ deflateSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
22
+ inflate(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
23
+ inflateSync(buf: Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
24
+ inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
25
+ inflateToStringSync(buf: Buffer, options?: ZlibOptions): string;
26
+ /**
27
+ * gzipBuffer uses `gzip`
28
+ * It's 9 bytes longer than `deflate`.
29
+ */
30
+ gzip(input: string | Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
31
+ /**
32
+ * gzipBuffer uses `gzip`
33
+ * It's 9 bytes longer than `deflate`.
34
+ */
35
+ gzipSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
36
+ gunzip(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
37
+ gunzipSync(buf: Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
38
+ gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
39
+ gunzipToStringSync(buf: Buffer, options?: ZlibOptions): string;
40
+ zstdCompress(input: Buffer | string, level?: Integer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
41
+ zstdCompressSync(input: Buffer | string, level?: Integer, options?: ZstdOptions): Buffer<ArrayBuffer>;
42
+ zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
43
+ zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
44
+ zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
45
+ zstdDecompressToStringSync(input: Buffer, options?: ZstdOptions): string;
46
+ zstdDecompressSync(input: Buffer, options?: ZstdOptions): Buffer<ArrayBuffer>;
47
+ isZstdBuffer(input: Buffer): boolean;
48
+ isGzipBuffer(input: Buffer): boolean;
49
+ }
50
+ export declare const zip2: Zip2;
51
+ export {};
package/dist/zip/zip2.js ADDED
@@ -0,0 +1,123 @@
1
+ import { promisify } from 'node:util';
2
+ import zlib from 'node:zlib';
3
+ const deflateAsync = promisify(zlib.deflate.bind(zlib));
4
+ const inflateAsync = promisify(zlib.inflate.bind(zlib));
5
+ const gzipAsync = promisify(zlib.gzip.bind(zlib));
6
+ const gunzipAsync = promisify(zlib.gunzip.bind(zlib));
7
+ const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib));
8
+ const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
9
+ class Zip2 {
10
+ async decompressZstdOrInflateToString(buf) {
11
+ return (await this.decompressZstdOrInflate(buf)).toString();
12
+ }
13
+ decompressZstdOrInflateToStringSync(buf) {
14
+ return this.decompressZstdOrInflateSync(buf).toString();
15
+ }
16
+ /**
17
+ * Detects if Buffer is zstd-compressed.
18
+ * Otherwise attempts to Inflate.
19
+ */
20
+ async decompressZstdOrInflate(buf) {
21
+ if (this.isZstdBuffer(buf)) {
22
+ return await zstdDecompressAsync(buf);
23
+ }
24
+ return await inflateAsync(buf);
25
+ }
26
+ decompressZstdOrInflateSync(buf) {
27
+ if (this.isZstdBuffer(buf)) {
28
+ return zlib.zstdDecompressSync(buf);
29
+ }
30
+ return zlib.inflateSync(buf);
31
+ }
32
+ /**
33
+ * deflateBuffer uses `deflate`.
34
+ * It's 9 bytes shorter than `gzip`.
35
+ */
36
+ async deflate(input, options = {}) {
37
+ return await deflateAsync(input, options);
38
+ }
39
+ /**
40
+ * deflateSync uses `deflate`.
41
+ * It's 9 bytes shorter than `gzip`.
42
+ */
43
+ deflateSync(input, options) {
44
+ return zlib.deflateSync(input, options);
45
+ }
46
+ async inflate(buf, options = {}) {
47
+ return await inflateAsync(buf, options);
48
+ }
49
+ inflateSync(buf, options = {}) {
50
+ return zlib.inflateSync(buf, options);
51
+ }
52
+ async inflateToString(buf, options) {
53
+ return (await this.inflate(buf, options)).toString();
54
+ }
55
+ inflateToStringSync(buf, options) {
56
+ return zlib.inflateSync(buf, options).toString();
57
+ }
58
+ /**
59
+ * gzipBuffer uses `gzip`
60
+ * It's 9 bytes longer than `deflate`.
61
+ */
62
+ async gzip(input, options = {}) {
63
+ return await gzipAsync(input, options);
64
+ }
65
+ /**
66
+ * gzipBuffer uses `gzip`
67
+ * It's 9 bytes longer than `deflate`.
68
+ */
69
+ gzipSync(input, options = {}) {
70
+ return zlib.gzipSync(input, options);
71
+ }
72
+ async gunzip(buf, options = {}) {
73
+ return await gunzipAsync(buf, options);
74
+ }
75
+ gunzipSync(buf, options = {}) {
76
+ return zlib.gunzipSync(buf, options);
77
+ }
78
+ async gunzipToString(buf, options) {
79
+ return (await this.gunzip(buf, options)).toString();
80
+ }
81
+ gunzipToStringSync(buf, options) {
82
+ return zlib.gunzipSync(buf, options).toString();
83
+ }
84
+ async zstdCompress(input, level, // defaults to 3
85
+ options = {}) {
86
+ return await zstdCompressAsync(input, this.zstdLevelToOptions(level, options));
87
+ }
88
+ zstdCompressSync(input, level, // defaults to 3
89
+ options = {}) {
90
+ return zlib.zstdCompressSync(input, this.zstdLevelToOptions(level, options));
91
+ }
92
+ zstdLevelToOptions(level, opt = {}) {
93
+ if (!level)
94
+ return opt;
95
+ return {
96
+ ...opt,
97
+ params: {
98
+ ...opt.params,
99
+ [zlib.constants.ZSTD_c_compressionLevel]: level,
100
+ },
101
+ };
102
+ }
103
+ async zstdDecompressToString(input, options = {}) {
104
+ return (await zstdDecompressAsync(input, options)).toString();
105
+ }
106
+ async zstdDecompress(input, options = {}) {
107
+ return await zstdDecompressAsync(input, options);
108
+ }
109
+ zstdDecompressToStringSync(input, options = {}) {
110
+ return zlib.zstdDecompressSync(input, options).toString();
111
+ }
112
+ zstdDecompressSync(input, options = {}) {
113
+ return zlib.zstdDecompressSync(input, options);
114
+ }
115
+ isZstdBuffer(input) {
116
+ return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER;
117
+ }
118
+ isGzipBuffer(input) {
119
+ return input[0] === 0x1f && input[1] === 0x8b;
120
+ }
121
+ }
122
+ export const zip2 = new Zip2();
123
+ const ZSTD_MAGIC_NUMBER = 0xfd2fb528;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@naturalcycles/nodejs-lib",
3
3
  "type": "module",
4
- "version": "15.104.0",
4
+ "version": "15.105.0",
5
5
  "dependencies": {
6
6
  "@naturalcycles/js-lib": "^15",
7
7
  "@standard-schema/spec": "^1",
@@ -38,7 +38,7 @@
38
38
  "./stream/*.js": "./dist/stream/*.js",
39
39
  "./yargs": "./dist/yargs/yargs.util.js",
40
40
  "./ajv": "./dist/validation/ajv/index.js",
41
- "./zip": "./dist/zip/zip.util.js"
41
+ "./zip": "./dist/zip/zip2.js"
42
42
  },
43
43
  "bin": {
44
44
  "kpy": "dist/bin/kpy.js",
package/src/stream/pipeline.ts CHANGED
@@ -19,7 +19,7 @@ import type {
19
19
  SKIP,
20
20
  } from '@naturalcycles/js-lib/types'
21
21
  import { fs2 } from '../fs/fs2.js'
22
- import { zstdLevelToOptions } from '../zip/zip.util.js'
22
+ import { zip2 } from '../zip/zip2.js'
23
23
  import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
24
24
  import { transformJsonParse } from './ndjson/transformJsonParse.js'
25
25
  import { transformToNDJson } from './ndjson/transformToNDJson.js'
@@ -359,7 +359,7 @@ export class Pipeline<T = unknown> {
359
359
  level?: Integer, // defaults to 3
360
360
  opt?: ZstdOptions,
361
361
  ): Pipeline<Uint8Array> {
362
- this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)))
362
+ this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level, opt)))
363
363
  this.objectMode = false
364
364
  return this as any
365
365
  }
@@ -406,7 +406,7 @@ export class Pipeline<T = unknown> {
406
406
  }),
407
407
  )
408
408
  } else if (outputFilePath.endsWith('.zst')) {
409
- this.transforms.push(createZstdCompress(zstdLevelToOptions(level)))
409
+ this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level)))
410
410
  }
411
411
  this.destination = fs2.createWriteStream(outputFilePath, {
412
412
  // highWaterMark: 64 * 1024, // no observed speedup
package/src/stream/progressLogger.ts CHANGED
@@ -6,7 +6,6 @@ import type { CommonLogger } from '@naturalcycles/js-lib/log'
6
6
  import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
7
7
  import type { AnyObject, PositiveInteger, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
8
8
  import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js'
9
- import { SizeStack } from './sizeStack.js'
10
9
  import type { ReadableMapper } from './stream.model.js'
11
10
 
12
11
  export interface ProgressLoggerCfg<T = any> {
@@ -112,29 +111,6 @@ export interface ProgressLoggerCfg<T = any> {
112
111
  * Defaults to 1.
113
112
  */
114
113
  chunkSize?: PositiveInteger
115
-
116
- /**
117
- * Experimental logging of item (shunk) sizes, when json-stringified.
118
- *
119
- * Defaults to false.
120
- *
121
- * @experimental
122
- */
123
- logSizes?: boolean
124
-
125
- /**
126
- * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
127
- * Defaults to 100_000.
128
- * Cannot be Infinity.
129
- */
130
- logSizesBuffer?: number
131
-
132
- /**
133
- * Works in addition to `logSizes`. Adds "zipped sizes".
134
- *
135
- * @experimental
136
- */
137
- logZippedSizes?: boolean
138
114
  }
139
115
 
140
116
  export interface ProgressLogItem extends AnyObject {
@@ -189,8 +165,6 @@ export class ProgressLogger<T> implements Disposable {
189
165
  private processedLastSecond!: number
190
166
  private progress!: number
191
167
  private peakRSS!: number
192
- private sizes?: SizeStack
193
- private sizesZipped?: SizeStack
194
168
 
195
169
  private start(): void {
196
170
  this.started = Date.now() as UnixTimestampMillis
@@ -199,21 +173,12 @@ export class ProgressLogger<T> implements Disposable {
199
173
  this.processedLastSecond = 0
200
174
  this.progress = 0
201
175
  this.peakRSS = 0
202
- this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined
203
- this.sizesZipped = this.cfg.logZippedSizes
204
- ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
205
- : undefined
206
176
  }
207
177
 
208
178
  log(chunk?: T): void {
209
179
  this.progress++
210
180
  this.processedLastSecond++
211
181
 
212
- if (this.sizes) {
213
- // Check it, cause gzipping might be delayed here..
214
- void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped)
215
- }
216
-
217
182
  if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
218
183
  this.logStats(chunk, false, this.progress % this.logEvery10 === 0)
219
184
  }
@@ -275,14 +240,6 @@ export class ProgressLogger<T> implements Disposable {
275
240
 
276
241
  logger.log(inspect(o, inspectOpt))
277
242
 
278
- if (this.sizes?.items.length) {
279
- logger.log(this.sizes.getStats())
280
-
281
- if (this.sizesZipped?.items.length) {
282
- logger.log(this.sizesZipped.getStats())
283
- }
284
- }
285
-
286
243
  if (tenx) {
287
244
  const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started))
288
245
 
package/src/zip/zip2.ts ADDED
@@ -0,0 +1,159 @@
1
+ import { promisify } from 'node:util'
2
+ import type { ZlibOptions, ZstdOptions } from 'node:zlib'
3
+ import zlib from 'node:zlib'
4
+ import type { Integer } from '@naturalcycles/js-lib/types'
5
+
6
+ const deflateAsync = promisify(zlib.deflate.bind(zlib))
7
+ const inflateAsync = promisify(zlib.inflate.bind(zlib))
8
+ const gzipAsync = promisify(zlib.gzip.bind(zlib))
9
+ const gunzipAsync = promisify(zlib.gunzip.bind(zlib))
10
+ const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
11
+ const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
12
+
13
+ class Zip2 {
14
+ async decompressZstdOrInflateToString(buf: Buffer): Promise<string> {
15
+ return (await this.decompressZstdOrInflate(buf)).toString()
16
+ }
17
+
18
+ decompressZstdOrInflateToStringSync(buf: Buffer): string {
19
+ return this.decompressZstdOrInflateSync(buf).toString()
20
+ }
21
+
22
+ /**
23
+ * Detects if Buffer is zstd-compressed.
24
+ * Otherwise attempts to Inflate.
25
+ */
26
+ async decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>> {
27
+ if (this.isZstdBuffer(buf)) {
28
+ return await zstdDecompressAsync(buf)
29
+ }
30
+ return await inflateAsync(buf)
31
+ }
32
+
33
+ decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer> {
34
+ if (this.isZstdBuffer(buf)) {
35
+ return zlib.zstdDecompressSync(buf)
36
+ }
37
+ return zlib.inflateSync(buf)
38
+ }
39
+
40
+ /**
41
+ * deflateBuffer uses `deflate`.
42
+ * It's 9 bytes shorter than `gzip`.
43
+ */
44
+ async deflate(input: string | Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
45
+ return await deflateAsync(input, options)
46
+ }
47
+
48
+ /**
49
+ * deflateSync uses `deflate`.
50
+ * It's 9 bytes shorter than `gzip`.
51
+ */
52
+ deflateSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer> {
53
+ return zlib.deflateSync(input, options)
54
+ }
55
+
56
+ async inflate(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
57
+ return await inflateAsync(buf, options)
58
+ }
59
+
60
+ inflateSync(buf: Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
61
+ return zlib.inflateSync(buf, options)
62
+ }
63
+
64
+ async inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
65
+ return (await this.inflate(buf, options)).toString()
66
+ }
67
+
68
+ inflateToStringSync(buf: Buffer, options?: ZlibOptions): string {
69
+ return zlib.inflateSync(buf, options).toString()
70
+ }
71
+
72
+ /**
73
+ * gzipBuffer uses `gzip`
74
+ * It's 9 bytes longer than `deflate`.
75
+ */
76
+ async gzip(input: string | Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
77
+ return await gzipAsync(input, options)
78
+ }
79
+
80
+ /**
81
+ * gzipBuffer uses `gzip`
82
+ * It's 9 bytes longer than `deflate`.
83
+ */
84
+ gzipSync(input: string | Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
85
+ return zlib.gzipSync(input, options)
86
+ }
87
+
88
+ async gunzip(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
89
+ return await gunzipAsync(buf, options)
90
+ }
91
+
92
+ gunzipSync(buf: Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
93
+ return zlib.gunzipSync(buf, options)
94
+ }
95
+
96
+ async gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
97
+ return (await this.gunzip(buf, options)).toString()
98
+ }
99
+
100
+ gunzipToStringSync(buf: Buffer, options?: ZlibOptions): string {
101
+ return zlib.gunzipSync(buf, options).toString()
102
+ }
103
+
104
+ async zstdCompress(
105
+ input: Buffer | string,
106
+ level?: Integer, // defaults to 3
107
+ options: ZstdOptions = {},
108
+ ): Promise<Buffer<ArrayBuffer>> {
109
+ return await zstdCompressAsync(input, this.zstdLevelToOptions(level, options))
110
+ }
111
+
112
+ zstdCompressSync(
113
+ input: Buffer | string,
114
+ level?: Integer, // defaults to 3
115
+ options: ZstdOptions = {},
116
+ ): Buffer<ArrayBuffer> {
117
+ return zlib.zstdCompressSync(input, this.zstdLevelToOptions(level, options))
118
+ }
119
+
120
+ zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
121
+ if (!level) return opt
122
+
123
+ return {
124
+ ...opt,
125
+ params: {
126
+ ...opt.params,
127
+ [zlib.constants.ZSTD_c_compressionLevel]: level,
128
+ },
129
+ }
130
+ }
131
+
132
+ async zstdDecompressToString(input: Buffer, options: ZstdOptions = {}): Promise<string> {
133
+ return (await zstdDecompressAsync(input, options)).toString()
134
+ }
135
+
136
+ async zstdDecompress(input: Buffer, options: ZstdOptions = {}): Promise<Buffer<ArrayBuffer>> {
137
+ return await zstdDecompressAsync(input, options)
138
+ }
139
+
140
+ zstdDecompressToStringSync(input: Buffer, options: ZstdOptions = {}): string {
141
+ return zlib.zstdDecompressSync(input, options).toString()
142
+ }
143
+
144
+ zstdDecompressSync(input: Buffer, options: ZstdOptions = {}): Buffer<ArrayBuffer> {
145
+ return zlib.zstdDecompressSync(input, options)
146
+ }
147
+
148
+ isZstdBuffer(input: Buffer): boolean {
149
+ return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER
150
+ }
151
+
152
+ isGzipBuffer(input: Buffer): boolean {
153
+ return input[0] === 0x1f && input[1] === 0x8b
154
+ }
155
+ }
156
+
157
+ export const zip2 = new Zip2()
158
+
159
+ const ZSTD_MAGIC_NUMBER = 0xfd2fb528
package/dist/stream/sizeStack.d.ts DELETED
@@ -1,10 +0,0 @@
1
- import type { CommonLogger } from '@naturalcycles/js-lib/log';
2
- import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js';
3
- export declare class SizeStack extends NumberStack {
4
- name: string;
5
- constructor(name: string, size: number);
6
- total: number;
7
- push(item: any): this;
8
- getStats(): string;
9
- static countItem(item: any, logger: CommonLogger, sizes?: SizeStack, sizesZipped?: SizeStack): Promise<void>;
10
- }
package/dist/stream/sizeStack.js DELETED
@@ -1,46 +0,0 @@
1
- import { _hb } from '@naturalcycles/js-lib';
2
- import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js';
3
- import { yellow } from '../colors/colors.js';
4
- import { gzipBuffer } from '../zip/zip.util.js';
5
- export class SizeStack extends NumberStack {
6
- name;
7
- constructor(name, size) {
8
- super(size);
9
- this.name = name;
10
- }
11
- total = 0;
12
- push(item) {
13
- this.total += item;
14
- return super.push(item);
15
- }
16
- getStats() {
17
- // const pcs = this.percentiles([50, 90])
18
- return [
19
- ' ' + this.name,
20
- 'avg',
21
- yellow(_hb(this.avg())),
22
- // 'p50',
23
- // yellow(_hb(pcs[50])),
24
- // 'p90',
25
- // yellow(_hb(pcs[90])),
26
- 'total',
27
- yellow(_hb(this.total)),
28
- ].join(' ');
29
- }
30
- static async countItem(item, logger, sizes, sizesZipped) {
31
- if (!sizes)
32
- return;
33
- // try-catch, because we don't want to fail the pipeline on logProgress
34
- try {
35
- const buf = Buffer.from(JSON.stringify(item));
36
- sizes.push(buf.byteLength);
37
- if (sizesZipped) {
38
- const { byteLength } = await gzipBuffer(buf);
39
- sizesZipped.push(byteLength);
40
- }
41
- }
42
- catch (err) {
43
- logger.warn(`transformLogProgress failed to JSON.stringify the chunk: ${err.message}`);
44
- }
45
- }
46
- }
package/dist/zip/zip.util.d.ts DELETED
@@ -1,43 +0,0 @@
1
- import type { ZlibOptions, ZstdOptions } from 'node:zlib';
2
- import type { Integer } from '@naturalcycles/js-lib/types';
3
- export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
4
- export declare function decompressZstdOrInflateToStringSync(buf: Buffer): string;
5
- /**
6
- * Detects if Buffer is zstd-compressed.
7
- * Otherwise attempts to Inflate.
8
- */
9
- export declare function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>>;
10
- export declare function decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer>;
11
- /**
12
- * deflateBuffer uses `deflate`.
13
- * It's 9 bytes shorter than `gzip`.
14
- */
15
- export declare function deflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
16
- export declare function inflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
17
- /**
18
- * deflateString uses `deflate`.
19
- * It's 9 bytes shorter than `gzip`.
20
- */
21
- export declare function deflateString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
22
- export declare function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
23
- /**
24
- * gzipBuffer uses `gzip`
25
- * It's 9 bytes longer than `deflate`.
26
- */
27
- export declare function gzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
28
- export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
29
- /**
30
- * gzipString uses `gzip`.
31
- * It's 9 bytes longer than `deflate`.
32
- */
33
- export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
34
- export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
35
- export declare function zstdCompress(input: Buffer | string, level?: Integer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
36
- export declare function zstdCompressSync(input: Buffer | string, level?: Integer, options?: ZstdOptions): Buffer<ArrayBuffer>;
37
- export declare function zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
38
- export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
39
- export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
40
- export declare function zstdDecompressToStringSync(input: Buffer, options?: ZstdOptions): string;
41
- export declare function zstdDecompressSync(input: Buffer, options?: ZstdOptions): Buffer<ArrayBuffer>;
42
- export declare function isZstdBuffer(input: Buffer): boolean;
43
- export declare function isGzipBuffer(input: Buffer): boolean;
package/dist/zip/zip.util.js DELETED
@@ -1,108 +0,0 @@
1
- import { promisify } from 'node:util';
2
- import zlib from 'node:zlib';
3
- const deflate = promisify(zlib.deflate.bind(zlib));
4
- const inflate = promisify(zlib.inflate.bind(zlib));
5
- const gzip = promisify(zlib.gzip.bind(zlib));
6
- const gunzip = promisify(zlib.gunzip.bind(zlib));
7
- const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib));
8
- const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
9
- export async function decompressZstdOrInflateToString(buf) {
10
- return (await decompressZstdOrInflate(buf)).toString();
11
- }
12
- export function decompressZstdOrInflateToStringSync(buf) {
13
- return decompressZstdOrInflateSync(buf).toString();
14
- }
15
- /**
16
- * Detects if Buffer is zstd-compressed.
17
- * Otherwise attempts to Inflate.
18
- */
19
- export async function decompressZstdOrInflate(buf) {
20
- if (isZstdBuffer(buf)) {
21
- return await zstdDecompressAsync(buf);
22
- }
23
- return await inflate(buf);
24
- }
25
- export function decompressZstdOrInflateSync(buf) {
26
- if (isZstdBuffer(buf)) {
27
- return zlib.zstdDecompressSync(buf);
28
- }
29
- return zlib.inflateSync(buf);
30
- }
31
- /**
32
- * deflateBuffer uses `deflate`.
33
- * It's 9 bytes shorter than `gzip`.
34
- */
35
- export async function deflateBuffer(buf, options = {}) {
36
- return await deflate(buf, options);
37
- }
38
- export async function inflateBuffer(buf, options = {}) {
39
- return await inflate(buf, options);
40
- }
41
- /**
42
- * deflateString uses `deflate`.
43
- * It's 9 bytes shorter than `gzip`.
44
- */
45
- export async function deflateString(s, options) {
46
- return await deflate(s, options);
47
- }
48
- export async function inflateToString(buf, options) {
49
- return (await inflateBuffer(buf, options)).toString();
50
- }
51
- /**
52
- * gzipBuffer uses `gzip`
53
- * It's 9 bytes longer than `deflate`.
54
- */
55
- export async function gzipBuffer(buf, options = {}) {
56
- return await gzip(buf, options);
57
- }
58
- export async function gunzipBuffer(buf, options = {}) {
59
- return await gunzip(buf, options);
60
- }
61
- /**
62
- * gzipString uses `gzip`.
63
- * It's 9 bytes longer than `deflate`.
64
- */
65
- export async function gzipString(s, options) {
66
- return await gzip(s, options);
67
- }
68
- export async function gunzipToString(buf, options) {
69
- return (await gunzipBuffer(buf, options)).toString();
70
- }
71
- export async function zstdCompress(input, level, // defaults to 3
72
- options = {}) {
73
- return await zstdCompressAsync(input, zstdLevelToOptions(level, options));
74
- }
75
- export function zstdCompressSync(input, level, // defaults to 3
76
- options = {}) {
77
- return zlib.zstdCompressSync(input, zstdLevelToOptions(level, options));
78
- }
79
- export function zstdLevelToOptions(level, opt = {}) {
80
- if (!level)
81
- return opt;
82
- return {
83
- ...opt,
84
- params: {
85
- ...opt.params,
86
- [zlib.constants.ZSTD_c_compressionLevel]: level,
87
- },
88
- };
89
- }
90
- export async function zstdDecompressToString(input, options = {}) {
91
- return (await zstdDecompressAsync(input, options)).toString();
92
- }
93
- export async function zstdDecompress(input, options = {}) {
94
- return await zstdDecompressAsync(input, options);
95
- }
96
- export function zstdDecompressToStringSync(input, options = {}) {
97
- return zlib.zstdDecompressSync(input, options).toString();
98
- }
99
- export function zstdDecompressSync(input, options = {}) {
100
- return zlib.zstdDecompressSync(input, options);
101
- }
102
- const ZSTD_MAGIC_NUMBER = 0xfd2fb528;
103
- export function isZstdBuffer(input) {
104
- return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER;
105
- }
106
- export function isGzipBuffer(input) {
107
- return input[0] === 0x1f && input[1] === 0x8b;
108
- }
package/src/stream/sizeStack.ts DELETED
@@ -1,61 +0,0 @@
1
- import { _hb } from '@naturalcycles/js-lib'
2
- import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
- import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js'
4
- import { yellow } from '../colors/colors.js'
5
- import { gzipBuffer } from '../zip/zip.util.js'
6
-
7
- export class SizeStack extends NumberStack {
8
- constructor(
9
- public name: string,
10
- size: number,
11
- ) {
12
- super(size)
13
- }
14
-
15
- total = 0
16
-
17
- override push(item: any): this {
18
- this.total += item
19
- return super.push(item)
20
- }
21
-
22
- getStats(): string {
23
- // const pcs = this.percentiles([50, 90])
24
-
25
- return [
26
- ' ' + this.name,
27
- 'avg',
28
- yellow(_hb(this.avg())),
29
- // 'p50',
30
- // yellow(_hb(pcs[50])),
31
- // 'p90',
32
- // yellow(_hb(pcs[90])),
33
- 'total',
34
- yellow(_hb(this.total)),
35
- ].join(' ')
36
- }
37
-
38
- static async countItem(
39
- item: any,
40
- logger: CommonLogger,
41
- sizes?: SizeStack,
42
- sizesZipped?: SizeStack,
43
- ): Promise<void> {
44
- if (!sizes) return
45
-
46
- // try-catch, because we don't want to fail the pipeline on logProgress
47
- try {
48
- const buf = Buffer.from(JSON.stringify(item))
49
- sizes.push(buf.byteLength)
50
-
51
- if (sizesZipped) {
52
- const { byteLength } = await gzipBuffer(buf)
53
- sizesZipped.push(byteLength)
54
- }
55
- } catch (err) {
56
- logger.warn(
57
- `transformLogProgress failed to JSON.stringify the chunk: ${(err as Error).message}`,
58
- )
59
- }
60
- }
61
- }
package/src/zip/zip.util.ts DELETED
@@ -1,160 +0,0 @@
1
- import { promisify } from 'node:util'
2
- import type { ZlibOptions, ZstdOptions } from 'node:zlib'
3
- import zlib from 'node:zlib'
4
- import type { Integer } from '@naturalcycles/js-lib/types'
5
-
6
- const deflate = promisify(zlib.deflate.bind(zlib))
7
- const inflate = promisify(zlib.inflate.bind(zlib))
8
- const gzip = promisify(zlib.gzip.bind(zlib))
9
- const gunzip = promisify(zlib.gunzip.bind(zlib))
10
- const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
11
- const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
12
-
13
- export async function decompressZstdOrInflateToString(buf: Buffer): Promise<string> {
14
- return (await decompressZstdOrInflate(buf)).toString()
15
- }
16
-
17
- export function decompressZstdOrInflateToStringSync(buf: Buffer): string {
18
- return decompressZstdOrInflateSync(buf).toString()
19
- }
20
-
21
- /**
22
- * Detects if Buffer is zstd-compressed.
23
- * Otherwise attempts to Inflate.
24
- */
25
- export async function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>> {
26
- if (isZstdBuffer(buf)) {
27
- return await zstdDecompressAsync(buf)
28
- }
29
- return await inflate(buf)
30
- }
31
-
32
- export function decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer> {
33
- if (isZstdBuffer(buf)) {
34
- return zlib.zstdDecompressSync(buf)
35
- }
36
- return zlib.inflateSync(buf)
37
- }
38
-
39
- /**
40
- * deflateBuffer uses `deflate`.
41
- * It's 9 bytes shorter than `gzip`.
42
- */
43
- export async function deflateBuffer(
44
- buf: Buffer,
45
- options: ZlibOptions = {},
46
- ): Promise<Buffer<ArrayBuffer>> {
47
- return await deflate(buf, options)
48
- }
49
-
50
- export async function inflateBuffer(
51
- buf: Buffer,
52
- options: ZlibOptions = {},
53
- ): Promise<Buffer<ArrayBuffer>> {
54
- return await inflate(buf, options)
55
- }
56
-
57
- /**
58
- * deflateString uses `deflate`.
59
- * It's 9 bytes shorter than `gzip`.
60
- */
61
- export async function deflateString(
62
- s: string,
63
- options?: ZlibOptions,
64
- ): Promise<Buffer<ArrayBuffer>> {
65
- return await deflate(s, options)
66
- }
67
-
68
- export async function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
69
- return (await inflateBuffer(buf, options)).toString()
70
- }
71
-
72
- /**
73
- * gzipBuffer uses `gzip`
74
- * It's 9 bytes longer than `deflate`.
75
- */
76
- export async function gzipBuffer(
77
- buf: Buffer,
78
- options: ZlibOptions = {},
79
- ): Promise<Buffer<ArrayBuffer>> {
80
- return await gzip(buf, options)
81
- }
82
-
83
- export async function gunzipBuffer(
84
- buf: Buffer,
85
- options: ZlibOptions = {},
86
- ): Promise<Buffer<ArrayBuffer>> {
87
- return await gunzip(buf, options)
88
- }
89
-
90
- /**
91
- * gzipString uses `gzip`.
92
- * It's 9 bytes longer than `deflate`.
93
- */
94
- export async function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>> {
95
- return await gzip(s, options)
96
- }
97
-
98
- export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
99
- return (await gunzipBuffer(buf, options)).toString()
100
- }
101
-
102
- export async function zstdCompress(
103
- input: Buffer | string,
104
- level?: Integer, // defaults to 3
105
- options: ZstdOptions = {},
106
- ): Promise<Buffer<ArrayBuffer>> {
107
- return await zstdCompressAsync(input, zstdLevelToOptions(level, options))
108
- }
109
-
110
- export function zstdCompressSync(
111
- input: Buffer | string,
112
- level?: Integer, // defaults to 3
113
- options: ZstdOptions = {},
114
- ): Buffer<ArrayBuffer> {
115
- return zlib.zstdCompressSync(input, zstdLevelToOptions(level, options))
116
- }
117
-
118
- export function zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
119
- if (!level) return opt
120
-
121
- return {
122
- ...opt,
123
- params: {
124
- ...opt.params,
125
- [zlib.constants.ZSTD_c_compressionLevel]: level,
126
- },
127
- }
128
- }
129
-
130
- export async function zstdDecompressToString(
131
- input: Buffer,
132
- options: ZstdOptions = {},
133
- ): Promise<string> {
134
- return (await zstdDecompressAsync(input, options)).toString()
135
- }
136
-
137
- export async function zstdDecompress(
138
- input: Buffer,
139
- options: ZstdOptions = {},
140
- ): Promise<Buffer<ArrayBuffer>> {
141
- return await zstdDecompressAsync(input, options)
142
- }
143
-
144
- export function zstdDecompressToStringSync(input: Buffer, options: ZstdOptions = {}): string {
145
- return zlib.zstdDecompressSync(input, options).toString()
146
- }
147
-
148
- export function zstdDecompressSync(input: Buffer, options: ZstdOptions = {}): Buffer<ArrayBuffer> {
149
- return zlib.zstdDecompressSync(input, options)
150
- }
151
-
152
- const ZSTD_MAGIC_NUMBER = 0xfd2fb528
153
-
154
- export function isZstdBuffer(input: Buffer): boolean {
155
- return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER
156
- }
157
-
158
- export function isGzipBuffer(input: Buffer): boolean {
159
- return input[0] === 0x1f && input[1] === 0x8b
160
- }