@naturalcycles/nodejs-lib 15.103.0 → 15.105.0

package/dist/stream/pipeline.js

@@ -4,7 +4,7 @@ import { createGzip, createUnzip, createZstdCompress, createZstdDecompress } fro
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { _passthroughPredicate } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
-import { zstdLevelToOptions } from '../zip/zip.util.js';
+import { zip2 } from '../zip/zip2.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
@@ -262,7 +262,7 @@ export class Pipeline {
     }
     zstdCompress(level, // defaults to 3
     opt) {
-        this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)));
+        this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level, opt)));
         this.objectMode = false;
         return this;
     }
@@ -302,7 +302,7 @@ export class Pipeline {
             }));
         }
         else if (outputFilePath.endsWith('.zst')) {
-            this.transforms.push(createZstdCompress(zstdLevelToOptions(level)));
+            this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level)));
         }
         this.destination = fs2.createWriteStream(outputFilePath, {});
         await this.run();

package/dist/stream/progressLogger.d.ts

@@ -90,26 +90,6 @@ export interface ProgressLoggerCfg<T = any> {
      * Defaults to 1.
      */
     chunkSize?: PositiveInteger;
-    /**
-     * Experimental logging of item (shunk) sizes, when json-stringified.
-     *
-     * Defaults to false.
-     *
-     * @experimental
-     */
-    logSizes?: boolean;
-    /**
-     * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
-     * Defaults to 100_000.
-     * Cannot be Infinity.
-     */
-    logSizesBuffer?: number;
-    /**
-     * Works in addition to `logSizes`. Adds "zipped sizes".
-     *
-     * @experimental
-     */
-    logZippedSizes?: boolean;
 }
 export interface ProgressLogItem extends AnyObject {
     heapUsed?: number;
@@ -138,8 +118,6 @@ export declare class ProgressLogger<T> implements Disposable {
     private processedLastSecond;
     private progress;
     private peakRSS;
-    private sizes?;
-    private sizesZipped?;
     private start;
     log(chunk?: T): void;
     done(): void;

package/dist/stream/progressLogger.js

@@ -3,7 +3,6 @@ import { _hc, _mb } from '@naturalcycles/js-lib';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
-import { SizeStack } from './sizeStack.js';
 const inspectOpt = {
     colors: hasColors,
     breakLength: 300,
@@ -34,8 +33,6 @@ export class ProgressLogger {
     processedLastSecond;
     progress;
     peakRSS;
-    sizes;
-    sizesZipped;
     start() {
         this.started = Date.now();
         this.lastSecondStarted = Date.now();
@@ -43,18 +40,10 @@ export class ProgressLogger {
         this.processedLastSecond = 0;
         this.progress = 0;
         this.peakRSS = 0;
-        this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined;
-        this.sizesZipped = this.cfg.logZippedSizes
-            ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
-            : undefined;
     }
     log(chunk) {
         this.progress++;
         this.processedLastSecond++;
-        if (this.sizes) {
-            // Check it, cause gzipping might be delayed here..
-            void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped);
-        }
         if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
             this.logStats(chunk, false, this.progress % this.logEvery10 === 0);
         }
@@ -101,12 +90,6 @@ export class ProgressLogger {
         if (logRPS)
             Object.assign(o, { rps10, rpsTotal });
         logger.log(inspect(o, inspectOpt));
-        if (this.sizes?.items.length) {
-            logger.log(this.sizes.getStats());
-            if (this.sizesZipped?.items.length) {
-                logger.log(this.sizesZipped.getStats());
-            }
-        }
         if (tenx) {
             const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started));
             logger.log(`${dimGrey(localTime.now().toPretty())} ${white(metric)} took ${yellow(_since(this.started))} so far to process ${yellow(_hc(batchedProgress))} rows, ~${yellow(perHour)}/hour`);

package/dist/stream/transform/transformChunk.js

@@ -11,6 +11,7 @@ export function transformChunk(chunkSize, opt) {
     let buf = [];
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             buf.push(chunk);

package/dist/stream/transform/transformFilter.js

@@ -16,6 +16,7 @@ export function transformFilterSync(predicate, opt = {}) {
     let index = 0;
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             try {

package/dist/stream/transform/transformFlatten.js

@@ -2,6 +2,7 @@ import { Transform } from 'node:stream';
 export function transformFlatten() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             for (const item of chunk) {
                 this.push(item);
@@ -13,6 +14,7 @@ export function transformFlatten() {
 export function transformFlattenIfNeeded() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             if (Array.isArray(chunk)) {
                 for (const item of chunk) {

package/dist/stream/transform/transformFork.js

@@ -12,7 +12,7 @@ import { createReadable } from '../readable/createReadable.js';
  * @experimental
  */
 export function transformFork(fn, opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
+    const { objectMode = true, highWaterMark = 1 } = opt;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let lock;
     const fork = createReadable([], {}, () => {

package/dist/stream/transform/transformLimit.js

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 import { transformNoOp } from './transformNoOp.js';
 export function transformLimit(opt) {
-    const { limit, signal, objectMode = true, highWaterMark } = opt;
+    const { limit, signal, objectMode = true, highWaterMark = 1 } = opt;
     if (!limit) {
         return transformNoOp();
     }

package/dist/stream/transform/transformMapSimple.js

@@ -11,7 +11,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
  */
 export function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark, } = opt;
+    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark = 1, } = opt;
     return new Transform({
         objectMode,
         highWaterMark,

package/dist/stream/transform/transformMapSync.js

@@ -20,6 +20,7 @@ export function transformMapSync(mapper, opt = {}) {
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     return new Transform({
         objectMode,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             // Stop processing if isSettled

package/dist/stream/transform/transformNoOp.js

@@ -7,6 +7,7 @@ import { Transform } from 'node:stream';
 export function transformNoOp() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             cb(null, chunk);
         },

package/dist/stream/transform/transformOffset.js

@@ -9,6 +9,7 @@ export function transformOffset(opt) {
     let i = 0; // so we start first chunk with 1
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             if (++i <= offset) {

package/dist/stream/transform/transformThrottle.js

@@ -20,7 +20,7 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  * @experimental
  */
 export function transformThrottle(opt) {
-    const { throughput, interval, objectMode = true, highWaterMark } = opt;
+    const { throughput, interval, objectMode = true, highWaterMark = 1 } = opt;
     let count = 0;
     let start;
     let lock;

package/dist/stream/transform/transformThrottleByRSS.js

@@ -15,7 +15,7 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  */
 export function transformThrottleByRSS(opt) {
     const { maxRSS, pollInterval = 5000, pollTimeout = 30 * 60_000, // 30 min
-    onPollTimeout = 'open-the-floodgates', objectMode = true, highWaterMark, } = opt;
+    onPollTimeout = 'open-the-floodgates', objectMode = true, highWaterMark = 1, } = opt;
     const maxRSSBytes = maxRSS * 1024 * 1024;
     let lock;
     let pollTimer;

package/dist/stream/transform/transformWarmup.js

@@ -12,7 +12,7 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  * @experimental
  */
 export function transformWarmup(opt) {
-    const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt;
+    const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt;
     const warmupMs = warmupSeconds * 1000;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let startTime = 0;

package/dist/zip/zip2.d.ts

@@ -0,0 +1,51 @@
+import type { ZlibOptions, ZstdOptions } from 'node:zlib';
+import type { Integer } from '@naturalcycles/js-lib/types';
+declare class Zip2 {
+    decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
+    decompressZstdOrInflateToStringSync(buf: Buffer): string;
+    /**
+     * Detects if Buffer is zstd-compressed.
+     * Otherwise attempts to Inflate.
+     */
+    decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>>;
+    decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer>;
+    /**
+     * deflateBuffer uses `deflate`.
+     * It's 9 bytes shorter than `gzip`.
+     */
+    deflate(input: string | Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
+    /**
+     * deflateSync uses `deflate`.
+     * It's 9 bytes shorter than `gzip`.
+     */
+    deflateSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
+    inflate(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
+    inflateSync(buf: Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
+    inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
+    inflateToStringSync(buf: Buffer, options?: ZlibOptions): string;
+    /**
+     * gzipBuffer uses `gzip`
+     * It's 9 bytes longer than `deflate`.
+     */
+    gzip(input: string | Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
+    /**
+     * gzipBuffer uses `gzip`
+     * It's 9 bytes longer than `deflate`.
+     */
+    gzipSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
+    gunzip(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
+    gunzipSync(buf: Buffer, options?: ZlibOptions): Buffer<ArrayBuffer>;
+    gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
+    gunzipToStringSync(buf: Buffer, options?: ZlibOptions): string;
+    zstdCompress(input: Buffer | string, level?: Integer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+    zstdCompressSync(input: Buffer | string, level?: Integer, options?: ZstdOptions): Buffer<ArrayBuffer>;
+    zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
+    zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
+    zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+    zstdDecompressToStringSync(input: Buffer, options?: ZstdOptions): string;
+    zstdDecompressSync(input: Buffer, options?: ZstdOptions): Buffer<ArrayBuffer>;
+    isZstdBuffer(input: Buffer): boolean;
+    isGzipBuffer(input: Buffer): boolean;
+}
+export declare const zip2: Zip2;
+export {};

package/dist/zip/zip2.js

@@ -0,0 +1,123 @@
+import { promisify } from 'node:util';
+import zlib from 'node:zlib';
+const deflateAsync = promisify(zlib.deflate.bind(zlib));
+const inflateAsync = promisify(zlib.inflate.bind(zlib));
+const gzipAsync = promisify(zlib.gzip.bind(zlib));
+const gunzipAsync = promisify(zlib.gunzip.bind(zlib));
+const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib));
+const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
+class Zip2 {
+    async decompressZstdOrInflateToString(buf) {
+        return (await this.decompressZstdOrInflate(buf)).toString();
+    }
+    decompressZstdOrInflateToStringSync(buf) {
+        return this.decompressZstdOrInflateSync(buf).toString();
+    }
+    /**
+     * Detects if Buffer is zstd-compressed.
+     * Otherwise attempts to Inflate.
+     */
+    async decompressZstdOrInflate(buf) {
+        if (this.isZstdBuffer(buf)) {
+            return await zstdDecompressAsync(buf);
+        }
+        return await inflateAsync(buf);
+    }
+    decompressZstdOrInflateSync(buf) {
+        if (this.isZstdBuffer(buf)) {
+            return zlib.zstdDecompressSync(buf);
+        }
+        return zlib.inflateSync(buf);
+    }
+    /**
+     * deflateBuffer uses `deflate`.
+     * It's 9 bytes shorter than `gzip`.
+     */
+    async deflate(input, options = {}) {
+        return await deflateAsync(input, options);
+    }
+    /**
+     * deflateSync uses `deflate`.
+     * It's 9 bytes shorter than `gzip`.
+     */
+    deflateSync(input, options) {
+        return zlib.deflateSync(input, options);
+    }
+    async inflate(buf, options = {}) {
+        return await inflateAsync(buf, options);
+    }
+    inflateSync(buf, options = {}) {
+        return zlib.inflateSync(buf, options);
+    }
+    async inflateToString(buf, options) {
+        return (await this.inflate(buf, options)).toString();
+    }
+    inflateToStringSync(buf, options) {
+        return zlib.inflateSync(buf, options).toString();
+    }
+    /**
+     * gzipBuffer uses `gzip`
+     * It's 9 bytes longer than `deflate`.
+     */
+    async gzip(input, options = {}) {
+        return await gzipAsync(input, options);
+    }
+    /**
+     * gzipBuffer uses `gzip`
+     * It's 9 bytes longer than `deflate`.
+     */
+    gzipSync(input, options = {}) {
+        return zlib.gzipSync(input, options);
+    }
+    async gunzip(buf, options = {}) {
+        return await gunzipAsync(buf, options);
+    }
+    gunzipSync(buf, options = {}) {
+        return zlib.gunzipSync(buf, options);
+    }
+    async gunzipToString(buf, options) {
+        return (await this.gunzip(buf, options)).toString();
+    }
+    gunzipToStringSync(buf, options) {
+        return zlib.gunzipSync(buf, options).toString();
+    }
+    async zstdCompress(input, level, // defaults to 3
+    options = {}) {
+        return await zstdCompressAsync(input, this.zstdLevelToOptions(level, options));
+    }
+    zstdCompressSync(input, level, // defaults to 3
+    options = {}) {
+        return zlib.zstdCompressSync(input, this.zstdLevelToOptions(level, options));
+    }
+    zstdLevelToOptions(level, opt = {}) {
+        if (!level)
+            return opt;
+        return {
+            ...opt,
+            params: {
+                ...opt.params,
+                [zlib.constants.ZSTD_c_compressionLevel]: level,
+            },
+        };
+    }
+    async zstdDecompressToString(input, options = {}) {
+        return (await zstdDecompressAsync(input, options)).toString();
+    }
+    async zstdDecompress(input, options = {}) {
+        return await zstdDecompressAsync(input, options);
+    }
+    zstdDecompressToStringSync(input, options = {}) {
+        return zlib.zstdDecompressSync(input, options).toString();
+    }
+    zstdDecompressSync(input, options = {}) {
+        return zlib.zstdDecompressSync(input, options);
+    }
+    isZstdBuffer(input) {
+        return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER;
+    }
+    isGzipBuffer(input) {
+        return input[0] === 0x1f && input[1] === 0x8b;
+    }
+}
+export const zip2 = new Zip2();
+const ZSTD_MAGIC_NUMBER = 0xfd2fb528;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.103.0",
+  "version": "15.105.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",
@@ -38,7 +38,7 @@
     "./stream/*.js": "./dist/stream/*.js",
     "./yargs": "./dist/yargs/yargs.util.js",
     "./ajv": "./dist/validation/ajv/index.js",
-    "./zip": "./dist/zip/zip.util.js"
+    "./zip": "./dist/zip/zip2.js"
   },
   "bin": {
     "kpy": "dist/bin/kpy.js",

package/src/stream/pipeline.ts

@@ -19,7 +19,7 @@ import type {
   SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
-import { zstdLevelToOptions } from '../zip/zip.util.js'
+import { zip2 } from '../zip/zip2.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
@@ -359,7 +359,7 @@ export class Pipeline<T = unknown> {
     level?: Integer, // defaults to 3
     opt?: ZstdOptions,
   ): Pipeline<Uint8Array> {
-    this.transforms.push(createZstdCompress(zstdLevelToOptions(level, opt)))
+    this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level, opt)))
     this.objectMode = false
     return this as any
   }
@@ -406,7 +406,7 @@ export class Pipeline<T = unknown> {
         }),
       )
     } else if (outputFilePath.endsWith('.zst')) {
-      this.transforms.push(createZstdCompress(zstdLevelToOptions(level)))
+      this.transforms.push(createZstdCompress(zip2.zstdLevelToOptions(level)))
     }
     this.destination = fs2.createWriteStream(outputFilePath, {
       // highWaterMark: 64 * 1024, // no observed speedup

package/src/stream/progressLogger.ts

@@ -6,7 +6,6 @@ import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
 import type { AnyObject, PositiveInteger, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js'
-import { SizeStack } from './sizeStack.js'
 import type { ReadableMapper } from './stream.model.js'
 
 export interface ProgressLoggerCfg<T = any> {
@@ -112,29 +111,6 @@ export interface ProgressLoggerCfg<T = any> {
    * Defaults to 1.
    */
   chunkSize?: PositiveInteger
-
-  /**
-   * Experimental logging of item (shunk) sizes, when json-stringified.
-   *
-   * Defaults to false.
-   *
-   * @experimental
-   */
-  logSizes?: boolean
-
-  /**
-   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
-   * Defaults to 100_000.
-   * Cannot be Infinity.
-   */
-  logSizesBuffer?: number
-
-  /**
-   * Works in addition to `logSizes`. Adds "zipped sizes".
-   *
-   * @experimental
-   */
-  logZippedSizes?: boolean
 }
 
 export interface ProgressLogItem extends AnyObject {
@@ -189,8 +165,6 @@ export class ProgressLogger<T> implements Disposable {
   private processedLastSecond!: number
   private progress!: number
   private peakRSS!: number
-  private sizes?: SizeStack
-  private sizesZipped?: SizeStack
 
   private start(): void {
     this.started = Date.now() as UnixTimestampMillis
@@ -199,21 +173,12 @@ export class ProgressLogger<T> implements Disposable {
     this.processedLastSecond = 0
     this.progress = 0
     this.peakRSS = 0
-    this.sizes = this.cfg.logSizes ? new SizeStack('json', this.cfg.logSizesBuffer) : undefined
-    this.sizesZipped = this.cfg.logZippedSizes
-      ? new SizeStack('json.gz', this.cfg.logSizesBuffer)
-      : undefined
   }
 
   log(chunk?: T): void {
     this.progress++
     this.processedLastSecond++
 
-    if (this.sizes) {
-      // Check it, cause gzipping might be delayed here..
-      void SizeStack.countItem(chunk, this.cfg.logger, this.sizes, this.sizesZipped)
-    }
-
     if (this.cfg.logProgress && this.progress % this.cfg.logEvery === 0) {
       this.logStats(chunk, false, this.progress % this.logEvery10 === 0)
     }
@@ -275,14 +240,6 @@ export class ProgressLogger<T> implements Disposable {
 
     logger.log(inspect(o, inspectOpt))
 
-    if (this.sizes?.items.length) {
-      logger.log(this.sizes.getStats())
-
-      if (this.sizesZipped?.items.length) {
-        logger.log(this.sizesZipped.getStats())
-      }
-    }
-
     if (tenx) {
       const perHour = _hc((batchedProgress * 1000 * 60 * 60) / (now - this.started))
 

package/src/stream/transform/transformChunk.ts

@@ -18,6 +18,7 @@ export function transformChunk<IN = any>(
 
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk, _, cb) {
       buf.push(chunk)

package/src/stream/transform/transformFilter.ts

@@ -28,6 +28,7 @@ export function transformFilterSync<IN = any>(
 
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       try {

package/src/stream/transform/transformFlatten.ts

@@ -4,6 +4,7 @@ import type { TransformTyped } from '../stream.model.js'
 export function transformFlatten<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T[], _, cb) {
       for (const item of chunk) {
         this.push(item)
@@ -16,6 +17,7 @@ export function transformFlatten<T>(): TransformTyped<T[], T> {
 export function transformFlattenIfNeeded<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T[], _, cb) {
       if (Array.isArray(chunk)) {
         for (const item of chunk) {

package/src/stream/transform/transformFork.ts

@@ -18,7 +18,7 @@ export function transformFork<T>(
   fn: (pipeline: Pipeline<T>) => Promise<void>,
   opt: TransformOptions = {},
 ): TransformTyped<T, T> {
-  const { objectMode = true, highWaterMark } = opt
+  const { objectMode = true, highWaterMark = 1 } = opt
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   let lock: DeferredPromise | undefined

package/src/stream/transform/transformLimit.ts

@@ -18,7 +18,7 @@ export interface TransformLimitOptions extends TransformOptions {
 }
 
 export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
-  const { limit, signal, objectMode = true, highWaterMark } = opt
+  const { limit, signal, objectMode = true, highWaterMark = 1 } = opt
 
   if (!limit) {
     return transformNoOp()

package/src/stream/transform/transformMapSimple.ts

@@ -30,7 +30,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     logger = console,
     objectMode = true,
-    highWaterMark,
+    highWaterMark = 1,
   } = opt
 
   return new Transform({

package/src/stream/transform/transformMapSync.ts

@@ -88,6 +88,7 @@ export function transformMapSync<IN = any, OUT = IN>(
 
   return new Transform({
     objectMode,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       // Stop processing if isSettled

package/src/stream/transform/transformNoOp.ts

@@ -9,6 +9,7 @@ import type { TransformTyped } from '../stream.model.js'
 export function transformNoOp<T = any>(): TransformTyped<T, T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T, _, cb) {
       cb(null, chunk)
     },

package/src/stream/transform/transformOffset.ts

@@ -22,6 +22,7 @@ export function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped
   let i = 0 // so we start first chunk with 1
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       if (++i <= offset) {

package/src/stream/transform/transformThrottle.ts

@@ -40,7 +40,7 @@ export interface TransformThrottleOptions extends TransformOptions {
  * @experimental
  */
 export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTyped<T, T> {
-  const { throughput, interval, objectMode = true, highWaterMark } = opt
+  const { throughput, interval, objectMode = true, highWaterMark = 1 } = opt
 
   let count = 0
   let start: UnixTimestampMillis

package/src/stream/transform/transformThrottleByRSS.ts

@@ -60,7 +60,7 @@ export function transformThrottleByRSS<T>(
     pollTimeout = 30 * 60_000, // 30 min
     onPollTimeout = 'open-the-floodgates',
     objectMode = true,
-    highWaterMark,
+    highWaterMark = 1,
   } = opt
 
   const maxRSSBytes = maxRSS * 1024 * 1024

package/src/stream/transform/transformWarmup.ts

@@ -29,7 +29,7 @@ export interface TransformWarmupOptions extends TransformOptions {
  * @experimental
  */
 export function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T> {
-  const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt
+  const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt
   const warmupMs = warmupSeconds * 1000
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 

package/src/zip/zip2.ts

@@ -0,0 +1,159 @@
+import { promisify } from 'node:util'
+import type { ZlibOptions, ZstdOptions } from 'node:zlib'
+import zlib from 'node:zlib'
+import type { Integer } from '@naturalcycles/js-lib/types'
+
+const deflateAsync = promisify(zlib.deflate.bind(zlib))
+const inflateAsync = promisify(zlib.inflate.bind(zlib))
+const gzipAsync = promisify(zlib.gzip.bind(zlib))
+const gunzipAsync = promisify(zlib.gunzip.bind(zlib))
+const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
+const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
+
+class Zip2 {
+  async decompressZstdOrInflateToString(buf: Buffer): Promise<string> {
+    return (await this.decompressZstdOrInflate(buf)).toString()
+  }
+
+  decompressZstdOrInflateToStringSync(buf: Buffer): string {
+    return this.decompressZstdOrInflateSync(buf).toString()
+  }
+
+  /**
+   * Detects if Buffer is zstd-compressed.
+   * Otherwise attempts to Inflate.
+   */
+  async decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>> {
+    if (this.isZstdBuffer(buf)) {
+      return await zstdDecompressAsync(buf)
+    }
+    return await inflateAsync(buf)
+  }
+
+  decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer> {
+    if (this.isZstdBuffer(buf)) {
+      return zlib.zstdDecompressSync(buf)
+    }
+    return zlib.inflateSync(buf)
+  }
+
+  /**
+   * deflateBuffer uses `deflate`.
+   * It's 9 bytes shorter than `gzip`.
+   */
+  async deflate(input: string | Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
+    return await deflateAsync(input, options)
+  }
+
+  /**
+   * deflateSync uses `deflate`.
+   * It's 9 bytes shorter than `gzip`.
+   */
+  deflateSync(input: string | Buffer, options?: ZlibOptions): Buffer<ArrayBuffer> {
+    return zlib.deflateSync(input, options)
+  }
+
+  async inflate(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
+    return await inflateAsync(buf, options)
+  }
+
+  inflateSync(buf: Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
+    return zlib.inflateSync(buf, options)
+  }
+
+  async inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
+    return (await this.inflate(buf, options)).toString()
+  }
+
+  inflateToStringSync(buf: Buffer, options?: ZlibOptions): string {
+    return zlib.inflateSync(buf, options).toString()
+  }
+
+  /**
+   * gzipBuffer uses `gzip`
+   * It's 9 bytes longer than `deflate`.
+   */
+  async gzip(input: string | Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
+    return await gzipAsync(input, options)
+  }
+
+  /**
+   * gzipBuffer uses `gzip`
+   * It's 9 bytes longer than `deflate`.
+   */
+  gzipSync(input: string | Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
+    return zlib.gzipSync(input, options)
+  }
+
+  async gunzip(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer<ArrayBuffer>> {
+    return await gunzipAsync(buf, options)
+  }
+
+  gunzipSync(buf: Buffer, options: ZlibOptions = {}): Buffer<ArrayBuffer> {
+    return zlib.gunzipSync(buf, options)
+  }
+
+  async gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
+    return (await this.gunzip(buf, options)).toString()
+  }
+
+  gunzipToStringSync(buf: Buffer, options?: ZlibOptions): string {
+    return zlib.gunzipSync(buf, options).toString()
+  }
+
+  async zstdCompress(
+    input: Buffer | string,
+    level?: Integer, // defaults to 3
+    options: ZstdOptions = {},
+  ): Promise<Buffer<ArrayBuffer>> {
+    return await zstdCompressAsync(input, this.zstdLevelToOptions(level, options))
+  }
+
+  zstdCompressSync(
+    input: Buffer | string,
+    level?: Integer, // defaults to 3
+    options: ZstdOptions = {},
+  ): Buffer<ArrayBuffer> {
+    return zlib.zstdCompressSync(input, this.zstdLevelToOptions(level, options))
+  }
+
+  zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
+    if (!level) return opt
+
+    return {
+      ...opt,
+      params: {
+        ...opt.params,
+        [zlib.constants.ZSTD_c_compressionLevel]: level,
+      },
+    }
+  }
+
+  async zstdDecompressToString(input: Buffer, options: ZstdOptions = {}): Promise<string> {
+    return (await zstdDecompressAsync(input, options)).toString()
+  }
+
+  async zstdDecompress(input: Buffer, options: ZstdOptions = {}): Promise<Buffer<ArrayBuffer>> {
+    return await zstdDecompressAsync(input, options)
+  }
+
+  zstdDecompressToStringSync(input: Buffer, options: ZstdOptions = {}): string {
+    return zlib.zstdDecompressSync(input, options).toString()
+  }
+
+  zstdDecompressSync(input: Buffer, options: ZstdOptions = {}): Buffer<ArrayBuffer> {
+    return zlib.zstdDecompressSync(input, options)
+  }
+
+  isZstdBuffer(input: Buffer): boolean {
+    return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER
+  }
+
+  isGzipBuffer(input: Buffer): boolean {
+    return input[0] === 0x1f && input[1] === 0x8b
+  }
+}
+
+export const zip2 = new Zip2()
+
+const ZSTD_MAGIC_NUMBER = 0xfd2fb528

package/dist/stream/sizeStack.d.ts

@@ -1,10 +0,0 @@
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js';
-export declare class SizeStack extends NumberStack {
-    name: string;
-    constructor(name: string, size: number);
-    total: number;
-    push(item: any): this;
-    getStats(): string;
-    static countItem(item: any, logger: CommonLogger, sizes?: SizeStack, sizesZipped?: SizeStack): Promise<void>;
-}

package/dist/stream/sizeStack.js

@@ -1,46 +0,0 @@
-import { _hb } from '@naturalcycles/js-lib';
-import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js';
-import { yellow } from '../colors/colors.js';
-import { gzipBuffer } from '../zip/zip.util.js';
-export class SizeStack extends NumberStack {
-    name;
-    constructor(name, size) {
-        super(size);
-        this.name = name;
-    }
-    total = 0;
-    push(item) {
-        this.total += item;
-        return super.push(item);
-    }
-    getStats() {
-        // const pcs = this.percentiles([50, 90])
-        return [
-            '  ' + this.name,
-            'avg',
-            yellow(_hb(this.avg())),
-            // 'p50',
-            // yellow(_hb(pcs[50])),
-            // 'p90',
-            // yellow(_hb(pcs[90])),
-            'total',
-            yellow(_hb(this.total)),
-        ].join(' ');
-    }
-    static async countItem(item, logger, sizes, sizesZipped) {
-        if (!sizes)
-            return;
-        // try-catch, because we don't want to fail the pipeline on logProgress
-        try {
-            const buf = Buffer.from(JSON.stringify(item));
-            sizes.push(buf.byteLength);
-            if (sizesZipped) {
-                const { byteLength } = await gzipBuffer(buf);
-                sizesZipped.push(byteLength);
-            }
-        }
-        catch (err) {
-            logger.warn(`transformLogProgress failed to JSON.stringify the chunk: ${err.message}`);
-        }
-    }
-}

package/dist/zip/zip.util.d.ts

@@ -1,41 +0,0 @@
-import type { ZlibOptions, ZstdOptions } from 'node:zlib';
-import type { Integer } from '@naturalcycles/js-lib/types';
-export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
-/**
- * Detects if Buffer is zstd-compressed.
- * Otherwise attempts to Inflate.
- */
-export declare function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>>;
-/**
- * deflateBuffer uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export declare function deflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function inflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-/**
- * deflateString uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export declare function deflateString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
-/**
- * gzipBuffer uses `gzip`
- * It's 9 bytes longer than `deflate`.
- */
-export declare function gzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-/**
- * gzipString uses `gzip`.
- * It's 9 bytes longer than `deflate`.
- */
-export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
-export declare function zstdCompress(input: Buffer | string, level?: Integer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function zstdCompressSync(input: Buffer | string, level?: Integer, options?: ZstdOptions): Buffer<ArrayBuffer>;
-export declare function zstdLevelToOptions(level: Integer | undefined, opt?: ZstdOptions): ZstdOptions;
-export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
-export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
-export declare function zstdDecompressToStringSync(input: Buffer, options?: ZstdOptions): string;
-export declare function zstdDecompressSync(input: Buffer, options?: ZstdOptions): Buffer<ArrayBuffer>;
-export declare function isZstdBuffer(input: Buffer): boolean;
-export declare function isGzipBuffer(input: Buffer): boolean;

package/dist/zip/zip.util.js

@@ -1,99 +0,0 @@
-import { promisify } from 'node:util';
-import zlib from 'node:zlib';
-const deflate = promisify(zlib.deflate.bind(zlib));
-const inflate = promisify(zlib.inflate.bind(zlib));
-const gzip = promisify(zlib.gzip.bind(zlib));
-const gunzip = promisify(zlib.gunzip.bind(zlib));
-const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib));
-const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
-export async function decompressZstdOrInflateToString(buf) {
-    return (await decompressZstdOrInflate(buf)).toString();
-}
-/**
- * Detects if Buffer is zstd-compressed.
- * Otherwise attempts to Inflate.
- */
-export async function decompressZstdOrInflate(buf) {
-    if (isZstdBuffer(buf)) {
-        return await zstdDecompressAsync(buf);
-    }
-    return await inflate(buf);
-}
-/**
- * deflateBuffer uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export async function deflateBuffer(buf, options = {}) {
-    return await deflate(buf, options);
-}
-export async function inflateBuffer(buf, options = {}) {
-    return await inflate(buf, options);
-}
-/**
- * deflateString uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export async function deflateString(s, options) {
-    return await deflate(s, options);
-}
-export async function inflateToString(buf, options) {
-    return (await inflateBuffer(buf, options)).toString();
-}
-/**
- * gzipBuffer uses `gzip`
- * It's 9 bytes longer than `deflate`.
- */
-export async function gzipBuffer(buf, options = {}) {
-    return await gzip(buf, options);
-}
-export async function gunzipBuffer(buf, options = {}) {
-    return await gunzip(buf, options);
-}
-/**
- * gzipString uses `gzip`.
- * It's 9 bytes longer than `deflate`.
- */
-export async function gzipString(s, options) {
-    return await gzip(s, options);
-}
-export async function gunzipToString(buf, options) {
-    return (await gunzipBuffer(buf, options)).toString();
-}
-export async function zstdCompress(input, level, // defaults to 3
-options = {}) {
-    return await zstdCompressAsync(input, zstdLevelToOptions(level, options));
-}
-export function zstdCompressSync(input, level, // defaults to 3
-options = {}) {
-    return zlib.zstdCompressSync(input, zstdLevelToOptions(level, options));
-}
-export function zstdLevelToOptions(level, opt = {}) {
-    if (!level)
-        return opt;
-    return {
-        ...opt,
-        params: {
-            ...opt.params,
-            [zlib.constants.ZSTD_c_compressionLevel]: level,
-        },
-    };
-}
-export async function zstdDecompressToString(input, options = {}) {
-    return (await zstdDecompressAsync(input, options)).toString();
-}
-export async function zstdDecompress(input, options = {}) {
-    return await zstdDecompressAsync(input, options);
-}
-export function zstdDecompressToStringSync(input, options = {}) {
-    return zlib.zstdDecompressSync(input, options).toString();
-}
-export function zstdDecompressSync(input, options = {}) {
-    return zlib.zstdDecompressSync(input, options);
-}
-const ZSTD_MAGIC_NUMBER = 0xfd2fb528;
-export function isZstdBuffer(input) {
-    return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER;
-}
-export function isGzipBuffer(input) {
-    return input[0] === 0x1f && input[1] === 0x8b;
-}

package/src/stream/sizeStack.ts

@@ -1,61 +0,0 @@
-import { _hb } from '@naturalcycles/js-lib'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js'
-import { yellow } from '../colors/colors.js'
-import { gzipBuffer } from '../zip/zip.util.js'
-
-export class SizeStack extends NumberStack {
-  constructor(
-    public name: string,
-    size: number,
-  ) {
-    super(size)
-  }
-
-  total = 0
-
-  override push(item: any): this {
-    this.total += item
-    return super.push(item)
-  }
-
-  getStats(): string {
-    // const pcs = this.percentiles([50, 90])
-
-    return [
-      '  ' + this.name,
-      'avg',
-      yellow(_hb(this.avg())),
-      // 'p50',
-      // yellow(_hb(pcs[50])),
-      // 'p90',
-      // yellow(_hb(pcs[90])),
-      'total',
-      yellow(_hb(this.total)),
-    ].join(' ')
-  }
-
-  static async countItem(
-    item: any,
-    logger: CommonLogger,
-    sizes?: SizeStack,
-    sizesZipped?: SizeStack,
-  ): Promise<void> {
-    if (!sizes) return
-
-    // try-catch, because we don't want to fail the pipeline on logProgress
-    try {
-      const buf = Buffer.from(JSON.stringify(item))
-      sizes.push(buf.byteLength)
-
-      if (sizesZipped) {
-        const { byteLength } = await gzipBuffer(buf)
-        sizesZipped.push(byteLength)
-      }
-    } catch (err) {
-      logger.warn(
-        `transformLogProgress failed to JSON.stringify the chunk: ${(err as Error).message}`,
-      )
-    }
-  }
-}

package/src/zip/zip.util.ts

@@ -1,149 +0,0 @@
-import { promisify } from 'node:util'
-import type { ZlibOptions, ZstdOptions } from 'node:zlib'
-import zlib from 'node:zlib'
-import type { Integer } from '@naturalcycles/js-lib/types'
-
-const deflate = promisify(zlib.deflate.bind(zlib))
-const inflate = promisify(zlib.inflate.bind(zlib))
-const gzip = promisify(zlib.gzip.bind(zlib))
-const gunzip = promisify(zlib.gunzip.bind(zlib))
-const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
-const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
-
-export async function decompressZstdOrInflateToString(buf: Buffer): Promise<string> {
-  return (await decompressZstdOrInflate(buf)).toString()
-}
-
-/**
- * Detects if Buffer is zstd-compressed.
- * Otherwise attempts to Inflate.
- */
-export async function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>> {
-  if (isZstdBuffer(buf)) {
-    return await zstdDecompressAsync(buf)
-  }
-  return await inflate(buf)
-}
-
-/**
- * deflateBuffer uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export async function deflateBuffer(
-  buf: Buffer,
-  options: ZlibOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await deflate(buf, options)
-}
-
-export async function inflateBuffer(
-  buf: Buffer,
-  options: ZlibOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await inflate(buf, options)
-}
-
-/**
- * deflateString uses `deflate`.
- * It's 9 bytes shorter than `gzip`.
- */
-export async function deflateString(
-  s: string,
-  options?: ZlibOptions,
-): Promise<Buffer<ArrayBuffer>> {
-  return await deflate(s, options)
-}
-
-export async function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
-  return (await inflateBuffer(buf, options)).toString()
-}
-
-/**
- * gzipBuffer uses `gzip`
- * It's 9 bytes longer than `deflate`.
- */
-export async function gzipBuffer(
-  buf: Buffer,
-  options: ZlibOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await gzip(buf, options)
-}
-
-export async function gunzipBuffer(
-  buf: Buffer,
-  options: ZlibOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await gunzip(buf, options)
-}
-
-/**
- * gzipString uses `gzip`.
- * It's 9 bytes longer than `deflate`.
- */
-export async function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>> {
-  return await gzip(s, options)
-}
-
-export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
-  return (await gunzipBuffer(buf, options)).toString()
-}
-
-export async function zstdCompress(
-  input: Buffer | string,
-  level?: Integer, // defaults to 3
-  options: ZstdOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await zstdCompressAsync(input, zstdLevelToOptions(level, options))
-}
-
-export function zstdCompressSync(
-  input: Buffer | string,
-  level?: Integer, // defaults to 3
-  options: ZstdOptions = {},
-): Buffer<ArrayBuffer> {
-  return zlib.zstdCompressSync(input, zstdLevelToOptions(level, options))
-}
-
-export function zstdLevelToOptions(level: Integer | undefined, opt: ZstdOptions = {}): ZstdOptions {
-  if (!level) return opt
-
-  return {
-    ...opt,
-    params: {
-      ...opt.params,
-      [zlib.constants.ZSTD_c_compressionLevel]: level,
-    },
-  }
-}
-
-export async function zstdDecompressToString(
-  input: Buffer,
-  options: ZstdOptions = {},
-): Promise<string> {
-  return (await zstdDecompressAsync(input, options)).toString()
-}
-
-export async function zstdDecompress(
-  input: Buffer,
-  options: ZstdOptions = {},
-): Promise<Buffer<ArrayBuffer>> {
-  return await zstdDecompressAsync(input, options)
-}
-
-export function zstdDecompressToStringSync(input: Buffer, options: ZstdOptions = {}): string {
-  return zlib.zstdDecompressSync(input, options).toString()
-}
-
-export function zstdDecompressSync(input: Buffer, options: ZstdOptions = {}): Buffer<ArrayBuffer> {
-  return zlib.zstdDecompressSync(input, options)
-}
-
-const ZSTD_MAGIC_NUMBER = 0xfd2fb528
-
-export function isZstdBuffer(input: Buffer): boolean {
-  return input.readUInt32LE(0) === ZSTD_MAGIC_NUMBER
-}
-
-export function isGzipBuffer(input: Buffer): boolean {
-  return input[0] === 0x1f && input[1] === 0x8b
-}