@naturalcycles/nodejs-lib 15.65.2 → 15.66.0

package/dist/stream/ndjson/createReadStreamAsNDJson.d.ts

@@ -1,16 +1,16 @@
 import type { ReadableTyped } from '../stream.model.js';
 /**
  Returns a Readable of [already parsed] NDJSON objects.
- 
+
  Replaces a list of operations:
  - requireFileToExist(inputPath)
  - fs.createReadStream
  - createUnzip (only if path ends with '.gz')
  - transformSplitOnNewline
  - transformJsonParse
- 
+
  To add a Limit or Offset: just add .take() or .drop(), example:
- 
+
  createReadStreamAsNDJson().take(100)
  */
 export declare function createReadStreamAsNDJson<ROW = any>(inputPath: string): ReadableTyped<ROW>;

package/dist/stream/ndjson/createReadStreamAsNDJson.js

@@ -1,18 +1,18 @@
-import { createUnzip } from 'node:zlib';
+import { createUnzip, createZstdDecompress } from 'node:zlib';
 import { fs2 } from '../../fs/fs2.js';
 import { transformSplitOnNewline } from '../transform/transformSplit.js';
 /**
  Returns a Readable of [already parsed] NDJSON objects.
- 
+
  Replaces a list of operations:
  - requireFileToExist(inputPath)
  - fs.createReadStream
  - createUnzip (only if path ends with '.gz')
  - transformSplitOnNewline
  - transformJsonParse
- 
+
  To add a Limit or Offset: just add .take() or .drop(), example:
- 
+
  createReadStreamAsNDJson().take(100)
  */
 export function createReadStreamAsNDJson(inputPath) {
@@ -27,6 +27,11 @@ export function createReadStreamAsNDJson(inputPath) {
             chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
         }));
     }
+    else if (inputPath.endsWith('.zst')) {
+        stream = stream.pipe(createZstdDecompress({
+            chunkSize: 64 * 1024, // todo: test it
+        }));
+    }
     return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
     // For some crazy reason .map is much faster than transformJsonParse!
     // ~5000 vs ~4000 rps !!!

package/dist/stream/pipeline.d.ts

@@ -1,6 +1,6 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
-import { type ZlibOptions } from 'node:zlib';
+import { type ZlibOptions, type ZstdOptions } from 'node:zlib';
 import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
@@ -92,6 +92,8 @@ export declare class Pipeline<T = unknown> {
     parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
     gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
+    zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
+    zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;
     toFile(outputFilePath: string): Promise<void>;
     toNDJsonFile(outputFilePath: string): Promise<void>;

package/dist/stream/pipeline.js

@@ -1,6 +1,6 @@
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
-import { createGzip, createUnzip } from 'node:zlib';
+import { createGzip, createUnzip, createZstdCompress, createZstdDecompress, } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
@@ -25,12 +25,10 @@ import { transformThrottle } from './transform/transformThrottle.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
 import { writableVoid } from './writable/writableVoid.js';
 export class Pipeline {
-    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     source;
     transforms = [];
     destination;
     readableLimit;
-    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     objectMode;
     abortableSignal = createAbortableSignal();
     constructor(source, objectMode = true) {
@@ -247,6 +245,22 @@ export class Pipeline {
         this.objectMode = false;
         return this;
     }
+    zstdCompress(opt) {
+        this.transforms.push(createZstdCompress({
+            // chunkSize: 64 * 1024, // no observed speedup
+            ...opt,
+        }));
+        this.objectMode = false;
+        return this;
+    }
+    zstdDecompress(opt) {
+        this.transforms.push(createZstdDecompress({
+            chunkSize: 64 * 1024, // todo: test it
+            ...opt,
+        }));
+        this.objectMode = false;
+        return this;
+    }
     async toArray(opt) {
         const arr = [];
         this.destination = writablePushToArray(arr, opt);
@@ -266,6 +280,11 @@ export class Pipeline {
             // chunkSize: 64 * 1024, // no observed speedup
             }));
         }
+        else if (outputFilePath.endsWith('.zst')) {
+            this.transforms.push(createZstdCompress({
+            // chunkSize: 64 * 1024, // no observed speedup
+            }));
+        }
         this.destination = fs2.createWriteStream(outputFilePath, {
         // highWaterMark: 64 * 1024, // no observed speedup
         });

package/dist/stream/readable/readableCombined.js

@@ -32,11 +32,8 @@ export class ReadableCombined extends Readable {
      * If not defined - we are in Flowing mode, no limits in data flow.
      */
     lock;
-    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     countIn = 0;
-    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     countOut = 0;
-    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     countReads = 0;
     async run() {
         const { logger } = this;

package/dist/validation/ajv/getAjv.js

@@ -428,10 +428,21 @@ export function createAjv(opt) {
         modifying: false,
         errors: true,
         schemaType: 'number',
-        validate: function validate(minProperties, data, _schema, _ctx) {
+        validate: function validate(minProperties, data, _schema, ctx) {
             if (typeof data !== 'object')
                 return true;
-            return Object.getOwnPropertyNames(data).length >= minProperties;
+            const numberOfProperties = Object.getOwnPropertyNames(data).length;
+            const isValid = numberOfProperties >= minProperties;
+            if (!isValid) {
+                ;
+                validate.errors = [
+                    {
+                        instancePath: ctx?.instancePath ?? '',
+                        message: `must NOT have fewer than ${minProperties} properties`,
+                    },
+                ];
+            }
+            return isValid;
         },
     });
     return ajv;

package/dist/zip/zip.util.d.ts

@@ -1,4 +1,4 @@
-import type { ZlibOptions } from 'node:zlib';
+import type { ZlibOptions, ZstdOptions } from 'node:zlib';
 /**
  * deflateBuffer uses `deflate`.
  * It's 9 bytes shorter than `gzip`.
@@ -23,3 +23,6 @@ export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promis
  */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer<ArrayBuffer>>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
+export declare function zstdCompress(input: Buffer | string, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;
+export declare function zstdDecompressToString(input: Buffer, options?: ZstdOptions): Promise<string>;
+export declare function zstdDecompress(input: Buffer, options?: ZstdOptions): Promise<Buffer<ArrayBuffer>>;

package/dist/zip/zip.util.js

@@ -4,15 +4,17 @@ const deflate = promisify(zlib.deflate.bind(zlib));
 const inflate = promisify(zlib.inflate.bind(zlib));
 const gzip = promisify(zlib.gzip.bind(zlib));
 const gunzip = promisify(zlib.gunzip.bind(zlib));
+const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib));
+const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
 /**
  * deflateBuffer uses `deflate`.
  * It's 9 bytes shorter than `gzip`.
  */
 export async function deflateBuffer(buf, options = {}) {
-    return (await deflate(buf, options));
+    return await deflate(buf, options);
 }
 export async function inflateBuffer(buf, options = {}) {
-    return (await inflate(buf, options));
+    return await inflate(buf, options);
 }
 /**
  * deflateString uses `deflate`.
@@ -29,10 +31,10 @@ export async function inflateToString(buf, options) {
  * It's 9 bytes longer than `deflate`.
  */
 export async function gzipBuffer(buf, options = {}) {
-    return (await gzip(buf, options));
+    return await gzip(buf, options);
 }
 export async function gunzipBuffer(buf, options = {}) {
-    return (await gunzip(buf, options));
+    return await gunzip(buf, options);
 }
 /**
  * gzipString uses `gzip`.
@@ -44,3 +46,12 @@ export async function gzipString(s, options) {
 export async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }
+export async function zstdCompress(input, options = {}) {
+    return await zstdCompressAsync(input, options);
+}
+export async function zstdDecompressToString(input, options = {}) {
+    return (await zstdDecompressAsync(input, options)).toString();
+}
+export async function zstdDecompress(input, options = {}) {
+    return await zstdDecompressAsync(input, options);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.65.2",
+  "version": "15.66.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/ndjson/createReadStreamAsNDJson.ts

@@ -1,20 +1,20 @@
-import { createUnzip } from 'node:zlib'
+import { createUnzip, createZstdDecompress } from 'node:zlib'
 import { fs2 } from '../../fs/fs2.js'
 import type { ReadableTyped } from '../stream.model.js'
 import { transformSplitOnNewline } from '../transform/transformSplit.js'
 
 /**
  Returns a Readable of [already parsed] NDJSON objects.
- 
+
  Replaces a list of operations:
  - requireFileToExist(inputPath)
  - fs.createReadStream
  - createUnzip (only if path ends with '.gz')
  - transformSplitOnNewline
  - transformJsonParse
- 
+
  To add a Limit or Offset: just add .take() or .drop(), example:
- 
+
  createReadStreamAsNDJson().take(100)
  */
 
@@ -33,6 +33,12 @@ export function createReadStreamAsNDJson<ROW = any>(inputPath: string): Readable
         chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
       }),
     )
+  } else if (inputPath.endsWith('.zst')) {
+    stream = stream.pipe(
+      createZstdDecompress({
+        chunkSize: 64 * 1024, // todo: test it
+      }),
+    )
   }
 
   return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))

package/src/stream/pipeline.ts

@@ -1,7 +1,14 @@
 import { Readable, type Transform } from 'node:stream'
 import { pipeline } from 'node:stream/promises'
 import type { ReadableStream as WebReadableStream } from 'node:stream/web'
-import { createGzip, createUnzip, type ZlibOptions } from 'node:zlib'
+import {
+  createGzip,
+  createUnzip,
+  createZstdCompress,
+  createZstdDecompress,
+  type ZlibOptions,
+  type ZstdOptions,
+} from 'node:zlib'
 import { createAbortableSignal } from '@naturalcycles/js-lib'
 import {
   _passthroughPredicate,
@@ -51,12 +58,11 @@ import { writablePushToArray } from './writable/writablePushToArray.js'
 import { writableVoid } from './writable/writableVoid.js'
 
 export class Pipeline<T = unknown> {
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
   private readonly source: Readable
   private transforms: NodeJS.ReadWriteStream[] = []
   private destination?: NodeJS.WritableStream
   private readableLimit?: Integer
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+
   private objectMode: boolean
   private abortableSignal = createAbortableSignal()
 
@@ -334,6 +340,28 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
+  zstdCompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
+    this.transforms.push(
+      createZstdCompress({
+        // chunkSize: 64 * 1024, // no observed speedup
+        ...opt,
+      }),
+    )
+    this.objectMode = false
+    return this as any
+  }
+
+  zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
+    this.transforms.push(
+      createZstdDecompress({
+        chunkSize: 64 * 1024, // todo: test it
+        ...opt,
+      }),
+    )
+    this.objectMode = false
+    return this as any
+  }
+
   async toArray(opt?: TransformOptions): Promise<T[]> {
     const arr: T[] = []
     this.destination = writablePushToArray(arr, opt)
@@ -356,6 +384,12 @@ export class Pipeline<T = unknown> {
           // chunkSize: 64 * 1024, // no observed speedup
         }),
       )
+    } else if (outputFilePath.endsWith('.zst')) {
+      this.transforms.push(
+        createZstdCompress({
+          // chunkSize: 64 * 1024, // no observed speedup
+        }),
+      )
     }
     this.destination = fs2.createWriteStream(outputFilePath, {
       // highWaterMark: 64 * 1024, // no observed speedup

package/src/stream/readable/readableCombined.ts

@@ -40,11 +40,10 @@ export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
    */
   private lock?: DeferredPromise
 
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
   private countIn = 0
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+
   private countOut = 0
-  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+
   private countReads = 0
 
   private async run(): Promise<void> {

package/src/stream/transform/worker/transformMultiThreaded.ts

@@ -75,7 +75,7 @@ export function transformMultiThreaded<IN, OUT>(
 
     worker.on('error', err => {
       console.error(`Worker ${workerIndex} error`, err)
-      workerDonePromises[workerIndex]!.reject(err)
+      workerDonePromises[workerIndex]!.reject(err as Error)
     })
 
     worker.on('exit', _exitCode => {

package/src/validation/ajv/getAjv.ts

@@ -492,9 +492,21 @@ export function createAjv(opt?: Options): Ajv2020 {
     modifying: false,
     errors: true,
     schemaType: 'number',
-    validate: function validate(minProperties: number, data: AnyObject, _schema, _ctx) {
+    validate: function validate(minProperties: number, data: AnyObject, _schema, ctx) {
       if (typeof data !== 'object') return true
-      return Object.getOwnPropertyNames(data).length >= minProperties
+
+      const numberOfProperties = Object.getOwnPropertyNames(data).length
+      const isValid = numberOfProperties >= minProperties
+      if (!isValid) {
+        ;(validate as any).errors = [
+          {
+            instancePath: ctx?.instancePath ?? '',
+            message: `must NOT have fewer than ${minProperties} properties`,
+          },
+        ]
+      }
+
+      return isValid
     },
   })
 

package/src/zip/zip.util.ts

@@ -1,11 +1,13 @@
 import { promisify } from 'node:util'
-import type { ZlibOptions } from 'node:zlib'
+import type { ZlibOptions, ZstdOptions } from 'node:zlib'
 import zlib from 'node:zlib'
 
 const deflate = promisify(zlib.deflate.bind(zlib))
 const inflate = promisify(zlib.inflate.bind(zlib))
 const gzip = promisify(zlib.gzip.bind(zlib))
 const gunzip = promisify(zlib.gunzip.bind(zlib))
+const zstdCompressAsync = promisify(zlib.zstdCompress.bind(zlib))
+const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib))
 
 /**
  * deflateBuffer uses `deflate`.
@@ -15,14 +17,14 @@ export async function deflateBuffer(
   buf: Buffer,
   options: ZlibOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return (await deflate(buf, options)) as Buffer<ArrayBuffer>
+  return await deflate(buf, options)
 }
 
 export async function inflateBuffer(
   buf: Buffer,
   options: ZlibOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return (await inflate(buf, options)) as Buffer<ArrayBuffer>
+  return await inflate(buf, options)
 }
 
 /**
@@ -48,14 +50,14 @@ export async function gzipBuffer(
   buf: Buffer,
   options: ZlibOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return (await gzip(buf, options)) as Buffer<ArrayBuffer>
+  return await gzip(buf, options)
 }
 
 export async function gunzipBuffer(
   buf: Buffer,
   options: ZlibOptions = {},
 ): Promise<Buffer<ArrayBuffer>> {
-  return (await gunzip(buf, options)) as Buffer<ArrayBuffer>
+  return await gunzip(buf, options)
 }
 
 /**
@@ -69,3 +71,24 @@ export async function gzipString(s: string, options?: ZlibOptions): Promise<Buff
 export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
   return (await gunzipBuffer(buf, options)).toString()
 }
+
+export async function zstdCompress(
+  input: Buffer | string,
+  options: ZstdOptions = {},
+): Promise<Buffer<ArrayBuffer>> {
+  return await zstdCompressAsync(input, options)
+}
+
+export async function zstdDecompressToString(
+  input: Buffer,
+  options: ZstdOptions = {},
+): Promise<string> {
+  return (await zstdDecompressAsync(input, options)).toString()
+}
+
+export async function zstdDecompress(
+  input: Buffer,
+  options: ZstdOptions = {},
+): Promise<Buffer<ArrayBuffer>> {
+  return await zstdDecompressAsync(input, options)
+}