@naturalcycles/nodejs-lib 15.34.0 → 15.36.0

package/dist/stream/index.d.ts

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/index.js

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
+export * from './writable/writableChunk.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/writable/writableChunk.d.ts

@@ -0,0 +1,15 @@
+import type { Predicate } from '@naturalcycles/js-lib/types';
+import { type TransformOptions, type TransformTyped, type WritableTyped } from '../index.js';
+export interface WritableChunkOptions<T> extends TransformOptions {
+    splitPredicate: Predicate<T>;
+    transformFactories?: (() => TransformTyped<T, T>)[];
+    writableFactory: (splitIndex: number) => WritableTyped<T>;
+}
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export declare function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T>;

package/dist/stream/writable/writableChunk.js

@@ -0,0 +1,83 @@
+import { Writable } from 'node:stream';
+import { _first, _last } from '@naturalcycles/js-lib/array';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { _deepCopy } from '@naturalcycles/js-lib/object';
+import { transformNoOp, } from '../index.js';
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk(opt) {
+    const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let indexWritten = 0;
+    let currentSplitIndex = 0;
+    // We don't want to have an empty chain, so we add a no-op transform
+    if (transformFactories.length === 0) {
+        transformFactories.push((transformNoOp));
+    }
+    // Create the transforms as well as the Writable, and pipe them together
+    let currentWritable = writableFactory(currentSplitIndex);
+    let transforms = transformFactories.map(f => f());
+    generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+    _last(transforms).pipe(currentWritable);
+    // We keep track of all the pending writables, so we can await them in the final method
+    const writablesFinish = [awaitFinish(currentWritable)];
+    return new Writable({
+        objectMode: true,
+        highWaterMark,
+        write(chunk, _, cb) {
+            // pipe will take care of piping the data through the different streams correctly
+            transforms[0].write(chunk, cb);
+            if (splitPredicate(chunk, ++indexWritten)) {
+                logger.log(`writableChunk: splitting at index ${currentSplitIndex}`);
+                currentSplitIndex++;
+                transforms[0].end();
+                currentWritable = writableFactory(currentSplitIndex);
+                transforms = transformFactories.map(f => f());
+                generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2));
+                _last(transforms).pipe(currentWritable);
+                writablesFinish.push(awaitFinish(currentWritable));
+            }
+        },
+        async final(cb) {
+            try {
+                transforms[0].end();
+                await Promise.all(writablesFinish);
+                logger.log('writableChunk: all writables are finished');
+                cb();
+            }
+            catch (err) {
+                cb(err);
+            }
+        },
+    });
+}
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream) {
+    return await new Promise(resolve => {
+        stream.on('finish', resolve);
+    });
+}
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples(arr) {
+    const tuples = [];
+    const arrCopy = _deepCopy(arr);
+    for (let i = 1; i < arrCopy.length; i++) {
+        tuples.push([arrCopy[i - 1], arrCopy[i]]);
+    }
+    return tuples;
+}

package/dist/validation/ajv/ajvSchema.d.ts

@@ -1,8 +1,10 @@
 import { type ValidationFunction, type ValidationFunctionResult } from '@naturalcycles/js-lib';
 import type { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib/json-schema';
+import type { AnyObject } from '@naturalcycles/js-lib/types';
 import { ZodType } from '@naturalcycles/js-lib/zod';
 import type { Ajv } from 'ajv';
 import { AjvValidationError } from './ajvValidationError.js';
+export type SchemaHandledByAjv<T> = JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>;
 export interface AjvValidationOptions {
     /**
      * Defaults to true,
@@ -63,11 +65,15 @@ export declare class AjvSchema<T = unknown> {
      * Implementation note: JsonSchemaBuilder goes first in the union type, otherwise TypeScript fails to infer <T> type
      * correctly for some reason.
      */
-    static create<T>(schema: JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T>;
+    static create<T>(schema: SchemaHandledByAjv<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T>;
     /**
-     * @experimental
+     * @deprecated
+     *
+     * Use `AjvSchema.create`
      */
     static createFromZod<T>(zodSchema: ZodType<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T>;
+    static isJsonSchemaBuilder<T>(schema: unknown): schema is JsonSchemaBuilder<T>;
+    static isZodSchema<T>(schema: unknown): schema is ZodType<T>;
     readonly cfg: AjvSchemaCfg;
     /**
      * It returns the original object just for convenience.
@@ -81,12 +87,12 @@ export declare class AjvSchema<T = unknown> {
     isValid(input: T, opt?: AjvValidationOptions): boolean;
     getValidationResult(input: T, opt?: AjvValidationOptions): ValidationFunctionResult<T, AjvValidationError>;
     getValidationFunction(): ValidationFunction<T, AjvValidationError>;
-    static isZodSchemaWithAjvSchema<T>(schema: ZodType<T>): schema is ZodTypeWithAjvSchema<T>;
-    static cacheAjvSchemaInZodSchema<T>(zodSchema: ZodType<T>, ajvSchema: AjvSchema<T>): ZodTypeWithAjvSchema<T>;
-    static requireCachedAjvSchemaFromZodSchema<T>(zodSchema: ZodTypeWithAjvSchema<T>): AjvSchema<T>;
+    static isSchemaWithCachedAjvSchema<Base, T>(schema: Base): schema is WithCachedAjvSchema<Base, T>;
+    static cacheAjvSchema<Base extends AnyObject, T>(schema: Base, ajvSchema: AjvSchema<T>): WithCachedAjvSchema<Base, T>;
+    static requireCachedAjvSchema<Base, T>(schema: WithCachedAjvSchema<Base, T>): AjvSchema<T>;
     private getAJVValidateFunction;
 }
 export declare const HIDDEN_AJV_SCHEMA: unique symbol;
-export interface ZodTypeWithAjvSchema<T> extends ZodType<T> {
+export type WithCachedAjvSchema<Base, T> = Base & {
     [HIDDEN_AJV_SCHEMA]: AjvSchema<T>;
-}
+};

package/dist/validation/ajv/ajvSchema.js

@@ -46,24 +46,36 @@ export class AjvSchema {
     static create(schema, cfg) {
         if (schema instanceof AjvSchema)
             return schema;
-        if (schema instanceof JsonSchemaAnyBuilder) {
-            return new AjvSchema(schema.build(), cfg);
+        if (AjvSchema.isSchemaWithCachedAjvSchema(schema)) {
+            return AjvSchema.requireCachedAjvSchema(schema);
         }
-        if (schema instanceof ZodType)
-            return AjvSchema.createFromZod(schema);
-        return new AjvSchema(schema, cfg);
+        let jsonSchema;
+        if (AjvSchema.isJsonSchemaBuilder(schema)) {
+            jsonSchema = schema.build();
+        }
+        else if (AjvSchema.isZodSchema(schema)) {
+            jsonSchema = z.toJSONSchema(schema, { target: 'draft-7' });
+        }
+        else {
+            jsonSchema = schema;
+        }
+        const ajvSchema = new AjvSchema(jsonSchema, cfg);
+        AjvSchema.cacheAjvSchema(schema, ajvSchema);
+        return ajvSchema;
     }
     /**
-     * @experimental
+     * @deprecated
+     *
+     * Use `AjvSchema.create`
      */
     static createFromZod(zodSchema, cfg) {
-        if (AjvSchema.isZodSchemaWithAjvSchema(zodSchema)) {
-            return AjvSchema.requireCachedAjvSchemaFromZodSchema(zodSchema);
-        }
-        const jsonSchema = z.toJSONSchema(zodSchema, { target: 'draft-7' });
-        const ajvSchema = new AjvSchema(jsonSchema, cfg);
-        AjvSchema.cacheAjvSchemaInZodSchema(zodSchema, ajvSchema);
-        return ajvSchema;
+        return AjvSchema.create(zodSchema, cfg);
+    }
+    static isJsonSchemaBuilder(schema) {
+        return schema instanceof JsonSchemaAnyBuilder;
+    }
+    static isZodSchema(schema) {
+        return schema instanceof ZodType;
     }
     cfg;
     /**
@@ -121,14 +133,14 @@ export class AjvSchema {
             });
         };
     }
-    static isZodSchemaWithAjvSchema(schema) {
-        return !!schema[HIDDEN_AJV_SCHEMA];
+    static isSchemaWithCachedAjvSchema(schema) {
+        return !!schema?.[HIDDEN_AJV_SCHEMA];
     }
-    static cacheAjvSchemaInZodSchema(zodSchema, ajvSchema) {
-        return Object.assign(zodSchema, { [HIDDEN_AJV_SCHEMA]: ajvSchema });
+    static cacheAjvSchema(schema, ajvSchema) {
+        return Object.assign(schema, { [HIDDEN_AJV_SCHEMA]: ajvSchema });
     }
-    static requireCachedAjvSchemaFromZodSchema(zodSchema) {
-        return zodSchema[HIDDEN_AJV_SCHEMA];
+    static requireCachedAjvSchema(schema) {
+        return schema[HIDDEN_AJV_SCHEMA];
     }
     getAJVValidateFunction = _lazyValue(() => this.cfg.ajv.compile(this.schema));
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.34.0",
+  "version": "15.36.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -26,5 +26,6 @@ export * from './transform/transformThrottle.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'
+export * from './writable/writableChunk.js'
 export * from './writable/writablePushToArray.js'
 export * from './writable/writableVoid.js'

package/src/stream/writable/writableChunk.ts

@@ -0,0 +1,104 @@
+import { Writable } from 'node:stream'
+import { _first, _last } from '@naturalcycles/js-lib/array'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import { _deepCopy } from '@naturalcycles/js-lib/object'
+import type { Predicate } from '@naturalcycles/js-lib/types'
+import {
+  transformNoOp,
+  type TransformOptions,
+  type TransformTyped,
+  type WritableTyped,
+} from '../index.js'
+
+export interface WritableChunkOptions<T> extends TransformOptions {
+  splitPredicate: Predicate<T>
+  transformFactories?: (() => TransformTyped<T, T>)[]
+  writableFactory: (splitIndex: number) => WritableTyped<T>
+}
+
+/**
+ * Allows to split the output to multiple files by splitting into chunks
+ * based on `shouldSplitFn`.
+ * `transformFactories` are used to create a chain of transforms for each chunk.
+ * It was meant to be used with createGzip, which needs a proper start and end for each chunk
+ * for the output file to be a valid gzip file.
+ */
+export function writableChunk<T>(opt: WritableChunkOptions<T>): WritableTyped<T> {
+  const { highWaterMark, splitPredicate, transformFactories = [], writableFactory } = opt
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  let indexWritten = 0
+  let currentSplitIndex = 0
+  // We don't want to have an empty chain, so we add a no-op transform
+  if (transformFactories.length === 0) {
+    transformFactories.push(transformNoOp<T>)
+  }
+
+  // Create the transforms as well as the Writable, and pipe them together
+  let currentWritable = writableFactory(currentSplitIndex)
+  let transforms = transformFactories.map(f => f())
+  generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+  _last(transforms).pipe(currentWritable)
+
+  // We keep track of all the pending writables, so we can await them in the final method
+  const writablesFinish: Promise<void>[] = [awaitFinish(currentWritable)]
+
+  return new Writable({
+    objectMode: true,
+    highWaterMark,
+    write(chunk: T, _, cb) {
+      // pipe will take care of piping the data through the different streams correctly
+      transforms[0]!.write(chunk, cb)
+
+      if (splitPredicate(chunk, ++indexWritten)) {
+        logger.log(`writableChunk: splitting at index ${currentSplitIndex}`)
+        currentSplitIndex++
+        transforms[0]!.end()
+
+        currentWritable = writableFactory(currentSplitIndex)
+        transforms = transformFactories.map(f => f())
+        generateTuples(transforms).forEach(([t1, t2]) => t1.pipe(t2))
+        _last(transforms).pipe(currentWritable)
+
+        writablesFinish.push(awaitFinish(currentWritable))
+      }
+    },
+    async final(cb) {
+      try {
+        transforms[0]!.end()
+        await Promise.all(writablesFinish)
+        logger.log('writableChunk: all writables are finished')
+        cb()
+      } catch (err) {
+        cb(err as Error)
+      }
+    },
+  })
+}
+
+/**
+ * This is a helper function to create a promise which resolves when the stream emits a 'finish'
+ * event.
+ * This is used to await all the writables in the final method of the writableChunk
+ */
+async function awaitFinish(stream: Writable): Promise<void> {
+  return await new Promise(resolve => {
+    stream.on('finish', resolve)
+  })
+}
+
+/**
+ * Generates an array of [arr[i], arr[i+1]] tuples from the input array.
+ * The resulting array will have a length of `arr.length - 1`.
+ * ```ts
+ * generateTuples([1, 2, 3, 4]) // [[1, 2], [2, 3], [3, 4]]
+ * ```
+ */
+function generateTuples<T>(arr: T[]): [T, T][] {
+  const tuples: [T, T][] = []
+  const arrCopy = _deepCopy(arr)
+  for (let i = 1; i < arrCopy.length; i++) {
+    tuples.push([arrCopy[i - 1]!, arrCopy[i]!])
+  }
+  return tuples
+}

package/src/validation/ajv/ajvSchema.ts

@@ -8,12 +8,15 @@ import type { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib/json-s
 import { JsonSchemaAnyBuilder } from '@naturalcycles/js-lib/json-schema'
 import { _deepCopy, _filterNullishValues } from '@naturalcycles/js-lib/object'
 import { _substringBefore } from '@naturalcycles/js-lib/string'
+import type { AnyObject } from '@naturalcycles/js-lib/types'
 import { z, ZodType } from '@naturalcycles/js-lib/zod'
 import type { Ajv } from 'ajv'
 import { _inspect } from '../../string/inspect.js'
 import { AjvValidationError } from './ajvValidationError.js'
 import { getAjv } from './getAjv.js'
 
+export type SchemaHandledByAjv<T> = JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>
+
 export interface AjvValidationOptions {
   /**
    * Defaults to true,
@@ -104,32 +107,44 @@ export class AjvSchema<T = unknown> {
    * Implementation note: JsonSchemaBuilder goes first in the union type, otherwise TypeScript fails to infer <T> type
    * correctly for some reason.
    */
-  static create<T>(
-    schema: JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>,
-    cfg?: Partial<AjvSchemaCfg>,
-  ): AjvSchema<T> {
+  static create<T>(schema: SchemaHandledByAjv<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T> {
     if (schema instanceof AjvSchema) return schema
-    if (schema instanceof JsonSchemaAnyBuilder) {
-      return new AjvSchema<T>(schema.build(), cfg)
+
+    if (AjvSchema.isSchemaWithCachedAjvSchema<typeof schema, T>(schema)) {
+      return AjvSchema.requireCachedAjvSchema<typeof schema, T>(schema)
     }
-    if (schema instanceof ZodType) return AjvSchema.createFromZod(schema)
-    return new AjvSchema<T>(schema as JsonSchema<T>, cfg)
+
+    let jsonSchema: JsonSchema<T>
+
+    if (AjvSchema.isJsonSchemaBuilder(schema)) {
+      jsonSchema = schema.build()
+    } else if (AjvSchema.isZodSchema(schema)) {
+      jsonSchema = z.toJSONSchema(schema, { target: 'draft-7' }) as JsonSchema<T>
+    } else {
+      jsonSchema = schema
+    }
+
+    const ajvSchema = new AjvSchema<T>(jsonSchema, cfg)
+    AjvSchema.cacheAjvSchema(schema, ajvSchema)
+
+    return ajvSchema
   }
 
   /**
-   * @experimental
+   * @deprecated
+   *
+   * Use `AjvSchema.create`
    */
   static createFromZod<T>(zodSchema: ZodType<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T> {
-    if (AjvSchema.isZodSchemaWithAjvSchema(zodSchema)) {
-      return AjvSchema.requireCachedAjvSchemaFromZodSchema(zodSchema)
-    }
-
-    const jsonSchema = z.toJSONSchema(zodSchema, { target: 'draft-7' })
-    const ajvSchema = new AjvSchema<T>(jsonSchema as JsonSchema<T>, cfg)
+    return AjvSchema.create(zodSchema, cfg)
+  }
 
-    AjvSchema.cacheAjvSchemaInZodSchema(zodSchema, ajvSchema)
+  static isJsonSchemaBuilder<T>(schema: unknown): schema is JsonSchemaBuilder<T> {
+    return schema instanceof JsonSchemaAnyBuilder
+  }
 
-    return ajvSchema
+  static isZodSchema<T>(schema: unknown): schema is ZodType<T> {
+    return schema instanceof ZodType
   }
 
   readonly cfg: AjvSchemaCfg
@@ -208,19 +223,21 @@ export class AjvSchema<T = unknown> {
     }
   }
 
-  static isZodSchemaWithAjvSchema<T>(schema: ZodType<T>): schema is ZodTypeWithAjvSchema<T> {
-    return !!(schema as any)[HIDDEN_AJV_SCHEMA]
+  static isSchemaWithCachedAjvSchema<Base, T>(
+    schema: Base,
+  ): schema is WithCachedAjvSchema<Base, T> {
+    return !!(schema as any)?.[HIDDEN_AJV_SCHEMA]
   }
 
-  static cacheAjvSchemaInZodSchema<T>(
-    zodSchema: ZodType<T>,
+  static cacheAjvSchema<Base extends AnyObject, T>(
+    schema: Base,
     ajvSchema: AjvSchema<T>,
-  ): ZodTypeWithAjvSchema<T> {
-    return Object.assign(zodSchema, { [HIDDEN_AJV_SCHEMA]: ajvSchema })
+  ): WithCachedAjvSchema<Base, T> {
+    return Object.assign(schema, { [HIDDEN_AJV_SCHEMA]: ajvSchema })
   }
 
-  static requireCachedAjvSchemaFromZodSchema<T>(zodSchema: ZodTypeWithAjvSchema<T>): AjvSchema<T> {
-    return zodSchema[HIDDEN_AJV_SCHEMA]
+  static requireCachedAjvSchema<Base, T>(schema: WithCachedAjvSchema<Base, T>): AjvSchema<T> {
+    return schema[HIDDEN_AJV_SCHEMA]
   }
 
   private getAJVValidateFunction = _lazyValue(() => this.cfg.ajv.compile<T>(this.schema))
@@ -230,6 +247,6 @@ const separator = '\n'
 
 export const HIDDEN_AJV_SCHEMA = Symbol('HIDDEN_AJV_SCHEMA')
 
-export interface ZodTypeWithAjvSchema<T> extends ZodType<T> {
+export type WithCachedAjvSchema<Base, T> = Base & {
   [HIDDEN_AJV_SCHEMA]: AjvSchema<T>
 }