@naturalcycles/nodejs-lib 12.43.0 → 12.46.0

package/CHANGELOG.md

@@ -1,3 +1,32 @@
+# [12.46.0](https://github.com/NaturalCycles/nodejs-lib/compare/v12.45.1...v12.46.0) (2021-10-25)
+
+
+### Features
+
+* stringSchema.stripHTML extension, export sanitizeHTML ([586d576](https://github.com/NaturalCycles/nodejs-lib/commit/586d576585652af822d974607ee913c16a81691f))
+
+## [12.45.1](https://github.com/NaturalCycles/nodejs-lib/compare/v12.45.0...v12.45.1) (2021-10-25)
+
+
+### Bug Fixes
+
+* make JoiValidationError.annotation non-enumerable ([9337fd3](https://github.com/NaturalCycles/nodejs-lib/commit/9337fd3e1a60deb0d9e91b17cf5e7b42f72b5459))
+
+# [12.45.0](https://github.com/NaturalCycles/nodejs-lib/compare/v12.44.0...v12.45.0) (2021-10-22)
+
+
+### Features
+
+* ndjsonMap to take mapper as option ([eb06ae1](https://github.com/NaturalCycles/nodejs-lib/commit/eb06ae1e6ff02ef0338e764e5f7d39142ede4760))
+
+# [12.44.0](https://github.com/NaturalCycles/nodejs-lib/compare/v12.43.0...v12.44.0) (2021-10-22)
+
+
+### Features
+
+* hashAsBuffer, md5AsBuffer ([67bd66b](https://github.com/NaturalCycles/nodejs-lib/commit/67bd66b3a9bdf85016970b7a49d78eca703ec071))
+* remove `ndjson-map` bin ([5ee734b](https://github.com/NaturalCycles/nodejs-lib/commit/5ee734bcaf9ace2bf9a18ffef9f8d09cca6cc315))
+
 # [12.43.0](https://github.com/NaturalCycles/nodejs-lib/compare/v12.42.0...v12.43.0) (2021-10-20)
 
 

package/dist/diff/tableDiff.d.ts

@@ -1,3 +1,4 @@
+import { AnyObject } from '@naturalcycles/js-lib';
 export interface TableDiffOptions {
     /**
      * @default false
@@ -29,4 +30,4 @@ export interface TableDiffOptions {
  *
  * Function is located in nodejs-lib (not js-lib), because it's planned to improve in the future and add e.g colors (via chalk).
  */
-export declare function tableDiff(a: Record<string, any>, b: Record<string, any>, opt?: TableDiffOptions): void;
+export declare function tableDiff(a: AnyObject, b: AnyObject, opt?: TableDiffOptions): void;

package/dist/got/got.model.d.ts

@@ -1,3 +1,4 @@
+import { AnyObject } from '@naturalcycles/js-lib';
 import type { Options } from 'got';
 export interface GetGotOptions extends Options {
     /**
@@ -33,7 +34,7 @@ export interface GetGotOptions extends Options {
      */
     maxResponseLength?: number;
 }
-export interface GotRequestContext extends Record<string, any> {
+export interface GotRequestContext extends AnyObject {
     /**
      * Millisecond-timestamp of when the request was started. To be able to count "time spent".
      */

package/dist/index.d.ts

@@ -8,7 +8,7 @@ import { getGot } from './got/getGot';
 import { GetGotOptions } from './got/got.model';
 import { memoryUsage, memoryUsageFull, processSharedUtil } from './infra/process.util';
 import { Debug, DebugLogLevel, IDebug, IDebugger } from './log/debug';
-import { base64ToBuffer, base64ToString, bufferToBase64, hash, md5, stringToBase64 } from './security/hash.util';
+import { base64ToBuffer, base64ToString, bufferToBase64, hash, md5, hashAsBuffer, md5AsBuffer, stringToBase64 } from './security/hash.util';
 import { ALPHABET_ALPHANUMERIC, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_LOWERCASE, ALPHABET_NUMBER, ALPHABET_UPPERCASE, stringId, stringIdAsync, stringIdUnsafe } from './security/id.util';
 import { getSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, setSecretMap } from './security/secret.util';
 import { hasColors } from './colors/colors';
@@ -17,6 +17,7 @@ import { SlackApiBody, SlackMessage, SlackMessagePrefixHook, SlackMessageProps,
 import { NDJsonStats } from './stream/ndjson/ndjson.model';
 import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead';
 import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite';
+import { ndjsonMap } from './stream/ndjson/ndjsonMap';
 import { ndjsonStreamForEach, NDJSONStreamForEachOptions } from './stream/ndjson/ndjsonStreamForEach';
 import { pipelineFromNDJsonFile, PipelineFromNDJsonFileOptions } from './stream/ndjson/pipelineFromNDJsonFile';
 import { pipelineToNDJsonFile, PipelineToNDJsonFileOptions } from './stream/ndjson/pipelineToNDJsonFile';
@@ -63,5 +64,6 @@ import { AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped
 import { anyObjectSchema, anySchema, arraySchema, oneOfSchema, binarySchema, booleanDefaultToFalseSchema, booleanSchema, dateStringSchema, emailSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, integerSchema, ipAddressSchema, numberSchema, objectSchema, percentageSchema, semVerSchema, SEM_VER_PATTERN, slugSchema, stringSchema, unixTimestampSchema, urlSchema, userAgentSchema, utcOffsetSchema, verSchema } from './validation/joi/joi.shared.schemas';
 import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi.validation.error';
 import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIfInvalid, validate } from './validation/joi/joi.validation.util';
-export type { JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, };
-export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, DebugLogLevel, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, };
+import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
+export type { JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
+export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, DebugLogLevel, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, };

package/dist/index.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
-exports.writableForEach = exports.transformNoOp = exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.DebugLogLevel = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
-exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = void 0;
+exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.DebugLogLevel = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
+exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = exports.transformMapSimple = void 0;
 const ajv_1 = require("ajv");
 exports.Ajv = ajv_1.default;
 const got_1 = require("got");
@@ -27,6 +27,8 @@ Object.defineProperty(exports, "base64ToString", { enumerable: true, get: functi
 Object.defineProperty(exports, "bufferToBase64", { enumerable: true, get: function () { return hash_util_1.bufferToBase64; } });
 Object.defineProperty(exports, "hash", { enumerable: true, get: function () { return hash_util_1.hash; } });
 Object.defineProperty(exports, "md5", { enumerable: true, get: function () { return hash_util_1.md5; } });
+Object.defineProperty(exports, "hashAsBuffer", { enumerable: true, get: function () { return hash_util_1.hashAsBuffer; } });
+Object.defineProperty(exports, "md5AsBuffer", { enumerable: true, get: function () { return hash_util_1.md5AsBuffer; } });
 Object.defineProperty(exports, "stringToBase64", { enumerable: true, get: function () { return hash_util_1.stringToBase64; } });
 const id_util_1 = require("./security/id.util");
 Object.defineProperty(exports, "ALPHABET_ALPHANUMERIC", { enumerable: true, get: function () { return id_util_1.ALPHABET_ALPHANUMERIC; } });
@@ -57,6 +59,8 @@ const ndJsonFileRead_1 = require("./stream/ndjson/ndJsonFileRead");
 Object.defineProperty(exports, "ndJsonFileRead", { enumerable: true, get: function () { return ndJsonFileRead_1.ndJsonFileRead; } });
 const ndJsonFileWrite_1 = require("./stream/ndjson/ndJsonFileWrite");
 Object.defineProperty(exports, "ndJsonFileWrite", { enumerable: true, get: function () { return ndJsonFileWrite_1.ndJsonFileWrite; } });
+const ndjsonMap_1 = require("./stream/ndjson/ndjsonMap");
+Object.defineProperty(exports, "ndjsonMap", { enumerable: true, get: function () { return ndjsonMap_1.ndjsonMap; } });
 const ndjsonStreamForEach_1 = require("./stream/ndjson/ndjsonStreamForEach");
 Object.defineProperty(exports, "ndjsonStreamForEach", { enumerable: true, get: function () { return ndjsonStreamForEach_1.ndjsonStreamForEach; } });
 const pipelineFromNDJsonFile_1 = require("./stream/ndjson/pipelineFromNDJsonFile");
@@ -185,3 +189,5 @@ Object.defineProperty(exports, "getValidationResult", { enumerable: true, get: f
 Object.defineProperty(exports, "isValid", { enumerable: true, get: function () { return joi_validation_util_1.isValid; } });
 Object.defineProperty(exports, "undefinedIfInvalid", { enumerable: true, get: function () { return joi_validation_util_1.undefinedIfInvalid; } });
 Object.defineProperty(exports, "validate", { enumerable: true, get: function () { return joi_validation_util_1.validate; } });
+const sanitize_util_1 = require("./validation/sanitize.util");
+Object.defineProperty(exports, "sanitizeHTML", { enumerable: true, get: function () { return sanitize_util_1.sanitizeHTML; } });

package/dist/security/hash.util.d.ts

@@ -1,6 +1,8 @@
 /// <reference types="node" />
 export declare function md5(s: string | Buffer): string;
+export declare function md5AsBuffer(s: string | Buffer): Buffer;
 export declare function hash(s: string | Buffer, algorithm: string): string;
+export declare function hashAsBuffer(s: string | Buffer, algorithm: string): Buffer;
 export declare function stringToBase64(s: string): string;
 export declare function base64ToString(strBase64: string): string;
 export declare function bufferToBase64(b: Buffer): string;

package/dist/security/hash.util.js

@@ -1,15 +1,23 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.hash = exports.md5 = void 0;
+exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.hashAsBuffer = exports.hash = exports.md5AsBuffer = exports.md5 = void 0;
 const crypto = require("crypto");
 function md5(s) {
     return hash(s, 'md5');
 }
 exports.md5 = md5;
+function md5AsBuffer(s) {
+    return hashAsBuffer(s, 'md5');
+}
+exports.md5AsBuffer = md5AsBuffer;
 function hash(s, algorithm) {
     return crypto.createHash(algorithm).update(s).digest('hex');
 }
 exports.hash = hash;
+function hashAsBuffer(s, algorithm) {
+    return crypto.createHash(algorithm).update(s).digest();
+}
+exports.hashAsBuffer = hashAsBuffer;
 function stringToBase64(s) {
     return Buffer.from(s, 'utf8').toString('base64');
 }

package/dist/stream/ndjson/ndjsonMap.d.ts

@@ -1,21 +1,23 @@
-import { TransformMapOptions } from '../..';
-export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT> {
+import { AsyncMapper } from '@naturalcycles/js-lib';
+import { TransformMapOptions, TransformLogProgressOptions } from '../..';
+export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT>, TransformLogProgressOptions<IN> {
     inputFilePath: string;
     outputFilePath: string;
-    mapperFilePath: string;
     limitInput?: number;
     limitOutput?: number;
-    /**
-     * @default 1000
-     */
-    logEveryInput?: number;
     /**
      * @default 100_000
      */
     logEveryOutput?: number;
+    /**
+     * Defaults to `true` for ndjsonMap
+     *
+     * @default true
+     */
+    flattenArrayOutput?: boolean;
 }
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
  */
-export declare function ndjsonMap<IN = any, OUT = any>(opt: NDJSONMapOptions<IN, OUT>): Promise<void>;
+export declare function ndjsonMap<IN = any, OUT = any>(mapper: AsyncMapper<IN, OUT>, opt: NDJSONMapOptions<IN, OUT>): Promise<void>;

package/dist/stream/ndjson/ndjsonMap.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ndjsonMap = void 0;
 const fs_1 = require("fs");
-const path = require("path");
 const zlib_1 = require("zlib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const __1 = require("../..");
@@ -10,27 +9,13 @@ const __1 = require("../..");
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
  */
-async function ndjsonMap(opt) {
-    const { inputFilePath, outputFilePath, mapperFilePath, logEveryInput = 1000, logEveryOutput = 100000, limitInput, limitOutput, } = opt;
+async function ndjsonMap(mapper, opt) {
+    const { inputFilePath, outputFilePath, logEveryOutput = 100000, limitInput, limitOutput } = opt;
     (0, __1.requireFileToExist)(inputFilePath);
-    (0, __1.requireFileToExist)(mapperFilePath);
-    const resolvedMapperPath = path.resolve(mapperFilePath);
     console.log({
         inputFilePath,
         outputFilePath,
-        mapperFilePath,
-        resolvedMapperPath,
     });
-    // This is to allow importing *.ts mappers
-    try {
-        require('ts-node/register/transpile-only');
-        require('tsconfig-paths/register');
-    }
-    catch { } // require if exists
-    const { mapper } = require(resolvedMapperPath);
-    if (!mapper) {
-        throw new Error(`Mapper file should export "mapper" function`);
-    }
     const transformUnzip = inputFilePath.endsWith('.gz') ? [(0, zlib_1.createUnzip)()] : [];
     const transformZip = outputFilePath.endsWith('.gz') ? [(0, zlib_1.createGzip)()] : [];
     await (0, __1._pipeline)([
@@ -39,7 +24,7 @@ async function ndjsonMap(opt) {
         (0, __1.transformSplit)(),
         (0, __1.transformJsonParse)(),
         (0, __1.transformLimit)(limitInput),
-        (0, __1.transformLogProgress)({ metric: 'read', logEvery: logEveryInput }),
+        (0, __1.transformLogProgress)({ metric: 'read', ...opt }),
         (0, __1.transformMap)(mapper, {
             flattenArrayOutput: true,
             errorMode: js_lib_1.ErrorMode.SUPPRESS,

package/dist/stream/ndjson/transformJsonParse.d.ts

@@ -3,8 +3,9 @@ import { Reviver } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformJsonParseOptions {
     /**
-     * @default true
      * If true - will throw an error on JSON.parse / stringify error
+     *
+     * @default true
      */
     strict?: boolean;
     reviver?: Reviver;
@@ -23,5 +24,5 @@ export interface TransformJsonParseOptions {
  *   consumeYourStream...
  * [)
  */
-export declare function transformJsonParse<OUT = Record<string, any>>(opt?: TransformJsonParseOptions): TransformTyped<string | Buffer, OUT>;
+export declare function transformJsonParse<OUT = any>(opt?: TransformJsonParseOptions): TransformTyped<string | Buffer, OUT>;
 export declare const bufferReviver: Reviver;

package/dist/stream/ndjson/transformToNDJson.d.ts

@@ -1,13 +1,15 @@
 import { TransformTyped } from '../stream.model';
 export interface TransformToNDJsonOptions {
     /**
-     * @default true
      * If true - will throw an error on JSON.parse / stringify error
+     *
+     * @default true
      */
     strict?: boolean;
     /**
-     * @default false
      * If true - will run `sortObjectDeep()` on each object to achieve deterministic sort
+     *
+     * @default false
      */
     sortObjects?: boolean;
     /**

package/dist/stream/transform/transformBuffer.d.ts

@@ -7,4 +7,4 @@ export interface TransformBufferOptions extends TransformOptions {
  *
  * @default batchSize is 10
  */
-export declare function transformBuffer<IN = Record<string, any>>(opt: TransformBufferOptions): TransformTyped<IN, IN[]>;
+export declare function transformBuffer<IN = any>(opt: TransformBufferOptions): TransformTyped<IN, IN[]>;

package/dist/stream/transform/transformLogProgress.d.ts

@@ -1,3 +1,4 @@
+import { AnyObject } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
     /**
@@ -9,7 +10,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
     /**
      * Include `heapUsed` in log.
      *
-     * @default true
+     * @default false
      */
     heapUsed?: boolean;
     /**
@@ -27,7 +28,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
     /**
      * Incude Peak RSS in log.
      *
-     * @default false
+     * @default true
      */
     peakRSS?: boolean;
     /**
@@ -73,7 +74,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      *
      * chunk is undefined for "final" stats, otherwise is defined.
      */
-    extra?: (chunk: IN | undefined, index: number) => Record<string, any>;
+    extra?: (chunk: IN | undefined, index: number) => AnyObject;
 }
 /**
  * Pass-through transform that optionally logs progress.

package/dist/stream/transform/transformLogProgress.js

@@ -15,7 +15,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = true, rss: logRss = true, peakRSS: logPeakRSS = false, logRPS = true, logEvery = 1000, extra, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, extra, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();

package/dist/stream/transform/transformMap.d.ts

@@ -2,17 +2,18 @@ import { AsyncMapper, AsyncPredicate, ErrorMode } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
-     * @default false
      * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
+     *
+     * @default false
      */
     flattenArrayOutput?: boolean;
     /**
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
      *
-     * @default to filter out undefined/null values, but pass anything else
-     *
      * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+     *
+     * @default to filter out undefined/null values, but pass anything else
      */
     predicate?: AsyncPredicate<OUT>;
     /**

package/dist/stream/transform/worker/transformMultiThreaded.d.ts

@@ -1,3 +1,4 @@
+import { AnyObject } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../../stream.model';
 export interface TransformMultiThreadedOptions {
     /**
@@ -19,7 +20,7 @@ export interface TransformMultiThreadedOptions {
     /**
      * Passed to the Worker as `workerData` property (initial data).
      */
-    workerData?: Record<string, any>;
+    workerData?: AnyObject;
 }
 /**
  * Spawns a pool of Workers (threads).

package/dist/validation/joi/joi.validation.error.d.ts

@@ -17,6 +17,9 @@ export interface JoiValidationErrorData extends ErrorData {
     /**
      * Error "annotation" is stripped in Error.message.
      * This field contains the "full" annotation.
+     *
+     * This field is non-enumerable, won't be printed or included in JSON by default,
+     * but still accessible programmatically (via `err.data.annotation`) when needed!
      */
     annotation?: string;
 }

package/dist/validation/joi/joi.validation.util.js

@@ -122,10 +122,16 @@ function createError(value, err, objectName) {
         tokens.push(annotation);
     }
     const msg = tokens.join('\n');
-    return new joi_validation_error_1.JoiValidationError(msg, {
+    const data = {
         joiValidationErrorItems: err.details,
         ...(objectName && { joiValidationObjectName: objectName }),
         ...(objectId && { joiValidationObjectId: objectId }),
-        annotation,
+    };
+    // Make annotation non-enumerable, to not get it automatically printed,
+    // but still accessible
+    Object.defineProperty(data, 'annotation', {
+        enumerable: false,
+        value: annotation,
     });
+    return new joi_validation_error_1.JoiValidationError(msg, data);
 }

package/dist/validation/joi/string.extensions.d.ts

@@ -3,9 +3,18 @@ import * as Joi from 'joi';
 import { AnySchemaTyped } from './joi.model';
 export interface ExtendedStringSchema extends StringSchema, AnySchemaTyped<string> {
     dateString(min?: string, max?: string): this;
+    stripHTML(opt?: JoiStripHTMLOptions): this;
 }
-export interface DateStringParams {
+export interface JoiDateStringOptions {
     min?: string;
     max?: string;
 }
+export interface JoiStripHTMLOptions {
+    /**
+     * 'Strict' would throw an error if it detects any HTML.
+     * Non-strict (default) does not error, but DOES convert the string to the string without HTML.
+     * Internally uses `sanitize-html` library, with allowedTags = [], and method = 'discard'.
+     */
+    strict?: boolean;
+}
 export declare function stringExtensions(joi: typeof Joi): Extension;

package/dist/validation/joi/string.extensions.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.stringExtensions = void 0;
 const time_lib_1 = require("@naturalcycles/time-lib");
+const sanitize = require("sanitize-html");
 function stringExtensions(joi) {
     return {
         type: 'string',
@@ -11,6 +12,7 @@ function stringExtensions(joi) {
             'string.dateStringMin': '"{{#label}}" must be not earlier than {{#min}}',
             'string.dateStringMax': '"{{#label}}" must be not later than {{#max}}',
             'string.dateStringCalendarAccuracy': '"{{#label}}" must be a VALID calendar date',
+            'string.stripHTML': '"{{#label}}" must NOT contain any HTML tags',
         },
         rules: {
             dateString: {
@@ -57,6 +59,7 @@ function stringExtensions(joi) {
                         err = 'string.dateStringMax';
                     }
                     else if (!(0, time_lib_1.dayjs)(v).isValid()) {
+                        // todo: replace with another regex (from ajv-validators) for speed
                         err = 'string.dateStringCalendarAccuracy';
                     }
                     if (err) {
@@ -65,6 +68,37 @@ function stringExtensions(joi) {
                     return v; // validation passed
                 },
             },
+            stripHTML: {
+                method(opt) {
+                    return this.$_addRule({
+                        name: 'stripHTML',
+                        args: {
+                            strict: false,
+                            ...opt,
+                        },
+                    });
+                },
+                args: [
+                    {
+                        name: 'strict',
+                        ref: true,
+                        assert: v => typeof v === 'boolean',
+                        message: 'must be a boolean',
+                    },
+                ],
+                validate(v, helpers, args) {
+                    console.log('!!! stripHTML', args, v);
+                    const { strict = false } = args;
+                    const r = sanitize(v, {
+                        allowedTags: [], // no html tags allowed at all
+                        // disallowedTagsMode: 'discard' // discard is default
+                    });
+                    if (strict && r !== v) {
+                        return helpers.error('string.stripHTML', args);
+                    }
+                    return r; // return converted value (or the same, if there was nothing to sanitize)
+                },
+            },
         },
     };
 }

package/dist/validation/sanitize.util.d.ts

@@ -0,0 +1,8 @@
+import * as sanitize from 'sanitize-html';
+export declare type SanitizeHTMLOptions = sanitize.IOptions;
+/**
+ * Simply a wrapper around `sanitize-html` library.
+ *
+ * @experimental
+ */
+export declare function sanitizeHTML(s: string, opt?: SanitizeHTMLOptions): string;

package/dist/validation/sanitize.util.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeHTML = void 0;
+const sanitize = require("sanitize-html");
+/**
+ * Simply a wrapper around `sanitize-html` library.
+ *
+ * @experimental
+ */
+function sanitizeHTML(s, opt) {
+    return sanitize(s, opt);
+}
+exports.sanitizeHTML = sanitizeHTML;

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.43.0",
+  "version": "12.46.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",
     "docs-build": "vuepress build docs",
-    "ndjson-map-debug": "tsn ./src/bin/ndjson-map.ts --in ./tmp/ndjsonMapIn.ndjson --out ./tmp/ndjsonMapOut.ndjson --mapper ./src/test/ndjson/ndjsonMapper.ts",
     "slack-this-debug": "tsn ./src/bin/slack-this.ts --channel test --msg 'Hello slack!'",
     "secrets-gen-key-debug": "tsn ./src/bin/secrets-gen-key.ts",
     "secrets-encrypt-debug": "tsn ./src/bin/secrets-encrypt.ts",
@@ -18,6 +17,7 @@
     "@naturalcycles/js-lib": "^14.0.0",
     "@naturalcycles/time-lib": "^3.0.1",
     "@types/lru-cache": "^5.1.0",
+    "@types/sanitize-html": "^2.5.0",
     "@types/through2-concurrent": "^2.0.0",
     "ajv": "^8.6.2",
     "ajv-formats": "^2.1.0",
@@ -36,6 +36,7 @@
     "lru-cache": "^6.0.0",
     "move-file": "^2.0.0",
     "nanoid": "^3.0.0",
+    "sanitize-html": "^2.5.2",
     "supports-color": "^8.0.0",
     "through2-concurrent": "^2.0.0",
     "yargs": "^17.0.0"
@@ -61,7 +62,6 @@
     "del": "dist/bin/del.js",
     "kpy": "dist/bin/kpy.js",
     "json2env": "dist/bin/json2env.js",
-    "ndjson-map": "dist/bin/ndjson-map.js",
     "slack-this": "dist/bin/slack-this.js",
     "secrets-gen-key": "dist/bin/secrets-gen-key.js",
     "secrets-encrypt": "dist/bin/secrets-encrypt.js",

package/src/diff/tableDiff.ts

@@ -1,4 +1,4 @@
-import { StringMap, _truncate } from '@naturalcycles/js-lib'
+import { StringMap, _truncate, AnyObject } from '@naturalcycles/js-lib'
 
 export interface TableDiffOptions {
   /**
@@ -35,11 +35,7 @@ export interface TableDiffOptions {
  *
  * Function is located in nodejs-lib (not js-lib), because it's planned to improve in the future and add e.g colors (via chalk).
  */
-export function tableDiff(
-  a: Record<string, any>,
-  b: Record<string, any>,
-  opt: TableDiffOptions = {},
-): void {
+export function tableDiff(a: AnyObject, b: AnyObject, opt: TableDiffOptions = {}): void {
   const { maxFieldLen, aTitle = 'a', bTitle = 'b' } = opt
   const diff: StringMap<any> = {}
 

package/src/got/got.model.ts

@@ -1,3 +1,4 @@
+import { AnyObject } from '@naturalcycles/js-lib'
 import type { Options } from 'got'
 
 export interface GetGotOptions extends Options {
@@ -40,7 +41,7 @@ export interface GetGotOptions extends Options {
   maxResponseLength?: number
 }
 
-export interface GotRequestContext extends Record<string, any> {
+export interface GotRequestContext extends AnyObject {
   /**
    * Millisecond-timestamp of when the request was started. To be able to count "time spent".
    */

package/src/index.ts

@@ -14,6 +14,8 @@ import {
   bufferToBase64,
   hash,
   md5,
+  hashAsBuffer,
+  md5AsBuffer,
   stringToBase64,
 } from './security/hash.util'
 import {
@@ -48,6 +50,7 @@ import {
 import { NDJsonStats } from './stream/ndjson/ndjson.model'
 import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead'
 import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite'
+import { ndjsonMap } from './stream/ndjson/ndjsonMap'
 import {
   ndjsonStreamForEach,
   NDJSONStreamForEachOptions,
@@ -169,6 +172,7 @@ import {
   undefinedIfInvalid,
   validate,
 } from './validation/joi/joi.validation.util'
+import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util'
 
 export type {
   JoiValidationErrorData,
@@ -216,6 +220,7 @@ export type {
   AjvValidationOptions,
   AjvSchemaCfg,
   AjvValidationErrorData,
+  SanitizeHTMLOptions,
 }
 
 export {
@@ -275,6 +280,8 @@ export {
   ALPHABET_ALPHANUMERIC,
   md5,
   hash,
+  hashAsBuffer,
+  md5AsBuffer,
   stringToBase64,
   base64ToString,
   bufferToBase64,
@@ -302,6 +309,7 @@ export {
   readableMapToArray,
   _pipeline,
   transformBuffer,
+  ndjsonMap,
   ndJsonFileRead,
   ndJsonFileWrite,
   ndjsonStreamForEach,
@@ -343,4 +351,5 @@ export {
   readJsonSchemas,
   readAjvSchemas,
   hasColors,
+  sanitizeHTML,
 }

package/src/security/hash.util.ts

@@ -4,10 +4,18 @@ export function md5(s: string | Buffer): string {
   return hash(s, 'md5')
 }
 
+export function md5AsBuffer(s: string | Buffer): Buffer {
+  return hashAsBuffer(s, 'md5')
+}
+
 export function hash(s: string | Buffer, algorithm: string): string {
   return crypto.createHash(algorithm).update(s).digest('hex')
 }
 
+export function hashAsBuffer(s: string | Buffer, algorithm: string): Buffer {
+  return crypto.createHash(algorithm).update(s).digest()
+}
+
 export function stringToBase64(s: string): string {
   return Buffer.from(s, 'utf8').toString('base64')
 }

package/src/stream/ndjson/ndjsonMap.ts

@@ -1,5 +1,4 @@
 import { createReadStream, createWriteStream } from 'fs'
-import * as path from 'path'
 import { createGzip, createUnzip } from 'zlib'
 import { AsyncMapper, ErrorMode } from '@naturalcycles/js-lib'
 import {
@@ -12,28 +11,29 @@ import {
   transformSplit,
   transformToNDJson,
   _pipeline,
+  TransformLogProgressOptions,
 } from '../..'
 
-interface NDJSONMapperFile<IN = any, OUT = any> {
-  mapper: AsyncMapper<IN, OUT>
-}
-
-export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOptions<IN, OUT> {
+export interface NDJSONMapOptions<IN = any, OUT = IN>
+  extends TransformMapOptions<IN, OUT>,
+    TransformLogProgressOptions<IN> {
   inputFilePath: string
   outputFilePath: string
-  mapperFilePath: string
+
   limitInput?: number
   limitOutput?: number
 
   /**
-   * @default 1000
+   * @default 100_000
    */
-  logEveryInput?: number
+  logEveryOutput?: number
 
   /**
-   * @default 100_000
+   * Defaults to `true` for ndjsonMap
+   *
+   * @default true
    */
-  logEveryOutput?: number
+  flattenArrayOutput?: boolean
 }
 
 /**
@@ -41,42 +41,18 @@ export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOption
  * Zips output file automatically, if it ends with `.gz`.
  */
 export async function ndjsonMap<IN = any, OUT = any>(
+  mapper: AsyncMapper<IN, OUT>,
   opt: NDJSONMapOptions<IN, OUT>,
 ): Promise<void> {
-  const {
-    inputFilePath,
-    outputFilePath,
-    mapperFilePath,
-    logEveryInput = 1000,
-    logEveryOutput = 100_000,
-    limitInput,
-    limitOutput,
-  } = opt
+  const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt
 
   requireFileToExist(inputFilePath)
-  requireFileToExist(mapperFilePath)
-
-  const resolvedMapperPath = path.resolve(mapperFilePath)
 
   console.log({
     inputFilePath,
     outputFilePath,
-    mapperFilePath,
-    resolvedMapperPath,
   })
 
-  // This is to allow importing *.ts mappers
-  try {
-    require('ts-node/register/transpile-only')
-    require('tsconfig-paths/register')
-  } catch {} // require if exists
-
-  const { mapper } = require(resolvedMapperPath) as NDJSONMapperFile<IN, OUT>
-
-  if (!mapper) {
-    throw new Error(`Mapper file should export "mapper" function`)
-  }
-
   const transformUnzip = inputFilePath.endsWith('.gz') ? [createUnzip()] : []
   const transformZip = outputFilePath.endsWith('.gz') ? [createGzip()] : []
 
@@ -86,7 +62,7 @@ export async function ndjsonMap<IN = any, OUT = any>(
     transformSplit(), // splits by \n
     transformJsonParse(),
     transformLimit(limitInput),
-    transformLogProgress({ metric: 'read', logEvery: logEveryInput }),
+    transformLogProgress({ metric: 'read', ...opt }),
     transformMap(mapper, {
       flattenArrayOutput: true,
       errorMode: ErrorMode.SUPPRESS,

package/src/stream/ndjson/transformJsonParse.ts

@@ -4,8 +4,9 @@ import { TransformTyped } from '../stream.model'
 
 export interface TransformJsonParseOptions {
   /**
-   * @default true
    * If true - will throw an error on JSON.parse / stringify error
+   *
+   * @default true
    */
   strict?: boolean
 
@@ -26,7 +27,7 @@ export interface TransformJsonParseOptions {
  *   consumeYourStream...
  * [)
  */
-export function transformJsonParse<OUT = Record<string, any>>(
+export function transformJsonParse<OUT = any>(
   opt: TransformJsonParseOptions = {},
 ): TransformTyped<string | Buffer, OUT> {
   const { strict = true, reviver } = opt

package/src/stream/ndjson/transformToNDJson.ts

@@ -4,14 +4,16 @@ import { TransformTyped } from '../stream.model'
 
 export interface TransformToNDJsonOptions {
   /**
-   * @default true
    * If true - will throw an error on JSON.parse / stringify error
+   *
+   * @default true
    */
   strict?: boolean
 
   /**
-   * @default false
    * If true - will run `sortObjectDeep()` on each object to achieve deterministic sort
+   *
+   * @default false
    */
   sortObjects?: boolean
 

package/src/stream/transform/transformBuffer.ts

@@ -10,9 +10,7 @@ export interface TransformBufferOptions extends TransformOptions {
  *
  * @default batchSize is 10
  */
-export function transformBuffer<IN = Record<string, any>>(
-  opt: TransformBufferOptions,
-): TransformTyped<IN, IN[]> {
+export function transformBuffer<IN = any>(opt: TransformBufferOptions): TransformTyped<IN, IN[]> {
   const { batchSize } = opt
 
   let buf: IN[] = []

package/src/stream/transform/transformLogProgress.ts

@@ -1,6 +1,6 @@
 import { Transform } from 'stream'
 import { inspect, InspectOptions } from 'util'
-import { SimpleMovingAverage, _mb, _since } from '@naturalcycles/js-lib'
+import { SimpleMovingAverage, _mb, _since, AnyObject } from '@naturalcycles/js-lib'
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
@@ -17,7 +17,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
   /**
    * Include `heapUsed` in log.
    *
-   * @default true
+   * @default false
    */
   heapUsed?: boolean
 
@@ -38,7 +38,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
   /**
    * Incude Peak RSS in log.
    *
-   * @default false
+   * @default true
    */
   peakRSS?: boolean
 
@@ -91,7 +91,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    *
    * chunk is undefined for "final" stats, otherwise is defined.
    */
-  extra?: (chunk: IN | undefined, index: number) => Record<string, any>
+  extra?: (chunk: IN | undefined, index: number) => AnyObject
 }
 
 const inspectOpt: InspectOptions = {
@@ -108,9 +108,9 @@ export function transformLogProgress<IN = any>(
   const {
     metric = 'progress',
     heapTotal: logHeapTotal = false,
-    heapUsed: logHeapUsed = true,
+    heapUsed: logHeapUsed = false,
     rss: logRss = true,
-    peakRSS: logPeakRSS = false,
+    peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
     extra,

package/src/stream/transform/transformMap.ts

@@ -12,8 +12,9 @@ import { TransformTyped } from '../stream.model'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
-   * @default false
    * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
+   *
+   * @default false
    */
   flattenArrayOutput?: boolean
 
@@ -21,9 +22,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * Predicate to filter outgoing results (after mapper).
    * Allows to not emit all results.
    *
-   * @default to filter out undefined/null values, but pass anything else
-   *
    * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+   *
+   * @default to filter out undefined/null values, but pass anything else
    */
   predicate?: AsyncPredicate<OUT>
 

package/src/stream/transform/worker/transformMultiThreaded.ts

@@ -1,5 +1,5 @@
 import { Worker } from 'worker_threads'
-import { DeferredPromise, pDefer, _range } from '@naturalcycles/js-lib'
+import { DeferredPromise, pDefer, _range, AnyObject } from '@naturalcycles/js-lib'
 import through2Concurrent = require('through2-concurrent')
 import { TransformTyped } from '../../stream.model'
 import { WorkerInput, WorkerOutput } from './transformMultiThreaded.model'
@@ -30,7 +30,7 @@ export interface TransformMultiThreadedOptions {
   /**
    * Passed to the Worker as `workerData` property (initial data).
    */
-  workerData?: Record<string, any>
+  workerData?: AnyObject
 }
 
 const workerProxyFilePath = `${__dirname}/workerClassProxy.js`

package/src/validation/joi/joi.validation.error.ts

@@ -18,6 +18,9 @@ export interface JoiValidationErrorData extends ErrorData {
   /**
    * Error "annotation" is stripped in Error.message.
    * This field contains the "full" annotation.
+   *
+   * This field is non-enumerable, won't be printed or included in JSON by default,
+   * but still accessible programmatically (via `err.data.annotation`) when needed!
    */
   annotation?: string
 }

package/src/validation/joi/joi.validation.util.ts

@@ -9,7 +9,7 @@
 import { _hb, _isObject, _truncateMiddle } from '@naturalcycles/js-lib'
 import { ValidationError, ValidationOptions } from 'joi'
 import { AnySchemaTyped } from './joi.model'
-import { JoiValidationError } from './joi.validation.error'
+import { JoiValidationError, JoiValidationErrorData } from './joi.validation.error'
 
 // todo: consider replacing with Tuple of [error, value]
 export interface JoiValidationResult<T = any> {
@@ -163,10 +163,18 @@ function createError(value: any, err: ValidationError, objectName?: string): Joi
 
   const msg = tokens.join('\n')
 
-  return new JoiValidationError(msg, {
+  const data: JoiValidationErrorData = {
     joiValidationErrorItems: err.details,
     ...(objectName && { joiValidationObjectName: objectName }),
     ...(objectId && { joiValidationObjectId: objectId }),
-    annotation,
+  }
+
+  // Make annotation non-enumerable, to not get it automatically printed,
+  // but still accessible
+  Object.defineProperty(data, 'annotation', {
+    enumerable: false,
+    value: annotation,
   })
+
+  return new JoiValidationError(msg, data)
 }

package/src/validation/joi/string.extensions.ts

@@ -1,17 +1,28 @@
 import { dayjs } from '@naturalcycles/time-lib'
 import { Extension, StringSchema } from 'joi'
 import * as Joi from 'joi'
+import * as sanitize from 'sanitize-html'
 import { AnySchemaTyped } from './joi.model'
 
 export interface ExtendedStringSchema extends StringSchema, AnySchemaTyped<string> {
   dateString(min?: string, max?: string): this
+  stripHTML(opt?: JoiStripHTMLOptions): this
 }
 
-export interface DateStringParams {
+export interface JoiDateStringOptions {
   min?: string
   max?: string
 }
 
+export interface JoiStripHTMLOptions {
+  /**
+   * 'Strict' would throw an error if it detects any HTML.
+   * Non-strict (default) does not error, but DOES convert the string to the string without HTML.
+   * Internally uses `sanitize-html` library, with allowedTags = [], and method = 'discard'.
+   */
+  strict?: boolean
+}
+
 export function stringExtensions(joi: typeof Joi): Extension {
   return {
     type: 'string',
@@ -21,13 +32,14 @@ export function stringExtensions(joi: typeof Joi): Extension {
       'string.dateStringMin': '"{{#label}}" must be not earlier than {{#min}}',
       'string.dateStringMax': '"{{#label}}" must be not later than {{#max}}',
       'string.dateStringCalendarAccuracy': '"{{#label}}" must be a VALID calendar date',
+      'string.stripHTML': '"{{#label}}" must NOT contain any HTML tags',
     },
     rules: {
       dateString: {
         method(min?: string, max?: string) {
           return this.$_addRule({
             name: 'dateString',
-            args: { min, max } as DateStringParams,
+            args: { min, max } as JoiDateStringOptions,
           })
         },
         args: [
@@ -44,7 +56,7 @@ export function stringExtensions(joi: typeof Joi): Extension {
             message: 'must be a string',
           },
         ],
-        validate(v: string, helpers, args: DateStringParams) {
+        validate(v: string, helpers, args: JoiDateStringOptions) {
           // console.log('dateString validate called', {v, args})
 
           let err: string | undefined
@@ -67,6 +79,7 @@ export function stringExtensions(joi: typeof Joi): Extension {
           } else if (max && v > max) {
             err = 'string.dateStringMax'
           } else if (!dayjs(v).isValid()) {
+            // todo: replace with another regex (from ajv-validators) for speed
             err = 'string.dateStringCalendarAccuracy'
           }
 
@@ -77,6 +90,40 @@ export function stringExtensions(joi: typeof Joi): Extension {
           return v // validation passed
         },
       },
+      stripHTML: {
+        method(opt?: JoiStripHTMLOptions) {
+          return this.$_addRule({
+            name: 'stripHTML',
+            args: {
+              strict: false,
+              ...opt,
+            },
+          })
+        },
+        args: [
+          {
+            name: 'strict',
+            ref: true,
+            assert: v => typeof v === 'boolean',
+            message: 'must be a boolean',
+          },
+        ],
+        validate(v: string, helpers, args: JoiStripHTMLOptions) {
+          console.log('!!! stripHTML', args, v)
+          const { strict = false } = args
+
+          const r = sanitize(v, {
+            allowedTags: [], // no html tags allowed at all
+            // disallowedTagsMode: 'discard' // discard is default
+          })
+
+          if (strict && r !== v) {
+            return helpers.error('string.stripHTML', args)
+          }
+
+          return r // return converted value (or the same, if there was nothing to sanitize)
+        },
+      },
     },
   }
 }

package/src/validation/sanitize.util.ts

@@ -0,0 +1,12 @@
+import * as sanitize from 'sanitize-html'
+
+export type SanitizeHTMLOptions = sanitize.IOptions
+
+/**
+ * Simply a wrapper around `sanitize-html` library.
+ *
+ * @experimental
+ */
+export function sanitizeHTML(s: string, opt?: SanitizeHTMLOptions): string {
+  return sanitize(s, opt)
+}

package/dist/bin/ndjson-map.d.ts

@@ -1,2 +0,0 @@
-#!/usr/bin/env node
-export {};

package/dist/bin/ndjson-map.js

@@ -1,48 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const yargs = require("yargs");
-const script_1 = require("../script");
-const ndjsonMap_1 = require("../stream/ndjson/ndjsonMap");
-(0, script_1.runScript)(async () => {
-    const { in: inputFilePath, out: outputFilePath, mapper: mapperFilePath, logEveryInput, logEveryOutput, limitInput, limitOutput, } = yargs.options({
-        in: {
-            type: 'string',
-            demandOption: true,
-            desc: 'Input ndjson file path',
-        },
-        out: {
-            type: 'string',
-            desc: 'Output ndjson file path',
-            demandOption: true,
-        },
-        mapper: {
-            type: 'string',
-            desc: 'Mapper file path',
-            demandOption: true,
-        },
-        logEveryInput: {
-            type: 'number',
-            default: 1000,
-        },
-        logEveryOutput: {
-            type: 'number',
-            default: 100000,
-        },
-        limitInput: {
-            type: 'number',
-        },
-        limitOutput: {
-            type: 'number',
-        },
-    }).argv;
-    await (0, ndjsonMap_1.ndjsonMap)({
-        inputFilePath,
-        outputFilePath,
-        mapperFilePath,
-        logEveryInput,
-        logEveryOutput,
-        limitInput,
-        limitOutput,
-    });
-});

package/src/bin/ndjson-map.ts

@@ -1,57 +0,0 @@
-#!/usr/bin/env node
-
-import * as yargs from 'yargs'
-import { runScript } from '../script'
-import { ndjsonMap } from '../stream/ndjson/ndjsonMap'
-
-runScript(async () => {
-  const {
-    in: inputFilePath,
-    out: outputFilePath,
-    mapper: mapperFilePath,
-    logEveryInput,
-    logEveryOutput,
-    limitInput,
-    limitOutput,
-  } = yargs.options({
-    in: {
-      type: 'string',
-      demandOption: true,
-      desc: 'Input ndjson file path',
-    },
-    out: {
-      type: 'string',
-      desc: 'Output ndjson file path',
-      demandOption: true,
-    },
-    mapper: {
-      type: 'string',
-      desc: 'Mapper file path',
-      demandOption: true,
-    },
-    logEveryInput: {
-      type: 'number',
-      default: 1000,
-    },
-    logEveryOutput: {
-      type: 'number',
-      default: 100_000,
-    },
-    limitInput: {
-      type: 'number',
-    },
-    limitOutput: {
-      type: 'number',
-    },
-  }).argv
-
-  await ndjsonMap({
-    inputFilePath,
-    outputFilePath,
-    mapperFilePath,
-    logEveryInput,
-    logEveryOutput,
-    limitInput,
-    limitOutput,
-  })
-})