@naturalcycles/nodejs-lib 12.62.1 → 12.64.0

package/dist/got/getGot.d.ts

@@ -6,5 +6,6 @@ import { GetGotOptions } from './got.model';
  * 1. Error handler hook that prints helpful errors.
  * 2. Hooks that log start/end of request (optional, false by default).
  * 3. Reasonable defaults(tm), e.g non-infinite Timeout
+ * 4. Preserves error stack traces (!) (experimental!)
  */
 export declare function getGot(opt?: GetGotOptions): Got;

package/dist/got/getGot.js

@@ -10,11 +10,12 @@ const __1 = require("..");
  * 1. Error handler hook that prints helpful errors.
  * 2. Hooks that log start/end of request (optional, false by default).
  * 3. Reasonable defaults(tm), e.g non-infinite Timeout
+ * 4. Preserves error stack traces (!) (experimental!)
  */
 function getGot(opt = {}) {
     opt.logger || (opt.logger = console);
     if (opt.debug) {
-        opt.logStart = opt.logFinished = opt.logResponse = true;
+        opt.logStart = opt.logFinished = opt.logResponse = opt.logRequest = true;
     }
     return got_1.default.extend({
         // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
@@ -40,6 +41,18 @@ function getGot(opt = {}) {
         // Which definitely doesn't fit into default "RequestTimeout"
         timeout: 60000,
         ...opt,
+        handlers: [
+            (options, next) => {
+                options.context = {
+                    ...options.context,
+                    started: Date.now(),
+                    // This is to preserve original stack trace
+                    // https://github.com/sindresorhus/got/blob/main/documentation/async-stack-traces.md
+                    err: new Error('RequestError'),
+                };
+                return next(options);
+            },
+        ],
         hooks: {
             ...opt.hooks,
             beforeError: [
@@ -120,20 +133,26 @@ function gotErrorHook(opt = {}) {
         err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
             .filter(Boolean)
             .join('\n');
+        const stack = err.options.context?.err?.stack;
+        if (stack) {
+            err.stack += '\n    --' + stack.replace('Error: RequestError', '');
+        }
         return err;
     };
 }
 function gotBeforeRequestHook(opt) {
     return options => {
-        options.context = {
-            ...options.context,
-            started: Date.now(),
-        };
         if (opt.logStart) {
             const { retryCount } = options.context;
             const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
             opt.logger.log([' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '));
         }
+        if (opt.logRequest) {
+            const body = options.json || options.body;
+            if (body) {
+                opt.logger.log(body);
+            }
+        }
     };
 }
 // Here we log always, because it's similar to ErrorHook - we always log errors

package/dist/got/got.model.d.ts

@@ -16,6 +16,12 @@ export interface GetGotOptions extends Options {
      * @default false
      */
     logFinished?: boolean;
+    /**
+     * Log request object.
+     *
+     * @default false
+     */
+    logRequest?: boolean;
     /**
      * Log actual response object.
      *
@@ -48,5 +54,6 @@ export interface GotRequestContext extends AnyObject {
      * Millisecond-timestamp of when the request was started. To be able to count "time spent".
      */
     started: number;
+    err?: Error;
     retryCount?: number;
 }

package/dist/index.d.ts

@@ -65,7 +65,6 @@ import { AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped
 export * from './validation/joi/joi.shared.schemas';
 import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi.validation.error';
 import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIfInvalid, validate } from './validation/joi/joi.validation.util';
-import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
 import { runScript, RunScriptOptions } from './script';
-export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
-export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, RequestError, TimeoutError, _chunkBuffer, Ajv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, sanitizeHTML, runScript, };
+export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, };
+export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, RequestError, TimeoutError, _chunkBuffer, Ajv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, runScript, };

package/dist/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.runScript = exports.sanitizeHTML = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.RequestError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
+exports.runScript = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.RequestError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
 const tslib_1 = require("tslib");
 const ajv_1 = require("ajv");
 exports.Ajv = ajv_1.default;
@@ -96,7 +96,5 @@ Object.defineProperty(exports, "getValidationResult", { enumerable: true, get: f
 Object.defineProperty(exports, "isValid", { enumerable: true, get: function () { return joi_validation_util_1.isValid; } });
 Object.defineProperty(exports, "undefinedIfInvalid", { enumerable: true, get: function () { return joi_validation_util_1.undefinedIfInvalid; } });
 Object.defineProperty(exports, "validate", { enumerable: true, get: function () { return joi_validation_util_1.validate; } });
-const sanitize_util_1 = require("./validation/sanitize.util");
-Object.defineProperty(exports, "sanitizeHTML", { enumerable: true, get: function () { return sanitize_util_1.sanitizeHTML; } });
 const script_1 = require("./script");
 Object.defineProperty(exports, "runScript", { enumerable: true, get: function () { return script_1.runScript; } });

package/dist/validation/joi/joi.shared.schemas.d.ts

@@ -43,5 +43,5 @@ export declare const semVerSchema: import("./string.extensions").ExtendedStringS
 export declare const userAgentSchema: import("./string.extensions").ExtendedStringSchema;
 export declare const utcOffsetSchema: import("./number.extensions").ExtendedNumberSchema;
 export declare const ipAddressSchema: import("./string.extensions").ExtendedStringSchema;
-export declare const baseDBEntitySchema: ObjectSchemaTyped<Partial<SavedDBEntity>, Partial<SavedDBEntity>>;
-export declare const savedDBEntitySchema: ObjectSchemaTyped<SavedDBEntity, SavedDBEntity>;
+export declare const baseDBEntitySchema: ObjectSchemaTyped<Partial<SavedDBEntity<string>>, Partial<SavedDBEntity<string>>>;
+export declare const savedDBEntitySchema: ObjectSchemaTyped<SavedDBEntity<string>, SavedDBEntity<string>>;

package/dist/validation/joi/string.extensions.d.ts

@@ -3,18 +3,9 @@ import * as Joi from 'joi';
 import { AnySchemaTyped } from './joi.model';
 export interface ExtendedStringSchema extends StringSchema, AnySchemaTyped<string> {
     dateString(min?: string, max?: string): this;
-    stripHTML(opt?: JoiStripHTMLOptions): this;
 }
 export interface JoiDateStringOptions {
     min?: string;
     max?: string;
 }
-export interface JoiStripHTMLOptions {
-    /**
-     * 'Strict' would throw an error if it detects any HTML.
-     * Non-strict (default) does not error, but DOES convert the string to the string without HTML.
-     * Internally uses `sanitize-html` library, with allowedTags = [], and method = 'discard'.
-     */
-    strict?: boolean;
-}
 export declare function stringExtensions(joi: typeof Joi): Extension;

package/dist/validation/joi/string.extensions.js

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.stringExtensions = void 0;
 const time_lib_1 = require("@naturalcycles/time-lib");
-const sanitize = require("sanitize-html");
 function stringExtensions(joi) {
     return {
         type: 'string',
@@ -68,39 +67,6 @@ function stringExtensions(joi) {
                     return v; // validation passed
                 },
             },
-            stripHTML: {
-                method(opt) {
-                    return this.$_addRule({
-                        name: 'stripHTML',
-                        args: {
-                            strict: false,
-                            ...opt,
-                        },
-                    });
-                },
-                args: [
-                    {
-                        name: 'strict',
-                        ref: true,
-                        assert: v => typeof v === 'boolean',
-                        message: 'must be a boolean',
-                    },
-                ],
-                validate(v, helpers, args) {
-                    // console.log('!!! stripHTML', args, v)
-                    const r = sanitize(v, {
-                        allowedTags: [],
-                        // disallowedTagsMode: 'discard' // discard is default
-                        parser: {
-                            decodeEntities: false, // prevent decoding/changing of &<>"'
-                        },
-                    });
-                    if (args.strict && r !== v) {
-                        return helpers.error('string.stripHTML', args);
-                    }
-                    return r; // return converted value (or the same, if there was nothing to sanitize)
-                },
-            },
         },
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.62.1",
+  "version": "12.64.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",
@@ -17,7 +17,6 @@
     "@naturalcycles/js-lib": "^14.0.0",
     "@naturalcycles/time-lib": "^3.0.1",
     "@types/lru-cache": "^5.1.0",
-    "@types/sanitize-html": "^2.5.0",
     "@types/through2-concurrent": "^2.0.0",
     "ajv": "^8.6.2",
     "ajv-formats": "^2.1.0",
@@ -27,7 +26,7 @@
     "chalk": "^4.0.0",
     "cp-file": "^9.0.0",
     "debug": "^4.1.1",
-    "dotenv": "^10.0.0",
+    "dotenv": "^14.2.0",
     "execa": "^5.0.0",
     "fs-extra": "^10.0.0",
     "globby": "^11.0.0",
@@ -36,14 +35,13 @@
     "lru-cache": "^6.0.0",
     "move-file": "^2.0.0",
     "nanoid": "^3.0.0",
-    "sanitize-html": "^2.5.2",
     "through2-concurrent": "^2.0.0",
     "yargs": "^17.0.0"
   },
   "devDependencies": {
     "@naturalcycles/bench-lib": "^1.0.7",
     "@naturalcycles/dev-lib": "^12.0.0",
-    "@types/node": "^16.0.0",
+    "@types/node": "^17.0.0",
     "@types/yargs": "^16.0.0",
     "jest": "^27.0.1",
     "nock": "^13.0.2",

package/src/got/getGot.ts

@@ -16,12 +16,13 @@ import { GetGotOptions, GotRequestContext } from './got.model'
  * 1. Error handler hook that prints helpful errors.
  * 2. Hooks that log start/end of request (optional, false by default).
  * 3. Reasonable defaults(tm), e.g non-infinite Timeout
+ * 4. Preserves error stack traces (!) (experimental!)
  */
 export function getGot(opt: GetGotOptions = {}): Got {
   opt.logger ||= console
 
   if (opt.debug) {
-    opt.logStart = opt.logFinished = opt.logResponse = true
+    opt.logStart = opt.logFinished = opt.logResponse = opt.logRequest = true
   }
 
   return got.extend({
@@ -48,6 +49,19 @@ export function getGot(opt: GetGotOptions = {}): Got {
     // Which definitely doesn't fit into default "RequestTimeout"
     timeout: 60_000,
     ...opt,
+    handlers: [
+      (options, next) => {
+        options.context = {
+          ...options.context,
+          started: Date.now(),
+          // This is to preserve original stack trace
+          // https://github.com/sindresorhus/got/blob/main/documentation/async-stack-traces.md
+          err: new Error('RequestError'),
+        } as GotRequestContext
+
+        return next(options)
+      },
+    ],
     hooks: {
       ...opt.hooks,
       beforeError: [
@@ -135,17 +149,17 @@ function gotErrorHook(opt: GetGotOptions = {}): BeforeErrorHook {
       .filter(Boolean)
       .join('\n')
 
+    const stack = (err.options.context as GotRequestContext)?.err?.stack
+    if (stack) {
+      err.stack += '\n    --' + stack.replace('Error: RequestError', '')
+    }
+
     return err
   }
 }
 
 function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
   return options => {
-    options.context = {
-      ...options.context,
-      started: Date.now(),
-    } as GotRequestContext
-
     if (opt.logStart) {
       const { retryCount } = options.context as GotRequestContext
       const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
@@ -153,6 +167,14 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
         [' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '),
       )
     }
+
+    if (opt.logRequest) {
+      const body = options.json || options.body
+
+      if (body) {
+        opt.logger!.log(body)
+      }
+    }
   }
 }
 

package/src/got/got.model.ts

@@ -20,6 +20,13 @@ export interface GetGotOptions extends Options {
    */
   logFinished?: boolean
 
+  /**
+   * Log request object.
+   *
+   * @default false
+   */
+  logRequest?: boolean
+
   /**
    * Log actual response object.
    *
@@ -58,5 +65,7 @@ export interface GotRequestContext extends AnyObject {
    */
   started: number
 
+  err?: Error
+
   retryCount?: number
 }

package/src/index.ts

@@ -107,7 +107,6 @@ import {
   undefinedIfInvalid,
   validate,
 } from './validation/joi/joi.validation.util'
-import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util'
 import { runScript, RunScriptOptions } from './script'
 
 export type {
@@ -156,7 +155,6 @@ export type {
   AjvValidationOptions,
   AjvSchemaCfg,
   AjvValidationErrorData,
-  SanitizeHTMLOptions,
 }
 
 export {
@@ -194,6 +192,5 @@ export {
   AjvValidationError,
   readJsonSchemas,
   readAjvSchemas,
-  sanitizeHTML,
   runScript,
 }

package/src/validation/joi/string.extensions.ts

@@ -1,12 +1,10 @@
 import { dayjs } from '@naturalcycles/time-lib'
 import { Extension, StringSchema } from 'joi'
 import * as Joi from 'joi'
-import * as sanitize from 'sanitize-html'
 import { AnySchemaTyped } from './joi.model'
 
 export interface ExtendedStringSchema extends StringSchema, AnySchemaTyped<string> {
   dateString(min?: string, max?: string): this
-  stripHTML(opt?: JoiStripHTMLOptions): this
 }
 
 export interface JoiDateStringOptions {
@@ -14,15 +12,6 @@ export interface JoiDateStringOptions {
   max?: string
 }
 
-export interface JoiStripHTMLOptions {
-  /**
-   * 'Strict' would throw an error if it detects any HTML.
-   * Non-strict (default) does not error, but DOES convert the string to the string without HTML.
-   * Internally uses `sanitize-html` library, with allowedTags = [], and method = 'discard'.
-   */
-  strict?: boolean
-}
-
 export function stringExtensions(joi: typeof Joi): Extension {
   return {
     type: 'string',
@@ -90,42 +79,6 @@ export function stringExtensions(joi: typeof Joi): Extension {
           return v // validation passed
         },
       },
-      stripHTML: {
-        method(opt?: JoiStripHTMLOptions) {
-          return this.$_addRule({
-            name: 'stripHTML',
-            args: {
-              strict: false,
-              ...opt,
-            },
-          })
-        },
-        args: [
-          {
-            name: 'strict',
-            ref: true,
-            assert: v => typeof v === 'boolean',
-            message: 'must be a boolean',
-          },
-        ],
-        validate(v: string, helpers, args: JoiStripHTMLOptions) {
-          // console.log('!!! stripHTML', args, v)
-
-          const r = sanitize(v, {
-            allowedTags: [], // no html tags allowed at all
-            // disallowedTagsMode: 'discard' // discard is default
-            parser: {
-              decodeEntities: false, // prevent decoding/changing of &<>"'
-            },
-          })
-
-          if (args.strict && r !== v) {
-            return helpers.error('string.stripHTML', args)
-          }
-
-          return r // return converted value (or the same, if there was nothing to sanitize)
-        },
-      },
     },
   }
 }

package/dist/validation/sanitize.util.d.ts

@@ -1,8 +0,0 @@
-import * as sanitize from 'sanitize-html';
-export declare type SanitizeHTMLOptions = sanitize.IOptions;
-/**
- * Simply a wrapper around `sanitize-html` library.
- *
- * @experimental
- */
-export declare function sanitizeHTML(s: string, opt?: SanitizeHTMLOptions): string;

package/dist/validation/sanitize.util.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeHTML = void 0;
-const sanitize = require("sanitize-html");
-/**
- * Simply a wrapper around `sanitize-html` library.
- *
- * @experimental
- */
-function sanitizeHTML(s, opt) {
-    return sanitize(s, opt);
-}
-exports.sanitizeHTML = sanitizeHTML;

package/src/validation/sanitize.util.ts

@@ -1,12 +0,0 @@
-import * as sanitize from 'sanitize-html'
-
-export type SanitizeHTMLOptions = sanitize.IOptions
-
-/**
- * Simply a wrapper around `sanitize-html` library.
- *
- * @experimental
- */
-export function sanitizeHTML(s: string, opt?: SanitizeHTMLOptions): string {
-  return sanitize(s, opt)
-}