@naturalcycles/nodejs-lib 12.60.0 → 12.62.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/dist/got/getGot.js +98 -28
  2. package/dist/got/got.model.d.ts +6 -0
  3. package/dist/index.d.ts +19 -19
  4. package/dist/index.js +19 -39
  5. package/dist/stream/sizeStack.js +6 -6
  6. package/dist/util/zip.util.d.ts +9 -9
  7. package/dist/util/zip.util.js +21 -23
  8. package/package.json +4 -4
  9. package/src/got/getGot.ts +120 -31
  10. package/src/got/got.model.ts +8 -0
  11. package/src/index.ts +19 -38
  12. package/src/stream/sizeStack.ts +6 -6
  13. package/src/util/zip.util.ts +19 -21
package/dist/got/getGot.js CHANGED
@@ -4,7 +4,6 @@ exports.getGot = void 0;
4
4
  const js_lib_1 = require("@naturalcycles/js-lib");
5
5
  const got_1 = require("got");
6
6
  const __1 = require("..");
7
- const colors_1 = require("../colors");
8
7
  /**
9
8
  * Returns instance of Got with "reasonable defaults":
10
9
  *
@@ -14,12 +13,32 @@ const colors_1 = require("../colors");
14
13
  */
15
14
  function getGot(opt = {}) {
16
15
  opt.logger || (opt.logger = console);
16
+ if (opt.debug) {
17
+ opt.logStart = opt.logFinished = opt.logResponse = true;
18
+ }
17
19
  return got_1.default.extend({
18
20
  // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
19
21
  // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
20
22
  // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
21
23
  // before server times out with 503).
22
- timeout: 90000,
24
+ //
25
+ // UPD 2021-11-27
26
+ // There are 2 types/strategies for requests:
27
+ // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
28
+ // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
29
+ //
30
+ // Here it's hard to set the default timeout right for both use-cases.
31
+ // So, if it's important, you should override it according to your use-cases:
32
+ // - set it longer for Type 1 (e.g 120 seconds)
33
+ // - set it shorter for Type 2 (e.g 10/20 seconds)
34
+ // Please beware of default Retry strategy of Got:
35
+ // by default it will retry 2 times (after first try)
36
+ // First delay between tries will be ~1 second, then ~2 seconds
37
+ // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
38
+ // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
39
+ // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
40
+ // Which definitely doesn't fit into default "RequestTimeout"
41
+ timeout: 60000,
23
42
  ...opt,
24
43
  hooks: {
25
44
  ...opt.hooks,
@@ -33,6 +52,11 @@ function getGot(opt = {}) {
33
52
  // User hooks go AFTER
34
53
  ...(opt.hooks?.beforeRequest || []),
35
54
  ],
55
+ beforeRetry: [
56
+ gotBeforeRetryHook(opt),
57
+ // User hooks go AFTER
58
+ ...(opt.hooks?.beforeRetry || []),
59
+ ],
36
60
  afterResponse: [
37
61
  ...(opt.hooks?.afterResponse || []),
38
62
  // User hooks go BEFORE
@@ -69,20 +93,33 @@ exports.getGot = getGot;
69
93
  function gotErrorHook(opt = {}) {
70
94
  const { maxResponseLength = 10000 } = opt;
71
95
  return err => {
72
- if (err instanceof got_1.HTTPError) {
73
- const { statusCode } = err.response;
74
- const { method, url, prefixUrl } = err.options;
75
- const shortUrl = getShortUrl(opt, url, prefixUrl);
76
- // const { started } = context as GotRequestContext
77
- const body = (0, __1.inspectAny)(err.response.body, {
96
+ const statusCode = err.response?.statusCode || 0;
97
+ const { method, url, prefixUrl } = err.options;
98
+ const shortUrl = getShortUrl(opt, url, prefixUrl);
99
+ const { started, retryCount } = (err.request?.options.context || {});
100
+ const body = err.response?.body
101
+ ? (0, __1.inspectAny)(err.response.body, {
78
102
  maxLen: maxResponseLength,
79
103
  colors: false,
80
- });
81
- // timings are not part of err.message to allow automatic error grouping in Sentry
82
- err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
83
- .filter(Boolean)
84
- .join('\n');
85
- }
104
+ })
105
+ : err.message;
106
+ // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
107
+ opt.logger.log([
108
+ ' <<',
109
+ statusCode,
110
+ method,
111
+ shortUrl,
112
+ retryCount && `(retry ${retryCount})`,
113
+ 'error',
114
+ started && 'in ' + (0, js_lib_1._since)(started),
115
+ ]
116
+ .filter(Boolean)
117
+ .join(' '));
118
+ // timings are not part of err.message to allow automatic error grouping in Sentry
119
+ // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
120
+ err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
121
+ .filter(Boolean)
122
+ .join('\n');
86
123
  return err;
87
124
  };
88
125
  }
@@ -93,24 +130,64 @@ function gotBeforeRequestHook(opt) {
93
130
  started: Date.now(),
94
131
  };
95
132
  if (opt.logStart) {
133
+ const { retryCount } = options.context;
96
134
  const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
97
- opt.logger.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
135
+ opt.logger.log([' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '));
98
136
  }
99
137
  };
100
138
  }
139
+ // Here we log always, because it's similar to ErrorHook - we always log errors
140
+ // Because Retries are always result of some Error
141
+ function gotBeforeRetryHook(opt) {
142
+ const { maxResponseLength = 10000 } = opt;
143
+ return (options, err, retryCount) => {
144
+ // opt.logger!.log('beforeRetry', retryCount)
145
+ const statusCode = err?.response?.statusCode || 0;
146
+ const { method, url, prefixUrl } = options;
147
+ const shortUrl = getShortUrl(opt, url, prefixUrl);
148
+ const { started } = options.context;
149
+ Object.assign(options.context, { retryCount });
150
+ const body = err?.response?.body
151
+ ? (0, __1.inspectAny)(err.response.body, {
152
+ maxLen: maxResponseLength,
153
+ colors: false,
154
+ })
155
+ : err?.message;
156
+ // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
157
+ opt.logger.warn([
158
+ [
159
+ ' <<',
160
+ statusCode,
161
+ method,
162
+ shortUrl,
163
+ retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
164
+ 'error',
165
+ started && 'in ' + (0, js_lib_1._since)(started),
166
+ ]
167
+ .filter(Boolean)
168
+ .join(' '),
169
+ body,
170
+ ]
171
+ .filter(Boolean)
172
+ .join('\n'));
173
+ };
174
+ }
175
+ // AfterResponseHook is never called on Error
176
+ // So, coloredHttpCode(resp.statusCode) is probably useless
101
177
  function gotAfterResponseHook(opt = {}) {
102
178
  return resp => {
103
179
  const success = resp.statusCode >= 200 && resp.statusCode < 400;
104
180
  if (opt.logFinished) {
105
- const { started } = resp.request.options.context;
181
+ const { started, retryCount } = resp.request.options.context;
106
182
  const { url, prefixUrl, method } = resp.request.options;
107
183
  const shortUrl = getShortUrl(opt, url, prefixUrl);
108
184
  opt.logger.log([
109
- (0, colors_1.dimGrey)(' <<'),
110
- coloredHttpCode(resp.statusCode),
111
- (0, colors_1.dimGrey)(method),
112
- (0, colors_1.grey)(shortUrl),
113
- started && (0, colors_1.dimGrey)('in ' + (0, js_lib_1._since)(started)),
185
+ ' <<',
186
+ resp.statusCode,
187
+ method,
188
+ shortUrl,
189
+ retryCount && `(retry ${retryCount - 1})`,
190
+ started && 'in ' + (0, js_lib_1._since)(started),
114
191
  ]
115
192
  .filter(Boolean)
116
193
  .join(' '));
@@ -123,13 +200,6 @@ function gotAfterResponseHook(opt = {}) {
123
200
  return resp;
124
201
  };
125
202
  }
126
- function coloredHttpCode(statusCode) {
127
- if (statusCode < 400)
128
- return (0, colors_1.dimGrey)(statusCode); // default
129
- if (statusCode < 500)
130
- return (0, colors_1.yellow)(statusCode);
131
- return (0, colors_1.red)(statusCode);
132
- }
133
203
  function getShortUrl(opt, url, prefixUrl) {
134
204
  let shortUrl = url.toString();
135
205
  if (opt.logWithSearchParams === false) {
package/dist/got/got.model.d.ts CHANGED
@@ -1,6 +1,11 @@
1
1
  import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
2
2
  import type { Options } from 'got';
3
3
  export interface GetGotOptions extends Options {
4
+ /**
5
+ * Set to `true` to enable all possible debug logging.
6
+ * Not safe in prod (as it logs Responses), but great to use during development.
7
+ */
8
+ debug?: boolean;
4
9
  /**
5
10
  * @default false
6
11
  */
@@ -43,4 +48,5 @@ export interface GotRequestContext extends AnyObject {
43
48
  * Millisecond-timestamp of when the request was started. To be able to count "time spent".
44
49
  */
45
50
  started: number;
51
+ retryCount?: number;
46
52
  }
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import Ajv from 'ajv';
2
- import { HTTPError, TimeoutError } from 'got';
2
+ import { RequestError, TimeoutError } from 'got';
3
3
  import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got } from 'got';
4
4
  import { AnySchema, ValidationErrorItem } from 'joi';
5
5
  import { _chunkBuffer } from './buffer/buffer.util';
@@ -11,14 +11,14 @@ import { Debug, IDebug, IDebugger } from './log/debug';
11
11
  export * from './security/hash.util';
12
12
  export * from './security/id.util';
13
13
  export * from './security/secret.util';
14
- import { hasColors } from './colors/colors';
14
+ export * from './colors/colors';
15
15
  export * from './log/log.util';
16
16
  import { slackDefaultMessagePrefixHook, SlackService } from './slack/slack.service';
17
17
  import { SlackApiBody, SlackMessage, SlackMessagePrefixHook, SlackMessageProps, SlackServiceCfg } from './slack/slack.service.model';
18
18
  import { NDJsonStats } from './stream/ndjson/ndjson.model';
19
- import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead';
20
- import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite';
21
- import { ndjsonMap } from './stream/ndjson/ndjsonMap';
19
+ export * from './stream/ndjson/ndJsonFileRead';
20
+ export * from './stream/ndjson/ndJsonFileWrite';
21
+ export * from './stream/ndjson/ndjsonMap';
22
22
  import { ndjsonStreamForEach, NDJSONStreamForEachOptions } from './stream/ndjson/ndjsonStreamForEach';
23
23
  import { pipelineFromNDJsonFile, PipelineFromNDJsonFileOptions } from './stream/ndjson/pipelineFromNDJsonFile';
24
24
  import { pipelineToNDJsonFile, PipelineToNDJsonFileOptions } from './stream/ndjson/pipelineToNDJsonFile';
@@ -26,25 +26,25 @@ import { streamToNDJsonFile } from './stream/ndjson/streamToNDJsonFile';
26
26
  import { bufferReviver, transformJsonParse, TransformJsonParseOptions } from './stream/ndjson/transformJsonParse';
27
27
  import { transformToNDJson, TransformToNDJsonOptions } from './stream/ndjson/transformToNDJson';
28
28
  export * from './stream/pipeline/pipeline';
29
- import { readableCreate, readableFrom } from './stream/readable/readableCreate';
30
- import { readableForEach, readableForEachSync } from './stream/readable/readableForEach';
31
- import { readableFromArray } from './stream/readable/readableFromArray';
32
- import { readableMap } from './stream/readable/readableMap';
33
- import { readableMapToArray } from './stream/readable/readableMapToArray';
34
- import { readableToArray } from './stream/readable/readableToArray';
29
+ export * from './stream/readable/readableCreate';
30
+ export * from './stream/readable/readableForEach';
31
+ export * from './stream/readable/readableFromArray';
32
+ export * from './stream/readable/readableMap';
33
+ export * from './stream/readable/readableMapToArray';
34
+ export * from './stream/readable/readableToArray';
35
35
  import { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream/stream.model';
36
36
  export * from './stream/transform/transformBuffer';
37
37
  export * from './stream/transform/transformFilter';
38
38
  export * from './stream/transform/transformLimit';
39
39
  export * from './stream/transform/transformLogProgress';
40
40
  import { transformMap, TransformMapOptions } from './stream/transform/transformMap';
41
- import { transformMapSimple } from './stream/transform/transformMapSimple';
42
- import { transformNoOp } from './stream/transform/transformNoOp';
41
+ export * from './stream/transform/transformMapSimple';
42
+ export * from './stream/transform/transformNoOp';
43
43
  import { transformMapSync, TransformMapSyncOptions } from './stream/transform/transformMapSync';
44
- import { transformSplit } from './stream/transform/transformSplit';
45
- import { transformTap } from './stream/transform/transformTap';
46
- import { transformToArray } from './stream/transform/transformToArray';
47
- import { transformToString } from './stream/transform/transformToString';
44
+ export * from './stream/transform/transformSplit';
45
+ export * from './stream/transform/transformTap';
46
+ export * from './stream/transform/transformToArray';
47
+ export * from './stream/transform/transformToString';
48
48
  import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass';
49
49
  import { transformMultiThreaded, TransformMultiThreadedOptions } from './stream/transform/worker/transformMultiThreaded';
50
50
  import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model';
@@ -59,7 +59,7 @@ export * from './util/zip.util';
59
59
  import { readAjvSchemas, readJsonSchemas } from './validation/ajv/ajv.util';
60
60
  import { AjvSchema, AjvSchemaCfg, AjvValidationOptions } from './validation/ajv/ajvSchema';
61
61
  import { AjvValidationError, AjvValidationErrorData } from './validation/ajv/ajvValidationError';
62
- import { getAjv } from './validation/ajv/getAjv';
62
+ export * from './validation/ajv/getAjv';
63
63
  import { ExtendedJoi, Joi } from './validation/joi/joi.extensions';
64
64
  import { AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, SchemaTyped, StringSchemaTyped } from './validation/joi/joi.model';
65
65
  export * from './validation/joi/joi.shared.schemas';
@@ -68,4 +68,4 @@ import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIf
68
68
  import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
69
69
  import { runScript, RunScriptOptions } from './script';
70
70
  export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
71
- export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMapSimple, transformNoOp, transformSplit, transformToString, transformToArray, transformTap, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
71
+ export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, RequestError, TimeoutError, _chunkBuffer, Ajv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, sanitizeHTML, runScript, };
package/dist/index.js CHANGED
@@ -1,12 +1,11 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.transformNoOp = exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
4
- exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = void 0;
3
+ exports.runScript = exports.sanitizeHTML = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.RequestError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
5
4
  const tslib_1 = require("tslib");
6
5
  const ajv_1 = require("ajv");
7
6
  exports.Ajv = ajv_1.default;
8
7
  const got_1 = require("got");
9
- Object.defineProperty(exports, "HTTPError", { enumerable: true, get: function () { return got_1.HTTPError; } });
8
+ Object.defineProperty(exports, "RequestError", { enumerable: true, get: function () { return got_1.RequestError; } });
10
9
  Object.defineProperty(exports, "TimeoutError", { enumerable: true, get: function () { return got_1.TimeoutError; } });
11
10
  const buffer_util_1 = require("./buffer/buffer.util");
12
11
  Object.defineProperty(exports, "_chunkBuffer", { enumerable: true, get: function () { return buffer_util_1._chunkBuffer; } });
@@ -19,20 +18,16 @@ Object.defineProperty(exports, "Debug", { enumerable: true, get: function () { r
19
18
  (0, tslib_1.__exportStar)(require("./security/hash.util"), exports);
20
19
  (0, tslib_1.__exportStar)(require("./security/id.util"), exports);
21
20
  (0, tslib_1.__exportStar)(require("./security/secret.util"), exports);
22
- const colors_1 = require("./colors/colors");
23
- Object.defineProperty(exports, "hasColors", { enumerable: true, get: function () { return colors_1.hasColors; } });
21
+ (0, tslib_1.__exportStar)(require("./colors/colors"), exports);
24
22
  (0, tslib_1.__exportStar)(require("./log/log.util"), exports);
25
23
  const slack_service_1 = require("./slack/slack.service");
26
24
  Object.defineProperty(exports, "slackDefaultMessagePrefixHook", { enumerable: true, get: function () { return slack_service_1.slackDefaultMessagePrefixHook; } });
27
25
  Object.defineProperty(exports, "SlackService", { enumerable: true, get: function () { return slack_service_1.SlackService; } });
28
26
  const ndjson_model_1 = require("./stream/ndjson/ndjson.model");
29
27
  Object.defineProperty(exports, "NDJsonStats", { enumerable: true, get: function () { return ndjson_model_1.NDJsonStats; } });
30
- const ndJsonFileRead_1 = require("./stream/ndjson/ndJsonFileRead");
31
- Object.defineProperty(exports, "ndJsonFileRead", { enumerable: true, get: function () { return ndJsonFileRead_1.ndJsonFileRead; } });
32
- const ndJsonFileWrite_1 = require("./stream/ndjson/ndJsonFileWrite");
33
- Object.defineProperty(exports, "ndJsonFileWrite", { enumerable: true, get: function () { return ndJsonFileWrite_1.ndJsonFileWrite; } });
34
- const ndjsonMap_1 = require("./stream/ndjson/ndjsonMap");
35
- Object.defineProperty(exports, "ndjsonMap", { enumerable: true, get: function () { return ndjsonMap_1.ndjsonMap; } });
28
+ (0, tslib_1.__exportStar)(require("./stream/ndjson/ndJsonFileRead"), exports);
29
+ (0, tslib_1.__exportStar)(require("./stream/ndjson/ndJsonFileWrite"), exports);
30
+ (0, tslib_1.__exportStar)(require("./stream/ndjson/ndjsonMap"), exports);
36
31
  const ndjsonStreamForEach_1 = require("./stream/ndjson/ndjsonStreamForEach");
37
32
  Object.defineProperty(exports, "ndjsonStreamForEach", { enumerable: true, get: function () { return ndjsonStreamForEach_1.ndjsonStreamForEach; } });
38
33
  const pipelineFromNDJsonFile_1 = require("./stream/ndjson/pipelineFromNDJsonFile");
@@ -47,40 +42,26 @@ Object.defineProperty(exports, "transformJsonParse", { enumerable: true, get: fu
47
42
  const transformToNDJson_1 = require("./stream/ndjson/transformToNDJson");
48
43
  Object.defineProperty(exports, "transformToNDJson", { enumerable: true, get: function () { return transformToNDJson_1.transformToNDJson; } });
49
44
  (0, tslib_1.__exportStar)(require("./stream/pipeline/pipeline"), exports);
50
- const readableCreate_1 = require("./stream/readable/readableCreate");
51
- Object.defineProperty(exports, "readableCreate", { enumerable: true, get: function () { return readableCreate_1.readableCreate; } });
52
- Object.defineProperty(exports, "readableFrom", { enumerable: true, get: function () { return readableCreate_1.readableFrom; } });
53
- const readableForEach_1 = require("./stream/readable/readableForEach");
54
- Object.defineProperty(exports, "readableForEach", { enumerable: true, get: function () { return readableForEach_1.readableForEach; } });
55
- Object.defineProperty(exports, "readableForEachSync", { enumerable: true, get: function () { return readableForEach_1.readableForEachSync; } });
56
- const readableFromArray_1 = require("./stream/readable/readableFromArray");
57
- Object.defineProperty(exports, "readableFromArray", { enumerable: true, get: function () { return readableFromArray_1.readableFromArray; } });
58
- const readableMap_1 = require("./stream/readable/readableMap");
59
- Object.defineProperty(exports, "readableMap", { enumerable: true, get: function () { return readableMap_1.readableMap; } });
60
- const readableMapToArray_1 = require("./stream/readable/readableMapToArray");
61
- Object.defineProperty(exports, "readableMapToArray", { enumerable: true, get: function () { return readableMapToArray_1.readableMapToArray; } });
62
- const readableToArray_1 = require("./stream/readable/readableToArray");
63
- Object.defineProperty(exports, "readableToArray", { enumerable: true, get: function () { return readableToArray_1.readableToArray; } });
45
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableCreate"), exports);
46
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableForEach"), exports);
47
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableFromArray"), exports);
48
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableMap"), exports);
49
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableMapToArray"), exports);
50
+ (0, tslib_1.__exportStar)(require("./stream/readable/readableToArray"), exports);
64
51
  (0, tslib_1.__exportStar)(require("./stream/transform/transformBuffer"), exports);
65
52
  (0, tslib_1.__exportStar)(require("./stream/transform/transformFilter"), exports);
66
53
  (0, tslib_1.__exportStar)(require("./stream/transform/transformLimit"), exports);
67
54
  (0, tslib_1.__exportStar)(require("./stream/transform/transformLogProgress"), exports);
68
55
  const transformMap_1 = require("./stream/transform/transformMap");
69
56
  Object.defineProperty(exports, "transformMap", { enumerable: true, get: function () { return transformMap_1.transformMap; } });
70
- const transformMapSimple_1 = require("./stream/transform/transformMapSimple");
71
- Object.defineProperty(exports, "transformMapSimple", { enumerable: true, get: function () { return transformMapSimple_1.transformMapSimple; } });
72
- const transformNoOp_1 = require("./stream/transform/transformNoOp");
73
- Object.defineProperty(exports, "transformNoOp", { enumerable: true, get: function () { return transformNoOp_1.transformNoOp; } });
57
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformMapSimple"), exports);
58
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformNoOp"), exports);
74
59
  const transformMapSync_1 = require("./stream/transform/transformMapSync");
75
60
  Object.defineProperty(exports, "transformMapSync", { enumerable: true, get: function () { return transformMapSync_1.transformMapSync; } });
76
- const transformSplit_1 = require("./stream/transform/transformSplit");
77
- Object.defineProperty(exports, "transformSplit", { enumerable: true, get: function () { return transformSplit_1.transformSplit; } });
78
- const transformTap_1 = require("./stream/transform/transformTap");
79
- Object.defineProperty(exports, "transformTap", { enumerable: true, get: function () { return transformTap_1.transformTap; } });
80
- const transformToArray_1 = require("./stream/transform/transformToArray");
81
- Object.defineProperty(exports, "transformToArray", { enumerable: true, get: function () { return transformToArray_1.transformToArray; } });
82
- const transformToString_1 = require("./stream/transform/transformToString");
83
- Object.defineProperty(exports, "transformToString", { enumerable: true, get: function () { return transformToString_1.transformToString; } });
61
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformSplit"), exports);
62
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformTap"), exports);
63
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformToArray"), exports);
64
+ (0, tslib_1.__exportStar)(require("./stream/transform/transformToString"), exports);
84
65
  const baseWorkerClass_1 = require("./stream/transform/worker/baseWorkerClass");
85
66
  Object.defineProperty(exports, "BaseWorkerClass", { enumerable: true, get: function () { return baseWorkerClass_1.BaseWorkerClass; } });
86
67
  const transformMultiThreaded_1 = require("./stream/transform/worker/transformMultiThreaded");
@@ -103,8 +84,7 @@ const ajvSchema_1 = require("./validation/ajv/ajvSchema");
103
84
  Object.defineProperty(exports, "AjvSchema", { enumerable: true, get: function () { return ajvSchema_1.AjvSchema; } });
104
85
  const ajvValidationError_1 = require("./validation/ajv/ajvValidationError");
105
86
  Object.defineProperty(exports, "AjvValidationError", { enumerable: true, get: function () { return ajvValidationError_1.AjvValidationError; } });
106
- const getAjv_1 = require("./validation/ajv/getAjv");
107
- Object.defineProperty(exports, "getAjv", { enumerable: true, get: function () { return getAjv_1.getAjv; } });
87
+ (0, tslib_1.__exportStar)(require("./validation/ajv/getAjv"), exports);
108
88
  const joi_extensions_1 = require("./validation/joi/joi.extensions");
109
89
  Object.defineProperty(exports, "Joi", { enumerable: true, get: function () { return joi_extensions_1.Joi; } });
110
90
  (0, tslib_1.__exportStar)(require("./validation/joi/joi.shared.schemas"), exports);
package/dist/stream/sizeStack.js CHANGED
@@ -15,15 +15,15 @@ class SizeStack extends js_lib_1.NumberStack {
15
15
  return super.push(item);
16
16
  }
17
17
  getStats() {
18
- const pcs = this.percentiles([50, 90]);
18
+ // const pcs = this.percentiles([50, 90])
19
19
  return [
20
- this.name,
20
+ ' ' + this.name,
21
21
  'avg',
22
22
  (0, colors_1.yellow)((0, js_lib_1._hb)(this.avg())),
23
- 'p50',
24
- (0, colors_1.yellow)((0, js_lib_1._hb)(pcs[50])),
25
- 'p90',
26
- (0, colors_1.yellow)((0, js_lib_1._hb)(pcs[90])),
23
+ // 'p50',
24
+ // yellow(_hb(pcs[50])),
25
+ // 'p90',
26
+ // yellow(_hb(pcs[90])),
27
27
  'total',
28
28
  (0, colors_1.yellow)((0, js_lib_1._hb)(this.total)),
29
29
  ].join(' ');
package/dist/util/zip.util.d.ts CHANGED
@@ -1,26 +1,26 @@
1
1
  /// <reference types="node" />
2
2
  import { ZlibOptions } from 'zlib';
3
3
  /**
4
- * zipBuffer uses `deflate`.
4
+ * deflateBuffer uses `deflate`.
5
5
  * It's 9 bytes shorter than `gzip`.
6
6
  */
7
- export declare function zipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
7
+ export declare function deflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
8
+ export declare function inflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
9
+ /**
10
+ * deflateString uses `deflate`.
11
+ * It's 9 bytes shorter than `gzip`.
12
+ */
13
+ export declare function deflateString(s: string, options?: ZlibOptions): Promise<Buffer>;
14
+ export declare function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
8
15
  /**
9
16
  * gzipBuffer uses `gzip`
10
17
  * It's 9 bytes longer than `deflate`.
11
18
  */
12
19
  export declare function gzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
13
- export declare function unzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
14
20
  export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
15
- /**
16
- * zipString uses `deflate`.
17
- * It's 9 bytes shorter than `gzip`.
18
- */
19
- export declare function zipString(s: string, options?: ZlibOptions): Promise<Buffer>;
20
21
  /**
21
22
  * gzipString uses `gzip`.
22
23
  * It's 9 bytes longer than `deflate`.
23
24
  */
24
25
  export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer>;
25
- export declare function unzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
26
26
  export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
package/dist/util/zip.util.js CHANGED
@@ -1,21 +1,37 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = void 0;
3
+ exports.gunzipToString = exports.gzipString = exports.gunzipBuffer = exports.gzipBuffer = exports.inflateToString = exports.deflateString = exports.inflateBuffer = exports.deflateBuffer = void 0;
4
4
  const util_1 = require("util");
5
5
  const zlib = require("zlib");
6
6
  const deflate = (0, util_1.promisify)(zlib.deflate.bind(zlib));
7
7
  const inflate = (0, util_1.promisify)(zlib.inflate.bind(zlib));
8
8
  const gzip = (0, util_1.promisify)(zlib.gzip.bind(zlib));
9
9
  const gunzip = (0, util_1.promisify)(zlib.gunzip.bind(zlib));
10
- // string > zip
10
+ // string > compressed buffer
11
11
  /**
12
- * zipBuffer uses `deflate`.
12
+ * deflateBuffer uses `deflate`.
13
13
  * It's 9 bytes shorter than `gzip`.
14
14
  */
15
- async function zipBuffer(buf, options = {}) {
15
+ async function deflateBuffer(buf, options = {}) {
16
16
  return await deflate(buf, options);
17
17
  }
18
- exports.zipBuffer = zipBuffer;
18
+ exports.deflateBuffer = deflateBuffer;
19
+ async function inflateBuffer(buf, options = {}) {
20
+ return await inflate(buf, options);
21
+ }
22
+ exports.inflateBuffer = inflateBuffer;
23
+ /**
24
+ * deflateString uses `deflate`.
25
+ * It's 9 bytes shorter than `gzip`.
26
+ */
27
+ async function deflateString(s, options) {
28
+ return await deflateBuffer(Buffer.from(s), options);
29
+ }
30
+ exports.deflateString = deflateString;
31
+ async function inflateToString(buf, options) {
32
+ return (await inflateBuffer(buf, options)).toString();
33
+ }
34
+ exports.inflateToString = inflateToString;
19
35
  /**
20
36
  * gzipBuffer uses `gzip`
21
37
  * It's 9 bytes longer than `deflate`.
@@ -24,23 +40,10 @@ async function gzipBuffer(buf, options = {}) {
24
40
  return await gzip(buf, options);
25
41
  }
26
42
  exports.gzipBuffer = gzipBuffer;
27
- // zip > buffer
28
- async function unzipBuffer(buf, options = {}) {
29
- return await inflate(buf, options);
30
- }
31
- exports.unzipBuffer = unzipBuffer;
32
43
  async function gunzipBuffer(buf, options = {}) {
33
44
  return await gunzip(buf, options);
34
45
  }
35
46
  exports.gunzipBuffer = gunzipBuffer;
36
- /**
37
- * zipString uses `deflate`.
38
- * It's 9 bytes shorter than `gzip`.
39
- */
40
- async function zipString(s, options) {
41
- return await zipBuffer(Buffer.from(s), options);
42
- }
43
- exports.zipString = zipString;
44
47
  /**
45
48
  * gzipString uses `gzip`.
46
49
  * It's 9 bytes longer than `deflate`.
@@ -49,11 +52,6 @@ async function gzipString(s, options) {
49
52
  return await gzipBuffer(Buffer.from(s), options);
50
53
  }
51
54
  exports.gzipString = gzipString;
52
- // convenience
53
- async function unzipToString(buf, options) {
54
- return (await unzipBuffer(buf, options)).toString();
55
- }
56
- exports.unzipToString = unzipToString;
57
55
  async function gunzipToString(buf, options) {
58
56
  return (await gunzipBuffer(buf, options)).toString();
59
57
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@naturalcycles/nodejs-lib",
3
- "version": "12.60.0",
3
+ "version": "12.62.2",
4
4
  "scripts": {
5
5
  "prepare": "husky install",
6
6
  "docs-serve": "vuepress dev docs",
@@ -27,12 +27,12 @@
27
27
  "chalk": "^4.0.0",
28
28
  "cp-file": "^9.0.0",
29
29
  "debug": "^4.1.1",
30
- "dotenv": "^10.0.0",
30
+ "dotenv": "^11.0.0",
31
31
  "execa": "^5.0.0",
32
32
  "fs-extra": "^10.0.0",
33
33
  "globby": "^11.0.0",
34
34
  "got": "^11.0.1",
35
- "joi": "^17.2.1",
35
+ "joi": "17.4.2",
36
36
  "lru-cache": "^6.0.0",
37
37
  "move-file": "^2.0.0",
38
38
  "nanoid": "^3.0.0",
@@ -43,7 +43,7 @@
43
43
  "devDependencies": {
44
44
  "@naturalcycles/bench-lib": "^1.0.7",
45
45
  "@naturalcycles/dev-lib": "^12.0.0",
46
- "@types/node": "^16.0.0",
46
+ "@types/node": "^17.0.0",
47
47
  "@types/yargs": "^16.0.0",
48
48
  "jest": "^27.0.1",
49
49
  "nock": "^13.0.2",
package/src/got/getGot.ts CHANGED
@@ -1,8 +1,13 @@
1
1
  import { URL } from 'url'
2
2
  import { _since } from '@naturalcycles/js-lib'
3
- import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
3
+ import got, {
4
+ AfterResponseHook,
5
+ BeforeErrorHook,
6
+ BeforeRequestHook,
7
+ BeforeRetryHook,
8
+ Got,
9
+ } from 'got'
4
10
  import { inspectAny } from '..'
5
- import { dimGrey, grey, red, yellow } from '../colors'
6
11
  import { GetGotOptions, GotRequestContext } from './got.model'
7
12
 
8
13
  /**
@@ -15,12 +20,33 @@ import { GetGotOptions, GotRequestContext } from './got.model'
15
20
  export function getGot(opt: GetGotOptions = {}): Got {
16
21
  opt.logger ||= console
17
22
 
23
+ if (opt.debug) {
24
+ opt.logStart = opt.logFinished = opt.logResponse = true
25
+ }
26
+
18
27
  return got.extend({
19
28
  // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
20
29
  // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
21
30
  // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
22
31
  // before server times out with 503).
23
- timeout: 90_000,
32
+ //
33
+ // UPD 2021-11-27
34
+ // There are 2 types/strategies for requests:
35
+ // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
36
+ // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
37
+ //
38
+ // Here it's hard to set the default timeout right for both use-cases.
39
+ // So, if it's important, you should override it according to your use-cases:
40
+ // - set it longer for Type 1 (e.g 120 seconds)
41
+ // - set it shorter for Type 2 (e.g 10/20 seconds)
42
+ // Please beware of default Retry strategy of Got:
43
+ // by default it will retry 2 times (after first try)
44
+ // First delay between tries will be ~1 second, then ~2 seconds
45
+ // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
46
+ // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
47
+ // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
48
+ // Which definitely doesn't fit into default "RequestTimeout"
49
+ timeout: 60_000,
24
50
  ...opt,
25
51
  hooks: {
26
52
  ...opt.hooks,
@@ -34,6 +60,11 @@ export function getGot(opt: GetGotOptions = {}): Got {
34
60
  // User hooks go AFTER
35
61
  ...(opt.hooks?.beforeRequest || []),
36
62
  ],
63
+ beforeRetry: [
64
+ gotBeforeRetryHook(opt),
65
+ // User hooks go AFTER
66
+ ...(opt.hooks?.beforeRetry || []),
67
+ ],
37
68
  afterResponse: [
38
69
  ...(opt.hooks?.afterResponse || []),
39
70
  // User hooks go BEFORE
@@ -71,22 +102,38 @@ function gotErrorHook(opt: GetGotOptions = {}): BeforeErrorHook {
71
102
  const { maxResponseLength = 10_000 } = opt
72
103
 
73
104
  return err => {
74
- if (err instanceof HTTPError) {
75
- const { statusCode } = err.response
76
- const { method, url, prefixUrl } = err.options
77
- const shortUrl = getShortUrl(opt, url, prefixUrl)
78
- // const { started } = context as GotRequestContext
79
-
80
- const body = inspectAny(err.response.body, {
81
- maxLen: maxResponseLength,
82
- colors: false,
83
- })
84
-
85
- // timings are not part of err.message to allow automatic error grouping in Sentry
86
- err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
105
+ const statusCode = err.response?.statusCode || 0
106
+ const { method, url, prefixUrl } = err.options
107
+ const shortUrl = getShortUrl(opt, url, prefixUrl)
108
+ const { started, retryCount } = (err.request?.options.context || {}) as GotRequestContext
109
+
110
+ const body = err.response?.body
111
+ ? inspectAny(err.response.body, {
112
+ maxLen: maxResponseLength,
113
+ colors: false,
114
+ })
115
+ : err.message
116
+
117
+ // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
118
+ opt.logger!.log(
119
+ [
120
+ ' <<',
121
+ statusCode,
122
+ method,
123
+ shortUrl,
124
+ retryCount && `(retry ${retryCount})`,
125
+ 'error',
126
+ started && 'in ' + _since(started),
127
+ ]
87
128
  .filter(Boolean)
88
- .join('\n')
89
- }
129
+ .join(' '),
130
+ )
131
+
132
+ // timings are not part of err.message to allow automatic error grouping in Sentry
133
+ // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
134
+ err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
135
+ .filter(Boolean)
136
+ .join('\n')
90
137
 
91
138
  return err
92
139
  }
@@ -100,28 +147,76 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
100
147
  } as GotRequestContext
101
148
 
102
149
  if (opt.logStart) {
150
+ const { retryCount } = options.context as GotRequestContext
103
151
  const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
104
- opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
152
+ opt.logger!.log(
153
+ [' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '),
154
+ )
105
155
  }
106
156
  }
107
157
  }
108
158
 
159
+ // Here we log always, because it's similar to ErrorHook - we always log errors
160
+ // Because Retries are always result of some Error
161
+ function gotBeforeRetryHook(opt: GetGotOptions): BeforeRetryHook {
162
+ const { maxResponseLength = 10_000 } = opt
163
+
164
+ return (options, err, retryCount) => {
165
+ // opt.logger!.log('beforeRetry', retryCount)
166
+ const statusCode = err?.response?.statusCode || 0
167
+ const { method, url, prefixUrl } = options
168
+ const shortUrl = getShortUrl(opt, url, prefixUrl)
169
+ const { started } = options.context as GotRequestContext
170
+ Object.assign(options.context, { retryCount })
171
+
172
+ const body = err?.response?.body
173
+ ? inspectAny(err.response.body, {
174
+ maxLen: maxResponseLength,
175
+ colors: false,
176
+ })
177
+ : err?.message
178
+
179
+ // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
180
+ opt.logger!.warn(
181
+ [
182
+ [
183
+ ' <<',
184
+ statusCode,
185
+ method,
186
+ shortUrl,
187
+ retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
188
+ 'error',
189
+ started && 'in ' + _since(started),
190
+ ]
191
+ .filter(Boolean)
192
+ .join(' '),
193
+ body,
194
+ ]
195
+ .filter(Boolean)
196
+ .join('\n'),
197
+ )
198
+ }
199
+ }
200
+
201
+ // AfterResponseHook is never called on Error
202
+ // So, coloredHttpCode(resp.statusCode) is probably useless
109
203
  function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
110
204
  return resp => {
111
205
  const success = resp.statusCode >= 200 && resp.statusCode < 400
112
206
 
113
207
  if (opt.logFinished) {
114
- const { started } = resp.request.options.context as GotRequestContext
208
+ const { started, retryCount } = resp.request.options.context as GotRequestContext
115
209
  const { url, prefixUrl, method } = resp.request.options
116
210
  const shortUrl = getShortUrl(opt, url, prefixUrl)
117
211
 
118
212
  opt.logger!.log(
119
213
  [
120
- dimGrey(' <<'),
121
- coloredHttpCode(resp.statusCode),
122
- dimGrey(method),
123
- grey(shortUrl),
124
- started && dimGrey('in ' + _since(started)),
214
+ ' <<',
215
+ resp.statusCode,
216
+ method,
217
+ shortUrl,
218
+ retryCount && `(retry ${retryCount - 1})`,
219
+ started && 'in ' + _since(started),
125
220
  ]
126
221
  .filter(Boolean)
127
222
  .join(' '),
@@ -138,12 +233,6 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
138
233
  }
139
234
  }
140
235
 
141
- function coloredHttpCode(statusCode: number): string {
142
- if (statusCode < 400) return dimGrey(statusCode) // default
143
- if (statusCode < 500) return yellow(statusCode)
144
- return red(statusCode)
145
- }
146
-
147
236
  function getShortUrl(opt: GetGotOptions, url: URL, prefixUrl?: string): string {
148
237
  let shortUrl = url.toString()
149
238
 
package/src/got/got.model.ts CHANGED
@@ -2,6 +2,12 @@ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
2
2
  import type { Options } from 'got'
3
3
 
4
4
  export interface GetGotOptions extends Options {
5
+ /**
6
+ * Set to `true` to enable all possible debug logging.
7
+ * Not safe in prod (as it logs Responses), but great to use during development.
8
+ */
9
+ debug?: boolean
10
+
5
11
  /**
6
12
  * @default false
7
13
  */
@@ -51,4 +57,6 @@ export interface GotRequestContext extends AnyObject {
51
57
  * Millisecond-timestamp of when the request was started. To be able to count "time spent".
52
58
  */
53
59
  started: number
60
+
61
+ retryCount?: number
54
62
  }
package/src/index.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import Ajv from 'ajv'
2
- import { HTTPError, TimeoutError } from 'got'
2
+ import { RequestError, TimeoutError } from 'got'
3
3
  import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got } from 'got'
4
4
  import { AnySchema, ValidationErrorItem } from 'joi'
5
5
  import { _chunkBuffer } from './buffer/buffer.util'
@@ -11,7 +11,7 @@ import { Debug, IDebug, IDebugger } from './log/debug'
11
11
  export * from './security/hash.util'
12
12
  export * from './security/id.util'
13
13
  export * from './security/secret.util'
14
- import { hasColors } from './colors/colors'
14
+ export * from './colors/colors'
15
15
  export * from './log/log.util'
16
16
  import { slackDefaultMessagePrefixHook, SlackService } from './slack/slack.service'
17
17
  import {
@@ -22,9 +22,9 @@ import {
22
22
  SlackServiceCfg,
23
23
  } from './slack/slack.service.model'
24
24
  import { NDJsonStats } from './stream/ndjson/ndjson.model'
25
- import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead'
26
- import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite'
27
- import { ndjsonMap } from './stream/ndjson/ndjsonMap'
25
+ export * from './stream/ndjson/ndJsonFileRead'
26
+ export * from './stream/ndjson/ndJsonFileWrite'
27
+ export * from './stream/ndjson/ndjsonMap'
28
28
  import {
29
29
  ndjsonStreamForEach,
30
30
  NDJSONStreamForEachOptions,
@@ -45,12 +45,12 @@ import {
45
45
  } from './stream/ndjson/transformJsonParse'
46
46
  import { transformToNDJson, TransformToNDJsonOptions } from './stream/ndjson/transformToNDJson'
47
47
  export * from './stream/pipeline/pipeline'
48
- import { readableCreate, readableFrom } from './stream/readable/readableCreate'
49
- import { readableForEach, readableForEachSync } from './stream/readable/readableForEach'
50
- import { readableFromArray } from './stream/readable/readableFromArray'
51
- import { readableMap } from './stream/readable/readableMap'
52
- import { readableMapToArray } from './stream/readable/readableMapToArray'
53
- import { readableToArray } from './stream/readable/readableToArray'
48
+ export * from './stream/readable/readableCreate'
49
+ export * from './stream/readable/readableForEach'
50
+ export * from './stream/readable/readableFromArray'
51
+ export * from './stream/readable/readableMap'
52
+ export * from './stream/readable/readableMapToArray'
53
+ export * from './stream/readable/readableToArray'
54
54
  import {
55
55
  ReadableTyped,
56
56
  TransformOptions,
@@ -62,13 +62,13 @@ export * from './stream/transform/transformFilter'
62
62
  export * from './stream/transform/transformLimit'
63
63
  export * from './stream/transform/transformLogProgress'
64
64
  import { transformMap, TransformMapOptions } from './stream/transform/transformMap'
65
- import { transformMapSimple } from './stream/transform/transformMapSimple'
66
- import { transformNoOp } from './stream/transform/transformNoOp'
65
+ export * from './stream/transform/transformMapSimple'
66
+ export * from './stream/transform/transformNoOp'
67
67
  import { transformMapSync, TransformMapSyncOptions } from './stream/transform/transformMapSync'
68
- import { transformSplit } from './stream/transform/transformSplit'
69
- import { transformTap } from './stream/transform/transformTap'
70
- import { transformToArray } from './stream/transform/transformToArray'
71
- import { transformToString } from './stream/transform/transformToString'
68
+ export * from './stream/transform/transformSplit'
69
+ export * from './stream/transform/transformTap'
70
+ export * from './stream/transform/transformToArray'
71
+ export * from './stream/transform/transformToString'
72
72
  import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass'
73
73
  import {
74
74
  transformMultiThreaded,
@@ -86,7 +86,7 @@ export * from './util/zip.util'
86
86
  import { readAjvSchemas, readJsonSchemas } from './validation/ajv/ajv.util'
87
87
  import { AjvSchema, AjvSchemaCfg, AjvValidationOptions } from './validation/ajv/ajvSchema'
88
88
  import { AjvValidationError, AjvValidationErrorData } from './validation/ajv/ajvValidationError'
89
- import { getAjv } from './validation/ajv/getAjv'
89
+ export * from './validation/ajv/getAjv'
90
90
  import { ExtendedJoi, Joi } from './validation/joi/joi.extensions'
91
91
  import {
92
92
  AnySchemaTyped,
@@ -171,17 +171,6 @@ export {
171
171
  Debug,
172
172
  SlackService,
173
173
  slackDefaultMessagePrefixHook,
174
- readableCreate,
175
- readableFrom,
176
- readableFromArray,
177
- readableToArray,
178
- readableForEach,
179
- readableForEachSync,
180
- readableMap,
181
- readableMapToArray,
182
- ndjsonMap,
183
- ndJsonFileRead,
184
- ndJsonFileWrite,
185
174
  ndjsonStreamForEach,
186
175
  pipelineFromNDJsonFile,
187
176
  pipelineToNDJsonFile,
@@ -192,27 +181,19 @@ export {
192
181
  transformToNDJson,
193
182
  transformMap,
194
183
  transformMapSync,
195
- transformMapSimple,
196
- transformNoOp,
197
- transformSplit,
198
- transformToString,
199
- transformToArray,
200
- transformTap,
201
184
  transformMultiThreaded,
202
185
  BaseWorkerClass,
203
186
  tableDiff,
204
187
  inspectAny,
205
188
  inspectAnyStringifyFn,
206
- HTTPError,
189
+ RequestError,
207
190
  TimeoutError,
208
191
  _chunkBuffer,
209
192
  Ajv,
210
- getAjv,
211
193
  AjvSchema,
212
194
  AjvValidationError,
213
195
  readJsonSchemas,
214
196
  readAjvSchemas,
215
- hasColors,
216
197
  sanitizeHTML,
217
198
  runScript,
218
199
  }
package/src/stream/sizeStack.ts CHANGED
@@ -15,16 +15,16 @@ export class SizeStack extends NumberStack {
15
15
  }
16
16
 
17
17
  getStats(): string {
18
- const pcs = this.percentiles([50, 90])
18
+ // const pcs = this.percentiles([50, 90])
19
19
 
20
20
  return [
21
- this.name,
21
+ ' ' + this.name,
22
22
  'avg',
23
23
  yellow(_hb(this.avg())),
24
- 'p50',
25
- yellow(_hb(pcs[50])),
26
- 'p90',
27
- yellow(_hb(pcs[90])),
24
+ // 'p50',
25
+ // yellow(_hb(pcs[50])),
26
+ // 'p90',
27
+ // yellow(_hb(pcs[90])),
28
28
  'total',
29
29
  yellow(_hb(this.total)),
30
30
  ].join(' ')
package/src/util/zip.util.ts CHANGED
@@ -7,16 +7,32 @@ const inflate = promisify(zlib.inflate.bind(zlib))
7
7
  const gzip = promisify(zlib.gzip.bind(zlib))
8
8
  const gunzip = promisify(zlib.gunzip.bind(zlib))
9
9
 
10
- // string > zip
10
+ // string > compressed buffer
11
11
 
12
12
  /**
13
- * zipBuffer uses `deflate`.
13
+ * deflateBuffer uses `deflate`.
14
14
  * It's 9 bytes shorter than `gzip`.
15
15
  */
16
- export async function zipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
16
+ export async function deflateBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
17
17
  return await deflate(buf, options)
18
18
  }
19
19
 
20
+ export async function inflateBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
21
+ return await inflate(buf, options)
22
+ }
23
+
24
+ /**
25
+ * deflateString uses `deflate`.
26
+ * It's 9 bytes shorter than `gzip`.
27
+ */
28
+ export async function deflateString(s: string, options?: ZlibOptions): Promise<Buffer> {
29
+ return await deflateBuffer(Buffer.from(s), options)
30
+ }
31
+
32
+ export async function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
33
+ return (await inflateBuffer(buf, options)).toString()
34
+ }
35
+
20
36
  /**
21
37
  * gzipBuffer uses `gzip`
22
38
  * It's 9 bytes longer than `deflate`.
@@ -25,23 +41,10 @@ export async function gzipBuffer(buf: Buffer, options: ZlibOptions = {}): Promis
25
41
  return await gzip(buf, options)
26
42
  }
27
43
 
28
- // zip > buffer
29
- export async function unzipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
30
- return await inflate(buf, options)
31
- }
32
-
33
44
  export async function gunzipBuffer(buf: Buffer, options: ZlibOptions = {}): Promise<Buffer> {
34
45
  return await gunzip(buf, options)
35
46
  }
36
47
 
37
- /**
38
- * zipString uses `deflate`.
39
- * It's 9 bytes shorter than `gzip`.
40
- */
41
- export async function zipString(s: string, options?: ZlibOptions): Promise<Buffer> {
42
- return await zipBuffer(Buffer.from(s), options)
43
- }
44
-
45
48
  /**
46
49
  * gzipString uses `gzip`.
47
50
  * It's 9 bytes longer than `deflate`.
@@ -50,11 +53,6 @@ export async function gzipString(s: string, options?: ZlibOptions): Promise<Buff
50
53
  return await gzipBuffer(Buffer.from(s), options)
51
54
  }
52
55
 
53
- // convenience
54
- export async function unzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
55
- return (await unzipBuffer(buf, options)).toString()
56
- }
57
-
58
56
  export async function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string> {
59
57
  return (await gunzipBuffer(buf, options)).toString()
60
58
  }