@naturalcycles/nodejs-lib 15.123.0 → 15.124.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/infra/process.util.d.ts +2 -1
- package/dist/infra/process.util.js +4 -3
- package/dist/log/log.util.d.ts +7 -0
- package/dist/log/log.util.js +16 -2
- package/dist/script/runScript.d.ts +1 -1
- package/dist/script/runScript.js +6 -5
- package/dist/slack/slack.service.js +21 -10
- package/dist/stream/progressLogger.js +3 -2
- package/dist/stream/stream.model.d.ts +1 -1
- package/dist/stream/transform/transformMap.js +2 -2
- package/dist/stream/transform/transformMapSimple.js +3 -2
- package/dist/stream/transform/transformMapSync.js +4 -4
- package/dist/stream/transform/transformTap.js +5 -4
- package/dist/stream/transform/transformThrottle.js +2 -1
- package/package.json +2 -2
- package/src/infra/process.util.ts +6 -4
- package/src/log/log.util.ts +19 -2
- package/src/script/runScript.ts +7 -6
- package/src/slack/slack.service.ts +25 -14
- package/src/stream/progressLogger.ts +3 -2
- package/src/stream/stream.model.ts +1 -1
- package/src/stream/transform/transformMap.ts +2 -2
- package/src/stream/transform/transformMapSimple.ts +3 -2
- package/src/stream/transform/transformMapSync.ts +4 -4
- package/src/stream/transform/transformTap.ts +5 -4
- package/src/stream/transform/transformThrottle.ts +2 -1
|
@@ -3,15 +3,16 @@ export declare function memoryUsage(): {
|
|
|
3
3
|
heapTotal: number;
|
|
4
4
|
heapUsed: number;
|
|
5
5
|
external: number;
|
|
6
|
+
arrayBuffers: number;
|
|
6
7
|
};
|
|
7
8
|
export declare function memoryUsageFull(): {
|
|
8
9
|
rss: number;
|
|
9
10
|
heapTotal: number;
|
|
10
11
|
heapUsed: number;
|
|
11
12
|
external: number;
|
|
13
|
+
arrayBuffers: number;
|
|
12
14
|
totalMem: number;
|
|
13
15
|
freeMem: number;
|
|
14
|
-
usedMem: number;
|
|
15
16
|
};
|
|
16
17
|
declare class ProcessUtil {
|
|
17
18
|
private timer;
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
2
|
import { _mb } from '@naturalcycles/js-lib';
|
|
3
3
|
export function memoryUsage() {
|
|
4
|
-
const { rss, external, heapUsed, heapTotal } = process.memoryUsage();
|
|
4
|
+
const { rss, external, arrayBuffers, heapUsed, heapTotal } = process.memoryUsage();
|
|
5
5
|
return {
|
|
6
6
|
rss: _mb(rss),
|
|
7
7
|
heapTotal: _mb(heapTotal),
|
|
8
8
|
heapUsed: _mb(heapUsed),
|
|
9
9
|
external: _mb(external),
|
|
10
|
+
arrayBuffers: _mb(arrayBuffers),
|
|
10
11
|
};
|
|
11
12
|
}
|
|
12
13
|
export function memoryUsageFull() {
|
|
13
|
-
const { rss, external, heapUsed, heapTotal } = process.memoryUsage();
|
|
14
|
+
const { rss, external, arrayBuffers, heapUsed, heapTotal } = process.memoryUsage();
|
|
14
15
|
const totalMem = os.totalmem();
|
|
15
16
|
const freeMem = os.freemem();
|
|
16
17
|
return {
|
|
@@ -18,9 +19,9 @@ export function memoryUsageFull() {
|
|
|
18
19
|
heapTotal: _mb(heapTotal),
|
|
19
20
|
heapUsed: _mb(heapUsed),
|
|
20
21
|
external: _mb(external),
|
|
22
|
+
arrayBuffers: _mb(arrayBuffers),
|
|
21
23
|
totalMem: _mb(totalMem),
|
|
22
24
|
freeMem: _mb(freeMem),
|
|
23
|
-
usedMem: _mb(totalMem - freeMem),
|
|
24
25
|
};
|
|
25
26
|
}
|
|
26
27
|
class ProcessUtil {
|
package/dist/log/log.util.d.ts
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
+
import type { LogEntry } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { InspectAnyOptions } from '../string/inspect.js';
|
|
1
3
|
/**
|
|
2
4
|
* CommonLogger that logs to process.stdout directly (bypassing console.log)!
|
|
3
5
|
*/
|
|
4
6
|
export declare const stdoutLogger: any;
|
|
7
|
+
/**
|
|
8
|
+
* Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
|
|
9
|
+
* separated by a space.
|
|
10
|
+
*/
|
|
11
|
+
export declare function logEntryToString({ level: _level, msg, err, ...fields }: LogEntry, opt?: InspectAnyOptions): string;
|
package/dist/log/log.util.js
CHANGED
|
@@ -3,6 +3,20 @@ import { _inspect } from '../string/inspect.js';
|
|
|
3
3
|
/**
|
|
4
4
|
* CommonLogger that logs to process.stdout directly (bypassing console.log)!
|
|
5
5
|
*/
|
|
6
|
-
export const stdoutLogger = commonLoggerCreate(
|
|
7
|
-
process.stdout.write(
|
|
6
|
+
export const stdoutLogger = commonLoggerCreate(entry => {
|
|
7
|
+
process.stdout.write(logEntryToString(entry) + '\n');
|
|
8
8
|
});
|
|
9
|
+
/**
|
|
10
|
+
* Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
|
|
11
|
+
* separated by a space.
|
|
12
|
+
*/
|
|
13
|
+
export function logEntryToString({ level: _level, msg, err, ...fields }, opt = {}) {
|
|
14
|
+
const parts = [];
|
|
15
|
+
if (msg !== undefined)
|
|
16
|
+
parts.push(msg);
|
|
17
|
+
if (err !== undefined)
|
|
18
|
+
parts.push(_inspect(err, { includeErrorStack: true, ...opt }));
|
|
19
|
+
if (Object.keys(fields).length)
|
|
20
|
+
parts.push(_inspect(fields, opt));
|
|
21
|
+
return parts.join(' ');
|
|
22
|
+
}
|
package/dist/script/runScript.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
2
|
import { AsyncManager } from '@naturalcycles/js-lib';
|
|
3
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log';
|
|
3
4
|
import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js';
|
|
4
5
|
import { dimGrey } from '../colors/colors.js';
|
|
5
6
|
import { loadEnvFileIfExists } from '../node.util.js';
|
|
@@ -27,13 +28,13 @@ const { DEBUG_RUN_SCRIPT } = process.env;
|
|
|
27
28
|
export function runScript(fn, opt = {}) {
|
|
28
29
|
checkAndlogEnvironment();
|
|
29
30
|
setGlobalStringifyFunction(inspectStringifyFn);
|
|
30
|
-
const { logger =
|
|
31
|
+
const { logger = consoleLogger, noExit, registerUncaughtExceptionHandlers = true } = opt;
|
|
31
32
|
if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
|
|
32
33
|
process.on('uncaughtException', err => {
|
|
33
|
-
logger.error('runScript uncaughtException:', err);
|
|
34
|
+
logger.error({ msg: 'runScript uncaughtException:', err });
|
|
34
35
|
});
|
|
35
36
|
process.on('unhandledRejection', err => {
|
|
36
|
-
logger.error('runScript unhandledRejection:', err);
|
|
37
|
+
logger.error({ msg: 'runScript unhandledRejection:', err });
|
|
37
38
|
});
|
|
38
39
|
}
|
|
39
40
|
if (DEBUG_RUN_SCRIPT) {
|
|
@@ -54,7 +55,7 @@ export function runScript(fn, opt = {}) {
|
|
|
54
55
|
}
|
|
55
56
|
}
|
|
56
57
|
catch (err) {
|
|
57
|
-
logger.error('runScript error:', err);
|
|
58
|
+
logger.error({ msg: 'runScript error:', err });
|
|
58
59
|
process.exitCode = 1;
|
|
59
60
|
if (!noExit) {
|
|
60
61
|
setImmediate(() => process.exit(1));
|
|
@@ -68,7 +69,7 @@ export function runScript(fn, opt = {}) {
|
|
|
68
69
|
function checkAndlogEnvironment() {
|
|
69
70
|
const { platform, arch, versions: { node }, env: { CPU_LIMIT, NODE_OPTIONS, TZ }, } = process;
|
|
70
71
|
const cpuLimit = Number(CPU_LIMIT) || undefined;
|
|
71
|
-
const availableParallelism = os.availableParallelism
|
|
72
|
+
const availableParallelism = os.availableParallelism();
|
|
72
73
|
const cpus = os.cpus().length;
|
|
73
74
|
console.log(dimGrey(formatObject({
|
|
74
75
|
node: `${node} ${platform} ${arch}`,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
2
2
|
import { getFetcher } from '@naturalcycles/js-lib/http';
|
|
3
|
-
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
4
|
-
import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
|
|
3
|
+
import { commonLoggerCreate, consoleLogger, createCommonLoggerAtLevel, } from '@naturalcycles/js-lib/log';
|
|
4
|
+
import { _filterFalsyValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
|
|
5
5
|
import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js';
|
|
6
6
|
import { _inspect } from '../index.js';
|
|
7
7
|
const GAE = !!process.env['GAE_INSTANCE'];
|
|
@@ -31,7 +31,7 @@ export class SlackService {
|
|
|
31
31
|
constructor(cfg) {
|
|
32
32
|
this.cfg = {
|
|
33
33
|
messagePrefixHook: slackDefaultMessagePrefixHook,
|
|
34
|
-
logger:
|
|
34
|
+
logger: consoleLogger,
|
|
35
35
|
...cfg,
|
|
36
36
|
inspectOptions: {
|
|
37
37
|
...INSPECT_OPT,
|
|
@@ -60,7 +60,8 @@ export class SlackService {
|
|
|
60
60
|
if (ctx !== undefined) {
|
|
61
61
|
Object.assign(msg, { ctx });
|
|
62
62
|
}
|
|
63
|
-
|
|
63
|
+
const { items, kv, attachments, mentions } = msg;
|
|
64
|
+
this.cfg.logger.log(_filterFalsyValues({ msg: _inspect(items, inspectOptions), kv, attachments, mentions }));
|
|
64
65
|
if (!webhookUrl)
|
|
65
66
|
return;
|
|
66
67
|
// Transform msg.kv into msg.attachments
|
|
@@ -122,12 +123,22 @@ export class SlackService {
|
|
|
122
123
|
const q = new PQueue({
|
|
123
124
|
concurrency: 1,
|
|
124
125
|
});
|
|
125
|
-
|
|
126
|
-
debug:
|
|
127
|
-
log:
|
|
128
|
-
warn:
|
|
129
|
-
error:
|
|
130
|
-
}
|
|
126
|
+
const channelByLevel = {
|
|
127
|
+
debug: debugChannel || defaultChannel,
|
|
128
|
+
log: logChannel || defaultChannel,
|
|
129
|
+
warn: warnChannel || defaultChannel,
|
|
130
|
+
error: errorChannel || defaultChannel,
|
|
131
|
+
};
|
|
132
|
+
return createCommonLoggerAtLevel(commonLoggerCreate(({ level, msg, err, ...fields }) => {
|
|
133
|
+
const items = [];
|
|
134
|
+
if (msg !== undefined)
|
|
135
|
+
items.push(msg);
|
|
136
|
+
if (err !== undefined)
|
|
137
|
+
items.push(err);
|
|
138
|
+
if (Object.keys(fields).length)
|
|
139
|
+
items.push(fields);
|
|
140
|
+
void q.push(() => this.send({ items, channel: channelByLevel[level] }));
|
|
141
|
+
}), minLogLevel);
|
|
131
142
|
}
|
|
132
143
|
}
|
|
133
144
|
export function slackDefaultMessagePrefixHook(msg) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { inspect } from 'node:util';
|
|
2
2
|
import { _hc, _mb } from '@naturalcycles/js-lib';
|
|
3
3
|
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
|
|
4
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log';
|
|
4
5
|
import { SimpleMovingAverage } from '@naturalcycles/js-lib/math';
|
|
5
6
|
import { boldWhite, dimGrey, hasColors, white, yellow } from '../colors/colors.js';
|
|
6
7
|
const inspectOpt = {
|
|
@@ -17,7 +18,7 @@ export class ProgressLogger {
|
|
|
17
18
|
logEvery: 1000,
|
|
18
19
|
logSizesBuffer: 100_000,
|
|
19
20
|
chunkSize: 1,
|
|
20
|
-
logger:
|
|
21
|
+
logger: consoleLogger,
|
|
21
22
|
logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
|
|
22
23
|
...cfg,
|
|
23
24
|
};
|
|
@@ -100,7 +101,7 @@ export class ProgressLogger {
|
|
|
100
101
|
this.cfg.onProgressDone?.(o);
|
|
101
102
|
}
|
|
102
103
|
catch (err) {
|
|
103
|
-
logger.error(err);
|
|
104
|
+
logger.error({ err });
|
|
104
105
|
}
|
|
105
106
|
}
|
|
106
107
|
}
|
|
@@ -85,7 +85,7 @@ export function transformMap(mapper, opt = {}) {
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
catch (err) {
|
|
88
|
-
logger.error(err);
|
|
88
|
+
logger.error({ err });
|
|
89
89
|
errors++;
|
|
90
90
|
logErrorStats();
|
|
91
91
|
if (onError) {
|
|
@@ -167,7 +167,7 @@ export function transformMap(mapper, opt = {}) {
|
|
|
167
167
|
});
|
|
168
168
|
}
|
|
169
169
|
catch (err) {
|
|
170
|
-
logger.error(err);
|
|
170
|
+
logger.error({ err });
|
|
171
171
|
}
|
|
172
172
|
}
|
|
173
173
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
2
|
import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
|
|
3
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log';
|
|
3
4
|
/**
|
|
4
5
|
* Simplest version of `transformMap`.
|
|
5
6
|
* errorMode: IMMEDIATE
|
|
@@ -11,7 +12,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
|
|
|
11
12
|
*/
|
|
12
13
|
export function transformMapSimple(mapper, opt = {}) {
|
|
13
14
|
let index = -1;
|
|
14
|
-
const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger =
|
|
15
|
+
const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = consoleLogger, objectMode = true, highWaterMark = 1, } = opt;
|
|
15
16
|
return new Transform({
|
|
16
17
|
objectMode,
|
|
17
18
|
highWaterMark,
|
|
@@ -20,7 +21,7 @@ export function transformMapSimple(mapper, opt = {}) {
|
|
|
20
21
|
cb(null, mapper(chunk, ++index));
|
|
21
22
|
}
|
|
22
23
|
catch (err) {
|
|
23
|
-
logger.error(err);
|
|
24
|
+
logger.error({ err });
|
|
24
25
|
if (errorMode === ErrorMode.SUPPRESS) {
|
|
25
26
|
cb(); // suppress the error
|
|
26
27
|
}
|
|
@@ -48,7 +48,7 @@ export function transformMapSync(mapper, opt = {}) {
|
|
|
48
48
|
cb(); // done processing
|
|
49
49
|
}
|
|
50
50
|
catch (err) {
|
|
51
|
-
logger.error(err);
|
|
51
|
+
logger.error({ err });
|
|
52
52
|
errors++;
|
|
53
53
|
logErrorStats();
|
|
54
54
|
if (onError) {
|
|
@@ -70,7 +70,7 @@ export function transformMapSync(mapper, opt = {}) {
|
|
|
70
70
|
});
|
|
71
71
|
}
|
|
72
72
|
catch (err) {
|
|
73
|
-
logger.error(err);
|
|
73
|
+
logger.error({ err });
|
|
74
74
|
}
|
|
75
75
|
// Emit error immediately
|
|
76
76
|
return cb(err);
|
|
@@ -96,7 +96,7 @@ export function transformMapSync(mapper, opt = {}) {
|
|
|
96
96
|
});
|
|
97
97
|
}
|
|
98
98
|
catch (err) {
|
|
99
|
-
logger.error(err);
|
|
99
|
+
logger.error({ err });
|
|
100
100
|
}
|
|
101
101
|
// emit Aggregated error
|
|
102
102
|
cb(new AggregateError(collectedErrors, `transformMapSync resulted in ${collectedErrors.length} error(s)`));
|
|
@@ -114,7 +114,7 @@ export function transformMapSync(mapper, opt = {}) {
|
|
|
114
114
|
});
|
|
115
115
|
}
|
|
116
116
|
catch (err) {
|
|
117
|
-
logger.error(err);
|
|
117
|
+
logger.error({ err });
|
|
118
118
|
}
|
|
119
119
|
cb();
|
|
120
120
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log';
|
|
2
3
|
/**
|
|
3
4
|
* Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
|
|
4
5
|
* Item is passed through to the output.
|
|
@@ -6,7 +7,7 @@ import { Transform } from 'node:stream';
|
|
|
6
7
|
* Can also act as a counter, since `index` is passed to `fn`
|
|
7
8
|
*/
|
|
8
9
|
export function transformTap(fn, opt = {}) {
|
|
9
|
-
const { logger =
|
|
10
|
+
const { logger = consoleLogger, highWaterMark = 1 } = opt;
|
|
10
11
|
let index = -1;
|
|
11
12
|
return new Transform({
|
|
12
13
|
objectMode: true,
|
|
@@ -16,7 +17,7 @@ export function transformTap(fn, opt = {}) {
|
|
|
16
17
|
await fn(chunk, ++index);
|
|
17
18
|
}
|
|
18
19
|
catch (err) {
|
|
19
|
-
logger.error(err);
|
|
20
|
+
logger.error({ err });
|
|
20
21
|
// suppressed error
|
|
21
22
|
}
|
|
22
23
|
cb(null, chunk);
|
|
@@ -27,7 +28,7 @@ export function transformTap(fn, opt = {}) {
|
|
|
27
28
|
* Sync version of transformTap
|
|
28
29
|
*/
|
|
29
30
|
export function transformTapSync(fn, opt = {}) {
|
|
30
|
-
const { logger =
|
|
31
|
+
const { logger = consoleLogger, highWaterMark = 1 } = opt;
|
|
31
32
|
let index = -1;
|
|
32
33
|
return new Transform({
|
|
33
34
|
objectMode: true,
|
|
@@ -37,7 +38,7 @@ export function transformTapSync(fn, opt = {}) {
|
|
|
37
38
|
fn(chunk, ++index);
|
|
38
39
|
}
|
|
39
40
|
catch (err) {
|
|
40
|
-
logger.error(err);
|
|
41
|
+
logger.error({ err });
|
|
41
42
|
// suppressed error
|
|
42
43
|
}
|
|
43
44
|
cb(null, chunk);
|
|
@@ -34,7 +34,8 @@ export function transformThrottle(opt) {
|
|
|
34
34
|
if (!start) {
|
|
35
35
|
start = localTime.nowUnixMillis();
|
|
36
36
|
timeout = setTimeout(() => onInterval(), interval * 1000);
|
|
37
|
-
logger.log(
|
|
37
|
+
logger.log({
|
|
38
|
+
msg: `${localTime.now().toPretty()} transformThrottle started with`,
|
|
38
39
|
throughput,
|
|
39
40
|
interval,
|
|
40
41
|
rps: Math.round(throughput / interval),
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/nodejs-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "15.
|
|
4
|
+
"version": "15.124.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@naturalcycles/js-lib": "^15",
|
|
7
7
|
"@standard-schema/spec": "^1",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"yaml": "^2"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@naturalcycles/dev-lib": "
|
|
17
|
+
"@naturalcycles/dev-lib": "20.64.0",
|
|
18
18
|
"typescript": "^7"
|
|
19
19
|
},
|
|
20
20
|
"exports": {
|
|
@@ -6,13 +6,15 @@ export function memoryUsage(): {
|
|
|
6
6
|
heapTotal: number
|
|
7
7
|
heapUsed: number
|
|
8
8
|
external: number
|
|
9
|
+
arrayBuffers: number
|
|
9
10
|
} {
|
|
10
|
-
const { rss, external, heapUsed, heapTotal } = process.memoryUsage()
|
|
11
|
+
const { rss, external, arrayBuffers, heapUsed, heapTotal } = process.memoryUsage()
|
|
11
12
|
return {
|
|
12
13
|
rss: _mb(rss),
|
|
13
14
|
heapTotal: _mb(heapTotal),
|
|
14
15
|
heapUsed: _mb(heapUsed),
|
|
15
16
|
external: _mb(external),
|
|
17
|
+
arrayBuffers: _mb(arrayBuffers),
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
20
|
|
|
@@ -21,11 +23,11 @@ export function memoryUsageFull(): {
|
|
|
21
23
|
heapTotal: number
|
|
22
24
|
heapUsed: number
|
|
23
25
|
external: number
|
|
26
|
+
arrayBuffers: number
|
|
24
27
|
totalMem: number
|
|
25
28
|
freeMem: number
|
|
26
|
-
usedMem: number
|
|
27
29
|
} {
|
|
28
|
-
const { rss, external, heapUsed, heapTotal } = process.memoryUsage()
|
|
30
|
+
const { rss, external, arrayBuffers, heapUsed, heapTotal } = process.memoryUsage()
|
|
29
31
|
const totalMem = os.totalmem()
|
|
30
32
|
const freeMem = os.freemem()
|
|
31
33
|
return {
|
|
@@ -33,9 +35,9 @@ export function memoryUsageFull(): {
|
|
|
33
35
|
heapTotal: _mb(heapTotal),
|
|
34
36
|
heapUsed: _mb(heapUsed),
|
|
35
37
|
external: _mb(external),
|
|
38
|
+
arrayBuffers: _mb(arrayBuffers),
|
|
36
39
|
totalMem: _mb(totalMem),
|
|
37
40
|
freeMem: _mb(freeMem),
|
|
38
|
-
usedMem: _mb(totalMem - freeMem),
|
|
39
41
|
}
|
|
40
42
|
}
|
|
41
43
|
|
package/src/log/log.util.ts
CHANGED
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
import { commonLoggerCreate } from '@naturalcycles/js-lib/log'
|
|
2
|
+
import type { LogEntry } from '@naturalcycles/js-lib/log'
|
|
2
3
|
import { _inspect } from '../string/inspect.js'
|
|
4
|
+
import type { InspectAnyOptions } from '../string/inspect.js'
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* CommonLogger that logs to process.stdout directly (bypassing console.log)!
|
|
6
8
|
*/
|
|
7
|
-
export const stdoutLogger = commonLoggerCreate(
|
|
8
|
-
process.stdout.write(
|
|
9
|
+
export const stdoutLogger = commonLoggerCreate(entry => {
|
|
10
|
+
process.stdout.write(logEntryToString(entry) + '\n')
|
|
9
11
|
})
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Renders a LogEntry as one line: the msg, the inspected err and the inspected remaining fields,
|
|
15
|
+
* separated by a space.
|
|
16
|
+
*/
|
|
17
|
+
export function logEntryToString(
|
|
18
|
+
{ level: _level, msg, err, ...fields }: LogEntry,
|
|
19
|
+
opt: InspectAnyOptions = {},
|
|
20
|
+
): string {
|
|
21
|
+
const parts: string[] = []
|
|
22
|
+
if (msg !== undefined) parts.push(msg)
|
|
23
|
+
if (err !== undefined) parts.push(_inspect(err, { includeErrorStack: true, ...opt }))
|
|
24
|
+
if (Object.keys(fields).length) parts.push(_inspect(fields, opt))
|
|
25
|
+
return parts.join(' ')
|
|
26
|
+
}
|
package/src/script/runScript.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os from 'node:os'
|
|
2
2
|
import { AsyncManager } from '@naturalcycles/js-lib'
|
|
3
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log'
|
|
3
4
|
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
4
5
|
import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js'
|
|
5
6
|
import type { AnyObject } from '@naturalcycles/js-lib/types'
|
|
@@ -18,7 +19,7 @@ export interface RunScriptOptions {
|
|
|
18
19
|
noExit?: boolean
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
|
-
* Default to `
|
|
22
|
+
* Default to `consoleLogger`
|
|
22
23
|
*/
|
|
23
24
|
logger?: CommonLogger
|
|
24
25
|
|
|
@@ -53,14 +54,14 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
53
54
|
checkAndlogEnvironment()
|
|
54
55
|
setGlobalStringifyFunction(inspectStringifyFn)
|
|
55
56
|
|
|
56
|
-
const { logger =
|
|
57
|
+
const { logger = consoleLogger, noExit, registerUncaughtExceptionHandlers = true } = opt
|
|
57
58
|
|
|
58
59
|
if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
|
|
59
60
|
process.on('uncaughtException', err => {
|
|
60
|
-
logger.error('runScript uncaughtException:', err)
|
|
61
|
+
logger.error({ msg: 'runScript uncaughtException:', err })
|
|
61
62
|
})
|
|
62
63
|
process.on('unhandledRejection', err => {
|
|
63
|
-
logger.error('runScript unhandledRejection:', err)
|
|
64
|
+
logger.error({ msg: 'runScript unhandledRejection:', err })
|
|
64
65
|
})
|
|
65
66
|
}
|
|
66
67
|
|
|
@@ -85,7 +86,7 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
85
86
|
setImmediate(() => process.exit(0))
|
|
86
87
|
}
|
|
87
88
|
} catch (err) {
|
|
88
|
-
logger.error('runScript error:', err)
|
|
89
|
+
logger.error({ msg: 'runScript error:', err })
|
|
89
90
|
process.exitCode = 1
|
|
90
91
|
if (!noExit) {
|
|
91
92
|
setImmediate(() => process.exit(1))
|
|
@@ -105,7 +106,7 @@ function checkAndlogEnvironment(): void {
|
|
|
105
106
|
} = process
|
|
106
107
|
|
|
107
108
|
const cpuLimit = Number(CPU_LIMIT) || undefined
|
|
108
|
-
const availableParallelism = os.availableParallelism
|
|
109
|
+
const availableParallelism = os.availableParallelism()
|
|
109
110
|
const cpus = os.cpus().length
|
|
110
111
|
console.log(
|
|
111
112
|
dimGrey(
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
2
2
|
import { getFetcher } from '@naturalcycles/js-lib/http'
|
|
3
3
|
import type { Fetcher } from '@naturalcycles/js-lib/http'
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
commonLoggerCreate,
|
|
6
|
+
consoleLogger,
|
|
7
|
+
createCommonLoggerAtLevel,
|
|
8
|
+
} from '@naturalcycles/js-lib/log'
|
|
5
9
|
import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
|
|
6
|
-
import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
|
|
10
|
+
import { _filterFalsyValues, _omit } from '@naturalcycles/js-lib/object/object.util.js'
|
|
7
11
|
import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js'
|
|
8
12
|
import type { AnyObject } from '@naturalcycles/js-lib/types'
|
|
9
13
|
import { _inspect } from '../index.js'
|
|
@@ -45,7 +49,7 @@ export class SlackService<CTX = any> {
|
|
|
45
49
|
constructor(cfg: Partial<SlackServiceCfg<CTX>>) {
|
|
46
50
|
this.cfg = {
|
|
47
51
|
messagePrefixHook: slackDefaultMessagePrefixHook,
|
|
48
|
-
logger:
|
|
52
|
+
logger: consoleLogger,
|
|
49
53
|
...cfg,
|
|
50
54
|
inspectOptions: {
|
|
51
55
|
...INSPECT_OPT,
|
|
@@ -82,7 +86,10 @@ export class SlackService<CTX = any> {
|
|
|
82
86
|
Object.assign(msg, { ctx })
|
|
83
87
|
}
|
|
84
88
|
|
|
85
|
-
|
|
89
|
+
const { items, kv, attachments, mentions } = msg
|
|
90
|
+
this.cfg.logger.log(
|
|
91
|
+
_filterFalsyValues({ msg: _inspect(items, inspectOptions), kv, attachments, mentions }),
|
|
92
|
+
)
|
|
86
93
|
|
|
87
94
|
if (!webhookUrl) return
|
|
88
95
|
|
|
@@ -163,17 +170,21 @@ export class SlackService<CTX = any> {
|
|
|
163
170
|
concurrency: 1,
|
|
164
171
|
})
|
|
165
172
|
|
|
173
|
+
const channelByLevel: Record<CommonLogLevel, string> = {
|
|
174
|
+
debug: debugChannel || defaultChannel,
|
|
175
|
+
log: logChannel || defaultChannel,
|
|
176
|
+
warn: warnChannel || defaultChannel,
|
|
177
|
+
error: errorChannel || defaultChannel,
|
|
178
|
+
}
|
|
179
|
+
|
|
166
180
|
return createCommonLoggerAtLevel(
|
|
167
|
-
{
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
error: (...args) =>
|
|
175
|
-
q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
|
|
176
|
-
},
|
|
181
|
+
commonLoggerCreate(({ level, msg, err, ...fields }) => {
|
|
182
|
+
const items: unknown[] = []
|
|
183
|
+
if (msg !== undefined) items.push(msg)
|
|
184
|
+
if (err !== undefined) items.push(err)
|
|
185
|
+
if (Object.keys(fields).length) items.push(fields)
|
|
186
|
+
void q.push(() => this.send({ items, channel: channelByLevel[level] }))
|
|
187
|
+
}),
|
|
177
188
|
minLogLevel,
|
|
178
189
|
)
|
|
179
190
|
}
|
|
@@ -2,6 +2,7 @@ import type { InspectOptions } from 'node:util'
|
|
|
2
2
|
import { inspect } from 'node:util'
|
|
3
3
|
import { _hc, _mb } from '@naturalcycles/js-lib'
|
|
4
4
|
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
5
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log'
|
|
5
6
|
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
6
7
|
import { SimpleMovingAverage } from '@naturalcycles/js-lib/math'
|
|
7
8
|
import type { AnyObject, PositiveInteger, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
|
|
@@ -140,7 +141,7 @@ export class ProgressLogger<T> implements Disposable {
|
|
|
140
141
|
logEvery: 1000,
|
|
141
142
|
logSizesBuffer: 100_000,
|
|
142
143
|
chunkSize: 1,
|
|
143
|
-
logger:
|
|
144
|
+
logger: consoleLogger,
|
|
144
145
|
logProgress: cfg.logProgress !== false && cfg.logEvery !== 0,
|
|
145
146
|
...cfg,
|
|
146
147
|
}
|
|
@@ -258,7 +259,7 @@ export class ProgressLogger<T> implements Disposable {
|
|
|
258
259
|
try {
|
|
259
260
|
this.cfg.onProgressDone?.(o)
|
|
260
261
|
} catch (err) {
|
|
261
|
-
logger.error(err)
|
|
262
|
+
logger.error({ err })
|
|
262
263
|
}
|
|
263
264
|
}
|
|
264
265
|
}
|
|
@@ -186,7 +186,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
186
186
|
this.push(res)
|
|
187
187
|
}
|
|
188
188
|
} catch (err) {
|
|
189
|
-
logger.error(err)
|
|
189
|
+
logger.error({ err })
|
|
190
190
|
errors++
|
|
191
191
|
logErrorStats()
|
|
192
192
|
|
|
@@ -278,7 +278,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
278
278
|
started,
|
|
279
279
|
})
|
|
280
280
|
} catch (err) {
|
|
281
|
-
logger.error(err)
|
|
281
|
+
logger.error({ err })
|
|
282
282
|
}
|
|
283
283
|
}
|
|
284
284
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
2
|
import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
|
|
3
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log'
|
|
3
4
|
import type { IndexedMapper } from '@naturalcycles/js-lib/types'
|
|
4
5
|
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
5
6
|
|
|
@@ -28,7 +29,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
|
|
|
28
29
|
let index = -1
|
|
29
30
|
const {
|
|
30
31
|
errorMode = ErrorMode.THROW_IMMEDIATELY,
|
|
31
|
-
logger =
|
|
32
|
+
logger = consoleLogger,
|
|
32
33
|
objectMode = true,
|
|
33
34
|
highWaterMark = 1,
|
|
34
35
|
} = opt
|
|
@@ -40,7 +41,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
|
|
|
40
41
|
try {
|
|
41
42
|
cb(null, mapper(chunk, ++index))
|
|
42
43
|
} catch (err) {
|
|
43
|
-
logger.error(err)
|
|
44
|
+
logger.error({ err })
|
|
44
45
|
|
|
45
46
|
if (errorMode === ErrorMode.SUPPRESS) {
|
|
46
47
|
cb() // suppress the error
|
|
@@ -120,7 +120,7 @@ export function transformMapSync<IN = any, OUT = IN>(
|
|
|
120
120
|
|
|
121
121
|
cb() // done processing
|
|
122
122
|
} catch (err) {
|
|
123
|
-
logger.error(err)
|
|
123
|
+
logger.error({ err })
|
|
124
124
|
errors++
|
|
125
125
|
|
|
126
126
|
logErrorStats()
|
|
@@ -144,7 +144,7 @@ export function transformMapSync<IN = any, OUT = IN>(
|
|
|
144
144
|
started,
|
|
145
145
|
})
|
|
146
146
|
} catch (err) {
|
|
147
|
-
logger.error(err)
|
|
147
|
+
logger.error({ err })
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
// Emit error immediately
|
|
@@ -174,7 +174,7 @@ export function transformMapSync<IN = any, OUT = IN>(
|
|
|
174
174
|
started,
|
|
175
175
|
})
|
|
176
176
|
} catch (err) {
|
|
177
|
-
logger.error(err)
|
|
177
|
+
logger.error({ err })
|
|
178
178
|
}
|
|
179
179
|
|
|
180
180
|
// emit Aggregated error
|
|
@@ -197,7 +197,7 @@ export function transformMapSync<IN = any, OUT = IN>(
|
|
|
197
197
|
started,
|
|
198
198
|
})
|
|
199
199
|
} catch (err) {
|
|
200
|
-
logger.error(err)
|
|
200
|
+
logger.error({ err })
|
|
201
201
|
}
|
|
202
202
|
|
|
203
203
|
cb()
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
|
+
import { consoleLogger } from '@naturalcycles/js-lib/log'
|
|
2
3
|
import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
|
|
3
4
|
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
4
5
|
|
|
@@ -12,7 +13,7 @@ export function transformTap<IN>(
|
|
|
12
13
|
fn: AsyncIndexedMapper<IN, any>,
|
|
13
14
|
opt: TransformOptions = {},
|
|
14
15
|
): TransformTyped<IN, IN> {
|
|
15
|
-
const { logger =
|
|
16
|
+
const { logger = consoleLogger, highWaterMark = 1 } = opt
|
|
16
17
|
let index = -1
|
|
17
18
|
|
|
18
19
|
return new Transform({
|
|
@@ -22,7 +23,7 @@ export function transformTap<IN>(
|
|
|
22
23
|
try {
|
|
23
24
|
await fn(chunk, ++index)
|
|
24
25
|
} catch (err) {
|
|
25
|
-
logger.error(err)
|
|
26
|
+
logger.error({ err })
|
|
26
27
|
// suppressed error
|
|
27
28
|
}
|
|
28
29
|
|
|
@@ -38,7 +39,7 @@ export function transformTapSync<IN>(
|
|
|
38
39
|
fn: IndexedMapper<IN, any>,
|
|
39
40
|
opt: TransformOptions = {},
|
|
40
41
|
): TransformTyped<IN, IN> {
|
|
41
|
-
const { logger =
|
|
42
|
+
const { logger = consoleLogger, highWaterMark = 1 } = opt
|
|
42
43
|
let index = -1
|
|
43
44
|
|
|
44
45
|
return new Transform({
|
|
@@ -48,7 +49,7 @@ export function transformTapSync<IN>(
|
|
|
48
49
|
try {
|
|
49
50
|
fn(chunk, ++index)
|
|
50
51
|
} catch (err) {
|
|
51
|
-
logger.error(err)
|
|
52
|
+
logger.error({ err })
|
|
52
53
|
// suppressed error
|
|
53
54
|
}
|
|
54
55
|
|
|
@@ -56,7 +56,8 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
|
|
|
56
56
|
if (!start) {
|
|
57
57
|
start = localTime.nowUnixMillis()
|
|
58
58
|
timeout = setTimeout(() => onInterval(), interval * 1000)
|
|
59
|
-
logger.log(
|
|
59
|
+
logger.log({
|
|
60
|
+
msg: `${localTime.now().toPretty()} transformThrottle started with`,
|
|
60
61
|
throughput,
|
|
61
62
|
interval,
|
|
62
63
|
rps: Math.round(throughput / interval),
|