@naturalcycles/nodejs-lib 14.1.0 → 14.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/script/runScript.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import 'dotenv/config';
|
|
1
2
|
import os from 'node:os';
|
|
2
3
|
import { pDelay, setGlobalStringifyFunction } from '@naturalcycles/js-lib';
|
|
3
4
|
import { dimGrey } from '../colors/colors.js';
|
|
@@ -22,7 +23,7 @@ const { DEBUG_RUN_SCRIPT } = process.env;
|
|
|
22
23
|
* Set env DEBUG_RUN_SCRIPT for extra debugging.
|
|
23
24
|
*/
|
|
24
25
|
export function runScript(fn, opt = {}) {
|
|
25
|
-
|
|
26
|
+
checkAndlogEnvironment();
|
|
26
27
|
setGlobalStringifyFunction(inspectStringifyFn);
|
|
27
28
|
const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt;
|
|
28
29
|
if (registerUncaughtExceptionHandlers || DEBUG_RUN_SCRIPT) {
|
|
@@ -40,7 +41,6 @@ export function runScript(fn, opt = {}) {
|
|
|
40
41
|
// fake timeout, to ensure node.js process won't exit until runScript main promise is resolved
|
|
41
42
|
const timeout = setTimeout(() => { }, 10000000);
|
|
42
43
|
void (async () => {
|
|
43
|
-
await import('dotenv/config');
|
|
44
44
|
try {
|
|
45
45
|
await fn();
|
|
46
46
|
await pDelay(); // to ensure all async operations are completed
|
|
@@ -62,24 +62,40 @@ export function runScript(fn, opt = {}) {
|
|
|
62
62
|
}
|
|
63
63
|
})();
|
|
64
64
|
}
|
|
65
|
-
function
|
|
66
|
-
const { platform, arch, versions: { node }, env: { CPU_LIMIT, NODE_OPTIONS }, } = process;
|
|
65
|
+
function checkAndlogEnvironment() {
|
|
66
|
+
const { platform, arch, versions: { node }, env: { CPU_LIMIT, NODE_OPTIONS, TZ }, } = process;
|
|
67
67
|
const cpuLimit = Number(CPU_LIMIT) || undefined;
|
|
68
68
|
const availableParallelism = os.availableParallelism?.();
|
|
69
69
|
const cpus = os.cpus().length;
|
|
70
|
-
console.log(dimGrey(
|
|
70
|
+
console.log(dimGrey(formatObject({
|
|
71
71
|
node: `${node} ${platform} ${arch}`,
|
|
72
|
-
NODE_OPTIONS: NODE_OPTIONS || 'not defined',
|
|
73
72
|
cpus,
|
|
74
73
|
availableParallelism,
|
|
75
74
|
cpuLimit,
|
|
76
|
-
})
|
|
77
|
-
|
|
78
|
-
|
|
75
|
+
}) +
|
|
76
|
+
'\n' +
|
|
77
|
+
formatObject({
|
|
78
|
+
NODE_OPTIONS: NODE_OPTIONS || 'not defined',
|
|
79
|
+
TZ,
|
|
80
|
+
})));
|
|
79
81
|
if (!NODE_OPTIONS) {
|
|
80
82
|
console.warn(`NODE_OPTIONS env variable is not defined. You may run into out-of-memory issues when running memory-intensive scripts. It's recommended to set it to:\n--max-old-space-size=12000`);
|
|
81
83
|
}
|
|
82
84
|
else if (NODE_OPTIONS.includes('max_old')) {
|
|
83
85
|
console.warn(`It looks like you're using "max_old_space_size" syntax with underscores instead of dashes - it's WRONG and doesn't work in environment variables. Strongly advised to rename it to "max-old-space-size"`);
|
|
84
86
|
}
|
|
87
|
+
if (!TZ) {
|
|
88
|
+
console.error([
|
|
89
|
+
'!!! TZ environment variable is required to be set, but was not set.',
|
|
90
|
+
'The runScript will exit and not continue further because of that,',
|
|
91
|
+
'please ensure the TZ variable and try again.',
|
|
92
|
+
'If you are running locally, you can add TZ=UTC to the local .env file.',
|
|
93
|
+
].join('\n'));
|
|
94
|
+
process.exit(1);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function formatObject(obj) {
|
|
98
|
+
return Object.entries(obj)
|
|
99
|
+
.map(([k, v]) => `${k}: ${v}`)
|
|
100
|
+
.join(', ');
|
|
85
101
|
}
|
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
import { dirname } from 'node:path';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
1
|
import { Worker } from 'node:worker_threads';
|
|
4
2
|
import { _range, pDefer } from '@naturalcycles/js-lib';
|
|
5
3
|
import through2Concurrent from 'through2-concurrent';
|
|
6
|
-
const
|
|
7
|
-
const __dirname = dirname(__filename);
|
|
8
|
-
const workerProxyFilePath = `${__dirname}/workerClassProxy.js`;
|
|
4
|
+
const workerProxyFilePath = `${import.meta.dirname}/workerClassProxy.js`;
|
|
9
5
|
/**
|
|
10
6
|
* Spawns a pool of Workers (threads).
|
|
11
7
|
* Distributes (using round-robin, equally) all inputs over Workers.
|
package/package.json
CHANGED
package/src/script/runScript.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import 'dotenv/config'
|
|
1
2
|
import os from 'node:os'
|
|
2
|
-
import type { CommonLogger } from '@naturalcycles/js-lib'
|
|
3
|
+
import type { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
|
|
3
4
|
import { pDelay, setGlobalStringifyFunction } from '@naturalcycles/js-lib'
|
|
4
5
|
import { dimGrey } from '../colors/colors.js'
|
|
5
6
|
import { inspectStringifyFn } from '../string/inspect.js'
|
|
@@ -45,7 +46,7 @@ const { DEBUG_RUN_SCRIPT } = process.env
|
|
|
45
46
|
* Set env DEBUG_RUN_SCRIPT for extra debugging.
|
|
46
47
|
*/
|
|
47
48
|
export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {}): void {
|
|
48
|
-
|
|
49
|
+
checkAndlogEnvironment()
|
|
49
50
|
setGlobalStringifyFunction(inspectStringifyFn)
|
|
50
51
|
|
|
51
52
|
const { logger = console, noExit, registerUncaughtExceptionHandlers = true } = opt
|
|
@@ -68,8 +69,6 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
68
69
|
const timeout = setTimeout(() => {}, 10000000)
|
|
69
70
|
|
|
70
71
|
void (async () => {
|
|
71
|
-
await import('dotenv/config')
|
|
72
|
-
|
|
73
72
|
try {
|
|
74
73
|
await fn()
|
|
75
74
|
|
|
@@ -92,12 +91,12 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
|
|
|
92
91
|
})()
|
|
93
92
|
}
|
|
94
93
|
|
|
95
|
-
function
|
|
94
|
+
function checkAndlogEnvironment(): void {
|
|
96
95
|
const {
|
|
97
96
|
platform,
|
|
98
97
|
arch,
|
|
99
98
|
versions: { node },
|
|
100
|
-
env: { CPU_LIMIT, NODE_OPTIONS },
|
|
99
|
+
env: { CPU_LIMIT, NODE_OPTIONS, TZ },
|
|
101
100
|
} = process
|
|
102
101
|
|
|
103
102
|
const cpuLimit = Number(CPU_LIMIT) || undefined
|
|
@@ -105,15 +104,17 @@ function logEnvironment(): void {
|
|
|
105
104
|
const cpus = os.cpus().length
|
|
106
105
|
console.log(
|
|
107
106
|
dimGrey(
|
|
108
|
-
|
|
107
|
+
formatObject({
|
|
109
108
|
node: `${node} ${platform} ${arch}`,
|
|
110
|
-
NODE_OPTIONS: NODE_OPTIONS || 'not defined',
|
|
111
109
|
cpus,
|
|
112
110
|
availableParallelism,
|
|
113
111
|
cpuLimit,
|
|
114
|
-
})
|
|
115
|
-
|
|
116
|
-
|
|
112
|
+
}) +
|
|
113
|
+
'\n' +
|
|
114
|
+
formatObject({
|
|
115
|
+
NODE_OPTIONS: NODE_OPTIONS || 'not defined',
|
|
116
|
+
TZ,
|
|
117
|
+
}),
|
|
117
118
|
),
|
|
118
119
|
)
|
|
119
120
|
|
|
@@ -126,4 +127,22 @@ function logEnvironment(): void {
|
|
|
126
127
|
`It looks like you're using "max_old_space_size" syntax with underscores instead of dashes - it's WRONG and doesn't work in environment variables. Strongly advised to rename it to "max-old-space-size"`,
|
|
127
128
|
)
|
|
128
129
|
}
|
|
130
|
+
|
|
131
|
+
if (!TZ) {
|
|
132
|
+
console.error(
|
|
133
|
+
[
|
|
134
|
+
'!!! TZ environment variable is required to be set, but was not set.',
|
|
135
|
+
'The runScript will exit and not continue further because of that,',
|
|
136
|
+
'please ensure the TZ variable and try again.',
|
|
137
|
+
'If you are running locally, you can add TZ=UTC to the local .env file.',
|
|
138
|
+
].join('\n'),
|
|
139
|
+
)
|
|
140
|
+
process.exit(1)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function formatObject(obj: AnyObject): string {
|
|
145
|
+
return Object.entries(obj)
|
|
146
|
+
.map(([k, v]) => `${k}: ${v}`)
|
|
147
|
+
.join(', ')
|
|
129
148
|
}
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { dirname } from 'node:path'
|
|
2
|
-
import { fileURLToPath } from 'node:url'
|
|
3
1
|
import { Worker } from 'node:worker_threads'
|
|
4
2
|
import type { AnyObject, DeferredPromise } from '@naturalcycles/js-lib'
|
|
5
3
|
import { _range, pDefer } from '@naturalcycles/js-lib'
|
|
@@ -7,9 +5,6 @@ import through2Concurrent from 'through2-concurrent'
|
|
|
7
5
|
import type { TransformTyped } from '../../stream.model.js'
|
|
8
6
|
import type { WorkerInput, WorkerOutput } from './transformMultiThreaded.model.js'
|
|
9
7
|
|
|
10
|
-
const __filename = fileURLToPath(import.meta.url)
|
|
11
|
-
const __dirname = dirname(__filename)
|
|
12
|
-
|
|
13
8
|
export interface TransformMultiThreadedOptions {
|
|
14
9
|
/**
|
|
15
10
|
* Absolute path to a js file with worker code
|
|
@@ -37,7 +32,7 @@ export interface TransformMultiThreadedOptions {
|
|
|
37
32
|
workerData?: AnyObject
|
|
38
33
|
}
|
|
39
34
|
|
|
40
|
-
const workerProxyFilePath = `${
|
|
35
|
+
const workerProxyFilePath = `${import.meta.dirname}/workerClassProxy.js`
|
|
41
36
|
|
|
42
37
|
/**
|
|
43
38
|
* Spawns a pool of Workers (threads).
|