@naturalcycles/nodejs-lib 15.138.0 → 15.140.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exec2/exec2.js +5 -4
- package/dist/fs/kpy.js +3 -3
- package/dist/jwt/jwt.service.js +1 -0
- package/dist/security/secret.util.js +1 -3
- package/dist/stream/progressLogger.js +3 -3
- package/dist/stream/transform/transformMap.js +3 -3
- package/dist/stream/transform/transformMapSync.js +2 -1
- package/dist/stream/transform/transformSplit.js +2 -3
- package/dist/stream/transform/transformThrottle.js +1 -1
- package/dist/stream/transform/transformThrottleByRSS.js +2 -2
- package/dist/zip/zipInternal.js +1 -0
- package/dist/zip/zipReader.js +3 -6
- package/dist/zip/zipWriter.js +6 -8
- package/package.json +2 -2
- package/src/exec2/exec2.ts +5 -4
- package/src/fs/kpy.ts +3 -3
- package/src/jwt/jwt.service.ts +1 -0
- package/src/security/secret.util.ts +1 -3
- package/src/stream/progressLogger.ts +3 -3
- package/src/stream/transform/transformMap.ts +3 -3
- package/src/stream/transform/transformMapSync.ts +3 -2
- package/src/stream/transform/transformSplit.ts +2 -3
- package/src/stream/transform/transformThrottle.ts +1 -1
- package/src/stream/transform/transformThrottleByRSS.ts +2 -2
- package/src/stream/transform/worker/workerClassProxy.js +4 -0
- package/src/zip/zipInternal.ts +2 -0
- package/src/zip/zipReader.ts +6 -6
- package/src/zip/zipWriter.ts +9 -6
package/dist/exec2/exec2.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { execSync, spawn, spawnSync } from 'node:child_process';
|
|
2
|
+
import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
2
3
|
import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
|
|
3
4
|
import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
|
|
4
5
|
import { _substringAfterLast } from '@naturalcycles/js-lib/string/string.util.js';
|
|
@@ -53,7 +54,7 @@ class Exec2 {
|
|
|
53
54
|
const log = opt.log ?? true; // by default log should be true, as we are printing the output
|
|
54
55
|
// Copy, to not mutate the input options
|
|
55
56
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log };
|
|
56
|
-
const started =
|
|
57
|
+
const started = nowUnixMillis();
|
|
57
58
|
this.logStart(cmd, opt);
|
|
58
59
|
const r = spawnSync(cmd, opt.args, {
|
|
59
60
|
encoding: 'utf8',
|
|
@@ -98,7 +99,7 @@ class Exec2 {
|
|
|
98
99
|
logStart: opt.logStart ?? opt.log ?? false,
|
|
99
100
|
logFinish: opt.logFinish ?? opt.log ?? false,
|
|
100
101
|
};
|
|
101
|
-
const started =
|
|
102
|
+
const started = nowUnixMillis();
|
|
102
103
|
this.logStart(cmd, opt);
|
|
103
104
|
try {
|
|
104
105
|
const s = execSync(cmd, {
|
|
@@ -148,7 +149,7 @@ class Exec2 {
|
|
|
148
149
|
const log = opt.log ?? true; // by default log should be true, as we are printing the output
|
|
149
150
|
// Copy, to not mutate the input options
|
|
150
151
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log };
|
|
151
|
-
const started =
|
|
152
|
+
const started = nowUnixMillis();
|
|
152
153
|
this.logStart(cmd, opt);
|
|
153
154
|
await new Promise((resolve, reject) => {
|
|
154
155
|
const p = spawn(cmd, opt.args || [], {
|
|
@@ -203,7 +204,7 @@ class Exec2 {
|
|
|
203
204
|
const log = opt.log ?? printWhileRunning; // by default log should be true, as we are printing the output
|
|
204
205
|
// Copy, to not mutate the input options
|
|
205
206
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log };
|
|
206
|
-
const started =
|
|
207
|
+
const started = nowUnixMillis();
|
|
207
208
|
this.logStart(cmd, opt);
|
|
208
209
|
let stdout = '';
|
|
209
210
|
let stderr = '';
|
package/dist/fs/kpy.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import { _since,
|
|
2
|
+
import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime';
|
|
3
3
|
import { glob, globSync } from 'tinyglobby';
|
|
4
4
|
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js';
|
|
5
5
|
import { fs2 } from './fs2.js';
|
|
6
6
|
export async function kpy(opt) {
|
|
7
|
-
const started =
|
|
7
|
+
const started = nowUnixMillis();
|
|
8
8
|
opt = kpyPrepare(opt);
|
|
9
9
|
const filenames = await glob(opt.inputPatterns, {
|
|
10
10
|
cwd: opt.baseDir,
|
|
@@ -33,7 +33,7 @@ export async function kpy(opt) {
|
|
|
33
33
|
kpyLogResult(opt, filenames, started);
|
|
34
34
|
}
|
|
35
35
|
export function kpySync(opt) {
|
|
36
|
-
const started =
|
|
36
|
+
const started = nowUnixMillis();
|
|
37
37
|
opt = kpyPrepare(opt);
|
|
38
38
|
const filenames = globSync(opt.inputPatterns, {
|
|
39
39
|
cwd: opt.baseDir,
|
package/dist/jwt/jwt.service.js
CHANGED
|
@@ -78,6 +78,7 @@ export class JWTService {
|
|
|
78
78
|
const { payload } = await jwtVerify(token, key, {
|
|
79
79
|
algorithms: this.cfg.verifyAlgorithms || [this.cfg.algorithm],
|
|
80
80
|
...joseOpt,
|
|
81
|
+
// oxlint-disable-next-line no-restricted-globals -- jose expects a Date
|
|
81
82
|
currentDate: now === undefined ? undefined : new Date(now * 1000),
|
|
82
83
|
});
|
|
83
84
|
data = payload;
|
|
@@ -106,7 +106,5 @@ export function setSecretMap(map) {
|
|
|
106
106
|
.join(', ')}`);
|
|
107
107
|
}
|
|
108
108
|
function requireLoaded() {
|
|
109
|
-
|
|
110
|
-
throw new Error(`Secrets were not loaded! Call loadSecrets() before accessing secrets.`);
|
|
111
|
-
}
|
|
109
|
+
_assert(loaded, `Secrets were not loaded! Call loadSecrets() before accessing secrets.`);
|
|
112
110
|
}
|
|
@@ -35,8 +35,8 @@ export class ProgressLogger {
|
|
|
35
35
|
progress;
|
|
36
36
|
peakRSS;
|
|
37
37
|
start() {
|
|
38
|
-
this.started =
|
|
39
|
-
this.lastSecondStarted =
|
|
38
|
+
this.started = localTime.nowUnixMillis();
|
|
39
|
+
this.lastSecondStarted = localTime.nowUnixMillis();
|
|
40
40
|
this.sma = new SimpleMovingAverage(10);
|
|
41
41
|
this.processedLastSecond = 0;
|
|
42
42
|
this.progress = 0;
|
|
@@ -60,7 +60,7 @@ export class ProgressLogger {
|
|
|
60
60
|
return;
|
|
61
61
|
const { metric, extra, chunkSize, heapUsed: logHeapUsed, heapTotal: logHeapTotal, rss: logRss, peakRSS: logPeakRss, rssMinusHeap, external, arrayBuffers, logRPS, logger, } = this.cfg;
|
|
62
62
|
const mem = process.memoryUsage();
|
|
63
|
-
const now =
|
|
63
|
+
const now = localTime.nowUnixMillis();
|
|
64
64
|
const batchedProgress = this.progress * chunkSize;
|
|
65
65
|
const lastRPS = (this.processedLastSecond * chunkSize) / ((now - this.lastSecondStarted) / 1000) || 0;
|
|
66
66
|
const rpsTotal = Math.round(batchedProgress / ((now - this.started) / 1000)) || 0;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
2
|
import { _hc } from '@naturalcycles/js-lib';
|
|
3
|
-
import { _since } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime';
|
|
4
4
|
import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
|
|
5
5
|
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
6
6
|
import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
|
|
@@ -43,7 +43,7 @@ export function transformMap(mapper, opt = {}) {
|
|
|
43
43
|
async transform(chunk, _, cb) {
|
|
44
44
|
// Initialize start time on first item
|
|
45
45
|
if (started === 0) {
|
|
46
|
-
started =
|
|
46
|
+
started = nowUnixMillis();
|
|
47
47
|
lastWarmupCheck = started;
|
|
48
48
|
}
|
|
49
49
|
if (isSettled)
|
|
@@ -150,7 +150,7 @@ export function transformMap(mapper, opt = {}) {
|
|
|
150
150
|
},
|
|
151
151
|
});
|
|
152
152
|
function updateConcurrency() {
|
|
153
|
-
const now =
|
|
153
|
+
const now = nowUnixMillis();
|
|
154
154
|
if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS)
|
|
155
155
|
return;
|
|
156
156
|
lastWarmupCheck = now;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
|
+
import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
2
3
|
import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
|
|
3
4
|
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
4
5
|
import { END, SKIP } from '@naturalcycles/js-lib/types';
|
|
@@ -11,7 +12,7 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
|
|
|
11
12
|
export function transformMapSync(mapper, opt = {}) {
|
|
12
13
|
const { predicate, // defaults to "no predicate" (pass everything)
|
|
13
14
|
errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, signal, } = opt;
|
|
14
|
-
const started =
|
|
15
|
+
const started = nowUnixMillis();
|
|
15
16
|
let index = -1;
|
|
16
17
|
let countOut = 0;
|
|
17
18
|
let isSettled = false;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
2
3
|
// The code below is carefully adopted from: https://github.com/max-mapper/binary-split
|
|
3
4
|
// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
|
|
4
5
|
/**
|
|
@@ -22,9 +23,7 @@ export function transformSplitOnNewline() {
|
|
|
22
23
|
*/
|
|
23
24
|
export function transformSplit(separator = '\n') {
|
|
24
25
|
const matcher = Buffer.from(separator);
|
|
25
|
-
|
|
26
|
-
throw new Error('transformSplit: separator must not be empty');
|
|
27
|
-
}
|
|
26
|
+
_assert(matcher.length !== 0, 'transformSplit: separator must not be empty');
|
|
28
27
|
const matcherLength = matcher.length;
|
|
29
28
|
// indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
|
|
30
29
|
const needle = matcherLength === 1 ? matcher[0] : matcher;
|
|
@@ -48,7 +48,7 @@ export function transformThrottle(opt) {
|
|
|
48
48
|
if (++count >= throughput) {
|
|
49
49
|
// console.log('pausing now after', {item, count})
|
|
50
50
|
lock = pDefer();
|
|
51
|
-
logger.log(`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (
|
|
51
|
+
logger.log(`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (localTime.nowUnixMillis() - start))}`);
|
|
52
52
|
}
|
|
53
53
|
cb(null, item); // pass the item through
|
|
54
54
|
},
|
|
@@ -39,7 +39,7 @@ export function transformThrottleByRSS(opt) {
|
|
|
39
39
|
}
|
|
40
40
|
if (!disabled && lastRSS > maxRSSBytes && !lock) {
|
|
41
41
|
lock = pDefer();
|
|
42
|
-
pausedSince =
|
|
42
|
+
pausedSince = localTime.nowUnixMillis();
|
|
43
43
|
logger.log(`${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`);
|
|
44
44
|
pollTimer = setTimeout(() => pollRSS(), pollInterval);
|
|
45
45
|
}
|
|
@@ -67,7 +67,7 @@ export function transformThrottleByRSS(opt) {
|
|
|
67
67
|
lock.resolve();
|
|
68
68
|
lock = undefined;
|
|
69
69
|
}
|
|
70
|
-
else if (pollTimeout &&
|
|
70
|
+
else if (pollTimeout && localTime.nowUnixMillis() - pausedSince >= pollTimeout) {
|
|
71
71
|
clearTimeout(rssCheckTimer);
|
|
72
72
|
if (onPollTimeout === 'throw') {
|
|
73
73
|
lock.reject(new Error(`transformThrottleByRSS pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB`));
|
package/dist/zip/zipInternal.js
CHANGED
|
@@ -8,6 +8,7 @@ integer read/write helpers, DOS date/time conversion (both directions) and
|
|
|
8
8
|
entry-name normalization/validation.
|
|
9
9
|
|
|
10
10
|
*/
|
|
11
|
+
// oxlint-disable no-restricted-globals -- DOS date/time fields are local-time components, converted via the native Date
|
|
11
12
|
// oxlint-disable no-bitwise -- the ZIP format packs DOS date/time into bit fields
|
|
12
13
|
// Compression methods.
|
|
13
14
|
export const STORED = 0;
|
package/dist/zip/zipReader.js
CHANGED
|
@@ -25,6 +25,7 @@ import { Readable } from 'node:stream';
|
|
|
25
25
|
import { pipeline } from 'node:stream/promises';
|
|
26
26
|
import { promisify } from 'node:util';
|
|
27
27
|
import zlib from 'node:zlib';
|
|
28
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
28
29
|
import { assertSafeZipEntryName, CDFH_SIG, CDFH_SIZE, DEFLATE, dosDateTimeToUnix, EOCDR_SIG, EOCDR_SIZE, LOCAL_FILE_HEADER_SIG, MAX_COMMENT_SIZE, normalizeZipEntryName, readUInt64LE, STORED, ZIP64_EOCDL_SIG, ZIP64_EOCDL_SIZE, ZIP64_EOCDR_SIG, ZIP64_EOCDR_SIZE, } from './zipInternal.js';
|
|
29
30
|
// oxlint-disable no-bitwise -- parsing the binary ZIP format requires bitwise ops on bit flags and packed DOS date/time fields
|
|
30
31
|
const inflateRawAsync = promisify(zlib.inflateRaw.bind(zlib));
|
|
@@ -239,12 +240,8 @@ async function readEndOfCentralDirectory(source) {
|
|
|
239
240
|
if (locatorIndex >= 0 && buf.readUInt32LE(locatorIndex) === ZIP64_EOCDL_SIG) {
|
|
240
241
|
const zip64EocdrOffset = readUInt64LE(buf, locatorIndex + 8);
|
|
241
242
|
const zip64 = await source.read(zip64EocdrOffset, ZIP64_EOCDR_SIZE);
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
if (zip64.readUInt32LE(16) !== 0) {
|
|
246
|
-
throw new Error('multi-disk zip files are not supported');
|
|
247
|
-
}
|
|
243
|
+
_assert(zip64.readUInt32LE(0) === ZIP64_EOCDR_SIG, 'invalid zip64 end of central directory record signature');
|
|
244
|
+
_assert(zip64.readUInt32LE(16) === 0, 'multi-disk zip files are not supported');
|
|
248
245
|
entryCount = readUInt64LE(zip64, 32);
|
|
249
246
|
centralDirectoryOffset = readUInt64LE(zip64, 48);
|
|
250
247
|
centralDirectoryEnd = zip64EocdrOffset;
|
package/dist/zip/zipWriter.js
CHANGED
|
@@ -29,6 +29,8 @@ import { finished, pipeline } from 'node:stream/promises';
|
|
|
29
29
|
import { promisify } from 'node:util';
|
|
30
30
|
import zlib from 'node:zlib';
|
|
31
31
|
import { comparators } from '@naturalcycles/js-lib/array/sort.js';
|
|
32
|
+
import { nowUnix } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
33
|
+
import { _assert } from '@naturalcycles/js-lib/error/assert.js';
|
|
32
34
|
import { glob } from 'tinyglobby';
|
|
33
35
|
import { assertSafeZipEntryName, CDFH_SIG, CDFH_SIZE, DATA_DESCRIPTOR_SIG, DEFLATE, EOCDR_SIG, EOCDR_SIZE, LOCAL_FILE_HEADER_SIG, MAX_COMMENT_SIZE, normalizeZipEntryName, STORED, unixToDosDateTime, writeUInt64LE, ZIP64_EOCDL_SIG, ZIP64_EOCDL_SIZE, ZIP64_EOCDR_SIG, ZIP64_EOCDR_SIZE, } from './zipInternal.js';
|
|
34
36
|
// oxlint-disable no-bitwise, unicorn/prefer-math-trunc -- writing the binary ZIP format requires bitwise ops on bit flags, packed DOS date/time fields, and unsigned 32-bit attribute packing
|
|
@@ -187,9 +189,7 @@ export class ZipWriter {
|
|
|
187
189
|
}
|
|
188
190
|
// A comment containing this signature would confuse readers that scan
|
|
189
191
|
// backwards for the end-of-central-directory record.
|
|
190
|
-
|
|
191
|
-
throw new Error('archive comment must not contain the end-of-central-directory signature');
|
|
192
|
-
}
|
|
192
|
+
_assert(!comment.includes(EOCDR_SIG_BYTES), 'archive comment must not contain the end-of-central-directory signature');
|
|
193
193
|
this.finalized = true;
|
|
194
194
|
const centralDirectoryOffset = this.offset;
|
|
195
195
|
for (const entry of this.entries) {
|
|
@@ -215,8 +215,7 @@ export class ZipWriter {
|
|
|
215
215
|
assertWritable() {
|
|
216
216
|
if (this.streamError)
|
|
217
217
|
throw this.streamError;
|
|
218
|
-
|
|
219
|
-
throw new Error('ZipWriter has already been finalized');
|
|
218
|
+
_assert(!this.finalized, 'ZipWriter has already been finalized');
|
|
220
219
|
}
|
|
221
220
|
/** Build the in-memory representation of an entry from its name and options. */
|
|
222
221
|
createEntry(fileName, isDirectory, opt) {
|
|
@@ -224,7 +223,7 @@ export class ZipWriter {
|
|
|
224
223
|
if (nameBuf.length > MAX_COMMENT_SIZE) {
|
|
225
224
|
throw new Error(`zip entry name is too long: ${nameBuf.length} > ${MAX_COMMENT_SIZE} bytes`);
|
|
226
225
|
}
|
|
227
|
-
const mtime = opt.mtime ??
|
|
226
|
+
const mtime = opt.mtime ?? nowUnix();
|
|
228
227
|
const { date, time } = unixToDosDateTime(mtime);
|
|
229
228
|
const mode = opt.mode ?? (isDirectory ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE);
|
|
230
229
|
if ((mode & 0xffff) !== mode) {
|
|
@@ -508,8 +507,7 @@ function buildEndRecords(input) {
|
|
|
508
507
|
return Buffer.concat([zip64Eocdr, locator, eocdr]);
|
|
509
508
|
}
|
|
510
509
|
function normalizeAndValidateEntryName(fileName, isDirectory) {
|
|
511
|
-
|
|
512
|
-
throw new Error('zip entry name must not be empty');
|
|
510
|
+
_assert(fileName, 'zip entry name must not be empty');
|
|
513
511
|
let name = normalizeZipEntryName(fileName);
|
|
514
512
|
if (isDirectory) {
|
|
515
513
|
if (!name.endsWith('/'))
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/nodejs-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "15.
|
|
4
|
+
"version": "15.140.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@naturalcycles/js-lib": "^15",
|
|
7
7
|
"@standard-schema/spec": "^1",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"yaml": "^2"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@naturalcycles/dev-lib": "
|
|
17
|
+
"@naturalcycles/dev-lib": "20.71.0",
|
|
18
18
|
"typescript": "^7"
|
|
19
19
|
},
|
|
20
20
|
"exports": {
|
package/src/exec2/exec2.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { execSync, spawn, spawnSync } from 'node:child_process'
|
|
2
2
|
import type { StdioOptions } from 'node:child_process'
|
|
3
|
+
import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
3
4
|
import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
4
5
|
import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
|
|
5
6
|
import { _substringAfterLast } from '@naturalcycles/js-lib/string/string.util.js'
|
|
@@ -69,7 +70,7 @@ class Exec2 {
|
|
|
69
70
|
const log = opt.log ?? true // by default log should be true, as we are printing the output
|
|
70
71
|
// Copy, to not mutate the input options
|
|
71
72
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
|
|
72
|
-
const started =
|
|
73
|
+
const started = nowUnixMillis()
|
|
73
74
|
this.logStart(cmd, opt)
|
|
74
75
|
|
|
75
76
|
const r = spawnSync(cmd, opt.args, {
|
|
@@ -118,7 +119,7 @@ class Exec2 {
|
|
|
118
119
|
logStart: opt.logStart ?? opt.log ?? false,
|
|
119
120
|
logFinish: opt.logFinish ?? opt.log ?? false,
|
|
120
121
|
}
|
|
121
|
-
const started =
|
|
122
|
+
const started = nowUnixMillis()
|
|
122
123
|
this.logStart(cmd, opt)
|
|
123
124
|
|
|
124
125
|
try {
|
|
@@ -177,7 +178,7 @@ class Exec2 {
|
|
|
177
178
|
const log = opt.log ?? true // by default log should be true, as we are printing the output
|
|
178
179
|
// Copy, to not mutate the input options
|
|
179
180
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
|
|
180
|
-
const started =
|
|
181
|
+
const started = nowUnixMillis()
|
|
181
182
|
this.logStart(cmd, opt)
|
|
182
183
|
|
|
183
184
|
await new Promise<void>((resolve, reject) => {
|
|
@@ -245,7 +246,7 @@ class Exec2 {
|
|
|
245
246
|
const log = opt.log ?? printWhileRunning // by default log should be true, as we are printing the output
|
|
246
247
|
// Copy, to not mutate the input options
|
|
247
248
|
opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
|
|
248
|
-
const started =
|
|
249
|
+
const started = nowUnixMillis()
|
|
249
250
|
this.logStart(cmd, opt)
|
|
250
251
|
let stdout = ''
|
|
251
252
|
let stderr = ''
|
package/src/fs/kpy.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from 'node:path'
|
|
2
|
-
import { _since,
|
|
2
|
+
import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime'
|
|
3
3
|
import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
|
|
4
4
|
import { glob, globSync } from 'tinyglobby'
|
|
5
5
|
import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js'
|
|
@@ -45,7 +45,7 @@ export interface KpyOptions {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
export async function kpy(opt: KpyOptions): Promise<void> {
|
|
48
|
-
const started =
|
|
48
|
+
const started = nowUnixMillis()
|
|
49
49
|
|
|
50
50
|
opt = kpyPrepare(opt)
|
|
51
51
|
|
|
@@ -84,7 +84,7 @@ export async function kpy(opt: KpyOptions): Promise<void> {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
export function kpySync(opt: KpyOptions): void {
|
|
87
|
-
const started =
|
|
87
|
+
const started = nowUnixMillis()
|
|
88
88
|
|
|
89
89
|
opt = kpyPrepare(opt)
|
|
90
90
|
|
package/src/jwt/jwt.service.ts
CHANGED
|
@@ -263,6 +263,7 @@ export class JWTService<T extends AnyObject = AnyObject> {
|
|
|
263
263
|
const { payload } = await jwtVerify(token, key, {
|
|
264
264
|
algorithms: this.cfg.verifyAlgorithms || [this.cfg.algorithm],
|
|
265
265
|
...joseOpt,
|
|
266
|
+
// oxlint-disable-next-line no-restricted-globals -- jose expects a Date
|
|
266
267
|
currentDate: now === undefined ? undefined : new Date(now * 1000),
|
|
267
268
|
})
|
|
268
269
|
data = payload as TT
|
|
@@ -151,7 +151,5 @@ export function setSecretMap(map: StringMap): void {
|
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
function requireLoaded(): void {
|
|
154
|
-
|
|
155
|
-
throw new Error(`Secrets were not loaded! Call loadSecrets() before accessing secrets.`)
|
|
156
|
-
}
|
|
154
|
+
_assert(loaded, `Secrets were not loaded! Call loadSecrets() before accessing secrets.`)
|
|
157
155
|
}
|
|
@@ -168,8 +168,8 @@ export class ProgressLogger<T> implements Disposable {
|
|
|
168
168
|
private peakRSS!: number
|
|
169
169
|
|
|
170
170
|
private start(): void {
|
|
171
|
-
this.started =
|
|
172
|
-
this.lastSecondStarted =
|
|
171
|
+
this.started = localTime.nowUnixMillis()
|
|
172
|
+
this.lastSecondStarted = localTime.nowUnixMillis()
|
|
173
173
|
this.sma = new SimpleMovingAverage(10)
|
|
174
174
|
this.processedLastSecond = 0
|
|
175
175
|
this.progress = 0
|
|
@@ -213,7 +213,7 @@ export class ProgressLogger<T> implements Disposable {
|
|
|
213
213
|
|
|
214
214
|
const mem = process.memoryUsage()
|
|
215
215
|
|
|
216
|
-
const now =
|
|
216
|
+
const now = localTime.nowUnixMillis()
|
|
217
217
|
const batchedProgress = this.progress * chunkSize
|
|
218
218
|
const lastRPS =
|
|
219
219
|
(this.processedLastSecond * chunkSize) / ((now - this.lastSecondStarted) / 1000) || 0
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
2
|
import { _hc } from '@naturalcycles/js-lib'
|
|
3
3
|
import type { AbortableSignal } from '@naturalcycles/js-lib'
|
|
4
|
-
import { _since } from '@naturalcycles/js-lib/datetime'
|
|
4
|
+
import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime'
|
|
5
5
|
import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
|
|
6
6
|
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
7
7
|
import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
|
|
@@ -141,7 +141,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
141
141
|
async transform(this: Transform, chunk: IN, _, cb) {
|
|
142
142
|
// Initialize start time on first item
|
|
143
143
|
if (started === 0) {
|
|
144
|
-
started =
|
|
144
|
+
started = nowUnixMillis()
|
|
145
145
|
lastWarmupCheck = started
|
|
146
146
|
}
|
|
147
147
|
|
|
@@ -262,7 +262,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
262
262
|
})
|
|
263
263
|
|
|
264
264
|
function updateConcurrency(): void {
|
|
265
|
-
const now =
|
|
265
|
+
const now = nowUnixMillis()
|
|
266
266
|
if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS) return
|
|
267
267
|
lastWarmupCheck = now
|
|
268
268
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
2
|
import type { AbortableSignal } from '@naturalcycles/js-lib'
|
|
3
|
+
import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
3
4
|
import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
|
|
4
5
|
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
5
|
-
import type { IndexedMapper, Predicate
|
|
6
|
+
import type { IndexedMapper, Predicate } from '@naturalcycles/js-lib/types'
|
|
6
7
|
import { END, SKIP } from '@naturalcycles/js-lib/types'
|
|
7
8
|
import { yellow } from '../../colors/colors.js'
|
|
8
9
|
import type { TransformOptions, TransformTyped } from '../stream.model.js'
|
|
@@ -78,7 +79,7 @@ export function transformMapSync<IN = any, OUT = IN>(
|
|
|
78
79
|
signal,
|
|
79
80
|
} = opt
|
|
80
81
|
|
|
81
|
-
const started =
|
|
82
|
+
const started = nowUnixMillis()
|
|
82
83
|
let index = -1
|
|
83
84
|
let countOut = 0
|
|
84
85
|
let isSettled = false
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
2
3
|
import type { TransformTyped } from '../stream.model.js'
|
|
3
4
|
|
|
4
5
|
// The code below is carefully adopted from: https://github.com/max-mapper/binary-split
|
|
@@ -26,9 +27,7 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
|
|
|
26
27
|
*/
|
|
27
28
|
export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer> {
|
|
28
29
|
const matcher = Buffer.from(separator)
|
|
29
|
-
|
|
30
|
-
throw new Error('transformSplit: separator must not be empty')
|
|
31
|
-
}
|
|
30
|
+
_assert(matcher.length !== 0, 'transformSplit: separator must not be empty')
|
|
32
31
|
const matcherLength = matcher.length
|
|
33
32
|
// indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
|
|
34
33
|
const needle = matcherLength === 1 ? matcher[0]! : matcher
|
|
@@ -73,7 +73,7 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
|
|
|
73
73
|
// console.log('pausing now after', {item, count})
|
|
74
74
|
lock = pDefer()
|
|
75
75
|
logger.log(
|
|
76
|
-
`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (
|
|
76
|
+
`${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (localTime.nowUnixMillis() - start))}`,
|
|
77
77
|
)
|
|
78
78
|
}
|
|
79
79
|
|
|
@@ -87,7 +87,7 @@ export function transformThrottleByRSS<T>(
|
|
|
87
87
|
|
|
88
88
|
if (!disabled && lastRSS > maxRSSBytes && !lock) {
|
|
89
89
|
lock = pDefer()
|
|
90
|
-
pausedSince =
|
|
90
|
+
pausedSince = localTime.nowUnixMillis()
|
|
91
91
|
logger.log(
|
|
92
92
|
`${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`,
|
|
93
93
|
)
|
|
@@ -122,7 +122,7 @@ export function transformThrottleByRSS<T>(
|
|
|
122
122
|
)
|
|
123
123
|
lock!.resolve()
|
|
124
124
|
lock = undefined
|
|
125
|
-
} else if (pollTimeout &&
|
|
125
|
+
} else if (pollTimeout && localTime.nowUnixMillis() - pausedSince >= pollTimeout) {
|
|
126
126
|
clearTimeout(rssCheckTimer)
|
|
127
127
|
if (onPollTimeout === 'throw') {
|
|
128
128
|
lock!.reject(
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
// oxlint-disable-next-line no-restricted-globals -- the worker entry stays dependency-free
|
|
1
2
|
const started = Date.now()
|
|
2
3
|
import { inspect } from 'node:util'
|
|
3
4
|
import { workerData, parentPort } from 'node:worker_threads'
|
|
4
5
|
|
|
5
6
|
const { workerFile, workerIndex, logEvery = 1000, metric = 'worker', silent } = workerData || {}
|
|
6
7
|
|
|
8
|
+
// The worker entry stays dependency-free, so no `_assert` here
|
|
9
|
+
// oxlint-disable-next-line nc/prefer-assert
|
|
7
10
|
if (!workerFile) {
|
|
8
11
|
throw new Error('workerData.workerFile is required!')
|
|
9
12
|
}
|
|
@@ -18,6 +21,7 @@ try {
|
|
|
18
21
|
const { WorkerClass } = await import(workerFile)
|
|
19
22
|
const worker = new WorkerClass(workerData)
|
|
20
23
|
|
|
24
|
+
// oxlint-disable-next-line no-restricted-globals -- the worker entry stays dependency-free
|
|
21
25
|
log(`${metric}#${workerIndex} loaded in ${Date.now() - started} ms`)
|
|
22
26
|
|
|
23
27
|
let errors = 0
|
package/src/zip/zipInternal.ts
CHANGED
|
@@ -11,6 +11,8 @@ entry-name normalization/validation.
|
|
|
11
11
|
|
|
12
12
|
import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
13
13
|
|
|
14
|
+
// oxlint-disable no-restricted-globals -- DOS date/time fields are local-time components, converted via the native Date
|
|
15
|
+
|
|
14
16
|
// oxlint-disable no-bitwise -- the ZIP format packs DOS date/time into bit fields
|
|
15
17
|
|
|
16
18
|
// Compression methods.
|
package/src/zip/zipReader.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { Readable } from 'node:stream'
|
|
|
27
27
|
import { pipeline } from 'node:stream/promises'
|
|
28
28
|
import { promisify } from 'node:util'
|
|
29
29
|
import zlib from 'node:zlib'
|
|
30
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
30
31
|
import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
31
32
|
import {
|
|
32
33
|
assertSafeZipEntryName,
|
|
@@ -283,12 +284,11 @@ async function readEndOfCentralDirectory(source: ZipSource): Promise<EndOfCentra
|
|
|
283
284
|
if (locatorIndex >= 0 && buf.readUInt32LE(locatorIndex) === ZIP64_EOCDL_SIG) {
|
|
284
285
|
const zip64EocdrOffset = readUInt64LE(buf, locatorIndex + 8)
|
|
285
286
|
const zip64 = await source.read(zip64EocdrOffset, ZIP64_EOCDR_SIZE)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
}
|
|
287
|
+
_assert(
|
|
288
|
+
zip64.readUInt32LE(0) === ZIP64_EOCDR_SIG,
|
|
289
|
+
'invalid zip64 end of central directory record signature',
|
|
290
|
+
)
|
|
291
|
+
_assert(zip64.readUInt32LE(16) === 0, 'multi-disk zip files are not supported')
|
|
292
292
|
entryCount = readUInt64LE(zip64, 32)
|
|
293
293
|
centralDirectoryOffset = readUInt64LE(zip64, 48)
|
|
294
294
|
centralDirectoryEnd = zip64EocdrOffset
|
package/src/zip/zipWriter.ts
CHANGED
|
@@ -31,6 +31,8 @@ import { finished, pipeline } from 'node:stream/promises'
|
|
|
31
31
|
import { promisify } from 'node:util'
|
|
32
32
|
import zlib from 'node:zlib'
|
|
33
33
|
import { comparators } from '@naturalcycles/js-lib/array/sort.js'
|
|
34
|
+
import { nowUnix } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
35
|
+
import { _assert } from '@naturalcycles/js-lib/error/assert.js'
|
|
34
36
|
import type { UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
35
37
|
import { glob } from 'tinyglobby'
|
|
36
38
|
import {
|
|
@@ -250,9 +252,10 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
250
252
|
}
|
|
251
253
|
// A comment containing this signature would confuse readers that scan
|
|
252
254
|
// backwards for the end-of-central-directory record.
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
255
|
+
_assert(
|
|
256
|
+
!comment.includes(EOCDR_SIG_BYTES),
|
|
257
|
+
'archive comment must not contain the end-of-central-directory signature',
|
|
258
|
+
)
|
|
256
259
|
this.finalized = true
|
|
257
260
|
|
|
258
261
|
const centralDirectoryOffset = this.offset
|
|
@@ -282,7 +285,7 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
282
285
|
|
|
283
286
|
private assertWritable(): void {
|
|
284
287
|
if (this.streamError) throw this.streamError
|
|
285
|
-
|
|
288
|
+
_assert(!this.finalized, 'ZipWriter has already been finalized')
|
|
286
289
|
}
|
|
287
290
|
|
|
288
291
|
/** Build the in-memory representation of an entry from its name and options. */
|
|
@@ -295,7 +298,7 @@ export class ZipWriter implements AsyncDisposable {
|
|
|
295
298
|
if (nameBuf.length > MAX_COMMENT_SIZE) {
|
|
296
299
|
throw new Error(`zip entry name is too long: ${nameBuf.length} > ${MAX_COMMENT_SIZE} bytes`)
|
|
297
300
|
}
|
|
298
|
-
const mtime = opt.mtime ?? (
|
|
301
|
+
const mtime = opt.mtime ?? nowUnix()
|
|
299
302
|
const { date, time } = unixToDosDateTime(mtime)
|
|
300
303
|
const mode = opt.mode ?? (isDirectory ? DEFAULT_DIR_MODE : DEFAULT_FILE_MODE)
|
|
301
304
|
if ((mode & 0xffff) !== mode) {
|
|
@@ -659,7 +662,7 @@ function buildEndRecords(input: EndRecordsInput): Buffer {
|
|
|
659
662
|
}
|
|
660
663
|
|
|
661
664
|
function normalizeAndValidateEntryName(fileName: string, isDirectory: boolean): string {
|
|
662
|
-
|
|
665
|
+
_assert(fileName, 'zip entry name must not be empty')
|
|
663
666
|
let name = normalizeZipEntryName(fileName)
|
|
664
667
|
if (isDirectory) {
|
|
665
668
|
if (!name.endsWith('/')) name += '/'
|