@naturalcycles/nodejs-lib 15.33.0 → 15.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache/lruMemoCache.d.ts +1 -1
- package/dist/cache/lruMemoCache.js +1 -1
- package/dist/csv/csvReader.js +1 -1
- package/dist/exec2/exec2.js +1 -0
- package/dist/infra/process.util.js +1 -1
- package/dist/slack/slack.service.model.d.ts +1 -1
- package/dist/stream/ndjson/ndjson.model.js +1 -1
- package/dist/stream/pipeline.js +1 -2
- package/dist/stream/readable/createReadable.js +1 -2
- package/dist/stream/transform/transformThrottle.js +4 -4
- package/dist/util/env.util.js +1 -1
- package/dist/validation/ajv/ajvSchema.d.ts +13 -7
- package/dist/validation/ajv/ajvSchema.js +31 -19
- package/dist/validation/ajv/getAjv.js +129 -0
- package/package.json +1 -1
- package/src/cache/lruMemoCache.ts +1 -1
- package/src/csv/csvReader.ts +1 -1
- package/src/exec2/exec2.ts +1 -0
- package/src/infra/process.util.ts +1 -1
- package/src/slack/slack.service.model.ts +1 -1
- package/src/stream/ndjson/ndjson.model.ts +1 -1
- package/src/stream/pipeline.ts +1 -2
- package/src/stream/readable/createReadable.ts +1 -2
- package/src/stream/stream.model.ts +2 -2
- package/src/stream/transform/transformThrottle.ts +4 -4
- package/src/util/env.util.ts +1 -1
- package/src/validation/ajv/ajvSchema.ts +43 -26
- package/src/validation/ajv/getAjv.ts +130 -0
|
@@ -5,7 +5,7 @@ export type LRUMemoCacheOptions<KEY, VALUE> = Partial<LRUCache.Options<KEY, VALU
|
|
|
5
5
|
* @example
|
|
6
6
|
* Use it like this:
|
|
7
7
|
*
|
|
8
|
-
*
|
|
8
|
+
* `@_Memo({ cacheFactory: () => new LRUMemoCache({...}) })`
|
|
9
9
|
* method1 ()
|
|
10
10
|
*/
|
|
11
11
|
export declare class LRUMemoCache<KEY = any, VALUE = any> implements MemoCache<KEY, VALUE> {
|
package/dist/csv/csvReader.js
CHANGED
|
@@ -21,7 +21,7 @@ export function csvStringParse(str, cfg = {}) {
|
|
|
21
21
|
}
|
|
22
22
|
_assert(header, `firstRowIsHeader or columns is required`);
|
|
23
23
|
return arr.map(row => {
|
|
24
|
-
//
|
|
24
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
25
25
|
return header.reduce((obj, col, i) => {
|
|
26
26
|
;
|
|
27
27
|
obj[col] = row[i];
|
package/dist/exec2/exec2.js
CHANGED
|
@@ -67,7 +67,7 @@ class ProcessUtil {
|
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
getCPUInfo() {
|
|
70
|
-
//
|
|
70
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
71
71
|
return os.cpus().reduce((r, cpu) => {
|
|
72
72
|
r['idle'] += cpu.times.idle;
|
|
73
73
|
Object.values(cpu.times).forEach(m => (r['total'] += m));
|
|
@@ -42,7 +42,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
42
42
|
*/
|
|
43
43
|
kv?: AnyObject;
|
|
44
44
|
/**
|
|
45
|
-
* If specified - adds
|
|
45
|
+
* If specified - adds `@name1`, `@name2` in the end of the message
|
|
46
46
|
*/
|
|
47
47
|
mentions?: string[];
|
|
48
48
|
/**
|
|
@@ -9,7 +9,7 @@ export class NDJsonStats {
|
|
|
9
9
|
return new NDJsonStats();
|
|
10
10
|
}
|
|
11
11
|
static createCombined(stats) {
|
|
12
|
-
//
|
|
12
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
13
13
|
return stats.reduce((statsTotal, stats) => statsTotal.add(stats), new NDJsonStats());
|
|
14
14
|
}
|
|
15
15
|
tookMillis = 0;
|
package/dist/stream/pipeline.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { pipeline } from 'node:stream/promises';
|
|
3
|
-
import { createUnzip } from 'node:zlib';
|
|
4
|
-
import { createGzip } from 'node:zlib';
|
|
3
|
+
import { createGzip, createUnzip } from 'node:zlib';
|
|
5
4
|
import { createAbortableSignal } from '@naturalcycles/js-lib';
|
|
6
5
|
import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
|
|
7
6
|
import { fs2 } from '../fs/fs2.js';
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { Transform } from 'node:stream';
|
|
2
|
-
import { Readable } from 'node:stream';
|
|
1
|
+
import { Readable, Transform } from 'node:stream';
|
|
3
2
|
/**
|
|
4
3
|
* Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
|
|
5
4
|
* Push `null` to it to complete (similar to RxJS `.complete()`).
|
|
@@ -32,8 +32,8 @@ export function transformThrottle(opt) {
|
|
|
32
32
|
async transform(item, _, cb) {
|
|
33
33
|
// console.log('incoming', item, { paused: !!paused, count })
|
|
34
34
|
if (!start) {
|
|
35
|
-
start =
|
|
36
|
-
timeout = setTimeout(() => onInterval(
|
|
35
|
+
start = localTime.nowUnixMillis();
|
|
36
|
+
timeout = setTimeout(() => onInterval(), interval * 1000);
|
|
37
37
|
logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
|
|
38
38
|
throughput,
|
|
39
39
|
interval,
|
|
@@ -56,7 +56,7 @@ export function transformThrottle(opt) {
|
|
|
56
56
|
cb();
|
|
57
57
|
},
|
|
58
58
|
});
|
|
59
|
-
function onInterval(
|
|
59
|
+
function onInterval() {
|
|
60
60
|
if (lock) {
|
|
61
61
|
logger.log(`${localTime.now().toPretty()} transformThrottle resumed`);
|
|
62
62
|
lock.resolve();
|
|
@@ -67,6 +67,6 @@ export function transformThrottle(opt) {
|
|
|
67
67
|
}
|
|
68
68
|
count = 0;
|
|
69
69
|
start = localTime.nowUnixMillis();
|
|
70
|
-
timeout = setTimeout(() => onInterval(
|
|
70
|
+
timeout = setTimeout(() => onInterval(), interval * 1000);
|
|
71
71
|
}
|
|
72
72
|
}
|
package/dist/util/env.util.js
CHANGED
|
@@ -7,7 +7,7 @@ import { fs2 } from '../fs/fs2.js';
|
|
|
7
7
|
* Will throw if any of the passed keys is not defined.
|
|
8
8
|
*/
|
|
9
9
|
export function requireEnvKeys(...keys) {
|
|
10
|
-
//
|
|
10
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
11
11
|
return keys.reduce((r, k) => {
|
|
12
12
|
const v = process.env[k];
|
|
13
13
|
if (!v)
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { type ValidationFunction, type ValidationFunctionResult } from '@naturalcycles/js-lib';
|
|
2
2
|
import type { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib/json-schema';
|
|
3
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types';
|
|
3
4
|
import { ZodType } from '@naturalcycles/js-lib/zod';
|
|
4
5
|
import type { Ajv } from 'ajv';
|
|
5
6
|
import { AjvValidationError } from './ajvValidationError.js';
|
|
7
|
+
export type SchemaHandledByAjv<T> = JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>;
|
|
6
8
|
export interface AjvValidationOptions {
|
|
7
9
|
/**
|
|
8
10
|
* Defaults to true,
|
|
@@ -63,11 +65,15 @@ export declare class AjvSchema<T = unknown> {
|
|
|
63
65
|
* Implementation note: JsonSchemaBuilder goes first in the union type, otherwise TypeScript fails to infer <T> type
|
|
64
66
|
* correctly for some reason.
|
|
65
67
|
*/
|
|
66
|
-
static create<T>(schema:
|
|
68
|
+
static create<T>(schema: SchemaHandledByAjv<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T>;
|
|
67
69
|
/**
|
|
68
|
-
* @
|
|
70
|
+
* @deprecated
|
|
71
|
+
*
|
|
72
|
+
* Use `AjvSchema.create`
|
|
69
73
|
*/
|
|
70
74
|
static createFromZod<T>(zodSchema: ZodType<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T>;
|
|
75
|
+
static isJsonSchemaBuilder<T>(schema: unknown): schema is JsonSchemaBuilder<T>;
|
|
76
|
+
static isZodSchema<T>(schema: unknown): schema is ZodType<T>;
|
|
71
77
|
readonly cfg: AjvSchemaCfg;
|
|
72
78
|
/**
|
|
73
79
|
* It returns the original object just for convenience.
|
|
@@ -81,12 +87,12 @@ export declare class AjvSchema<T = unknown> {
|
|
|
81
87
|
isValid(input: T, opt?: AjvValidationOptions): boolean;
|
|
82
88
|
getValidationResult(input: T, opt?: AjvValidationOptions): ValidationFunctionResult<T, AjvValidationError>;
|
|
83
89
|
getValidationFunction(): ValidationFunction<T, AjvValidationError>;
|
|
84
|
-
static
|
|
85
|
-
static
|
|
86
|
-
static
|
|
90
|
+
static isSchemaWithCachedAjvSchema<Base, T>(schema: Base): schema is WithCachedAjvSchema<Base, T>;
|
|
91
|
+
static cacheAjvSchema<Base extends AnyObject, T>(schema: Base, ajvSchema: AjvSchema<T>): WithCachedAjvSchema<Base, T>;
|
|
92
|
+
static requireCachedAjvSchema<Base, T>(schema: WithCachedAjvSchema<Base, T>): AjvSchema<T>;
|
|
87
93
|
private getAJVValidateFunction;
|
|
88
94
|
}
|
|
89
95
|
export declare const HIDDEN_AJV_SCHEMA: unique symbol;
|
|
90
|
-
export
|
|
96
|
+
export type WithCachedAjvSchema<Base, T> = Base & {
|
|
91
97
|
[HIDDEN_AJV_SCHEMA]: AjvSchema<T>;
|
|
92
|
-
}
|
|
98
|
+
};
|
|
@@ -46,24 +46,36 @@ export class AjvSchema {
|
|
|
46
46
|
static create(schema, cfg) {
|
|
47
47
|
if (schema instanceof AjvSchema)
|
|
48
48
|
return schema;
|
|
49
|
-
if (schema
|
|
50
|
-
return
|
|
49
|
+
if (AjvSchema.isSchemaWithCachedAjvSchema(schema)) {
|
|
50
|
+
return AjvSchema.requireCachedAjvSchema(schema);
|
|
51
51
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
let jsonSchema;
|
|
53
|
+
if (AjvSchema.isJsonSchemaBuilder(schema)) {
|
|
54
|
+
jsonSchema = schema.build();
|
|
55
|
+
}
|
|
56
|
+
else if (AjvSchema.isZodSchema(schema)) {
|
|
57
|
+
jsonSchema = z.toJSONSchema(schema, { target: 'draft-7' });
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
jsonSchema = schema;
|
|
61
|
+
}
|
|
62
|
+
const ajvSchema = new AjvSchema(jsonSchema, cfg);
|
|
63
|
+
AjvSchema.cacheAjvSchema(schema, ajvSchema);
|
|
64
|
+
return ajvSchema;
|
|
55
65
|
}
|
|
56
66
|
/**
|
|
57
|
-
* @
|
|
67
|
+
* @deprecated
|
|
68
|
+
*
|
|
69
|
+
* Use `AjvSchema.create`
|
|
58
70
|
*/
|
|
59
71
|
static createFromZod(zodSchema, cfg) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return
|
|
72
|
+
return AjvSchema.create(zodSchema, cfg);
|
|
73
|
+
}
|
|
74
|
+
static isJsonSchemaBuilder(schema) {
|
|
75
|
+
return schema instanceof JsonSchemaAnyBuilder;
|
|
76
|
+
}
|
|
77
|
+
static isZodSchema(schema) {
|
|
78
|
+
return schema instanceof ZodType;
|
|
67
79
|
}
|
|
68
80
|
cfg;
|
|
69
81
|
/**
|
|
@@ -121,14 +133,14 @@ export class AjvSchema {
|
|
|
121
133
|
});
|
|
122
134
|
};
|
|
123
135
|
}
|
|
124
|
-
static
|
|
125
|
-
return !!schema[HIDDEN_AJV_SCHEMA];
|
|
136
|
+
static isSchemaWithCachedAjvSchema(schema) {
|
|
137
|
+
return !!schema?.[HIDDEN_AJV_SCHEMA];
|
|
126
138
|
}
|
|
127
|
-
static
|
|
128
|
-
return Object.assign(
|
|
139
|
+
static cacheAjvSchema(schema, ajvSchema) {
|
|
140
|
+
return Object.assign(schema, { [HIDDEN_AJV_SCHEMA]: ajvSchema });
|
|
129
141
|
}
|
|
130
|
-
static
|
|
131
|
-
return
|
|
142
|
+
static requireCachedAjvSchema(schema) {
|
|
143
|
+
return schema[HIDDEN_AJV_SCHEMA];
|
|
132
144
|
}
|
|
133
145
|
getAJVValidateFunction = _lazyValue(() => this.cfg.ajv.compile(this.schema));
|
|
134
146
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
|
|
2
|
+
/* eslint-disable unicorn/prefer-code-point */
|
|
1
3
|
import { _lazyValue } from '@naturalcycles/js-lib';
|
|
2
4
|
import { Ajv } from 'ajv';
|
|
3
5
|
import ajvFormats from 'ajv-formats';
|
|
@@ -66,6 +68,7 @@ const TS_2500 = 16725225600; // 2500-01-01
|
|
|
66
68
|
const TS_2500_MILLIS = TS_2500 * 1000;
|
|
67
69
|
const TS_2000 = 946684800; // 2000-01-01
|
|
68
70
|
const TS_2000_MILLIS = TS_2000 * 1000;
|
|
71
|
+
const monthLengths = [0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
69
72
|
function addCustomAjvFormats(ajv) {
|
|
70
73
|
return (ajv
|
|
71
74
|
.addFormat('id', /^[a-z0-9_]{6,64}$/)
|
|
@@ -116,5 +119,131 @@ function addCustomAjvFormats(ajv) {
|
|
|
116
119
|
// multipleOf 15 (minutes)
|
|
117
120
|
return n >= -14 && n <= 14 && Number.isInteger(n);
|
|
118
121
|
},
|
|
122
|
+
})
|
|
123
|
+
.addFormat('IsoDate', {
|
|
124
|
+
type: 'string',
|
|
125
|
+
validate: isIsoDateValid,
|
|
126
|
+
})
|
|
127
|
+
.addFormat('IsoDateTime', {
|
|
128
|
+
type: 'string',
|
|
129
|
+
validate: isIsoDateTimeValid,
|
|
119
130
|
}));
|
|
120
131
|
}
|
|
132
|
+
const DASH_CODE = '-'.charCodeAt(0);
|
|
133
|
+
const ZERO_CODE = '0'.charCodeAt(0);
|
|
134
|
+
const PLUS_CODE = '+'.charCodeAt(0);
|
|
135
|
+
const COLON_CODE = ':'.charCodeAt(0);
|
|
136
|
+
/**
|
|
137
|
+
* This is a performance optimized correct validation
|
|
138
|
+
* for ISO dates formatted as YYYY-MM-DD.
|
|
139
|
+
*
|
|
140
|
+
* - Slightly more performant than using `localDate`.
|
|
141
|
+
* - More performant than string splitting and `Number()` conversions
|
|
142
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
143
|
+
*/
|
|
144
|
+
function isIsoDateValid(s) {
|
|
145
|
+
// must be exactly "YYYY-MM-DD"
|
|
146
|
+
if (s.length !== 10)
|
|
147
|
+
return false;
|
|
148
|
+
if (s.charCodeAt(4) !== DASH_CODE || s.charCodeAt(7) !== DASH_CODE)
|
|
149
|
+
return false;
|
|
150
|
+
// fast parse numbers without substrings/Number()
|
|
151
|
+
const year = (s.charCodeAt(0) - ZERO_CODE) * 1000 +
|
|
152
|
+
(s.charCodeAt(1) - ZERO_CODE) * 100 +
|
|
153
|
+
(s.charCodeAt(2) - ZERO_CODE) * 10 +
|
|
154
|
+
(s.charCodeAt(3) - ZERO_CODE);
|
|
155
|
+
const month = (s.charCodeAt(5) - ZERO_CODE) * 10 + (s.charCodeAt(6) - ZERO_CODE);
|
|
156
|
+
const day = (s.charCodeAt(8) - ZERO_CODE) * 10 + (s.charCodeAt(9) - ZERO_CODE);
|
|
157
|
+
if (month < 1 || month > 12 || day < 1)
|
|
158
|
+
return false;
|
|
159
|
+
if (month !== 2) {
|
|
160
|
+
return day <= monthLengths[month];
|
|
161
|
+
}
|
|
162
|
+
const isLeap = (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;
|
|
163
|
+
return day <= (isLeap ? 29 : 28);
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* This is a performance optimized correct validation
|
|
167
|
+
* for ISO datetimes formatted as "YYYY-MM-DDTHH:MM:SS" followed by
|
|
168
|
+
* nothing, "Z" or "+hh:mm" or "-hh:mm".
|
|
169
|
+
*
|
|
170
|
+
* - Slightly more performant than using `localTime`.
|
|
171
|
+
* - More performant than string splitting and `Number()` conversions
|
|
172
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
173
|
+
*/
|
|
174
|
+
function isIsoDateTimeValid(s) {
|
|
175
|
+
if (s.length < 19 || s.length > 25)
|
|
176
|
+
return false;
|
|
177
|
+
if (s.charCodeAt(10) !== 84)
|
|
178
|
+
return false; // 'T'
|
|
179
|
+
const datePart = s.slice(0, 10); // YYYY-MM-DD
|
|
180
|
+
if (!isIsoDateValid(datePart))
|
|
181
|
+
return false;
|
|
182
|
+
const timePart = s.slice(11, 19); // HH:MM:SS
|
|
183
|
+
if (!isIsoTimeValid(timePart))
|
|
184
|
+
return false;
|
|
185
|
+
const zonePart = s.slice(19); // nothing or Z or +/-hh:mm
|
|
186
|
+
if (!isIsoTimezoneValid(zonePart))
|
|
187
|
+
return false;
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* This is a performance optimized correct validation
|
|
192
|
+
* for ISO times formatted as "HH:MM:SS".
|
|
193
|
+
*
|
|
194
|
+
* - Slightly more performant than using `localTime`.
|
|
195
|
+
* - More performant than string splitting and `Number()` conversions
|
|
196
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
197
|
+
*/
|
|
198
|
+
function isIsoTimeValid(s) {
|
|
199
|
+
if (s.length !== 8)
|
|
200
|
+
return false;
|
|
201
|
+
if (s.charCodeAt(2) !== COLON_CODE || s.charCodeAt(5) !== COLON_CODE)
|
|
202
|
+
return false;
|
|
203
|
+
const hour = (s.charCodeAt(0) - ZERO_CODE) * 10 + (s.charCodeAt(1) - ZERO_CODE);
|
|
204
|
+
if (hour < 0 || hour > 23)
|
|
205
|
+
return false;
|
|
206
|
+
const minute = (s.charCodeAt(3) - ZERO_CODE) * 10 + (s.charCodeAt(4) - ZERO_CODE);
|
|
207
|
+
if (minute < 0 || minute > 59)
|
|
208
|
+
return false;
|
|
209
|
+
const second = (s.charCodeAt(6) - ZERO_CODE) * 10 + (s.charCodeAt(7) - ZERO_CODE);
|
|
210
|
+
if (second < 0 || second > 59)
|
|
211
|
+
return false;
|
|
212
|
+
return true;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* This is a performance optimized correct validation
|
|
216
|
+
* for the timezone suffix of ISO times
|
|
217
|
+
* formatted as "Z" or "+HH:MM" or "-HH:MM".
|
|
218
|
+
*
|
|
219
|
+
* It also accepts an empty string.
|
|
220
|
+
*/
|
|
221
|
+
function isIsoTimezoneValid(s) {
|
|
222
|
+
if (s === '')
|
|
223
|
+
return true;
|
|
224
|
+
if (s === 'Z')
|
|
225
|
+
return true;
|
|
226
|
+
if (s.length !== 6)
|
|
227
|
+
return false;
|
|
228
|
+
if (s.charCodeAt(0) !== PLUS_CODE && s.charCodeAt(0) !== DASH_CODE)
|
|
229
|
+
return false;
|
|
230
|
+
if (s.charCodeAt(3) !== COLON_CODE)
|
|
231
|
+
return false;
|
|
232
|
+
const isWestern = s[0] === '-';
|
|
233
|
+
const isEastern = s[0] === '+';
|
|
234
|
+
const hour = (s.charCodeAt(1) - ZERO_CODE) * 10 + (s.charCodeAt(2) - ZERO_CODE);
|
|
235
|
+
if (hour < 0)
|
|
236
|
+
return false;
|
|
237
|
+
if (isWestern && hour > 12)
|
|
238
|
+
return false;
|
|
239
|
+
if (isEastern && hour > 14)
|
|
240
|
+
return false;
|
|
241
|
+
const minute = (s.charCodeAt(4) - ZERO_CODE) * 10 + (s.charCodeAt(5) - ZERO_CODE);
|
|
242
|
+
if (minute < 0 || minute > 59)
|
|
243
|
+
return false;
|
|
244
|
+
if (isEastern && hour === 14 && minute > 0)
|
|
245
|
+
return false; // max is +14:00
|
|
246
|
+
if (isWestern && hour === 12 && minute > 0)
|
|
247
|
+
return false; // min is -12:00
|
|
248
|
+
return true;
|
|
249
|
+
}
|
package/package.json
CHANGED
|
@@ -8,7 +8,7 @@ export type LRUMemoCacheOptions<KEY, VALUE> = Partial<LRUCache.Options<KEY, VALU
|
|
|
8
8
|
* @example
|
|
9
9
|
* Use it like this:
|
|
10
10
|
*
|
|
11
|
-
*
|
|
11
|
+
* `@_Memo({ cacheFactory: () => new LRUMemoCache({...}) })`
|
|
12
12
|
* method1 ()
|
|
13
13
|
*/
|
|
14
14
|
export class LRUMemoCache<KEY = any, VALUE = any> implements MemoCache<KEY, VALUE> {
|
package/src/csv/csvReader.ts
CHANGED
|
@@ -48,7 +48,7 @@ export function csvStringParse<T extends AnyObject = any>(
|
|
|
48
48
|
_assert(header, `firstRowIsHeader or columns is required`)
|
|
49
49
|
|
|
50
50
|
return arr.map(row => {
|
|
51
|
-
//
|
|
51
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
52
52
|
return header.reduce((obj, col, i) => {
|
|
53
53
|
;(obj as any)[col] = row[i]
|
|
54
54
|
return obj
|
package/src/exec2/exec2.ts
CHANGED
|
@@ -51,7 +51,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
51
51
|
kv?: AnyObject
|
|
52
52
|
|
|
53
53
|
/**
|
|
54
|
-
* If specified - adds
|
|
54
|
+
* If specified - adds `@name1`, `@name2` in the end of the message
|
|
55
55
|
*/
|
|
56
56
|
mentions?: string[]
|
|
57
57
|
|
|
@@ -12,7 +12,7 @@ export class NDJsonStats {
|
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
static createCombined(stats: NDJsonStats[]): NDJsonStats {
|
|
15
|
-
//
|
|
15
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
16
16
|
return stats.reduce((statsTotal, stats) => statsTotal.add(stats), new NDJsonStats())
|
|
17
17
|
}
|
|
18
18
|
|
package/src/stream/pipeline.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { Readable, type Transform } from 'node:stream'
|
|
2
2
|
import { pipeline } from 'node:stream/promises'
|
|
3
3
|
import type { ReadableStream as WebReadableStream } from 'node:stream/web'
|
|
4
|
-
import { createUnzip, type ZlibOptions } from 'node:zlib'
|
|
5
|
-
import { createGzip } from 'node:zlib'
|
|
4
|
+
import { createGzip, createUnzip, type ZlibOptions } from 'node:zlib'
|
|
6
5
|
import { createAbortableSignal } from '@naturalcycles/js-lib'
|
|
7
6
|
import {
|
|
8
7
|
_passthroughPredicate,
|
|
@@ -49,11 +49,11 @@ export interface ReadableTyped<T = unknown> extends Readable {
|
|
|
49
49
|
drop: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
//
|
|
52
|
+
// oxlint-disable no-unused-vars
|
|
53
53
|
export interface WritableTyped<T> extends Writable {}
|
|
54
54
|
|
|
55
|
-
// biome-ignore lint/correctness/noUnusedVariables: ok
|
|
56
55
|
export interface TransformTyped<IN = unknown, OUT = unknown> extends Transform {}
|
|
56
|
+
// oxlint-enable
|
|
57
57
|
|
|
58
58
|
export interface TransformOptions {
|
|
59
59
|
/**
|
|
@@ -54,8 +54,8 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
|
|
|
54
54
|
async transform(item: T, _, cb) {
|
|
55
55
|
// console.log('incoming', item, { paused: !!paused, count })
|
|
56
56
|
if (!start) {
|
|
57
|
-
start =
|
|
58
|
-
timeout = setTimeout(() => onInterval(
|
|
57
|
+
start = localTime.nowUnixMillis()
|
|
58
|
+
timeout = setTimeout(() => onInterval(), interval * 1000)
|
|
59
59
|
logger.log(`${localTime.now().toPretty()} transformThrottle started with`, {
|
|
60
60
|
throughput,
|
|
61
61
|
interval,
|
|
@@ -84,7 +84,7 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
|
|
|
84
84
|
},
|
|
85
85
|
})
|
|
86
86
|
|
|
87
|
-
function onInterval(
|
|
87
|
+
function onInterval(): void {
|
|
88
88
|
if (lock) {
|
|
89
89
|
logger.log(`${localTime.now().toPretty()} transformThrottle resumed`)
|
|
90
90
|
lock.resolve()
|
|
@@ -97,6 +97,6 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
|
|
|
97
97
|
|
|
98
98
|
count = 0
|
|
99
99
|
start = localTime.nowUnixMillis()
|
|
100
|
-
timeout = setTimeout(() => onInterval(
|
|
100
|
+
timeout = setTimeout(() => onInterval(), interval * 1000)
|
|
101
101
|
}
|
|
102
102
|
}
|
package/src/util/env.util.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { fs2 } from '../fs/fs2.js'
|
|
|
11
11
|
export function requireEnvKeys<T extends readonly string[]>(
|
|
12
12
|
...keys: T
|
|
13
13
|
): { [k in ValuesOf<T>]: string } {
|
|
14
|
-
//
|
|
14
|
+
// oxlint-disable-next-line unicorn/no-array-reduce
|
|
15
15
|
return keys.reduce(
|
|
16
16
|
(r, k) => {
|
|
17
17
|
const v = process.env[k]
|
|
@@ -8,12 +8,15 @@ import type { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib/json-s
|
|
|
8
8
|
import { JsonSchemaAnyBuilder } from '@naturalcycles/js-lib/json-schema'
|
|
9
9
|
import { _deepCopy, _filterNullishValues } from '@naturalcycles/js-lib/object'
|
|
10
10
|
import { _substringBefore } from '@naturalcycles/js-lib/string'
|
|
11
|
+
import type { AnyObject } from '@naturalcycles/js-lib/types'
|
|
11
12
|
import { z, ZodType } from '@naturalcycles/js-lib/zod'
|
|
12
13
|
import type { Ajv } from 'ajv'
|
|
13
14
|
import { _inspect } from '../../string/inspect.js'
|
|
14
15
|
import { AjvValidationError } from './ajvValidationError.js'
|
|
15
16
|
import { getAjv } from './getAjv.js'
|
|
16
17
|
|
|
18
|
+
export type SchemaHandledByAjv<T> = JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>
|
|
19
|
+
|
|
17
20
|
export interface AjvValidationOptions {
|
|
18
21
|
/**
|
|
19
22
|
* Defaults to true,
|
|
@@ -104,32 +107,44 @@ export class AjvSchema<T = unknown> {
|
|
|
104
107
|
* Implementation note: JsonSchemaBuilder goes first in the union type, otherwise TypeScript fails to infer <T> type
|
|
105
108
|
* correctly for some reason.
|
|
106
109
|
*/
|
|
107
|
-
static create<T>(
|
|
108
|
-
schema: JsonSchemaBuilder<T> | JsonSchema<T> | AjvSchema<T> | ZodType<T>,
|
|
109
|
-
cfg?: Partial<AjvSchemaCfg>,
|
|
110
|
-
): AjvSchema<T> {
|
|
110
|
+
static create<T>(schema: SchemaHandledByAjv<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T> {
|
|
111
111
|
if (schema instanceof AjvSchema) return schema
|
|
112
|
-
|
|
113
|
-
|
|
112
|
+
|
|
113
|
+
if (AjvSchema.isSchemaWithCachedAjvSchema<typeof schema, T>(schema)) {
|
|
114
|
+
return AjvSchema.requireCachedAjvSchema<typeof schema, T>(schema)
|
|
114
115
|
}
|
|
115
|
-
|
|
116
|
-
|
|
116
|
+
|
|
117
|
+
let jsonSchema: JsonSchema<T>
|
|
118
|
+
|
|
119
|
+
if (AjvSchema.isJsonSchemaBuilder(schema)) {
|
|
120
|
+
jsonSchema = schema.build()
|
|
121
|
+
} else if (AjvSchema.isZodSchema(schema)) {
|
|
122
|
+
jsonSchema = z.toJSONSchema(schema, { target: 'draft-7' }) as JsonSchema<T>
|
|
123
|
+
} else {
|
|
124
|
+
jsonSchema = schema
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const ajvSchema = new AjvSchema<T>(jsonSchema, cfg)
|
|
128
|
+
AjvSchema.cacheAjvSchema(schema, ajvSchema)
|
|
129
|
+
|
|
130
|
+
return ajvSchema
|
|
117
131
|
}
|
|
118
132
|
|
|
119
133
|
/**
|
|
120
|
-
* @
|
|
134
|
+
* @deprecated
|
|
135
|
+
*
|
|
136
|
+
* Use `AjvSchema.create`
|
|
121
137
|
*/
|
|
122
138
|
static createFromZod<T>(zodSchema: ZodType<T>, cfg?: Partial<AjvSchemaCfg>): AjvSchema<T> {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
const jsonSchema = z.toJSONSchema(zodSchema, { target: 'draft-7' })
|
|
128
|
-
const ajvSchema = new AjvSchema<T>(jsonSchema as JsonSchema<T>, cfg)
|
|
139
|
+
return AjvSchema.create(zodSchema, cfg)
|
|
140
|
+
}
|
|
129
141
|
|
|
130
|
-
|
|
142
|
+
static isJsonSchemaBuilder<T>(schema: unknown): schema is JsonSchemaBuilder<T> {
|
|
143
|
+
return schema instanceof JsonSchemaAnyBuilder
|
|
144
|
+
}
|
|
131
145
|
|
|
132
|
-
|
|
146
|
+
static isZodSchema<T>(schema: unknown): schema is ZodType<T> {
|
|
147
|
+
return schema instanceof ZodType
|
|
133
148
|
}
|
|
134
149
|
|
|
135
150
|
readonly cfg: AjvSchemaCfg
|
|
@@ -208,19 +223,21 @@ export class AjvSchema<T = unknown> {
|
|
|
208
223
|
}
|
|
209
224
|
}
|
|
210
225
|
|
|
211
|
-
static
|
|
212
|
-
|
|
226
|
+
static isSchemaWithCachedAjvSchema<Base, T>(
|
|
227
|
+
schema: Base,
|
|
228
|
+
): schema is WithCachedAjvSchema<Base, T> {
|
|
229
|
+
return !!(schema as any)?.[HIDDEN_AJV_SCHEMA]
|
|
213
230
|
}
|
|
214
231
|
|
|
215
|
-
static
|
|
216
|
-
|
|
232
|
+
static cacheAjvSchema<Base extends AnyObject, T>(
|
|
233
|
+
schema: Base,
|
|
217
234
|
ajvSchema: AjvSchema<T>,
|
|
218
|
-
):
|
|
219
|
-
return Object.assign(
|
|
235
|
+
): WithCachedAjvSchema<Base, T> {
|
|
236
|
+
return Object.assign(schema, { [HIDDEN_AJV_SCHEMA]: ajvSchema })
|
|
220
237
|
}
|
|
221
238
|
|
|
222
|
-
static
|
|
223
|
-
return
|
|
239
|
+
static requireCachedAjvSchema<Base, T>(schema: WithCachedAjvSchema<Base, T>): AjvSchema<T> {
|
|
240
|
+
return schema[HIDDEN_AJV_SCHEMA]
|
|
224
241
|
}
|
|
225
242
|
|
|
226
243
|
private getAJVValidateFunction = _lazyValue(() => this.cfg.ajv.compile<T>(this.schema))
|
|
@@ -230,6 +247,6 @@ const separator = '\n'
|
|
|
230
247
|
|
|
231
248
|
export const HIDDEN_AJV_SCHEMA = Symbol('HIDDEN_AJV_SCHEMA')
|
|
232
249
|
|
|
233
|
-
export
|
|
250
|
+
export type WithCachedAjvSchema<Base, T> = Base & {
|
|
234
251
|
[HIDDEN_AJV_SCHEMA]: AjvSchema<T>
|
|
235
252
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/prefer-string-starts-ends-with */
|
|
2
|
+
/* eslint-disable unicorn/prefer-code-point */
|
|
1
3
|
import { _lazyValue } from '@naturalcycles/js-lib'
|
|
2
4
|
import type { Options } from 'ajv'
|
|
3
5
|
import { Ajv } from 'ajv'
|
|
@@ -79,6 +81,8 @@ const TS_2500_MILLIS = TS_2500 * 1000
|
|
|
79
81
|
const TS_2000 = 946684800 // 2000-01-01
|
|
80
82
|
const TS_2000_MILLIS = TS_2000 * 1000
|
|
81
83
|
|
|
84
|
+
const monthLengths = [0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
|
|
85
|
+
|
|
82
86
|
function addCustomAjvFormats(ajv: Ajv): Ajv {
|
|
83
87
|
return (
|
|
84
88
|
ajv
|
|
@@ -131,5 +135,131 @@ function addCustomAjvFormats(ajv: Ajv): Ajv {
|
|
|
131
135
|
return n >= -14 && n <= 14 && Number.isInteger(n)
|
|
132
136
|
},
|
|
133
137
|
})
|
|
138
|
+
.addFormat('IsoDate', {
|
|
139
|
+
type: 'string',
|
|
140
|
+
validate: isIsoDateValid,
|
|
141
|
+
})
|
|
142
|
+
.addFormat('IsoDateTime', {
|
|
143
|
+
type: 'string',
|
|
144
|
+
validate: isIsoDateTimeValid,
|
|
145
|
+
})
|
|
134
146
|
)
|
|
135
147
|
}
|
|
148
|
+
|
|
149
|
+
const DASH_CODE = '-'.charCodeAt(0)
|
|
150
|
+
const ZERO_CODE = '0'.charCodeAt(0)
|
|
151
|
+
const PLUS_CODE = '+'.charCodeAt(0)
|
|
152
|
+
const COLON_CODE = ':'.charCodeAt(0)
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* This is a performance optimized correct validation
|
|
156
|
+
* for ISO dates formatted as YYYY-MM-DD.
|
|
157
|
+
*
|
|
158
|
+
* - Slightly more performant than using `localDate`.
|
|
159
|
+
* - More performant than string splitting and `Number()` conversions
|
|
160
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
161
|
+
*/
|
|
162
|
+
function isIsoDateValid(s: string): boolean {
|
|
163
|
+
// must be exactly "YYYY-MM-DD"
|
|
164
|
+
if (s.length !== 10) return false
|
|
165
|
+
if (s.charCodeAt(4) !== DASH_CODE || s.charCodeAt(7) !== DASH_CODE) return false
|
|
166
|
+
|
|
167
|
+
// fast parse numbers without substrings/Number()
|
|
168
|
+
const year =
|
|
169
|
+
(s.charCodeAt(0) - ZERO_CODE) * 1000 +
|
|
170
|
+
(s.charCodeAt(1) - ZERO_CODE) * 100 +
|
|
171
|
+
(s.charCodeAt(2) - ZERO_CODE) * 10 +
|
|
172
|
+
(s.charCodeAt(3) - ZERO_CODE)
|
|
173
|
+
|
|
174
|
+
const month = (s.charCodeAt(5) - ZERO_CODE) * 10 + (s.charCodeAt(6) - ZERO_CODE)
|
|
175
|
+
const day = (s.charCodeAt(8) - ZERO_CODE) * 10 + (s.charCodeAt(9) - ZERO_CODE)
|
|
176
|
+
|
|
177
|
+
if (month < 1 || month > 12 || day < 1) return false
|
|
178
|
+
|
|
179
|
+
if (month !== 2) {
|
|
180
|
+
return day <= monthLengths[month]!
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const isLeap = (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0
|
|
184
|
+
return day <= (isLeap ? 29 : 28)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* This is a performance optimized correct validation
|
|
189
|
+
* for ISO datetimes formatted as "YYYY-MM-DDTHH:MM:SS" followed by
|
|
190
|
+
* nothing, "Z" or "+hh:mm" or "-hh:mm".
|
|
191
|
+
*
|
|
192
|
+
* - Slightly more performant than using `localTime`.
|
|
193
|
+
* - More performant than string splitting and `Number()` conversions
|
|
194
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
195
|
+
*/
|
|
196
|
+
function isIsoDateTimeValid(s: string): boolean {
|
|
197
|
+
if (s.length < 19 || s.length > 25) return false
|
|
198
|
+
if (s.charCodeAt(10) !== 84) return false // 'T'
|
|
199
|
+
|
|
200
|
+
const datePart = s.slice(0, 10) // YYYY-MM-DD
|
|
201
|
+
if (!isIsoDateValid(datePart)) return false
|
|
202
|
+
|
|
203
|
+
const timePart = s.slice(11, 19) // HH:MM:SS
|
|
204
|
+
if (!isIsoTimeValid(timePart)) return false
|
|
205
|
+
|
|
206
|
+
const zonePart = s.slice(19) // nothing or Z or +/-hh:mm
|
|
207
|
+
if (!isIsoTimezoneValid(zonePart)) return false
|
|
208
|
+
|
|
209
|
+
return true
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* This is a performance optimized correct validation
|
|
214
|
+
* for ISO times formatted as "HH:MM:SS".
|
|
215
|
+
*
|
|
216
|
+
* - Slightly more performant than using `localTime`.
|
|
217
|
+
* - More performant than string splitting and `Number()` conversions
|
|
218
|
+
* - Less performant than regex, but it does not allow invalid dates.
|
|
219
|
+
*/
|
|
220
|
+
function isIsoTimeValid(s: string): boolean {
|
|
221
|
+
if (s.length !== 8) return false
|
|
222
|
+
if (s.charCodeAt(2) !== COLON_CODE || s.charCodeAt(5) !== COLON_CODE) return false
|
|
223
|
+
|
|
224
|
+
const hour = (s.charCodeAt(0) - ZERO_CODE) * 10 + (s.charCodeAt(1) - ZERO_CODE)
|
|
225
|
+
if (hour < 0 || hour > 23) return false
|
|
226
|
+
|
|
227
|
+
const minute = (s.charCodeAt(3) - ZERO_CODE) * 10 + (s.charCodeAt(4) - ZERO_CODE)
|
|
228
|
+
if (minute < 0 || minute > 59) return false
|
|
229
|
+
|
|
230
|
+
const second = (s.charCodeAt(6) - ZERO_CODE) * 10 + (s.charCodeAt(7) - ZERO_CODE)
|
|
231
|
+
if (second < 0 || second > 59) return false
|
|
232
|
+
|
|
233
|
+
return true
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* This is a performance optimized correct validation
|
|
238
|
+
* for the timezone suffix of ISO times
|
|
239
|
+
* formatted as "Z" or "+HH:MM" or "-HH:MM".
|
|
240
|
+
*
|
|
241
|
+
* It also accepts an empty string.
|
|
242
|
+
*/
|
|
243
|
+
function isIsoTimezoneValid(s: string): boolean {
|
|
244
|
+
if (s === '') return true
|
|
245
|
+
if (s === 'Z') return true
|
|
246
|
+
if (s.length !== 6) return false
|
|
247
|
+
if (s.charCodeAt(0) !== PLUS_CODE && s.charCodeAt(0) !== DASH_CODE) return false
|
|
248
|
+
if (s.charCodeAt(3) !== COLON_CODE) return false
|
|
249
|
+
|
|
250
|
+
const isWestern = s[0] === '-'
|
|
251
|
+
const isEastern = s[0] === '+'
|
|
252
|
+
|
|
253
|
+
const hour = (s.charCodeAt(1) - ZERO_CODE) * 10 + (s.charCodeAt(2) - ZERO_CODE)
|
|
254
|
+
if (hour < 0) return false
|
|
255
|
+
if (isWestern && hour > 12) return false
|
|
256
|
+
if (isEastern && hour > 14) return false
|
|
257
|
+
|
|
258
|
+
const minute = (s.charCodeAt(4) - ZERO_CODE) * 10 + (s.charCodeAt(5) - ZERO_CODE)
|
|
259
|
+
if (minute < 0 || minute > 59) return false
|
|
260
|
+
|
|
261
|
+
if (isEastern && hour === 14 && minute > 0) return false // max is +14:00
|
|
262
|
+
if (isWestern && hour === 12 && minute > 0) return false // min is -12:00
|
|
263
|
+
|
|
264
|
+
return true
|
|
265
|
+
}
|