@naturalcycles/nodejs-lib 15.139.0 → 15.141.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.139.0",
+  "version": "15.141.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",
@@ -14,7 +14,7 @@
     "yaml": "^2"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "20.70.0",
+    "@naturalcycles/dev-lib": "0.0.0",
     "typescript": "^7"
   },
   "exports": {

package/src/exec2/exec2.ts

@@ -1,5 +1,6 @@
 import { execSync, spawn, spawnSync } from 'node:child_process'
 import type { StdioOptions } from 'node:child_process'
+import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import { _substringAfterLast } from '@naturalcycles/js-lib/string/string.util.js'
@@ -69,7 +70,7 @@ class Exec2 {
     const log = opt.log ?? true // by default log should be true, as we are printing the output
     // Copy, to not mutate the input options
     opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
-    const started = Date.now() as UnixTimestampMillis
+    const started = nowUnixMillis()
     this.logStart(cmd, opt)
 
     const r = spawnSync(cmd, opt.args, {
@@ -118,7 +119,7 @@ class Exec2 {
       logStart: opt.logStart ?? opt.log ?? false,
       logFinish: opt.logFinish ?? opt.log ?? false,
     }
-    const started = Date.now() as UnixTimestampMillis
+    const started = nowUnixMillis()
     this.logStart(cmd, opt)
 
     try {
@@ -177,7 +178,7 @@ class Exec2 {
     const log = opt.log ?? true // by default log should be true, as we are printing the output
     // Copy, to not mutate the input options
     opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
-    const started = Date.now() as UnixTimestampMillis
+    const started = nowUnixMillis()
     this.logStart(cmd, opt)
 
     await new Promise<void>((resolve, reject) => {
@@ -245,7 +246,7 @@ class Exec2 {
     const log = opt.log ?? printWhileRunning // by default log should be true, as we are printing the output
     // Copy, to not mutate the input options
     opt = { ...opt, log, logStart: opt.logStart ?? log, logFinish: opt.logFinish ?? log }
-    const started = Date.now() as UnixTimestampMillis
+    const started = nowUnixMillis()
     this.logStart(cmd, opt)
     let stdout = ''
     let stderr = ''

package/src/fs/kpy.ts

@@ -1,5 +1,5 @@
 import path from 'node:path'
-import { _since, localTime } from '@naturalcycles/js-lib/datetime'
+import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime'
 import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { glob, globSync } from 'tinyglobby'
 import { boldWhite, dimGrey, grey, yellow } from '../colors/colors.js'
@@ -45,7 +45,7 @@ export interface KpyOptions {
 }
 
 export async function kpy(opt: KpyOptions): Promise<void> {
-  const started = localTime.nowUnixMillis()
+  const started = nowUnixMillis()
 
   opt = kpyPrepare(opt)
 
@@ -84,7 +84,7 @@ export async function kpy(opt: KpyOptions): Promise<void> {
 }
 
 export function kpySync(opt: KpyOptions): void {
-  const started = localTime.nowUnixMillis()
+  const started = nowUnixMillis()
 
   opt = kpyPrepare(opt)
 

package/src/jwt/jwt.service.ts

@@ -263,6 +263,7 @@ export class JWTService<T extends AnyObject = AnyObject> {
       const { payload } = await jwtVerify(token, key, {
         algorithms: this.cfg.verifyAlgorithms || [this.cfg.algorithm],
         ...joseOpt,
+        // oxlint-disable-next-line no-restricted-globals -- jose expects a Date
         currentDate: now === undefined ? undefined : new Date(now * 1000),
       })
       data = payload as TT

package/src/stream/progressLogger.ts

@@ -168,8 +168,8 @@ export class ProgressLogger<T> implements Disposable {
   private peakRSS!: number
 
   private start(): void {
-    this.started = Date.now() as UnixTimestampMillis
-    this.lastSecondStarted = Date.now() as UnixTimestampMillis
+    this.started = localTime.nowUnixMillis()
+    this.lastSecondStarted = localTime.nowUnixMillis()
     this.sma = new SimpleMovingAverage(10)
     this.processedLastSecond = 0
     this.progress = 0
@@ -213,7 +213,7 @@ export class ProgressLogger<T> implements Disposable {
 
     const mem = process.memoryUsage()
 
-    const now = Date.now() as UnixTimestampMillis
+    const now = localTime.nowUnixMillis()
     const batchedProgress = this.progress * chunkSize
     const lastRPS =
       (this.processedLastSecond * chunkSize) / ((now - this.lastSecondStarted) / 1000) || 0

package/src/stream/transform/transformMap.ts

@@ -1,7 +1,7 @@
 import { Transform } from 'node:stream'
 import { _hc } from '@naturalcycles/js-lib'
 import type { AbortableSignal } from '@naturalcycles/js-lib'
-import { _since } from '@naturalcycles/js-lib/datetime'
+import { _since, nowUnixMillis } from '@naturalcycles/js-lib/datetime'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
@@ -141,7 +141,7 @@ export function transformMap<IN = any, OUT = IN>(
     async transform(this: Transform, chunk: IN, _, cb) {
       // Initialize start time on first item
       if (started === 0) {
-        started = Date.now() as UnixTimestampMillis
+        started = nowUnixMillis()
         lastWarmupCheck = started
       }
 
@@ -262,7 +262,7 @@ export function transformMap<IN = any, OUT = IN>(
   })
 
   function updateConcurrency(): void {
-    const now = Date.now()
+    const now = nowUnixMillis()
     if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS) return
     lastWarmupCheck = now
 

package/src/stream/transform/transformMapSync.ts

@@ -1,8 +1,9 @@
 import { Transform } from 'node:stream'
 import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { nowUnixMillis } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
-import type { IndexedMapper, Predicate, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
+import type { IndexedMapper, Predicate } from '@naturalcycles/js-lib/types'
 import { END, SKIP } from '@naturalcycles/js-lib/types'
 import { yellow } from '../../colors/colors.js'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
@@ -78,7 +79,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     signal,
   } = opt
 
-  const started = Date.now() as UnixTimestampMillis
+  const started = nowUnixMillis()
   let index = -1
   let countOut = 0
   let isSettled = false

package/src/stream/transform/transformThrottle.ts

@@ -73,7 +73,7 @@ export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTy
         // console.log('pausing now after', {item, count})
         lock = pDefer()
         logger.log(
-          `${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`,
+          `${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_since(start)}, will pause for ${_ms(interval * 1000 - (localTime.nowUnixMillis() - start))}`,
         )
       }
 

package/src/stream/transform/transformThrottleByRSS.ts

@@ -87,7 +87,7 @@ export function transformThrottleByRSS<T>(
 
       if (!disabled && lastRSS > maxRSSBytes && !lock) {
         lock = pDefer()
-        pausedSince = Date.now()
+        pausedSince = localTime.nowUnixMillis()
         logger.log(
           `${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`,
         )
@@ -122,7 +122,7 @@ export function transformThrottleByRSS<T>(
       )
       lock!.resolve()
       lock = undefined
-    } else if (pollTimeout && Date.now() - pausedSince >= pollTimeout) {
+    } else if (pollTimeout && localTime.nowUnixMillis() - pausedSince >= pollTimeout) {
       clearTimeout(rssCheckTimer)
       if (onPollTimeout === 'throw') {
         lock!.reject(

package/src/stream/transform/worker/workerClassProxy.js

@@ -1,3 +1,4 @@
+// oxlint-disable-next-line no-restricted-globals -- the worker entry stays dependency-free
 const started = Date.now()
 import { inspect } from 'node:util'
 import { workerData, parentPort } from 'node:worker_threads'
@@ -20,6 +21,7 @@ try {
 const { WorkerClass } = await import(workerFile)
 const worker = new WorkerClass(workerData)
 
+// oxlint-disable-next-line no-restricted-globals -- the worker entry stays dependency-free
 log(`${metric}#${workerIndex} loaded in ${Date.now() - started} ms`)
 
 let errors = 0

package/src/validation/ajv/getAjv.ts

@@ -3,15 +3,14 @@ import { _assert } from '@naturalcycles/js-lib/error'
 import { _deepCopy, _mapObject, Set2 } from '@naturalcycles/js-lib/object'
 import { _substringAfterLast } from '@naturalcycles/js-lib/string'
 import type { AnyObject } from '@naturalcycles/js-lib/types'
-import type { Options, ValidateFunction } from 'ajv/dist/2020.js'
-import { Ajv2020 } from 'ajv/dist/2020.js'
+import type { Code, KeywordCxt, Options, ValidateFunction } from 'ajv/dist/2020.js'
+import { _, Ajv2020, str } from 'ajv/dist/2020.js'
 import { validTLDs } from '../tlds.js'
 import type {
   CustomConverterFn,
   CustomValidatorFn,
   JSchema,
   JsonSchemaIsoDateOptions,
-  JsonSchemaIsoMonthOptions,
   JsonSchemaStringEmailOptions,
   JsonSchemaUtcOffsetOptions,
 } from './jSchema.js'
@@ -310,140 +309,79 @@ export function createAjv(opt?: Options): Ajv2020 {
     },
   })
 
+  // The date/time keywords are `code` keywords, which Ajv inlines into the compiled validator.
+  // A `validate` keyword is a function call with a context object, whose instancePath Ajv builds
+  // on every call (escaping the key of every map entry), even for valid data.
   ajv.addKeyword({
     keyword: 'IsoDate',
     type: 'string',
-    modifying: false,
-    errors: true,
     schemaType: 'object',
-    validate: function validate(opt: JsonSchemaIsoDateOptions, data: string, _schema, ctx) {
-      const hasOptions = Object.keys(opt).length > 0
-
-      const isValid = isIsoDateValid(data)
-      if (!isValid) {
-        ;(validate as any).errors = [
-          {
-            instancePath: ctx?.instancePath ?? '',
-            message: `is an invalid IsoDate`,
-          },
-        ]
-
-        return false
-      }
-
-      if (!hasOptions) return true
-
-      const { before, sameOrBefore, after, sameOrAfter } = opt
-      const errors: any[] = []
-
-      if (before) {
-        const isParamValid = isIsoDateValid(before)
-        const isRuleValid = isParamValid && before > data
-
-        if (!isRuleValid) {
-          errors.push({
-            instancePath: ctx?.instancePath ?? '',
-            message: `is not before ${before}`,
-          })
-        }
-      }
-
-      if (sameOrBefore) {
-        const isParamValid = isIsoDateValid(sameOrBefore)
-        const isRuleValid = isParamValid && sameOrBefore >= data
-
-        if (!isRuleValid) {
-          errors.push({
-            instancePath: ctx?.instancePath ?? '',
-            message: `is not the same or before ${sameOrBefore}`,
-          })
-        }
-      }
-
-      if (after) {
-        const isParamValid = isIsoDateValid(after)
-        const isRuleValid = isParamValid && after < data
-
-        if (!isRuleValid) {
-          errors.push({
-            instancePath: ctx?.instancePath ?? '',
-            message: `is not after ${after}`,
-          })
-        }
-      }
-
-      if (sameOrAfter) {
-        const isParamValid = isIsoDateValid(sameOrAfter)
-        const isRuleValid = isParamValid && sameOrAfter <= data
-
-        if (!isRuleValid) {
-          errors.push({
-            instancePath: ctx?.instancePath ?? '',
-            message: `is not the same or after ${sameOrAfter}`,
-          })
-        }
-      }
-
-      if (errors.length === 0) return true
-      ;(validate as any).errors = errors
-      return false
+    error: {
+      message: ({ params }) =>
+        params['rule']
+          ? str`is not ${isoDateRuleLabels[params['rule'] as IsoDateRule]} ${params['limit']}`
+          : str`is an invalid IsoDate`,
+      params: ({ params }) => (params['rule'] ? _`{${params['rule']}: ${params['limit']}}` : _`{}`),
+    },
+    code(cxt: KeywordCxt) {
+      const { gen, data } = cxt
+      const opt: JsonSchemaIsoDateOptions = cxt.schema
+      const isValid = gen.scopeValue('func', { ref: isIsoDateValid })
+
+      gen.if(
+        _`!${isValid}(${data})`,
+        () => cxt.error(),
+        () => {
+          for (const rule of isoDateRules) {
+            const limit = opt[rule]
+            if (!limit) continue
+
+            const params = { rule, limit }
+            if (!isIsoDateValid(limit)) {
+              // An invalid limit rejects every date
+              cxt.error(false, params)
+              continue
+            }
+            gen.if(isoDateRuleFails(rule, data, limit), () => cxt.error(false, params))
+          }
+        },
+      )
     },
   })
 
   ajv.addKeyword({
     keyword: 'IsoDateTime',
     type: 'string',
-    modifying: false,
-    errors: true,
     schemaType: 'boolean',
-    validate: function validate(_opt: JsonSchemaIsoMonthOptions, data: string, _schema, ctx) {
-      const isValid = isIsoDateTimeValid(data)
-      if (isValid) return true
-      ;(validate as any).errors = [
-        {
-          instancePath: ctx?.instancePath ?? '',
-          message: `is an invalid IsoDateTime`,
-        },
-      ]
-      return false
+    error: { message: 'is an invalid IsoDateTime' },
+    code(cxt: KeywordCxt) {
+      const isValid = cxt.gen.scopeValue('func', { ref: isIsoDateTimeValid })
+      cxt.fail(_`!${isValid}(${cxt.data})`)
     },
   })
 
   ajv.addKeyword({
     keyword: 'IsoMonth',
     type: 'string',
-    modifying: false,
-    errors: true,
     schemaType: 'object',
-    validate: function validate(_opt: true, data: string, _schema, ctx) {
-      const isValid = isIsoMonthValid(data)
-      if (isValid) return true
-      ;(validate as any).errors = [
-        {
-          instancePath: ctx?.instancePath ?? '',
-          message: `is an invalid IsoMonth`,
-        },
-      ]
-      return false
+    error: { message: 'is an invalid IsoMonth' },
+    code(cxt: KeywordCxt) {
+      const isValid = cxt.gen.scopeValue('func', { ref: isIsoMonthValid })
+      cxt.fail(_`!${isValid}(${cxt.data})`)
     },
   })
 
   ajv.addKeyword({
     keyword: 'UtcOffset',
     type: 'string',
-    modifying: false,
-    errors: true,
     schemaType: 'object',
-    validate: function validate(opt: JsonSchemaUtcOffsetOptions, data: string, _schema, ctx) {
-      const isValid = data === 'Z' ? opt.allowZ : data !== '' && isIsoTimezoneValid(data)
-      if (isValid) return true
-      ;(validate as any).errors = [
-        {
-          instancePath: ctx?.instancePath ?? '',
-          message: `is an invalid UtcOffset`,
-        },
-      ]
-      return false
+    error: { message: 'is an invalid UtcOffset' },
+    code(cxt: KeywordCxt) {
+      const opt: JsonSchemaUtcOffsetOptions = cxt.schema
+      const isValid = cxt.gen.scopeValue('func', {
+        ref: opt.allowZ ? isUtcOffsetOrZValid : isUtcOffsetValid,
+      })
+      cxt.fail(_`!${isValid}(${cxt.data})`)
     },
   })
 
@@ -496,39 +434,69 @@ export function createAjv(opt?: Options): Ajv2020 {
     },
   })
 
+  // Strips the keys that don't match the key schema; the values are validated by `additionalProperties`.
+  // The key schema is inlined per key with error reporting off, and the error count is reset after
+  // a mismatch, so a stripped key never surfaces as an error. Keys listed in `properties` are skipped.
   ajv.addKeyword({
     keyword: 'keySchema',
     type: 'object',
-    modifying: true,
-    errors: false,
     schemaType: 'object',
-    compile(innerSchema, _parentSchema, _it) {
-      const isValidKeyFn: ValidateFunction = ajv.compile(innerSchema)
+    trackErrors: true,
+    code(cxt: KeywordCxt) {
+      const { gen, data, parentSchema } = cxt
+      const propertyNames = Object.keys(parentSchema['properties'] || {})
+      const valid = gen.name('valid')
+
+      gen.forIn('key', data, key => {
+        const checkKey = (): void => {
+          cxt.subschema(
+            {
+              keyword: 'keySchema',
+              data: key,
+              dataTypes: ['string'],
+              compositeRule: true,
+              createErrors: false,
+              allErrors: false,
+            },
+            valid,
+          )
+          gen.if(_`!${valid}`, () => {
+            cxt.reset()
+            gen.code(_`delete ${data}[${key}]`)
+          })
+        }
 
-      function validate(data: AnyObject, _ctx: any): boolean {
-        if (typeof data !== 'object' || data === null) return true
+        if (!propertyNames.length) return checkKey()
 
-        for (const key of Object.keys(data)) {
-          if (!isValidKeyFn(key)) {
-            delete data[key]
-          }
+        let isNotProperty: Code = _`${key} !== ${propertyNames[0]}`
+        for (const name of propertyNames.slice(1)) {
+          isNotProperty = _`${isNotProperty} && ${key} !== ${name}`
         }
+        gen.if(isNotProperty, checkKey)
+      })
+    },
+  })
 
-        return true
-      }
-
-      return validate
+  // Validates the inner schema only when the value is defined. Used for the optional values of
+  // `stringMap`/`record`, whose `additionalProperties` loop visits the keys with undefined values too.
+  ajv.addKeyword({
+    keyword: 'ifDefined',
+    schemaType: ['object', 'boolean'],
+    code(cxt: KeywordCxt) {
+      const { gen, data } = cxt
+      gen.if(_`${data} !== undefined`, () => {
+        cxt.subschema({ keyword: 'ifDefined' }, gen.name('valid'))
+      })
     },
   })
 
-  // Validates that the value is undefined. Used in record/stringMap with optional value schemas
-  // to allow undefined values in patternProperties via anyOf.
+  // Validates that the value is undefined. Superseded by `ifDefined`, kept for raw JSON schemas.
   ajv.addKeyword({
     keyword: 'isUndefined',
-    modifying: false,
-    errors: false,
     schemaType: 'boolean',
-    validate: (_schema: boolean, data: unknown) => data === undefined,
+    code(cxt: KeywordCxt) {
+      cxt.fail(_`${cxt.data} !== undefined`)
+    },
   })
 
   // This is added because Ajv validates the `min/maxProperties` before validating the properties.
@@ -777,6 +745,27 @@ export function createAjv(opt?: Options): Ajv2020 {
   return ajv
 }
 
+type IsoDateRule = keyof JsonSchemaIsoDateOptions
+
+const isoDateRules: IsoDateRule[] = ['before', 'sameOrBefore', 'after', 'sameOrAfter']
+
+const isoDateRuleLabels: Record<IsoDateRule, string> = {
+  before: 'before',
+  sameOrBefore: 'the same or before',
+  after: 'after',
+  sameOrAfter: 'the same or after',
+}
+
+/**
+ * IsoDate strings compare chronologically as plain strings.
+ */
+function isoDateRuleFails(rule: IsoDateRule, data: Code, limit: string): Code {
+  if (rule === 'before') return _`${data} >= ${limit}`
+  if (rule === 'sameOrBefore') return _`${data} > ${limit}`
+  if (rule === 'after') return _`${data} <= ${limit}`
+  return _`${data} < ${limit}`
+}
+
 const monthLengths = [0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
 
 const DASH_CODE = '-'.charCodeAt(0)
@@ -904,6 +893,14 @@ function isIsoTimezoneValid(s: string): boolean {
   return true
 }
 
+function isUtcOffsetValid(s: string): boolean {
+  return s !== 'Z' && s !== '' && isIsoTimezoneValid(s)
+}
+
+function isUtcOffsetOrZValid(s: string): boolean {
+  return s === 'Z' || (s !== '' && isIsoTimezoneValid(s))
+}
+
 /**
  * This is a performance optimized correct validation
  * for ISO month formatted as "YYYY-MM".