@naturalcycles/nodejs-lib 13.23.0 → 13.24.0

package/dist/fs/fs2.d.ts

@@ -4,7 +4,7 @@
 import type { RmOptions } from 'node:fs';
 import fs from 'node:fs';
 import { DumpOptions } from 'js-yaml';
-import { ReadableTyped, WritableTyped } from '../stream/stream.model';
+import { ReadableTyped, TransformTyped } from '../stream/stream.model';
 /**
  * fs2 conveniently groups filesystem functions together.
  * Supposed to be almost a drop-in replacement for these things together:
@@ -77,7 +77,7 @@ declare class FS2 {
     createWriteStream: typeof fs.createWriteStream;
     createReadStream: typeof fs.createReadStream;
     createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;
-    createWriteStreamAsNDJSON(outputPath: string): WritableTyped<any>;
+    createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];
 }
 export declare const fs2: FS2;
 export interface JsonOptions {

package/dist/fs/fs2.js

@@ -311,7 +311,8 @@ class FS2 {
         // .pipe(transformJsonParse<ROW>())
     }
     /*
-    Returns a Writable.
+    Returns an array of Transforms, so that you can ...destructure them at
+    the end of the _pipeline.
   
     Replaces a list of operations:
     - transformToNDJson
@@ -320,17 +321,17 @@ class FS2 {
      */
     createWriteStreamAsNDJSON(outputPath) {
         this.ensureFile(outputPath);
-        const transform1 = (0, transformToNDJson_1.transformToNDJson)();
-        let transform = transform1;
-        if (outputPath.endsWith('.gz')) {
-            transform = transform.pipe((0, node_zlib_1.createGzip)({
-            // chunkSize: 64 * 1024, // no observed speedup
-            }));
-        }
-        transform.pipe(node_fs_1.default.createWriteStream(outputPath, {
-        // highWaterMark: 64 * 1024, // no observed speedup
-        }));
-        return transform1;
+        return [
+            (0, transformToNDJson_1.transformToNDJson)(),
+            outputPath.endsWith('.gz')
+                ? (0, node_zlib_1.createGzip)({
+                // chunkSize: 64 * 1024, // no observed speedup
+                })
+                : undefined,
+            node_fs_1.default.createWriteStream(outputPath, {
+            // highWaterMark: 64 * 1024, // no observed speedup
+            }),
+        ].filter(js_lib_1._isTruthy);
     }
 }
 exports.fs2 = new FS2();

package/dist/stream/ndjson/ndjsonMap.js

@@ -26,7 +26,7 @@ async function ndjsonMap(mapper, opt) {
         }),
         (0, __1.transformLimit)({ limit: limitOutput, sourceReadable: readable }),
         (0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }),
-        __1.fs2.createWriteStreamAsNDJSON(outputFilePath),
+        ...__1.fs2.createWriteStreamAsNDJSON(outputFilePath),
     ]);
 }
 exports.ndjsonMap = ndjsonMap;

package/dist/validation/joi/string.extensions.js

@@ -41,14 +41,14 @@ function stringExtensions(joi) {
                     let { min, max } = args;
                     // Today allows +-14 hours gap to account for different timezones
                     if (max === 'today') {
-                        max = (0, js_lib_1.localTimeNow)().plus(14, 'hour').toISODate();
+                        max = getTodayStrPlus15();
                     }
                     if (min === 'today') {
-                        min = (0, js_lib_1.localTimeNow)().minus(14, 'hour').toISODate();
+                        min = getTodayStrMinus15();
                     }
                     // console.log('min/max', min, max)
-                    const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v);
-                    if (!m || m.length <= 1) {
+                    const parts = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v);
+                    if (!parts || parts.length < 4) {
                         err = 'string.dateString';
                     }
                     else if (min && v < min) {
@@ -57,8 +57,7 @@ function stringExtensions(joi) {
                     else if (max && v > max) {
                         err = 'string.dateStringMax';
                     }
-                    else if (!js_lib_1.LocalDate.isValid(v)) {
-                        // todo: replace with another regex (from ajv-validators) for speed
+                    else if (!isValidDate(parts)) {
                         err = 'string.dateStringCalendarAccuracy';
                     }
                     if (err) {
@@ -71,3 +70,39 @@ function stringExtensions(joi) {
     };
 }
 exports.stringExtensions = stringExtensions;
+const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+// Based on: https://github.com/ajv-validator
+function isValidDate(parts) {
+    const year = Number(parts[1]);
+    const month = Number(parts[2]);
+    const day = Number(parts[3]);
+    return (month >= 1 &&
+        month <= 12 &&
+        day >= 1 &&
+        day <= (month === 2 && isLeapYear(year) ? 29 : DAYS[month]));
+}
+function isLeapYear(year) {
+    return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+}
+let lastCheckedPlus = 0;
+let todayStrPlusCached;
+let lastCheckedMinus = 0;
+let todayStrMinusCached;
+function getTodayStrPlus15() {
+    const now = Date.now();
+    if (now - lastCheckedPlus < 3_600_000) {
+        // cached for 1 hour
+        return todayStrPlusCached;
+    }
+    lastCheckedPlus = now;
+    return (todayStrPlusCached = (0, js_lib_1.localTimeNow)().plus(15, 'hour').toISODate());
+}
+function getTodayStrMinus15() {
+    const now = Date.now();
+    if (now - lastCheckedMinus < 3_600_000) {
+        // cached for 1 hour
+        return todayStrMinusCached;
+    }
+    lastCheckedMinus = now;
+    return (todayStrMinusCached = (0, js_lib_1.localTimeNow)().plus(-15, 'hour').toISODate());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.23.0",
+  "version": "13.24.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/fs/fs2.ts

@@ -19,10 +19,10 @@ import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import path from 'node:path'
 import { createGzip, createUnzip } from 'node:zlib'
-import { _jsonParse } from '@naturalcycles/js-lib'
+import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib'
 import yaml, { DumpOptions } from 'js-yaml'
 import { transformToNDJson } from '../stream/ndjson/transformToNDJson'
-import { ReadableTyped, WritableTyped } from '../stream/stream.model'
+import { ReadableTyped, TransformTyped } from '../stream/stream.model'
 import { transformSplitOnNewline } from '../stream/transform/transformSplit'
 import { requireFileToExist } from '../util/env.util'
 
@@ -353,31 +353,28 @@ class FS2 {
   }
 
   /*
-  Returns a Writable.
+  Returns an array of Transforms, so that you can ...destructure them at
+  the end of the _pipeline.
 
   Replaces a list of operations:
   - transformToNDJson
   - createGzip (only if path ends with '.gz')
   - fs.createWriteStream
    */
-  createWriteStreamAsNDJSON(outputPath: string): WritableTyped<any> {
+  createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
     this.ensureFile(outputPath)
 
-    const transform1 = transformToNDJson()
-    let transform = transform1
-    if (outputPath.endsWith('.gz')) {
-      transform = transform.pipe(
-        createGzip({
-          // chunkSize: 64 * 1024, // no observed speedup
-        }),
-      )
-    }
-    transform.pipe(
+    return [
+      transformToNDJson(),
+      outputPath.endsWith('.gz')
+        ? createGzip({
+            // chunkSize: 64 * 1024, // no observed speedup
+          })
+        : undefined,
       fs.createWriteStream(outputPath, {
         // highWaterMark: 64 * 1024, // no observed speedup
       }),
-    )
-    return transform1
+    ].filter(_isTruthy) as TransformTyped<any, any>[]
   }
 }
 

package/src/stream/ndjson/ndjsonMap.ts

@@ -60,6 +60,6 @@ export async function ndjsonMap<IN = any, OUT = any>(
     }),
     transformLimit({ limit: limitOutput, sourceReadable: readable }),
     transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
-    fs2.createWriteStreamAsNDJSON(outputFilePath),
+    ...fs2.createWriteStreamAsNDJSON(outputFilePath),
   ])
 }

package/src/validation/joi/string.extensions.ts

@@ -1,4 +1,4 @@
-import { LocalDate, localTimeNow } from '@naturalcycles/js-lib'
+import { localTimeNow } from '@naturalcycles/js-lib'
 import Joi, { Extension, StringSchema as JoiStringSchema } from 'joi'
 
 export interface StringSchema<TSchema = string> extends JoiStringSchema<TSchema> {
@@ -51,22 +51,21 @@ export function stringExtensions(joi: typeof Joi): Extension {
 
           // Today allows +-14 hours gap to account for different timezones
           if (max === 'today') {
-            max = localTimeNow().plus(14, 'hour').toISODate()
+            max = getTodayStrPlus15()
           }
           if (min === 'today') {
-            min = localTimeNow().minus(14, 'hour').toISODate()
+            min = getTodayStrMinus15()
           }
           // console.log('min/max', min, max)
 
-          const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v)
-          if (!m || m.length <= 1) {
+          const parts = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v)
+          if (!parts || parts.length < 4) {
             err = 'string.dateString'
           } else if (min && v < min) {
             err = 'string.dateStringMin'
           } else if (max && v > max) {
             err = 'string.dateStringMax'
-          } else if (!LocalDate.isValid(v)) {
-            // todo: replace with another regex (from ajv-validators) for speed
+          } else if (!isValidDate(parts)) {
             err = 'string.dateStringCalendarAccuracy'
           }
 
@@ -80,3 +79,48 @@ export function stringExtensions(joi: typeof Joi): Extension {
     },
   }
 }
+
+const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+// Based on: https://github.com/ajv-validator
+function isValidDate(parts: string[]): boolean {
+  const year = Number(parts[1])
+  const month = Number(parts[2])
+  const day = Number(parts[3])
+  return (
+    month >= 1 &&
+    month <= 12 &&
+    day >= 1 &&
+    day <= (month === 2 && isLeapYear(year) ? 29 : DAYS[month]!)
+  )
+}
+
+function isLeapYear(year: number): boolean {
+  return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0)
+}
+
+let lastCheckedPlus = 0
+let todayStrPlusCached: string
+let lastCheckedMinus = 0
+let todayStrMinusCached: string
+
+function getTodayStrPlus15(): string {
+  const now = Date.now()
+  if (now - lastCheckedPlus < 3_600_000) {
+    // cached for 1 hour
+    return todayStrPlusCached
+  }
+
+  lastCheckedPlus = now
+  return (todayStrPlusCached = localTimeNow().plus(15, 'hour').toISODate())
+}
+
+function getTodayStrMinus15(): string {
+  const now = Date.now()
+  if (now - lastCheckedMinus < 3_600_000) {
+    // cached for 1 hour
+    return todayStrMinusCached
+  }
+
+  lastCheckedMinus = now
+  return (todayStrMinusCached = localTimeNow().plus(-15, 'hour').toISODate())
+}