@naturalcycles/nodejs-lib 13.4.0 → 13.6.0

package/src/fs/del.ts

@@ -1,8 +1,6 @@
-import fs from 'node:fs'
-import fsp from 'node:fs/promises'
 import { pFilter, pMap, _since } from '@naturalcycles/js-lib'
 import { dimGrey, yellow } from '../colors/colors'
-import { _pathExists, _pathExistsSync, globby } from '../index'
+import { fs2, globby } from '../index'
 
 export interface DelOptions {
   /**
@@ -71,14 +69,7 @@ export async function del(_opt: DelOptions | DelSingleOption): Promise<void> {
 
   if (dry) return
 
-  await pMap(
-    filenames,
-    filepath =>
-      fsp.rm(filepath, {
-        force: true,
-      }),
-    { concurrency },
-  )
+  await pMap(filenames, filepath => fs2.removePath(filepath), { concurrency })
 
   // 2. glob only dirs, expand, delete only empty!
   let dirnames = await globby(patterns, {
@@ -90,7 +81,7 @@ export async function del(_opt: DelOptions | DelSingleOption): Promise<void> {
   // Add original patterns (if any of them are dirs)
   dirnames = dirnames.concat(
     await pFilter(patterns, async pattern => {
-      return (await _pathExists(pattern)) && (await fsp.lstat(pattern)).isDirectory()
+      return (await fs2.pathExistsAsync(pattern)) && (await fs2.lstatAsync(pattern)).isDirectory()
     }),
   )
 
@@ -102,7 +93,7 @@ export async function del(_opt: DelOptions | DelSingleOption): Promise<void> {
   for await (const dirpath of dirnamesSorted) {
     if (await isEmptyDir(dirpath)) {
       // console.log(`empty dir: ${dirpath}`)
-      await fsp.rm(dirpath, { force: true, recursive: true })
+      await fs2.removePathAsync(dirpath)
       deletedDirs.push(dirpath)
     }
   }
@@ -152,7 +143,7 @@ export function delSync(_opt: DelOptions | DelSingleOption): void {
 
   if (dry) return
 
-  filenames.forEach(filepath => fs.rmSync(filepath, { force: true }))
+  filenames.forEach(filepath => fs2.removePath(filepath))
 
   // 2. glob only dirs, expand, delete only empty!
   let dirnames = globby.sync(patterns, {
@@ -162,9 +153,7 @@ export function delSync(_opt: DelOptions | DelSingleOption): void {
   })
 
   // Add original patterns (if any of them are dirs)
-  dirnames = dirnames.concat(
-    patterns.filter(p => _pathExistsSync(p) && fs.lstatSync(p).isDirectory()),
-  )
+  dirnames = dirnames.concat(patterns.filter(p => fs2.pathExists(p) && fs2.lstat(p).isDirectory()))
 
   const dirnamesSorted = dirnames.sort().reverse()
 
@@ -174,7 +163,7 @@ export function delSync(_opt: DelOptions | DelSingleOption): void {
   for (const dirpath of dirnamesSorted) {
     if (isEmptyDirSync(dirpath)) {
       // console.log(`empty dir: ${dirpath}`)
-      fs.rmSync(dirpath, { force: true, recursive: true })
+      fs2.removePath(dirpath)
       deletedDirs.push(dirpath)
     }
   }
@@ -196,9 +185,9 @@ export function delSync(_opt: DelOptions | DelSingleOption): void {
 // 3. test each original pattern, if it exists and is directory and is empty - delete
 
 async function isEmptyDir(dir: string): Promise<boolean> {
-  return (await fsp.readdir(dir)).length === 0
+  return (await fs2.readdirAsync(dir)).length === 0
 }
 
 function isEmptyDirSync(dir: string): boolean {
-  return fs.readdirSync(dir).length === 0
+  return fs2.readdir(dir).length === 0
 }

package/src/fs/fs2.ts

@@ -0,0 +1,303 @@
+/*
+
+Why?
+
+Convenience re-export/re-implementation of most common fs functions from
+node:fs, node:fs/promises and fs-extra
+
+Defaults to string input/output, as it's used in 80% of time. For the rest - you can use native fs.
+
+Allows to import it easier, so you don't have to choose between the 3 import locations.
+That's why function names are slightly renamed, to avoid conflict.
+
+Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
+
+ */
+
+import type { RmOptions } from 'node:fs'
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
+import path from 'node:path'
+import { _jsonParse } from '@naturalcycles/js-lib'
+import yaml, { DumpOptions } from 'js-yaml'
+
+/**
+ * fs2 conveniently groups filesystem functions together.
+ * Supposed to be almost a drop-in replacement for these things together:
+ *
+ * 1. node:fs
+ * 2. node:fs/promises
+ * 3. fs-extra
+ */
+class FS2 {
+  // Naming convention is:
+  // - async function has Async in the name, e.g readTextAsync
+  // - sync function has postfix in the name, e.g readText
+
+  /**
+   * Convenience wrapper that defaults to utf-8 string output.
+   */
+  readText(filePath: string): string {
+    return fs.readFileSync(filePath, 'utf8')
+  }
+
+  /**
+   * Convenience wrapper that defaults to utf-8 string output.
+   */
+  async readTextAsync(filePath: string): Promise<string> {
+    return await fsp.readFile(filePath, 'utf8')
+  }
+
+  readBuffer(filePath: string): Buffer {
+    return fs.readFileSync(filePath)
+  }
+
+  async readBufferAsync(filePath: string): Promise<Buffer> {
+    return await fsp.readFile(filePath)
+  }
+
+  readJson<T = unknown>(filePath: string): T {
+    const str = fs.readFileSync(filePath, 'utf8')
+    return _jsonParse(str)
+  }
+
+  async readJsonAsync<T = unknown>(filePath: string): Promise<T> {
+    const str = await fsp.readFile(filePath, 'utf8')
+    return _jsonParse(str)
+  }
+
+  readYaml<T = unknown>(filePath: string): T {
+    return yaml.load(fs.readFileSync(filePath, 'utf8')) as T
+  }
+
+  async readYamlAsync<T = unknown>(filePath: string): Promise<T> {
+    return yaml.load(await fsp.readFile(filePath, 'utf8')) as T
+  }
+
+  writeFile(filePath: string, data: string | Buffer): void {
+    fs.writeFileSync(filePath, data)
+  }
+
+  async writeFileAsync(filePath: string, data: string | Buffer): Promise<void> {
+    await fsp.writeFile(filePath, data)
+  }
+
+  writeJson(filePath: string, data: any, opt?: JsonOptions): void {
+    const str = stringify(data, opt)
+    fs.writeFileSync(filePath, str)
+  }
+
+  async writeJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void> {
+    const str = stringify(data, opt)
+    await fsp.writeFile(filePath, str)
+  }
+
+  writeYaml(filePath: string, data: any, opt?: DumpOptions): void {
+    const str = yaml.dump(data, opt)
+    fs.writeFileSync(filePath, str)
+  }
+
+  async writeYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
+    const str = yaml.dump(data, opt)
+    await fsp.writeFile(filePath, str)
+  }
+
+  outputJson(filePath: string, data: any, opt?: JsonOptions): void {
+    const str = stringify(data, opt)
+    this.outputFile(filePath, str)
+  }
+
+  async outputJsonAsync(filePath: string, data: any, opt?: JsonOptions): Promise<void> {
+    const str = stringify(data, opt)
+    await this.outputFileAsync(filePath, str)
+  }
+
+  outputYaml(filePath: string, data: any, opt?: DumpOptions): void {
+    const str = yaml.dump(data, opt)
+    this.outputFile(filePath, str)
+  }
+
+  async outputYamlAsync(filePath: string, data: any, opt?: DumpOptions): Promise<void> {
+    const str = yaml.dump(data, opt)
+    await this.outputFileAsync(filePath, str)
+  }
+
+  outputFile(filePath: string, data: string | Buffer): void {
+    const dirPath = path.dirname(filePath)
+    if (!fs.existsSync(dirPath)) {
+      this.ensureDir(dirPath)
+    }
+
+    fs.writeFileSync(filePath, data)
+  }
+
+  async outputFileAsync(filePath: string, data: string | Buffer): Promise<void> {
+    const dirPath = path.dirname(filePath)
+    if (!(await this.pathExistsAsync(dirPath))) {
+      await this.ensureDirAsync(dirPath)
+    }
+
+    await fsp.writeFile(filePath, data)
+  }
+
+  pathExists(filePath: string): boolean {
+    return fs.existsSync(filePath)
+  }
+
+  async pathExistsAsync(filePath: string): Promise<boolean> {
+    try {
+      await fsp.access(filePath)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  ensureDir(dirPath: string): void {
+    fs.mkdirSync(dirPath, {
+      mode: 0o777,
+      recursive: true,
+    })
+  }
+
+  async ensureDirAsync(dirPath: string): Promise<void> {
+    await fsp.mkdir(dirPath, {
+      mode: 0o777,
+      recursive: true,
+    })
+  }
+
+  ensureFile(filePath: string): void {
+    let stats
+    try {
+      stats = fs.statSync(filePath)
+    } catch {}
+    if (stats?.isFile()) return
+
+    const dir = path.dirname(filePath)
+    try {
+      if (!fs.statSync(dir).isDirectory()) {
+        // parent is not a directory
+        // This is just to cause an internal ENOTDIR error to be thrown
+        fs.readdirSync(dir)
+      }
+    } catch (err) {
+      // If the stat call above failed because the directory doesn't exist, create it
+      if ((err as any)?.code === 'ENOENT') return this.ensureDir(dir)
+      throw err
+    }
+
+    fs.writeFileSync(filePath, '')
+  }
+
+  async ensureFileAsync(filePath: string): Promise<void> {
+    let stats
+    try {
+      stats = await fsp.stat(filePath)
+    } catch {}
+    if (stats?.isFile()) return
+
+    const dir = path.dirname(filePath)
+    try {
+      if (!(await fsp.stat(dir)).isDirectory()) {
+        // parent is not a directory
+        // This is just to cause an internal ENOTDIR error to be thrown
+        await fsp.readdir(dir)
+      }
+    } catch (err) {
+      // If the stat call above failed because the directory doesn't exist, create it
+      if ((err as any)?.code === 'ENOENT') return await this.ensureDirAsync(dir)
+      throw err
+    }
+
+    await fsp.writeFile(filePath, '')
+  }
+
+  removePath(fileOrDirPath: string, opt?: RmOptions): void {
+    fs.rmSync(fileOrDirPath, { recursive: true, force: true, ...opt })
+  }
+
+  async removePathAsync(fileOrDirPath: string, opt?: RmOptions): Promise<void> {
+    await fsp.rm(fileOrDirPath, { recursive: true, force: true, ...opt })
+  }
+
+  emptyDir(dirPath: string): void {
+    let items
+    try {
+      items = fs.readdirSync(dirPath)
+    } catch {
+      return this.ensureDir(dirPath)
+    }
+
+    items.forEach(item => this.removePath(path.join(dirPath, item)))
+  }
+
+  async emptyDirAsync(dirPath: string): Promise<void> {
+    let items
+    try {
+      items = await fsp.readdir(dirPath)
+    } catch {
+      return await this.ensureDirAsync(dirPath)
+    }
+
+    await Promise.all(items.map(item => this.removePathAsync(path.join(dirPath, item))))
+  }
+
+  /**
+   * Cautious, underlying Node function is currently Experimental.
+   */
+  copyPath(src: string, dest: string, opt?: fs.CopySyncOptions): void {
+    fs.cpSync(src, dest, {
+      recursive: true,
+      ...opt,
+    })
+  }
+
+  /**
+   * Cautious, underlying Node function is currently Experimental.
+   */
+  async copyPathAsync(src: string, dest: string, opt?: fs.CopyOptions): Promise<void> {
+    await fsp.cp(src, dest, {
+      recursive: true,
+      ...opt,
+    })
+  }
+
+  movePath(src: string, dest: string, opt?: fs.CopySyncOptions): void {
+    this.copyPath(src, dest, opt)
+    this.removePath(src)
+  }
+
+  async movePathAsync(src: string, dest: string, opt?: fs.CopyOptions): Promise<void> {
+    await this.copyPathAsync(src, dest, opt)
+    await this.removePathAsync(src)
+  }
+
+  // Re-export the whole fs/fsp, for the edge cases where they are needed
+  fs = fs
+  fsp = fsp
+
+  // Re-export existing fs/fsp functions
+  // rm/rmAsync are replaced with removePath/removePathAsync
+  lstat = fs.lstatSync
+  lstatAsync = fsp.lstat
+  stat = fs.statSync
+  statAsync = fsp.stat
+  mkdir = fs.mkdirSync
+  mkdirAsync = fsp.mkdir
+  readdir = fs.readdirSync
+  readdirAsync = fsp.readdir
+  createWriteStream = fs.createWriteStream
+  createReadStream = fs.createReadStream
+}
+
+export const fs2 = new FS2()
+
+export interface JsonOptions {
+  spaces?: number
+}
+
+function stringify(data: any, opt?: JsonOptions): string {
+  // If pretty-printing is enabled (spaces) - also add a newline at the end (to match our prettier config)
+  return JSON.stringify(data, null, opt?.spaces) + (opt?.spaces ? '\n' : '')
+}

package/src/fs/json2env.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import { AnyObject } from '@naturalcycles/js-lib'
 import { dimGrey } from '../colors/colors'
-import { _pathExistsSync, _readJsonSync, _writeFileSync } from './fs.util'
+import { fs2 } from './fs2'
 
 export interface Json2EnvOptions {
   jsonPath: string
@@ -46,7 +46,7 @@ export function json2env(opt: Json2EnvOptions): void {
     ...opt,
   }
 
-  if (!_pathExistsSync(jsonPath)) {
+  if (!fs2.pathExists(jsonPath)) {
     if (fail) {
       throw new Error(`Path doesn't exist: ${jsonPath}`)
     }
@@ -63,7 +63,7 @@ export function json2env(opt: Json2EnvOptions): void {
   }
 
   // read file
-  const json: AnyObject = _readJsonSync(jsonPath)
+  const json: AnyObject = fs2.readJson(jsonPath)
 
   if (debug) {
     console.log(json)
@@ -72,7 +72,7 @@ export function json2env(opt: Json2EnvOptions): void {
   if (saveEnvFile) {
     const shPath = `${jsonPath}.sh`
     const exportStr = objectToShellExport(json, prefix)
-    _writeFileSync(shPath, exportStr)
+    fs2.writeFile(shPath, exportStr)
 
     if (!silent) {
       console.log(`json2env created ${dimGrey(shPath)}:`)

package/src/fs/kpy.ts

@@ -1,15 +1,7 @@
 import path from 'node:path'
 import { _since } from '@naturalcycles/js-lib'
 import { boldWhite, dimGrey, grey, yellow } from '../colors/colors'
-import {
-  _copyPath,
-  _copyPathSync,
-  _ensureDirSync,
-  _movePath,
-  _movePathSync,
-  _pathExistsSync,
-  globby,
-} from '../index'
+import { fs2, globby } from '../index'
 
 /**
  * Everything defaults to `undefined`.
@@ -72,11 +64,11 @@ export async function kpy(opt: KpyOptions): Promise<void> {
 
       if (!opt.dry) {
         if (opt.move) {
-          await _movePath(srcFilename, destFilename, {
+          await fs2.movePathAsync(srcFilename, destFilename, {
             force: overwrite,
           })
         } else {
-          await _copyPath(srcFilename, destFilename, { force: overwrite })
+          await fs2.copyPathAsync(srcFilename, destFilename, { force: overwrite })
         }
       }
 
@@ -110,9 +102,9 @@ export function kpySync(opt: KpyOptions): void {
 
     if (!opt.dry) {
       if (opt.move) {
-        _movePathSync(srcFilename, destFilename, { force: overwrite })
+        fs2.movePath(srcFilename, destFilename, { force: overwrite })
       } else {
-        _copyPathSync(srcFilename, destFilename, { force: overwrite })
+        fs2.copyPath(srcFilename, destFilename, { force: overwrite })
       }
     }
 
@@ -132,12 +124,12 @@ function kpyPrepare(opt: KpyOptions): void {
   opt.baseDir ||= '.'
   opt.outputDir ||= '.'
 
-  if (!_pathExistsSync(opt.baseDir)) {
+  if (!fs2.pathExists(opt.baseDir)) {
     console.log(`kpy: baseDir doesn't exist: ${boldWhite(opt.baseDir)}`)
     return
   }
 
-  _ensureDirSync(opt.outputDir)
+  fs2.ensureDir(opt.outputDir)
 }
 
 function kpyLogFilenames(opt: KpyOptions, filenames: string[]): void {

package/src/index.ts

@@ -86,7 +86,7 @@ export * from './validation/joi/joi.validation.error'
 export * from './validation/joi/joi.validation.util'
 export * from './script/runScript'
 export * from './jwt/jwt.service'
-export * from './fs/fs.util'
+export * from './fs/fs2'
 export * from './fs/del'
 export * from './fs/json2env'
 export * from './fs/kpy'

package/src/secret/secrets-decrypt.util.ts

@@ -2,7 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { _assert } from '@naturalcycles/js-lib'
 import { dimGrey, yellow } from '../colors/colors'
-import { _readJsonSync, _writeJsonSync, fastGlob } from '../index'
+import { fastGlob, fs2 } from '../index'
 import { decryptObject, decryptRandomIVBuffer } from '../security/crypto.util'
 
 export interface DecryptCLIOptions {
@@ -47,9 +47,9 @@ export function secretsDecrypt(
       )
       plainFilename = filename.replace('.json', '.plain.json')
 
-      const json = decryptObject(_readJsonSync(filename), encKeyBuffer)
+      const json = decryptObject(fs2.readJson(filename), encKeyBuffer)
 
-      _writeJsonSync(plainFilename, json, { spaces: 2 })
+      fs2.writeJson(plainFilename, json, { spaces: 2 })
     } else {
       const enc = fs.readFileSync(filename)
       const plain = decryptRandomIVBuffer(enc, encKeyBuffer)

package/src/secret/secrets-encrypt.util.ts

@@ -2,7 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { _assert } from '@naturalcycles/js-lib'
 import { dimGrey, yellow } from '../colors/colors'
-import { _readJsonSync, _writeJsonSync, fastGlob } from '../index'
+import { fastGlob, fs2 } from '../index'
 import { encryptObject, encryptRandomIVBuffer } from '../security/crypto.util'
 
 export interface EncryptCLIOptions {
@@ -41,9 +41,9 @@ export function secretsEncrypt(
       )
       encFilename = filename.replace('.plain', '')
 
-      const json = encryptObject(_readJsonSync(filename), encKeyBuffer)
+      const json = encryptObject(fs2.readJson(filename), encKeyBuffer)
 
-      _writeJsonSync(encFilename, json, { spaces: 2 })
+      fs2.writeJson(encFilename, json, { spaces: 2 })
     } else {
       const plain = fs.readFileSync(filename)
       const enc = encryptRandomIVBuffer(plain, encKeyBuffer)

package/src/slack/slack.service.ts

@@ -185,7 +185,7 @@ export function slackDefaultMessagePrefixHook(msg: SlackMessage): string[] {
 
   // AppEngine-specific decoration
   if (GAE && ctx && typeof ctx === 'object' && typeof ctx.header === 'function') {
-    tokens.push(ctx.header('x-appengine-country')!, ctx.header('x-appengine-city')!)
+    tokens.push(ctx.header('x-appengine-country'), ctx.header('x-appengine-city'))
   }
 
   return tokens.filter(Boolean)

package/src/stream/ndjson/pipelineToNDJsonFile.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import { createGzip, ZlibOptions } from 'node:zlib'
 import { AppError } from '@naturalcycles/js-lib'
-import { transformTap, _pipeline, _pathExistsSync, _ensureFileSync } from '../..'
+import { transformTap, _pipeline, fs2 } from '../..'
 import { grey } from '../../colors/colors'
 import { NDJsonStats } from './ndjson.model'
 import { transformToNDJson, TransformToNDJsonOptions } from './transformToNDJson'
@@ -37,14 +37,14 @@ export async function pipelineToNDJsonFile(
 ): Promise<NDJsonStats> {
   const { filePath, gzip, protectFromOverwrite = false } = opt
 
-  if (protectFromOverwrite && _pathExistsSync(filePath)) {
+  if (protectFromOverwrite && fs2.pathExists(filePath)) {
     throw new AppError(`pipelineToNDJsonFile: output file exists: ${filePath}`)
   }
 
   const started = Date.now()
   let rows = 0
 
-  _ensureFileSync(filePath)
+  fs2.ensureFile(filePath)
 
   console.log(`>> ${grey(filePath)} started...`)
 

package/src/util/buildInfo.util.ts

@@ -5,7 +5,7 @@ import {
   localTimeOrNow,
   UnixTimestampNumber,
 } from '@naturalcycles/js-lib'
-import { _pathExistsSync, _readJsonSync } from '../fs/fs.util'
+import { fs2 } from '../fs/fs2'
 import {
   gitCurrentBranchName,
   gitCurrentCommitSha,
@@ -36,8 +36,8 @@ export function generateBuildInfo(opt: GenerateBuildInfoOptions = {}): BuildInfo
   if (!env) {
     // Attempt to read `envByBranch` from package.json root
     try {
-      if (_pathExistsSync('package.json')) {
-        const packageJson = _readJsonSync<AnyObject>('package.json')
+      if (fs2.pathExists('package.json')) {
+        const packageJson = fs2.readJson<AnyObject>('package.json')
         env = packageJson?.['envByBranch']?.[branchName] || packageJson?.['envByBranch']?.['*']
       }
     } catch {}

package/src/validation/ajv/ajv.util.ts

@@ -1,5 +1,5 @@
 import { JsonSchema } from '@naturalcycles/js-lib'
-import { _readJsonSync, fastGlob } from '../..'
+import { fastGlob, fs2 } from '../..'
 import type { FastGlobOptions } from '../..'
 import { AjvSchema, AjvSchemaCfg } from './ajvSchema'
 
@@ -12,7 +12,7 @@ import { AjvSchema, AjvSchemaCfg } from './ajvSchema'
  * @experimental
  */
 export function readJsonSchemas(patterns: string | string[], opt?: FastGlobOptions): JsonSchema[] {
-  return fastGlob.sync(patterns, opt).map(fileName => _readJsonSync(fileName))
+  return fastGlob.sync(patterns, opt).map(fileName => fs2.readJson(fileName))
 }
 
 /**

package/src/validation/ajv/ajvSchema.ts

@@ -8,7 +8,7 @@ import {
   CommonLogger,
 } from '@naturalcycles/js-lib'
 import Ajv, { ValidateFunction } from 'ajv'
-import { _readJsonSync, _inspect, requireFileToExist } from '../../index'
+import { _inspect, fs2, requireFileToExist } from '../../index'
 import { AjvValidationError } from './ajvValidationError'
 import { getAjv } from './getAjv'
 
@@ -134,7 +134,7 @@ export class AjvSchema<T = unknown> {
     cfg: Partial<AjvSchemaCfg> = {},
   ): AjvSchema<T> {
     requireFileToExist(filePath)
-    const schema = _readJsonSync<JsonSchema<T>>(filePath)
+    const schema = fs2.readJson<JsonSchema<T>>(filePath)
     return new AjvSchema<T>(schema, cfg)
   }