@naturalcycles/nodejs-lib 13.32.0 → 13.33.0

package/src/csv/csvReader.ts

@@ -56,10 +56,7 @@ export function csvStringParse<T extends AnyObject = any>(
 }
 
 export function csvStringToArray(str: string): string[][] {
-  const objPattern = new RegExp(
-    String.raw`(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))`,
-    'gi',
-  )
+  const objPattern = /(,|\r?\n|\r|^)(?:"([^"]*(?:""[^"]*)*)"|([^,\r\n]*))/gi
   let matches: RegExpExecArray | null
   const arr: any[][] = [[]]
 
@@ -67,9 +64,7 @@ export function csvStringToArray(str: string): string[][] {
     if (matches[1]!.length && matches[1] !== ',') {
       arr.push([])
     }
-    arr[arr.length - 1]!.push(
-      matches[2] ? matches[2].replaceAll(new RegExp('""', 'g'), '"') : matches[3],
-    )
+    arr[arr.length - 1]!.push(matches[2] ? matches[2].replaceAll('""', '"') : matches[3])
   }
   return arr
 }

package/src/fs/fs2.ts

@@ -14,7 +14,7 @@ Credit to: fs-extra (https://github.com/jprichardson/node-fs-extra)
 
  */
 
-import type { RmOptions } from 'node:fs'
+import type { RmOptions, Stats } from 'node:fs'
 import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import path from 'node:path'
@@ -182,7 +182,7 @@ class FS2 {
   }
 
   ensureFile(filePath: string): void {
-    let stats
+    let stats: Stats | undefined
     try {
       stats = fs.statSync(filePath)
     } catch {}
@@ -197,7 +197,10 @@ class FS2 {
       }
     } catch (err) {
       // If the stat call above failed because the directory doesn't exist, create it
-      if ((err as any)?.code === 'ENOENT') return this.ensureDir(dir)
+      if ((err as any)?.code === 'ENOENT') {
+        this.ensureDir(dir)
+        return
+      }
       throw err
     }
 
@@ -205,7 +208,7 @@ class FS2 {
   }
 
   async ensureFileAsync(filePath: string): Promise<void> {
-    let stats
+    let stats: Stats | undefined
     try {
       stats = await fsp.stat(filePath)
     } catch {}
@@ -236,18 +239,19 @@ class FS2 {
   }
 
   emptyDir(dirPath: string): void {
-    let items
+    let items: string[]
     try {
       items = fs.readdirSync(dirPath)
     } catch {
-      return this.ensureDir(dirPath)
+      this.ensureDir(dirPath)
+      return
     }
 
     items.forEach(item => this.removePath(path.join(dirPath, item)))
   }
 
   async emptyDirAsync(dirPath: string): Promise<void> {
-    let items
+    let items: string[]
     try {
       items = await fsp.readdir(dirPath)
     } catch {

package/src/index.ts

@@ -72,8 +72,8 @@ export * from './stream/writable/writableVoid'
 export * from './string/inspect'
 export * from './util/buildInfo.util'
 export * from './util/env.util'
-export * from './util/exec.util'
-export * from './util/git.util'
+export * from './util/exec2'
+export * from './util/git2'
 export * from './util/lruMemoCache'
 export * from './util/zip.util'
 export * from './validation/ajv/ajv.util'

package/src/secret/secrets-decrypt.util.ts

@@ -37,7 +37,7 @@ export function secretsDecrypt(
   const filenames = fastGlob.sync(patterns)
 
   filenames.forEach(filename => {
-    let plainFilename
+    let plainFilename: string
 
     if (jsonMode) {
       _assert(filename.endsWith('.json'), `${path.basename(filename)} MUST end with '.json'`)

package/src/secret/secrets-encrypt.util.ts

@@ -31,7 +31,7 @@ export function secretsEncrypt(
         `!**/*.enc`, // excluding already encoded
       ]
   const filenames = fastGlob.sync(patterns)
-  let encFilename
+  let encFilename: string
 
   filenames.forEach(filename => {
     if (jsonMode) {

package/src/slack/slack.service.ts

@@ -92,7 +92,7 @@ export class SlackService<CTX = any> {
     if (msg.kv) {
       ;(msg.attachments ||= []).push({ fields: this.kvToFields(msg.kv) })
 
-      delete msg.kv // to not pass it all the way to Slack Api
+      msg.kv = undefined // to not pass it all the way to Slack Api
     }
 
     let text: string
@@ -135,9 +135,8 @@ export class SlackService<CTX = any> {
         // ignore (unless throwOnError is set)
         if (msg.throwOnError) {
           throw err
-        } else {
-          console.log(err)
         }
+        console.log(err)
       })
   }
 

package/src/stream/ndjson/transformJsonParse.ts

@@ -55,7 +55,7 @@ export function transformJsonParse<ROW = any>(
 }
 
 // Based on: https://stackoverflow.com/a/34557997/4919972
-export const bufferReviver: Reviver = (k, v) => {
+export const bufferReviver: Reviver = (_k, v) => {
   if (v !== null && typeof v === 'object' && v.type === 'Buffer' && Array.isArray(v.data)) {
     return Buffer.from(v.data)
   }

package/src/stream/stream.model.ts

@@ -45,7 +45,7 @@ export interface ReadableTyped<T> extends Readable {
   drop: (limit: number, opt?: ReadableSignalOptions) => ReadableTyped<T>
 }
 
-// eslint-disable-next-line unused-imports/no-unused-vars
+// biome-ignore lint/correctness/noUnusedVariables: ok
 export interface WritableTyped<T> extends Writable {}
 
 /**
@@ -59,7 +59,7 @@ export type ReadableBinary = Readable
  */
 export type WritableBinary = Writable
 
-// eslint-disable-next-line unused-imports/no-unused-vars
+// biome-ignore lint/correctness/noUnusedVariables: ok
 export interface TransformTyped<IN, OUT> extends Transform {}
 
 export interface TransformOptions {

package/src/stream/transform/transformSplit.ts

@@ -18,7 +18,7 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
     readableObjectMode: true,
     writableHighWaterMark: 64 * 1024,
 
-    transform(buf: Buffer, enc, done) {
+    transform(buf: Buffer, _enc, done) {
       let offset = 0
       let lastMatch = 0
       if (buffered) {
@@ -66,7 +66,7 @@ export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer>
     readableObjectMode: true,
     writableHighWaterMark: 64 * 1024,
 
-    transform(buf: Buffer, enc, done) {
+    transform(buf: Buffer, _enc, done) {
       let offset = 0
       let lastMatch = 0
       if (buffered) {
@@ -119,7 +119,7 @@ function firstNewlineMatch(buf: Buffer, offset: number): number {
 
 function firstMatch(buf: Buffer, offset: number, matcher: Buffer): number {
   if (offset >= buf.length) return -1
-  let i
+  let i: number
   for (i = offset; i < buf.length; i++) {
     if (buf[i] === matcher[0]) {
       if (matcher.length > 1) {

package/src/stream/transform/worker/workerClassProxy.js

@@ -1,6 +1,6 @@
 const started = Date.now()
-const { workerData, parentPort } = require('worker_threads')
-const { inspect } = require('util')
+const { workerData, parentPort } = require('node:worker_threads')
+const { inspect } = require('node:util')
 const { workerFile, workerIndex, logEvery = 1000, metric = 'worker' } = workerData || {}
 
 if (!workerFile) {

package/src/stream/writable/writableLimit.ts

@@ -11,7 +11,7 @@ export function writableLimit<T>(readable: Readable, limit: number): WritableTyp
 
   return new Writable({
     objectMode: true,
-    write(chunk, _, cb) {
+    write(_chunk, _, cb) {
       if (limit === 0) return cb() // no limit, just passthrough
 
       i++

package/src/stream/writable/writableVoid.ts

@@ -18,7 +18,7 @@ export function writableVoid(opt: WritableVoidOptions = {}): Writable {
   return new Writable({
     objectMode: true,
     ...opt,
-    write(chunk, _, cb) {
+    write(_chunk, _, cb) {
       cb()
     },
     final(cb) {

package/src/util/buildInfo.util.ts

@@ -6,12 +6,7 @@ import {
   UnixTimestampNumber,
 } from '@naturalcycles/js-lib'
 import { fs2 } from '../fs/fs2'
-import {
-  gitCurrentBranchName,
-  gitCurrentCommitSha,
-  gitCurrentCommitTimestamp,
-  gitCurrentRepoName,
-} from './git.util'
+import { git2 } from './git2'
 
 export interface GenerateBuildInfoOptions {
   /**
@@ -24,10 +19,10 @@ export function generateBuildInfo(opt: GenerateBuildInfoOptions = {}): BuildInfo
   const now = localTime.orNow(opt.overrideTimestamp)
   const ts = now.unix
 
-  const rev = gitCurrentCommitSha()
-  const branchName = gitCurrentBranchName()
-  const repoName = gitCurrentRepoName()
-  const tsCommit = gitCurrentCommitTimestamp()
+  const rev = git2.gitCurrentCommitSha()
+  const branchName = git2.gitCurrentBranchName()
+  const repoName = git2.gitCurrentRepoName()
+  const tsCommit = git2.gitCurrentCommitTimestamp()
 
   const ver = [now.toStringCompact(), repoName, branchName, rev].join('_')
 

package/src/util/exec2.ts

@@ -0,0 +1,326 @@
+import cp from 'node:child_process'
+import {
+  _since,
+  AnyObject,
+  AppError,
+  NumberOfMilliseconds,
+  UnixTimestampMillisNumber,
+} from '@naturalcycles/js-lib'
+import { dimGrey, white } from '../colors/colors'
+
+/**
+ * Set of utility functions to work with Spawn / Exec.
+ *
+ * How to decide between Spawn and Exec?
+ *
+ * Long-running job that prints output, and no need to return the output - use Spawn.
+ *
+ * Short-running job, no need to print the output, might want to return the output - use Exec.
+ *
+ * Need to both print and return the output - use SpawnAsync.
+ *
+ * ***
+ *
+ * Spawn is good for long-running large-output processes, that continuously output data.
+ * E.g running `jest`.
+ *
+ * Exec is the opposite - good for short-running processes that output small data.
+ * Exec allows to return the output as a string.
+ * Exec doesn't stream data during execution, so the output/error will only be printed
+ * at the end.
+ * Exec always uses the shell (there's no option to disable it).
+ */
+class Exec2 {
+  /**
+   * Advanced/async version of Spawn.
+   * Consider simpler `spawn` or `exec` first, which are also sync.
+   *
+   * spawnAsync features:
+   *
+   * 1. Async
+   * 2. Allows to collect the output AND print it while running.
+   * 3. Returns SpawnOutput with stdout, stderr and exitCode.
+   * 4. Allows to not throw on error, but just return SpawnOutput for further inspection.
+   *
+   * Defaults:
+   *
+   * shell: true
+   * printWhileRunning: true
+   * collectOutputWhileRunning: true
+   * throwOnNonZeroCode: true
+   * log: true
+   */
+  async spawnAsync(cmd: string, opt: SpawnAsyncOptions = {}): Promise<SpawnOutput> {
+    const started = Date.now()
+    this.logStart(cmd, opt)
+    const {
+      shell = true,
+      printWhileRunning = true,
+      collectOutputWhileRunning = true,
+      throwOnNonZeroCode = true,
+      cwd,
+      env,
+    } = opt
+    let stdout = ''
+    let stderr = ''
+
+    return await new Promise<SpawnOutput>((resolve, reject) => {
+      const p = cp.spawn(cmd, opt.args || [], {
+        shell,
+        cwd,
+        env,
+        // ...process.env, // not passing by default for security reasons
+      })
+
+      p.stdout.on('data', data => {
+        if (collectOutputWhileRunning) {
+          stdout += data.toString()
+          // console.log('stdout:', data.toString())
+        }
+        if (printWhileRunning) {
+          process.stdout.write(data)
+          // console.log('stderr:', data.toString())
+        }
+      })
+      p.stderr.on('data', data => {
+        if (collectOutputWhileRunning) {
+          stderr += data.toString()
+        }
+        if (printWhileRunning) {
+          process.stderr.write(data)
+        }
+      })
+
+      p.on('close', code => {
+        this.logFinish(cmd, opt, started)
+        const exitCode = code || 0
+        const o: SpawnOutput = {
+          exitCode,
+          stdout: stdout.trim(),
+          stderr: stderr.trim(),
+        }
+        if (throwOnNonZeroCode && code) {
+          return reject(new SpawnError(`spawnAsync exited with code ${code}: ${cmd}`, o))
+        }
+        resolve(o)
+      })
+    })
+  }
+
+  /**
+   * Reasons to use it:
+   * - Sync
+   * - Need to print output while running
+   *
+   * Limitations:
+   * - Cannot return stdout/stderr (use exec or spawnAsync for that)
+   *
+   * Defaults:
+   *
+   * shell: true
+   * log: true
+   */
+  spawn(cmd: string, opt: SpawnOptions = {}): void {
+    const started = Date.now()
+    this.logStart(cmd, opt)
+    const { shell = true, cwd, env } = opt
+
+    const r = cp.spawnSync(cmd, opt.args, {
+      encoding: 'utf8',
+      stdio: 'inherit',
+      shell,
+      cwd,
+      env,
+      // ...process.env, // not passing by default for security reasons
+    })
+
+    this.logFinish(cmd, opt, started)
+
+    if (r.error) {
+      throw r.error
+    }
+    if (r.status) {
+      throw new Error(`spawn exited with code ${r.status}: ${cmd}`)
+    }
+  }
+
+  /**
+   * Reasons to use it:
+   *
+   * - Sync
+   * - Need to return output
+   *
+   * Limitations:
+   * - Cannot print while running (use spawn or spawnAsync for that)
+   *
+   * Defaults:
+   *
+   * shell: true
+   * log: true
+   */
+  exec(cmd: string, opt: ExecOptions = {}): string {
+    const started = Date.now()
+    this.logStart(cmd, opt)
+    const { cwd, env, timeout } = opt
+
+    try {
+      return cp
+        .execSync(cmd, {
+          encoding: 'utf8',
+          // stdio: 'inherit', // no, otherwise we don't get the output returned
+          stdio: undefined,
+          // shell: undefined,
+          cwd,
+          timeout,
+          env,
+          // ...process.env, // not passing by default for security reasons
+        })
+        .trim()
+    } catch (err) {
+      // Not logging stderr, as it's printed by execSync by default (somehow)
+      // stdout is not printed by execSync though, therefor we print it here
+      // if ((err as any).stderr) {
+      //   process.stderr.write((err as any).stderr)
+      // }
+      if ((err as any).stdout) {
+        process.stdout.write((err as any).stdout)
+      }
+      throw new Error(`exec exited with code ${(err as any).status}: ${cmd}`)
+    } finally {
+      this.logFinish(cmd, opt, started)
+    }
+  }
+
+  throwOnNonZeroExitCode(o: SpawnOutput): void {
+    if (o.exitCode) {
+      throw new SpawnError(`spawn exited with code ${o.exitCode}`, o)
+    }
+  }
+
+  private logStart(cmd: string, opt: SpawnOptions | ExecOptions): void {
+    if (!opt.logStart && !opt.log) return
+
+    console.log(
+      [
+        dimGrey(...Object.entries(opt.env || {}).map(([k, v]) => [k, v].join('='))),
+        white(opt.name || cmd),
+        ...((!opt.name && (opt as SpawnOptions).args) || []),
+      ]
+        .filter(Boolean)
+        .join(' '),
+    )
+  }
+
+  private logFinish(
+    cmd: string,
+    opt: SpawnOptions | ExecOptions,
+    started: UnixTimestampMillisNumber,
+  ): void {
+    if (!opt.logFinish && !opt.log) return
+
+    console.log([white(opt.name || cmd), dimGrey('took ' + _since(started))].join(' '))
+  }
+}
+
+export const exec2 = new Exec2()
+
+export class SpawnError extends AppError<SpawnErrorData> {
+  constructor(message: string, data: SpawnErrorData) {
+    super(message, data, { name: 'SpawnError' })
+  }
+}
+
+export interface SpawnErrorData extends SpawnOutput {}
+
+export interface SpawnOutput {
+  /**
+   * Exit code of the spawned process.
+   * 0 means success, anything else means failure.
+   */
+  exitCode: number
+  stdout: string
+  stderr: string
+}
+
+export interface SpawnAsyncOptions extends SpawnOptions {
+  /**
+   * Defaults to true.
+   * If true - prints both stdout and stderr to console while running,
+   * otherwise runs "silently".
+   * Returns SpawnOutput in the same way, regardless of `printWhileRunning` setting.
+   */
+  printWhileRunning?: boolean
+
+  /**
+   * Defaults to true.
+   * If true - collects stdout and stderr while running, and return it in the end.
+   * stdout/stderr are collected and returned regardless if it returns with error or not.
+   * On success - stdout/stderr are available from `SpawnOutput`.
+   * On error - stdout/stderr are available from `SpawnError.data`.
+   */
+  collectOutputWhileRunning?: boolean
+
+  /**
+   * Defaults to true.
+   * If true - throws SpawnError if non-zero code is returned.
+   * SpawnError conveniently contains .data.stdout and .data.strerr for inspection.
+   * If false - will not throw, but return SpawnOutput with stdout, stderr and exitCode.
+   */
+  throwOnNonZeroCode?: boolean
+}
+
+export interface SpawnOptions {
+  args?: string[]
+  /**
+   * Defaults to true.
+   */
+  logStart?: boolean
+  /**
+   * Defaults to true.
+   */
+  logFinish?: boolean
+  /**
+   * Defaults to true.
+   * Controls/overrides both logStart and logFinish simultaneously.
+   */
+  log?: boolean
+  /**
+   * Defaults to true.
+   */
+  shell?: boolean
+
+  /**
+   * If specified - will be used as "command name" for logging purposes,
+   * instead of "cmd + args"
+   */
+  name?: string
+  cwd?: string
+
+  env?: AnyObject
+}
+
+export interface ExecOptions {
+  /**
+   * Defaults to false.
+   */
+  logStart?: boolean
+  /**
+   * Defaults to false.
+   */
+  logFinish?: boolean
+  /**
+   * Defaults to false.
+   * Controls/overrides both logStart and logFinish simultaneously.
+   */
+  log?: boolean
+
+  /**
+   * If specified - will be used as "command name" for logging purposes,
+   * instead of "cmd + args"
+   */
+  name?: string
+  cwd?: string
+  timeout?: NumberOfMilliseconds
+
+  env?: AnyObject
+}

package/src/util/git2.ts

@@ -0,0 +1,105 @@
+import cp from 'node:child_process'
+import path from 'node:path'
+import type { UnixTimestampNumber } from '@naturalcycles/js-lib'
+import { grey } from '../colors/colors'
+import { exec2 } from './exec2'
+
+/**
+ * Set of utility functions to work with git.
+ */
+class Git2 {
+  getLastGitCommitMsg(): string {
+    return exec2.exec('git log -1 --pretty=%B')
+  }
+
+  commitMessageToTitleMessage(msg: string): string {
+    const firstLine = msg.split('\n')[0]!
+    const [preTitle, title] = firstLine.split(': ')
+    return title || preTitle!
+  }
+
+  gitHasUncommittedChanges(): boolean {
+    // git diff-index --quiet HEAD -- || echo "untracked"
+    try {
+      cp.execSync('git diff-index --quiet HEAD --', {
+        encoding: 'utf8',
+      })
+      return false
+    } catch {
+      return true
+    }
+  }
+
+  /**
+   * Returns true if there were changes
+   */
+  gitCommitAll(msg: string): boolean {
+    // git commit -a -m "style(lint-all): $GIT_MSG" || true
+    const cmd = `git commit -a --no-verify -m "${msg}"`
+    // const cmd = `git`
+    // const args = ['commit', '-a', '--no-verify', '-m', msg]
+    console.log(grey(cmd))
+
+    try {
+      cp.execSync(cmd, {
+        stdio: 'inherit',
+      })
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * @returns true if there are not pushed commits.
+   */
+  gitIsAhead(): boolean {
+    // ahead=`git rev-list HEAD --not --remotes | wc -l | awk '{print $1}'`
+    const cmd = `git rev-list HEAD --not --remotes | wc -l | awk '{print $1}'`
+    const stdout = exec2.exec(cmd)
+    // console.log(`gitIsAhead: ${stdout}`)
+    return Number(stdout) > 0
+  }
+
+  gitPull(): void {
+    const cmd = 'git pull'
+    try {
+      cp.execSync(cmd, {
+        stdio: 'inherit',
+      })
+    } catch {}
+  }
+
+  gitPush(): void {
+    // git push --set-upstream origin $CIRCLE_BRANCH && echo "pushed, exiting" && exit 0
+    let cmd = 'git push'
+
+    const branchName = this.gitCurrentBranchName()
+
+    if (branchName) {
+      cmd += ` --set-upstream origin ${branchName}`
+    }
+
+    exec2.spawn(cmd, { logStart: true })
+  }
+
+  gitCurrentCommitSha(full = false): string {
+    const sha = exec2.exec('git rev-parse HEAD')
+    return full ? sha : sha.slice(0, 7)
+  }
+
+  gitCurrentCommitTimestamp(): UnixTimestampNumber {
+    return Number(exec2.exec('git log -1 --format=%ct'))
+  }
+
+  gitCurrentBranchName(): string {
+    return exec2.exec('git rev-parse --abbrev-ref HEAD')
+  }
+
+  gitCurrentRepoName(): string {
+    const originUrl = exec2.exec('git config --get remote.origin.url')
+    return path.basename(originUrl, '.git')
+  }
+}
+
+export const git2 = new Git2()

package/dist/util/exec.util.d.ts

@@ -1,10 +0,0 @@
-import type { SpawnOptions } from 'node:child_process';
-export interface ExecOptions extends SpawnOptions {
-    /**
-     * Defaults to true.
-     * Set to false to skip logging.
-     */
-    log?: boolean;
-}
-export declare function execVoidCommand(cmd: string, args?: string[], opt?: ExecOptions): Promise<void>;
-export declare function execVoidCommandSync(cmd: string, args?: string[], opt?: ExecOptions): void;