bajo-extra 0.2.15 → 0.2.17

package/bajo/config.json

@@ -1,5 +1,8 @@
 {
   "alias": "x",
+  "fetch": {
+    "agent": {}
+  },
   "stream": {
     "export": {
       "maxBatch": 1000,

package/bajo/helper/count-file-lines.js

@@ -0,0 +1,23 @@
+// taken from: https://stackoverflow.com/a/41439945
+import fs from 'fs'
+
+function countFileLines (file) {
+  return new Promise((resolve, reject) => {
+    let lineCount = 0
+    fs.createReadStream(file)
+      .on('data', (buffer) => {
+        let idx = -1
+        lineCount--
+        do {
+          idx = buffer.indexOf(10, idx + 1)
+          lineCount++
+        } while (idx !== -1)
+      })
+      .on('end', () => {
+        resolve(lineCount)
+      })
+      .on('error', reject)
+  })
+}
+
+export default countFileLines

package/bajo/helper/download.js

@@ -0,0 +1,45 @@
+import path from 'path'
+
+async function download (url, opts = {}) {
+  const { getPluginDataDir, importPkg, error, generateId } = this.bajo.helper
+  const { fetch, formatByte, formatPercentage } = this.bajoExtra.helper
+  const { isFunction } = await importPkg('lodash-es')
+  if (typeof opts === 'string') opts = { dir: opts }
+  const [fs, increment] = await importPkg('fs-extra', 'add-filename-increment')
+  if (!opts.dir) {
+    opts.dir = `${getPluginDataDir('bajoExtra')}/download`
+    fs.ensureDirSync(opts.dir)
+  }
+  const fetchOpts = opts.fetchOpts ?? {}
+  if (!fs.existsSync(opts.dir)) throw error('Download dir \'%s\' doesn\'t exists', opts.dir)
+  if (opts.randomFileName) {
+    const ext = path.extname(url)
+    opts.fileName = `${generateId()}${ext}`
+  }
+  if (!opts.fileName) opts.fileName = path.basename(url)
+  const file = path.resolve(increment(`${opts.dir}/${opts.fileName}`, { fs: true }))
+  const writer = fs.createWriteStream(file)
+  fetchOpts.responseType = 'stream'
+  const { headers, data } = await fetch(url, fetchOpts, { rawResponse: true })
+  const total = headers['content-length'] ?? 0
+  let length = 0
+  data.on('data', chunk => {
+    length += chunk.length
+    if (isFunction(opts.progressFn)) opts.progressFn.call(this, length, total)
+    else if (opts.spin) {
+      opts.spinText = opts.spinText ?? 'Downloading...'
+      if (total === 0) opts.spin.setText(`${opts.spinText} %s`, formatByte(length))
+      else opts.spin.setText(`${opts.spinText} %s of %s (%s)`, formatByte(length), formatByte(total), formatPercentage(length / total))
+    }
+  })
+  data.pipe(writer)
+
+  return new Promise((resolve, reject) => {
+    writer.on('error', reject)
+    writer.on('finish', () => {
+      resolve(file)
+    })
+  })
+}
+
+export default download

package/bajo/helper/fetch.js

@@ -1,8 +1,11 @@
 import axios from 'axios'
+import http from 'http'
+import https from 'https'
 
 async function fetch (url, opts = {}, ext = {}) {
-  const { importPkg } = this.bajo.helper
-  const { has, isPlainObject, cloneDeep } = await importPkg('lodash-es')
+  const { importPkg, getConfig } = this.bajo.helper
+  const { has, isPlainObject, cloneDeep, isEmpty } = await importPkg('lodash-es')
+  const cfg = getConfig('bajoExtra')
   if (isPlainObject(url)) {
     ext = cloneDeep(opts)
     opts = cloneDeep(url)
@@ -10,6 +13,10 @@ async function fetch (url, opts = {}, ext = {}) {
   opts.params = opts.params ?? {}
   if (!has(ext, 'cacheBuster')) ext.cacheBuster = true
   if (ext.cacheBuster) opts.params[ext.cacheBusterKey ?? '_'] = Date.now()
+  if (!isEmpty(cfg.fetch.agent)) {
+    opts.httpAgent = opts.httpAgent ?? new http.Agent(cfg.fetch.agent)
+    opts.httpsAgent = opts.httpsAgent ?? new https.Agent(cfg.fetch.agent)
+  }
   const resp = await axios(opts)
   if (ext.rawResponse) return resp
   return resp.data

package/bajo/helper/format/byte.js

@@ -0,0 +1,11 @@
+import numbro from 'numbro'
+
+function byte (value, opts = {}) {
+  opts.output = 'byte'
+  opts.base = 'binary'
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.spaceSeparated = opts.spaceSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default byte

package/bajo/helper/format/float.js

@@ -0,0 +1,9 @@
+import numbro from 'numbro'
+
+function float (value, opts = {}) {
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.thousandSeparated = opts.thousandSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default float

package/bajo/helper/format/integer.js

@@ -0,0 +1,9 @@
+import numbro from 'numbro'
+
+function integer (value, opts = {}) {
+  opts.mantissa = 0
+  opts.thousandSeparated = opts.thousandSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default integer

package/bajo/helper/format/percentage.js

@@ -0,0 +1,10 @@
+import numbro from 'numbro'
+
+function percentage (value, opts = {}) {
+  opts.output = 'percent'
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.spaceSeparated = opts.spaceSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default percentage

package/bajo/helper/gzip.js

@@ -1,14 +1,14 @@
 import { createGzip, createGunzip } from 'zlib'
 
-function gzip (file, deleteOld, unzip) {
+function gzip (file, deleteOld, expand) {
   return new Promise((resolve, reject) => {
     const { importPkg } = this.bajo.helper
     importPkg('fs-extra')
       .then(fs => {
-        const newFile = unzip ? file.slice(0, file.length - 3) : (file + '.gz')
+        const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
         const reader = fs.createReadStream(file)
         const writer = fs.createWriteStream(newFile)
-        const method = unzip ? createGunzip() : createGzip()
+        const method = expand ? createGunzip() : createGzip()
         reader.pipe(method).pipe(writer)
         writer.on('error', reject)
         writer.on('finish', err => {

package/bajo/helper/hash.js

@@ -1,10 +1,16 @@
 import bcrypt from 'bcrypt'
 import crypto from 'crypto'
 
-async function hash (text, type = 'md5', { digest = 'hex', salt = 10 } = {}) {
+async function hash (text, type = 'md5', options = {}) {
+  options.digest = options.digest ?? 'hex'
+  options.salt = options.hash ?? 10
   if (typeof text !== 'string') text = JSON.stringify(text)
-  if (type === 'bcrypt') return await bcrypt.hash(text, salt)
-  return crypto.createHash(type).update(text).digest(digest)
+  if (type === 'bcrypt') return await bcrypt.hash(text, options.salt)
+  if (type === 'short') {
+    type = 'shake256'
+    options.outputLength = 6
+  }
+  return crypto.createHash(type, options).update(text).digest(options.digest)
 }
 
 export default hash

package/bajo/helper/import-from.js

@@ -9,9 +9,10 @@ const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
 
 async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
   const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
-  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-  const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
-  await getInfo(dest)
+  if (dest !== false) {
+    if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
+    await this.bajoDb.helper.getInfo(dest)
+  }
   const { merge } = await importPkg('lodash-es')
   const fs = await importPkg('fs-extra')
   const cfg = getConfig('bajoExtra')
@@ -30,7 +31,7 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
     decompress = true
   }
   if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  if (trashOld) await recordClear(dest)
+  if (trashOld && dest !== false) await this.bajoDb.helper.recordClear(dest)
   const reader = fs.createReadStream(file)
   batch = parseInt(batch) || 100
   if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
@@ -46,6 +47,7 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
 
   const stream = DataStream.pipeline(...pipes)
   let batchNo = 1
+  const data = []
   await stream
     .batch(batch)
     .map(async items => {
@@ -54,14 +56,15 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
       for (let item of items) {
         count++
         item = converterFn ? await converterFn.call(this, item) : item
-        await recordCreate(dest, item, createOpts)
+        if (dest !== false) await this.bajoDb.helper.recordCreate(dest, item, createOpts)
+        else data.push(item)
       }
       if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
       batchNo++
     })
     .run()
 
-  return { file, count }
+  return dest === false ? data : { file, count }
 }
 
 export default importFrom

package/bajoCli/tool/download.js

@@ -0,0 +1,17 @@
+async function download ({ path, args }) {
+  const { spinner } = this.bajo.helper
+  const { download } = this.bajoExtra.helper
+  const url = args[0]
+  const spinText = 'Downloading file...'
+  const spin = spinner({ showCounter: true }).start(spinText)
+
+  let dest
+  try {
+    dest = await download(url, { spin, spinText })
+  } catch (err) {
+    spin.fatal('Error: %s', err.message)
+  }
+  spin.succeed('File saved as \'%s\'', dest)
+}
+
+export default download

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.15",
+  "version": "0.2.17",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -34,8 +34,8 @@
     "fast-csv": "^5.0.1",
     "fast-jwt": "^3.2.0",
     "fast-xml-parser": "^4.3.2",
-    "littlehash": "^1.0.1",
     "ndjson": "^2.0.0",
+    "numbro": "^2.5.0",
     "performant-array-to-tree": "^1.11.0",
     "query-string": "^8.1.0",
     "scramjet": "^4.36.9",