bajo-extra 0.2.16 → 0.3.0

package/bajo/config.json

@@ -1,5 +1,8 @@
 {
   "alias": "x",
+  "fetch": {
+    "agent": {}
+  },
   "stream": {
     "export": {
       "maxBatch": 1000,

package/bajo/helper/count-file-lines.js

@@ -0,0 +1,23 @@
+// taken from: https://stackoverflow.com/a/41439945
+import fs from 'fs'
+
+function countFileLines (file) {
+  return new Promise((resolve, reject) => {
+    let lineCount = 0
+    fs.createReadStream(file)
+      .on('data', (buffer) => {
+        let idx = -1
+        lineCount--
+        do {
+          idx = buffer.indexOf(10, idx + 1)
+          lineCount++
+        } while (idx !== -1)
+      })
+      .on('end', () => {
+        resolve(lineCount)
+      })
+      .on('error', reject)
+  })
+}
+
+export default countFileLines

package/bajo/helper/download.js

@@ -0,0 +1,45 @@
+import path from 'path'
+
+async function download (url, opts = {}) {
+  const { fs, getPluginDataDir, importPkg, error, generateId } = this.bajo.helper
+  const { fetch, formatByte, formatPercentage } = this.bajoExtra.helper
+  const { isFunction } = this.bajo.helper._
+  if (typeof opts === 'string') opts = { dir: opts }
+  const increment = await importPkg('add-filename-increment')
+  if (!opts.dir) {
+    opts.dir = `${getPluginDataDir('bajoExtra')}/download`
+    fs.ensureDirSync(opts.dir)
+  }
+  const fetchOpts = opts.fetchOpts ?? {}
+  if (!fs.existsSync(opts.dir)) throw error('Download dir \'%s\' doesn\'t exists', opts.dir)
+  if (opts.randomFileName) {
+    const ext = path.extname(url)
+    opts.fileName = `${generateId()}${ext}`
+  }
+  if (!opts.fileName) opts.fileName = path.basename(url)
+  const file = path.resolve(increment(`${opts.dir}/${opts.fileName}`, { fs: true }))
+  const writer = fs.createWriteStream(file)
+  fetchOpts.responseType = 'stream'
+  const { headers, data } = await fetch(url, fetchOpts, { rawResponse: true })
+  const total = headers['content-length'] ?? 0
+  let length = 0
+  data.on('data', chunk => {
+    length += chunk.length
+    if (isFunction(opts.progressFn)) opts.progressFn.call(this, length, total)
+    else if (opts.spin) {
+      opts.spinText = opts.spinText ?? 'Downloading...'
+      if (total === 0) opts.spin.setText(`${opts.spinText} %s`, formatByte(length))
+      else opts.spin.setText(`${opts.spinText} %s of %s (%s)`, formatByte(length), formatByte(total), formatPercentage(length / total))
+    }
+  })
+  data.pipe(writer)
+
+  return new Promise((resolve, reject) => {
+    writer.on('error', reject)
+    writer.on('finish', () => {
+      resolve(file)
+    })
+  })
+}
+
+export default download

package/bajo/helper/export-to.js

@@ -9,8 +9,8 @@ const { DataStream } = scramjet
 const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
 
 async function getFile (dest, ensureDir) {
-  const { importPkg, error, getPluginDataDir } = this.bajo.helper
-  const [fs, increment] = await importPkg('fs-extra', 'add-filename-increment')
+  const { fs, importPkg, error, getPluginDataDir } = this.bajo.helper
+  const increment = await importPkg('add-filename-increment')
   let file
   if (path.isAbsolute(dest)) file = dest
   else {
@@ -51,7 +51,7 @@ async function getData ({ source, filter, count, stream, progressFn }) {
 }
 
 function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
-  const { error, importPkg, getConfig } = this.bajo.helper
+  const { fs, error, getConfig } = this.bajo.helper
   const cfg = getConfig('bajoExtra')
   if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
   filter.page = 1
@@ -59,27 +59,18 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
   if (batch > cfg.stream.export.maxBatch) batch = cfg.stream.export.maxBatch
   if (batch < 0) batch = 1
   filter.limit = batch
+  const { merge } = this.bajo.helper._
 
   return new Promise((resolve, reject) => {
     const { getInfo } = this.bajoDb.helper
     let count = 0
-    let fs
-    let merge
     let file
     let ext
     let stream
     let compress
     let writer
     getInfo(source)
-      .then(() => {
-        return importPkg('lodash-es')
-      })
-      .then(l => {
-        merge = l.merge
-        return importPkg('fs-extra')
-      })
       .then(res => {
-        fs = res
         return getFile.call(this, dest, ensureDir)
       })
       .then(res => {

package/bajo/helper/fetch-and-save.js

@@ -1,6 +1,5 @@
 async function handler (rec, bulk) {
-  const { importPkg } = this.bajo.helper
-  const { isFunction, set } = await importPkg('lodash-es')
+  const { isFunction, set } = this.bajo.helper._
   const { recordCreate, recordFind, recordUpdate } = this.bajoDb.helper
   const save = bulk.save ?? {}
   const current = save.current ?? {}
@@ -52,9 +51,9 @@ async function handler (rec, bulk) {
 }
 
 async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
-  const { importPkg, getConfig, importModule } = this.bajo.helper
+  const { getConfig, importModule } = this.bajo.helper
   const { fetchBulk } = this.bajoExtra.helper
-  const { merge } = await importPkg('lodash-es')
+  const { merge } = this.bajo.helper._
   merge(bulk, { handler, save })
   const cfgDb = getConfig('bajoDb', { full: true })
   const start = await importModule(`${cfgDb.dir.pkg}/bajo/start.js`)

package/bajo/helper/fetch-bulk.js

@@ -1,6 +1,6 @@
 async function fetching ({ url, opts, bulk, spin }) {
-  const { setImmediate, importPkg, print } = this.bajo.helper
-  const { isEmpty, isFunction, has } = await importPkg('lodash-es')
+  const { setImmediate, print } = this.bajo.helper
+  const { isEmpty, isFunction, has } = this.bajo.helper._
   const { validationErrorMessage } = this.bajoDb.helper
   const { fetch } = this.bajoExtra.helper
   const resp = await fetch(url, opts ?? {})
@@ -49,8 +49,8 @@ async function fetching ({ url, opts, bulk, spin }) {
 }
 
 async function fetchBulk (url, bulk = {}, opts = {}) {
-  const { print, spinner, importPkg, error } = this.bajo.helper
-  const { isFunction } = await importPkg('lodash-es')
+  const { print, spinner, error } = this.bajo.helper
+  const { isFunction } = this.bajo.helper._
   opts.params = opts.params ?? {}
   bulk.maxStep = bulk.maxStep ?? 0
   if (!isFunction(bulk.handler)) throw error('A function handler must be provided')

package/bajo/helper/fetch.js

@@ -1,8 +1,11 @@
 import axios from 'axios'
+import http from 'http'
+import https from 'https'
 
 async function fetch (url, opts = {}, ext = {}) {
-  const { importPkg } = this.bajo.helper
-  const { has, isPlainObject, cloneDeep } = await importPkg('lodash-es')
+  const { getConfig } = this.bajo.helper
+  const { has, isPlainObject, cloneDeep, isEmpty } = this.bajo.helper._
+  const cfg = getConfig('bajoExtra')
   if (isPlainObject(url)) {
     ext = cloneDeep(opts)
     opts = cloneDeep(url)
@@ -10,6 +13,10 @@ async function fetch (url, opts = {}, ext = {}) {
   opts.params = opts.params ?? {}
   if (!has(ext, 'cacheBuster')) ext.cacheBuster = true
   if (ext.cacheBuster) opts.params[ext.cacheBusterKey ?? '_'] = Date.now()
+  if (!isEmpty(cfg.fetch.agent)) {
+    opts.httpAgent = opts.httpAgent ?? new http.Agent(cfg.fetch.agent)
+    opts.httpsAgent = opts.httpsAgent ?? new https.Agent(cfg.fetch.agent)
+  }
   const resp = await axios(opts)
   if (ext.rawResponse) return resp
   return resp.data

package/bajo/helper/format/byte.js

@@ -0,0 +1,11 @@
+import numbro from 'numbro'
+
+function byte (value, opts = {}) {
+  opts.output = 'byte'
+  opts.base = 'binary'
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.spaceSeparated = opts.spaceSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default byte

package/bajo/helper/format/float.js

@@ -0,0 +1,9 @@
+import numbro from 'numbro'
+
+function float (value, opts = {}) {
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.thousandSeparated = opts.thousandSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default float

package/bajo/helper/format/integer.js

@@ -0,0 +1,9 @@
+import numbro from 'numbro'
+
+function integer (value, opts = {}) {
+  opts.mantissa = 0
+  opts.thousandSeparated = opts.thousandSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default integer

package/bajo/helper/format/percentage.js

@@ -0,0 +1,10 @@
+import numbro from 'numbro'
+
+function percentage (value, opts = {}) {
+  opts.output = 'percent'
+  opts.mantissa = opts.mantissa ?? opts.scale ?? 2
+  opts.spaceSeparated = opts.spaceSeparated ?? true
+  return numbro(value).format(opts)
+}
+
+export default percentage

package/bajo/helper/gzip.js

@@ -1,22 +1,19 @@
 import { createGzip, createGunzip } from 'zlib'
 
-function gzip (file, deleteOld, unzip) {
+function gzip (file, deleteOld, expand) {
+  const { fs } = this.bajo.helper
   return new Promise((resolve, reject) => {
-    const { importPkg } = this.bajo.helper
-    importPkg('fs-extra')
-      .then(fs => {
-        const newFile = unzip ? file.slice(0, file.length - 3) : (file + '.gz')
-        const reader = fs.createReadStream(file)
-        const writer = fs.createWriteStream(newFile)
-        const method = unzip ? createGunzip() : createGzip()
-        reader.pipe(method).pipe(writer)
-        writer.on('error', reject)
-        writer.on('finish', err => {
-          if (err) return reject(err)
-          if (deleteOld) fs.unlinkSync(file)
-          resolve()
-        })
-      })
+    const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
+    const reader = fs.createReadStream(file)
+    const writer = fs.createWriteStream(newFile)
+    const method = expand ? createGunzip() : createGzip()
+    reader.pipe(method).pipe(writer)
+    writer.on('error', reject)
+    writer.on('finish', err => {
+      if (err) return reject(err)
+      if (deleteOld) fs.unlinkSync(file)
+      resolve()
+    })
   })
 }
 

package/bajo/helper/import-from.js

@@ -8,12 +8,12 @@ const { DataStream } = scramjet
 const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
 
 async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
-  const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
-  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-  const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
-  await getInfo(dest)
-  const { merge } = await importPkg('lodash-es')
-  const fs = await importPkg('fs-extra')
+  const { fs, error, getConfig, getPluginDataDir } = this.bajo.helper
+  if (dest !== false) {
+    if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
+    await this.bajoDb.helper.getInfo(dest)
+  }
+  const { merge } = this.bajo.helper._
   const cfg = getConfig('bajoExtra')
 
   let file
@@ -30,7 +30,7 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
     decompress = true
   }
   if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  if (trashOld) await recordClear(dest)
+  if (trashOld && dest !== false) await this.bajoDb.helper.recordClear(dest)
   const reader = fs.createReadStream(file)
   batch = parseInt(batch) || 100
   if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
@@ -46,6 +46,7 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
 
   const stream = DataStream.pipeline(...pipes)
   let batchNo = 1
+  const data = []
   await stream
     .batch(batch)
     .map(async items => {
@@ -54,14 +55,15 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
       for (let item of items) {
         count++
         item = converterFn ? await converterFn.call(this, item) : item
-        await recordCreate(dest, item, createOpts)
+        if (dest !== false) await this.bajoDb.helper.recordCreate(dest, item, createOpts)
+        else data.push(item)
       }
       if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
       batchNo++
     })
     .run()
 
-  return { file, count }
+  return dest === false ? data : { file, count }
 }
 
 export default importFrom

package/bajoCli/tool/download.js

@@ -0,0 +1,17 @@
+async function download ({ path, args }) {
+  const { spinner } = this.bajo.helper
+  const { download } = this.bajoExtra.helper
+  const url = args[0]
+  const spinText = 'Downloading file...'
+  const spin = spinner({ showCounter: true }).start(spinText)
+
+  let dest
+  try {
+    dest = await download(url, { spin, spinText })
+  } catch (err) {
+    spin.fatal('Error: %s', err.message)
+  }
+  spin.succeed('File saved as \'%s\'', dest)
+}
+
+export default download

package/bajoCli/tool/export-to.js

@@ -12,8 +12,8 @@ function makeProgress (spin) {
 
 async function exportTo ({ path, args }) {
   const { importPkg, print, dayjs, getConfig, importModule, spinner } = this.bajo.helper
-  const { isEmpty, map } = await importPkg('lodash-es')
-  const [input, select] = await importPkg('bajo-cli:@inquirer/input',
+  const { isEmpty, map } = this.bajo.helper._
+  const [input, select] = await importPkg('bajoCli:@inquirer/input',
     'bajo-cli:@inquirer/select')
   const config = getConfig()
   if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })

package/bajoCli/tool/import-from.js

@@ -11,9 +11,9 @@ function makeProgress (spin) {
 
 async function importFrom ({ path, args }) {
   const { importPkg, print, importModule, getConfig, spinner } = this.bajo.helper
-  const { isEmpty, map } = await importPkg('lodash-es')
-  const [input, select, confirm] = await importPkg('bajo-cli:@inquirer/input',
-    'bajo-cli:@inquirer/select', 'bajo-cli:@inquirer/confirm')
+  const { isEmpty, map } = this.bajo.helper._
+  const [input, select, confirm] = await importPkg('bajoCli:@inquirer/input',
+    'bajoCli:@inquirer/select', 'bajoCli:@inquirer/confirm')
   const config = getConfig()
   if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
   const schemas = map(this.bajoDb.schemas, 'name')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.16",
+  "version": "0.3.0",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -35,6 +35,7 @@
     "fast-jwt": "^3.2.0",
     "fast-xml-parser": "^4.3.2",
     "ndjson": "^2.0.0",
+    "numbro": "^2.5.0",
     "performant-array-to-tree": "^1.11.0",
     "query-string": "^8.1.0",
     "scramjet": "^4.36.9",