bajo-extra 0.2.17 → 0.3.1

package/bajo/config.json

@@ -2,18 +2,5 @@
   "alias": "x",
   "fetch": {
     "agent": {}
-  },
-  "stream": {
-    "export": {
-      "maxBatch": 1000,
-      "stringify": {
-        "open": "[\n",
-        "sep": ",\n",
-        "close": "\n]\n"
-      }
-    },
-    "import": {
-      "maxBatch": 1000
-    }
   }
 }

package/bajo/helper/download.js

@@ -1,11 +1,11 @@
 import path from 'path'
 
 async function download (url, opts = {}) {
-  const { getPluginDataDir, importPkg, error, generateId } = this.bajo.helper
+  const { fs, getPluginDataDir, importPkg, error, generateId } = this.bajo.helper
   const { fetch, formatByte, formatPercentage } = this.bajoExtra.helper
-  const { isFunction } = await importPkg('lodash-es')
+  const { isFunction } = this.bajo.helper._
   if (typeof opts === 'string') opts = { dir: opts }
-  const [fs, increment] = await importPkg('fs-extra', 'add-filename-increment')
+  const increment = await importPkg('add-filename-increment')
   if (!opts.dir) {
     opts.dir = `${getPluginDataDir('bajoExtra')}/download`
     fs.ensureDirSync(opts.dir)

package/bajo/helper/fetch-and-save.js

@@ -1,6 +1,5 @@
 async function handler (rec, bulk) {
-  const { importPkg } = this.bajo.helper
-  const { isFunction, set } = await importPkg('lodash-es')
+  const { isFunction, set } = this.bajo.helper._
   const { recordCreate, recordFind, recordUpdate } = this.bajoDb.helper
   const save = bulk.save ?? {}
   const current = save.current ?? {}
@@ -52,13 +51,11 @@ async function handler (rec, bulk) {
 }
 
 async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
-  const { importPkg, getConfig, importModule } = this.bajo.helper
+  const { startPlugin } = this.bajo.helper
   const { fetchBulk } = this.bajoExtra.helper
-  const { merge } = await importPkg('lodash-es')
+  const { merge } = this.bajo.helper._
   merge(bulk, { handler, save })
-  const cfgDb = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfgDb.dir.pkg}/bajo/start.js`)
-  await start.call(this, 'all')
+  await startPlugin('bajoDb')
 
   await fetchBulk(url, bulk, opts)
 }

package/bajo/helper/fetch-bulk.js

@@ -1,6 +1,6 @@
 async function fetching ({ url, opts, bulk, spin }) {
-  const { setImmediate, importPkg, print } = this.bajo.helper
-  const { isEmpty, isFunction, has } = await importPkg('lodash-es')
+  const { setImmediate, print } = this.bajo.helper
+  const { isEmpty, isFunction, has } = this.bajo.helper._
   const { validationErrorMessage } = this.bajoDb.helper
   const { fetch } = this.bajoExtra.helper
   const resp = await fetch(url, opts ?? {})
@@ -49,8 +49,8 @@ async function fetching ({ url, opts, bulk, spin }) {
 }
 
 async function fetchBulk (url, bulk = {}, opts = {}) {
-  const { print, spinner, importPkg, error } = this.bajo.helper
-  const { isFunction } = await importPkg('lodash-es')
+  const { print, spinner, error } = this.bajo.helper
+  const { isFunction } = this.bajo.helper._
   opts.params = opts.params ?? {}
   bulk.maxStep = bulk.maxStep ?? 0
   if (!isFunction(bulk.handler)) throw error('A function handler must be provided')

package/bajo/helper/fetch.js

@@ -3,8 +3,8 @@ import http from 'http'
 import https from 'https'
 
 async function fetch (url, opts = {}, ext = {}) {
-  const { importPkg, getConfig } = this.bajo.helper
-  const { has, isPlainObject, cloneDeep, isEmpty } = await importPkg('lodash-es')
+  const { getConfig } = this.bajo.helper
+  const { has, isPlainObject, cloneDeep, isEmpty } = this.bajo.helper._
   const cfg = getConfig('bajoExtra')
   if (isPlainObject(url)) {
     ext = cloneDeep(opts)

package/bajo/helper/gzip.js

@@ -1,22 +1,19 @@
 import { createGzip, createGunzip } from 'zlib'
 
 function gzip (file, deleteOld, expand) {
+  const { fs } = this.bajo.helper
   return new Promise((resolve, reject) => {
-    const { importPkg } = this.bajo.helper
-    importPkg('fs-extra')
-      .then(fs => {
-        const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
-        const reader = fs.createReadStream(file)
-        const writer = fs.createWriteStream(newFile)
-        const method = expand ? createGunzip() : createGzip()
-        reader.pipe(method).pipe(writer)
-        writer.on('error', reject)
-        writer.on('finish', err => {
-          if (err) return reject(err)
-          if (deleteOld) fs.unlinkSync(file)
-          resolve()
-        })
-      })
+    const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
+    const reader = fs.createReadStream(file)
+    const writer = fs.createWriteStream(newFile)
+    const method = expand ? createGunzip() : createGzip()
+    reader.pipe(method).pipe(writer)
+    writer.on('error', reject)
+    writer.on('finish', err => {
+      if (err) return reject(err)
+      if (deleteOld) fs.unlinkSync(file)
+      resolve()
+    })
   })
 }
 

package/bajoCli/tool.js

@@ -1,8 +1 @@
-async function tool ({ path, args = [] }) {
-  const { currentLoc } = this.bajo.helper
-  const { runToolMethod } = this.bajoCli.helper
-  const options = { demonize: ['shell'] }
-  await runToolMethod({ path, args, dir: `${currentLoc(import.meta).dir}/tool`, options })
-}
-
-export default tool
+export default 'defCliHandler'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.17",
+  "version": "0.3.1",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -26,21 +26,13 @@
   },
   "homepage": "https://github.com/ardhi/bajo-extra#readme",
   "dependencies": {
-    "@atomictech/xlsx-write-stream": "^2.0.2",
     "async": "^3.2.4",
     "axios": "^1.4.0",
     "bcrypt": "^5.1.1",
     "email-addresses": "^5.0.0",
-    "fast-csv": "^5.0.1",
-    "fast-jwt": "^3.2.0",
-    "fast-xml-parser": "^4.3.2",
-    "ndjson": "^2.0.0",
+    "fast-xml-parser": "^4.3.6",
     "numbro": "^2.5.0",
     "performant-array-to-tree": "^1.11.0",
-    "query-string": "^8.1.0",
-    "scramjet": "^4.36.9",
-    "stream-chain": "^2.2.5",
-    "stream-json": "^1.8.0",
-    "xlsx-parse-stream": "^1.1.0"
+    "query-string": "^8.1.0"
   }
 }

package/bajo/helper/export-to.js

@@ -1,115 +0,0 @@
-import path from 'path'
-import scramjet from 'scramjet'
-import format from '../../lib/ndjson-csv-xlsx.js'
-import { createGzip } from 'zlib'
-
-const { json, ndjson, csv, xlsx } = format
-const { DataStream } = scramjet
-
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
-
-async function getFile (dest, ensureDir) {
-  const { importPkg, error, getPluginDataDir } = this.bajo.helper
-  const [fs, increment] = await importPkg('fs-extra', 'add-filename-increment')
-  let file
-  if (path.isAbsolute(dest)) file = dest
-  else {
-    file = `${getPluginDataDir('bajoExtra')}/export/${dest}`
-    fs.ensureDirSync(path.dirname(file))
-  }
-  file = increment(file, { fs: true })
-  const dir = path.dirname(file)
-  if (!fs.existsSync(dir)) {
-    if (ensureDir) fs.ensureDirSync(dir)
-    else throw error('Directory \'%s\' doesn\'t exist', dir)
-  }
-  let compress = false
-  let ext = path.extname(file)
-  if (ext === '.gz') {
-    compress = true
-    ext = path.extname(path.basename(file).replace('.gz', ''))
-    // file = file.slice(0, file.length - 3)
-  }
-  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  return { file, ext, compress }
-}
-
-async function getData ({ source, filter, count, stream, progressFn }) {
-  let cnt = count ?? 0
-  const { recordFind } = this.bajoDb.helper
-  for (;;) {
-    const batchStart = new Date()
-    const { data, page } = await recordFind(source, filter, { dataOnly: false })
-    if (data.length === 0) break
-    cnt += data.length
-    await stream.pull(data)
-    if (progressFn) await progressFn.call(this, { batchNo: page, data, batchStart, batchEnd: new Date() })
-    filter.page++
-  }
-  await stream.end()
-  return cnt
-}
-
-function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
-  const { error, importPkg, getConfig } = this.bajo.helper
-  const cfg = getConfig('bajoExtra')
-  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-  filter.page = 1
-  batch = parseInt(batch) ?? 500
-  if (batch > cfg.stream.export.maxBatch) batch = cfg.stream.export.maxBatch
-  if (batch < 0) batch = 1
-  filter.limit = batch
-
-  return new Promise((resolve, reject) => {
-    const { getInfo } = this.bajoDb.helper
-    let count = 0
-    let fs
-    let merge
-    let file
-    let ext
-    let stream
-    let compress
-    let writer
-    getInfo(source)
-      .then(() => {
-        return importPkg('lodash-es')
-      })
-      .then(l => {
-        merge = l.merge
-        return importPkg('fs-extra')
-      })
-      .then(res => {
-        fs = res
-        return getFile.call(this, dest, ensureDir)
-      })
-      .then(res => {
-        file = res.file
-        ext = res.ext
-        compress = res.compress
-        writer = fs.createWriteStream(file)
-        writer.on('error', err => {
-          reject(err)
-        })
-        writer.on('finish', () => {
-          resolve({ file, count })
-        })
-        stream = new DataStream()
-        stream = stream.flatMap(items => (items))
-        const pipes = []
-        if (ext === '.json') pipes.push(json.stringify(opts))
-        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify(opts))
-        else if (ext === '.csv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, opts)))
-        else if (ext === '.tsv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
-        else if (ext === '.xlsx') pipes.push(xlsx.stringify(merge({}, { header: useHeader }, opts)))
-        if (compress) pipes.push(createGzip())
-        DataStream.pipeline(stream, ...pipes).pipe(writer)
-        return getData.call(this, { source, filter, count, stream, progressFn })
-      })
-      .then(cnt => {
-        count = cnt
-      })
-      .catch(reject)
-  })
-}
-
-export default exportTo

package/bajo/helper/import-from.js

@@ -1,70 +0,0 @@
-import path from 'path'
-import scramjet from 'scramjet'
-import format from '../../lib/ndjson-csv-xlsx.js'
-import { createGunzip } from 'zlib'
-
-const { json, ndjson, csv, xlsx } = format
-const { DataStream } = scramjet
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
-
-async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
-  const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
-  if (dest !== false) {
-    if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-    await this.bajoDb.helper.getInfo(dest)
-  }
-  const { merge } = await importPkg('lodash-es')
-  const fs = await importPkg('fs-extra')
-  const cfg = getConfig('bajoExtra')
-
-  let file
-  if (path.isAbsolute(source)) file = source
-  else {
-    file = `${getPluginDataDir('bajoExtra')}/import/${source}`
-    fs.ensureDirSync(path.dirname(file))
-  }
-  if (!fs.existsSync(file)) throw error('Source file \'%s\' doesn\'t exist', file)
-  let ext = fileType ? `.${fileType}` : path.extname(file)
-  let decompress = false
-  if (ext === '.gz') {
-    ext = path.extname(path.basename(file, '.gz'))
-    decompress = true
-  }
-  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  if (trashOld && dest !== false) await this.bajoDb.helper.recordClear(dest)
-  const reader = fs.createReadStream(file)
-  batch = parseInt(batch) || 100
-  if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
-  if (batch < 0) batch = 1
-  let count = 0
-  const pipes = [reader]
-  if (decompress) pipes.push(createGunzip())
-  if (ext === '.json') pipes.push(json.parse(opts))
-  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
-  else if (ext === '.csv') pipes.push(csv.parse(merge({}, { headers: useHeader }, opts)))
-  else if (ext === '.tsv') pipes.push(csv.parse(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
-  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({}, { header: useHeader }, opts)))
-
-  const stream = DataStream.pipeline(...pipes)
-  let batchNo = 1
-  const data = []
-  await stream
-    .batch(batch)
-    .map(async items => {
-      if (items.length === 0) return null
-      const batchStart = new Date()
-      for (let item of items) {
-        count++
-        item = converterFn ? await converterFn.call(this, item) : item
-        if (dest !== false) await this.bajoDb.helper.recordCreate(dest, item, createOpts)
-        else data.push(item)
-      }
-      if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
-      batchNo++
-    })
-    .run()
-
-  return dest === false ? data : { file, count }
-}
-
-export default importFrom

package/bajoCli/tool/export-to.js

@@ -1,57 +0,0 @@
-import Path from 'path'
-
-const batch = 100
-
-function makeProgress (spin) {
-  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
-    const { secToHms } = this.bajo.helper
-    if (data.length === 0) return
-    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
-  }
-}
-
-async function exportTo ({ path, args }) {
-  const { importPkg, print, dayjs, getConfig, importModule, spinner } = this.bajo.helper
-  const { isEmpty, map } = await importPkg('lodash-es')
-  const [input, select] = await importPkg('bajo-cli:@inquirer/input',
-    'bajo-cli:@inquirer/select')
-  const config = getConfig()
-  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
-  const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
-  let [coll, dest, query] = args
-  if (isEmpty(coll)) {
-    coll = await select({
-      message: print.__('Please choose collection:'),
-      choices: map(schemas, s => ({ value: s }))
-    })
-  }
-  if (isEmpty(dest)) {
-    dest = await input({
-      message: print.__('Please enter destination file:'),
-      default: `${coll}-${dayjs().format('YYYYMMDD')}.ndjson`,
-      validate: (item) => !isEmpty(item)
-    })
-  }
-  if (isEmpty(query)) {
-    query = await input({
-      message: print.__('Please enter a query (if any):')
-    })
-  }
-  const spin = spinner().start('Exporting...')
-  const progressFn = makeProgress.call(this, spin)
-  const cfg = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
-  const { connection } = await this.bajoDb.helper.getInfo(coll)
-  await start.call(this, connection.name)
-  try {
-    const filter = { query }
-    const result = await this.bajoExtra.helper.exportTo(coll, dest, { filter, batch, progressFn })
-    spin.succeed('%d records successfully exported to \'%s\'', result.count, Path.resolve(result.file))
-  } catch (err) {
-    console.log(err)
-    spin.fail('Error: %s', err.message, { exit: config.tool })
-  }
-}
-
-export default exportTo

package/bajoCli/tool/import-from.js

@@ -1,53 +0,0 @@
-import Path from 'path'
-
-const batch = 100
-
-function makeProgress (spin) {
-  const { secToHms } = this.bajo.helper
-  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
-    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
-  }
-}
-
-async function importFrom ({ path, args }) {
-  const { importPkg, print, importModule, getConfig, spinner } = this.bajo.helper
-  const { isEmpty, map } = await importPkg('lodash-es')
-  const [input, select, confirm] = await importPkg('bajo-cli:@inquirer/input',
-    'bajo-cli:@inquirer/select', 'bajo-cli:@inquirer/confirm')
-  const config = getConfig()
-  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
-  const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
-  let [dest, coll] = args
-  if (isEmpty(dest)) {
-    dest = await input({
-      message: print.__('Please enter source file:'),
-      validate: (item) => !isEmpty(item)
-    })
-  }
-  if (isEmpty(coll)) {
-    coll = await select({
-      message: print.__('Please choose collection:'),
-      choices: map(schemas, s => ({ value: s }))
-    })
-  }
-  const answer = await confirm({
-    message: print.__('You\'re about to replace ALL records with the new ones. Are you really sure?'),
-    default: false
-  })
-  if (!answer) return print.fail('Aborted!', { exit: config.tool })
-  const spin = spinner({ showCounter: true }).start('Importing...')
-  const progressFn = makeProgress.call(this, spin)
-  const cfg = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
-  const { connection } = await this.bajoDb.helper.getInfo(coll)
-  await start.call(this, connection.name)
-  try {
-    const result = await this.bajoExtra.helper.importFrom(dest, coll, { batch, progressFn })
-    spin.succeed('%d records successfully imported from \'%s\'', result.count, Path.resolve(result.file))
-  } catch (err) {
-    spin.fail('Error: %s', err.message, { exit: config.tool })
-  }
-}
-
-export default importFrom

package/lib/ndjson-csv-xlsx.js

@@ -1,35 +0,0 @@
-// Borrowed from: https://github.com/fanlia/ndjson-csv-xlsx/blob/main/index.js
-
-import ndjson from 'ndjson'
-import csv from 'fast-csv'
-import xlsxparse from 'xlsx-parse-stream'
-import XLSXWriteStream from '@atomictech/xlsx-write-stream'
-import StreamArray from 'stream-json/streamers/StreamArray.js'
-import stringer from 'stream-json/Stringer.js'
-import disassembler from 'stream-json/Disassembler.js'
-import chain from 'stream-chain'
-
-export default {
-  ndjson: {
-    parse: (...args) => ndjson.parse(...args),
-    stringify: (...args) => ndjson.stringify(...args)
-  },
-  csv: {
-    parse: (...args) => csv.parse(...args),
-    stringify: (...args) => csv.format(...args)
-  },
-  xlsx: {
-    parse: (...args) => xlsxparse(...args),
-    stringify: (...args) => new XLSXWriteStream(...args)
-  },
-  json: {
-    parse: (...args) => chain([
-      StreamArray.withParser(...args),
-      data => data.value
-    ]),
-    stringify: (options, ...args) => chain([
-      disassembler(),
-      stringer({ ...options, makeArray: true })
-    ])
-  }
-}