bajo-extra 0.3.0 → 0.3.1

package/bajo/config.json

@@ -2,18 +2,5 @@
   "alias": "x",
   "fetch": {
     "agent": {}
-  },
-  "stream": {
-    "export": {
-      "maxBatch": 1000,
-      "stringify": {
-        "open": "[\n",
-        "sep": ",\n",
-        "close": "\n]\n"
-      }
-    },
-    "import": {
-      "maxBatch": 1000
-    }
   }
 }

package/bajo/helper/fetch-and-save.js

@@ -51,13 +51,11 @@ async function handler (rec, bulk) {
 }
 
 async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
-  const { getConfig, importModule } = this.bajo.helper
+  const { startPlugin } = this.bajo.helper
   const { fetchBulk } = this.bajoExtra.helper
   const { merge } = this.bajo.helper._
   merge(bulk, { handler, save })
-  const cfgDb = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfgDb.dir.pkg}/bajo/start.js`)
-  await start.call(this, 'all')
+  await startPlugin('bajoDb')
 
   await fetchBulk(url, bulk, opts)
 }

package/bajoCli/tool.js

@@ -1,8 +1 @@
-async function tool ({ path, args = [] }) {
-  const { currentLoc } = this.bajo.helper
-  const { runToolMethod } = this.bajoCli.helper
-  const options = { demonize: ['shell'] }
-  await runToolMethod({ path, args, dir: `${currentLoc(import.meta).dir}/tool`, options })
-}
-
-export default tool
+export default 'defCliHandler'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -26,21 +26,13 @@
   },
   "homepage": "https://github.com/ardhi/bajo-extra#readme",
   "dependencies": {
-    "@atomictech/xlsx-write-stream": "^2.0.2",
     "async": "^3.2.4",
     "axios": "^1.4.0",
     "bcrypt": "^5.1.1",
     "email-addresses": "^5.0.0",
-    "fast-csv": "^5.0.1",
-    "fast-jwt": "^3.2.0",
-    "fast-xml-parser": "^4.3.2",
-    "ndjson": "^2.0.0",
+    "fast-xml-parser": "^4.3.6",
     "numbro": "^2.5.0",
     "performant-array-to-tree": "^1.11.0",
-    "query-string": "^8.1.0",
-    "scramjet": "^4.36.9",
-    "stream-chain": "^2.2.5",
-    "stream-json": "^1.8.0",
-    "xlsx-parse-stream": "^1.1.0"
+    "query-string": "^8.1.0"
   }
 }

package/bajo/helper/export-to.js

@@ -1,106 +0,0 @@
-import path from 'path'
-import scramjet from 'scramjet'
-import format from '../../lib/ndjson-csv-xlsx.js'
-import { createGzip } from 'zlib'
-
-const { json, ndjson, csv, xlsx } = format
-const { DataStream } = scramjet
-
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
-
-async function getFile (dest, ensureDir) {
-  const { fs, importPkg, error, getPluginDataDir } = this.bajo.helper
-  const increment = await importPkg('add-filename-increment')
-  let file
-  if (path.isAbsolute(dest)) file = dest
-  else {
-    file = `${getPluginDataDir('bajoExtra')}/export/${dest}`
-    fs.ensureDirSync(path.dirname(file))
-  }
-  file = increment(file, { fs: true })
-  const dir = path.dirname(file)
-  if (!fs.existsSync(dir)) {
-    if (ensureDir) fs.ensureDirSync(dir)
-    else throw error('Directory \'%s\' doesn\'t exist', dir)
-  }
-  let compress = false
-  let ext = path.extname(file)
-  if (ext === '.gz') {
-    compress = true
-    ext = path.extname(path.basename(file).replace('.gz', ''))
-    // file = file.slice(0, file.length - 3)
-  }
-  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  return { file, ext, compress }
-}
-
-async function getData ({ source, filter, count, stream, progressFn }) {
-  let cnt = count ?? 0
-  const { recordFind } = this.bajoDb.helper
-  for (;;) {
-    const batchStart = new Date()
-    const { data, page } = await recordFind(source, filter, { dataOnly: false })
-    if (data.length === 0) break
-    cnt += data.length
-    await stream.pull(data)
-    if (progressFn) await progressFn.call(this, { batchNo: page, data, batchStart, batchEnd: new Date() })
-    filter.page++
-  }
-  await stream.end()
-  return cnt
-}
-
-function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
-  const { fs, error, getConfig } = this.bajo.helper
-  const cfg = getConfig('bajoExtra')
-  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-  filter.page = 1
-  batch = parseInt(batch) ?? 500
-  if (batch > cfg.stream.export.maxBatch) batch = cfg.stream.export.maxBatch
-  if (batch < 0) batch = 1
-  filter.limit = batch
-  const { merge } = this.bajo.helper._
-
-  return new Promise((resolve, reject) => {
-    const { getInfo } = this.bajoDb.helper
-    let count = 0
-    let file
-    let ext
-    let stream
-    let compress
-    let writer
-    getInfo(source)
-      .then(res => {
-        return getFile.call(this, dest, ensureDir)
-      })
-      .then(res => {
-        file = res.file
-        ext = res.ext
-        compress = res.compress
-        writer = fs.createWriteStream(file)
-        writer.on('error', err => {
-          reject(err)
-        })
-        writer.on('finish', () => {
-          resolve({ file, count })
-        })
-        stream = new DataStream()
-        stream = stream.flatMap(items => (items))
-        const pipes = []
-        if (ext === '.json') pipes.push(json.stringify(opts))
-        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify(opts))
-        else if (ext === '.csv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, opts)))
-        else if (ext === '.tsv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
-        else if (ext === '.xlsx') pipes.push(xlsx.stringify(merge({}, { header: useHeader }, opts)))
-        if (compress) pipes.push(createGzip())
-        DataStream.pipeline(stream, ...pipes).pipe(writer)
-        return getData.call(this, { source, filter, count, stream, progressFn })
-      })
-      .then(cnt => {
-        count = cnt
-      })
-      .catch(reject)
-  })
-}
-
-export default exportTo

package/bajo/helper/import-from.js

@@ -1,69 +0,0 @@
-import path from 'path'
-import scramjet from 'scramjet'
-import format from '../../lib/ndjson-csv-xlsx.js'
-import { createGunzip } from 'zlib'
-
-const { json, ndjson, csv, xlsx } = format
-const { DataStream } = scramjet
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
-
-async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
-  const { fs, error, getConfig, getPluginDataDir } = this.bajo.helper
-  if (dest !== false) {
-    if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
-    await this.bajoDb.helper.getInfo(dest)
-  }
-  const { merge } = this.bajo.helper._
-  const cfg = getConfig('bajoExtra')
-
-  let file
-  if (path.isAbsolute(source)) file = source
-  else {
-    file = `${getPluginDataDir('bajoExtra')}/import/${source}`
-    fs.ensureDirSync(path.dirname(file))
-  }
-  if (!fs.existsSync(file)) throw error('Source file \'%s\' doesn\'t exist', file)
-  let ext = fileType ? `.${fileType}` : path.extname(file)
-  let decompress = false
-  if (ext === '.gz') {
-    ext = path.extname(path.basename(file, '.gz'))
-    decompress = true
-  }
-  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
-  if (trashOld && dest !== false) await this.bajoDb.helper.recordClear(dest)
-  const reader = fs.createReadStream(file)
-  batch = parseInt(batch) || 100
-  if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
-  if (batch < 0) batch = 1
-  let count = 0
-  const pipes = [reader]
-  if (decompress) pipes.push(createGunzip())
-  if (ext === '.json') pipes.push(json.parse(opts))
-  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
-  else if (ext === '.csv') pipes.push(csv.parse(merge({}, { headers: useHeader }, opts)))
-  else if (ext === '.tsv') pipes.push(csv.parse(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
-  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({}, { header: useHeader }, opts)))
-
-  const stream = DataStream.pipeline(...pipes)
-  let batchNo = 1
-  const data = []
-  await stream
-    .batch(batch)
-    .map(async items => {
-      if (items.length === 0) return null
-      const batchStart = new Date()
-      for (let item of items) {
-        count++
-        item = converterFn ? await converterFn.call(this, item) : item
-        if (dest !== false) await this.bajoDb.helper.recordCreate(dest, item, createOpts)
-        else data.push(item)
-      }
-      if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
-      batchNo++
-    })
-    .run()
-
-  return dest === false ? data : { file, count }
-}
-
-export default importFrom

package/bajoCli/tool/export-to.js

@@ -1,57 +0,0 @@
-import Path from 'path'
-
-const batch = 100
-
-function makeProgress (spin) {
-  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
-    const { secToHms } = this.bajo.helper
-    if (data.length === 0) return
-    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
-  }
-}
-
-async function exportTo ({ path, args }) {
-  const { importPkg, print, dayjs, getConfig, importModule, spinner } = this.bajo.helper
-  const { isEmpty, map } = this.bajo.helper._
-  const [input, select] = await importPkg('bajoCli:@inquirer/input',
-    'bajo-cli:@inquirer/select')
-  const config = getConfig()
-  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
-  const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
-  let [coll, dest, query] = args
-  if (isEmpty(coll)) {
-    coll = await select({
-      message: print.__('Please choose collection:'),
-      choices: map(schemas, s => ({ value: s }))
-    })
-  }
-  if (isEmpty(dest)) {
-    dest = await input({
-      message: print.__('Please enter destination file:'),
-      default: `${coll}-${dayjs().format('YYYYMMDD')}.ndjson`,
-      validate: (item) => !isEmpty(item)
-    })
-  }
-  if (isEmpty(query)) {
-    query = await input({
-      message: print.__('Please enter a query (if any):')
-    })
-  }
-  const spin = spinner().start('Exporting...')
-  const progressFn = makeProgress.call(this, spin)
-  const cfg = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
-  const { connection } = await this.bajoDb.helper.getInfo(coll)
-  await start.call(this, connection.name)
-  try {
-    const filter = { query }
-    const result = await this.bajoExtra.helper.exportTo(coll, dest, { filter, batch, progressFn })
-    spin.succeed('%d records successfully exported to \'%s\'', result.count, Path.resolve(result.file))
-  } catch (err) {
-    console.log(err)
-    spin.fail('Error: %s', err.message, { exit: config.tool })
-  }
-}
-
-export default exportTo

package/bajoCli/tool/import-from.js

@@ -1,53 +0,0 @@
-import Path from 'path'
-
-const batch = 100
-
-function makeProgress (spin) {
-  const { secToHms } = this.bajo.helper
-  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
-    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
-  }
-}
-
-async function importFrom ({ path, args }) {
-  const { importPkg, print, importModule, getConfig, spinner } = this.bajo.helper
-  const { isEmpty, map } = this.bajo.helper._
-  const [input, select, confirm] = await importPkg('bajoCli:@inquirer/input',
-    'bajoCli:@inquirer/select', 'bajoCli:@inquirer/confirm')
-  const config = getConfig()
-  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
-  const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
-  let [dest, coll] = args
-  if (isEmpty(dest)) {
-    dest = await input({
-      message: print.__('Please enter source file:'),
-      validate: (item) => !isEmpty(item)
-    })
-  }
-  if (isEmpty(coll)) {
-    coll = await select({
-      message: print.__('Please choose collection:'),
-      choices: map(schemas, s => ({ value: s }))
-    })
-  }
-  const answer = await confirm({
-    message: print.__('You\'re about to replace ALL records with the new ones. Are you really sure?'),
-    default: false
-  })
-  if (!answer) return print.fail('Aborted!', { exit: config.tool })
-  const spin = spinner({ showCounter: true }).start('Importing...')
-  const progressFn = makeProgress.call(this, spin)
-  const cfg = getConfig('bajoDb', { full: true })
-  const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
-  const { connection } = await this.bajoDb.helper.getInfo(coll)
-  await start.call(this, connection.name)
-  try {
-    const result = await this.bajoExtra.helper.importFrom(dest, coll, { batch, progressFn })
-    spin.succeed('%d records successfully imported from \'%s\'', result.count, Path.resolve(result.file))
-  } catch (err) {
-    spin.fail('Error: %s', err.message, { exit: config.tool })
-  }
-}
-
-export default importFrom

package/lib/ndjson-csv-xlsx.js

@@ -1,35 +0,0 @@
-// Borrowed from: https://github.com/fanlia/ndjson-csv-xlsx/blob/main/index.js
-
-import ndjson from 'ndjson'
-import csv from 'fast-csv'
-import xlsxparse from 'xlsx-parse-stream'
-import XLSXWriteStream from '@atomictech/xlsx-write-stream'
-import StreamArray from 'stream-json/streamers/StreamArray.js'
-import stringer from 'stream-json/Stringer.js'
-import disassembler from 'stream-json/Disassembler.js'
-import chain from 'stream-chain'
-
-export default {
-  ndjson: {
-    parse: (...args) => ndjson.parse(...args),
-    stringify: (...args) => ndjson.stringify(...args)
-  },
-  csv: {
-    parse: (...args) => csv.parse(...args),
-    stringify: (...args) => csv.format(...args)
-  },
-  xlsx: {
-    parse: (...args) => xlsxparse(...args),
-    stringify: (...args) => new XLSXWriteStream(...args)
-  },
-  json: {
-    parse: (...args) => chain([
-      StreamArray.withParser(...args),
-      data => data.value
-    ]),
-    stringify: (options, ...args) => chain([
-      disassembler(),
-      stringer({ ...options, makeArray: true })
-    ])
-  }
-}