bajo-extra 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Ardhi Lukianto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1 @@
+# bajo-extra

package/bajo/config.json

@@ -0,0 +1,16 @@
+{
+  "alias": "x",
+  "stream": {
+    "export": {
+      "maxBatch": 1000,
+      "stringify": {
+        "open": "[\n",
+        "sep": ",\n",
+        "close": "\n]\n"
+      }
+    },
+    "import": {
+      "maxBatch": 100
+    }
+  }
+}

package/bajo/helper/export-to.js

@@ -0,0 +1,109 @@
+import path from 'path'
+import scramjet from 'scramjet'
+import format from 'ndjson-csv-xlsx'
+import { createGzip } from 'zlib'
+
+const { json, ndjson, csv, xlsx } = format
+const { DataStream } = scramjet
+
+const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
+
+async function getFile (dest, ensureDir) {
+  const { importPkg, getConfig, error } = this.bajo.helper
+  const [fs, increment] = await importPkg('fs-extra', 'add-filename-increment')
+  const config = getConfig()
+  let file
+  if (path.isAbsolute(dest)) file = dest
+  else {
+    file = `${config.dir.data}/plugins/bajoDb/export/${dest}`
+    fs.ensureDirSync(path.dirname(file))
+  }
+  file = increment(file, { fs: true })
+  const dir = path.dirname(file)
+  if (!fs.existsSync(dir)) {
+    if (ensureDir) fs.ensureDirSync(dir)
+    else throw error('Directory \'%s\' doesn\'t exist', dir)
+  }
+  let compress = false
+  let ext = path.extname(file)
+  if (ext === '.gz') {
+    compress = true
+    ext = path.extname(path.basename(file).replace('.gz', ''))
+    // file = file.slice(0, file.length - 3)
+  }
+  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
+  return { file, ext, compress }
+}
+
+async function getData ({ source, filter, count, stream, progressFn }) {
+  let cnt = count ?? 0
+  const { recordFind } = this.bajoDb.helper
+  for (;;) {
+    const { data, pages, page } = await recordFind(source, filter, { dataOnly: false })
+    if (data.length === 0) break
+    cnt += data.length
+    await stream.pull(data)
+    if (progressFn) await progressFn.call(this, { batchTotal: pages, batchNo: page, data })
+    filter.page++
+  }
+  await stream.end()
+  return cnt
+}
+
+function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}) {
+  const { error, importPkg, getConfig } = this.bajo.helper
+  const cfg = getConfig('bajoExtra')
+  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
+  filter.page = 1
+  batch = parseInt(batch) ?? 500
+  if (batch > cfg.stream.export.maxBatch) batch = cfg.stream.export.maxBatch
+  if (batch < 0) batch = 1
+  filter.limit = batch
+
+  return new Promise((resolve, reject) => {
+    const { getInfo } = this.bajoDb.helper
+    let count = 0
+    let fs
+    let file
+    let ext
+    let stream
+    let compress
+    let writer
+    getInfo(source)
+      .then(() => {
+        return importPkg('fs-extra')
+      })
+      .then(res => {
+        fs = res
+        return getFile.call(this, dest, ensureDir)
+      })
+      .then(res => {
+        file = res.file
+        ext = res.ext
+        compress = res.compress
+        writer = fs.createWriteStream(file)
+        writer.on('error', err => {
+          reject(err)
+        })
+        writer.on('finish', () => {
+          resolve({ file, count })
+        })
+        stream = new DataStream()
+        stream = stream.flatMap(items => (items))
+        const pipes = []
+        if (ext === '.json') pipes.push(json.stringify())
+        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify())
+        else if (ext === '.csv') pipes.push(csv.stringify({ headers: useHeader }))
+        else if (ext === '.xlsx') pipes.push(xlsx.stringify({ header: useHeader }))
+        if (compress) pipes.push(createGzip())
+        DataStream.pipeline(stream, ...pipes).pipe(writer)
+        return getData.call(this, { source, filter, count, stream, progressFn })
+      })
+      .then(cnt => {
+        count = cnt
+      })
+      .catch(reject)
+  })
+}
+
+export default exportTo

package/bajo/helper/fetch.js

@@ -0,0 +1,18 @@
+import axios from 'axios'
+
+async function fetch (url, opts = {}, ext = {}) {
+  const { importPkg } = this.bajo.helper
+  const { has, isPlainObject, cloneDeep } = await importPkg('lodash-es')
+  if (isPlainObject(url)) {
+    ext = cloneDeep(opts)
+    opts = cloneDeep(url)
+  } else opts.url = url
+  opts.params = opts.params ?? {}
+  if (!has(ext, 'cacheBuster')) ext.cacheBuster = true
+  if (ext.cacheBuster) opts.params[ext.cacheBusterKey ?? '_'] = Date.now()
+  const resp = await axios(opts)
+  if (ext.rawResponse) return resp
+  return resp.data
+}
+
+export default fetch

package/bajo/helper/gunzip.js

@@ -0,0 +1,7 @@
+import gzip from './gzip.js'
+
+async function gunzip (file, deleteOld) {
+  await gzip.call(this, file, deleteOld, true)
+}
+
+export default gunzip

package/bajo/helper/gzip.js

@@ -0,0 +1,23 @@
+import { createGzip, createGunzip } from 'zlib'
+
+function gzip (file, deleteOld, unzip) {
+  return new Promise((resolve, reject) => {
+    const { importPkg } = this.bajo.helper
+    importPkg('fs-extra')
+      .then(fs => {
+        const newFile = unzip ? file.slice(0, file.length - 3) : (file + '.gz')
+        const reader = fs.createReadStream(file)
+        const writer = fs.createWriteStream(newFile)
+        const method = unzip ? createGunzip() : createGzip()
+        reader.pipe(method).pipe(writer)
+        writer.on('error', reject)
+        writer.on('finish', err => {
+          if (err) return reject(err)
+          if (deleteOld) fs.unlinkSync(file)
+          resolve()
+        })
+      })
+  })
+}
+
+export default gzip

package/bajo/helper/import-from.js

@@ -0,0 +1,65 @@
+import path from 'path'
+import scramjet from 'scramjet'
+import format from 'ndjson-csv-xlsx'
+import { createGunzip } from 'zlib'
+
+const { json, ndjson, csv, xlsx } = format
+const { DataStream } = scramjet
+const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
+
+async function importFrom (source, dest, { trashOld = true, batch, progressFn, useHeader = true } = {}) {
+  const { error, importPkg, getConfig } = this.bajo.helper
+  if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
+  const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
+  await getInfo(dest)
+  const fs = await importPkg('fs-extra')
+  const config = getConfig()
+  const cfg = getConfig('bajoExtra')
+
+  let file
+  if (path.isAbsolute(source)) file = source
+  else {
+    file = `${config.dir.data}/plugins/bajoDb/import/${source}`
+    fs.ensureDirSync(path.dirname(file))
+  }
+  if (!fs.existsSync(file)) throw error('Source file \'%s\' doesn\'t exist', file)
+  let ext = path.extname(file)
+  let decompress = false
+  if (ext === '.gz') {
+    ext = path.extname(path.basename(file, '.gz'))
+    decompress = true
+  }
+  if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
+  if (trashOld) await recordClear(dest)
+  const reader = fs.createReadStream(file)
+  batch = parseInt(batch) ?? 100
+  if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
+  if (batch < 0) batch = 1
+  let count = 0
+  const pipes = [reader]
+  if (decompress) pipes.push(createGunzip())
+  if (ext === '.json') pipes.push(json.parse())
+  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse())
+  else if (ext === '.csv') pipes.push(csv.parse({ headers: useHeader }))
+  else if (ext === '.xlsx') pipes.push(xlsx.parse({ header: useHeader }))
+
+  const stream = DataStream.pipeline(...pipes)
+  let batchNo = 1
+  await stream
+    .batch(batch)
+    .map(async items => {
+      if (items.length === 0) return null
+      if (progressFn) await progressFn.call(this, { batchNo, data: items })
+      for (let i = 0; i < items.length; i++) {
+        count++
+        await recordCreate(dest, items[i])
+      }
+      batchNo++
+      return null
+    })
+    .run()
+
+  return { file, count }
+}
+
+export default importFrom

package/bajoCli/tool/export-to.js

@@ -0,0 +1,54 @@
+import Path from 'path'
+
+function makeProgress (spinner) {
+  return async function ({ batchNo, batchTotal, data } = {}) {
+    if (batchTotal === 0) return
+    spinner.setText('Batch %d of %d (%d records)', batchNo, batchTotal, data.length)
+  }
+}
+
+async function exportTo (path, args) {
+  const { importPkg, print, dayjs, getConfig, importModule } = this.bajo.helper
+  const { isEmpty, map } = await importPkg('lodash-es')
+  const [input, select] = await importPkg('bajo-cli:@inquirer/input',
+    'bajo-cli:@inquirer/select')
+  if (!this.bajoDb) print.fatal('Bajo DB isn\'t loaded')
+  const schemas = map(this.bajoDb.schemas, 'name')
+  if (isEmpty(schemas)) print.fatal('No schema found!')
+  let [repo, dest, query] = args
+  if (isEmpty(repo)) {
+    repo = await select({
+      message: print.__('Please choose repository:'),
+      choices: map(schemas, s => ({ value: s }))
+    })
+  }
+  if (isEmpty(dest)) {
+    dest = await input({
+      message: print.__('Please enter destination file:'),
+      default: `${repo}-${dayjs().format('YYYYMMDD')}.ndjson`,
+      validate: (item) => !isEmpty(item)
+    })
+  }
+  if (isEmpty(query)) {
+    query = await input({
+      message: print.__('Please enter a query (if any):')
+    })
+  }
+  const spinner = print.bora('Exporting...').start()
+  const progressFn = makeProgress.call(this, spinner)
+  const cfg = getConfig('bajoDb', { full: true })
+  const { batch } = getConfig()
+  const start = await importModule(`${cfg.dir}/bajo/start.js`)
+  const { connection } = await this.bajoDb.helper.getInfo(repo)
+  await start.call(this, connection.name)
+  try {
+    const filter = { query }
+    const result = await this.bajoExtra.helper.exportTo(repo, dest, { filter, batch, progressFn })
+    spinner.succeed('%d records successfully exported to \'%s\'', result.count, Path.resolve(result.file))
+  } catch (err) {
+    console.log(err)
+    spinner.fatal('Error: %s', err.message)
+  }
+}
+
+export default exportTo

package/bajoCli/tool/import-from.js

@@ -0,0 +1,50 @@
+import Path from 'path'
+
+function makeProgress (spinner) {
+  return async function ({ batchNo, data } = {}) {
+    spinner.setText('Batch %d (%d records)', batchNo, data.length)
+  }
+}
+
+async function importFrom (path, args) {
+  const { importPkg, print, importModule, getConfig } = this.bajo.helper
+  const { isEmpty, map } = await importPkg('lodash-es')
+  const [input, select, confirm] = await importPkg('bajo-cli:@inquirer/input',
+    'bajo-cli:@inquirer/select', 'bajo-cli:@inquirer/confirm')
+  if (!this.bajoDb) print.fatal('Bajo DB isn\'t loaded')
+  const schemas = map(this.bajoDb.schemas, 'name')
+  if (isEmpty(schemas)) print.fatal('No schema found!')
+  let [dest, repo] = args
+  if (isEmpty(dest)) {
+    dest = await input({
+      message: print.__('Please enter source file:'),
+      validate: (item) => !isEmpty(item)
+    })
+  }
+  if (isEmpty(repo)) {
+    repo = await select({
+      message: print.__('Please choose repository:'),
+      choices: map(schemas, s => ({ value: s }))
+    })
+  }
+  const answer = await confirm({
+    message: print.__('You\'re about to replace ALL records with the new ones. Are you really sure?'),
+    default: false
+  })
+  if (!answer) print.fatal('Aborted!')
+  const spinner = print.bora('Importing...').start()
+  const progressFn = makeProgress.call(this, spinner)
+  const cfg = getConfig('bajoDb', { full: true })
+  const { batch } = getConfig()
+  const start = await importModule(`${cfg.dir}/bajo/start.js`)
+  const { connection } = await this.bajoDb.helper.getInfo(repo)
+  await start.call(this, connection.name)
+  try {
+    const result = await this.bajoExtra.helper.importFrom(dest, repo, { batch, progressFn })
+    spinner.succeed('%d records successfully imported from \'%s\'', result.count, Path.resolve(result.file))
+  } catch (err) {
+    spinner.fatal('Error: %s', err.message)
+  }
+}
+
+export default importFrom

package/bajoCli/tool.js

@@ -0,0 +1,8 @@
+async function tool ({ path, args = [] }) {
+  const { currentLoc } = this.bajo.helper
+  const { runToolMethod } = this.bajoCli.helper
+  const options = { demonize: ['shell'] }
+  await runToolMethod({ path, args, dir: `${currentLoc(import.meta).dir}/tool`, options })
+}
+
+export default tool

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "bajo-extra",
+  "version": "0.1.0",
+  "description": "Extra package for Bajo Framework",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ardhi/bajo-extra.git"
+  },
+  "keywords": [
+    "bajo",
+    "extra",
+    "nodejs",
+    "modular",
+    "module",
+    "package"
+  ],
+  "author": "Ardhi Lukianto <ardhi@lukianto.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ardhi/bajo-extra/issues"
+  },
+  "homepage": "https://github.com/ardhi/bajo-extra#readme",
+  "dependencies": {
+    "async": "^3.2.4",
+    "axios": "^1.4.0",
+    "ndjson-csv-xlsx": "^1.1.1",
+    "query-string": "^8.1.0",
+    "scramjet": "^4.36.9"
+  }
+}