dobo-extra 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ardhi Lukianto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,23 @@
+# dobo-extra
+
+Plugin name: **doboExtra**, alias: **dbxtra**
+
+![GitHub package.json version](https://img.shields.io/github/package-json/v/ardhi/dobo-extra) ![NPM Version](https://img.shields.io/npm/v/dobo-extra)
+
+> <br />**Attention**: I do NOT accept any pull request at the moment, thanks!<br /><br />
+
+Tools & utility for [Dobo](https://github.com/ardhi/dobo)
+
+## Installation
+
+Goto your ```<bajo-base-dir>``` and type:
+
+```bash
+$ npm install dobo-extra
+```
+
+Now open your ```<bajo-data-dir>/config/.plugins``` and put ```dobo-extra``` in it
+
+## License
+
+[MIT](LICENSE)

package/bajo/config.json

@@ -0,0 +1,20 @@
+{
+  "alias": "dbx",
+  "export": {
+    "maxBatch": 1000,
+    "stringify": {
+      "open": "[\n",
+      "sep": ",\n",
+      "close": "\n]\n"
+    }
+  },
+  "import": {
+    "maxBatch": 1000
+  },
+  "archive": {
+    "tasks": [],
+    "checkInterval": false,
+    "runEarly": true
+  },
+  "dependencies": ["dobo", "bajo-extra"]
+}

package/bajo/init.js

@@ -0,0 +1,25 @@
+const types = ['datetime', 'date', 'timestamp']
+
+async function handler ({ item }) {
+  const { join } = this.app.bajo
+  const { getSchema } = this.app.dobo
+  const { has } = this.app.bajo.lib._
+  for (const f of ['source', 'destination']) {
+    if (!has(item, f)) throw this.error('Task must have %s model', f)
+    const key = `${f}Field`
+    item[key] = item[key] ?? 'createdAt'
+    const schema = getSchema(item[f])
+    const prop = schema.properties.find(p => p.name === item[key])
+    if (!prop) throw this.error('Unknown field \'%s@%s\'', item[key], item[f])
+    if (!types.includes(prop.type)) throw this.error('\'%s@%s (%s)\' is not supported (must be one of %s)', item[key], item[f], prop.type, join(types))
+  }
+  if (item.source === item.destination) throw this.error('Source & destination must be different')
+  item.maxAge = item.maxAge ?? 1 // in days, less then 1 is ignored
+}
+
+async function init () {
+  const { buildCollections } = this.app.bajo
+  this.archivers = await buildCollections({ ns: this.name, handler, container: 'archive.tasks', dupChecks: ['source'], useDefaultName: false })
+}
+
+export default init

package/bajo/method/export-to.js

@@ -0,0 +1,105 @@
+import path from 'path'
+import format from '../../lib/ndjson-csv-xlsx.js'
+import { createGzip } from 'node:zlib'
+import scramjet from 'scramjet'
+import supportedExt from '../../lib/io-exts.js'
+
+const { DataStream } = scramjet
+const { json, ndjson, csv, xlsx } = format
+
+async function getFile (dest, ensureDir) {
+  const { importPkg, getPluginDataDir } = this.app.bajo
+  const { fs } = this.app.bajo.lib
+  const increment = await importPkg('add-filename-increment')
+  let file
+  if (path.isAbsolute(dest)) file = dest
+  else {
+    file = `${getPluginDataDir(this.name)}/export/${dest}`
+    fs.ensureDirSync(path.dirname(file))
+  }
+  file = increment(file, { fs: true })
+  const dir = path.dirname(file)
+  if (!fs.existsSync(dir)) {
+    if (ensureDir) fs.ensureDirSync(dir)
+    else throw this.error('Directory \'%s\' doesn\'t exist', dir)
+  }
+  let compress = false
+  let ext = path.extname(file)
+  if (ext === '.gz') {
+    compress = true
+    ext = path.extname(path.basename(file).replace('.gz', ''))
+    // file = file.slice(0, file.length - 3)
+  }
+  if (!supportedExt.includes(ext)) throw this.error('Unsupported format \'%s\'', ext.slice(1))
+  return { file, ext, compress }
+}
+
+async function getData ({ source, filter, count, stream, progressFn }) {
+  let cnt = count ?? 0
+  const { recordFind } = this.app.dobo
+  for (;;) {
+    const batchStart = new Date()
+    const { data, page } = await recordFind(source, filter, { dataOnly: false })
+    if (data.length === 0) break
+    cnt += data.length
+    await stream.pull(data)
+    if (progressFn) await progressFn.call(this, { batchNo: page, data, batchStart, batchEnd: new Date() })
+    filter.page++
+  }
+  await stream.end()
+  return cnt
+}
+
+function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
+  const { fs } = this.app.bajo.lib
+  const { merge } = this.app.bajo.lib._
+
+  filter.page = 1
+  batch = parseInt(batch) ?? 500
+  if (batch > this.config.export.maxBatch) batch = this.config.export.maxBatch
+  if (batch < 0) batch = 1
+  filter.limit = batch
+
+  return new Promise((resolve, reject) => {
+    const { getInfo } = this.app.dobo
+    let count = 0
+    let file
+    let ext
+    let stream
+    let compress
+    let writer
+    getInfo(source)
+      .then(res => {
+        return getFile.call(this, dest, ensureDir)
+      })
+      .then(res => {
+        file = res.file
+        ext = res.ext
+        compress = res.compress
+        writer = fs.createWriteStream(file)
+        writer.on('error', err => {
+          reject(err)
+        })
+        writer.on('finish', () => {
+          resolve({ file, count })
+        })
+        stream = new DataStream()
+        stream = stream.flatMap(items => (items))
+        const pipes = []
+        if (ext === '.json') pipes.push(json.stringify(opts))
+        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify(opts))
+        else if (ext === '.csv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, opts)))
+        else if (ext === '.tsv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
+        else if (ext === '.xlsx') pipes.push(xlsx.stringify(merge({}, { header: useHeader }, opts)))
+        if (compress) pipes.push(createGzip())
+        DataStream.pipeline(stream, ...pipes).pipe(writer)
+        return getData.call(this, { source, filter, count, stream, progressFn })
+      })
+      .then(cnt => {
+        count = cnt
+      })
+      .catch(reject)
+  })
+}
+
+export default exportTo

package/bajo/method/import-from.js

@@ -0,0 +1,67 @@
+import path from 'path'
+import format from '../../lib/ndjson-csv-xlsx.js'
+import { createGunzip } from 'zlib'
+import supportedExt from '../../lib/io-exts.js'
+import scramjet from 'scramjet'
+
+const { DataStream } = scramjet
+const { json, ndjson, csv, xlsx } = format
+
+async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
+  const { getPluginDataDir } = this.app.bajo
+  const { recordCreate } = this.app.dobo
+  const { merge } = this.app.bajo.lib._
+  const { fs } = this.app.bajo.lib
+
+  if (dest !== false) this.app.dobo.getInfo(dest) // make sure dest model is valid
+  let file
+  if (path.isAbsolute(source)) file = source
+  else {
+    file = `${getPluginDataDir(this.name)}/import/${source}`
+    fs.ensureDirSync(path.dirname(file))
+  }
+  if (!fs.existsSync(file)) throw this.error('Source file \'%s\' doesn\'t exist', file)
+  let ext = fileType ? `.${fileType}` : path.extname(file)
+  let decompress = false
+  if (ext === '.gz') {
+    ext = path.extname(path.basename(file, '.gz'))
+    decompress = true
+  }
+  if (!supportedExt.includes(ext)) throw this.error('Unsupported format \'%s\'', ext.slice(1))
+  if (trashOld && dest !== false) await this.app.dobo.modelClear(dest)
+  const reader = fs.createReadStream(file)
+  batch = parseInt(batch) || 100
+  if (batch > this.config.import.maxBatch) batch = this.config.import.maxBatch
+  if (batch < 0) batch = 1
+  let count = 0
+  const pipes = [reader]
+  if (decompress) pipes.push(createGunzip())
+  if (ext === '.json') pipes.push(json.parse(opts))
+  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
+  else if (ext === '.csv') pipes.push(csv.parse(merge({}, { headers: useHeader }, opts)))
+  else if (ext === '.tsv') pipes.push(csv.parse(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
+  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({}, { header: useHeader }, opts)))
+
+  const stream = DataStream.pipeline(...pipes)
+  let batchNo = 1
+  const data = []
+  await stream
+    .batch(batch)
+    .map(async items => {
+      if (items.length === 0) return null
+      const batchStart = new Date()
+      for (let item of items) {
+        count++
+        item = converterFn ? await converterFn.call(this, item) : item
+        if (dest !== false) await recordCreate(dest, item, createOpts)
+        else data.push(item)
+      }
+      if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
+      batchNo++
+    })
+    .run()
+
+  return dest === false ? data : { file, count }
+}
+
+export default importFrom

package/bajo/start.js

@@ -0,0 +1,14 @@
+async function start () {
+  if (this.config.archive.checkInterval === false || this.config.archive.checkInterval <= 0) {
+    this.log.warn('Automatic archiving is disabled')
+    return
+  }
+  if (this.config.archive.runEarly) await this.archive()
+  this.archiveIntv = setInterval(() => {
+    this.archive().then().catch(err => {
+      this.log.error('Archive error: %s', err.message)
+    })
+  }, this.config.archive.checkInterval * 60 * 1000)
+}
+
+export default start

package/bajoCli/applet/archive.js

@@ -0,0 +1,95 @@
+async function move (task) {
+  const { importPkg } = this.app.bajo
+  const { dayjs } = this.app.bajo.lib
+  const { set } = this.app.bajo.lib._
+  const { formatInteger } = this.app.bajoExtra
+  const { recordFind, recordCreate, recordRemove, statAggregate } = this.app.dobo
+  const prompts = await importPkg('bajoCli:@inquirer/prompts')
+  const { confirm } = prompts
+  // get relevant record
+  this.print.info('Copying %s -> %s...', task.source, task.destination)
+  const mark = dayjs().subtract(task.maxAge, 'day').toDate()
+  const query = set({}, task.sourceField, { $lt: mark })
+  let count = await statAggregate(task.source, { query }, { aggregate: 'count' })
+  count = count[0].count
+  if (count === 0) {
+    this.print.warn('No record found, skipped')
+    return
+  }
+  if (this.config.prompt !== false) {
+    const answer = await confirm({
+      message: this.print.write('%d records will be archived. Continue?', count),
+      default: true
+    })
+    if (!answer) {
+      this.print.warn('Task %s -> %s cancelled', task.source, task.destination)
+      return
+    }
+  } else this.print.info('%d records will be archived', count)
+  let page = 1
+  let total = 0
+  let affected = 0
+  let error = 0
+  let isMax = false
+  const ids = []
+  const spin = this.print.spinner({ showCounter: true }).start()
+  for (;;) {
+    const results = await recordFind(task.source, { query, page, limit: 100 }, { noCache: true, noHook: true })
+    if (results.length === 0) break
+    if (this.bajo.config.tool && total % 1000 === 0 && total !== 0) this.print.succeed('Milestone #%s records copied', formatInteger(total))
+    for (const r of results) {
+      if (task.maxRecords && task.maxRecords < total) {
+        isMax = true
+        break
+      }
+      total++
+      try {
+        await recordCreate(task.destination, r, { noSanitize: true, noHook: true, noResult: true, noCheckUnique: true })
+        ids.push(r.id)
+        spin.setText('ID #%s', r.id)
+      } catch (err) {
+        error++
+      }
+    }
+    if (isMax) break
+    affected = total - error
+    page++
+  }
+  if (isMax) this.print.warn('Max of %d records reached', task.maxRecords)
+  this.print.info('Removing %s...', task.source)
+  for (const idx in ids) {
+    const id = ids[idx]
+    try {
+      await recordRemove(task.source, id, { noHook: true, noResult: true })
+      spin.setText('ID #%s', id)
+      if (this.bajo.config.tool && idx % 1000 === 0 && idx !== '0') this.print.succeed('Milestone #%s records removed', formatInteger(idx))
+    } catch (err) {}
+  }
+  spin.stop()
+  this.print.info('Archiving %s -> %s ended, moved: %s, error: %s', task.source, task.destination, formatInteger(affected), formatInteger(error))
+}
+
+async function archive (...args) {
+  const { importPkg } = this.app.bajo
+  const prompts = await importPkg('bajoCli:@inquirer/prompts')
+  const { confirm } = prompts
+  if (this.config.prompt !== false) {
+    const answer = await confirm({
+      message: this.print.write('You\'re about to manually run task archiver. Continue?'),
+      default: false
+    })
+
+    if (!answer) this.print.fatal('Aborted!')
+  }
+  await this.app.dobo.start()
+  if (this.archivers.length === 0) this.print.fatal('Nothing to archive')
+  for (const t of this.archivers.tasks) {
+    if (t.maxAge < 1) {
+      this.print.warn('Archive %s -> %s is disabled', t.source, t.destination)
+      continue
+    }
+    await move.call(this, t)
+  }
+}
+
+export default archive

package/bajoCli/applet/export-to.js

@@ -0,0 +1,56 @@
+import _path from 'path'
+
+const batch = 100
+
+function makeProgress (spin) {
+  const { secToHms } = this.app.bajo
+  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
+    if (data.length === 0) return
+    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
+  }
+}
+
+async function exportTo (...args) {
+  const { importPkg } = this.app.bajo
+  const { dayjs } = this.app.bajo.lib
+  const { isEmpty, map } = this.app.bajo.lib._
+  const { getInfo, start } = this.app.dobo
+
+  const [input, select] = await importPkg('bajoCli:@inquirer/input',
+    'bajoCli:@inquirer/select')
+  const schemas = map(this.app.dobo.schemas, 'name')
+  if (isEmpty(schemas)) return this.print.fatal('No schema found!')
+  let [model, dest, query] = args
+  if (isEmpty(model)) {
+    model = await select({
+      message: this.print.write('Please choose model:'),
+      choices: map(schemas, s => ({ value: s }))
+    })
+  }
+  if (isEmpty(dest)) {
+    dest = await input({
+      message: this.print.write('Please enter destination file:'),
+      default: `${model}-${dayjs().format('YYYYMMDD')}.ndjson`,
+      validate: (item) => !isEmpty(item)
+    })
+  }
+  if (isEmpty(query)) {
+    query = await input({
+      message: this.print.write('Please enter a query (if any):')
+    })
+  }
+  const spin = this.print.spinner().start('Exporting...')
+  const progressFn = makeProgress.call(this, spin)
+  const { connection } = getInfo(model)
+  await start(connection.name)
+  try {
+    const filter = { query }
+    const result = await this.exportTo(model, dest, { filter, batch, progressFn })
+    spin.succeed('%d records successfully exported to \'%s\'', result.count, _path.resolve(result.file))
+  } catch (err) {
+    console.log(err)
+    spin.fatal('Error: %s', err.message)
+  }
+}
+
+export default exportTo

package/bajoCli/applet/import-from.js

@@ -0,0 +1,51 @@
+import _path from 'path'
+
+const batch = 100
+
+function makeProgress (spin) {
+  const { secToHms } = this.app.bajo
+  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
+    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
+  }
+}
+
+async function importFrom (...args) {
+  const { importPkg } = this.app.bajo
+  const { isEmpty, map } = this.app.bajo.lib._
+  const { getInfo, start } = this.app.dobo
+
+  const [input, select, confirm] = await importPkg('bajoCli:@inquirer/input',
+    'bajoCli:@inquirer/select', 'bajoCli:@inquirer/confirm')
+  const schemas = map(this.app.dobo.schemas, 'name')
+  if (isEmpty(schemas)) return this.print.fatal('No schema found!')
+  let [dest, model] = args
+  if (isEmpty(dest)) {
+    dest = await input({
+      message: this.print.write('Please enter source file:'),
+      validate: (item) => !isEmpty(item)
+    })
+  }
+  if (isEmpty(model)) {
+    model = await select({
+      message: this.print.write('Please choose model:'),
+      choices: map(schemas, s => ({ value: s }))
+    })
+  }
+  const answer = await confirm({
+    message: this.print.write('You\'re about to replace ALL records with the new ones. Are you really sure?'),
+    default: false
+  })
+  if (!answer) return this.print.fatal('Aborted!')
+  const spin = this.print.spinner({ showCounter: true }).start('Importing...')
+  const progressFn = makeProgress.call(this, spin)
+  const { connection } = getInfo(model)
+  await start(connection.name)
+  try {
+    const result = await importFrom(dest, model, { batch, progressFn })
+    spin.succeed('%d records successfully imported from \'%s\'', result.count, _path.resolve(result.file))
+  } catch (err) {
+    spin.fatal('Error: %s', err.message)
+  }
+}
+
+export default importFrom

package/bajoCli/applet.js

@@ -0,0 +1 @@
+export default 'default'

package/bajoI18N/resource/id.json

@@ -0,0 +1,37 @@
+{
+  "Directory '%s' doesn't exist": "",
+  "Unsupported format '%s'": "",
+  "Source file '%s' doesn't exist": "",
+  "Task must have %s model": "",
+  "Unknown field '%s@%s'": "",
+  "'%s@%s (%s)' is not supported (must be one of %s)": "",
+  "Source & destination must be different": "",
+  "Automatic archiving is disabled": "",
+  "Archive error: %s": "",
+  "Copying %s -> %s...": "",
+  "No record found, skipped": "",
+  "%d records will be archived. Continue?": "",
+  "Task %s -> %s cancelled": "",
+  "%d records will be archived": "",
+  "Milestone #%s records copied": "",
+  "ID #%s": "",
+  "Max of %d records reached": "",
+  "Removing %s...": "",
+  "Milestone #%s records removed": "",
+  "Archiving %s -> %s ended, moved: %s, error: %s": "",
+  "You're about to manually run task archiver. Continue?": "",
+  "Nothing to archive": "",
+  "Archive %s -> %s is disabled": "",
+  "Batch #%d (%s)": "",
+  "No schema found!": "",
+  "Please choose model:": "",
+  "Please enter destination file:": "",
+  "Please enter a query (if any):": "",
+  "Exporting...": "",
+  "%d records successfully exported to '%s'": "",
+  "Error: %s": "",
+  "Please enter source file:": "",
+  "You're about to replace ALL records with the new ones. Are you really sure?": "",
+  "Importing...": "",
+  "%d records successfully imported from '%s'": ""
+}

package/lib/io-exts.js

@@ -0,0 +1 @@
+export default ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']

package/lib/ndjson-csv-xlsx.js

@@ -0,0 +1,35 @@
+// Borrowed from: https://github.com/fanlia/ndjson-csv-xlsx/blob/main/index.js
+
+import ndjson from 'ndjson'
+import csv from 'fast-csv'
+import xlsxparse from 'xlsx-parse-stream'
+import XLSXWriteStream from '@atomictech/xlsx-write-stream'
+import StreamArray from 'stream-json/streamers/StreamArray.js'
+import stringer from 'stream-json/Stringer.js'
+import disassembler from 'stream-json/Disassembler.js'
+import chain from 'stream-chain'
+
+export default {
+  ndjson: {
+    parse: (...args) => ndjson.parse(...args),
+    stringify: (...args) => ndjson.stringify(...args)
+  },
+  csv: {
+    parse: (...args) => csv.parse(...args),
+    stringify: (...args) => csv.format(...args)
+  },
+  xlsx: {
+    parse: (...args) => xlsxparse(...args),
+    stringify: (...args) => new XLSXWriteStream(...args)
+  },
+  json: {
+    parse: (...args) => chain([
+      StreamArray.withParser(...args),
+      data => data.value
+    ]),
+    stringify: (options, ...args) => chain([
+      disassembler(),
+      stringer({ ...options, makeArray: true })
+    ])
+  }
+}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "dobo-extra",
+  "version": "1.0.0",
+  "description": "Bajo DB Extra Tools/Utility",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ardhi/dobo-extra.git"
+  },
+  "keywords": [
+    "extra",
+    "db",
+    "driver",
+    "bajo",
+    "framework",
+    "modular"
+  ],
+  "author": "Ardhi Lukianto <ardhi@lukianto.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ardhi/dobo-extra/issues"
+  },
+  "homepage": "https://github.com/ardhi/dobo-extra#readme",
+  "dependencies": {
+    "@atomictech/xlsx-write-stream": "^2.0.2",
+    "fast-csv": "^5.0.1",
+    "fast-jwt": "^4.0.2",
+    "ndjson": "^2.0.0",
+    "scramjet": "^4.37.0",
+    "stream-chain": "^2.2.5",
+    "stream-json": "^1.8.0",
+    "xlsx-parse-stream": "^1.1.0"
+  }
+}