bajo-extra 0.2.11 → 0.2.13

package/bajo/config.json

@@ -10,7 +10,7 @@
       }
     },
     "import": {
-      "maxBatch": 100
+      "maxBatch": 1000
     }
   }
 }

package/bajo/helper/export-to.js

@@ -1,6 +1,6 @@
 import path from 'path'
 import scramjet from 'scramjet'
-import format from 'ndjson-csv-xlsx'
+import format from '../../lib/ndjson-csv-xlsx.js'
 import { createGzip } from 'zlib'
 
 const { json, ndjson, csv, xlsx } = format
@@ -14,7 +14,7 @@ async function getFile (dest, ensureDir) {
   let file
   if (path.isAbsolute(dest)) file = dest
   else {
-    file = `${getPluginDataDir('bajoDb')}/export/${dest}`
+    file = `${getPluginDataDir('bajoExtra')}/export/${dest}`
     fs.ensureDirSync(path.dirname(file))
   }
   file = increment(file, { fs: true })

package/bajo/helper/fetch-and-save.js

@@ -10,24 +10,34 @@ async function handler (rec, bulk) {
   save.checkUnique = save.checkUnique ?? 'id'
   if (['unique', 'upsert'].includes(save.mode)) {
     const query = isFunction(save.checkUnique) ? await save.checkUnique.call(this, rec, save) : set({}, save.checkUnique, rec[save.checkUnique])
-    const resp = await recordFind(save.coll, { query, limit: 1 }, { skipCache: true })
+    const resp = await recordFind(save.coll, { query, limit: 1 }, { noCache: true })
     if (resp.length > 0) existing = resp[0]
   }
   if (existing) {
     if (save.mode === 'upsert') {
       const body = save.updateConverter ? await save.updateConverter.call(this, rec, save) : rec
-      record = await recordUpdate(save.coll, existing.id, body)
-      method = 'updated'
+      try {
+        record = await recordUpdate(save.coll, existing.id, body)
+        method = 'updated'
+      } catch (err) {
+        console.error(err)
+        method = 'error'
+      }
     } else {
       method = 'skipped'
     }
   } else {
-    record = await recordCreate(save.coll, rec)
-    method = 'created'
+    try {
+      record = await recordCreate(save.coll, rec)
+      method = 'created'
+    } catch (err) {
+      console.error(err)
+      method = 'error'
+    }
   }
   if (record && current.coll && current.query) {
     const query = await current.query.call(this, { body: rec, record, opts: save })
-    const recs = await recordFind(current.coll, { query }, { skipCache: true })
+    const recs = await recordFind(current.coll, { query }, { noCache: true })
     const rc = current.converter ? await current.converter.call(this, { body: rec, record, opts: save }) : rec
     if (rc) {
       if (recs.length > 0) {
@@ -42,10 +52,14 @@ async function handler (rec, bulk) {
 }
 
 async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
-  const { importPkg } = this.bajo.helper
+  const { importPkg, getConfig, importModule } = this.bajo.helper
   const { fetchBulk } = this.bajoExtra.helper
   const { merge } = await importPkg('lodash-es')
   merge(bulk, { handler, save })
+  const cfgDb = getConfig('bajoDb', { full: true })
+  const start = await importModule(`${cfgDb.dir.pkg}/bajo/start.js`)
+  await start.call(this, 'all')
+
   await fetchBulk(url, bulk, opts)
 }
 

package/bajo/helper/fetch-bulk.js

@@ -18,6 +18,7 @@ async function fetching ({ url, opts, bulk, spin }) {
   let count = 0
   const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
   bulk.dataKey = bulk.dataKey ?? 'data'
+  if (bulk.printCount === true) bulk.printCount = 100
   const data = isFunction(bulk.dataKey) ? await bulk.dataKey.call(this, resp) : resp[bulk.dataKey]
   if (data.length === 0) {
     print.warn('No records to process, abort')
@@ -43,7 +44,7 @@ async function fetching ({ url, opts, bulk, spin }) {
     }
   }
   print.succeed('[%s] %d/%d records processed', spin.getElapsed(), count, data.length)
-  print.succeed('[%s] Created: %d, Updated: %d, Skipped: %d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
+  if (!bulk.noStat) print.succeed('[%s] Created: %d, Updated: %d, Skipped: %d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
   return data.length
 }
 
@@ -53,7 +54,7 @@ async function fetchBulk (url, bulk = {}, opts = {}) {
   opts.params = opts.params ?? {}
   bulk.maxStep = bulk.maxStep ?? 0
   if (!isFunction(bulk.handler)) throw error('A function handler must be provided')
-  if (bulk.paramsIncFn && isFunction(bulk.ParamsFn)) {
+  if (isFunction(bulk.paramsIncFn)) {
     print.info('Bulk fetch starting')
     const spin = spinner({ showCounter: true }).start('Fetching starts...')
     let step = 1

package/bajo/helper/import-from.js

@@ -1,28 +1,29 @@
 import path from 'path'
 import scramjet from 'scramjet'
-import format from 'ndjson-csv-xlsx'
+import format from '../../lib/ndjson-csv-xlsx.js'
 import { createGunzip } from 'zlib'
 
 const { json, ndjson, csv, xlsx } = format
 const { DataStream } = scramjet
 const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
 
-async function importFrom (source, dest, { trashOld = true, batch, progressFn, useHeader = true } = {}) {
-  const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
+async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
+  const { error, importPkg, getConfig, getPluginDataDir, secToHms } = this.bajo.helper
   if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
   const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
   await getInfo(dest)
+  const { merge } = await importPkg('lodash-es')
   const fs = await importPkg('fs-extra')
   const cfg = getConfig('bajoExtra')
 
   let file
   if (path.isAbsolute(source)) file = source
   else {
-    file = `${getPluginDataDir('bajoDb')}/import/${source}`
+    file = `${getPluginDataDir('bajoExtra')}/import/${source}`
     fs.ensureDirSync(path.dirname(file))
   }
   if (!fs.existsSync(file)) throw error('Source file \'%s\' doesn\'t exist', file)
-  let ext = path.extname(file)
+  let ext = fileType ? `.${fileType}` : path.extname(file)
   let decompress = false
   if (ext === '.gz') {
     ext = path.extname(path.basename(file, '.gz'))
@@ -31,16 +32,16 @@ async function importFrom (source, dest, { trashOld = true, batch, progressFn, u
   if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
   if (trashOld) await recordClear(dest)
   const reader = fs.createReadStream(file)
-  batch = parseInt(batch) ?? 100
+  batch = parseInt(batch) || 100
   if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
   if (batch < 0) batch = 1
   let count = 0
   const pipes = [reader]
   if (decompress) pipes.push(createGunzip())
-  if (ext === '.json') pipes.push(json.parse())
-  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse())
-  else if (ext === '.csv') pipes.push(csv.parse({ headers: useHeader }))
-  else if (ext === '.xlsx') pipes.push(xlsx.parse({ header: useHeader }))
+  if (ext === '.json') pipes.push(json.parse(opts))
+  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
+  else if (ext === '.csv') pipes.push(csv.parse(merge({ headers: useHeader }, opts)))
+  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({ header: useHeader }, opts)))
 
   const stream = DataStream.pipeline(...pipes)
   let batchNo = 1
@@ -48,13 +49,15 @@ async function importFrom (source, dest, { trashOld = true, batch, progressFn, u
     .batch(batch)
     .map(async items => {
       if (items.length === 0) return null
-      if (progressFn) await progressFn.call(this, { batchNo, data: items })
-      for (let i = 0; i < items.length; i++) {
+      const start = Date.now()
+      for (let item of items) {
         count++
-        await recordCreate(dest, items[i])
+        item = converterFn ? await converterFn.call(this, item) : item
+        await recordCreate(dest, item, createOpts)
       }
+      const diff = Date.now() - start
+      if (progressFn) await progressFn.call(this, { batchNo, data: items, time: secToHms(diff, true), timeMsec: diff })
       batchNo++
-      return null
     })
     .run()
 

package/bajoCli/tool/import-from.js

@@ -33,7 +33,7 @@ async function importFrom ({ path, args }) {
     default: false
   })
   if (!answer) return print.fail('Aborted!', { exit: config.tool })
-  const spin = spinner().start('Importing...')
+  const spin = spinner({ showCounter: true }).start('Importing...')
   const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
   const { batch } = getConfig()

package/lib/ndjson-csv-xlsx.js

@@ -0,0 +1,35 @@
+// Borrowed from: https://github.com/fanlia/ndjson-csv-xlsx/blob/main/index.js
+
+import ndjson from 'ndjson'
+import csv from 'fast-csv'
+import xlsxparse from 'xlsx-parse-stream'
+import XLSXWriteStream from '@atomictech/xlsx-write-stream'
+import StreamArray from 'stream-json/streamers/StreamArray.js'
+import stringer from 'stream-json/Stringer.js'
+import disassembler from 'stream-json/Disassembler.js'
+import chain from 'stream-chain'
+
+export default {
+  ndjson: {
+    parse: (...args) => ndjson.parse(...args),
+    stringify: (...args) => ndjson.stringify(...args)
+  },
+  csv: {
+    parse: (...args) => csv.parse(...args),
+    stringify: (...args) => csv.format(...args)
+  },
+  xlsx: {
+    parse: (...args) => xlsxparse(...args),
+    stringify: (...args) => new XLSXWriteStream(...args)
+  },
+  json: {
+    parse: (...args) => chain([
+      StreamArray.withParser(...args),
+      data => data.value
+    ]),
+    stringify: (options, ...args) => chain([
+      disassembler(),
+      stringer({ ...options, makeArray: true })
+    ])
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -26,16 +26,21 @@
   },
   "homepage": "https://github.com/ardhi/bajo-extra#readme",
   "dependencies": {
+    "@atomictech/xlsx-write-stream": "^2.0.2",
     "async": "^3.2.4",
     "axios": "^1.4.0",
     "bcrypt": "^5.1.1",
     "email-addresses": "^5.0.0",
+    "fast-csv": "^5.0.1",
     "fast-jwt": "^3.2.0",
     "fast-xml-parser": "^4.3.2",
     "littlehash": "^1.0.1",
-    "ndjson-csv-xlsx": "^1.1.1",
+    "ndjson": "^2.0.0",
     "performant-array-to-tree": "^1.11.0",
     "query-string": "^8.1.0",
-    "scramjet": "^4.36.9"
+    "scramjet": "^4.36.9",
+    "stream-chain": "^2.2.5",
+    "stream-json": "^1.8.0",
+    "xlsx-parse-stream": "^1.1.0"
   }
 }