bajo-extra 0.2.12 → 0.2.14

package/bajo/config.json

@@ -10,7 +10,7 @@
       }
     },
     "import": {
-      "maxBatch": 100
+      "maxBatch": 1000
     }
   }
 }

package/bajo/helper/export-to.js

@@ -1,6 +1,6 @@
 import path from 'path'
 import scramjet from 'scramjet'
-import format from 'ndjson-csv-xlsx'
+import format from '../../lib/ndjson-csv-xlsx.js'
 import { createGzip } from 'zlib'
 
 const { json, ndjson, csv, xlsx } = format
@@ -14,7 +14,7 @@ async function getFile (dest, ensureDir) {
   let file
   if (path.isAbsolute(dest)) file = dest
   else {
-    file = `${getPluginDataDir('bajoDb')}/export/${dest}`
+    file = `${getPluginDataDir('bajoExtra')}/export/${dest}`
     fs.ensureDirSync(path.dirname(file))
   }
   file = increment(file, { fs: true })

package/bajo/helper/fetch-and-save.js

@@ -10,7 +10,7 @@ async function handler (rec, bulk) {
   save.checkUnique = save.checkUnique ?? 'id'
   if (['unique', 'upsert'].includes(save.mode)) {
     const query = isFunction(save.checkUnique) ? await save.checkUnique.call(this, rec, save) : set({}, save.checkUnique, rec[save.checkUnique])
-    const resp = await recordFind(save.coll, { query, limit: 1 }, { skipCache: true })
+    const resp = await recordFind(save.coll, { query, limit: 1 }, { noCache: true })
     if (resp.length > 0) existing = resp[0]
   }
   if (existing) {
@@ -37,7 +37,7 @@ async function handler (rec, bulk) {
   }
   if (record && current.coll && current.query) {
     const query = await current.query.call(this, { body: rec, record, opts: save })
-    const recs = await recordFind(current.coll, { query }, { skipCache: true })
+    const recs = await recordFind(current.coll, { query }, { noCache: true })
     const rc = current.converter ? await current.converter.call(this, { body: rec, record, opts: save }) : rec
     if (rc) {
       if (recs.length > 0) {

package/bajo/helper/import-from.js

@@ -1,28 +1,29 @@
 import path from 'path'
 import scramjet from 'scramjet'
-import format from 'ndjson-csv-xlsx'
+import format from '../../lib/ndjson-csv-xlsx.js'
 import { createGunzip } from 'zlib'
 
 const { json, ndjson, csv, xlsx } = format
 const { DataStream } = scramjet
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
+const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
 
-async function importFrom (source, dest, { trashOld = true, batch, progressFn, useHeader = true } = {}) {
+async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
   const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
   if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
   const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
   await getInfo(dest)
+  const { merge } = await importPkg('lodash-es')
   const fs = await importPkg('fs-extra')
   const cfg = getConfig('bajoExtra')
 
   let file
   if (path.isAbsolute(source)) file = source
   else {
-    file = `${getPluginDataDir('bajoDb')}/import/${source}`
+    file = `${getPluginDataDir('bajoExtra')}/import/${source}`
     fs.ensureDirSync(path.dirname(file))
   }
   if (!fs.existsSync(file)) throw error('Source file \'%s\' doesn\'t exist', file)
-  let ext = path.extname(file)
+  let ext = fileType ? `.${fileType}` : path.extname(file)
   let decompress = false
   if (ext === '.gz') {
     ext = path.extname(path.basename(file, '.gz'))
@@ -31,16 +32,17 @@ async function importFrom (source, dest, { trashOld = true, batch, progressFn, u
   if (!supportedExt.includes(ext)) throw error('Unsupported format \'%s\'', ext.slice(1))
   if (trashOld) await recordClear(dest)
   const reader = fs.createReadStream(file)
-  batch = parseInt(batch) ?? 100
+  batch = parseInt(batch) || 100
   if (batch > cfg.stream.import.maxBatch) batch = cfg.stream.import.maxBatch
   if (batch < 0) batch = 1
   let count = 0
   const pipes = [reader]
   if (decompress) pipes.push(createGunzip())
-  if (ext === '.json') pipes.push(json.parse())
-  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse())
-  else if (ext === '.csv') pipes.push(csv.parse({ headers: useHeader }))
-  else if (ext === '.xlsx') pipes.push(xlsx.parse({ header: useHeader }))
+  if (ext === '.json') pipes.push(json.parse(opts))
+  else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
+  else if (ext === '.csv') pipes.push(csv.parse(merge({ headers: useHeader }, opts)))
+  else if (ext === '.tsv') pipes.push(csv.parse(merge({ headers: useHeader }, merge(opts, { delimiter: '\t' }))))
+  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({ header: useHeader }, opts)))
 
   const stream = DataStream.pipeline(...pipes)
   let batchNo = 1
@@ -48,13 +50,13 @@ async function importFrom (source, dest, { trashOld = true, batch, progressFn, u
     .batch(batch)
     .map(async items => {
       if (items.length === 0) return null
-      if (progressFn) await progressFn.call(this, { batchNo, data: items })
-      for (let i = 0; i < items.length; i++) {
+      for (let item of items) {
         count++
-        await recordCreate(dest, items[i])
+        item = converterFn ? await converterFn.call(this, item) : item
+        await recordCreate(dest, item, createOpts)
       }
+      if (progressFn) await progressFn.call(this, { batchNo, data: items })
       batchNo++
-      return null
     })
     .run()
 

package/bajoCli/tool/import-from.js

@@ -33,7 +33,7 @@ async function importFrom ({ path, args }) {
     default: false
   })
   if (!answer) return print.fail('Aborted!', { exit: config.tool })
-  const spin = spinner().start('Importing...')
+  const spin = spinner({ showCounter: true }).start('Importing...')
   const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
   const { batch } = getConfig()

package/lib/ndjson-csv-xlsx.js

@@ -0,0 +1,35 @@
+// Borrowed from: https://github.com/fanlia/ndjson-csv-xlsx/blob/main/index.js
+
+import ndjson from 'ndjson'
+import csv from 'fast-csv'
+import xlsxparse from 'xlsx-parse-stream'
+import XLSXWriteStream from '@atomictech/xlsx-write-stream'
+import StreamArray from 'stream-json/streamers/StreamArray.js'
+import stringer from 'stream-json/Stringer.js'
+import disassembler from 'stream-json/Disassembler.js'
+import chain from 'stream-chain'
+
+export default {
+  ndjson: {
+    parse: (...args) => ndjson.parse(...args),
+    stringify: (...args) => ndjson.stringify(...args)
+  },
+  csv: {
+    parse: (...args) => csv.parse(...args),
+    stringify: (...args) => csv.format(...args)
+  },
+  xlsx: {
+    parse: (...args) => xlsxparse(...args),
+    stringify: (...args) => new XLSXWriteStream(...args)
+  },
+  json: {
+    parse: (...args) => chain([
+      StreamArray.withParser(...args),
+      data => data.value
+    ]),
+    stringify: (options, ...args) => chain([
+      disassembler(),
+      stringer({ ...options, makeArray: true })
+    ])
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -26,16 +26,21 @@
   },
   "homepage": "https://github.com/ardhi/bajo-extra#readme",
   "dependencies": {
+    "@atomictech/xlsx-write-stream": "^2.0.2",
     "async": "^3.2.4",
     "axios": "^1.4.0",
     "bcrypt": "^5.1.1",
     "email-addresses": "^5.0.0",
+    "fast-csv": "^5.0.1",
     "fast-jwt": "^3.2.0",
     "fast-xml-parser": "^4.3.2",
     "littlehash": "^1.0.1",
-    "ndjson-csv-xlsx": "^1.1.1",
+    "ndjson": "^2.0.0",
     "performant-array-to-tree": "^1.11.0",
     "query-string": "^8.1.0",
-    "scramjet": "^4.36.9"
+    "scramjet": "^4.36.9",
+    "stream-chain": "^2.2.5",
+    "stream-json": "^1.8.0",
+    "xlsx-parse-stream": "^1.1.0"
   }
 }