bajo-extra 0.2.14 → 0.2.15

package/bajo/helper/export-to.js

@@ -6,7 +6,7 @@ import { createGzip } from 'zlib'
 const { json, ndjson, csv, xlsx } = format
 const { DataStream } = scramjet
 
-const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
+const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
 
 async function getFile (dest, ensureDir) {
   const { importPkg, error, getPluginDataDir } = this.bajo.helper
@@ -38,18 +38,19 @@ async function getData ({ source, filter, count, stream, progressFn }) {
   let cnt = count ?? 0
   const { recordFind } = this.bajoDb.helper
   for (;;) {
-    const { data, pages, page } = await recordFind(source, filter, { dataOnly: false })
+    const batchStart = new Date()
+    const { data, page } = await recordFind(source, filter, { dataOnly: false })
     if (data.length === 0) break
     cnt += data.length
     await stream.pull(data)
-    if (progressFn) await progressFn.call(this, { batchTotal: pages, batchNo: page, data })
+    if (progressFn) await progressFn.call(this, { batchNo: page, data, batchStart, batchEnd: new Date() })
     filter.page++
   }
   await stream.end()
   return cnt
 }
 
-function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}) {
+function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
   const { error, importPkg, getConfig } = this.bajo.helper
   const cfg = getConfig('bajoExtra')
   if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
@@ -63,6 +64,7 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
     const { getInfo } = this.bajoDb.helper
     let count = 0
     let fs
+    let merge
     let file
     let ext
     let stream
@@ -70,6 +72,10 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
     let writer
     getInfo(source)
       .then(() => {
+        return importPkg('lodash-es')
+      })
+      .then(l => {
+        merge = l.merge
         return importPkg('fs-extra')
       })
       .then(res => {
@@ -90,10 +96,11 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
         stream = new DataStream()
         stream = stream.flatMap(items => (items))
         const pipes = []
-        if (ext === '.json') pipes.push(json.stringify())
-        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify())
-        else if (ext === '.csv') pipes.push(csv.stringify({ headers: useHeader }))
-        else if (ext === '.xlsx') pipes.push(xlsx.stringify({ header: useHeader }))
+        if (ext === '.json') pipes.push(json.stringify(opts))
+        else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.stringify(opts))
+        else if (ext === '.csv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, opts)))
+        else if (ext === '.tsv') pipes.push(csv.stringify(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
+        else if (ext === '.xlsx') pipes.push(xlsx.stringify(merge({}, { header: useHeader }, opts)))
         if (compress) pipes.push(createGzip())
         DataStream.pipeline(stream, ...pipes).pipe(writer)
         return getData.call(this, { source, filter, count, stream, progressFn })

package/bajo/helper/import-from.js

@@ -40,9 +40,9 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
   if (decompress) pipes.push(createGunzip())
   if (ext === '.json') pipes.push(json.parse(opts))
   else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
-  else if (ext === '.csv') pipes.push(csv.parse(merge({ headers: useHeader }, opts)))
-  else if (ext === '.tsv') pipes.push(csv.parse(merge({ headers: useHeader }, merge(opts, { delimiter: '\t' }))))
-  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({ header: useHeader }, opts)))
+  else if (ext === '.csv') pipes.push(csv.parse(merge({}, { headers: useHeader }, opts)))
+  else if (ext === '.tsv') pipes.push(csv.parse(merge({}, { headers: useHeader }, merge({}, opts, { delimiter: '\t' }))))
+  else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({}, { header: useHeader }, opts)))
 
   const stream = DataStream.pipeline(...pipes)
   let batchNo = 1
@@ -50,12 +50,13 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
     .batch(batch)
     .map(async items => {
       if (items.length === 0) return null
+      const batchStart = new Date()
       for (let item of items) {
         count++
         item = converterFn ? await converterFn.call(this, item) : item
         await recordCreate(dest, item, createOpts)
       }
-      if (progressFn) await progressFn.call(this, { batchNo, data: items })
+      if (progressFn) await progressFn.call(this, { batchNo, data: items, batchStart, batchEnd: new Date() })
       batchNo++
     })
     .run()

package/bajoCli/tool/export-to.js

@@ -1,9 +1,12 @@
 import Path from 'path'
 
+const batch = 100
+
 function makeProgress (spin) {
-  return async function ({ batchNo, batchTotal, data } = {}) {
-    if (batchTotal === 0) return
-    spin.setText('Batch %d of %d (%d records)', batchNo, batchTotal, data.length)
+  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
+    const { secToHms } = this.bajo.helper
+    if (data.length === 0) return
+    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
   }
 }
 
@@ -38,7 +41,6 @@ async function exportTo ({ path, args }) {
   const spin = spinner().start('Exporting...')
   const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
-  const { batch } = getConfig()
   const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
   const { connection } = await this.bajoDb.helper.getInfo(coll)
   await start.call(this, connection.name)

package/bajoCli/tool/import-from.js

@@ -1,8 +1,11 @@
 import Path from 'path'
 
+const batch = 100
+
 function makeProgress (spin) {
-  return async function ({ batchNo, data } = {}) {
-    spin.setText('Batch %d (%d records)', batchNo, data.length)
+  const { secToHms } = this.bajo.helper
+  return async function ({ batchNo, data, batchStart, batchEnd } = {}) {
+    spin.setText('Batch #%d (%s)', batchNo, secToHms(batchEnd.toTime() - batchStart.toTime(), true))
   }
 }
 
@@ -36,7 +39,6 @@ async function importFrom ({ path, args }) {
   const spin = spinner({ showCounter: true }).start('Importing...')
   const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
-  const { batch } = getConfig()
   const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
   const { connection } = await this.bajoDb.helper.getInfo(coll)
   await start.call(this, connection.name)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.14",
+  "version": "0.2.15",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {