dobo-extra 1.2.1 → 1.2.3

package/lib/ndjson-csv-xlsx.js

@@ -5,10 +5,12 @@ import csv from 'fast-csv'
 import xlsxparse from 'xlsx-parse-stream'
 import XLSXWriteStream from '@atomictech/xlsx-write-stream'
 import StreamArray from 'stream-json/streamers/StreamArray.js'
-import stringer from 'stream-json/Stringer.js'
-import disassembler from 'stream-json/Disassembler.js'
+import Stringer from 'stream-json/Stringer.js'
+import Disassembler from 'stream-json/Disassembler.js'
 import chain from 'stream-chain'
 
+const XLSXStreamer = XLSXWriteStream.default
+
 export default {
   ndjson: {
     parse: (...args) => ndjson.parse(...args),
@@ -20,7 +22,7 @@ export default {
   },
   xlsx: {
     parse: (...args) => xlsxparse(...args),
-    stringify: (...args) => new XLSXWriteStream(...args)
+    stringify: (...args) => new XLSXStreamer(...args)
   },
   json: {
     parse: (...args) => chain([
@@ -28,8 +30,8 @@ export default {
       data => data.value
     ]),
     stringify: (options, ...args) => chain([
-      disassembler(),
-      stringer({ ...options, makeArray: true })
+      new Disassembler(),
+      new Stringer({ ...options, makeArray: true })
     ])
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dobo-extra",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "Bajo DB Extra Tools/Utility",
   "main": "index.js",
   "scripts": {

package/plugin-method/export-to.js

@@ -27,20 +27,41 @@ async function getFile (dest, ensureDir) {
   let ext = path.extname(file)
   if (ext === '.gz') {
     compress = true
-    ext = path.extname(path.basename(file).replace('.gz', ''))
+    ext = path.extname(file.slice(0, -3))
     // file = file.slice(0, file.length - 3)
   }
   if (!supportedExt.includes(ext)) throw this.error('unsupportedFormat%s', ext.slice(1))
   return { file, ext, compress }
 }
 
-async function getData ({ source, filter, count, stream, progressFn }) {
+async function getData ({ source, filter, count, stream, progressFn, fields }) {
   let cnt = count ?? 0
-  const { recordFind } = this.app.dobo
+  const { find } = this.lib._
+  const { recordFind, getSchema } = this.app.dobo
+  const { maxLimit, hardLimit } = this.app.dobo.config.default.filter
+  filter.limit = maxLimit
+  let sort
+  const schema = getSchema(source)
+  const idField = find(schema.properties, { name: 'id' })
+  for (const name of ['createdAt', 'updatedAt', 'ts', 'dt']) {
+    const field = find(schema.properties, { name })
+    if (field) {
+      sort = field.name
+      break
+    }
+  }
+  filter.sort = `${sort ?? idField}:1`
   for (;;) {
     const batchStart = new Date()
-    const { data, page } = await recordFind(source, filter, { dataOnly: false })
+    const { data, page } = await recordFind(source, filter, { dataOnly: false, fields })
     if (data.length === 0) break
+    if (cnt + data.length > hardLimit) {
+      const sliced = data.slice(0, hardLimit - cnt)
+      await stream.pull(sliced)
+      cnt += sliced.length
+      if (progressFn) await progressFn.call(this, { batchNo: page, data: sliced, batchStart, batchEnd: new Date() })
+      break
+    }
     cnt += data.length
     await stream.pull(data)
     if (progressFn) await progressFn.call(this, { batchNo: page, data, batchStart, batchEnd: new Date() })
@@ -50,9 +71,10 @@ async function getData ({ source, filter, count, stream, progressFn }) {
   return cnt
 }
 
-function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn } = {}, opts = {}) {
+function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, batch = 500, progressFn, fields } = {}, opts = {}) {
   const { fs } = this.lib
   const { merge } = this.lib._
+  const { getInfo } = this.app.dobo
 
   filter.page = 1
   batch = parseInt(batch) ?? 500
@@ -61,7 +83,6 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
   filter.limit = batch
 
   return new Promise((resolve, reject) => {
-    const { getInfo } = this.app.dobo
     let count = 0
     let file
     let ext
@@ -69,9 +90,7 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
     let compress
     let writer
     getInfo(source)
-      .then(res => {
-        return getFile.call(this, dest, ensureDir)
-      })
+    getFile.call(this, dest, ensureDir)
       .then(res => {
         file = res.file
         ext = res.ext
@@ -93,7 +112,7 @@ function exportTo (source, dest, { filter = {}, ensureDir, useHeader = true, bat
         else if (ext === '.xlsx') pipes.push(xlsx.stringify(merge({}, { header: useHeader }, opts)))
         if (compress) pipes.push(createGzip())
         DataStream.pipeline(stream, ...pipes).pipe(writer)
-        return getData.call(this, { source, filter, count, stream, progressFn })
+        return getData.call(this, { source, filter, count, stream, fields, progressFn })
       })
       .then(cnt => {
         count = cnt