bajo-extra 0.2.8 → 0.2.10

package/bajo/helper/fetch-and-save.js

@@ -1,32 +1,59 @@
-async function fetchAndSave ({ source = {}, converter, coll, current = {}, options = {} } = {}) {
-  const { print, setImmediate, importPkg, getConfig } = this.bajo.helper
-  const { isEmpty, isFunction } = await importPkg('lodash-es')
+async function batching ({ source = {}, converter, coll, current = {}, options = {}, spin } = {}) {
+  const { setImmediate, importPkg, print } = this.bajo.helper
+  const { isEmpty, isFunction, set } = await importPkg('lodash-es')
   const { fetch } = this.bajoExtra.helper
   const { recordCreate, recordFind, recordUpdate, validationErrorMessage } = this.bajoDb.helper
-  const config = getConfig()
-  const opts = { type: options.returnEarly ? 'log' : 'bora', pkg: options.pkg }
-  const showDatetime = !config.tool
-  const spinner = print.bora('Fetching starts...', { showCounter: true, showDatetime, isSilent: options.returnEarly }).start()
-  if (options.returnEarly) print.succeed('Fetching starts...', opts)
   const resp = await fetch(source.url, source.options ?? {})
-  if (isEmpty(resp)) spinner.fatal('No result from server, aborted!')
+  if (isEmpty(resp)) spin.fatal('No result from server, aborted!')
   if (source.abort) {
     const aborted = await source.abort.call(this, resp)
-    if (aborted) spinner.fatal(aborted)
+    if (aborted) spin.fatal(aborted)
   }
   let count = 0
+  const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
   const iterator = isFunction(source.iterator) ? await source.iterator.call(this, resp) : resp[source.iterator]
-  spinner.setText('Got %d records, processing...', iterator.length)
+  if (iterator.length === 0) {
+    print.warn('No records to process, abort')
+    return 0
+  }
+  spin.setText('Got %d records, processing...', iterator.length)
   for (let r of iterator) {
     await setImmediate()
     if (converter) r = await converter.call(this, r, options)
-    if (isEmpty(r)) continue
+    if (isEmpty(r)) {
+      stat.skipped++
+      continue
+    }
     try {
-      await recordCreate(coll, r)
-      if (current.coll && current.query) {
-        const query = await current.query.call(this, r)
+      let existing
+      let record
+      let method
+      options.checkUnique = options.checkUnique ?? 'id'
+      if (['unique', 'upsert'].includes(options.mode)) {
+        const query = isFunction(options.checkUnique) ? await options.checkUnique.call(this, r, options) : set({}, options.checkUnique, r[options.checkUnique])
+        const resp = await recordFind(coll, { query, limit: 1 }, { skipCache: true })
+        if (resp.length > 0) existing = resp[0]
+      }
+      if (existing) {
+        if (options.mode === 'upsert') {
+          const body = options.updateConverter ? await options.updateConverter.call(this, r, options) : r
+          record = await recordUpdate(coll, existing.id, body)
+          method = 'updated'
+          stat.updated++
+        } else {
+          stat.skipped++
+          print.warn(`[${spin.getElapsed()}] Record %s exists, skipped`, JSON.stringify(r))
+          method = 'skipped'
+        }
+      } else {
+        stat.created++
+        record = await recordCreate(coll, r)
+        method = 'created'
+      }
+      if (record && current.coll && current.query) {
+        const query = await current.query.call(this, { body: r, record, options })
         const recs = await recordFind(current.coll, { query }, { skipCache: true })
-        const rc = current.converter ? await current.converter.call(this, r, options) : r
+        const rc = current.converter ? await current.converter.call(this, { body: r, record, options }) : r
         if (rc) {
           if (recs.length > 0) {
             const id = recs[0].id
@@ -36,17 +63,42 @@ async function fetchAndSave ({ source = {}, converter, coll, current = {}, optio
           }
         }
       }
-      if (options.printCount && (count % options.printCount === 0)) print.succeed(`[${spinner.getElapsed()}] Batch line %d/%d`, count, iterator.length, opts)
-      else spinner.setText('Record %d/%d...', count, iterator.length)
+      if (options.printCount && options.printCount < count && (count % options.printCount === 0)) print.succeed('[%s] Processed %d/%d', spin.getElapsed(), count, iterator.length)
+      else if (!spin.opts.isLog) spin.setText('Record %d/%d...', count, iterator.length, method)
       count++
     } catch (err) {
       console.log(err)
-      spinner.setText(validationErrorMessage(err) + ', continue')
+      spin.setText(validationErrorMessage(err) + ', continue')
+    }
+  }
+  print.succeed('[%s] %d/%d records processed', spin.getElapsed(), count, iterator.length)
+  print.succeed('[%s] Created: %d, Updated: %d, Skipped: %d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
+  return iterator.length
+}
+
+async function fetchAndSave ({ source = {}, converter, coll, current = {}, options = {} } = {}) {
+  const { print, spinner } = this.bajo.helper
+  source.options = source.options ?? {}
+  source.options.params = source.options.params ?? {}
+  if (options.batch) {
+    print.info('Batch starting')
+    const spin = spinner({ showCounter: true }).start('Fetching starts...')
+    let step = 1
+    for (;;) {
+      print.info('[%s] Fetch batch #%d', spin.getElapsed(), step)
+      const newSource = await options.batch.call(this, source)
+      if (newSource) source = newSource
+      const length = await batching.call(this, { source, converter, coll, current, options, spin })
+      if (length === 0) {
+        print.info('All done!')
+        break
+      }
+      step++
     }
+  } else {
+    const spin = spinner({ showCounter: true }).start('Fetching starts...')
+    await batching.call(this, { source, converter, coll, current, options, spin })
   }
-  spinner.info(`${count}/${iterator.length} records processed`)
-  if (options.returnEarly) print.info(`${count}/${iterator.length} records processed`, opts)
-  else spinner.succeed('Done!')
 }
 
 export default fetchAndSave

package/bajoCli/tool/export-to.js

@@ -1,26 +1,21 @@
 import Path from 'path'
 
-function makeProgress (spinner) {
+function makeProgress (spin) {
   return async function ({ batchNo, batchTotal, data } = {}) {
     if (batchTotal === 0) return
-    spinner.setText('Batch %d of %d (%d records)', batchNo, batchTotal, data.length)
+    spin.setText('Batch %d of %d (%d records)', batchNo, batchTotal, data.length)
   }
 }
 
-async function exportTo ({ path, args, returnEarly }) {
-  const { importPkg, print, dayjs, getConfig, importModule } = this.bajo.helper
+async function exportTo ({ path, args }) {
+  const { importPkg, print, dayjs, getConfig, importModule, spinner } = this.bajo.helper
   const { isEmpty, map } = await importPkg('lodash-es')
   const [input, select] = await importPkg('bajo-cli:@inquirer/input',
     'bajo-cli:@inquirer/select')
-  if (!this.bajoDb) {
-    print.fail('Bajo DB isn\'t loaded', { exit: !returnEarly })
-    if (returnEarly) return
-  }
+  const config = getConfig()
+  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
   const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) {
-    print.fail('No schema found!', { exit: !returnEarly })
-    if (returnEarly) return
-  }
+  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
   let [coll, dest, query] = args
   if (isEmpty(coll)) {
     coll = await select({
@@ -40,8 +35,8 @@ async function exportTo ({ path, args, returnEarly }) {
       message: print.__('Please enter a query (if any):')
     })
   }
-  const spinner = print.bora('Exporting...').start()
-  const progressFn = makeProgress.call(this, spinner)
+  const spin = spinner().start('Exporting...')
+  const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
   const { batch } = getConfig()
   const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
@@ -50,10 +45,10 @@ async function exportTo ({ path, args, returnEarly }) {
   try {
     const filter = { query }
     const result = await this.bajoExtra.helper.exportTo(coll, dest, { filter, batch, progressFn })
-    spinner.succeed('%d records successfully exported to \'%s\'', result.count, Path.resolve(result.file))
+    spin.succeed('%d records successfully exported to \'%s\'', result.count, Path.resolve(result.file))
   } catch (err) {
     console.log(err)
-    spinner.fail('Error: %s', err.message, { exit: !returnEarly })
+    spin.fail('Error: %s', err.message, { exit: config.tool })
   }
 }
 

package/bajoCli/tool/import-from.js

@@ -1,25 +1,20 @@
 import Path from 'path'
 
-function makeProgress (spinner) {
+function makeProgress (spin) {
   return async function ({ batchNo, data } = {}) {
-    spinner.setText('Batch %d (%d records)', batchNo, data.length)
+    spin.setText('Batch %d (%d records)', batchNo, data.length)
   }
 }
 
-async function importFrom ({ path, args, returnEarly }) {
-  const { importPkg, print, importModule, getConfig } = this.bajo.helper
+async function importFrom ({ path, args }) {
+  const { importPkg, print, importModule, getConfig, spinner } = this.bajo.helper
   const { isEmpty, map } = await importPkg('lodash-es')
   const [input, select, confirm] = await importPkg('bajo-cli:@inquirer/input',
     'bajo-cli:@inquirer/select', 'bajo-cli:@inquirer/confirm')
-  if (!this.bajoDb) {
-    print.fail('Bajo DB isn\'t loaded', { exit: !returnEarly })
-    if (returnEarly) return
-  }
+  const config = getConfig()
+  if (!this.bajoDb) return print.fail('Bajo DB isn\'t loaded', { exit: config.tool })
   const schemas = map(this.bajoDb.schemas, 'name')
-  if (isEmpty(schemas)) {
-    print.fail('No schema found!', { exit: !returnEarly })
-    if (returnEarly) return
-  }
+  if (isEmpty(schemas)) return print.fail('No schema found!', { exit: config.tool })
   let [dest, coll] = args
   if (isEmpty(dest)) {
     dest = await input({
@@ -37,12 +32,9 @@ async function importFrom ({ path, args, returnEarly }) {
     message: print.__('You\'re about to replace ALL records with the new ones. Are you really sure?'),
     default: false
   })
-  if (!answer) {
-    print.fail('Aborted!', { exit: !returnEarly })
-    if (returnEarly) return
-  }
-  const spinner = print.bora('Importing...').start()
-  const progressFn = makeProgress.call(this, spinner)
+  if (!answer) return print.fail('Aborted!', { exit: config.tool })
+  const spin = spinner().start('Importing...')
+  const progressFn = makeProgress.call(this, spin)
   const cfg = getConfig('bajoDb', { full: true })
   const { batch } = getConfig()
   const start = await importModule(`${cfg.dir.pkg}/bajo/start.js`)
@@ -50,9 +42,9 @@ async function importFrom ({ path, args, returnEarly }) {
   await start.call(this, connection.name)
   try {
     const result = await this.bajoExtra.helper.importFrom(dest, coll, { batch, progressFn })
-    spinner.succeed('%d records successfully imported from \'%s\'', result.count, Path.resolve(result.file))
+    spin.succeed('%d records successfully imported from \'%s\'', result.count, Path.resolve(result.file))
   } catch (err) {
-    spinner.fail('Error: %s', err.message, { exit: !returnEarly })
+    spin.fail('Error: %s', err.message, { exit: config.tool })
   }
 }
 

package/bajoCli/tool.js

@@ -1,8 +1,8 @@
-async function tool ({ path, args = [], returnEarly }) {
+async function tool ({ path, args = [] }) {
   const { currentLoc } = this.bajo.helper
   const { runToolMethod } = this.bajoCli.helper
   const options = { demonize: ['shell'] }
-  await runToolMethod({ path, args, dir: `${currentLoc(import.meta).dir}/tool`, options, returnEarly })
+  await runToolMethod({ path, args, dir: `${currentLoc(import.meta).dir}/tool`, options })
 }
 
 export default tool

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {