bajo-extra 0.2.10 → 0.2.12

package/bajo/helper/fetch-and-save.js

@@ -1,104 +1,66 @@
-async function batching ({ source = {}, converter, coll, current = {}, options = {}, spin } = {}) {
-  const { setImmediate, importPkg, print } = this.bajo.helper
-  const { isEmpty, isFunction, set } = await importPkg('lodash-es')
-  const { fetch } = this.bajoExtra.helper
-  const { recordCreate, recordFind, recordUpdate, validationErrorMessage } = this.bajoDb.helper
-  const resp = await fetch(source.url, source.options ?? {})
-  if (isEmpty(resp)) spin.fatal('No result from server, aborted!')
-  if (source.abort) {
-    const aborted = await source.abort.call(this, resp)
-    if (aborted) spin.fatal(aborted)
+async function handler (rec, bulk) {
+  const { importPkg } = this.bajo.helper
+  const { isFunction, set } = await importPkg('lodash-es')
+  const { recordCreate, recordFind, recordUpdate } = this.bajoDb.helper
+  const save = bulk.save ?? {}
+  const current = save.current ?? {}
+  let existing
+  let record
+  let method
+  save.checkUnique = save.checkUnique ?? 'id'
+  if (['unique', 'upsert'].includes(save.mode)) {
+    const query = isFunction(save.checkUnique) ? await save.checkUnique.call(this, rec, save) : set({}, save.checkUnique, rec[save.checkUnique])
+    const resp = await recordFind(save.coll, { query, limit: 1 }, { skipCache: true })
+    if (resp.length > 0) existing = resp[0]
   }
-  let count = 0
-  const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
-  const iterator = isFunction(source.iterator) ? await source.iterator.call(this, resp) : resp[source.iterator]
-  if (iterator.length === 0) {
-    print.warn('No records to process, abort')
-    return 0
-  }
-  spin.setText('Got %d records, processing...', iterator.length)
-  for (let r of iterator) {
-    await setImmediate()
-    if (converter) r = await converter.call(this, r, options)
-    if (isEmpty(r)) {
-      stat.skipped++
-      continue
+  if (existing) {
+    if (save.mode === 'upsert') {
+      const body = save.updateConverter ? await save.updateConverter.call(this, rec, save) : rec
+      try {
+        record = await recordUpdate(save.coll, existing.id, body)
+        method = 'updated'
+      } catch (err) {
+        console.error(err)
+        method = 'error'
+      }
+    } else {
+      method = 'skipped'
     }
+  } else {
     try {
-      let existing
-      let record
-      let method
-      options.checkUnique = options.checkUnique ?? 'id'
-      if (['unique', 'upsert'].includes(options.mode)) {
-        const query = isFunction(options.checkUnique) ? await options.checkUnique.call(this, r, options) : set({}, options.checkUnique, r[options.checkUnique])
-        const resp = await recordFind(coll, { query, limit: 1 }, { skipCache: true })
-        if (resp.length > 0) existing = resp[0]
-      }
-      if (existing) {
-        if (options.mode === 'upsert') {
-          const body = options.updateConverter ? await options.updateConverter.call(this, r, options) : r
-          record = await recordUpdate(coll, existing.id, body)
-          method = 'updated'
-          stat.updated++
-        } else {
-          stat.skipped++
-          print.warn(`[${spin.getElapsed()}] Record %s exists, skipped`, JSON.stringify(r))
-          method = 'skipped'
-        }
-      } else {
-        stat.created++
-        record = await recordCreate(coll, r)
-        method = 'created'
-      }
-      if (record && current.coll && current.query) {
-        const query = await current.query.call(this, { body: r, record, options })
-        const recs = await recordFind(current.coll, { query }, { skipCache: true })
-        const rc = current.converter ? await current.converter.call(this, { body: r, record, options }) : r
-        if (rc) {
-          if (recs.length > 0) {
-            const id = recs[0].id
-            await recordUpdate(current.coll, id, rc)
-          } else {
-            await recordCreate(current.coll, rc)
-          }
-        }
-      }
-      if (options.printCount && options.printCount < count && (count % options.printCount === 0)) print.succeed('[%s] Processed %d/%d', spin.getElapsed(), count, iterator.length)
-      else if (!spin.opts.isLog) spin.setText('Record %d/%d...', count, iterator.length, method)
-      count++
+      record = await recordCreate(save.coll, rec)
+      method = 'created'
     } catch (err) {
-      console.log(err)
-      spin.setText(validationErrorMessage(err) + ', continue')
+      console.error(err)
+      method = 'error'
     }
   }
-  print.succeed('[%s] %d/%d records processed', spin.getElapsed(), count, iterator.length)
-  print.succeed('[%s] Created: %d, Updated: %d, Skipped: %d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
-  return iterator.length
-}
-
-async function fetchAndSave ({ source = {}, converter, coll, current = {}, options = {} } = {}) {
-  const { print, spinner } = this.bajo.helper
-  source.options = source.options ?? {}
-  source.options.params = source.options.params ?? {}
-  if (options.batch) {
-    print.info('Batch starting')
-    const spin = spinner({ showCounter: true }).start('Fetching starts...')
-    let step = 1
-    for (;;) {
-      print.info('[%s] Fetch batch #%d', spin.getElapsed(), step)
-      const newSource = await options.batch.call(this, source)
-      if (newSource) source = newSource
-      const length = await batching.call(this, { source, converter, coll, current, options, spin })
-      if (length === 0) {
-        print.info('All done!')
-        break
+  if (record && current.coll && current.query) {
+    const query = await current.query.call(this, { body: rec, record, opts: save })
+    const recs = await recordFind(current.coll, { query }, { skipCache: true })
+    const rc = current.converter ? await current.converter.call(this, { body: rec, record, opts: save }) : rec
+    if (rc) {
+      if (recs.length > 0) {
+        const id = recs[0].id
+        await recordUpdate(current.coll, id, rc)
+      } else {
+        await recordCreate(current.coll, rc)
       }
-      step++
     }
-  } else {
-    const spin = spinner({ showCounter: true }).start('Fetching starts...')
-    await batching.call(this, { source, converter, coll, current, options, spin })
   }
+  return method
+}
+
+async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
+  const { importPkg, getConfig, importModule } = this.bajo.helper
+  const { fetchBulk } = this.bajoExtra.helper
+  const { merge } = await importPkg('lodash-es')
+  merge(bulk, { handler, save })
+  const cfgDb = getConfig('bajoDb', { full: true })
+  const start = await importModule(`${cfgDb.dir.pkg}/bajo/start.js`)
+  await start.call(this, 'all')
+
+  await fetchBulk(url, bulk, opts)
 }
 
 export default fetchAndSave

package/bajo/helper/fetch-bulk.js

@@ -0,0 +1,78 @@
+async function fetching ({ url, opts, bulk, spin }) {
+  const { setImmediate, importPkg, print } = this.bajo.helper
+  const { isEmpty, isFunction, has } = await importPkg('lodash-es')
+  const { validationErrorMessage } = this.bajoDb.helper
+  const { fetch } = this.bajoExtra.helper
+  const resp = await fetch(url, opts ?? {})
+  if (isEmpty(resp)) {
+    spin.fatal('No result from server, aborted!')
+    return -1
+  }
+  if (bulk.abort) {
+    const aborted = await bulk.abort.call(this, resp)
+    if (aborted) {
+      spin.fatal(aborted)
+      return -1
+    }
+  }
+  let count = 0
+  const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
+  bulk.dataKey = bulk.dataKey ?? 'data'
+  if (bulk.printCount === true) bulk.printCount = 100
+  const data = isFunction(bulk.dataKey) ? await bulk.dataKey.call(this, resp) : resp[bulk.dataKey]
+  if (data.length === 0) {
+    print.warn('No records to process, abort')
+    return 0
+  }
+  spin.setText('Got %d records, processing...', data.length)
+  for (let r of data) {
+    await setImmediate()
+    if (bulk.converter) r = await bulk.converter.call(this, r, bulk)
+    if (isEmpty(r)) {
+      stat.skipped++
+      continue
+    }
+    try {
+      const result = await bulk.handler.call(this, r, bulk)
+      if (result && has(stat, result)) stat[result]++
+      if (bulk.printCount && bulk.printCount < count && (count % bulk.printCount === 0)) print.succeed('[%s] Processed %d/%d', spin.getElapsed(), count, data.length)
+      else if (!spin.opts.isLog) spin.setText('Record %d/%d...', count, data.length)
+      count++
+    } catch (err) {
+      console.log(err)
+      spin.setText(validationErrorMessage(err) + ', continue')
+    }
+  }
+  print.succeed('[%s] %d/%d records processed', spin.getElapsed(), count, data.length)
+  if (!bulk.noStat) print.succeed('[%s] Created: %d, Updated: %d, Skipped: %d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
+  return data.length
+}
+
+async function fetchBulk (url, bulk = {}, opts = {}) {
+  const { print, spinner, importPkg, error } = this.bajo.helper
+  const { isFunction } = await importPkg('lodash-es')
+  opts.params = opts.params ?? {}
+  bulk.maxStep = bulk.maxStep ?? 0
+  if (!isFunction(bulk.handler)) throw error('A function handler must be provided')
+  if (isFunction(bulk.paramsIncFn)) {
+    print.info('Bulk fetch starting')
+    const spin = spinner({ showCounter: true }).start('Fetching starts...')
+    let step = 1
+    for (;;) {
+      print.info('[%s] Batch #%d', spin.getElapsed(), step)
+      const newOpts = await bulk.paramsIncFn.call(this, { url, bulk, opts })
+      if (newOpts) opts = newOpts
+      const length = await fetching.call(this, { url, bulk, opts, spin })
+      if (length === 0 || (bulk.maxStep > 0 && step >= bulk.maxStep)) {
+        print.info('All done!')
+        break
+      }
+      step++
+    }
+  } else {
+    const spin = spinner({ showCounter: true }).start('Fetching starts...')
+    await fetching.call(this, { url, bulk, opts, spin })
+  }
+}
+
+export default fetchBulk

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bajo-extra",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "Extra package for Bajo Framework",
   "main": "index.js",
   "scripts": {
@@ -32,6 +32,7 @@
     "email-addresses": "^5.0.0",
     "fast-jwt": "^3.2.0",
     "fast-xml-parser": "^4.3.2",
+    "littlehash": "^1.0.1",
     "ndjson-csv-xlsx": "^1.1.1",
     "performant-array-to-tree": "^1.11.0",
     "query-string": "^8.1.0",