bajo-extra 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,10 +5,10 @@ import { createGunzip } from 'zlib'
|
|
|
5
5
|
|
|
6
6
|
const { json, ndjson, csv, xlsx } = format
|
|
7
7
|
const { DataStream } = scramjet
|
|
8
|
-
const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx']
|
|
8
|
+
const supportedExt = ['.json', '.jsonl', '.ndjson', '.csv', '.xlsx', '.tsv']
|
|
9
9
|
|
|
10
10
|
async function importFrom (source, dest, { trashOld = true, batch = 1, progressFn, converterFn, useHeader = true, fileType, createOpts = {} } = {}, opts = {}) {
|
|
11
|
-
const { error, importPkg, getConfig, getPluginDataDir
|
|
11
|
+
const { error, importPkg, getConfig, getPluginDataDir } = this.bajo.helper
|
|
12
12
|
if (!this.bajoDb) throw error('Bajo DB isn\'t loaded')
|
|
13
13
|
const { getInfo, recordClear, recordCreate } = this.bajoDb.helper
|
|
14
14
|
await getInfo(dest)
|
|
@@ -41,6 +41,7 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
|
|
|
41
41
|
if (ext === '.json') pipes.push(json.parse(opts))
|
|
42
42
|
else if (['.ndjson', '.jsonl'].includes(ext)) pipes.push(ndjson.parse(opts))
|
|
43
43
|
else if (ext === '.csv') pipes.push(csv.parse(merge({ headers: useHeader }, opts)))
|
|
44
|
+
else if (ext === '.tsv') pipes.push(csv.parse(merge({ headers: useHeader }, merge(opts, { delimiter: '\t' }))))
|
|
44
45
|
else if (ext === '.xlsx') pipes.push(xlsx.parse(merge({ header: useHeader }, opts)))
|
|
45
46
|
|
|
46
47
|
const stream = DataStream.pipeline(...pipes)
|
|
@@ -49,14 +50,12 @@ async function importFrom (source, dest, { trashOld = true, batch = 1, progressF
|
|
|
49
50
|
.batch(batch)
|
|
50
51
|
.map(async items => {
|
|
51
52
|
if (items.length === 0) return null
|
|
52
|
-
const start = Date.now()
|
|
53
53
|
for (let item of items) {
|
|
54
54
|
count++
|
|
55
55
|
item = converterFn ? await converterFn.call(this, item) : item
|
|
56
56
|
await recordCreate(dest, item, createOpts)
|
|
57
57
|
}
|
|
58
|
-
|
|
59
|
-
if (progressFn) await progressFn.call(this, { batchNo, data: items, time: secToHms(diff, true), timeMsec: diff })
|
|
58
|
+
if (progressFn) await progressFn.call(this, { batchNo, data: items })
|
|
60
59
|
batchNo++
|
|
61
60
|
})
|
|
62
61
|
.run()
|