bajo-extra 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugin/factory.js +342 -0
- package/plugin/.alias +0 -1
- package/plugin/config.json +0 -5
- package/plugin/method/count-file-lines.js +0 -22
- package/plugin/method/download.js +0 -49
- package/plugin/method/fetch-and-save.js +0 -62
- package/plugin/method/fetch-bulk.js +0 -76
- package/plugin/method/fetch.js +0 -51
- package/plugin/method/format/byte.js +0 -11
- package/plugin/method/format/float.js +0 -9
- package/plugin/method/format/integer.js +0 -9
- package/plugin/method/format/percentage.js +0 -10
- package/plugin/method/gunzip.js +0 -7
- package/plugin/method/gzip.js +0 -20
- package/plugin/method/hash.js +0 -16
- package/plugin/method/is-bcrypt.js +0 -6
- package/plugin/method/is-md5.js +0 -6
package/package.json
CHANGED
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import bcrypt from 'bcrypt'
|
|
2
|
+
import crypto from 'crypto'
|
|
3
|
+
import { createGzip, createGunzip } from 'zlib'
|
|
4
|
+
import path from 'path'
|
|
5
|
+
import { fetch, Agent } from 'undici'
|
|
6
|
+
import { Readable } from 'stream'
|
|
7
|
+
import numbro from 'numbro'
|
|
8
|
+
|
|
9
|
+
async function fetching ({ url, opts, bulk, spin }) {
|
|
10
|
+
const { setImmediate, print } = this.app.bajo
|
|
11
|
+
const { isEmpty, isFunction, has } = this.app.bajo.lib._
|
|
12
|
+
const { validationErrorMessage } = this.app.bajoDb
|
|
13
|
+
const resp = await this.fetch(url, opts ?? {})
|
|
14
|
+
if (isEmpty(resp)) {
|
|
15
|
+
spin.fatal('noServerResponse')
|
|
16
|
+
return -1
|
|
17
|
+
}
|
|
18
|
+
if (bulk.abort) {
|
|
19
|
+
const aborted = await bulk.abort.call(this, resp)
|
|
20
|
+
if (aborted) {
|
|
21
|
+
spin.fatal(aborted)
|
|
22
|
+
return -1
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
let count = 0
|
|
26
|
+
const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
|
|
27
|
+
bulk.dataKey = bulk.dataKey ?? 'data'
|
|
28
|
+
if (bulk.printCount === true) bulk.printCount = 100
|
|
29
|
+
const data = isFunction(bulk.dataKey) ? await bulk.dataKey.call(this, resp) : resp[bulk.dataKey]
|
|
30
|
+
if (data.length === 0) {
|
|
31
|
+
print.warn('noRecordToProcess')
|
|
32
|
+
return 0
|
|
33
|
+
}
|
|
34
|
+
spin.setText('gotRecordsProcessing%d', data.length)
|
|
35
|
+
for (let r of data) {
|
|
36
|
+
await setImmediate()
|
|
37
|
+
if (bulk.converter) r = await bulk.converter.call(this, r, bulk)
|
|
38
|
+
if (isEmpty(r)) {
|
|
39
|
+
stat.skipped++
|
|
40
|
+
continue
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const result = await bulk.handler.call(this, r, bulk)
|
|
44
|
+
if (result && has(stat, result)) stat[result]++
|
|
45
|
+
if (bulk.printCount && bulk.printCount < count && (count % bulk.printCount === 0)) print.succeed('[%s] Processed %d/%d', spin.getElapsed(), count, data.length)
|
|
46
|
+
else if (!spin.opts.isLog) spin.setText('rec%d%d', count, data.length)
|
|
47
|
+
count++
|
|
48
|
+
} catch (err) {
|
|
49
|
+
console.log(err)
|
|
50
|
+
spin.setText(validationErrorMessage(err) + ', continue')
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
print.succeed('recProcessed%s%d%d', spin.getElapsed(), count, data.length)
|
|
54
|
+
if (!bulk.noStat) print.succeed('createdUpdatedSkipped%s%d%d%d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
|
|
55
|
+
return data.length
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function handler (rec, bulk) {
|
|
59
|
+
const { isFunction, set } = this.app.bajo.lib._
|
|
60
|
+
const { recordCreate, recordFind, recordUpdate } = this.app.bajoDb
|
|
61
|
+
const save = bulk.save ?? {}
|
|
62
|
+
const current = save.current ?? {}
|
|
63
|
+
let existing
|
|
64
|
+
let record
|
|
65
|
+
let method
|
|
66
|
+
save.checkUnique = save.checkUnique ?? 'id'
|
|
67
|
+
if (['unique', 'upsert'].includes(save.mode)) {
|
|
68
|
+
const query = isFunction(save.checkUnique) ? await save.checkUnique.call(this, rec, save) : set({}, save.checkUnique, rec[save.checkUnique])
|
|
69
|
+
const resp = await recordFind(save.coll, { query, limit: 1 }, { noCache: true })
|
|
70
|
+
if (resp.length > 0) existing = resp[0]
|
|
71
|
+
}
|
|
72
|
+
if (existing) {
|
|
73
|
+
if (save.mode === 'upsert') {
|
|
74
|
+
const body = save.updateConverter ? await save.updateConverter.call(this, rec, save) : rec
|
|
75
|
+
try {
|
|
76
|
+
record = await recordUpdate(save.coll, existing.id, body)
|
|
77
|
+
method = 'updated'
|
|
78
|
+
} catch (err) {
|
|
79
|
+
console.error(err)
|
|
80
|
+
method = 'error'
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
method = 'skipped'
|
|
84
|
+
}
|
|
85
|
+
} else {
|
|
86
|
+
try {
|
|
87
|
+
record = await recordCreate(save.coll, rec)
|
|
88
|
+
method = 'created'
|
|
89
|
+
} catch (err) {
|
|
90
|
+
console.error(err)
|
|
91
|
+
method = 'error'
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (record && current.coll && current.query) {
|
|
95
|
+
const query = await current.query.call(this, { body: rec, record, opts: save })
|
|
96
|
+
const recs = await recordFind(current.coll, { query }, { noCache: true })
|
|
97
|
+
const rc = current.converter ? await current.converter.call(this, { body: rec, record, opts: save }) : rec
|
|
98
|
+
if (rc) {
|
|
99
|
+
if (recs.length > 0) {
|
|
100
|
+
const id = recs[0].id
|
|
101
|
+
await recordUpdate(current.coll, id, rc)
|
|
102
|
+
} else {
|
|
103
|
+
await recordCreate(current.coll, rc)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return method
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function factory (pkgName) {
|
|
111
|
+
const me = this
|
|
112
|
+
|
|
113
|
+
return class BajoExtra extends this.lib.BajoPlugin {
|
|
114
|
+
constructor () {
|
|
115
|
+
super(pkgName, me.app)
|
|
116
|
+
this.alias = 'extra'
|
|
117
|
+
this.config = {
|
|
118
|
+
fetch: {
|
|
119
|
+
agent: {}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
formatByte = (value, opts = {}) => {
|
|
125
|
+
opts.output = 'byte'
|
|
126
|
+
opts.base = 'binary'
|
|
127
|
+
opts.mantissa = opts.mantissa ?? opts.scale ?? 2
|
|
128
|
+
opts.spaceSeparated = opts.spaceSeparated ?? true
|
|
129
|
+
return numbro(value).format(opts)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
formatFloat = (value, opts = {}) => {
|
|
133
|
+
opts.mantissa = opts.mantissa ?? opts.scale ?? 2
|
|
134
|
+
opts.thousandSeparated = opts.thousandSeparated ?? true
|
|
135
|
+
return numbro(value).format(opts)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
formatInteger = (value, opts = {}) => {
|
|
139
|
+
opts.mantissa = 0
|
|
140
|
+
opts.thousandSeparated = opts.thousandSeparated ?? true
|
|
141
|
+
return numbro(value).format(opts)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
formatPercentage = (value, opts = {}) => {
|
|
145
|
+
opts.output = 'percent'
|
|
146
|
+
opts.mantissa = opts.mantissa ?? opts.scale ?? 2
|
|
147
|
+
opts.spaceSeparated = opts.spaceSeparated ?? true
|
|
148
|
+
return numbro(value).format(opts)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// taken from: https://stackoverflow.com/a/41439945
|
|
152
|
+
countFileLines = async (file) => {
|
|
153
|
+
const { fs } = this.app.bajo.lib
|
|
154
|
+
return new Promise((resolve, reject) => {
|
|
155
|
+
let lineCount = 0
|
|
156
|
+
fs.createReadStream(file)
|
|
157
|
+
.on('data', (buffer) => {
|
|
158
|
+
let idx = -1
|
|
159
|
+
lineCount--
|
|
160
|
+
do {
|
|
161
|
+
idx = buffer.indexOf(10, idx + 1)
|
|
162
|
+
lineCount++
|
|
163
|
+
} while (idx !== -1)
|
|
164
|
+
})
|
|
165
|
+
.on('end', () => {
|
|
166
|
+
resolve(lineCount)
|
|
167
|
+
})
|
|
168
|
+
.on('error', reject)
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
download = async (url, opts = {}, extra = {}) => {
|
|
173
|
+
const { getPluginDataDir, importPkg, generateId } = this.app.bajo
|
|
174
|
+
const { fs } = this.app.bajo.lib
|
|
175
|
+
const { isFunction, merge } = this.app.bajo.lib._
|
|
176
|
+
if (typeof opts === 'string') extra = { dir: opts }
|
|
177
|
+
const increment = await importPkg('add-filename-increment')
|
|
178
|
+
if (!extra.dir) {
|
|
179
|
+
extra.dir = `${getPluginDataDir('bajoExtra')}/download`
|
|
180
|
+
fs.ensureDirSync(extra.dir)
|
|
181
|
+
}
|
|
182
|
+
if (!fs.existsSync(extra.dir)) throw this.error('dlDirNotExists%s', extra.dir)
|
|
183
|
+
if (extra.randomFileName) {
|
|
184
|
+
const ext = path.extname(url)
|
|
185
|
+
extra.fileName = `${generateId()}${ext}`
|
|
186
|
+
}
|
|
187
|
+
if (!extra.fileName) extra.fileName = path.basename(url)
|
|
188
|
+
const file = path.resolve(increment(`${extra.dir}/${extra.fileName}`, { fs: true }))
|
|
189
|
+
const writer = fs.createWriteStream(file)
|
|
190
|
+
const { headers, body, ok, status } = await fetch(url, opts, merge({}, extra, { rawResponse: true }))
|
|
191
|
+
if (!ok) {
|
|
192
|
+
fs.removeSync(file)
|
|
193
|
+
throw this.error('gettingStatus%s', status)
|
|
194
|
+
}
|
|
195
|
+
const total = headers['content-length'] ?? 0
|
|
196
|
+
const data = Readable.fromWeb(body)
|
|
197
|
+
let length = 0
|
|
198
|
+
data.on('data', chunk => {
|
|
199
|
+
length += chunk.length
|
|
200
|
+
if (isFunction(extra.progressFn)) extra.progressFn.call(this, length, total)
|
|
201
|
+
else if (extra.spin) {
|
|
202
|
+
extra.spinText = extra.spinText ?? 'downloading'
|
|
203
|
+
if (total === 0) extra.spin.setText(`${extra.spinText} %s`, this.formatByte(length))
|
|
204
|
+
else extra.spin.setText(`${extra.spinText} %s of %s (%s)`, this.formatByte(length), this.formatByte(total), this.formatPercentage(length / total))
|
|
205
|
+
}
|
|
206
|
+
})
|
|
207
|
+
data.pipe(writer)
|
|
208
|
+
|
|
209
|
+
return new Promise((resolve, reject) => {
|
|
210
|
+
writer.on('error', reject)
|
|
211
|
+
writer.on('finish', () => {
|
|
212
|
+
resolve(file)
|
|
213
|
+
})
|
|
214
|
+
})
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
fetchAndSave = async ({ url, bulk, save = {}, opts = {} } = {}) => {
|
|
218
|
+
const { startPlugin } = this.bajo
|
|
219
|
+
const { merge } = this.bajo.lib._
|
|
220
|
+
merge(bulk, { handler, save })
|
|
221
|
+
await startPlugin('dobo')
|
|
222
|
+
await this.fetchBulk(url, bulk, opts)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
fetchBulk = async (url, bulk = {}, opts = {}) => {
|
|
226
|
+
const { isFunction } = this.bajo.lib._
|
|
227
|
+
opts.params = opts.params ?? {}
|
|
228
|
+
bulk.maxStep = bulk.maxStep ?? 0
|
|
229
|
+
if (!isFunction(bulk.handler)) throw this.error('handlerMustBeProvided')
|
|
230
|
+
if (isFunction(bulk.paramsIncFn)) {
|
|
231
|
+
this.print.info('bulkFetchStarting')
|
|
232
|
+
const spin = this.print.spinner({ showCounter: true }).start('fetchingStarts')
|
|
233
|
+
let step = 1
|
|
234
|
+
for (;;) {
|
|
235
|
+
this.print.info('batch%s%d', spin.getElapsed(), step)
|
|
236
|
+
const newOpts = await bulk.paramsIncFn.call(this, { url, bulk, opts })
|
|
237
|
+
if (newOpts) opts = newOpts
|
|
238
|
+
const length = await fetching.call(this, { url, bulk, opts, spin })
|
|
239
|
+
if (length === 0 || (bulk.maxStep > 0 && step >= bulk.maxStep)) {
|
|
240
|
+
this.print.info('allDone')
|
|
241
|
+
break
|
|
242
|
+
}
|
|
243
|
+
step++
|
|
244
|
+
}
|
|
245
|
+
} else {
|
|
246
|
+
const spin = this.print.spinner({ showCounter: true }).start('fetchingStarts')
|
|
247
|
+
await fetching.call(this, { url, bulk, opts, spin })
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
fetchUrl = async (url, opts = {}, extra = {}) => {
|
|
252
|
+
const { isSet } = this.app.bajo
|
|
253
|
+
const { fs } = this.app.bajo.lib
|
|
254
|
+
const { has, isArray, isPlainObject, isString, cloneDeep, merge } = this.app.bajo.lib._
|
|
255
|
+
if (isPlainObject(url)) {
|
|
256
|
+
extra = cloneDeep(opts)
|
|
257
|
+
opts = cloneDeep(url)
|
|
258
|
+
url = opts.url
|
|
259
|
+
delete opts.url
|
|
260
|
+
}
|
|
261
|
+
if (opts.method) opts.method = opts.method.toUpperCase()
|
|
262
|
+
if (opts.auth) {
|
|
263
|
+
opts.headers.Authorization = `Basic ${Buffer.from(`${opts.auth.username}:${opts.auth.password}`).toString('base64')}`
|
|
264
|
+
delete opts.auth
|
|
265
|
+
}
|
|
266
|
+
opts.query = merge({}, opts.query, opts.params ?? {})
|
|
267
|
+
delete opts.params
|
|
268
|
+
if (!has(extra, 'cacheBuster')) extra.cacheBuster = true
|
|
269
|
+
if (extra.cacheBuster) opts.query[extra.cacheBusterKey ?? '_'] = Date.now()
|
|
270
|
+
if (this.config.fetch.agent || extra.agent) {
|
|
271
|
+
opts.dispatcher = new Agent(extra.agent ?? this.config.fetch.agent)
|
|
272
|
+
}
|
|
273
|
+
if (opts.body && extra.formData) {
|
|
274
|
+
const formData = new FormData()
|
|
275
|
+
for (const key in opts.body) {
|
|
276
|
+
let fname
|
|
277
|
+
let val = opts.body[key]
|
|
278
|
+
if (!isSet(val)) continue
|
|
279
|
+
if (isString(val) && val.startsWith('file:///')) {
|
|
280
|
+
fname = path.basename(val)
|
|
281
|
+
val = new Blob([fs.readFileSync(val.slice(8))])
|
|
282
|
+
} else if (isPlainObject(val) || isArray(val)) val = JSON.stringify(val)
|
|
283
|
+
if (fname) formData.append(key, val, fname)
|
|
284
|
+
else formData.append(key, val)
|
|
285
|
+
}
|
|
286
|
+
opts.body = formData
|
|
287
|
+
}
|
|
288
|
+
if (opts.query) {
|
|
289
|
+
// todo: what if url already contain query string?
|
|
290
|
+
const query = new URLSearchParams(opts.query)
|
|
291
|
+
url += '?' + query
|
|
292
|
+
}
|
|
293
|
+
const resp = await fetch(url, opts)
|
|
294
|
+
if (extra.rawResponse) return resp
|
|
295
|
+
return await resp.json()
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
gunzip = async (file, deleteOld) => {
|
|
299
|
+
await this.gzip(file, deleteOld, true)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
gzip = async (file, deleteOld, expand) => {
|
|
303
|
+
const { fs } = this.app.bajo.lib
|
|
304
|
+
return new Promise((resolve, reject) => {
|
|
305
|
+
const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
|
|
306
|
+
const reader = fs.createReadStream(file)
|
|
307
|
+
const writer = fs.createWriteStream(newFile)
|
|
308
|
+
const method = expand ? createGunzip() : createGzip()
|
|
309
|
+
reader.pipe(method).pipe(writer)
|
|
310
|
+
writer.on('error', reject)
|
|
311
|
+
writer.on('finish', err => {
|
|
312
|
+
if (err) return reject(err)
|
|
313
|
+
if (deleteOld) fs.unlinkSync(file)
|
|
314
|
+
resolve()
|
|
315
|
+
})
|
|
316
|
+
})
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
hash = async (text, type = 'md5', options = {}) => {
|
|
320
|
+
options.digest = options.digest ?? 'hex'
|
|
321
|
+
options.salt = options.hash ?? 10
|
|
322
|
+
if (typeof text !== 'string') text = JSON.stringify(text)
|
|
323
|
+
if (type === 'bcrypt') return await bcrypt.hash(text, options.salt)
|
|
324
|
+
if (type === 'short') {
|
|
325
|
+
type = 'shake256'
|
|
326
|
+
options.outputLength = 6
|
|
327
|
+
}
|
|
328
|
+
return crypto.createHash(type, options).update(text).digest(options.digest)
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
isBcryptString = (text) => {
|
|
332
|
+
// return /^\$2[ayb]\$.{56}$/.test(text)
|
|
333
|
+
return /^\$2[aby]?\$\d{1,2}\$[./A-Za-z0-9]{53}$/.test(text)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
isMd5 = (text) => {
|
|
337
|
+
return /^[a-f0-9]{32}$/i.test(text)
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
export default factory
|
package/plugin/.alias
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
extra
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
// taken from: https://stackoverflow.com/a/41439945
|
|
2
|
-
function countFileLines (file) {
|
|
3
|
-
const { fs } = this.app.bajo.lib
|
|
4
|
-
return new Promise((resolve, reject) => {
|
|
5
|
-
let lineCount = 0
|
|
6
|
-
fs.createReadStream(file)
|
|
7
|
-
.on('data', (buffer) => {
|
|
8
|
-
let idx = -1
|
|
9
|
-
lineCount--
|
|
10
|
-
do {
|
|
11
|
-
idx = buffer.indexOf(10, idx + 1)
|
|
12
|
-
lineCount++
|
|
13
|
-
} while (idx !== -1)
|
|
14
|
-
})
|
|
15
|
-
.on('end', () => {
|
|
16
|
-
resolve(lineCount)
|
|
17
|
-
})
|
|
18
|
-
.on('error', reject)
|
|
19
|
-
})
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export default countFileLines
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
import path from 'path'
|
|
2
|
-
import { Readable } from 'node:stream'
|
|
3
|
-
|
|
4
|
-
async function download (url, opts = {}, extra = {}) {
|
|
5
|
-
const { getPluginDataDir, importPkg, generateId } = this.app.bajo
|
|
6
|
-
const { fs } = this.app.bajo.lib
|
|
7
|
-
const { isFunction, merge } = this.app.bajo.lib._
|
|
8
|
-
if (typeof opts === 'string') extra = { dir: opts }
|
|
9
|
-
const increment = await importPkg('add-filename-increment')
|
|
10
|
-
if (!extra.dir) {
|
|
11
|
-
extra.dir = `${getPluginDataDir('bajoExtra')}/download`
|
|
12
|
-
fs.ensureDirSync(extra.dir)
|
|
13
|
-
}
|
|
14
|
-
if (!fs.existsSync(extra.dir)) throw this.error('dlDirNotExists%s', extra.dir)
|
|
15
|
-
if (extra.randomFileName) {
|
|
16
|
-
const ext = path.extname(url)
|
|
17
|
-
extra.fileName = `${generateId()}${ext}`
|
|
18
|
-
}
|
|
19
|
-
if (!extra.fileName) extra.fileName = path.basename(url)
|
|
20
|
-
const file = path.resolve(increment(`${extra.dir}/${extra.fileName}`, { fs: true }))
|
|
21
|
-
const writer = fs.createWriteStream(file)
|
|
22
|
-
const { headers, body, ok, status } = await fetch(url, opts, merge({}, extra, { rawResponse: true }))
|
|
23
|
-
if (!ok) {
|
|
24
|
-
fs.removeSync(file)
|
|
25
|
-
throw this.error('gettingStatus%s', status)
|
|
26
|
-
}
|
|
27
|
-
const total = headers['content-length'] ?? 0
|
|
28
|
-
const data = Readable.fromWeb(body)
|
|
29
|
-
let length = 0
|
|
30
|
-
data.on('data', chunk => {
|
|
31
|
-
length += chunk.length
|
|
32
|
-
if (isFunction(extra.progressFn)) extra.progressFn.call(this, length, total)
|
|
33
|
-
else if (extra.spin) {
|
|
34
|
-
extra.spinText = extra.spinText ?? 'downloading'
|
|
35
|
-
if (total === 0) extra.spin.setText(`${extra.spinText} %s`, this.formatByte(length))
|
|
36
|
-
else extra.spin.setText(`${extra.spinText} %s of %s (%s)`, this.formatByte(length), this.formatByte(total), this.formatPercentage(length / total))
|
|
37
|
-
}
|
|
38
|
-
})
|
|
39
|
-
data.pipe(writer)
|
|
40
|
-
|
|
41
|
-
return new Promise((resolve, reject) => {
|
|
42
|
-
writer.on('error', reject)
|
|
43
|
-
writer.on('finish', () => {
|
|
44
|
-
resolve(file)
|
|
45
|
-
})
|
|
46
|
-
})
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export default download
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
async function handler (rec, bulk) {
|
|
2
|
-
const { isFunction, set } = this.app.bajo.lib._
|
|
3
|
-
const { recordCreate, recordFind, recordUpdate } = this.app.bajoDb
|
|
4
|
-
const save = bulk.save ?? {}
|
|
5
|
-
const current = save.current ?? {}
|
|
6
|
-
let existing
|
|
7
|
-
let record
|
|
8
|
-
let method
|
|
9
|
-
save.checkUnique = save.checkUnique ?? 'id'
|
|
10
|
-
if (['unique', 'upsert'].includes(save.mode)) {
|
|
11
|
-
const query = isFunction(save.checkUnique) ? await save.checkUnique.call(this, rec, save) : set({}, save.checkUnique, rec[save.checkUnique])
|
|
12
|
-
const resp = await recordFind(save.coll, { query, limit: 1 }, { noCache: true })
|
|
13
|
-
if (resp.length > 0) existing = resp[0]
|
|
14
|
-
}
|
|
15
|
-
if (existing) {
|
|
16
|
-
if (save.mode === 'upsert') {
|
|
17
|
-
const body = save.updateConverter ? await save.updateConverter.call(this, rec, save) : rec
|
|
18
|
-
try {
|
|
19
|
-
record = await recordUpdate(save.coll, existing.id, body)
|
|
20
|
-
method = 'updated'
|
|
21
|
-
} catch (err) {
|
|
22
|
-
console.error(err)
|
|
23
|
-
method = 'error'
|
|
24
|
-
}
|
|
25
|
-
} else {
|
|
26
|
-
method = 'skipped'
|
|
27
|
-
}
|
|
28
|
-
} else {
|
|
29
|
-
try {
|
|
30
|
-
record = await recordCreate(save.coll, rec)
|
|
31
|
-
method = 'created'
|
|
32
|
-
} catch (err) {
|
|
33
|
-
console.error(err)
|
|
34
|
-
method = 'error'
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
if (record && current.coll && current.query) {
|
|
38
|
-
const query = await current.query.call(this, { body: rec, record, opts: save })
|
|
39
|
-
const recs = await recordFind(current.coll, { query }, { noCache: true })
|
|
40
|
-
const rc = current.converter ? await current.converter.call(this, { body: rec, record, opts: save }) : rec
|
|
41
|
-
if (rc) {
|
|
42
|
-
if (recs.length > 0) {
|
|
43
|
-
const id = recs[0].id
|
|
44
|
-
await recordUpdate(current.coll, id, rc)
|
|
45
|
-
} else {
|
|
46
|
-
await recordCreate(current.coll, rc)
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
return method
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
async function fetchAndSave ({ url, bulk, save = {}, opts = {} } = {}) {
|
|
54
|
-
const { startPlugin } = this.bajo
|
|
55
|
-
const { merge } = this.bajo.lib._
|
|
56
|
-
merge(bulk, { handler, save })
|
|
57
|
-
await startPlugin('bajoDb')
|
|
58
|
-
|
|
59
|
-
await this.fetchBulk(url, bulk, opts)
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export default fetchAndSave
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
async function fetching ({ url, opts, bulk, spin }) {
|
|
2
|
-
const { setImmediate, print } = this.app.bajo
|
|
3
|
-
const { isEmpty, isFunction, has } = this.app.bajo.lib._
|
|
4
|
-
const { validationErrorMessage } = this.app.bajoDb
|
|
5
|
-
const resp = await this.fetch(url, opts ?? {})
|
|
6
|
-
if (isEmpty(resp)) {
|
|
7
|
-
spin.fatal('noServerResponse')
|
|
8
|
-
return -1
|
|
9
|
-
}
|
|
10
|
-
if (bulk.abort) {
|
|
11
|
-
const aborted = await bulk.abort.call(this, resp)
|
|
12
|
-
if (aborted) {
|
|
13
|
-
spin.fatal(aborted)
|
|
14
|
-
return -1
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
let count = 0
|
|
18
|
-
const stat = { created: 0, updated: 0, skipped: 0, error: 0 }
|
|
19
|
-
bulk.dataKey = bulk.dataKey ?? 'data'
|
|
20
|
-
if (bulk.printCount === true) bulk.printCount = 100
|
|
21
|
-
const data = isFunction(bulk.dataKey) ? await bulk.dataKey.call(this, resp) : resp[bulk.dataKey]
|
|
22
|
-
if (data.length === 0) {
|
|
23
|
-
print.warn('noRecordToProcess')
|
|
24
|
-
return 0
|
|
25
|
-
}
|
|
26
|
-
spin.setText('gotRecordsProcessing%d', data.length)
|
|
27
|
-
for (let r of data) {
|
|
28
|
-
await setImmediate()
|
|
29
|
-
if (bulk.converter) r = await bulk.converter.call(this, r, bulk)
|
|
30
|
-
if (isEmpty(r)) {
|
|
31
|
-
stat.skipped++
|
|
32
|
-
continue
|
|
33
|
-
}
|
|
34
|
-
try {
|
|
35
|
-
const result = await bulk.handler.call(this, r, bulk)
|
|
36
|
-
if (result && has(stat, result)) stat[result]++
|
|
37
|
-
if (bulk.printCount && bulk.printCount < count && (count % bulk.printCount === 0)) print.succeed('[%s] Processed %d/%d', spin.getElapsed(), count, data.length)
|
|
38
|
-
else if (!spin.opts.isLog) spin.setText('rec%d%d', count, data.length)
|
|
39
|
-
count++
|
|
40
|
-
} catch (err) {
|
|
41
|
-
console.log(err)
|
|
42
|
-
spin.setText(validationErrorMessage(err) + ', continue')
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
print.succeed('recProcessed%s%d%d', spin.getElapsed(), count, data.length)
|
|
46
|
-
if (!bulk.noStat) print.succeed('createdUpdatedSkipped%s%d%d%d', spin.getElapsed(), stat.created, stat.updated, stat.skipped)
|
|
47
|
-
return data.length
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
async function fetchBulk (url, bulk = {}, opts = {}) {
|
|
51
|
-
const { isFunction } = this.bajo.lib._
|
|
52
|
-
opts.params = opts.params ?? {}
|
|
53
|
-
bulk.maxStep = bulk.maxStep ?? 0
|
|
54
|
-
if (!isFunction(bulk.handler)) throw this.error('handlerMustBeProvided')
|
|
55
|
-
if (isFunction(bulk.paramsIncFn)) {
|
|
56
|
-
this.print.info('bulkFetchStarting')
|
|
57
|
-
const spin = this.print.spinner({ showCounter: true }).start('fetchingStarts')
|
|
58
|
-
let step = 1
|
|
59
|
-
for (;;) {
|
|
60
|
-
this.print.info('batch%s%d', spin.getElapsed(), step)
|
|
61
|
-
const newOpts = await bulk.paramsIncFn.call(this, { url, bulk, opts })
|
|
62
|
-
if (newOpts) opts = newOpts
|
|
63
|
-
const length = await fetching.call(this, { url, bulk, opts, spin })
|
|
64
|
-
if (length === 0 || (bulk.maxStep > 0 && step >= bulk.maxStep)) {
|
|
65
|
-
this.print.info('allDone')
|
|
66
|
-
break
|
|
67
|
-
}
|
|
68
|
-
step++
|
|
69
|
-
}
|
|
70
|
-
} else {
|
|
71
|
-
const spin = this.print.spinner({ showCounter: true }).start('fetchingStarts')
|
|
72
|
-
await fetching.call(this, { url, bulk, opts, spin })
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export default fetchBulk
|
package/plugin/method/fetch.js
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import path from 'path'
|
|
2
|
-
import { fetch, Agent } from 'undici'
|
|
3
|
-
|
|
4
|
-
async function fetchUrl (url, opts = {}, extra = {}) {
|
|
5
|
-
const { isSet } = this.app.bajo
|
|
6
|
-
const { fs } = this.app.bajo.lib
|
|
7
|
-
const { has, isArray, isPlainObject, isString, cloneDeep, merge } = this.app.bajo.lib._
|
|
8
|
-
if (isPlainObject(url)) {
|
|
9
|
-
extra = cloneDeep(opts)
|
|
10
|
-
opts = cloneDeep(url)
|
|
11
|
-
url = opts.url
|
|
12
|
-
delete opts.url
|
|
13
|
-
}
|
|
14
|
-
if (opts.method) opts.method = opts.method.toUpperCase()
|
|
15
|
-
if (opts.auth) {
|
|
16
|
-
opts.headers.Authorization = `Basic ${Buffer.from(`${opts.auth.username}:${opts.auth.password}`).toString('base64')}`
|
|
17
|
-
delete opts.auth
|
|
18
|
-
}
|
|
19
|
-
opts.query = merge({}, opts.query, opts.params ?? {})
|
|
20
|
-
delete opts.params
|
|
21
|
-
if (!has(extra, 'cacheBuster')) extra.cacheBuster = true
|
|
22
|
-
if (extra.cacheBuster) opts.query[extra.cacheBusterKey ?? '_'] = Date.now()
|
|
23
|
-
if (this.config.fetch.agent || extra.agent) {
|
|
24
|
-
opts.dispatcher = new Agent(extra.agent ?? this.config.fetch.agent)
|
|
25
|
-
}
|
|
26
|
-
if (opts.body && extra.formData) {
|
|
27
|
-
const formData = new FormData()
|
|
28
|
-
for (const key in opts.body) {
|
|
29
|
-
let fname
|
|
30
|
-
let val = opts.body[key]
|
|
31
|
-
if (!isSet(val)) continue
|
|
32
|
-
if (isString(val) && val.startsWith('file:///')) {
|
|
33
|
-
fname = path.basename(val)
|
|
34
|
-
val = new Blob([fs.readFileSync(val.slice(8))])
|
|
35
|
-
} else if (isPlainObject(val) || isArray(val)) val = JSON.stringify(val)
|
|
36
|
-
if (fname) formData.append(key, val, fname)
|
|
37
|
-
else formData.append(key, val)
|
|
38
|
-
}
|
|
39
|
-
opts.body = formData
|
|
40
|
-
}
|
|
41
|
-
if (opts.query) {
|
|
42
|
-
// todo: what if url already contain query string?
|
|
43
|
-
const query = new URLSearchParams(opts.query)
|
|
44
|
-
url += '?' + query
|
|
45
|
-
}
|
|
46
|
-
const resp = await fetch(url, opts)
|
|
47
|
-
if (extra.rawResponse) return resp
|
|
48
|
-
return await resp.json()
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
export default fetchUrl
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import numbro from 'numbro'
|
|
2
|
-
|
|
3
|
-
function byte (value, opts = {}) {
|
|
4
|
-
opts.output = 'byte'
|
|
5
|
-
opts.base = 'binary'
|
|
6
|
-
opts.mantissa = opts.mantissa ?? opts.scale ?? 2
|
|
7
|
-
opts.spaceSeparated = opts.spaceSeparated ?? true
|
|
8
|
-
return numbro(value).format(opts)
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export default byte
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import numbro from 'numbro'
|
|
2
|
-
|
|
3
|
-
function percentage (value, opts = {}) {
|
|
4
|
-
opts.output = 'percent'
|
|
5
|
-
opts.mantissa = opts.mantissa ?? opts.scale ?? 2
|
|
6
|
-
opts.spaceSeparated = opts.spaceSeparated ?? true
|
|
7
|
-
return numbro(value).format(opts)
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
export default percentage
|
package/plugin/method/gunzip.js
DELETED
package/plugin/method/gzip.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { createGzip, createGunzip } from 'zlib'
|
|
2
|
-
|
|
3
|
-
function gzip (file, deleteOld, expand) {
|
|
4
|
-
const { fs } = this.app.bajo.lib
|
|
5
|
-
return new Promise((resolve, reject) => {
|
|
6
|
-
const newFile = expand ? file.slice(0, file.length - 3) : (file + '.gz')
|
|
7
|
-
const reader = fs.createReadStream(file)
|
|
8
|
-
const writer = fs.createWriteStream(newFile)
|
|
9
|
-
const method = expand ? createGunzip() : createGzip()
|
|
10
|
-
reader.pipe(method).pipe(writer)
|
|
11
|
-
writer.on('error', reject)
|
|
12
|
-
writer.on('finish', err => {
|
|
13
|
-
if (err) return reject(err)
|
|
14
|
-
if (deleteOld) fs.unlinkSync(file)
|
|
15
|
-
resolve()
|
|
16
|
-
})
|
|
17
|
-
})
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
export default gzip
|
package/plugin/method/hash.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import bcrypt from 'bcrypt'
|
|
2
|
-
import crypto from 'crypto'
|
|
3
|
-
|
|
4
|
-
async function hash (text, type = 'md5', options = {}) {
|
|
5
|
-
options.digest = options.digest ?? 'hex'
|
|
6
|
-
options.salt = options.hash ?? 10
|
|
7
|
-
if (typeof text !== 'string') text = JSON.stringify(text)
|
|
8
|
-
if (type === 'bcrypt') return await bcrypt.hash(text, options.salt)
|
|
9
|
-
if (type === 'short') {
|
|
10
|
-
type = 'shake256'
|
|
11
|
-
options.outputLength = 6
|
|
12
|
-
}
|
|
13
|
-
return crypto.createHash(type, options).update(text).digest(options.digest)
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export default hash
|