mobiletto 1.2.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/lib/cjs/index.d.ts +2 -0
  2. package/lib/cjs/index.js +30 -0
  3. package/lib/cjs/package.json +16 -0
  4. package/lib/esm/index.d.ts +2 -0
  5. package/lib/esm/index.js +12 -0
  6. package/package.json +27 -13
  7. package/.babelrc +0 -5
  8. package/.eslintrc.js +0 -14
  9. package/docs/README.md +0 -113
  10. package/docs/af/README.md +0 -481
  11. package/docs/am/README.md +0 -481
  12. package/docs/ar/README.md +0 -481
  13. package/docs/az/README.md +0 -481
  14. package/docs/be/README.md +0 -481
  15. package/docs/bg/README.md +0 -481
  16. package/docs/bn/README.md +0 -481
  17. package/docs/bs/README.md +0 -481
  18. package/docs/ca/README.md +0 -481
  19. package/docs/ceb/README.md +0 -481
  20. package/docs/co/README.md +0 -481
  21. package/docs/cs/README.md +0 -481
  22. package/docs/cy/README.md +0 -481
  23. package/docs/da/README.md +0 -481
  24. package/docs/de/README.md +0 -481
  25. package/docs/el/README.md +0 -481
  26. package/docs/eo/README.md +0 -481
  27. package/docs/es/README.md +0 -481
  28. package/docs/et/README.md +0 -481
  29. package/docs/eu/README.md +0 -481
  30. package/docs/fa/README.md +0 -481
  31. package/docs/fi/README.md +0 -481
  32. package/docs/fr/README.md +0 -481
  33. package/docs/fy/README.md +0 -481
  34. package/docs/ga/README.md +0 -481
  35. package/docs/gd/README.md +0 -481
  36. package/docs/gl/README.md +0 -481
  37. package/docs/gu/README.md +0 -481
  38. package/docs/ha/README.md +0 -481
  39. package/docs/haw/README.md +0 -481
  40. package/docs/he/README.md +0 -481
  41. package/docs/hi/README.md +0 -481
  42. package/docs/hmn/README.md +0 -481
  43. package/docs/hr/README.md +0 -481
  44. package/docs/ht/README.md +0 -481
  45. package/docs/hu/README.md +0 -481
  46. package/docs/hy/README.md +0 -481
  47. package/docs/id/README.md +0 -481
  48. package/docs/ig/README.md +0 -481
  49. package/docs/is/README.md +0 -481
  50. package/docs/it/README.md +0 -481
  51. package/docs/iw/README.md +0 -481
  52. package/docs/ja/README.md +0 -481
  53. package/docs/jv/README.md +0 -481
  54. package/docs/ka/README.md +0 -481
  55. package/docs/kk/README.md +0 -481
  56. package/docs/km/README.md +0 -481
  57. package/docs/kn/README.md +0 -481
  58. package/docs/ko/README.md +0 -481
  59. package/docs/ku/README.md +0 -481
  60. package/docs/ky/README.md +0 -481
  61. package/docs/la/README.md +0 -481
  62. package/docs/lb/README.md +0 -481
  63. package/docs/lo/README.md +0 -481
  64. package/docs/lt/README.md +0 -481
  65. package/docs/lv/README.md +0 -481
  66. package/docs/mg/README.md +0 -481
  67. package/docs/mi/README.md +0 -481
  68. package/docs/mk/README.md +0 -481
  69. package/docs/ml/README.md +0 -481
  70. package/docs/mn/README.md +0 -481
  71. package/docs/mr/README.md +0 -481
  72. package/docs/ms/README.md +0 -481
  73. package/docs/mt/README.md +0 -481
  74. package/docs/my/README.md +0 -481
  75. package/docs/ne/README.md +0 -481
  76. package/docs/nl/README.md +0 -481
  77. package/docs/no/README.md +0 -481
  78. package/docs/ny/README.md +0 -481
  79. package/docs/or/README.md +0 -481
  80. package/docs/pa/README.md +0 -481
  81. package/docs/pl/README.md +0 -481
  82. package/docs/ps/README.md +0 -481
  83. package/docs/pt/README.md +0 -481
  84. package/docs/ro/README.md +0 -481
  85. package/docs/ru/README.md +0 -481
  86. package/docs/rw/README.md +0 -481
  87. package/docs/sd/README.md +0 -481
  88. package/docs/si/README.md +0 -481
  89. package/docs/sk/README.md +0 -481
  90. package/docs/sl/README.md +0 -481
  91. package/docs/sm/README.md +0 -481
  92. package/docs/sn/README.md +0 -481
  93. package/docs/so/README.md +0 -481
  94. package/docs/sq/README.md +0 -481
  95. package/docs/sr/README.md +0 -481
  96. package/docs/st/README.md +0 -481
  97. package/docs/su/README.md +0 -481
  98. package/docs/sv/README.md +0 -481
  99. package/docs/sw/README.md +0 -481
  100. package/docs/ta/README.md +0 -481
  101. package/docs/te/README.md +0 -481
  102. package/docs/tg/README.md +0 -481
  103. package/docs/th/README.md +0 -481
  104. package/docs/tk/README.md +0 -481
  105. package/docs/tl/README.md +0 -481
  106. package/docs/tr/README.md +0 -481
  107. package/docs/tt/README.md +0 -481
  108. package/docs/ug/README.md +0 -481
  109. package/docs/uk/README.md +0 -481
  110. package/docs/ur/README.md +0 -481
  111. package/docs/uz/README.md +0 -481
  112. package/docs/vi/README.md +0 -481
  113. package/docs/xh/README.md +0 -481
  114. package/docs/yi/README.md +0 -481
  115. package/docs/yo/README.md +0 -481
  116. package/docs/zh/README.md +0 -481
  117. package/docs/zu/README.md +0 -481
  118. package/drivers/b2/index.js +0 -451
  119. package/drivers/local/index.js +0 -315
  120. package/drivers/s3/index.js +0 -316
  121. package/hokey.json +0 -16
  122. package/index.js +0 -29
  123. package/test/api.spec.js +0 -512
  124. package/test/crypt.spec.js +0 -67
@@ -1,451 +0,0 @@
1
- const B2 = require('backblaze-b2')
2
-
3
- // this library is a godsend for B2 uploads
4
- B2.prototype.uploadAny = require('@gideo-llc/backblaze-b2-upload-any')
5
-
6
- const { Readable } = require('stream')
7
-
8
- const {
9
- M_FILE, M_DIR,
10
- readStream, isReadable, isAsyncGenerator,
11
- MobilettoError, MobilettoNotFoundError
12
- } = require('../../index')
13
-
14
- const { logger } = require('../../util/logger')
15
-
16
- // B2 tokens last 24 hours. Let's refresh ours after 23 hours and 50 minutes
17
- const AUTH_REFRESH = (1000 * 60 * 60 * 24) - (1000 * 60 * 10)
18
-
19
- class StorageClient {
20
- b2
21
- bucket
22
- prefix
23
- delimiter
24
- opts
25
- lastAuth
26
- absoluteMinimumPartSize
27
- recommendedPartSize
28
- configuredPartSize
29
- normalizeRegex
30
- constructor(keyId, appKey, opts) {
31
- if (!keyId || !appKey || !opts.bucket) throw new MobilettoError(`b2.StorageClient: key, secret and opts.bucket are required`)
32
- this.b2 = new B2({
33
- applicationKeyId: keyId,
34
- applicationKey: appKey
35
- })
36
- this.opts = opts
37
- this.configuredPartSize = opts.partSize ? opts.partSize : -1
38
- this.bucket = opts.bucket
39
- this.delimiter = opts.delimiter || '/'
40
- this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
41
- this.prefix = opts.prefix
42
- ? opts.prefix.endsWith(this.delimiter)
43
- ? opts.prefix
44
- : opts.prefix + this.delimiter
45
- : ''
46
- this.lastAuth = 0
47
- }
48
-
49
- auth = async () => {
50
- if (Date.now() - this.lastAuth > AUTH_REFRESH) {
51
- try {
52
- const response = await this.b2.authorize({})
53
- this.lastAuth = Date.now()
54
- this.absoluteMinimumPartSize = response.data.absoluteMinimumPartSize
55
- this.recommendedPartSize = response.data.recommendedPartSize
56
- if (this.configuredPartSize === -1) {
57
- logger.info(`partSize was not configured, using recommendedPartSize=${this.recommendedPartSize}`)
58
- this.configuredPartSize = this.recommendedPartSize
59
- } else if (this.configuredPartSize < this.absoluteMinimumPartSize) {
60
- logger.warn(`partSize was too small (${this.configuredPartSize}), using absoluteMinimumPartSize=${this.absoluteMinimumPartSize}`)
61
- this.configuredPartSize = this.absoluteMinimumPartSize
62
- }
63
- } catch (e) {
64
- logger.error(`auth error: ${e}`)
65
- throw e
66
- }
67
- }
68
- }
69
-
70
- testConfig = async () => await this.b2_list('', false, null, {max: 1})
71
-
72
- normalizePath = (path) => {
73
- const p = (path.startsWith(this.prefix)
74
- ? path
75
- : this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
76
- return p.replaceAll(this.normalizeRegex, this.delimiter)
77
- }
78
- denormalizePath = (path) => {
79
- const norm = path.startsWith(this.prefix) ? path.substring(this.prefix.length) : path
80
- return norm.startsWith(this.delimiter) ? norm.substring(1) : norm
81
- }
82
- bbError = (err, path, method) => {
83
- if (err.response && err.response.status && err.response.status === 404) {
84
- return new MobilettoNotFoundError(path)
85
- } else if (err.response && err.response.data && err.response.data.message) {
86
- return new MobilettoError(`${method} error: ${err.response.data.message}`)
87
- }
88
- return err
89
- }
90
-
91
- b2type = f => f.fileName.endsWith(this.delimiter) && f.contentType === null ? M_DIR : M_FILE
92
-
93
- file2object(f) {
94
- return {
95
- name: this.denormalizePath(f.fileName),
96
- size: f.contentLength,
97
- mtime: f.uploadTimestamp,
98
- type: this.b2type(f),
99
- b2id: f.fileId
100
- }
101
- }
102
-
103
- processList = async (path, response, visitor) => {
104
- const files = response && response.data && response.data.files ? response.data.files : null
105
- if (!files) {
106
- return []
107
- }
108
- const objects = files.map((f) => {
109
- logger.debug(`addListedFiles(${path}) adding file: ${f.fileName}`)
110
- return this.file2object(f)
111
- })
112
- if (visitor) {
113
- for (const obj of objects) { await visitor(obj) }
114
- }
115
- return objects
116
- }
117
-
118
- list = (pth = '', recursive = false, visitor = null) =>
119
- this.b2_list(pth, recursive, visitor)
120
-
121
- async b2_list (pth = '', recursive = false, visitor = null, opts = {}) {
122
- try {
123
- let optsDir = opts.dir || false
124
- let dirDiscovered = null
125
- let firstResult = null
126
- const files = []
127
- await this.auth()
128
- const hasMax = opts && opts.max
129
- const maxFileCount = hasMax ? opts.max : 10000
130
- const pfx = (this.prefix.length === 0 || this.prefix.endsWith(this.delimiter) ? this.prefix : this.prefix + this.delimiter) +
131
- (pth.startsWith(this.delimiter) ? pth.substring(0) : pth)
132
-
133
- let response = await this.b2.listFileNames({
134
- bucketId: this.bucket,
135
- maxFileCount,
136
- delimiter: recursive ? '' : this.delimiter,
137
- prefix: pfx
138
- })
139
- const handleResponse = async (files, pth, response, visitor) => {
140
- const list = await this.processList(pth, response, visitor)
141
- if (list.length === 0) {
142
- return true
143
- }
144
- if (!optsDir && !firstResult && list.length > 0) {
145
- firstResult = list[0]
146
- const exact = firstResult.name === pth || firstResult.name === pth + this.delimiter
147
- if (exact) {
148
- // we found exactly what we were looking for
149
- // if the next thing in the list starts with it plus delimiter,
150
- // it is a directory, return the contents as long as they match
151
- if (list.length > 1 && list[1].name.startsWith(pth + this.delimiter)) {
152
- const dirEntries = list.filter(o => o.name.startsWith(pth));
153
- files.push(...dirEntries)
154
- return false
155
- } else if (firstResult.name.endsWith(this.delimiter)) {
156
- const dirEntries = list.filter(o => o.name.startsWith(pth));
157
- files.push(...dirEntries)
158
- dirDiscovered = true
159
- return true
160
-
161
- } else {
162
- // it's an exact file-match, return it
163
- files.push(list[0])
164
- return true
165
- }
166
- }
167
- }
168
- if (hasMax && files.length + list.length >= maxFileCount) {
169
- files.push(...list.slice(0, (maxFileCount - files.length) + 1))
170
- return true
171
- } else {
172
- files.push(...list)
173
- return false
174
- }
175
- }
176
- while (response.data.nextFileName) {
177
- if (await handleResponse(files, pth, response, visitor)) {
178
- return files
179
- }
180
- response = await this.b2.listFileNames({
181
- bucketId: this.bucket,
182
- startFileName: response.data.nextFileName,
183
- maxFileCount,
184
- delimiter: recursive ? '' : this.delimiter,
185
- prefix: pfx
186
- })
187
- }
188
- await handleResponse(files, pth, response, visitor)
189
- if (dirDiscovered && !pth.endsWith(this.delimiter)) {
190
- const subdirFiles = await this.b2_list(pth + this.delimiter, recursive, null, { dir: true })
191
- files.shift()
192
- files.push(...subdirFiles)
193
- }
194
- return files
195
- } catch (err) {
196
- throw this.bbError(err, pth, 'list')
197
- }
198
- }
199
-
200
- b2_meta = (path) => (this.driver_metadata ? this.driver_metadata(path) : this.metadata(path))
201
-
202
- async metadata (pth) {
203
- await this.auth()
204
- let files = null
205
- const normPath = this.normalizePath(pth);
206
- try {
207
- files = (await this.b2.listFileNames({
208
- bucketId: this.bucket,
209
- startFileName: normPath,
210
- maxFileCount: 1,
211
- delimiter: '',
212
- prefix: this.prefix
213
- })).data.files
214
- } catch (err) {
215
- throw this.bbError(err, pth, 'metadata')
216
- }
217
- if (files.length === 0) {
218
- throw new MobilettoNotFoundError(pth)
219
- }
220
- const found = files.filter(f => f.fileName === normPath)
221
- if (!found || found.length === 0) {
222
- // if this is a directory, we may have returned the first item in it, that's OK
223
- if (files[0].fileName.startsWith(normPath.endsWith(this.delimiter) ? normPath : normPath + this.delimiter)) {
224
- let obj = this.file2object(files[0]);
225
- obj.name = pth
226
- obj.type = M_DIR
227
- return obj
228
- }
229
- throw new MobilettoNotFoundError(pth)
230
- }
231
- if (found.length > 1) {
232
- throw new MobilettoError(`metadata(${pth}) multiple files matched: ${JSON.stringify(found)}`)
233
- }
234
- return this.file2object(files[0])
235
- }
236
-
237
- async write (path, generatorOrReadable) {
238
- await this.auth()
239
- logger.debug(`write: writing path ${path}`)
240
- let stream = generatorOrReadable
241
- if (!isReadable(generatorOrReadable)) {
242
- stream = new Readable({
243
- async read (size) {
244
- let chunk = isAsyncGenerator(generatorOrReadable)
245
- ? (await generatorOrReadable.next()).value
246
- : generatorOrReadable.next().value
247
- this.push(typeof chunk !== 'undefined' ? chunk : null)
248
- }
249
- })
250
- }
251
- let response = null
252
- try {
253
- response = await this.b2.uploadAny({
254
- bucketId: this.bucket,
255
- fileName: this.normalizePath(path),
256
- partSize: this.configuredPartSize,
257
- data: stream
258
- });
259
- // response is from either b2_upload_file or b2_finish_large_file
260
- // It depends on how large the file was, see: https://github.com/gideo-llc/backblaze-b2-upload-any/blob/master/README.md#uploadanyoptions
261
- if (response.contentLength) {
262
- return response.contentLength
263
- }
264
- } catch (e) {
265
- logger.error(`write error: ${e}`)
266
- throw this.bbError(e, path, 'write')
267
- }
268
- if (response) {
269
- throw new MobilettoError(`write error: response status ${response.status} with data ${JSON.stringify(response.data)}`)
270
- }
271
- }
272
-
273
- async read (path, callback, endCallback = null) {
274
- await this.auth()
275
- const file = this.normalizePath(path)
276
- logger.debug(`read: reading path: ${path}`)
277
- let meta = null
278
- try {
279
- meta = await (this.b2_meta(file))
280
- } catch (e) {
281
- if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
282
- throw e
283
- }
284
- throw new MobilettoError(`read: error reading metadata for path ${path}`, e)
285
- }
286
- if (meta.type === M_DIR) {
287
- throw new MobilettoError(`read: cannot read directory: ${path}`)
288
- }
289
- try {
290
- const stream = await this.b2.downloadFileById({
291
- bucketId: this.bucket,
292
- fileId: meta.b2id,
293
- responseType: 'stream'
294
- })
295
- return await readStream(stream.data, callback, endCallback)
296
- } catch (err) {
297
- throw this.bbError(err, path, 'read')
298
- }
299
- }
300
-
301
- async deleteVersions (path, normPath, versions, quiet, errors = {}) {
302
- const removed = []
303
- for (const version of versions) {
304
- logger.debug(`deleteVersions(${path}, ${versions.length} versions) pushing version to delete: fileName=${version.fileName} fileId=${version.fileId}`)
305
- await this.auth()
306
- try {
307
- if (!version.fileName.startsWith(normPath)) {
308
- logger.debug(`deleteVersions(${path}, ${versions.length} versions) ending iteration on mismatched file: ${version.fileName}`)
309
- return removed
310
- }
311
- const delResponse = await this.b2.deleteFileVersion({
312
- bucketId: this.bucket,
313
- fileName: version.fileName,
314
- fileId: version.fileId
315
- })
316
- if (delResponse && delResponse.status !== 200) {
317
- const message = `deleteVersions(${path}) received HTTP status ${delResponse.status} trying to delete file version: ${JSON.stringify(version)}`
318
- logger.error(message)
319
- if (errors) {
320
- errors[path] = message
321
- }
322
- } else if (removed) {
323
- logger.debug(`deleteVersions(${path}, ${versions.length} versions) REMOVED: fileName=${version.fileName} fileId=${version.fileId}`)
324
- removed.push(this.denormalizePath(version.fileName))
325
- }
326
- } catch (err) {
327
- const message = `deleteVersions(${path}) error deleting version ${version.fileId}: ${err}`
328
- logger.error(message)
329
- if (errors) {
330
- errors[path] = message
331
- }
332
- if (!quiet) throw new MobilettoError(message, err)
333
- }
334
- }
335
- const errPaths = Object.keys(errors);
336
- if (errPaths.length > 0) {
337
- if (quiet) {
338
- logger.info(`deleteVersions(${path}) had ${errPaths.length} errors but quiet=true, returning ${removed.length} removed paths`)
339
- return removed
340
- }
341
- throw new MobilettoError(`deleteVersions(${path}) had errors: ${JSON.stringify(errors)}`)
342
- }
343
- return removed
344
- }
345
-
346
- async remove (path, recursive, quiet) {
347
- logger.debug(`b2.remove(${path}, recursive=${recursive}) starting...`)
348
- await this.auth()
349
- let meta = null
350
- const errors = {}
351
- const normPath = this.normalizePath(path)
352
- try {
353
- logger.debug(`b2.remove(${path}) fetching metadata to determine file/directory`)
354
- meta = await (this.b2_meta(path))
355
- } catch (e) {
356
- if (quiet) {
357
- logger.debug(`b2.remove(${path}) quietly returning, not reporting error fetching metadata for ${path}: ${e}`)
358
- return path
359
- }
360
- if (e instanceof MobilettoError || e instanceof MobilettoNotFoundError) {
361
- throw e
362
- }
363
- throw new MobilettoError(`b2.remove(${path}) error reading metadata for path ${path}`, e)
364
- }
365
- if (meta.type === M_FILE) {
366
- try {
367
- const response = await this.b2.listFileVersions({
368
- bucketId: this.bucket,
369
- startFileName: normPath,
370
- startFileId: meta.b2id,
371
- prefix: this.prefix,
372
- delimiter: this.delimiter,
373
- maxFileCount: 1000
374
- })
375
- const versions = response.data.files.filter(f => f.fileName === normPath)
376
- await this.deleteVersions(path, normPath, versions, quiet, errors)
377
- } catch (err) {
378
- if (err instanceof MobilettoError) {
379
- throw err
380
- }
381
- throw new MobilettoError(`b2.remove(${path}) error: ${err}`, err)
382
- }
383
- return path
384
- } else if (meta.type === M_DIR) {
385
- if (!recursive) {
386
- if (quiet) {
387
- logger.debug(`b2.remove(${path}) quietly returning, not deleting directory when recursive=false`)
388
- return path
389
- }
390
- throw new MobilettoError(`b2.remove(${path}) cannot remove directory (set recursive=true to remove)`)
391
- }
392
- const removed = []
393
- const errors = {}
394
- let versionsResponse
395
- try {
396
- logger.debug(`b2.remove(${path})`)
397
- let normPath = this.normalizePath(path);
398
- versionsResponse = await this.b2.listFileVersions({
399
- bucketId: this.bucket,
400
- startFileName: normPath,
401
- prefix: normPath,
402
- delimiter: '',
403
- maxFileCount: 1000
404
- })
405
- while (versionsResponse.data.nextFileName &&
406
- versionsResponse.data.nextFileId &&
407
- versionsResponse.data.nextFileName.startsWith(normPath)) {
408
- logger.debug(`b2.remove(${path}) removing a batch of files then asking for more`)
409
- removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
410
- logger.debug(`b2.remove(${path})`)
411
- versionsResponse = await this.b2.listFileVersions({
412
- bucketId: this.bucket,
413
- startFileName: versionsResponse.data.nextFileName,
414
- startFileId: versionsResponse.data.nextFileId,
415
- prefix: normPath,
416
- delimiter: ''
417
- })
418
- }
419
- logger.debug(`b2.remove(${path}) removing last batch of files`)
420
- removed.push(...(await this.deleteVersions(path, normPath, versionsResponse.data.files, quiet, errors)))
421
- } catch (e) {
422
- const message = `b2.remove(${path}) error listing/deleting: ${e}`
423
- if (quiet) {
424
- logger.warn(message)
425
- } else {
426
- throw new MobilettoError(message)
427
- }
428
- }
429
- if (Object.keys(errors).length > 0) {
430
- if (quiet) {
431
- logger.warn(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
432
- } else {
433
- throw new MobilettoError(`b2.remove(${path}) errors removing some paths: ${JSON.stringify(errors)}`)
434
- }
435
- }
436
- return removed
437
-
438
- } else {
439
- throw new MobilettoError(`b2.remove(${path}) cannot remove file of type: ${meta.type}`)
440
- }
441
- }
442
- }
443
-
444
- function storageClient (key, secret, opts) {
445
- if (!key || !secret || !opts || !opts.bucket) {
446
- throw new MobilettoError('b2.storageClient: key, secret, and opts.bucket are required')
447
- }
448
- return new StorageClient(key, secret, opts)
449
- }
450
-
451
- module.exports = { storageClient }