mobiletto 1.1.40 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -9,6 +9,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
9
9
  * [Mobiletto CLI](#mobiletto-cli)
10
10
  * [Source](#Source)
11
11
  * [Installation](#Installation)
12
+ * [Minimal install](#Minimal-install)
12
13
  * [Support and Funding](#Support-and-Funding)
13
14
  * [Basic usage](#Basic-usage)
14
15
  * [Metadata](#Metadata)
@@ -91,9 +92,42 @@ Current Mobiletto storage drivers:
91
92
  * `s3`: Amazon S3
92
93
  * `b2`: Backblaze B2
93
94
  * `local`: local filesystem
95
+ * `indexeddb`: IndexedDB storage
94
96
 
95
97
  *Contributions to support more cloud storage providers are very welcome!*
96
98
 
99
+ ## Quick Start
100
+ A short example using the mobiletto `s3` driver.
101
+
102
+ This code would run the same with any other mobiletto storage driver.
103
+
104
+ const storage = require('mobiletto')
105
+ const bucket = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
106
+
107
+ // list objects: returns array of metadata objects
108
+ const listing = await bucket.list()
109
+ const dirList = await bucket.list('some/dir/')
110
+ const everything = await bucket.list('', {recursive: true})
111
+
112
+ // write an entire file
113
+ let bytesWritten = await bucket.writeFile('some/path', someBufferOfData)
114
+
115
+ // write a file from a stream/generator
116
+ bytesWritten = await bucket.write('some/path', streamOrGenerator)
117
+
118
+ // read an entire file
119
+ // returns null if an exception would otherwise be thrown
120
+ const bufferOrNull = await bucket.safeReadFile('some/path')
121
+
122
+ // stream-read a file, passing data to callback
123
+ const bytesRead = await bucket.read('some/path', (chunk) => { ...do something with chunk... } )
124
+
125
+ // remove a file, returns the path removed
126
+ let removed = await bucket.remove('some/path') // removed is a string
127
+
128
+ // remove a directory, returns array of paths removed
129
+ removed = await bucket.remove('some/directory', {recursive: true}) // removed is now an array!
130
+
97
131
  ## mobiletto-cli
98
132
  Mobiletto is intended to be used as a library by other JavaScript code.
99
133
 
@@ -118,44 +152,44 @@ If you really want the README files in every language, install the full version:
118
152
  npm install mobiletto
119
153
  yarn add mobiletto
120
154
 
121
- ## Quick Start
122
- A short example using the mobiletto `s3` driver.
155
+ ### Minimal install
156
+ The `mobiletto` and `mobiletto-lite` packages include support for all mobiletto storage drivers.
123
157
 
124
- This code would run the same if the driver were `b2` or `local`.
158
+ If you know the storage driver(s) that you will need ahead of time, you can lighten your app's
159
+ footprint by installing the `mobiletto-base` package, and then one or more of the `mobiletto-driver-` packages:
125
160
 
126
- const storage = require('mobiletto')
127
- const bucket = await storage.connect('s3', aws_key, aws_secret, {bucket: 'bk'})
161
+ #### Minimal installation with npm
128
162
 
129
- // list objects: returns array of metadata objects
130
- const listing = await bucket.list()
131
- const dirList = await bucket.list('some/dir/')
132
- const everything = await bucket.list('', {recursive: true})
133
-
134
- // write an entire file
135
- let bytesWritten = await bucket.writeFile('some/path', someBufferOfData)
136
-
137
- // write a file from a stream/generator
138
- bytesWritten = await bucket.write('some/path', streamOrGenerator)
163
+ npm install mobiletto-base # install driver-less mobiletto
139
164
 
140
- // read an entire file
141
- // returns null if an exception would otherwise be thrown
142
- const bufferOrNull = await bucket.safeReadFile('some/path')
165
+ npm install mobiletto-driver-s3 # install Amazon S3 driver
166
+ npm install mobiletto-driver-b2 # install Backblaze B2 driver
167
+ npm install mobiletto-driver-local # install local filesystem driver
168
+ npm install mobiletto-driver-indexeddb # install IndexedDB driver
143
169
 
144
- // stream-read a file, passing data to callback
145
- const bytesRead = await bucket.read('some/path', (chunk) => { ...do something with chunk... } )
170
+ #### Minimal installation with yarn
146
171
 
147
- // remove a file, returns the path removed
148
- let removed = await bucket.remove('some/path') // removed is a string
172
+ yarn add mobiletto-base # install driver-less mobiletto
149
173
 
150
- // remove a directory, returns array of paths removed
151
- removed = await bucket.remove('some/directory', {recursive: true}) // removed is now an array!
174
+ yarn add mobiletto-driver-s3 # install Amazon S3 driver
175
+ yarn add mobiletto-driver-b2 # install Backblaze B2 driver
176
+ yarn add mobiletto-driver-local # install local filesystem driver
177
+ yarn add mobiletto-driver-indexeddb # install IndexedDB driver
152
178
 
153
179
  ----
154
180
  ## Basic usage
155
181
  A much more extensive example, showing most of the features offered:
156
182
 
183
+ // When using mobiletto or mobiletto-lite, all drivers are included
157
184
  const { mobiletto } = require('mobiletto')
158
185
 
186
+ // When using mobiletto-base, use registerDriver to load a storage driver before connecting
187
+ const { registerDriver, mobiletto } = = require('mobiletto-base')
188
+ registerDriver('s3', require('mobiletto-driver-s3'))
189
+ registerDriver('b2', require('mobiletto-driver-b2'))
190
+ registerDriver('local', require('mobiletto-driver-local'))
191
+ registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
192
+
159
193
  // General usage
160
194
  const api = await mobiletto(driverName, key, secret, opts)
161
195
 
@@ -190,6 +224,13 @@ A much more extensive example, showing most of the features offered:
190
224
  // * partSize: optional, large files will be split into chunks of this size when uploading
191
225
  const b3 = await mobiletto('b2', b2_key_id, b2_app_key, {bucket: 'bk', partSize: 10000000})
192
226
 
227
+ // To use 'indexeddb' driver:
228
+ // * key: IndexedDB database name
229
+ // * secret: ignored, can be null
230
+ // * opts object:
231
+ // * indexedDB: required, the indexedDB object (an instance of IDBFactory)
232
+ const idb = await mobiletto('indexeddb', 'my_db', null, { indexedDB })
233
+
193
234
  // List files
194
235
  api.list() // --> returns an array of metadata objects
195
236
 
@@ -431,11 +472,11 @@ The object that the storageClient function returns must define these functions:
431
472
  // Read metadata for a path
432
473
  async metadata (path)
433
474
 
434
- // Read a file
475
+ // Read a file, return bytes read
435
476
  // callback receives a chunk of data. endCallback is called at end-of-stream
436
477
  async read (path, callback, endCallback = null)
437
478
 
438
- // Write a file
479
+ // Write a file, return bytes written
439
480
  // driver must be able to handle a generator or a stream
440
481
  async write (path, generatorOrReadableStream)
441
482
 
package/index.js CHANGED
@@ -1,865 +1,10 @@
1
- const fs = require('fs')
2
- const { Readable, Transform } = require('stream')
3
- const Queue = require('bull')
4
- const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
5
- const { getRedis, teardown } = require('./util/redis')
6
- const { basename, dirname } = require('path')
7
- const shasum = require('shasum')
8
- const randomstring = require('randomstring')
1
+ const { registerDriver } = require('mobiletto-base')
9
2
 
10
- const crypt = require('./util/crypt')
11
- const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
- const LRU = require("lru-cache")
13
-
14
- const DIR_ENT_DIR_SUFFIX = '__.dirent'
15
- const DIR_ENT_FILE_PREFIX = 'dirent__'
16
- const ENC_PAD_SEP = ' ~ '
17
-
18
- const initMobilettoTempDir = () => {
19
- const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
20
- return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
21
- }
22
- const MOBILETTO_TMP = initMobilettoTempDir()
23
-
24
- const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
25
- const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
26
- const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
27
-
28
- // adapted from https://stackoverflow.com/a/27724419
29
- function MobilettoError (message, err) {
30
- this.message = `${message}: ${err ? err : ''}`
31
- // noinspection JSUnusedGlobalSymbols
32
- this.err = err
33
- // Use V8's native method if available, otherwise fallback
34
- if ('captureStackTrace' in Error) {
35
- Error.captureStackTrace(this, TypeError)
36
- } else {
37
- // noinspection JSUnusedGlobalSymbols
38
- this.stack = (new Error(this.message)).stack
39
- }
40
- MobilettoError.prototype.toString = () => JSON.stringify(this)
41
- }
42
-
43
- // adapted from https://stackoverflow.com/a/27724419
44
- function MobilettoNotFoundError (message) {
45
- this.message = `${message}`
46
- // Use V8's native method if available, otherwise fallback
47
- if ('captureStackTrace' in Error) {
48
- Error.captureStackTrace(this, TypeError)
49
- } else {
50
- // noinspection JSUnusedGlobalSymbols
51
- this.stack = (new Error(this.message)).stack
52
- }
53
- MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
54
- }
55
-
56
- const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
57
-
58
- const isAsyncGenerator = func => func[Symbol.toStringTag] === 'AsyncGenerator'
59
- const isReadable = thing => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
60
-
61
- const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
62
-
63
- class AwaitableLRU {
64
- lru
65
- constructor(size) {
66
- this.lru = new LRU({ max: size })
67
- }
68
- get = key => Promise.resolve(this.lru.get(key))
69
- set = (key, value) => Promise.resolve(this.lru.set(key, value))
70
- }
71
-
72
- const CACHE_FUNCTIONS = {
73
- redis: client => () => {
74
- if (typeof client.cache !== 'undefined') return client.cache
75
- const redisConfig = client.redisConfig || {}
76
- const enabled = redisConfig.enabled !== false
77
- if (!enabled) {
78
- logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
79
- client.cache = null
80
- return client.cache
81
- }
82
- const host = redisConfig.host || REDIS_HOST
83
- const port = redisConfig.port || REDIS_PORT
84
- const prefix = redisConfig.prefix || REDIS_PREFIX
85
- if (!client.id) {
86
- logger.warn(`redis: all nameless connections will share one cache`)
87
- client.cache = getRedis('~nameless~', host, port, prefix)
88
- } else {
89
- client.cache = getRedis(client.id, host, port, prefix)
90
- }
91
- return client.cache
92
- },
93
- scopedCache: client => (cacheName, size = 100) => {
94
- const redis = client.redis()
95
- return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
96
- },
97
- flush: client => async () => {
98
- const redis = client.redis()
99
- return redis ? redis.flush() : true
100
- }
101
- }
102
-
103
- // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
104
- const UTILITY_FUNCTIONS = {
105
- list: client => async (path, opts) => {
106
- const cache = client.scopedCache('list')
107
- let cached = cache ? await cache.get(path) : null
108
- if (cached) {
109
- if (Array.isArray(cached)) { return cached }
110
- throw cached
111
- }
112
- const recursive = opts && opts.recursive ? opts.recursive : false
113
- const visitor = opts && opts.visitor ? opts.visitor : null
114
- if (visitor && typeof visitor !== 'function') {
115
- throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
116
- }
117
- try {
118
- // noinspection JSUnresolvedFunction
119
- const results = await client.driver_list(path, recursive, visitor)
120
- if (cache) {
121
- cache.set(path, results).then(
122
- () => { logger.debug(`list(${path}) cached ${results ? results.length : `unknown? ${JSON.stringify(results)}`} results`) },
123
- (err) => { logger.error(`list(${path}) error: ${err}`) }
124
- )
125
- }
126
- return results
127
- } catch (e) {
128
- if (cache && e instanceof MobilettoNotFoundError) {
129
- cache.set(path, e).then(
130
- () => { logger.debug(`list(${path}) cached error ${e}`) },
131
- (err) => { logger.error(`list(${path}) error ${err} caching MobilettoNotFoundError`) }
132
- )
133
- }
134
- throw e
135
- }
136
- },
137
- safeList: client => async (path, opts) => {
138
- const recursive = opts && opts.recursive ? opts.recursive : false
139
- const visitor = opts && opts.visitor ? opts.visitor : null
140
- try {
141
- // noinspection JSUnresolvedFunction
142
- return await client.driver_list(path, recursive, visitor)
143
- } catch (e) {
144
- if (e instanceof MobilettoNotFoundError) {
145
- return []
146
- }
147
- throw e
148
- }
149
- },
150
- metadata: client => async (path) => {
151
- const cache = client.scopedCache('metadata')
152
- const cached = cache ? await cache.get(path) : null
153
- if (cached) {
154
- return cached
155
- }
156
- // noinspection JSUnresolvedFunction
157
- const meta = await client.driver_metadata(path);
158
- if (cache) {
159
- cache.set(path, meta).then(
160
- () => { logger.debug(`metadata(${path}) cached meta = ${JSON.stringify(meta)}`) },
161
- (err) => { logger.error(`metadata(${path}) error: ${err}`) }
162
- )
163
- }
164
- return meta
165
- },
166
- safeMetadata: client => async (path) => {
167
- try {
168
- return await client.metadata(path)
169
- } catch (e) {
170
- if (e instanceof MobilettoNotFoundError) {
171
- return null
172
- }
173
- throw e
174
- }
175
- },
176
- remove: client => async (path, opts) => {
177
- const recursive = opts && opts.recursive ? opts.recursive : false
178
- // noinspection JSUnresolvedVariable
179
- const quiet = opts && opts.quiet ? opts.quiet : false
180
- // noinspection JSUnresolvedFunction
181
- const result = await client.driver_remove(path, recursive, quiet)
182
- await client.flush()
183
- return result
184
- },
185
- readFile: client => async (path) => {
186
- const cache = client.scopedCache('readFile')
187
- const cached = cache ? await cache.get(path) : null
188
- if (cached) {
189
- return Buffer.from(cached, 'base64')
190
- }
191
- const chunks = []
192
- await client.read(path, reader(chunks))
193
- const data = Buffer.concat(chunks);
194
- if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
195
- const setVal = cache.set(path, data.toString('base64')) // redis returns promise, LRU does not
196
- if (setVal instanceof Promise) {
197
- setVal.then(
198
- () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
199
- (err) => { logger.error(`readFile(${path}) error: ${err}`) }
200
- )
201
- }
202
- }
203
- return data
204
- },
205
- safeReadFile: client => async (path) => {
206
- try {
207
- return await client.readFile(path)
208
- } catch (e) {
209
- logger.info(`safeReadFile(${path}) ${e}`)
210
- return null
211
- }
212
- },
213
- write: client => async (path, data) => {
214
- logger.debug(`util.write(${path}) starting ...`)
215
- const p = path.startsWith('/') ? path.substring(1) : path
216
- if (p !== path) {
217
- logger.debug(`util.write(${path}) removed leading ${this.delimiter}`)
218
- }
219
- // noinspection JSUnresolvedFunction
220
- const bytesWritten = await client.driver_write(p, data)
221
- await client.flush()
222
- logger.debug(`util.write(${p}) wrote ${bytesWritten} bytes`)
223
- return bytesWritten
224
- },
225
- writeFile: client => async (path, data) => {
226
- const readFunc = function* () { yield data }
227
- return await client.write(path, readFunc())
228
- },
229
- mirror: client => async (source, clientPath = '', sourcePath = '') => {
230
- logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
231
- const results = {
232
- success: 0,
233
- errors: 0
234
- }
235
- const visitor = async (obj) => {
236
- if (obj.type && obj.type === M_FILE) {
237
- logger.verbose(`mirror: mirroring file: ${obj.name}`)
238
- const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
239
- logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
240
- const destName = obj.name.startsWith(sourcePath)
241
- ? obj.name.substring(sourcePath.length)
242
- : obj.name
243
- const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
244
- (destName.startsWith('/') ? destName.substring(1) : destName)
245
- try {
246
- // if dest already exists and is the same size, don't copy it again
247
- let srcSize = null
248
- if (obj.size) {
249
- srcSize = obj.size
250
- } else {
251
- const srcMeta = source.safeMetadata(obj.name)
252
- if (srcMeta && srcMeta.size) {
253
- srcSize = srcMeta.size
254
- }
255
- }
256
- // only continue if we could determine the source size
257
- if (srcSize) {
258
- const destMeta = client.safeMetadata(destFullPath)
259
- if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
260
- logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
261
- return
262
- }
263
- }
264
-
265
- // write from source -> write to temp file
266
- const fd = fs.openSync(tempPath, 'wx', 0o0600)
267
- const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
268
- await source.read(obj.name, async (chunk) => {
269
- if (chunk) { writer.write(chunk) }
270
- }, () => {
271
- writer.close((err) => {
272
- if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
273
- logger.debug(`mirror: finished writing ${obj.name} to temp file ${tempPath}`)
274
- })
275
- }).then(async () => {
276
- // read from temp file -> write to mirror
277
- const fd = fs.openSync(tempPath, 'r')
278
- const reader = fs.createReadStream(tempPath, {fd})
279
- logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
280
- await client.write(destFullPath, reader)
281
- logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
282
- })
283
- results.success++
284
- } catch (e) {
285
- logger.warn(`mirror: error copying file: ${e}`)
286
- results.errors++
287
- } finally {
288
- logger.verbose(`mirror: file mirrored successfully: ${obj.name}`)
289
- fs.rmSync(tempPath, {force: true})
290
- }
291
- }
292
- }
293
- await mirrorDir(source, sourcePath, visitor)
294
- return results
295
- }
296
- }
297
-
298
- async function mirrorDir (source, sourcePath, visitor) {
299
- logger.verbose(`mirrorDir: mirroring dir: ${sourcePath}`)
300
- const listing = await source.list(sourcePath, {recursive: false, visitor})
301
- for (const obj of listing) {
302
- if (obj.type === M_DIR) {
303
- const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
304
- await mirrorDir(source, dir, visitor)
305
- }
306
- }
307
- }
308
-
309
- function utilityFunctionConflict (client, func) {
310
- if (typeof client[func] === 'function') {
311
- if (typeof client[`driver_${func}`] !== 'undefined') {
312
- logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
313
- return false
314
- } else {
315
- client[`driver_${func}`] = client[func] // save original driver function
316
- return true
317
- }
318
- } else if (typeof client[func] !== 'undefined') {
319
- throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
320
- }
321
- }
322
-
323
- const addUtilityFunctions = (client, readOnly = false) => {
324
- addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
325
- if (readOnly) {
326
- for (const writeFunc of ['write', 'remove', 'writeFile']) {
327
- client[writeFunc] = async () => {
328
- logger.warn(`${writeFunc} not supported in readOnly mode`)
329
- return false
330
- }
331
- }
332
- }
333
- return client
334
- }
335
-
336
- const addCacheFunctions = client => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
337
- logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
338
- return false
339
- })
340
-
341
- const addClientFunctions = (client, functions, conflictFunc) => {
342
- for (const func of Object.keys(functions)) {
343
- let add = true
344
- if (client[func]) {
345
- add = conflictFunc ? conflictFunc(client, func) : false
346
- }
347
- if (add) {
348
- client[func] = functions[func](client)
349
- }
350
- }
351
- return client
352
- }
353
-
354
- async function connect (driverPath, key, secret, opts, encryption = null) {
355
- return await mobiletto(driverPath, key, secret, opts, encryption)
3
+ const registerDrivers = () => {
4
+ registerDriver('local', require('mobiletto-driver-local'))
5
+ registerDriver('s3', require('mobiletto-driver-s3'))
6
+ registerDriver('b2', require('mobiletto-driver-b2'))
7
+ registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
356
8
  }
357
9
 
358
- async function mobiletto (driverPath, key, secret, opts, encryption = null) {
359
- logger.info(`mobiletto: connecting with driver ${driverPath}`)
360
- const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
361
- const client = driver.storageClient(key, secret, opts)
362
- const configValue = await client.testConfig();
363
- if (!configValue) {
364
- const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
365
- logger.error(message)
366
- throw new MobilettoError(message)
367
- }
368
- const readOnly = opts ? !!opts.readOnly : false
369
- client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
370
-
371
- const internalIdForDriver = () =>
372
- driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
373
-
374
- // If the driver didn't give the client a name, generate a unique internal name
375
- if (!client.id) {
376
- client.id = internalIdForDriver()
377
- }
378
- if (encryption === null) {
379
- logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
380
- return addUtilityFunctions(addCacheFunctions(client), readOnly)
381
- }
382
- const encKey = normalizeKey(encryption.key)
383
- if (!encKey) {
384
- const message = `mobiletto(${driverPath}) invalid encryption key`
385
- logger.error(message)
386
- throw new MobilettoError(message)
387
- }
388
- const iv = normalizeIV(encryption.iv, encKey)
389
- if (!iv) {
390
- const message = `mobiletto(${driverPath}) invalid encryption IV`
391
- logger.error(message)
392
- throw new MobilettoError(message)
393
- }
394
- const enc = {
395
- key: encKey,
396
- iv,
397
- algo: encryption.algo || DEFAULT_CRYPT_ALGO,
398
- dirLevels: encryption.dirLevels || 4,
399
- encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
400
- }
401
- function encryptPath (path) {
402
- const encrypted = shasum(enc.key + ' ' + path)
403
- let newPath = ''
404
- for (let i = 0; i <= enc.dirLevels; i++) {
405
- if (newPath.length > 0) newPath += '/'
406
- newPath += encrypted.substring(i*2, (i*2)+2)
407
- }
408
- return newPath + encrypted
409
- }
410
- const metaPath = path => encryptPath(path + ' ~ META')
411
- const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
412
- const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
413
-
414
- const outerClient = addCacheFunctions(client)
415
- const _metadata = client => async (path) => {
416
- const cache = outerClient.scopedCache('metadata')
417
- const cached = cache ? await cache.get(path) : null
418
- if (cached) {
419
- return cached
420
- }
421
- let metaObj
422
- try {
423
- let chunks = []
424
- await client.read(metaPath(path), reader(chunks))
425
- metaObj = JSON.parse(decrypt(Buffer.concat(chunks).toString(), enc))
426
- } catch (e) {
427
- metaObj = {}
428
- }
429
- let meta
430
- try {
431
- meta = await client.metadata(encryptPath(path))
432
- } catch (e) {
433
- if (e instanceof MobilettoNotFoundError) {
434
- const dd = direntDir(path);
435
- try {
436
- meta = await client.metadata(dd)
437
- } catch (err) {
438
- if (err instanceof MobilettoNotFoundError) {
439
- const contents = await client.list(dd)
440
- if (Array.isArray(contents) && contents.length > 0) {
441
- return { name: path, type: M_DIR }
442
- } else {
443
- throw err
444
- }
445
- } else {
446
- throw err
447
- }
448
- }
449
- } else {
450
- throw e
451
- }
452
- }
453
- meta.type ||= M_FILE
454
- meta.name = path // rewrite name back to plaintext name
455
- const finalMeta = Object.assign({}, meta, metaObj);
456
- if (cache) {
457
- cache.set(path, finalMeta).then(
458
- () => { logger.debug(`_metadata(${path}) cached meta = ${JSON.stringify(finalMeta)}`) },
459
- (err) => { logger.error(`_metadata(${path}) error: ${err}`) }
460
- )
461
- }
462
- return finalMeta
463
- }
464
-
465
- const _singleMeta = async (job) => {
466
- const dirent = job.data.dirent
467
- const entry = job.data.entry
468
- const logPrefix = `_singleMeta(${dirent}/${basename(entry.name)})`
469
- return new Promise((resolve, reject) => {
470
- const cipherText = []
471
- client.read(dirent + '/' + basename(entry.name), reader(cipherText))
472
- .then((bytesRead) => {
473
- if (!bytesRead) {
474
- logger.warn(`${logPrefix} returned no data`)
475
- resolve({})
476
- } else {
477
- const plain = decrypt(cipherText.toString(), enc)
478
- const realPath = plain.split(ENC_PAD_SEP)[0]
479
- _metadata(client)(realPath)
480
- .then(meta => resolve(meta))
481
- .catch((err) => {
482
- const message = `${logPrefix} error fetching _metadata: ${err}`
483
- logger.warn(message)
484
- reject(message)
485
- })
486
- }
487
- })
488
- .catch((err) => {
489
- const message = `${logPrefix} error reading file: ${err}`
490
- logger.warn(message)
491
- reject(message)
492
- })
493
- })
494
- }
495
-
496
- const META_LOAD_QUEUE_NAME = '_loadMetaQueue'
497
- const META_LOAD_JOB_NAME = '_loadMetaJob'
498
- const META_LOAD_CONCURRENCY = process.env.MOBILETTO_META_LOAD_CONCURRENCY || 10
499
- let META_LOAD_QUEUE = null
500
- const metaLoadQueue = () => {
501
- if (META_LOAD_QUEUE === null) {
502
- if (!client.redisConfig) {
503
- const message = 'metaLoadQueue: redis is required but not enabled'
504
- logger.error(message)
505
- throw new MobilettoError(message)
506
- }
507
- META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, `redis://${client.redisConfig.host || REDIS_HOST}:${client.redisConfig.port || REDIS_PORT}`)
508
- META_LOAD_QUEUE.mobilettoHandlers = {}
509
- META_LOAD_QUEUE.mobilettoErrorHandlers = {}
510
- META_LOAD_QUEUE.process(META_LOAD_JOB_NAME, META_LOAD_CONCURRENCY, _singleMeta)
511
- META_LOAD_QUEUE.on('completed', function (job, result) {
512
- logger.info(`${META_LOAD_JOB_NAME} completed with result: ${JSON.stringify(result)}`)
513
- if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID]) {
514
- META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID](result)
515
- }
516
- })
517
- META_LOAD_QUEUE.on('failed', function (job, result) {
518
- logger.info(`${META_LOAD_JOB_NAME} failed with result: ${JSON.stringify(result)}`)
519
- if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID]) {
520
- META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID](result)
521
- }
522
- })
523
- }
524
- return META_LOAD_QUEUE
525
- }
526
-
527
- const _loadMeta = async (dirent, entries) => {
528
- const files = []
529
-
530
- const waitForFiles = (resolve) => {
531
- if (files.length === entries.length) {
532
- resolve(files)
533
- } else {
534
- setTimeout(() => waitForFiles(resolve), 1000)
535
- }
536
- }
537
-
538
- const mobilettoJobID = randomstring.generate(10)
539
- const mq = metaLoadQueue()
540
- mq.mobilettoHandlers[mobilettoJobID] = meta => files.push(meta)
541
- mq.mobilettoErrorHandlers[mobilettoJobID] = err => files.push(err)
542
- for (const entry of entries) {
543
- const job = { mobilettoJobID, dirent, entry }
544
- mq.add(META_LOAD_JOB_NAME, job)
545
- }
546
- await new Promise(resolve => waitForFiles(resolve))
547
- delete mq.mobilettoHandlers[mobilettoJobID]
548
- delete mq.mobilettoErrorHandlers[mobilettoJobID]
549
- return files
550
- }
551
-
552
- async function removeDirentFile(path) {
553
- const df = direntFile(direntDir(dirname(path)), path);
554
- const recursive = false
555
- const quiet = true
556
-
557
- logger.debug(`removeDirentFile(${path}) removing df=${df}`)
558
- await client.remove(df, recursive, quiet)
559
-
560
- logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
561
- await client.remove(encryptPath(path), recursive, quiet)
562
-
563
- logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
564
- await client.remove(metaPath(path), recursive, quiet)
565
- }
566
-
567
- function stringGenerator (value, enc) {
568
- return function* () {
569
- yield encrypt(value, enc)
570
- }
571
- }
572
-
573
- // noinspection JSUnusedGlobalSymbols
574
- const encClient = {
575
- id: internalIdForDriver(),
576
- redisConfig: client.redisConfig,
577
- list: async (pth = '', recursive, visitor) => {
578
- const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
579
- const dirent = direntDir(p)
580
- let entries
581
-
582
- const cacheKey = `${p} ~ ${recursive}`
583
- const cache = visitor ? null : client.scopedCache('enc_list')
584
- const cached = cache && await cache.get(cacheKey)
585
-
586
- function cacheAndReturn (thing) {
587
- if (cache) {
588
- cache.set(cacheKey, thing).then(
589
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
590
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
591
- )
592
- }
593
- return thing
594
- }
595
-
596
- async function tryParentDirForSingleFile (p, visitor, e) {
597
- // it might be a single file, try listing the parent dir
598
- const parentDirent = direntDir(dirname(p))
599
- entries = await client.list(parentDirent)
600
- const objects = await _loadMeta(parentDirent, entries)
601
- const found = objects.find(o => o.name === p)
602
- if (found) {
603
- if (visitor) {
604
- await visitor(found)
605
- }
606
- logger.debug(`tryParentDirForSingleFile(${p}) found ${found.name}`)
607
- return cacheAndReturn([found])
608
- }
609
- logger.debug(`tryParentDirForSingleFile(${p}) nothing found! e=${e}`)
610
- throw e === null ? new MobilettoNotFoundError(p) : e
611
- }
612
-
613
- if (cached) {
614
- entries = cached
615
- } else {
616
- try {
617
- entries = await _loadMeta(dirent, await client.list(dirent))
618
- } catch (e) {
619
- if (e instanceof MobilettoNotFoundError) {
620
- if (p.includes('/')) {
621
- return await tryParentDirForSingleFile(p, visitor, e)
622
- }
623
- throw e
624
- } else {
625
- throw new MobilettoError(`encClient.list(${p}) ${e}`, e)
626
- }
627
- }
628
- if (recursive) {
629
- const dirs = entries.filter(obj => obj.type === M_DIR)
630
- while (dirs.length > 0) {
631
- const dir = dirs.shift()
632
- const subdir = direntDir(dir.name)
633
- const subdirListing = await client.list(subdir)
634
- if (subdirListing && subdirListing.length > 0) {
635
- const subdirEntries = await _loadMeta(subdir, subdirListing)
636
- if (visitor) {
637
- for (const obj of subdirEntries) {
638
- await visitor(obj)
639
- }
640
- }
641
- entries.push(...subdirEntries)
642
- const moreDirs = subdirEntries.filter(obj => obj.type === M_DIR)
643
- dirs.unshift(...moreDirs)
644
- }
645
- }
646
- if (cache) {
647
- cache.set(cacheKey, entries).then(
648
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
649
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
650
- )
651
- }
652
- if (!entries || entries.length === 0) {
653
- return []
654
- }
655
- }
656
- }
657
- if (entries.length === 0 && p.includes('/')) {
658
- return await tryParentDirForSingleFile(p, visitor, null)
659
- }
660
- if (cache) {
661
- cache.set(cacheKey, entries).then(
662
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
663
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
664
- )
665
- }
666
- if (!entries || entries.length === 0) {
667
- return []
668
- }
669
- if (visitor) {
670
- for (const obj of entries) {
671
- await visitor(obj)
672
- }
673
- }
674
- return entries
675
- },
676
- metadata: _metadata(client),
677
- read: async (path, callback) => {
678
- const realPath = encryptPath(path)
679
- const cipher = crypt.getDecipher(enc)
680
- return await client.read(realPath,
681
- (chunk) => {
682
- return callback(cipher.update(chunk))
683
- }, () => {
684
- callback(cipher.final())
685
- })
686
- },
687
- write: async (path, readFunc) => {
688
- // if encrypting paths, write dirent file(s) for all parent directories
689
- let p = path
690
- while (true) {
691
- const direntGenerator = stringGenerator(p + enc.encPathPadding(), enc)
692
- const dir = direntDir(dirname(p))
693
- const df = direntFile(dir, p);
694
- if (!(await client.write(df, direntGenerator()))) {
695
- throw new MobilettoError('write: error writing dirent file')
696
- }
697
- p = dirname(p)
698
- if (p === '.' || p === '') {
699
- break
700
- }
701
- }
702
-
703
- const cipher = crypt.getCipher(enc)
704
- const realPath = encryptPath(path)
705
- let generatorBytes = 0
706
- const createEncryptStream = (input) =>
707
- input.pipe(new Transform({
708
- transform(chunk, encoding, callback) {
709
- generatorBytes += chunk.length
710
- this.push(cipher.update(chunk))
711
- callback()
712
- },
713
- flush(callback) {
714
- this.push(cipher.final())
715
- callback()
716
- }
717
- }))
718
-
719
- // If we were passed a Readable, call write with a transform method
720
- if (isReadable(readFunc)) {
721
- await client.write(realPath, createEncryptStream(readFunc))
722
- } else {
723
- // If we were passed a generator, wrap it with another generator that encrypts
724
- function* cryptGenerator(plaintextGenerator) {
725
- let chunk = plaintextGenerator.next().value
726
- while (chunk) {
727
- generatorBytes += chunk.length
728
- yield cipher.update(chunk)
729
- chunk = plaintextGenerator.next().value
730
- }
731
- yield cipher.final()
732
- }
733
- await client.write(realPath, cryptGenerator(readFunc))
734
- }
735
- // write metadata
736
- const meta = { name: path, size: generatorBytes, type: M_FILE }
737
- await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
738
- return generatorBytes
739
- },
740
-
741
- remove: async (path, options) => {
742
- logger.debug(`enc.remove(${path}) starting`)
743
- const recursive = options === true || (options && options.recursive) || false
744
- // const quiet = (options && options.quiet) || false
745
- if (recursive) {
746
- // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
747
- const removed = []
748
- async function recRm (path) {
749
- const dirent = direntDir(path)
750
- let entries
751
- try {
752
- entries = await client.list(dirent)
753
- } catch (e) {
754
- if (!(e instanceof MobilettoNotFoundError)) {
755
- throw e instanceof MobilettoError
756
- ? e
757
- : new MobilettoError('error listing files for recursive deletion', e)
758
- }
759
- }
760
- if (entries && entries.length > 0) {
761
- const files = await _loadMeta(dirent, entries)
762
- for (const f of files) {
763
- await recRm(f.name)
764
- }
765
- }
766
- await removeDirentFile(path)
767
- removed.push(path)
768
- }
769
- await recRm(path)
770
- return removed
771
- }
772
-
773
- // remove the single file/dir
774
- await removeDirentFile(path)
775
-
776
- // if we were the last dirent file, then also remove dirent directory, and recursively upwards
777
- let parent = path
778
- let dirent = direntDir(parent)
779
- const df = direntFile(dirent, path)
780
- await client.remove(df, false, true)
781
- while (true) {
782
- try {
783
- const entries = await client.list(dirent)
784
- if (entries.length === 0) {
785
- await removeDirentFile(parent)
786
- }
787
- } catch (e) {
788
- if (!(e instanceof MobilettoNotFoundError)) {
789
- throw e
790
- }
791
- }
792
- if (parent === '.' || parent === '/') {
793
- // do not remove dirent for root dir
794
- break
795
- }
796
- parent = dirname(parent)
797
- dirent = direntDir(parent)
798
- }
799
- return path
800
- }
801
- }
802
- logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
803
- return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
804
- }
805
-
806
- async function readStream(stream, callback, endCallback) {
807
- const counter = {count: 0}
808
- const streamHandler = stream =>
809
- new Promise((resolve, reject) => {
810
- stream.on('data', (data) => {
811
- counter.count += data ? data.length : 0
812
- callback(data)
813
- })
814
- stream.on('error', reject)
815
- stream.on('end', () => {
816
- if (endCallback) {
817
- endCallback()
818
- }
819
- resolve()
820
- })
821
- })
822
- await streamHandler(stream)
823
- return counter.count
824
- }
825
-
826
- function writeStream (stream) {
827
- return (chunk) => {
828
- if (chunk) {
829
- stream.write(chunk, (err) => {
830
- if (err) {
831
- logger.error(`writeStream: error writing: ${err}`)
832
- throw err
833
- }
834
- })
835
- }
836
- }
837
- }
838
-
839
- function closeStream (stream) {
840
- return () => stream.close((err) => {
841
- if (err) {
842
- logger.error(`closeStream: error closing: ${err}`)
843
- throw err
844
- }
845
- })
846
- }
847
-
848
- const M_FILE = 'file'
849
- const M_DIR = 'dir'
850
- const M_LINK = 'link'
851
- const M_SPECIAL = 'special'
852
-
853
- const setLogLevel = level => setLoggerLevel(level)
854
-
855
- const closeRedis = async () => { await teardown() }
856
-
857
- module.exports = {
858
- M_FILE, M_DIR, M_LINK, M_SPECIAL,
859
- isAsyncGenerator, isReadable,
860
- mobiletto, connect,
861
- MobilettoError, MobilettoNotFoundError,
862
- setLogLevel,
863
- readStream, writeStream, closeStream,
864
- closeRedis
865
- }
10
+ module.exports = { registerDrivers }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.1.40",
3
+ "version": "1.2.0",
4
4
  "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
@@ -26,25 +26,20 @@
26
26
  "test": "mocha --timeout 60000 ./test/*.spec.js"
27
27
  },
28
28
  "devDependencies": {
29
- "@babel/core": "^7.18.10",
30
- "@babel/node": "^7.18.10",
31
- "@babel/preset-env": "^7.18.10",
32
- "chai": "^4.3.6",
33
- "dotenv": "^16.0.1",
34
- "mocha": "^10.0.0",
35
- "randomstring": "^1.2.2"
29
+ "@babel/core": "^7.22.5",
30
+ "@babel/node": "^7.22.5",
31
+ "@babel/preset-env": "^7.22.5",
32
+ "chai": "^4.3.7",
33
+ "dotenv": "^16.3.1",
34
+ "mocha": "^10.2.0",
35
+ "randomstring": "^1.3.0"
36
36
  },
37
37
  "dependencies": {
38
- "@aws-sdk/client-s3": "^3.150.0",
39
- "@aws-sdk/lib-storage": "^3.150.0",
40
- "@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
41
- "backblaze-b2": "^1.7.0",
42
- "bull": "^4.9.0",
43
- "ioredis": "^5.2.3",
44
- "lru-cache": "^7.14.0",
45
- "node-fetch-commonjs": "^3.1.1",
46
- "node-redis-scan": "^1.3.5",
47
- "shasum": "^1.0.2",
48
- "winston": "^3.8.1"
38
+ "mobiletto-base": "^1.0.13",
39
+ "mobiletto-common": "^1.0.2",
40
+ "mobiletto-driver-b2": "^1.0.6",
41
+ "mobiletto-driver-local": "^1.0.3",
42
+ "mobiletto-driver-s3": "^1.0.5",
43
+ "mobiletto-driver-indexeddb": "^1.0.1"
49
44
  }
50
45
  }
package/test/api.spec.js CHANGED
@@ -3,25 +3,27 @@ require('dotenv').config()
3
3
 
4
4
  const fs = require('fs')
5
5
  const { basename, dirname } = require('path')
6
-
7
6
  const randomstring = require('randomstring')
8
7
  const crypto = require('crypto')
9
- const redis = require('../util/redis')
10
-
11
- const { expect, should, assert } = require('chai')
12
8
 
13
9
  const {
14
- M_DIR, M_FILE,
15
- mobiletto, connect, MobilettoNotFoundError, closeRedis
16
- } = require("../index")
10
+ M_DIR, M_FILE, MobilettoNotFoundError, logger
11
+ } = require('mobiletto-common')
17
12
 
18
- const { logger } = require("../util/logger")
13
+ const { connect, mobiletto, redis, closeRedis } = require('mobiletto-base')
14
+
15
+ const { registerDrivers } = require("../index")
16
+ registerDrivers()
17
+
18
+ const { expect, should, assert } = require('chai')
19
19
 
20
20
  // chunk size used by generator function, used by driver's 'write' function
21
21
  // the temp file is also TEMP_SZ_MULTIPLE of this number
22
22
  const READ_SZ = 8 * 1024 // xfer data in 8k chunks
23
23
  const TEMP_SZ_MULTIPLE = 3 // temp file will be ~24k (READ_SZ * 3)
24
24
 
25
+ const SINGLE_DRIVER = process.env.SINGLE_DRIVER || null
26
+
25
27
  DRIVER_CONFIG = {
26
28
  local: {
27
29
  key: process.env.LOCAL_STORAGE || '/tmp'
@@ -130,7 +132,7 @@ if (!redisSetup.enabled) continue
130
132
 
131
133
  for (const driverName of DRIVER_NAMES) {
132
134
  // For testing single drivers
133
- // if (driverName !== 'local') continue
135
+ if (SINGLE_DRIVER != null && SINGLE_DRIVER !== driverName) continue
134
136
 
135
137
  const driverTest = `${driverName} [${redisSetup.name}]`
136
138
  const config = redisSetup.config(DRIVER_CONFIG[driverName])
@@ -205,6 +207,10 @@ describe(`${driverTest} test`, () => {
205
207
 
206
208
  for (const encryption of encryptionTests()) {
207
209
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
210
+ if (encryption && redisSetup.name !== 'redis-enabled' && driverName !== 'local') {
211
+ // encryption without redis is simply too slow, except for local filesystems
212
+ continue
213
+ }
208
214
  describe(`${driverTest} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
209
215
  // describe(`${driverTest} - ENC fail to write and delete files in readOnly mode`, () => {
210
216
  // const encryption = {key: rand(32)}
@@ -288,6 +294,10 @@ describe(`${driverTest} test`, () => {
288
294
 
289
295
  for (const encryption of encryptionTests()) {
290
296
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
297
+ if (encryption && redisSetup.name !== 'redis-enabled') {
298
+ // encryption without redis is simply too slow
299
+ continue
300
+ }
291
301
  const mirrorDriver = pickAnotherDriver(driverName)
292
302
  // const mirrorDriver = 'local' // for hard-coding the mirror driver
293
303
  const mirrorConfig = DRIVER_CONFIG[mirrorDriver]
@@ -10,7 +10,7 @@ const {
10
10
  getCipher, getDecipher
11
11
  } = require("../util/crypt")
12
12
 
13
- const { readStream } = require("../index")
13
+ const { readStream } = require('mobiletto-common')
14
14
 
15
15
  describe('crypto test', () => {
16
16
  const enc = {