mobiletto 1.1.40 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,865 +1,29 @@
1
- const fs = require('fs')
2
- const { Readable, Transform } = require('stream')
3
- const Queue = require('bull')
4
- const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
5
- const { getRedis, teardown } = require('./util/redis')
6
- const { basename, dirname } = require('path')
7
- const shasum = require('shasum')
8
- const randomstring = require('randomstring')
1
+ const {
2
+ registerDriver, mobiletto, connect,
3
+ redis, closeRedis
4
+ } = require('mobiletto-base')
9
5
 
10
- const crypt = require('./util/crypt')
11
- const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
- const LRU = require("lru-cache")
13
-
14
- const DIR_ENT_DIR_SUFFIX = '__.dirent'
15
- const DIR_ENT_FILE_PREFIX = 'dirent__'
16
- const ENC_PAD_SEP = ' ~ '
17
-
18
- const initMobilettoTempDir = () => {
19
- const dir = process.env.MOBILETTO_TMP || process.env.TMPDIR || '/tmp'
20
- return dir.endsWith('/') ? dir.substring(0, dir.length - 1) : dir
21
- }
22
- const MOBILETTO_TMP = initMobilettoTempDir()
23
-
24
- const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
25
- const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
26
- const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
27
-
28
- // adapted from https://stackoverflow.com/a/27724419
29
- function MobilettoError (message, err) {
30
- this.message = `${message}: ${err ? err : ''}`
31
- // noinspection JSUnusedGlobalSymbols
32
- this.err = err
33
- // Use V8's native method if available, otherwise fallback
34
- if ('captureStackTrace' in Error) {
35
- Error.captureStackTrace(this, TypeError)
36
- } else {
37
- // noinspection JSUnusedGlobalSymbols
38
- this.stack = (new Error(this.message)).stack
39
- }
40
- MobilettoError.prototype.toString = () => JSON.stringify(this)
41
- }
42
-
43
- // adapted from https://stackoverflow.com/a/27724419
44
- function MobilettoNotFoundError (message) {
45
- this.message = `${message}`
46
- // Use V8's native method if available, otherwise fallback
47
- if ('captureStackTrace' in Error) {
48
- Error.captureStackTrace(this, TypeError)
49
- } else {
50
- // noinspection JSUnusedGlobalSymbols
51
- this.stack = (new Error(this.message)).stack
52
- }
53
- MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
54
- }
55
-
56
- const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
57
-
58
- const isAsyncGenerator = func => func[Symbol.toStringTag] === 'AsyncGenerator'
59
- const isReadable = thing => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
60
-
61
- const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
62
-
63
- class AwaitableLRU {
64
- lru
65
- constructor(size) {
66
- this.lru = new LRU({ max: size })
67
- }
68
- get = key => Promise.resolve(this.lru.get(key))
69
- set = (key, value) => Promise.resolve(this.lru.set(key, value))
70
- }
71
-
72
- const CACHE_FUNCTIONS = {
73
- redis: client => () => {
74
- if (typeof client.cache !== 'undefined') return client.cache
75
- const redisConfig = client.redisConfig || {}
76
- const enabled = redisConfig.enabled !== false
77
- if (!enabled) {
78
- logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
79
- client.cache = null
80
- return client.cache
81
- }
82
- const host = redisConfig.host || REDIS_HOST
83
- const port = redisConfig.port || REDIS_PORT
84
- const prefix = redisConfig.prefix || REDIS_PREFIX
85
- if (!client.id) {
86
- logger.warn(`redis: all nameless connections will share one cache`)
87
- client.cache = getRedis('~nameless~', host, port, prefix)
88
- } else {
89
- client.cache = getRedis(client.id, host, port, prefix)
90
- }
91
- return client.cache
92
- },
93
- scopedCache: client => (cacheName, size = 100) => {
94
- const redis = client.redis()
95
- return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
96
- },
97
- flush: client => async () => {
98
- const redis = client.redis()
99
- return redis ? redis.flush() : true
100
- }
101
- }
102
-
103
- // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
104
- const UTILITY_FUNCTIONS = {
105
- list: client => async (path, opts) => {
106
- const cache = client.scopedCache('list')
107
- let cached = cache ? await cache.get(path) : null
108
- if (cached) {
109
- if (Array.isArray(cached)) { return cached }
110
- throw cached
111
- }
112
- const recursive = opts && opts.recursive ? opts.recursive : false
113
- const visitor = opts && opts.visitor ? opts.visitor : null
114
- if (visitor && typeof visitor !== 'function') {
115
- throw new MobilettoError(`list: visitor is not a function: ${typeof visitor}`)
116
- }
117
- try {
118
- // noinspection JSUnresolvedFunction
119
- const results = await client.driver_list(path, recursive, visitor)
120
- if (cache) {
121
- cache.set(path, results).then(
122
- () => { logger.debug(`list(${path}) cached ${results ? results.length : `unknown? ${JSON.stringify(results)}`} results`) },
123
- (err) => { logger.error(`list(${path}) error: ${err}`) }
124
- )
125
- }
126
- return results
127
- } catch (e) {
128
- if (cache && e instanceof MobilettoNotFoundError) {
129
- cache.set(path, e).then(
130
- () => { logger.debug(`list(${path}) cached error ${e}`) },
131
- (err) => { logger.error(`list(${path}) error ${err} caching MobilettoNotFoundError`) }
132
- )
133
- }
134
- throw e
135
- }
136
- },
137
- safeList: client => async (path, opts) => {
138
- const recursive = opts && opts.recursive ? opts.recursive : false
139
- const visitor = opts && opts.visitor ? opts.visitor : null
140
- try {
141
- // noinspection JSUnresolvedFunction
142
- return await client.driver_list(path, recursive, visitor)
143
- } catch (e) {
144
- if (e instanceof MobilettoNotFoundError) {
145
- return []
146
- }
147
- throw e
148
- }
149
- },
150
- metadata: client => async (path) => {
151
- const cache = client.scopedCache('metadata')
152
- const cached = cache ? await cache.get(path) : null
153
- if (cached) {
154
- return cached
155
- }
156
- // noinspection JSUnresolvedFunction
157
- const meta = await client.driver_metadata(path);
158
- if (cache) {
159
- cache.set(path, meta).then(
160
- () => { logger.debug(`metadata(${path}) cached meta = ${JSON.stringify(meta)}`) },
161
- (err) => { logger.error(`metadata(${path}) error: ${err}`) }
162
- )
163
- }
164
- return meta
165
- },
166
- safeMetadata: client => async (path) => {
167
- try {
168
- return await client.metadata(path)
169
- } catch (e) {
170
- if (e instanceof MobilettoNotFoundError) {
171
- return null
172
- }
173
- throw e
174
- }
175
- },
176
- remove: client => async (path, opts) => {
177
- const recursive = opts && opts.recursive ? opts.recursive : false
178
- // noinspection JSUnresolvedVariable
179
- const quiet = opts && opts.quiet ? opts.quiet : false
180
- // noinspection JSUnresolvedFunction
181
- const result = await client.driver_remove(path, recursive, quiet)
182
- await client.flush()
183
- return result
184
- },
185
- readFile: client => async (path) => {
186
- const cache = client.scopedCache('readFile')
187
- const cached = cache ? await cache.get(path) : null
188
- if (cached) {
189
- return Buffer.from(cached, 'base64')
190
- }
191
- const chunks = []
192
- await client.read(path, reader(chunks))
193
- const data = Buffer.concat(chunks);
194
- if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
195
- const setVal = cache.set(path, data.toString('base64')) // redis returns promise, LRU does not
196
- if (setVal instanceof Promise) {
197
- setVal.then(
198
- () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
199
- (err) => { logger.error(`readFile(${path}) error: ${err}`) }
200
- )
201
- }
202
- }
203
- return data
204
- },
205
- safeReadFile: client => async (path) => {
206
- try {
207
- return await client.readFile(path)
208
- } catch (e) {
209
- logger.info(`safeReadFile(${path}) ${e}`)
210
- return null
211
- }
212
- },
213
- write: client => async (path, data) => {
214
- logger.debug(`util.write(${path}) starting ...`)
215
- const p = path.startsWith('/') ? path.substring(1) : path
216
- if (p !== path) {
217
- logger.debug(`util.write(${path}) removed leading ${this.delimiter}`)
218
- }
219
- // noinspection JSUnresolvedFunction
220
- const bytesWritten = await client.driver_write(p, data)
221
- await client.flush()
222
- logger.debug(`util.write(${p}) wrote ${bytesWritten} bytes`)
223
- return bytesWritten
224
- },
225
- writeFile: client => async (path, data) => {
226
- const readFunc = function* () { yield data }
227
- return await client.write(path, readFunc())
228
- },
229
- mirror: client => async (source, clientPath = '', sourcePath = '') => {
230
- logger.info(`mirror: starting, sourcePath=${sourcePath} -> clientPath=${clientPath}`)
231
- const results = {
232
- success: 0,
233
- errors: 0
234
- }
235
- const visitor = async (obj) => {
236
- if (obj.type && obj.type === M_FILE) {
237
- logger.verbose(`mirror: mirroring file: ${obj.name}`)
238
- const tempPath = `${MOBILETTO_TMP}/mobiletto_${shasum(JSON.stringify(obj))}.${randomstring.generate(10)}`
239
- logger.debug(`mirror: writing ${obj.name} to temp file ${tempPath} ...`)
240
- const destName = obj.name.startsWith(sourcePath)
241
- ? obj.name.substring(sourcePath.length)
242
- : obj.name
243
- const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
244
- (destName.startsWith('/') ? destName.substring(1) : destName)
245
- try {
246
- // if dest already exists and is the same size, don't copy it again
247
- let srcSize = null
248
- if (obj.size) {
249
- srcSize = obj.size
250
- } else {
251
- const srcMeta = source.safeMetadata(obj.name)
252
- if (srcMeta && srcMeta.size) {
253
- srcSize = srcMeta.size
254
- }
255
- }
256
- // only continue if we could determine the source size
257
- if (srcSize) {
258
- const destMeta = client.safeMetadata(destFullPath)
259
- if (destMeta && destMeta.size && destMeta.size && destMeta.size === srcSize) {
260
- logger.info(`mirror: dest object (${destFullPath}) has same size (${srcSize}) as src object ${sourcePath}, not copying`)
261
- return
262
- }
263
- }
264
-
265
- // write from source -> write to temp file
266
- const fd = fs.openSync(tempPath, 'wx', 0o0600)
267
- const writer = fs.createWriteStream(tempPath, {fd, flags: 'wx'})
268
- await source.read(obj.name, async (chunk) => {
269
- if (chunk) { writer.write(chunk) }
270
- }, () => {
271
- writer.close((err) => {
272
- if (err) { throw new MobilettoError(`mirror: error closing temp file: ${err}`) }
273
- logger.debug(`mirror: finished writing ${obj.name} to temp file ${tempPath}`)
274
- })
275
- }).then(async () => {
276
- // read from temp file -> write to mirror
277
- const fd = fs.openSync(tempPath, 'r')
278
- const reader = fs.createReadStream(tempPath, {fd})
279
- logger.debug(`mirror: writing temp file ${tempPath} to destination: ${destFullPath}`)
280
- await client.write(destFullPath, reader)
281
- logger.debug(`mirror: finished writing temp file ${tempPath} to destination: ${destFullPath}`)
282
- })
283
- results.success++
284
- } catch (e) {
285
- logger.warn(`mirror: error copying file: ${e}`)
286
- results.errors++
287
- } finally {
288
- logger.verbose(`mirror: file mirrored successfully: ${obj.name}`)
289
- fs.rmSync(tempPath, {force: true})
290
- }
291
- }
292
- }
293
- await mirrorDir(source, sourcePath, visitor)
294
- return results
295
- }
296
- }
297
-
298
- async function mirrorDir (source, sourcePath, visitor) {
299
- logger.verbose(`mirrorDir: mirroring dir: ${sourcePath}`)
300
- const listing = await source.list(sourcePath, {recursive: false, visitor})
301
- for (const obj of listing) {
302
- if (obj.type === M_DIR) {
303
- const dir = obj.name.startsWith(sourcePath) ? obj.name : sourcePath + obj.name
304
- await mirrorDir(source, dir, visitor)
305
- }
306
- }
307
- }
308
-
309
- function utilityFunctionConflict (client, func) {
310
- if (typeof client[func] === 'function') {
311
- if (typeof client[`driver_${func}`] !== 'undefined') {
312
- logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
313
- return false
314
- } else {
315
- client[`driver_${func}`] = client[func] // save original driver function
316
- return true
317
- }
318
- } else if (typeof client[func] !== 'undefined') {
319
- throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
320
- }
321
- }
322
-
323
- const addUtilityFunctions = (client, readOnly = false) => {
324
- addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
325
- if (readOnly) {
326
- for (const writeFunc of ['write', 'remove', 'writeFile']) {
327
- client[writeFunc] = async () => {
328
- logger.warn(`${writeFunc} not supported in readOnly mode`)
329
- return false
330
- }
331
- }
332
- }
333
- return client
334
- }
335
-
336
- const addCacheFunctions = client => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
337
- logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
338
- return false
339
- })
340
-
341
- const addClientFunctions = (client, functions, conflictFunc) => {
342
- for (const func of Object.keys(functions)) {
343
- let add = true
344
- if (client[func]) {
345
- add = conflictFunc ? conflictFunc(client, func) : false
346
- }
347
- if (add) {
348
- client[func] = functions[func](client)
349
- }
350
- }
351
- return client
352
- }
353
-
354
- async function connect (driverPath, key, secret, opts, encryption = null) {
355
- return await mobiletto(driverPath, key, secret, opts, encryption)
356
- }
357
-
358
- async function mobiletto (driverPath, key, secret, opts, encryption = null) {
359
- logger.info(`mobiletto: connecting with driver ${driverPath}`)
360
- const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
361
- const client = driver.storageClient(key, secret, opts)
362
- const configValue = await client.testConfig();
363
- if (!configValue) {
364
- const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
365
- logger.error(message)
366
- throw new MobilettoError(message)
367
- }
368
- const readOnly = opts ? !!opts.readOnly : false
369
- client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
370
-
371
- const internalIdForDriver = () =>
372
- driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
373
-
374
- // If the driver didn't give the client a name, generate a unique internal name
375
- if (!client.id) {
376
- client.id = internalIdForDriver()
377
- }
378
- if (encryption === null) {
379
- logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
380
- return addUtilityFunctions(addCacheFunctions(client), readOnly)
381
- }
382
- const encKey = normalizeKey(encryption.key)
383
- if (!encKey) {
384
- const message = `mobiletto(${driverPath}) invalid encryption key`
385
- logger.error(message)
386
- throw new MobilettoError(message)
387
- }
388
- const iv = normalizeIV(encryption.iv, encKey)
389
- if (!iv) {
390
- const message = `mobiletto(${driverPath}) invalid encryption IV`
391
- logger.error(message)
392
- throw new MobilettoError(message)
393
- }
394
- const enc = {
395
- key: encKey,
396
- iv,
397
- algo: encryption.algo || DEFAULT_CRYPT_ALGO,
398
- dirLevels: encryption.dirLevels || 4,
399
- encPathPadding: () => ENC_PAD_SEP + randomstring.generate(1 + Math.floor(2*Math.random()))
400
- }
401
- function encryptPath (path) {
402
- const encrypted = shasum(enc.key + ' ' + path)
403
- let newPath = ''
404
- for (let i = 0; i <= enc.dirLevels; i++) {
405
- if (newPath.length > 0) newPath += '/'
406
- newPath += encrypted.substring(i*2, (i*2)+2)
407
- }
408
- return newPath + encrypted
409
- }
410
- const metaPath = path => encryptPath(path + ' ~ META')
411
- const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
412
- const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
413
-
414
- const outerClient = addCacheFunctions(client)
415
- const _metadata = client => async (path) => {
416
- const cache = outerClient.scopedCache('metadata')
417
- const cached = cache ? await cache.get(path) : null
418
- if (cached) {
419
- return cached
420
- }
421
- let metaObj
422
- try {
423
- let chunks = []
424
- await client.read(metaPath(path), reader(chunks))
425
- metaObj = JSON.parse(decrypt(Buffer.concat(chunks).toString(), enc))
426
- } catch (e) {
427
- metaObj = {}
428
- }
429
- let meta
430
- try {
431
- meta = await client.metadata(encryptPath(path))
432
- } catch (e) {
433
- if (e instanceof MobilettoNotFoundError) {
434
- const dd = direntDir(path);
435
- try {
436
- meta = await client.metadata(dd)
437
- } catch (err) {
438
- if (err instanceof MobilettoNotFoundError) {
439
- const contents = await client.list(dd)
440
- if (Array.isArray(contents) && contents.length > 0) {
441
- return { name: path, type: M_DIR }
442
- } else {
443
- throw err
444
- }
445
- } else {
446
- throw err
447
- }
448
- }
449
- } else {
450
- throw e
451
- }
452
- }
453
- meta.type ||= M_FILE
454
- meta.name = path // rewrite name back to plaintext name
455
- const finalMeta = Object.assign({}, meta, metaObj);
456
- if (cache) {
457
- cache.set(path, finalMeta).then(
458
- () => { logger.debug(`_metadata(${path}) cached meta = ${JSON.stringify(finalMeta)}`) },
459
- (err) => { logger.error(`_metadata(${path}) error: ${err}`) }
460
- )
461
- }
462
- return finalMeta
463
- }
464
-
465
- const _singleMeta = async (job) => {
466
- const dirent = job.data.dirent
467
- const entry = job.data.entry
468
- const logPrefix = `_singleMeta(${dirent}/${basename(entry.name)})`
469
- return new Promise((resolve, reject) => {
470
- const cipherText = []
471
- client.read(dirent + '/' + basename(entry.name), reader(cipherText))
472
- .then((bytesRead) => {
473
- if (!bytesRead) {
474
- logger.warn(`${logPrefix} returned no data`)
475
- resolve({})
476
- } else {
477
- const plain = decrypt(cipherText.toString(), enc)
478
- const realPath = plain.split(ENC_PAD_SEP)[0]
479
- _metadata(client)(realPath)
480
- .then(meta => resolve(meta))
481
- .catch((err) => {
482
- const message = `${logPrefix} error fetching _metadata: ${err}`
483
- logger.warn(message)
484
- reject(message)
485
- })
486
- }
487
- })
488
- .catch((err) => {
489
- const message = `${logPrefix} error reading file: ${err}`
490
- logger.warn(message)
491
- reject(message)
492
- })
493
- })
494
- }
495
-
496
- const META_LOAD_QUEUE_NAME = '_loadMetaQueue'
497
- const META_LOAD_JOB_NAME = '_loadMetaJob'
498
- const META_LOAD_CONCURRENCY = process.env.MOBILETTO_META_LOAD_CONCURRENCY || 10
499
- let META_LOAD_QUEUE = null
500
- const metaLoadQueue = () => {
501
- if (META_LOAD_QUEUE === null) {
502
- if (!client.redisConfig) {
503
- const message = 'metaLoadQueue: redis is required but not enabled'
504
- logger.error(message)
505
- throw new MobilettoError(message)
506
- }
507
- META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, `redis://${client.redisConfig.host || REDIS_HOST}:${client.redisConfig.port || REDIS_PORT}`)
508
- META_LOAD_QUEUE.mobilettoHandlers = {}
509
- META_LOAD_QUEUE.mobilettoErrorHandlers = {}
510
- META_LOAD_QUEUE.process(META_LOAD_JOB_NAME, META_LOAD_CONCURRENCY, _singleMeta)
511
- META_LOAD_QUEUE.on('completed', function (job, result) {
512
- logger.info(`${META_LOAD_JOB_NAME} completed with result: ${JSON.stringify(result)}`)
513
- if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID]) {
514
- META_LOAD_QUEUE.mobilettoHandlers[job.data.mobilettoJobID](result)
515
- }
516
- })
517
- META_LOAD_QUEUE.on('failed', function (job, result) {
518
- logger.info(`${META_LOAD_JOB_NAME} failed with result: ${JSON.stringify(result)}`)
519
- if (job.data.mobilettoJobID && META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID]) {
520
- META_LOAD_QUEUE.mobilettoErrorHandlers[job.data.mobilettoJobID](result)
521
- }
522
- })
523
- }
524
- return META_LOAD_QUEUE
525
- }
526
-
527
- const _loadMeta = async (dirent, entries) => {
528
- const files = []
529
-
530
- const waitForFiles = (resolve) => {
531
- if (files.length === entries.length) {
532
- resolve(files)
533
- } else {
534
- setTimeout(() => waitForFiles(resolve), 1000)
535
- }
536
- }
537
-
538
- const mobilettoJobID = randomstring.generate(10)
539
- const mq = metaLoadQueue()
540
- mq.mobilettoHandlers[mobilettoJobID] = meta => files.push(meta)
541
- mq.mobilettoErrorHandlers[mobilettoJobID] = err => files.push(err)
542
- for (const entry of entries) {
543
- const job = { mobilettoJobID, dirent, entry }
544
- mq.add(META_LOAD_JOB_NAME, job)
545
- }
546
- await new Promise(resolve => waitForFiles(resolve))
547
- delete mq.mobilettoHandlers[mobilettoJobID]
548
- delete mq.mobilettoErrorHandlers[mobilettoJobID]
549
- return files
550
- }
551
-
552
- async function removeDirentFile(path) {
553
- const df = direntFile(direntDir(dirname(path)), path);
554
- const recursive = false
555
- const quiet = true
556
-
557
- logger.debug(`removeDirentFile(${path}) removing df=${df}`)
558
- await client.remove(df, recursive, quiet)
559
-
560
- logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
561
- await client.remove(encryptPath(path), recursive, quiet)
562
-
563
- logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
564
- await client.remove(metaPath(path), recursive, quiet)
565
- }
566
-
567
- function stringGenerator (value, enc) {
568
- return function* () {
569
- yield encrypt(value, enc)
570
- }
571
- }
572
-
573
- // noinspection JSUnusedGlobalSymbols
574
- const encClient = {
575
- id: internalIdForDriver(),
576
- redisConfig: client.redisConfig,
577
- list: async (pth = '', recursive, visitor) => {
578
- const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
579
- const dirent = direntDir(p)
580
- let entries
581
-
582
- const cacheKey = `${p} ~ ${recursive}`
583
- const cache = visitor ? null : client.scopedCache('enc_list')
584
- const cached = cache && await cache.get(cacheKey)
585
-
586
- function cacheAndReturn (thing) {
587
- if (cache) {
588
- cache.set(cacheKey, thing).then(
589
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
590
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
591
- )
592
- }
593
- return thing
594
- }
595
-
596
- async function tryParentDirForSingleFile (p, visitor, e) {
597
- // it might be a single file, try listing the parent dir
598
- const parentDirent = direntDir(dirname(p))
599
- entries = await client.list(parentDirent)
600
- const objects = await _loadMeta(parentDirent, entries)
601
- const found = objects.find(o => o.name === p)
602
- if (found) {
603
- if (visitor) {
604
- await visitor(found)
605
- }
606
- logger.debug(`tryParentDirForSingleFile(${p}) found ${found.name}`)
607
- return cacheAndReturn([found])
608
- }
609
- logger.debug(`tryParentDirForSingleFile(${p}) nothing found! e=${e}`)
610
- throw e === null ? new MobilettoNotFoundError(p) : e
611
- }
612
-
613
- if (cached) {
614
- entries = cached
615
- } else {
616
- try {
617
- entries = await _loadMeta(dirent, await client.list(dirent))
618
- } catch (e) {
619
- if (e instanceof MobilettoNotFoundError) {
620
- if (p.includes('/')) {
621
- return await tryParentDirForSingleFile(p, visitor, e)
622
- }
623
- throw e
624
- } else {
625
- throw new MobilettoError(`encClient.list(${p}) ${e}`, e)
626
- }
627
- }
628
- if (recursive) {
629
- const dirs = entries.filter(obj => obj.type === M_DIR)
630
- while (dirs.length > 0) {
631
- const dir = dirs.shift()
632
- const subdir = direntDir(dir.name)
633
- const subdirListing = await client.list(subdir)
634
- if (subdirListing && subdirListing.length > 0) {
635
- const subdirEntries = await _loadMeta(subdir, subdirListing)
636
- if (visitor) {
637
- for (const obj of subdirEntries) {
638
- await visitor(obj)
639
- }
640
- }
641
- entries.push(...subdirEntries)
642
- const moreDirs = subdirEntries.filter(obj => obj.type === M_DIR)
643
- dirs.unshift(...moreDirs)
644
- }
645
- }
646
- if (cache) {
647
- cache.set(cacheKey, entries).then(
648
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
649
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
650
- )
651
- }
652
- if (!entries || entries.length === 0) {
653
- return []
654
- }
655
- }
656
- }
657
- if (entries.length === 0 && p.includes('/')) {
658
- return await tryParentDirForSingleFile(p, visitor, null)
659
- }
660
- if (cache) {
661
- cache.set(cacheKey, entries).then(
662
- () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
663
- (err) => { logger.error(`enc_list(${p}) error: ${err}`) }
664
- )
665
- }
666
- if (!entries || entries.length === 0) {
667
- return []
668
- }
669
- if (visitor) {
670
- for (const obj of entries) {
671
- await visitor(obj)
672
- }
673
- }
674
- return entries
675
- },
676
- metadata: _metadata(client),
677
- read: async (path, callback) => {
678
- const realPath = encryptPath(path)
679
- const cipher = crypt.getDecipher(enc)
680
- return await client.read(realPath,
681
- (chunk) => {
682
- return callback(cipher.update(chunk))
683
- }, () => {
684
- callback(cipher.final())
685
- })
686
- },
687
- write: async (path, readFunc) => {
688
- // if encrypting paths, write dirent file(s) for all parent directories
689
- let p = path
690
- while (true) {
691
- const direntGenerator = stringGenerator(p + enc.encPathPadding(), enc)
692
- const dir = direntDir(dirname(p))
693
- const df = direntFile(dir, p);
694
- if (!(await client.write(df, direntGenerator()))) {
695
- throw new MobilettoError('write: error writing dirent file')
696
- }
697
- p = dirname(p)
698
- if (p === '.' || p === '') {
699
- break
700
- }
701
- }
702
-
703
- const cipher = crypt.getCipher(enc)
704
- const realPath = encryptPath(path)
705
- let generatorBytes = 0
706
- const createEncryptStream = (input) =>
707
- input.pipe(new Transform({
708
- transform(chunk, encoding, callback) {
709
- generatorBytes += chunk.length
710
- this.push(cipher.update(chunk))
711
- callback()
712
- },
713
- flush(callback) {
714
- this.push(cipher.final())
715
- callback()
716
- }
717
- }))
718
-
719
- // If we were passed a Readable, call write with a transform method
720
- if (isReadable(readFunc)) {
721
- await client.write(realPath, createEncryptStream(readFunc))
722
- } else {
723
- // If we were passed a generator, wrap it with another generator that encrypts
724
- function* cryptGenerator(plaintextGenerator) {
725
- let chunk = plaintextGenerator.next().value
726
- while (chunk) {
727
- generatorBytes += chunk.length
728
- yield cipher.update(chunk)
729
- chunk = plaintextGenerator.next().value
730
- }
731
- yield cipher.final()
732
- }
733
- await client.write(realPath, cryptGenerator(readFunc))
734
- }
735
- // write metadata
736
- const meta = { name: path, size: generatorBytes, type: M_FILE }
737
- await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
738
- return generatorBytes
739
- },
740
-
741
- remove: async (path, options) => {
742
- logger.debug(`enc.remove(${path}) starting`)
743
- const recursive = options === true || (options && options.recursive) || false
744
- // const quiet = (options && options.quiet) || false
745
- if (recursive) {
746
- // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
747
- const removed = []
748
- async function recRm (path) {
749
- const dirent = direntDir(path)
750
- let entries
751
- try {
752
- entries = await client.list(dirent)
753
- } catch (e) {
754
- if (!(e instanceof MobilettoNotFoundError)) {
755
- throw e instanceof MobilettoError
756
- ? e
757
- : new MobilettoError('error listing files for recursive deletion', e)
758
- }
759
- }
760
- if (entries && entries.length > 0) {
761
- const files = await _loadMeta(dirent, entries)
762
- for (const f of files) {
763
- await recRm(f.name)
764
- }
765
- }
766
- await removeDirentFile(path)
767
- removed.push(path)
768
- }
769
- await recRm(path)
770
- return removed
771
- }
772
-
773
- // remove the single file/dir
774
- await removeDirentFile(path)
775
-
776
- // if we were the last dirent file, then also remove dirent directory, and recursively upwards
777
- let parent = path
778
- let dirent = direntDir(parent)
779
- const df = direntFile(dirent, path)
780
- await client.remove(df, false, true)
781
- while (true) {
782
- try {
783
- const entries = await client.list(dirent)
784
- if (entries.length === 0) {
785
- await removeDirentFile(parent)
786
- }
787
- } catch (e) {
788
- if (!(e instanceof MobilettoNotFoundError)) {
789
- throw e
790
- }
791
- }
792
- if (parent === '.' || parent === '/') {
793
- // do not remove dirent for root dir
794
- break
795
- }
796
- parent = dirname(parent)
797
- dirent = direntDir(parent)
798
- }
799
- return path
800
- }
801
- }
802
- logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
803
- return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
804
- }
805
-
806
- async function readStream(stream, callback, endCallback) {
807
- const counter = {count: 0}
808
- const streamHandler = stream =>
809
- new Promise((resolve, reject) => {
810
- stream.on('data', (data) => {
811
- counter.count += data ? data.length : 0
812
- callback(data)
813
- })
814
- stream.on('error', reject)
815
- stream.on('end', () => {
816
- if (endCallback) {
817
- endCallback()
818
- }
819
- resolve()
820
- })
821
- })
822
- await streamHandler(stream)
823
- return counter.count
824
- }
825
-
826
- function writeStream (stream) {
827
- return (chunk) => {
828
- if (chunk) {
829
- stream.write(chunk, (err) => {
830
- if (err) {
831
- logger.error(`writeStream: error writing: ${err}`)
832
- throw err
833
- }
834
- })
835
- }
836
- }
837
- }
6
+ const {
7
+ M_FILE, M_DIR, M_LINK, M_SPECIAL,
8
+ MobilettoError, MobilettoNotFoundError,
9
+ logger, setLogLevel
10
+ } = require('mobiletto-common')
838
11
 
839
- function closeStream (stream) {
840
- return () => stream.close((err) => {
841
- if (err) {
842
- logger.error(`closeStream: error closing: ${err}`)
843
- throw err
844
- }
845
- })
12
+ const registerDrivers = () => {
13
+ registerDriver('local', require('mobiletto-driver-local'))
14
+ registerDriver('s3', require('mobiletto-driver-s3'))
15
+ registerDriver('b2', require('mobiletto-driver-b2'))
16
+ registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
846
17
  }
847
18
 
848
- const M_FILE = 'file'
849
- const M_DIR = 'dir'
850
- const M_LINK = 'link'
851
- const M_SPECIAL = 'special'
852
-
853
- const setLogLevel = level => setLoggerLevel(level)
854
-
855
- const closeRedis = async () => { await teardown() }
856
-
857
19
  module.exports = {
858
20
  M_FILE, M_DIR, M_LINK, M_SPECIAL,
859
- isAsyncGenerator, isReadable,
860
- mobiletto, connect,
861
21
  MobilettoError, MobilettoNotFoundError,
862
- setLogLevel,
863
- readStream, writeStream, closeStream,
22
+ logger, setLogLevel,
23
+ registerDrivers,
24
+ registerDriver,
25
+ mobiletto,
26
+ connect,
27
+ redis,
864
28
  closeRedis
865
29
  }