mobiletto 1.0.46 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -32,10 +32,13 @@ class StorageClient {
32
32
  fileMode
33
33
  dirMode
34
34
  constructor(baseDir, opts) {
35
+ if (!baseDir) {
36
+ throw new MobilettoError('local.StorageClient: key (baseDir) is required')
37
+ }
35
38
  const resolved = this.resolveSymlinks(baseDir)
36
39
  if (!resolved.stat.isDirectory()) {
37
40
  const dest = resolved.path === baseDir ? null : resolved.path
38
- throw new MobilettoError(`local.storageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
41
+ throw new MobilettoError(`local.StorageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
39
42
  }
40
43
  const dir = resolved.path
41
44
  this.baseDir = dir.endsWith('/') ? dir : dir + '/'
@@ -109,7 +112,7 @@ class StorageClient {
109
112
  }
110
113
  return files
111
114
  } catch (e) {
112
- logger.error(`dirFiles: ${e}`)
115
+ logger.warn(`dirFiles: ${e}`)
113
116
  throw this.ioError(e, norm, 'dirFiles')
114
117
  }
115
118
  }
@@ -283,4 +286,4 @@ function storageClient (key, secret, opts) {
283
286
  return new StorageClient(key, opts)
284
287
  }
285
288
 
286
- module.exports = {storageClient}
289
+ module.exports = { storageClient }
@@ -30,7 +30,7 @@ class StorageClient {
30
30
  delimiter
31
31
  constructor(key, secret, opts) {
32
32
  if (!key || !secret || !opts || !opts.bucket) {
33
- throw new MobilettoError('key, secret, and opts.bucket are required')
33
+ throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
34
34
  }
35
35
  this.bucket = opts.bucket
36
36
  const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
@@ -305,9 +305,9 @@ class StorageClient {
305
305
 
306
306
  function storageClient (key, secret, opts) {
307
307
  if (!key || !secret || !opts || !opts.bucket) {
308
- throw new MobilettoError('key, secret, and opts.bucket are required')
308
+ throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
309
309
  }
310
310
  return new StorageClient(key, secret, opts)
311
311
  }
312
312
 
313
- module.exports = {storageClient}
313
+ module.exports = { storageClient }
package/index.js CHANGED
@@ -2,13 +2,14 @@ const fs = require('fs')
2
2
  const path = require('path')
3
3
  const { Readable, Transform } = require('stream')
4
4
  const { logger, setLogLevel: setLoggerLevel } = require('./util/logger')
5
- const redis = require('./util/redis')
5
+ const { getRedis } = require('./util/redis')
6
6
  const { basename, dirname } = require('path')
7
7
  const shasum = require('shasum')
8
8
  const randomstring = require('randomstring')
9
9
 
10
10
  const crypt = require('./util/crypt')
11
11
  const {DEFAULT_CRYPT_ALGO, normalizeKey, normalizeIV, encrypt, decrypt} = require("./util/crypt")
12
+ const LRU = require("lru-cache");
12
13
 
13
14
  const DIR_ENT_DIR_SUFFIX = '__.dirent'
14
15
  const DIR_ENT_FILE_PREFIX = 'dirent__'
@@ -20,6 +21,10 @@ const initMobilettoTempDir = () => {
20
21
  }
21
22
  const MOBILETTO_TMP = initMobilettoTempDir()
22
23
 
24
+ const REDIS_HOST = process.env.MOBILETTO_REDIS_HOST || '127.0.0.1'
25
+ const REDIS_PORT = process.env.MOBILETTO_REDIS_PORT || 6379
26
+ const REDIS_PREFIX = process.env.MOBILETTO_REDIS_PREFIX || '_mobiletto__'
27
+
23
28
  // adapted from https://stackoverflow.com/a/27724419
24
29
  function MobilettoError (message, err) {
25
30
  this.message = `${message}: ${err ? err : ''}`
@@ -33,6 +38,7 @@ function MobilettoError (message, err) {
33
38
  this.stack = (new Error(this.message)).stack
34
39
  }
35
40
  }
41
+ MobilettoError.prototype.toString = () => JSON.stringify(this)
36
42
 
37
43
  // adapted from https://stackoverflow.com/a/27724419
38
44
  function MobilettoNotFoundError (message) {
@@ -45,21 +51,59 @@ function MobilettoNotFoundError (message) {
45
51
  this.stack = (new Error(this.message)).stack
46
52
  }
47
53
  }
54
+ MobilettoNotFoundError.prototype.toString = () => JSON.stringify(this)
48
55
 
49
56
  const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
50
57
 
51
58
  const isAsyncGenerator = (func) => func[Symbol.toStringTag] === 'AsyncGenerator'
52
59
  const isReadable = (thing) => thing instanceof fs.ReadStream || thing instanceof Transform || thing instanceof Readable
53
60
 
54
- const getMetaCache = (size = 100) => redis.scopedCache('meta', size)
55
- const getListingCache = (size = 100) => redis.scopedCache('listingCache', size)
56
-
57
61
  const READ_FILE_CACHE_SIZE_THRESHOLD = 128 * 1024 // we can cache files of this size
58
62
 
63
+ class AwaitableLRU {
64
+ lru
65
+ constructor(size) {
66
+ this.lru = new LRU({ max: size })
67
+ }
68
+ get = (key) => Promise.resolve(this.lru.get(key))
69
+ set = (key, value) => Promise.resolve(this.lru.set(key, value))
70
+ }
71
+
72
+ const CACHE_FUNCTIONS = {
73
+ redis: (client) => () => {
74
+ if (typeof client.cache !== 'undefined') return client.cache
75
+ const redisConfig = client.redisConfig || {}
76
+ const enabled = redisConfig.enabled !== false
77
+ if (!enabled) {
78
+ logger.info(`redis: client.redisConfig.enabled === false, disabling cache`)
79
+ client.cache = null
80
+ return client.cache
81
+ }
82
+ const host = redisConfig.host || REDIS_HOST
83
+ const port = redisConfig.port || REDIS_PORT
84
+ const prefix = redisConfig.prefix || REDIS_PREFIX
85
+ if (!client.id) {
86
+ logger.warn(`redis: all nameless connections will share one cache`)
87
+ client.cache = getRedis('~nameless~', host, port, prefix)
88
+ } else {
89
+ client.cache = getRedis(client.id, host, port, prefix)
90
+ }
91
+ return client.cache
92
+ },
93
+ scopedCache: (client) => (cacheName, size = 100) => {
94
+ const redis = client.redis()
95
+ return redis ? redis.scopedCache(cacheName, size) : new AwaitableLRU(size)
96
+ },
97
+ flush: (client) => async () => {
98
+ const redis = client.redis()
99
+ return redis ? redis.flush() : true
100
+ }
101
+ }
102
+
59
103
  // noinspection JSUnusedGlobalSymbols,JSUnresolvedFunction
60
104
  const UTILITY_FUNCTIONS = {
61
105
  list: (client) => async (path, opts) => {
62
- const cache = getListingCache()
106
+ const cache = client.scopedCache('list')
63
107
  let cached = cache ? await cache.get(path) : null
64
108
  if (cached) {
65
109
  if (Array.isArray(cached)) { return cached }
@@ -104,7 +148,7 @@ const UTILITY_FUNCTIONS = {
104
148
  }
105
149
  },
106
150
  metadata: (client) => async (path) => {
107
- const cache = getMetaCache()
151
+ const cache = client.scopedCache('metadata')
108
152
  const cached = cache ? await cache.get(path) : null
109
153
  if (cached) {
110
154
  return cached
@@ -131,14 +175,15 @@ const UTILITY_FUNCTIONS = {
131
175
  },
132
176
  remove: (client) => async (path, opts) => {
133
177
  const recursive = opts && opts.recursive ? opts.recursive : false
178
+ // noinspection JSUnresolvedVariable
134
179
  const quiet = opts && opts.quiet ? opts.quiet : false
135
180
  // noinspection JSUnresolvedFunction
136
181
  const result = await client.driver_remove(path, recursive, quiet)
137
- await redis.flushMobiletto()
182
+ await client.flush()
138
183
  return result
139
184
  },
140
185
  readFile: (client) => async (path) => {
141
- const cache = redis.scopedCache('readFile')
186
+ const cache = client.scopedCache('readFile')
142
187
  const cached = cache ? await cache.get(path) : null
143
188
  if (cached) {
144
189
  return Buffer.from(cached, 'base64')
@@ -147,14 +192,19 @@ const UTILITY_FUNCTIONS = {
147
192
  await client.read(path, reader(chunks))
148
193
  const data = Buffer.concat(chunks);
149
194
  if (cache && data.length < READ_FILE_CACHE_SIZE_THRESHOLD) {
150
- cache.set(path, data.toString('base64'))
195
+ cache.set(path, data.toString('base64')).then(
196
+ () => { logger.debug(`readFile(${path}) cached ${data.length} bytes`) },
197
+ (err) => { logger.error(`readFile(${path}) error: ${err}`) }
198
+ )
151
199
  }
152
200
  return data
153
201
  },
154
202
  write: (client) => async (path, data) => {
203
+ logger.debug(`util.write(${path}) starting ...`)
155
204
  // noinspection JSUnresolvedFunction
156
205
  const bytesWritten = await client.driver_write(path, data)
157
- await redis.flushMobiletto()
206
+ await client.flush()
207
+ logger.debug(`util.write(${path}) wrote ${bytesWritten} bytes`)
158
208
  return bytesWritten
159
209
  },
160
210
  writeFile: (client) => async (path, data) => {
@@ -175,7 +225,8 @@ const UTILITY_FUNCTIONS = {
175
225
  const destName = obj.name.startsWith(sourcePath)
176
226
  ? obj.name.substring(sourcePath.length)
177
227
  : obj.name
178
- const destFullPath = clientPath + destName;
228
+ const destFullPath = (clientPath.endsWith('/') ? clientPath : clientPath + '/') +
229
+ (destName.startsWith('/') ? destName.substring(1) : destName)
179
230
  try {
180
231
  // if dest already exists and is the same size, don't copy it again
181
232
  let srcSize = null
@@ -240,11 +291,22 @@ async function mirrorDir (source, sourcePath, visitor) {
240
291
  }
241
292
  }
242
293
 
243
- function addUtilityFunctions (client, readOnly = false) {
244
- for (const func of Object.keys(UTILITY_FUNCTIONS)) {
245
- client[`driver_${func}`] = client[func] // save previous function, we will need `list` at least
246
- client[func] = UTILITY_FUNCTIONS[func](client)
294
+ function utilityFunctionConflict (client, func) {
295
+ if (typeof client[func] === 'function') {
296
+ if (typeof client[`driver_${func}`] !== 'undefined') {
297
+ logger.warn(`utilityFunctionConflict: driver_${func} has already been added`)
298
+ return false
299
+ } else {
300
+ client[`driver_${func}`] = client[func] // save original driver function
301
+ return true
302
+ }
303
+ } else if (typeof client[func] !== 'undefined') {
304
+ throw new MobilettoError(`utilityFunctionConflict: client defines a property ${func}, mobiletto function would overwrite`)
247
305
  }
306
+ }
307
+
308
+ const addUtilityFunctions = (client, readOnly = false) => {
309
+ addClientFunctions(client, UTILITY_FUNCTIONS, utilityFunctionConflict)
248
310
  if (readOnly) {
249
311
  for (const writeFunc of ['write', 'remove', 'writeFile']) {
250
312
  client[writeFunc] = async () => {
@@ -256,6 +318,24 @@ function addUtilityFunctions (client, readOnly = false) {
256
318
  return client
257
319
  }
258
320
 
321
+ const addCacheFunctions = (client) => addClientFunctions(client, CACHE_FUNCTIONS, (client, func) => {
322
+ logger.warn(`addCacheFunctions: ${func} already exists, not re-adding`)
323
+ return false
324
+ })
325
+
326
+ const addClientFunctions = (client, functions, conflictFunc) => {
327
+ for (const func of Object.keys(functions)) {
328
+ let add = true
329
+ if (client[func]) {
330
+ add = conflictFunc ? conflictFunc(client, func) : false
331
+ }
332
+ if (add) {
333
+ client[func] = functions[func](client)
334
+ }
335
+ }
336
+ return client
337
+ }
338
+
259
339
  async function connect (driverPath, key, secret, opts, encryption = null) {
260
340
  return await mobiletto(driverPath, key, secret, opts, encryption)
261
341
  }
@@ -264,15 +344,25 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
264
344
  logger.info(`mobiletto: connecting with driver ${driverPath}`)
265
345
  const driver = require(driverPath.includes('/') ? driverPath : `./drivers/${driverPath}/index.js`)
266
346
  const client = driver.storageClient(key, secret, opts)
267
- if (!(await client.testConfig())) {
268
- const message = `mobiletto(${driverPath}) error: test API call failed`
347
+ const configValue = await client.testConfig();
348
+ if (!configValue) {
349
+ const message = `mobiletto(${driverPath}) error: test API call failed: ${configValue}`
269
350
  logger.error(message)
270
351
  throw new MobilettoError(message)
271
352
  }
272
353
  const readOnly = opts ? !!opts.readOnly : false
354
+ client.redisConfig = opts && opts.redisConfig ? opts.redisConfig : {}
355
+
356
+ const internalIdForDriver = () =>
357
+ driverPath + '_' + shasum(`${key}\n${JSON.stringify(opts)}\n${(encryption ? JSON.stringify(encryption) : '')}`)
358
+
359
+ // If the driver didn't give the client a name, generate a unique internal name
360
+ if (!client.id) {
361
+ client.id = internalIdForDriver()
362
+ }
273
363
  if (encryption === null) {
274
364
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption not enabled)`)
275
- return addUtilityFunctions(client, readOnly)
365
+ return addUtilityFunctions(addCacheFunctions(client), readOnly)
276
366
  }
277
367
  const encKey = normalizeKey(encryption.key)
278
368
  if (!encKey) {
@@ -306,8 +396,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
306
396
  const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
307
397
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
308
398
 
399
+ const outerClient = addCacheFunctions(client)
309
400
  const _metadata = (client) => async (path) => {
310
- const cache = redis.scopedCache('_metadata')
401
+ const cache = outerClient.scopedCache('_metadata')
311
402
  const cached = cache ? await cache.get(path) : null
312
403
  if (cached) {
313
404
  return cached
@@ -373,8 +464,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
373
464
  const df = direntFile(direntDir(dirname(path)), path);
374
465
  const recursive = false
375
466
  const quiet = true
467
+
468
+ logger.debug(`removeDirentFile(${path}) removing df=${df}`)
376
469
  await client.remove(df, recursive, quiet)
470
+
471
+ logger.debug(`removeDirentFile(${path}) removing encryptPath(path)=${encryptPath(path)}`)
377
472
  await client.remove(encryptPath(path), recursive, quiet)
473
+
474
+ logger.debug(`removeDirentFile(${path}) removing metaPath(path)=${metaPath(path)}`)
378
475
  await client.remove(metaPath(path), recursive, quiet)
379
476
  }
380
477
 
@@ -384,14 +481,17 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
384
481
  }
385
482
  }
386
483
 
484
+ // noinspection JSUnusedGlobalSymbols
387
485
  const encClient = {
486
+ id: internalIdForDriver(),
487
+ redisConfig: client.redisConfig,
388
488
  list: async (pth = '', recursive, visitor) => {
389
489
  const p = pth === '' ? '.' : pth.endsWith('/') ? pth.substring(0, pth.length - 1) : pth
390
490
  const dirent = direntDir(p)
391
491
  let entries
392
492
 
393
493
  const cacheKey = `${p} ~ ${recursive}`
394
- const cache = visitor ? null : redis.scopedCache('enc_list')
494
+ const cache = visitor ? null : client.scopedCache('enc_list')
395
495
  const cached = cache && await cache.get(cacheKey)
396
496
 
397
497
  function cacheAndReturn (thing) {
@@ -404,6 +504,23 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
404
504
  return thing
405
505
  }
406
506
 
507
+ async function tryParentDirForSingleFile (path, visitor, e) {
508
+ // it might be a single file, try listing the parent dir
509
+ const parentDirent = direntDir(path.dirname(p));
510
+ entries = await client.list(parentDirent, false)
511
+ const objects = await _loadMeta(parentDirent, entries)
512
+ const found = objects.find(o => o.name === p)
513
+ if (found) {
514
+ if (visitor) {
515
+ await visitor(found)
516
+ }
517
+ logger.debug(`tryParentDirForSingleFile(${path}) found ${found.name}`)
518
+ return cacheAndReturn([found])
519
+ }
520
+ logger.debug(`tryParentDirForSingleFile(${path}) nothing found! e=${e}`)
521
+ throw e === null ? new MobilettoNotFoundError(path) : e
522
+ }
523
+
407
524
  if (cached) {
408
525
  entries = cached
409
526
  } else {
@@ -411,18 +528,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
411
528
  entries = await client.list(dirent, recursive)
412
529
  } catch (e) {
413
530
  if (e instanceof MobilettoNotFoundError && p.includes('/')) {
414
- // it might be a single file, try listing the parent dir
415
- const parentDirent = direntDir(path.dirname(p));
416
- entries = await client.list(parentDirent, false)
417
- const objects = await _loadMeta(parentDirent, entries)
418
- const found = objects.find(o => o.name === p)
419
- if (found) {
420
- return cacheAndReturn([found])
421
- }
422
- throw e
531
+ return await tryParentDirForSingleFile(path, visitor, e)
423
532
  }
424
533
  }
425
534
  }
535
+ if (entries.length === 0 && p.includes('/')) {
536
+ return await tryParentDirForSingleFile(path, visitor, null)
537
+ }
426
538
  if (cache) {
427
539
  cache.set(cacheKey, entries).then(
428
540
  () => { logger.debug(`enc_list: cached ${p} r=${recursive}`) },
@@ -506,12 +618,14 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
506
618
  await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
507
619
  return generatorBytes
508
620
  },
509
- // todo: remove should return an array of the paths that were actually removed
621
+
510
622
  remove: async (path, options) => {
623
+ logger.debug(`enc.remove(${path}) starting`)
511
624
  const recursive = options === true || (options && options.recursive) || false
512
625
  // const quiet = (options && options.quiet) || false
513
626
  if (recursive) {
514
627
  // ugh. we have to iterate over all dirent files, and remove each file/subdir one by one
628
+ const removed = []
515
629
  async function recRm (path) {
516
630
  const dirent = direntDir(path)
517
631
  let entries
@@ -530,13 +644,15 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
530
644
  await recRm(f.name)
531
645
  }
532
646
  }
533
- await removeDirentFile(path);
647
+ await removeDirentFile(path)
648
+ removed.push(path)
534
649
  }
535
650
  await recRm(path)
651
+ return removed
536
652
  }
537
653
 
538
654
  // remove the single file/dir
539
- await removeDirentFile(path);
655
+ await removeDirentFile(path)
540
656
 
541
657
  // if we were the last dirent file, then also remove dirent directory, and recursively upwards
542
658
  let parent = path
@@ -561,11 +677,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
561
677
  parent = dirname(parent)
562
678
  dirent = direntDir(parent)
563
679
  }
564
- return true
680
+ return path
565
681
  }
566
682
  }
567
683
  logger.info(`mobiletto: successfully connected using driver ${driverPath}, returning client (encryption enabled)`)
568
- return addUtilityFunctions(encClient, readOnly)
684
+ return addUtilityFunctions(addCacheFunctions(encClient), readOnly)
569
685
  }
570
686
 
571
687
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.46",
4
- "description": "A storage abstraction layer",
3
+ "version": "1.1.1",
4
+ "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, and local storage",
5
5
  "main": "index.js",
6
6
  "keywords": [
7
7
  "storage",
@@ -21,7 +21,7 @@
21
21
  },
22
22
  "license": "Apache-2.0",
23
23
  "scripts": {
24
- "test": "mocha --timeout 10000"
24
+ "test": "mocha --timeout 60000 ./test/*.spec.js"
25
25
  },
26
26
  "devDependencies": {
27
27
  "@babel/core": "^7.18.10",
@@ -35,6 +35,8 @@
35
35
  "dependencies": {
36
36
  "@aws-sdk/client-s3": "^3.150.0",
37
37
  "@aws-sdk/lib-storage": "^3.150.0",
38
+ "@gideo-llc/backblaze-b2-upload-any": "^0.1.4",
39
+ "backblaze-b2": "^1.7.0",
38
40
  "ioredis": "^5.2.3",
39
41
  "lru-cache": "^7.14.0",
40
42
  "node-fetch-commonjs": "^3.1.1",