mobiletto 1.0.33 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/drivers/local/index.js +18 -4
- package/drivers/s3/index.js +15 -1
- package/index.js +24 -8
- package/package.json +1 -1
- package/test/index.spec.js +21 -2
- package/util/crypt.js +10 -4
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
|
|
|
5
5
|
|
|
6
6
|
It supports apps that are agnostic to where files are stored.
|
|
7
7
|
|
|
8
|
-
All drivers are tested for identical behavior with over
|
|
8
|
+
All drivers are tested for identical behavior with over 50 tests for each driver.
|
|
9
9
|
|
|
10
10
|
Today the supported drivers are:
|
|
11
11
|
|
package/drivers/local/index.js
CHANGED
|
@@ -112,8 +112,8 @@ class StorageClient {
|
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
async list (
|
|
116
|
-
const dir = this.normalizePath(
|
|
115
|
+
async list (pth = '', recursive = false, visitor = null) {
|
|
116
|
+
const dir = this.normalizePath(pth)
|
|
117
117
|
try {
|
|
118
118
|
if (visitor === null) {
|
|
119
119
|
const results = []
|
|
@@ -123,7 +123,21 @@ class StorageClient {
|
|
|
123
123
|
return await this.readDirFiles(dir, recursive, visitor)
|
|
124
124
|
}
|
|
125
125
|
} catch (err) {
|
|
126
|
-
|
|
126
|
+
if (err instanceof MobilettoNotFoundError && !recursive && pth.includes('/')) {
|
|
127
|
+
// are we trying to list a single file?
|
|
128
|
+
const parentDir = this.normalizePath(dirname(pth))
|
|
129
|
+
const results = []
|
|
130
|
+
await this.readDirFiles(parentDir, false, (obj) => {
|
|
131
|
+
if (obj.name === pth) {
|
|
132
|
+
results.push(obj)
|
|
133
|
+
}
|
|
134
|
+
})
|
|
135
|
+
if (results.length === 0) {
|
|
136
|
+
throw err
|
|
137
|
+
}
|
|
138
|
+
return results
|
|
139
|
+
}
|
|
140
|
+
throw this.ioError(err, pth, 'list')
|
|
127
141
|
}
|
|
128
142
|
}
|
|
129
143
|
async metadata (path) {
|
|
@@ -228,7 +242,7 @@ class StorageClient {
|
|
|
228
242
|
: generator.next().value
|
|
229
243
|
}
|
|
230
244
|
closer()
|
|
231
|
-
return
|
|
245
|
+
return count
|
|
232
246
|
}
|
|
233
247
|
|
|
234
248
|
async read (path, callback, endCallback = null) {
|
package/drivers/s3/index.js
CHANGED
|
@@ -3,6 +3,8 @@ const {
|
|
|
3
3
|
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
4
|
} = require('../../index')
|
|
5
5
|
|
|
6
|
+
const { dirname } = require('path')
|
|
7
|
+
|
|
6
8
|
const {
|
|
7
9
|
S3Client,
|
|
8
10
|
ListObjectsCommand,
|
|
@@ -57,7 +59,19 @@ class StorageClient {
|
|
|
57
59
|
}
|
|
58
60
|
|
|
59
61
|
async list (path = '', recursive = false, visitor = null) {
|
|
60
|
-
|
|
62
|
+
try {
|
|
63
|
+
return await this._list(path, recursive, visitor)
|
|
64
|
+
} catch (e) {
|
|
65
|
+
if (e instanceof MobilettoNotFoundError && !recursive && path.includes('/')) {
|
|
66
|
+
// are we trying to list a single file?
|
|
67
|
+
const objects = await this._list(dirname(path), false)
|
|
68
|
+
const found = objects.find(o => o.name === path)
|
|
69
|
+
if (found) {
|
|
70
|
+
return [found]
|
|
71
|
+
}
|
|
72
|
+
throw e
|
|
73
|
+
}
|
|
74
|
+
}
|
|
61
75
|
}
|
|
62
76
|
|
|
63
77
|
async _list (path, recursive = false, visitor = null, params = {}) {
|
package/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
const fs = require('fs')
|
|
2
|
+
const path = require('path')
|
|
2
3
|
const winston = require('winston')
|
|
3
4
|
const LRU = require('lru-cache')
|
|
4
5
|
const { basename, dirname } = require('path')
|
|
@@ -347,9 +348,24 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
347
348
|
}
|
|
348
349
|
|
|
349
350
|
const encClient = {
|
|
350
|
-
list: async (
|
|
351
|
-
const dirent = direntDir(
|
|
352
|
-
|
|
351
|
+
list: async (pth, recursive, visitor) => {
|
|
352
|
+
const dirent = direntDir(pth)
|
|
353
|
+
let entries
|
|
354
|
+
try {
|
|
355
|
+
entries = await client.list(dirent, recursive)
|
|
356
|
+
} catch (e) {
|
|
357
|
+
if (e instanceof MobilettoNotFoundError) {
|
|
358
|
+
// it might be a single file, try listing the parent dir
|
|
359
|
+
const parentDirent = direntDir(path.dirname(pth));
|
|
360
|
+
entries = await client.list(parentDirent, false)
|
|
361
|
+
const objects = await _loadMeta(parentDirent, entries)
|
|
362
|
+
const found = objects.find(o => o.name === pth)
|
|
363
|
+
if (found) {
|
|
364
|
+
return [ found ]
|
|
365
|
+
}
|
|
366
|
+
throw e
|
|
367
|
+
}
|
|
368
|
+
}
|
|
353
369
|
if (!entries || entries.length === 0) {
|
|
354
370
|
return []
|
|
355
371
|
}
|
|
@@ -390,11 +406,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
390
406
|
}
|
|
391
407
|
}
|
|
392
408
|
|
|
393
|
-
let
|
|
409
|
+
let generatorBytes = 0
|
|
394
410
|
function* cryptGenerator(plaintextGenerator) {
|
|
395
411
|
let chunk = plaintextGenerator.next().value
|
|
396
412
|
while (chunk) {
|
|
397
|
-
|
|
413
|
+
generatorBytes += chunk.length
|
|
398
414
|
yield cipher.update(chunk)
|
|
399
415
|
chunk = plaintextGenerator.next().value
|
|
400
416
|
}
|
|
@@ -402,10 +418,10 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
|
|
|
402
418
|
}
|
|
403
419
|
const cipher = crypt.startEncryptStream(enc)
|
|
404
420
|
const realPath = encryptPath(path)
|
|
405
|
-
await client.write(realPath, cryptGenerator(readFunc))
|
|
406
|
-
const meta = { name: path, size:
|
|
421
|
+
const streamBytes = await client.write(realPath, isReadable(readFunc) ? readFunc : cryptGenerator(readFunc))
|
|
422
|
+
const meta = { name: path, size: generatorBytes, type: M_FILE }
|
|
407
423
|
await client.write(metaPath(path), stringGenerator(JSON.stringify(meta), enc)())
|
|
408
|
-
return
|
|
424
|
+
return isReadable(readFunc) ? streamBytes : generatorBytes
|
|
409
425
|
},
|
|
410
426
|
// todo: remove should return an array of the paths that were actually removed
|
|
411
427
|
remove: async (path, options) => {
|
package/package.json
CHANGED
package/test/index.spec.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// To run the tests, you need a .env file one level above this directory
|
|
2
2
|
// that contains env vars for all the process.env stuff in DRIVER_CONFIG below
|
|
3
|
+
const fs = require('fs')
|
|
3
4
|
require('dotenv').config()
|
|
4
5
|
|
|
5
6
|
const randomstring = require('randomstring')
|
|
@@ -148,7 +149,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
148
149
|
it("should read the file we just wrote", async () => {
|
|
149
150
|
const data = await fixture.api.readFile(fixture.name);
|
|
150
151
|
expect(data.length).is.equal(size, 'expected read API to return correct number of bytes read')
|
|
151
|
-
expect(data.toString(
|
|
152
|
+
expect(data.toString()).to.equal(fixture.randomData, 'expected to read back the same data we wrote')
|
|
152
153
|
})
|
|
153
154
|
it("should load metadata on the file we just wrote", async () => {
|
|
154
155
|
await assertMeta (fixture.api, fixture.name, fixture.randomData.length)
|
|
@@ -166,7 +167,7 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
166
167
|
const encDesc = encryption ? '(with encryption)' : '(without encryption)'
|
|
167
168
|
describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
|
|
168
169
|
// describe(`${driverName} - ENC fail to write and delete files in readOnly mode`, () => {
|
|
169
|
-
|
|
170
|
+
// const encryption = {key: rand(32)}
|
|
170
171
|
const size = 16
|
|
171
172
|
const randomData = rand(size)
|
|
172
173
|
const fileSuffix = '' + Date.now()
|
|
@@ -286,6 +287,24 @@ for (const driverName of DRIVER_NAMES) {
|
|
|
286
287
|
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
|
|
287
288
|
}
|
|
288
289
|
})
|
|
290
|
+
it('should write a file in a new directory using a stream', async () => {
|
|
291
|
+
// create a random temp file, write it
|
|
292
|
+
const data = randomstring.generate(READ_SZ)
|
|
293
|
+
const tempFile = `/tmp/${randomstring.generate(10)}_cool_${Date.now()}`;
|
|
294
|
+
fs.writeFileSync(tempFile, data)
|
|
295
|
+
const reader = fs.createReadStream(tempFile)
|
|
296
|
+
const streamFile = fixture.name + '_stream';
|
|
297
|
+
const bytesWritten = await fixture.api.write(streamFile, reader)
|
|
298
|
+
expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written for STREAM')
|
|
299
|
+
// remove stream file
|
|
300
|
+
const removed = await fixture.api.remove(streamFile)
|
|
301
|
+
expect(!!removed).to.be.true
|
|
302
|
+
})
|
|
303
|
+
it("should successfully list a file individually", async () => {
|
|
304
|
+
const singleFile = await fixture.api.list(fixture.name)
|
|
305
|
+
expect(singleFile).to.have.lengthOf(1)
|
|
306
|
+
expect(singleFile[0]).to.have.property('name', fixture.name)
|
|
307
|
+
})
|
|
289
308
|
it("should load metadata for one of the new files", async () => {
|
|
290
309
|
await assertMeta(fixture.api, fixture.name, READ_SZ, encryption ? ENC_SIZE_CLOSE_ENOUGH_PERCENT : null)
|
|
291
310
|
})
|
package/util/crypt.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// adapted from https://stackoverflow.com/a/64136185
|
|
2
2
|
const crypto = require('crypto')
|
|
3
3
|
const shasum = require('shasum')
|
|
4
|
+
const winston = require("winston");
|
|
4
5
|
|
|
5
6
|
const WARN_PLAINTEXT = !process.env.IGNORE_DISABLED_ENCRYPTION
|
|
6
7
|
const MIN_KEY_LEN = 16
|
|
@@ -8,13 +9,13 @@ const DEFAULT_CRYPT_ALGO = 'aes-256-cbc';
|
|
|
8
9
|
|
|
9
10
|
function normalizeKey (k) {
|
|
10
11
|
return (typeof k === 'string' && k.trim().length > MIN_KEY_LEN)
|
|
11
|
-
? Buffer.from(shasum(k)).subarray(0, 32)
|
|
12
|
+
? Buffer.from(shasum(k.trim())).subarray(0, 32)
|
|
12
13
|
: null
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
function normalizeIV (iv, key) {
|
|
16
|
-
return (typeof iv === 'string')
|
|
17
|
-
? Buffer.from(shasum(iv)).subarray(0, 16)
|
|
17
|
+
return (typeof iv === 'string' && iv.trim().length >= 16)
|
|
18
|
+
? Buffer.from(shasum(iv.trim())).subarray(0, 16)
|
|
18
19
|
: key
|
|
19
20
|
? Buffer.from(shasum(key)).subarray(0, 16)
|
|
20
21
|
: null
|
|
@@ -33,7 +34,12 @@ function setDefaultIV (iv) {
|
|
|
33
34
|
}
|
|
34
35
|
|
|
35
36
|
function getCipher(enc) {
|
|
36
|
-
|
|
37
|
+
try {
|
|
38
|
+
return crypto.createCipheriv(enc.algo, enc.key, enc.iv);
|
|
39
|
+
} catch (e) {
|
|
40
|
+
winston.error(`getCipher(${enc}): ${e}`)
|
|
41
|
+
throw e
|
|
42
|
+
}
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
function encrypt (plainText, encryption, outputEncoding = 'base64') {
|