mobiletto 1.0.7 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,7 +5,7 @@ Mobiletto is a JavaScript storage abstraction layer, with optional transparent c
5
5
 
6
6
  It supports apps that are agnostic to where files are stored.
7
7
 
8
- All drivers are tested for identical behavior with 30 tests for each driver.
8
+ All drivers are tested for identical behavior with over 35 tests for each driver.
9
9
 
10
10
  Today the supported drivers are:
11
11
 
@@ -22,6 +22,7 @@ Today the supported drivers are:
22
22
  // * key: base directory
23
23
  // * secret: ignored, can be null
24
24
  // * opts object:
25
+ // * readOnly: optional, never change anything on the filesystem; default is false
25
26
  // * mode: optional, filesystem permissions to set when creating new files and directories, default is 0700
26
27
  const local = await mobiletto('local', '/home/ubuntu/tmp', null, {mode: '0700'})
27
28
 
@@ -29,6 +30,7 @@ Today the supported drivers are:
29
30
  // * key: AWS access key
30
31
  // * secret: AWS secret key
31
32
  // * opts object:
33
+ // * readOnly: optional, never change anything on the bucket; default is false
32
34
  // * bucket: required, name of the S3 bucket
33
35
  // * region: optional, the AWS region to communicate with, default is us-east-1
34
36
  // * prefix: optional, all read/writes within the S3 bucket will be under this prefix
@@ -69,6 +71,10 @@ Today the supported drivers are:
69
71
  local.write(path, generator)
70
72
  s3.write(path, generator)
71
73
 
74
+ // Write an entire file at once (convenience method)
75
+ await local.writeFile(path, bufferOrString) // returns count of bytes written
76
+ await s3.writeFile(path, bufferOrString) // returns count of bytes written
77
+
72
78
  // Delete a file
73
79
  // Quiet param is optional (default false), when set errors will not be thrown if the path does not exist
74
80
  const quiet = true
@@ -45,9 +45,7 @@ class StorageClient {
45
45
  }
46
46
  return { path, stat }
47
47
  }
48
- async testConfig () {
49
- return await this.list()
50
- }
48
+ testConfig = async () => await this.list()
51
49
  fileType (stat) {
52
50
  if (stat.isDirectory()) {
53
51
  return M_DIR
@@ -43,9 +43,8 @@ class StorageClient {
43
43
  this.client = new S3Client({region: this.region, credentials})
44
44
  }
45
45
 
46
- async testConfig () {
47
- return await this._list('', { MaxKeys: 1 })
48
- }
46
+ // noinspection JSUnusedGlobalSymbols -- called by driver init
47
+ testConfig = async () => await this._list('', { MaxKeys: 1 })
49
48
 
50
49
  async list (path = '') {
51
50
  return (await this._list(path)).map(obj => {
package/index.js CHANGED
@@ -32,24 +32,32 @@ function MobilettoNotFoundError (message) {
32
32
  }
33
33
  }
34
34
 
35
- const reader = (chunks) => (chunk) => {
36
- if (chunk) {
37
- chunks.push(chunk)
38
- }
39
- }
35
+ const reader = (chunks) => (chunk) => { if (chunk) { chunks.push(chunk) } }
40
36
 
41
37
  const UTILITY_FUNCTIONS = {
42
38
  readFile: (client) => async (path) => {
43
39
  const chunks = []
44
40
  await client.read(path, reader(chunks))
45
41
  return Buffer.concat(chunks)
42
+ },
43
+ writeFile: (client) => async (path, data) => {
44
+ const readFunc = function* () { yield data }
45
+ return await client.write(path, readFunc)
46
46
  }
47
47
  }
48
48
 
49
- function addUtilityFunctions (client) {
49
+ function addUtilityFunctions (client, readOnly = false) {
50
50
  for (const func of Object.keys(UTILITY_FUNCTIONS)) {
51
51
  client[func] = UTILITY_FUNCTIONS[func](client)
52
52
  }
53
+ if (readOnly) {
54
+ for (const writeFunc of ['write', 'remove', 'writeFile']) {
55
+ client[writeFunc] = async () => {
56
+ // console.warn(`${writeFunc} not supported in readOnly mode`)
57
+ return false
58
+ }
59
+ }
60
+ }
53
61
  return client
54
62
  }
55
63
 
@@ -63,8 +71,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
63
71
  if (!(await client.testConfig())) {
64
72
  throw new MobilettoError(`mobiletto(${driverPath}) error: test API call failed`)
65
73
  }
74
+ const readOnly = opts ? !!opts.readOnly : false
66
75
  if (encryption === null) {
67
- return addUtilityFunctions(client)
76
+ return addUtilityFunctions(client, readOnly)
68
77
  }
69
78
  const encKey = normalizeKey(encryption.key)
70
79
  if (!encKey) {
@@ -90,10 +99,19 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
90
99
  }
91
100
  return newPath + encrypted
92
101
  }
102
+ const metaPath = path => encryptPath(path + ' ~ META')
93
103
  const direntDir = dir => encryptPath(dir + DIR_ENT_DIR_SUFFIX)
94
104
  const direntFile = (dirent, path) => dirent + '/' + shasum(DIR_ENT_FILE_PREFIX + ' ' + path)
95
105
 
96
106
  const _metadata = (client) => async (path) => {
107
+ let metaObj
108
+ try {
109
+ let chunks = []
110
+ await client.read(metaPath(path), reader(chunks))
111
+ metaObj = JSON.parse(decrypt(Buffer.concat(chunks).toString(), enc))
112
+ } catch (e) {
113
+ metaObj = {}
114
+ }
97
115
  let meta
98
116
  try {
99
117
  meta = await client.metadata(encryptPath(path))
@@ -119,7 +137,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
119
137
  }
120
138
  }
121
139
  meta.name = path // rewrite name back to plaintext name
122
- return meta
140
+ return Object.assign({}, meta, metaObj)
123
141
  }
124
142
 
125
143
  const _loadMeta = async (dirent, entries) => {
@@ -139,6 +157,13 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
139
157
  const df = direntFile(direntDir(dirname(path)), path);
140
158
  await client.remove(df, {recursive: false, quiet: true})
141
159
  await client.remove(encryptPath(path), {recursive: false, quiet: true})
160
+ await client.remove(metaPath(path), {recursive: false, quiet: true})
161
+ }
162
+
163
+ function stringGenerator (value, enc) {
164
+ return function* () {
165
+ yield encrypt(value, enc)
166
+ }
142
167
  }
143
168
 
144
169
  const encClient = {
@@ -165,9 +190,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
165
190
  // if encrypting paths, write dirent file(s) for all parent directories
166
191
  let p = path
167
192
  while (true) {
168
- const direntGenerator = function* () {
169
- yield encrypt(p + enc.encPathPadding(), enc)
170
- }
193
+ const direntGenerator = stringGenerator(p + enc.encPathPadding(), enc)
171
194
  const dir = direntDir(dirname(p))
172
195
  const df = direntFile(dir, p);
173
196
  if (!(await client.write(df, direntGenerator()))) {
@@ -179,9 +202,11 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
179
202
  }
180
203
  }
181
204
 
205
+ let bytesRead = 0
182
206
  function* cryptGenerator(plaintextGenerator) {
183
207
  let chunk = plaintextGenerator.next().value
184
208
  while (chunk) {
209
+ bytesRead += chunk.length
185
210
  yield cipher.update(chunk)
186
211
  chunk = plaintextGenerator.next().value
187
212
  }
@@ -189,7 +214,9 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
189
214
  }
190
215
  const cipher = crypt.startEncryptStream(enc)
191
216
  const realPath = encryptPath(path)
192
- return client.write(realPath, cryptGenerator(readFunc))
217
+ await client.write(realPath, cryptGenerator(readFunc))
218
+ await client.write(metaPath(path), stringGenerator(JSON.stringify({ size: bytesRead }), enc)())
219
+ return bytesRead
193
220
  },
194
221
  remove: async (path, options) => {
195
222
  const recursive = (options && options.recursive) || false
@@ -247,7 +274,7 @@ async function mobiletto (driverPath, key, secret, opts, encryption = null) {
247
274
  return true
248
275
  }
249
276
  }
250
- return addUtilityFunctions(encClient)
277
+ return addUtilityFunctions(encClient, readOnly)
251
278
  }
252
279
 
253
280
  async function readStream(stream, callback, endCallback) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobiletto",
3
- "version": "1.0.7",
3
+ "version": "1.0.10",
4
4
  "description": "A storage abstraction layer",
5
5
  "main": "index.js",
6
6
  "author": "Jonathan Cobb",
@@ -34,20 +34,11 @@ DRIVER_CONFIG = {
34
34
  }
35
35
  }
36
36
 
37
- function closeEnough (expected, actual, percent = ENC_SIZE_CLOSE_ENOUGH_PERCENT) {
38
- expect(Math.abs(expected - actual)).to.be.lessThan(Math.floor(expected * percent),
39
- 'expected size within 5% of actual (due to encryption)')
40
- }
41
-
42
- async function assertMeta (api, name, expectedSize, closeEnoughPercent = null) {
37
+ async function assertMeta (api, name, expectedSize) {
43
38
  const meta = await api.metadata(name)
44
39
  should().exist(meta, 'expected return value from metadata call')
45
40
  expect(meta.name).equals(name, 'expected name of written file to be correct')
46
- if (closeEnoughPercent) {
47
- closeEnough(expectedSize, meta.size, closeEnoughPercent)
48
- } else {
49
- expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
50
- }
41
+ expect(meta.size).equals(expectedSize, 'expected size of written file to equal size of randomData')
51
42
  }
52
43
 
53
44
  async function assertMetaFail (api, name) {
@@ -91,6 +82,8 @@ describe('crypto test', () => {
91
82
 
92
83
  })
93
84
 
85
+ const encryptionTests = () => [null, { key: rand(32) }]
86
+
94
87
  // To test a single driver:
95
88
  // - Uncomment one of the lines below to set driverName to the one you want to test
96
89
  // - Comment out the next `for` line, and its closing curly brace (just before EOF)
@@ -156,23 +149,70 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
156
149
  })
157
150
  })
158
151
 
152
+ for (const encryption of encryptionTests()) {
153
+ const encDesc = encryption ? '(with encryption)' : '(without encryption)'
154
+ describe(`${driverName} - ${encDesc} fail to write and delete files in readOnly mode`, () => {
155
+ // some random data, plus a bit extra
156
+ const size = 16
157
+ const randomData = rand(size)
158
+ const fileSuffix = '' + Date.now()
159
+ let fixture
160
+ beforeEach((done) => {
161
+ const name = `test_file_${fileSuffix}`
162
+ const opts = Object.assign({}, config.opts, {readOnly: true})
163
+ connect(driverName, config.key, config.secret, opts, encryption)
164
+ .then(api => { fixture = {api, name, randomData} })
165
+ .catch(err => { throw err })
166
+ .finally(done)
167
+ })
168
+ it("should try to write a file and fail", async () => {
169
+ const response = await writeRandomFile(fixture, size)
170
+ expect(response).to.be.false
171
+ })
172
+ it("should fail to read the file we did not write", async () => {
173
+ try {
174
+ await fixture.api.readFile(fixture.name)
175
+ assert.fail('should not have been able to read non-existent file that we did not write')
176
+ } catch (e) {
177
+ if (!(e instanceof MobilettoNotFoundError)) {
178
+ assert.fail(`unexpected error: ${e} trying to read non-existent file`)
179
+ }
180
+ }
181
+ })
182
+ it("should fail to load metadata on the file we did not write", async () => {
183
+ try {
184
+ await assertMeta(fixture.api, fixture.name, fixture.randomData.length)
185
+ assert.fail('should not have been able to read metadata for non-existent file that we did not write')
186
+ } catch (e) {
187
+ if (!(e instanceof MobilettoNotFoundError)) {
188
+ assert.fail(`unexpected error: ${e} trying to read metadata on non-existent file`)
189
+ }
190
+ }
191
+ })
192
+ it("should fail to delete the file we did not write", async () => {
193
+ const removed = await fixture.api.remove(fixture.name)
194
+ expect(removed).to.be.false
195
+ })
196
+ })
197
+ }
198
+
159
199
  describe(`${driverName} - write an encrypted file, read file, read metadata, delete file`, () => {
160
200
  // some random data, plus a bit extra
161
201
  const size = (READ_SZ * TEMP_SZ_MULTIPLE) + Math.floor(Math.random() * (READ_SZ/2))
162
202
  const randomData = rand(size)
163
203
  const fileSuffix = '' + Date.now()
164
204
  const encryptionKey = rand(32)
165
- let encryptedByteCount = null
166
205
  let fixture
167
206
  beforeEach((done) => {
168
207
  const name = `test_file_${fileSuffix}`
169
208
  mobiletto(driverName, config.key, config.secret, config.opts, {key: encryptionKey})
170
209
  .then(api => { fixture = {api, name, randomData} })
210
+ .catch((err) => { throw err })
171
211
  .finally(done)
172
212
  })
173
213
  it("should write an encrypted file", async () => {
174
- encryptedByteCount = await writeRandomFile(fixture, size);
175
- closeEnough(size, encryptedByteCount)
214
+ const bytesWritten = await writeRandomFile(fixture, size);
215
+ expect(bytesWritten).is.equal(size, 'expected write API to return correct number of bytes written')
176
216
  })
177
217
  it("should read the encrypted file we just wrote", async () => {
178
218
  const data = await fixture.api.readFile(fixture.name);
@@ -191,7 +231,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
191
231
  })
192
232
  })
193
233
 
194
- for (const encryption of [null, {key: rand(32)}]) {
234
+ for (const encryption of encryptionTests()) {
195
235
  const encDesc = encryption ? '(with encryption)' : '(without encryption)'
196
236
  describe(`${driverName} - ${encDesc} write files in a new dir, read metadata, recursively delete`, () => {
197
237
  // describe(`${driverName} - ENCRYPTION write files in a new dir, read metadata, recursively delete`, () => {
@@ -218,11 +258,7 @@ for (const driverName of Object.keys(DRIVER_CONFIG)) {
218
258
 
219
259
  for (let i = 0; i < fileCount; i++) {
220
260
  const bytesWritten = await fixture.api.write(tempFilename(fixture.name, i), dataGenerator())
221
- if (encryption) {
222
- closeEnough(READ_SZ, bytesWritten)
223
- } else {
224
- expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
225
- }
261
+ expect(bytesWritten).to.equal(READ_SZ, 'expected write API to return correct number of bytes written')
226
262
  }
227
263
  })
228
264
  it("should load metadata for one of the new files", async () => {