mobiletto 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cjs/index.d.ts +2 -0
- package/lib/cjs/index.js +30 -0
- package/lib/cjs/package.json +16 -0
- package/lib/esm/index.d.ts +2 -0
- package/lib/esm/index.js +12 -0
- package/package.json +27 -13
- package/.babelrc +0 -5
- package/.eslintrc.js +0 -14
- package/docs/README.md +0 -113
- package/docs/af/README.md +0 -481
- package/docs/am/README.md +0 -481
- package/docs/ar/README.md +0 -481
- package/docs/az/README.md +0 -481
- package/docs/be/README.md +0 -481
- package/docs/bg/README.md +0 -481
- package/docs/bn/README.md +0 -481
- package/docs/bs/README.md +0 -481
- package/docs/ca/README.md +0 -481
- package/docs/ceb/README.md +0 -481
- package/docs/co/README.md +0 -481
- package/docs/cs/README.md +0 -481
- package/docs/cy/README.md +0 -481
- package/docs/da/README.md +0 -481
- package/docs/de/README.md +0 -481
- package/docs/el/README.md +0 -481
- package/docs/eo/README.md +0 -481
- package/docs/es/README.md +0 -481
- package/docs/et/README.md +0 -481
- package/docs/eu/README.md +0 -481
- package/docs/fa/README.md +0 -481
- package/docs/fi/README.md +0 -481
- package/docs/fr/README.md +0 -481
- package/docs/fy/README.md +0 -481
- package/docs/ga/README.md +0 -481
- package/docs/gd/README.md +0 -481
- package/docs/gl/README.md +0 -481
- package/docs/gu/README.md +0 -481
- package/docs/ha/README.md +0 -481
- package/docs/haw/README.md +0 -481
- package/docs/he/README.md +0 -481
- package/docs/hi/README.md +0 -481
- package/docs/hmn/README.md +0 -481
- package/docs/hr/README.md +0 -481
- package/docs/ht/README.md +0 -481
- package/docs/hu/README.md +0 -481
- package/docs/hy/README.md +0 -481
- package/docs/id/README.md +0 -481
- package/docs/ig/README.md +0 -481
- package/docs/is/README.md +0 -481
- package/docs/it/README.md +0 -481
- package/docs/iw/README.md +0 -481
- package/docs/ja/README.md +0 -481
- package/docs/jv/README.md +0 -481
- package/docs/ka/README.md +0 -481
- package/docs/kk/README.md +0 -481
- package/docs/km/README.md +0 -481
- package/docs/kn/README.md +0 -481
- package/docs/ko/README.md +0 -481
- package/docs/ku/README.md +0 -481
- package/docs/ky/README.md +0 -481
- package/docs/la/README.md +0 -481
- package/docs/lb/README.md +0 -481
- package/docs/lo/README.md +0 -481
- package/docs/lt/README.md +0 -481
- package/docs/lv/README.md +0 -481
- package/docs/mg/README.md +0 -481
- package/docs/mi/README.md +0 -481
- package/docs/mk/README.md +0 -481
- package/docs/ml/README.md +0 -481
- package/docs/mn/README.md +0 -481
- package/docs/mr/README.md +0 -481
- package/docs/ms/README.md +0 -481
- package/docs/mt/README.md +0 -481
- package/docs/my/README.md +0 -481
- package/docs/ne/README.md +0 -481
- package/docs/nl/README.md +0 -481
- package/docs/no/README.md +0 -481
- package/docs/ny/README.md +0 -481
- package/docs/or/README.md +0 -481
- package/docs/pa/README.md +0 -481
- package/docs/pl/README.md +0 -481
- package/docs/ps/README.md +0 -481
- package/docs/pt/README.md +0 -481
- package/docs/ro/README.md +0 -481
- package/docs/ru/README.md +0 -481
- package/docs/rw/README.md +0 -481
- package/docs/sd/README.md +0 -481
- package/docs/si/README.md +0 -481
- package/docs/sk/README.md +0 -481
- package/docs/sl/README.md +0 -481
- package/docs/sm/README.md +0 -481
- package/docs/sn/README.md +0 -481
- package/docs/so/README.md +0 -481
- package/docs/sq/README.md +0 -481
- package/docs/sr/README.md +0 -481
- package/docs/st/README.md +0 -481
- package/docs/su/README.md +0 -481
- package/docs/sv/README.md +0 -481
- package/docs/sw/README.md +0 -481
- package/docs/ta/README.md +0 -481
- package/docs/te/README.md +0 -481
- package/docs/tg/README.md +0 -481
- package/docs/th/README.md +0 -481
- package/docs/tk/README.md +0 -481
- package/docs/tl/README.md +0 -481
- package/docs/tr/README.md +0 -481
- package/docs/tt/README.md +0 -481
- package/docs/ug/README.md +0 -481
- package/docs/uk/README.md +0 -481
- package/docs/ur/README.md +0 -481
- package/docs/uz/README.md +0 -481
- package/docs/vi/README.md +0 -481
- package/docs/xh/README.md +0 -481
- package/docs/yi/README.md +0 -481
- package/docs/yo/README.md +0 -481
- package/docs/zh/README.md +0 -481
- package/docs/zu/README.md +0 -481
- package/drivers/b2/index.js +0 -451
- package/drivers/local/index.js +0 -315
- package/drivers/s3/index.js +0 -316
- package/hokey.json +0 -16
- package/index.js +0 -29
- package/test/api.spec.js +0 -512
- package/test/crypt.spec.js +0 -67
package/drivers/local/index.js
DELETED
|
@@ -1,315 +0,0 @@
|
|
|
1
|
-
const { dirname, basename } = require('path')
|
|
2
|
-
|
|
3
|
-
const {
|
|
4
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL, isAsyncGenerator, isReadable,
|
|
5
|
-
MobilettoError, MobilettoNotFoundError, readStream, writeStream, closeStream
|
|
6
|
-
} = require('../../index')
|
|
7
|
-
|
|
8
|
-
const { logger } = require('../../util/logger')
|
|
9
|
-
|
|
10
|
-
const fs = require('fs')
|
|
11
|
-
|
|
12
|
-
const DEFAULT_FILE_MODE = '0600'
|
|
13
|
-
const DEFAULT_DIR_MODE = '0700'
|
|
14
|
-
|
|
15
|
-
const isNotExistError = (err) => err.code && (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
|
16
|
-
|
|
17
|
-
const fileType = (stat) => {
|
|
18
|
-
if (stat.isDirectory()) {
|
|
19
|
-
return M_DIR
|
|
20
|
-
}
|
|
21
|
-
if (stat.isFile()) {
|
|
22
|
-
return M_FILE
|
|
23
|
-
}
|
|
24
|
-
if (stat.isSymbolicLink()) {
|
|
25
|
-
return M_LINK
|
|
26
|
-
}
|
|
27
|
-
return M_SPECIAL
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
class StorageClient {
|
|
31
|
-
baseDir
|
|
32
|
-
fileMode
|
|
33
|
-
dirMode
|
|
34
|
-
constructor(baseDir, opts) {
|
|
35
|
-
if (!baseDir) {
|
|
36
|
-
throw new MobilettoError('local.StorageClient: key (baseDir) is required')
|
|
37
|
-
}
|
|
38
|
-
const resolved = this.resolveSymlinks(baseDir)
|
|
39
|
-
if (!resolved.stat.isDirectory()) {
|
|
40
|
-
const dest = resolved.path === baseDir ? null : resolved.path
|
|
41
|
-
throw new MobilettoError(`local.StorageClient: not a directory: ${baseDir}${dest ? ` (resolved to ${dest})` : ''}`)
|
|
42
|
-
}
|
|
43
|
-
const dir = resolved.path
|
|
44
|
-
this.baseDir = dir.endsWith('/') ? dir : dir + '/'
|
|
45
|
-
this.fileMode = opts && opts.fileMode ? opts.fileMode : DEFAULT_FILE_MODE
|
|
46
|
-
this.dirMode = opts && opts.dirMode ? opts.dirMode : DEFAULT_DIR_MODE
|
|
47
|
-
}
|
|
48
|
-
resolveSymlinks (path) {
|
|
49
|
-
let stat = fs.lstatSync(path)
|
|
50
|
-
let symlinksFollowed = false
|
|
51
|
-
while (!stat.isDirectory() && stat.isSymbolicLink()) {
|
|
52
|
-
path = fs.readlinkSync(path)
|
|
53
|
-
if (path.startsWith('private/')) {
|
|
54
|
-
path = '/' + path
|
|
55
|
-
}
|
|
56
|
-
stat = fs.lstatSync(path)
|
|
57
|
-
symlinksFollowed = true
|
|
58
|
-
}
|
|
59
|
-
return { path, stat }
|
|
60
|
-
}
|
|
61
|
-
testConfig = async () => await this.list()
|
|
62
|
-
|
|
63
|
-
normalizePath = (path) => path.startsWith(this.baseDir)
|
|
64
|
-
? path
|
|
65
|
-
: this.baseDir + (path.startsWith('/') ? path.substring(1) : path)
|
|
66
|
-
denormalizePath = (path) => {
|
|
67
|
-
const norm = path.startsWith(this.baseDir) ? path.substring(this.baseDir.length) : path
|
|
68
|
-
return norm.startsWith('/') ? norm.substring(1) : norm
|
|
69
|
-
}
|
|
70
|
-
ioError = (err, path, method) => isNotExistError(err)
|
|
71
|
-
? err instanceof MobilettoNotFoundError
|
|
72
|
-
? err
|
|
73
|
-
: new MobilettoNotFoundError(this.denormalizePath(path))
|
|
74
|
-
: err instanceof MobilettoError || err instanceof MobilettoNotFoundError
|
|
75
|
-
? err
|
|
76
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
fileToObject = (dir) => (f) => {
|
|
80
|
-
const normPath = (dir.endsWith('/') ? dir : dir + '/') + f;
|
|
81
|
-
const stat = fs.lstatSync(normPath)
|
|
82
|
-
const type = fileType(stat)
|
|
83
|
-
const entry = {
|
|
84
|
-
name: this.denormalizePath(normPath),
|
|
85
|
-
type
|
|
86
|
-
}
|
|
87
|
-
if (type === M_LINK) {
|
|
88
|
-
const resolved = this.resolveSymlinks(normPath)
|
|
89
|
-
entry.link = resolved.path
|
|
90
|
-
}
|
|
91
|
-
return entry
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
readDirFiles = async (dir, recursive, visitor) => {
|
|
95
|
-
if (!recursive) {
|
|
96
|
-
return await this.dirFiles(dir, visitor)
|
|
97
|
-
}
|
|
98
|
-
const files = await this.dirFiles(dir, visitor)
|
|
99
|
-
for (const file of files) {
|
|
100
|
-
if (file.type === M_DIR) {
|
|
101
|
-
await this.readDirFiles(file.name, true, visitor)
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
dirFiles = async (dir, visitor) => {
|
|
107
|
-
const norm = this.normalizePath(dir);
|
|
108
|
-
try {
|
|
109
|
-
const names = fs.readdirSync(norm);
|
|
110
|
-
const files = names.map(this.fileToObject(norm))
|
|
111
|
-
for (const f of files) {
|
|
112
|
-
await visitor(f)
|
|
113
|
-
}
|
|
114
|
-
return files
|
|
115
|
-
} catch (e) {
|
|
116
|
-
if (isNotExistError(e)) {
|
|
117
|
-
// try to list the parent directory and filter for just this file
|
|
118
|
-
const files = []
|
|
119
|
-
try {
|
|
120
|
-
const parent = dirname(norm)
|
|
121
|
-
const base = basename(norm)
|
|
122
|
-
const names = fs.readdirSync(parent)
|
|
123
|
-
for (const n of names) {
|
|
124
|
-
if (n === base) {
|
|
125
|
-
const f = this.fileToObject(parent)(base)
|
|
126
|
-
files.push(f)
|
|
127
|
-
await visitor(f)
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
} catch (e2) {
|
|
131
|
-
logger.warn(`dirFiles (try-file) error: ${JSON.stringify(e2)}`)
|
|
132
|
-
throw this.ioError(e2, norm, 'dirFiles')
|
|
133
|
-
}
|
|
134
|
-
if (files.length > 0) {
|
|
135
|
-
return files
|
|
136
|
-
} else {
|
|
137
|
-
logger.warn(`dirFiles (try-file) not found: ${norm}`)
|
|
138
|
-
throw this.ioError(e, norm, 'dirFiles')
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
logger.warn(`dirFiles error: ${JSON.stringify(e)}`)
|
|
142
|
-
throw this.ioError(e, norm, 'dirFiles')
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
async list (pth = '', recursive = false, visitor = null) {
|
|
147
|
-
const dir = this.normalizePath(pth)
|
|
148
|
-
try {
|
|
149
|
-
if (visitor === null) {
|
|
150
|
-
const results = []
|
|
151
|
-
await this.readDirFiles(dir, recursive, (obj) => results.push(obj))
|
|
152
|
-
return results
|
|
153
|
-
} else {
|
|
154
|
-
return await this.readDirFiles(dir, recursive, visitor)
|
|
155
|
-
}
|
|
156
|
-
} catch (err) {
|
|
157
|
-
if (err instanceof MobilettoNotFoundError && !recursive && pth.includes('/')) {
|
|
158
|
-
// are we trying to list a single file?
|
|
159
|
-
const parentDir = this.normalizePath(dirname(pth))
|
|
160
|
-
const results = []
|
|
161
|
-
await this.readDirFiles(parentDir, false, (obj) => {
|
|
162
|
-
if (obj.name === pth) {
|
|
163
|
-
results.push(obj)
|
|
164
|
-
}
|
|
165
|
-
})
|
|
166
|
-
if (results.length === 0) {
|
|
167
|
-
throw err
|
|
168
|
-
}
|
|
169
|
-
return results
|
|
170
|
-
}
|
|
171
|
-
throw this.ioError(err, pth, 'list')
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
async metadata (path) {
|
|
175
|
-
const file = this.normalizePath(path)
|
|
176
|
-
try {
|
|
177
|
-
const lstat = fs.lstatSync(file)
|
|
178
|
-
if (!lstat) {
|
|
179
|
-
throw new MobilettoError('metadata: lstat error')
|
|
180
|
-
}
|
|
181
|
-
const type = fileType(lstat)
|
|
182
|
-
if (type === M_DIR && path !== '') {
|
|
183
|
-
const contents = await this.list(path)
|
|
184
|
-
if (contents.length === 0) {
|
|
185
|
-
throw new MobilettoNotFoundError(path)
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
return {
|
|
189
|
-
name: this.denormalizePath(file),
|
|
190
|
-
type,
|
|
191
|
-
size: lstat.size,
|
|
192
|
-
mtime: lstat.mtimeMs
|
|
193
|
-
}
|
|
194
|
-
} catch (err) {
|
|
195
|
-
throw this.ioError(err, path, 'metadata')
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
mkdirs (path) {
|
|
200
|
-
try {
|
|
201
|
-
logger.debug(`mkdirs: creating directory: ${path}`)
|
|
202
|
-
fs.mkdirSync(path, {recursive: true, mode: this.dirMode})
|
|
203
|
-
} catch (err) {
|
|
204
|
-
throw new MobilettoError(`mkdirs: error creating directory ${path}: ${err}`, err)
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
async normalizePathAndEnsureParentDirs (path) {
|
|
209
|
-
const file = this.normalizePath(path)
|
|
210
|
-
|
|
211
|
-
logger.debug(`read: reading path: ${path} - ${file}`)
|
|
212
|
-
const parent = dirname(file)
|
|
213
|
-
let dirStat
|
|
214
|
-
try {
|
|
215
|
-
dirStat = fs.lstatSync(parent)
|
|
216
|
-
} catch (err) {
|
|
217
|
-
if (isNotExistError(err)) {
|
|
218
|
-
this.mkdirs(parent)
|
|
219
|
-
} else {
|
|
220
|
-
throw new MobilettoError(`write: lstat error on ${parent}: ${err}`, err)
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
if (typeof dirStat === 'undefined') {
|
|
224
|
-
this.mkdirs(parent)
|
|
225
|
-
} else if (!dirStat.isDirectory()) {
|
|
226
|
-
throw new MobilettoError(`write: not a directory: ${parent} (cannot write file ${file})`)
|
|
227
|
-
}
|
|
228
|
-
return file
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
async write (path, generatorOrReadableStream) {
|
|
232
|
-
const file = await this.normalizePathAndEnsureParentDirs(path)
|
|
233
|
-
logger.debug(`write: writing path ${path} -> ${file}`)
|
|
234
|
-
const stream = fs.createWriteStream(file, {mode: this.fileMode})
|
|
235
|
-
const writer = writeStream(stream)
|
|
236
|
-
const closer = closeStream(stream)
|
|
237
|
-
let count = 0
|
|
238
|
-
|
|
239
|
-
if (isReadable(generatorOrReadableStream)) {
|
|
240
|
-
const readable = generatorOrReadableStream
|
|
241
|
-
const streamHandler = stream =>
|
|
242
|
-
new Promise((resolve, reject) => {
|
|
243
|
-
stream.on('data', (data) => {
|
|
244
|
-
if (data) {
|
|
245
|
-
writer(data)
|
|
246
|
-
count += data.length
|
|
247
|
-
}
|
|
248
|
-
})
|
|
249
|
-
stream.on('error', reject)
|
|
250
|
-
stream.on('end', () => {
|
|
251
|
-
closer()
|
|
252
|
-
resolve()
|
|
253
|
-
})
|
|
254
|
-
})
|
|
255
|
-
await streamHandler(readable)
|
|
256
|
-
return count
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
const generator = generatorOrReadableStream
|
|
260
|
-
let chunk = isAsyncGenerator(generator)
|
|
261
|
-
? (await generator.next()).value
|
|
262
|
-
: generator.next().value
|
|
263
|
-
let nullCount = 0
|
|
264
|
-
while (chunk || nullCount < 5) {
|
|
265
|
-
if (chunk) {
|
|
266
|
-
count += chunk.length
|
|
267
|
-
writer(chunk)
|
|
268
|
-
} else {
|
|
269
|
-
nullCount++
|
|
270
|
-
}
|
|
271
|
-
chunk = isAsyncGenerator(generator)
|
|
272
|
-
? (await generator.next()).value
|
|
273
|
-
: generator.next().value
|
|
274
|
-
}
|
|
275
|
-
closer()
|
|
276
|
-
return count
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
async read (path, callback, endCallback = null) {
|
|
280
|
-
const file = this.normalizePath(path)
|
|
281
|
-
logger.debug(`read: reading path: ${path} - ${file}`)
|
|
282
|
-
try {
|
|
283
|
-
const stream = fs.createReadStream(file)
|
|
284
|
-
return await readStream(stream, callback, endCallback)
|
|
285
|
-
} catch (err) {
|
|
286
|
-
throw this.ioError(err, path, 'read')
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
async remove (path, recursive, quiet) {
|
|
291
|
-
const file = this.normalizePath(path)
|
|
292
|
-
logger.debug(`remove: deleting path: ${path} = ${file}`)
|
|
293
|
-
try {
|
|
294
|
-
fs.rmSync(file, {recursive: recursive, force: quiet, maxRetries: 2})
|
|
295
|
-
} catch (err) {
|
|
296
|
-
if (isNotExistError(err)) {
|
|
297
|
-
if (!quiet) {
|
|
298
|
-
throw new MobilettoNotFoundError(path)
|
|
299
|
-
}
|
|
300
|
-
} else {
|
|
301
|
-
throw new MobilettoError(`remove(${path}) error: ${err}`, err)
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return path
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
function storageClient (key, secret, opts) {
|
|
309
|
-
if (!key) {
|
|
310
|
-
throw new MobilettoError('local.storageClient: key is required')
|
|
311
|
-
}
|
|
312
|
-
return new StorageClient(key, opts)
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
module.exports = { storageClient }
|
package/drivers/s3/index.js
DELETED
|
@@ -1,316 +0,0 @@
|
|
|
1
|
-
const {
|
|
2
|
-
M_DIR, M_FILE,
|
|
3
|
-
MobilettoError, MobilettoNotFoundError, readStream
|
|
4
|
-
} = require('../../index')
|
|
5
|
-
|
|
6
|
-
const { logger } = require('../../util/logger')
|
|
7
|
-
|
|
8
|
-
const { dirname } = require('path')
|
|
9
|
-
|
|
10
|
-
const {
|
|
11
|
-
S3Client,
|
|
12
|
-
ListObjectsCommand,
|
|
13
|
-
HeadObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, NoSuchKey
|
|
14
|
-
} = require('@aws-sdk/client-s3')
|
|
15
|
-
|
|
16
|
-
const { Readable } = require('stream')
|
|
17
|
-
const {Upload} = require("@aws-sdk/lib-storage")
|
|
18
|
-
|
|
19
|
-
const DEFAULT_REGION = 'us-east-1'
|
|
20
|
-
const DEFAULT_PREFIX = ''
|
|
21
|
-
const DEFAULT_DELIMITER = '/'
|
|
22
|
-
|
|
23
|
-
const DELETE_OBJECTS_MAX_KEYS = 1000
|
|
24
|
-
|
|
25
|
-
class StorageClient {
|
|
26
|
-
client
|
|
27
|
-
region
|
|
28
|
-
bucket
|
|
29
|
-
prefix
|
|
30
|
-
delimiter
|
|
31
|
-
normalizeRegex
|
|
32
|
-
constructor(key, secret, opts) {
|
|
33
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
34
|
-
throw new MobilettoError('s3.StorageClient: key, secret, and opts.bucket are required')
|
|
35
|
-
}
|
|
36
|
-
this.bucket = opts.bucket
|
|
37
|
-
const delim = this.delimiter = opts.delimiter || DEFAULT_DELIMITER
|
|
38
|
-
this.normalizeRegex = new RegExp(`${this.delimiter}{2,}`, 'g')
|
|
39
|
-
this.prefix = opts.prefix || DEFAULT_PREFIX
|
|
40
|
-
? opts.prefix.endsWith(delim)
|
|
41
|
-
? opts.prefix
|
|
42
|
-
: opts.prefix + delim
|
|
43
|
-
: ''
|
|
44
|
-
this.region = opts.region || DEFAULT_REGION
|
|
45
|
-
const credentials = {
|
|
46
|
-
accessKeyId: key,
|
|
47
|
-
secretAccessKey: secret
|
|
48
|
-
}
|
|
49
|
-
this.client = new S3Client({region: this.region, credentials})
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
// noinspection JSUnusedGlobalSymbols -- called by driver init
|
|
53
|
-
testConfig = async () => await this._list('', false, null, { MaxKeys: 1 })
|
|
54
|
-
|
|
55
|
-
stripPrefix = (name) => name.startsWith(this.prefix) ? name.substring(this.prefix.length) : name
|
|
56
|
-
|
|
57
|
-
nameToObj = (name) => {
|
|
58
|
-
const relName = this.stripPrefix(name)
|
|
59
|
-
return {
|
|
60
|
-
name: relName,
|
|
61
|
-
type: relName.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
async list (path = '', recursive = false, visitor = null) {
|
|
66
|
-
try {
|
|
67
|
-
return await this._list(path, recursive, visitor)
|
|
68
|
-
} catch (e) {
|
|
69
|
-
if (e instanceof MobilettoNotFoundError && !recursive && path.includes(this.delimiter)) {
|
|
70
|
-
// are we trying to list a single file?
|
|
71
|
-
const objects = await this._list(dirname(path), false)
|
|
72
|
-
const found = objects.find(o => o.name === path)
|
|
73
|
-
if (found) {
|
|
74
|
-
return [found]
|
|
75
|
-
}
|
|
76
|
-
throw e
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async _list (path, recursive = false, visitor = null, params = {}) {
|
|
82
|
-
const logPrefix = `_list(path=${path})`
|
|
83
|
-
|
|
84
|
-
// Declare truncated as a flag that the while loop is based on.
|
|
85
|
-
let truncated = true
|
|
86
|
-
|
|
87
|
-
const Prefix = this.prefix +
|
|
88
|
-
(path.startsWith(this.delimiter) ? path.substring(0) : path) +
|
|
89
|
-
(path.length === 0 || path.endsWith(this.delimiter) ? '' : this.delimiter)
|
|
90
|
-
const bucketParams = Object.assign({}, params, {
|
|
91
|
-
Region: this.region,
|
|
92
|
-
Bucket: this.bucket,
|
|
93
|
-
Prefix
|
|
94
|
-
})
|
|
95
|
-
if (!recursive) {
|
|
96
|
-
bucketParams.Delimiter = this.delimiter
|
|
97
|
-
}
|
|
98
|
-
const objects = []
|
|
99
|
-
let objectCount = 0
|
|
100
|
-
logger.debug(`${logPrefix} bucketParams=${JSON.stringify(bucketParams)}`)
|
|
101
|
-
|
|
102
|
-
// while loop that runs until 'response.truncated' is false.
|
|
103
|
-
while (truncated) {
|
|
104
|
-
try {
|
|
105
|
-
const response = await this.client.send(new ListObjectsCommand(bucketParams))
|
|
106
|
-
const hasContents = typeof response.Contents !== 'undefined'
|
|
107
|
-
if (hasContents) {
|
|
108
|
-
for (const item of response.Contents) {
|
|
109
|
-
const obj = this.nameToObj(item.Key);
|
|
110
|
-
if (visitor) {
|
|
111
|
-
await visitor(obj)
|
|
112
|
-
}
|
|
113
|
-
objects.push(obj)
|
|
114
|
-
objectCount++
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
const hasCommonPrefixes = typeof response.CommonPrefixes !== 'undefined'
|
|
118
|
-
if (hasCommonPrefixes) {
|
|
119
|
-
for (const item of response.CommonPrefixes) {
|
|
120
|
-
const obj = this.nameToObj(item.Prefix)
|
|
121
|
-
if (visitor) {
|
|
122
|
-
await visitor(obj)
|
|
123
|
-
}
|
|
124
|
-
objects.push(obj)
|
|
125
|
-
objectCount++
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
truncated = response.IsTruncated
|
|
129
|
-
// If truncated is true, advance the marker
|
|
130
|
-
if (truncated) {
|
|
131
|
-
bucketParams.Marker = response.NextMarker
|
|
132
|
-
} else if (!hasContents && !hasCommonPrefixes) {
|
|
133
|
-
if (path === '') {
|
|
134
|
-
break
|
|
135
|
-
}
|
|
136
|
-
throw new MobilettoNotFoundError(path)
|
|
137
|
-
}
|
|
138
|
-
} catch (err) {
|
|
139
|
-
if (err instanceof MobilettoNotFoundError) {
|
|
140
|
-
throw err
|
|
141
|
-
}
|
|
142
|
-
throw new MobilettoError(`${logPrefix} Error: ${err}`)
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
if (recursive && objectCount === 0 && path !== '') {
|
|
146
|
-
throw new MobilettoNotFoundError(path)
|
|
147
|
-
}
|
|
148
|
-
const filtered = objects.filter(o => o.name !== path)
|
|
149
|
-
return filtered
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
normalizeKey = (path) => {
|
|
153
|
-
const p = (path.startsWith(this.prefix)
|
|
154
|
-
? path
|
|
155
|
-
: this.prefix + (path.startsWith(this.delimiter) ? path.substring(1) : path))
|
|
156
|
-
return p.replaceAll(this.normalizeRegex, this.delimiter)
|
|
157
|
-
}
|
|
158
|
-
denormalizeKey = (key) => {
|
|
159
|
-
return key.startsWith(this.prefix)
|
|
160
|
-
? key.substring(this.prefix.length)
|
|
161
|
-
: key
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
s3error (err, key, path, method) {
|
|
165
|
-
return (err instanceof MobilettoError || err instanceof MobilettoNotFoundError)
|
|
166
|
-
? err
|
|
167
|
-
: (err instanceof NoSuchKey) || (err.name && err.name === 'NotFound')
|
|
168
|
-
? new MobilettoNotFoundError(this.denormalizeKey(key))
|
|
169
|
-
: new MobilettoError(`${method}(${path}) error: ${err}`, err)
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async metadata (path) {
|
|
173
|
-
const Key = this.normalizeKey(path)
|
|
174
|
-
const bucketParams = {
|
|
175
|
-
Region: this.region,
|
|
176
|
-
Bucket: this.bucket,
|
|
177
|
-
Key,
|
|
178
|
-
Delimiter: this.delimiter
|
|
179
|
-
}
|
|
180
|
-
try {
|
|
181
|
-
const head = await this.client.send(new HeadObjectCommand(bucketParams))
|
|
182
|
-
const meta = {
|
|
183
|
-
name: this.stripPrefix(path),
|
|
184
|
-
size: head.ContentLength,
|
|
185
|
-
type: path.endsWith(this.delimiter) ? M_DIR : M_FILE
|
|
186
|
-
}
|
|
187
|
-
if (head.LastModified) {
|
|
188
|
-
meta.mtime = Date.parse(head.LastModified)
|
|
189
|
-
}
|
|
190
|
-
return meta
|
|
191
|
-
} catch (err) {
|
|
192
|
-
throw this.s3error(err, Key, path, 'metadata')
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
async write (path, generator) {
|
|
197
|
-
const Key = this.normalizeKey(path)
|
|
198
|
-
const bucketParams = {
|
|
199
|
-
Region: this.region,
|
|
200
|
-
Bucket: this.bucket,
|
|
201
|
-
Key,
|
|
202
|
-
Body: Readable.from(generator),
|
|
203
|
-
Delimiter: this.delimiter
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
const uploader = new Upload({
|
|
207
|
-
client: this.client,
|
|
208
|
-
params: bucketParams,
|
|
209
|
-
queueSize: 1, // optional concurrency configuration
|
|
210
|
-
partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
|
|
211
|
-
leavePartsOnError: false // optional manually handle dropped parts
|
|
212
|
-
})
|
|
213
|
-
let total = 0
|
|
214
|
-
uploader.on('httpUploadProgress', (progress) => {
|
|
215
|
-
logger.debug(`write(${bucketParams.Key}) ${JSON.stringify(progress)}`)
|
|
216
|
-
total += progress.loaded
|
|
217
|
-
})
|
|
218
|
-
const response = await uploader.done()
|
|
219
|
-
if (response.Key !== Key) {
|
|
220
|
-
throw new MobilettoError(`s3.write: after writing, expected Key=${Key} but found response.Key=${response.Key}`)
|
|
221
|
-
}
|
|
222
|
-
return total
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
async read (path, callback, endCallback = null) {
|
|
226
|
-
const Key = this.normalizeKey(path)
|
|
227
|
-
logger.debug(`read: reading Key: ${path} - ${Key}`)
|
|
228
|
-
const bucketParams = {
|
|
229
|
-
Region: this.region,
|
|
230
|
-
Bucket: this.bucket,
|
|
231
|
-
Key,
|
|
232
|
-
Delimiter: this.delimiter
|
|
233
|
-
}
|
|
234
|
-
try {
|
|
235
|
-
const data = await this.client.send(new GetObjectCommand(bucketParams))
|
|
236
|
-
return await readStream(data.Body, callback, endCallback)
|
|
237
|
-
} catch (err) {
|
|
238
|
-
throw this.s3error(err, Key, path, 'read')
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
async remove (path, recursive, quiet) {
|
|
243
|
-
if (recursive) {
|
|
244
|
-
const removed = []
|
|
245
|
-
let objects = await this._list(path, true, null, { MaxKeys: DELETE_OBJECTS_MAX_KEYS })
|
|
246
|
-
while (objects && objects.length > 0) {
|
|
247
|
-
const Delete = {
|
|
248
|
-
Objects: objects.map(obj => { return {Key: this.normalizeKey(obj.name)} })
|
|
249
|
-
}
|
|
250
|
-
if (quiet) {
|
|
251
|
-
Delete.Quiet = true
|
|
252
|
-
}
|
|
253
|
-
const bucketParams = {
|
|
254
|
-
Region: this.region,
|
|
255
|
-
Bucket: this.bucket,
|
|
256
|
-
Delete
|
|
257
|
-
}
|
|
258
|
-
logger.debug(`remove(${path}) deleting objects: ${JSON.stringify(objects)}`)
|
|
259
|
-
const response = await this.client.send(new DeleteObjectsCommand(bucketParams))
|
|
260
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
261
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
262
|
-
if (statusClass !== 2) {
|
|
263
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned HTTP status ${statusCode}`)
|
|
264
|
-
}
|
|
265
|
-
if (!quiet && response.Errors && response.Errors.length > 0) {
|
|
266
|
-
throw new MobilettoError(`remove(${path}) DeleteObjectsCommand returned Errors: ${JSON.stringify(response.Errors)}`)
|
|
267
|
-
}
|
|
268
|
-
if (response.Deleted) {
|
|
269
|
-
removed.push(...response.Deleted.map(del => this.denormalizeKey(del.Key)))
|
|
270
|
-
}
|
|
271
|
-
try {
|
|
272
|
-
objects = await this._list(path, true, null, {MaxKeys: DELETE_OBJECTS_MAX_KEYS})
|
|
273
|
-
} catch (e) {
|
|
274
|
-
if (!(e instanceof MobilettoNotFoundError)) {
|
|
275
|
-
throw e instanceof MobilettoError ? e : new MobilettoError(`remove(${path}) error listing: ${e}`)
|
|
276
|
-
}
|
|
277
|
-
objects = null
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
return removed
|
|
281
|
-
} else {
|
|
282
|
-
const Key = this.normalizeKey(path)
|
|
283
|
-
const bucketParams = {
|
|
284
|
-
Region: this.region,
|
|
285
|
-
Bucket: this.bucket,
|
|
286
|
-
Key
|
|
287
|
-
}
|
|
288
|
-
try {
|
|
289
|
-
// DeleteObjectCommand silently succeeds and returns HTTP 204 even for non-existent Keys
|
|
290
|
-
// Thus, if quiet is false, we must check metadata explicitly, which will fail with
|
|
291
|
-
// MobilettoNotFoundError, which is the correct behavior
|
|
292
|
-
if (!quiet) {
|
|
293
|
-
await this.metadata(path)
|
|
294
|
-
}
|
|
295
|
-
const response = await this.client.send(new DeleteObjectCommand(bucketParams))
|
|
296
|
-
let statusCode = response.$metadata.httpStatusCode
|
|
297
|
-
let statusClass = Math.floor(statusCode / 100)
|
|
298
|
-
if (statusClass !== 2) {
|
|
299
|
-
throw new MobilettoError(`remove: DeleteObjectCommand returned HTTP status ${statusCode}`)
|
|
300
|
-
}
|
|
301
|
-
} catch (err) {
|
|
302
|
-
throw this.s3error(err, Key, path, 'remove')
|
|
303
|
-
}
|
|
304
|
-
return path
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
function storageClient (key, secret, opts) {
|
|
310
|
-
if (!key || !secret || !opts || !opts.bucket) {
|
|
311
|
-
throw new MobilettoError('s3.storageClient: key, secret, and opts.bucket are required')
|
|
312
|
-
}
|
|
313
|
-
return new StorageClient(key, secret, opts)
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
module.exports = { storageClient }
|
package/hokey.json
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"inputLanguage": "en",
|
|
3
|
-
"languages": "all",
|
|
4
|
-
"hokey": [
|
|
5
|
-
{
|
|
6
|
-
"name": "README",
|
|
7
|
-
"infile": "README.md",
|
|
8
|
-
"outfile": "docs/LANG/",
|
|
9
|
-
"excludes": ["docs/", "node_modules/", "\\.git/", "tmp/"],
|
|
10
|
-
"filter": "relativizeMarkdownLinks docs",
|
|
11
|
-
"force": true,
|
|
12
|
-
"markdown": true,
|
|
13
|
-
"index": "docs/README.md"
|
|
14
|
-
}
|
|
15
|
-
]
|
|
16
|
-
}
|
package/index.js
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
const {
|
|
2
|
-
registerDriver, mobiletto, connect,
|
|
3
|
-
redis, closeRedis
|
|
4
|
-
} = require('mobiletto-base')
|
|
5
|
-
|
|
6
|
-
const {
|
|
7
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
8
|
-
MobilettoError, MobilettoNotFoundError,
|
|
9
|
-
logger, setLogLevel
|
|
10
|
-
} = require('mobiletto-common')
|
|
11
|
-
|
|
12
|
-
const registerDrivers = () => {
|
|
13
|
-
registerDriver('local', require('mobiletto-driver-local'))
|
|
14
|
-
registerDriver('s3', require('mobiletto-driver-s3'))
|
|
15
|
-
registerDriver('b2', require('mobiletto-driver-b2'))
|
|
16
|
-
registerDriver('indexeddb', require('mobiletto-driver-indexeddb'))
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
module.exports = {
|
|
20
|
-
M_FILE, M_DIR, M_LINK, M_SPECIAL,
|
|
21
|
-
MobilettoError, MobilettoNotFoundError,
|
|
22
|
-
logger, setLogLevel,
|
|
23
|
-
registerDrivers,
|
|
24
|
-
registerDriver,
|
|
25
|
-
mobiletto,
|
|
26
|
-
connect,
|
|
27
|
-
redis,
|
|
28
|
-
closeRedis
|
|
29
|
-
}
|