mbkbucket 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.js +1 -1
- package/lib/src/app.js +13 -25
- package/lib/src/controllers/api.controller.js +2 -1
- package/lib/src/controllers/dashboard.controller.js +2 -2
- package/lib/src/controllers/info.controller.js +2 -1
- package/lib/src/controllers/view.controller.js +2 -1
- package/lib/src/core/metrics.js +134 -134
- package/lib/src/core/storage/capabilities.js +58 -58
- package/lib/src/core/storage/errors.js +57 -57
- package/lib/src/core/storage/gdrive-auth.js +157 -157
- package/lib/src/core/storage/gdrive-provider.js +715 -714
- package/lib/src/core/storage/index.js +17 -17
- package/lib/src/core/storage/local-provider.js +507 -506
- package/lib/src/core/storage/models.js +121 -120
- package/lib/src/core/storage/path-strategy.js +85 -78
- package/lib/src/core/storage/s3-provider.js +472 -471
- package/lib/src/core/storage/storage-manager.js +226 -223
- package/lib/src/core/storage/storage-provider.js +146 -146
- package/lib/src/middleware/error-handler.js +15 -52
- package/lib/src/middleware/rate-limiter.js +1 -1
- package/lib/src/services/storage.service.js +1 -1
- package/lib/src/utils/cleanup-scheduler.js +91 -91
- package/lib/src/utils/helpers.js +2 -45
- package/package.json +3 -2
- package/views/bucket.hbs +4 -4
- package/views/bucketadmincontent.hbs +2 -2
- package/views/mbkbucket_info.hbs +4 -4
|
@@ -1,471 +1,472 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* MBKBucket - S3 / Cloudflare R2 / MinIO / iDrive E2 Storage Provider
|
|
3
|
-
* Copyright (c) 2026 Muhammad Bin Khalid, MBKTech.org and contributors
|
|
4
|
-
* Licensed under the MIT License.
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand, CopyObjectCommand } from '@aws-sdk/client-s3';
|
|
8
|
-
import { StorageProvider } from './storage-provider.js';
|
|
9
|
-
import { StorageCapabilities } from './capabilities.js';
|
|
10
|
-
import { StorageFile, StorageFolder, StorageListResult } from './models.js';
|
|
11
|
-
import { StorageNotFoundError, StorageConflictError, StorageAccessDeniedError, StorageValidationError, StorageError } from './errors.js';
|
|
12
|
-
import { createLogger } from "#logger";
|
|
13
|
-
import { nowIso
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
this.
|
|
22
|
-
this.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
'
|
|
114
|
-
'
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
...(
|
|
152
|
-
...(
|
|
153
|
-
...(
|
|
154
|
-
...(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
if (error.name === '
|
|
166
|
-
if (error
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
...(
|
|
273
|
-
...(
|
|
274
|
-
...(
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
'
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
cutoffDate
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* MBKBucket - S3 / Cloudflare R2 / MinIO / iDrive E2 Storage Provider
|
|
3
|
+
* Copyright (c) 2026 Muhammad Bin Khalid, MBKTech.org and contributors
|
|
4
|
+
* Licensed under the MIT License.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand, CopyObjectCommand } from '@aws-sdk/client-s3';
|
|
8
|
+
import { StorageProvider } from './storage-provider.js';
|
|
9
|
+
import { StorageCapabilities } from './capabilities.js';
|
|
10
|
+
import { StorageFile, StorageFolder, StorageListResult } from './models.js';
|
|
11
|
+
import { StorageNotFoundError, StorageConflictError, StorageAccessDeniedError, StorageValidationError, StorageError } from './errors.js';
|
|
12
|
+
import { createLogger } from "#logger";
|
|
13
|
+
import { nowIso } from "#helpers";
|
|
14
|
+
import { trimSlashes } from "mbkutils";
|
|
15
|
+
|
|
16
|
+
const debugS3 = createLogger('s3-provider');
|
|
17
|
+
|
|
18
|
+
export class S3StorageProvider extends StorageProvider {
|
|
19
|
+
constructor(name, config = {}) {
|
|
20
|
+
super(name, config);
|
|
21
|
+
this.bucket = config.BUCKET_NAME;
|
|
22
|
+
this.region = config.region || 'auto';
|
|
23
|
+
this.client = new S3Client({
|
|
24
|
+
region: this.region,
|
|
25
|
+
endpoint: config.ENDPOINT,
|
|
26
|
+
credentials: {
|
|
27
|
+
accessKeyId: config.ACCESS_KEY_ID,
|
|
28
|
+
secretAccessKey: config.SECRET_ACCESS_KEY,
|
|
29
|
+
},
|
|
30
|
+
maxAttempts: 3,
|
|
31
|
+
retryMode: 'adaptive',
|
|
32
|
+
requestTimeout: 120000,
|
|
33
|
+
requestHandler: {
|
|
34
|
+
connectionTimeout: 5000,
|
|
35
|
+
socketTimeout: 120000,
|
|
36
|
+
maxSockets: 50,
|
|
37
|
+
keepAlive: true,
|
|
38
|
+
keepAliveMsecs: 1000,
|
|
39
|
+
},
|
|
40
|
+
useAccelerateEndpoint: false,
|
|
41
|
+
forcePathStyle: true,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
get type() {
|
|
46
|
+
return 's3';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
get capabilities() {
|
|
50
|
+
return new StorageCapabilities({
|
|
51
|
+
multipart: true,
|
|
52
|
+
resumable: true,
|
|
53
|
+
nativeFolders: false,
|
|
54
|
+
ranges: true,
|
|
55
|
+
copy: true,
|
|
56
|
+
move: true,
|
|
57
|
+
search: false,
|
|
58
|
+
directDownloadUrl: false,
|
|
59
|
+
versioning: false,
|
|
60
|
+
publicUrls: false,
|
|
61
|
+
metadata: true
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async checkHealth() {
|
|
66
|
+
const startTime = Date.now();
|
|
67
|
+
try {
|
|
68
|
+
await this.client.send(new ListObjectsV2Command({
|
|
69
|
+
Bucket: this.bucket,
|
|
70
|
+
MaxKeys: 1
|
|
71
|
+
}));
|
|
72
|
+
return {
|
|
73
|
+
status: 'healthy',
|
|
74
|
+
providerType: 's3',
|
|
75
|
+
responseTime: Date.now() - startTime,
|
|
76
|
+
bucket: this.bucket,
|
|
77
|
+
region: this.region,
|
|
78
|
+
checkedAt: nowIso()
|
|
79
|
+
};
|
|
80
|
+
} catch (error) {
|
|
81
|
+
return {
|
|
82
|
+
status: 'unhealthy',
|
|
83
|
+
providerType: 's3',
|
|
84
|
+
error: error.message,
|
|
85
|
+
bucket: this.bucket || 'unknown',
|
|
86
|
+
checkedAt: nowIso()
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
92
|
+
try {
|
|
93
|
+
if (!key || !fileBuffer) throw new StorageValidationError('Key and file buffer are required');
|
|
94
|
+
|
|
95
|
+
const {
|
|
96
|
+
metadata = {},
|
|
97
|
+
cacheControl = 'public, max-age=31536000',
|
|
98
|
+
storageClass = 'STANDARD',
|
|
99
|
+
serverSideEncryption = 'AES256',
|
|
100
|
+
preventOverwrite = false
|
|
101
|
+
} = options;
|
|
102
|
+
|
|
103
|
+
debugS3('Uploading file to bucket=%s key=%s', this.bucket, key);
|
|
104
|
+
const uploadedAt = nowIso();
|
|
105
|
+
|
|
106
|
+
const commandParams = {
|
|
107
|
+
Bucket: this.bucket,
|
|
108
|
+
Key: key,
|
|
109
|
+
Body: fileBuffer,
|
|
110
|
+
ContentType: contentType,
|
|
111
|
+
CacheControl: cacheControl,
|
|
112
|
+
Metadata: {
|
|
113
|
+
'uploaded-at': uploadedAt,
|
|
114
|
+
'file-size': fileBuffer.length.toString(),
|
|
115
|
+
'upload-source': 'web-portal',
|
|
116
|
+
...metadata
|
|
117
|
+
},
|
|
118
|
+
ServerSideEncryption: serverSideEncryption,
|
|
119
|
+
StorageClass: storageClass,
|
|
120
|
+
...(preventOverwrite && { IfNoneMatch: '*' })
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const result = await this.client.send(new PutObjectCommand(commandParams));
|
|
124
|
+
return {
|
|
125
|
+
...result,
|
|
126
|
+
fileSize: fileBuffer.length,
|
|
127
|
+
key,
|
|
128
|
+
contentType,
|
|
129
|
+
uploadedAt
|
|
130
|
+
};
|
|
131
|
+
} catch (error) {
|
|
132
|
+
debugS3('Upload failed for key %s: %s', key, error.message);
|
|
133
|
+
if (error.name === 'PreconditionFailed' || error.$metadata?.httpStatusCode === 412) {
|
|
134
|
+
throw new StorageConflictError('File already exists');
|
|
135
|
+
}
|
|
136
|
+
if (error instanceof StorageError) throw error;
|
|
137
|
+
throw new StorageError(`Upload failed: ${error.message}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async downloadFile(key, options = {}) {
|
|
142
|
+
try {
|
|
143
|
+
if (!key) throw new StorageValidationError('Key is required');
|
|
144
|
+
|
|
145
|
+
const { range, ifNoneMatch, ifModifiedSince, responseCacheControl, responseContentType } = options;
|
|
146
|
+
if (range) debugS3('Range request for %s: %s', key, range);
|
|
147
|
+
|
|
148
|
+
const result = await this.client.send(new GetObjectCommand({
|
|
149
|
+
Bucket: this.bucket,
|
|
150
|
+
Key: key,
|
|
151
|
+
...(range && { Range: range }),
|
|
152
|
+
...(ifNoneMatch && { IfNoneMatch: ifNoneMatch }),
|
|
153
|
+
...(ifModifiedSince && { IfModifiedSince: ifModifiedSince }),
|
|
154
|
+
...(responseCacheControl && { ResponseCacheControl: responseCacheControl }),
|
|
155
|
+
...(responseContentType && { ResponseContentType: responseContentType }),
|
|
156
|
+
}));
|
|
157
|
+
|
|
158
|
+
return { ...result, key };
|
|
159
|
+
} catch (error) {
|
|
160
|
+
if (error?.$metadata?.httpStatusCode === 304 || error?.name === '304' || error?.name === 'NotModified') {
|
|
161
|
+
debugS3('File not modified (304) for key %s', key);
|
|
162
|
+
return { notModified: true, key };
|
|
163
|
+
}
|
|
164
|
+
debugS3('Download failed for key %s: %s', key, error.message);
|
|
165
|
+
if (error.name === 'NoSuchKey' || error.name === 'NotFound') throw new StorageNotFoundError(`File not found: ${key}`);
|
|
166
|
+
if (error.name === 'AccessDenied') throw new StorageAccessDeniedError(`Access denied for file: ${key}`);
|
|
167
|
+
if (error instanceof StorageError) throw error;
|
|
168
|
+
throw new StorageError(`Download failed: ${error.message}`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async deleteFile(key, _options = {}) {
|
|
173
|
+
try {
|
|
174
|
+
if (!key) throw new StorageValidationError('Key is required');
|
|
175
|
+
|
|
176
|
+
const result = await this.client.send(new DeleteObjectCommand({
|
|
177
|
+
Bucket: this.bucket,
|
|
178
|
+
Key: key,
|
|
179
|
+
}));
|
|
180
|
+
|
|
181
|
+
return { ...result, key, deletedAt: nowIso() };
|
|
182
|
+
} catch (error) {
|
|
183
|
+
debugS3('Delete failed for %s: %s', key, error.message);
|
|
184
|
+
if (error instanceof StorageError) throw error;
|
|
185
|
+
throw new StorageError(`Delete failed: ${error.message}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async deleteFiles(keys, _options = {}) {
|
|
190
|
+
try {
|
|
191
|
+
if (!keys || !Array.isArray(keys) || !keys.length) {
|
|
192
|
+
throw new StorageValidationError('Keys array is required and must not be empty');
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const maxBatchSize = 1000;
|
|
196
|
+
const results = [];
|
|
197
|
+
let deletedCount = 0;
|
|
198
|
+
const errors = [];
|
|
199
|
+
|
|
200
|
+
for (let i = 0; i < keys.length; i += maxBatchSize) {
|
|
201
|
+
const batch = keys.slice(i, i + maxBatchSize).map(k => ({ Key: k }));
|
|
202
|
+
const result = await this.client.send(new DeleteObjectsCommand({
|
|
203
|
+
Bucket: this.bucket,
|
|
204
|
+
Delete: { Objects: batch, Quiet: false }
|
|
205
|
+
}));
|
|
206
|
+
|
|
207
|
+
results.push(result);
|
|
208
|
+
deletedCount += result.Deleted?.length || 0;
|
|
209
|
+
if (result.Errors?.length) errors.push(...result.Errors);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
return { results, deletedCount, errors, deletedAt: nowIso() };
|
|
213
|
+
} catch (error) {
|
|
214
|
+
debugS3('Batch delete failed: %s', error.message);
|
|
215
|
+
if (error instanceof StorageError) throw error;
|
|
216
|
+
throw new StorageError(`Batch delete failed: ${error.message}`);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async deleteFolder(prefix, _options = {}) {
|
|
221
|
+
try {
|
|
222
|
+
if (!prefix) throw new StorageValidationError('Prefix is required');
|
|
223
|
+
const cleanPrefix = trimSlashes(prefix);
|
|
224
|
+
const effectivePrefix = cleanPrefix ? `${cleanPrefix}/` : '';
|
|
225
|
+
|
|
226
|
+
const keysToDelete = [];
|
|
227
|
+
let continuationToken = null;
|
|
228
|
+
|
|
229
|
+
do {
|
|
230
|
+
const resp = await this.client.send(new ListObjectsV2Command({
|
|
231
|
+
Bucket: this.bucket,
|
|
232
|
+
Prefix: effectivePrefix,
|
|
233
|
+
ContinuationToken: continuationToken,
|
|
234
|
+
MaxKeys: 1000
|
|
235
|
+
}));
|
|
236
|
+
(resp.Contents || []).forEach(obj => { if (obj.Key) keysToDelete.push(obj.Key); });
|
|
237
|
+
continuationToken = resp.IsTruncated ? resp.NextContinuationToken : null;
|
|
238
|
+
} while (continuationToken);
|
|
239
|
+
|
|
240
|
+
if (!keysToDelete.length) {
|
|
241
|
+
const exists = await this.fileExists(effectivePrefix);
|
|
242
|
+
if (exists) {
|
|
243
|
+
await this.deleteFile(effectivePrefix);
|
|
244
|
+
return { deletedCount: 1, deletedAt: nowIso(), prefix: effectivePrefix };
|
|
245
|
+
}
|
|
246
|
+
return { deletedCount: 0, deletedAt: nowIso(), prefix: effectivePrefix };
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const result = await this.deleteFiles(keysToDelete);
|
|
250
|
+
return { ...result, prefix: effectivePrefix, deletedAt: nowIso() };
|
|
251
|
+
} catch (error) {
|
|
252
|
+
debugS3('Delete folder failed for %s: %s', prefix, error.message);
|
|
253
|
+
if (error instanceof StorageError) throw error;
|
|
254
|
+
throw new StorageError(`Delete folder failed: ${error.message}`);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async listFiles(prefix = '', options = {}) {
|
|
259
|
+
try {
|
|
260
|
+
const {
|
|
261
|
+
maxKeys = 1000,
|
|
262
|
+
continuationToken = null,
|
|
263
|
+
delimiter = null,
|
|
264
|
+
fetchOwner = false,
|
|
265
|
+
startAfter = null
|
|
266
|
+
} = options;
|
|
267
|
+
|
|
268
|
+
const result = await this.client.send(new ListObjectsV2Command({
|
|
269
|
+
Bucket: this.bucket,
|
|
270
|
+
Prefix: prefix || '',
|
|
271
|
+
MaxKeys: maxKeys,
|
|
272
|
+
...(continuationToken && { ContinuationToken: continuationToken }),
|
|
273
|
+
...(delimiter && { Delimiter: delimiter }),
|
|
274
|
+
...(fetchOwner && { FetchOwner: fetchOwner }),
|
|
275
|
+
...(startAfter && { StartAfter: startAfter }),
|
|
276
|
+
}));
|
|
277
|
+
|
|
278
|
+
const contents = (result.Contents || []).map(item => new StorageFile({
|
|
279
|
+
key: item.Key,
|
|
280
|
+
size: item.Size,
|
|
281
|
+
lastModified: item.LastModified,
|
|
282
|
+
etag: item.ETag,
|
|
283
|
+
storageClass: item.StorageClass || 'STANDARD',
|
|
284
|
+
provider: 's3'
|
|
285
|
+
}));
|
|
286
|
+
|
|
287
|
+
const commonPrefixes = (result.CommonPrefixes || []).map(p => new StorageFolder(p.Prefix));
|
|
288
|
+
|
|
289
|
+
return new StorageListResult({
|
|
290
|
+
contents,
|
|
291
|
+
commonPrefixes,
|
|
292
|
+
nextToken: result.NextContinuationToken || null,
|
|
293
|
+
isTruncated: Boolean(result.IsTruncated),
|
|
294
|
+
totalFiles: result.KeyCount || contents.length
|
|
295
|
+
});
|
|
296
|
+
} catch (error) {
|
|
297
|
+
debugS3('List files failed for prefix %s: %s', prefix, error.message);
|
|
298
|
+
if (error instanceof StorageError) throw error;
|
|
299
|
+
throw new StorageError(`List files failed: ${error.message}`);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async getFileMetadata(key, _options = {}) {
|
|
304
|
+
try {
|
|
305
|
+
if (!key) throw new StorageValidationError('Key is required');
|
|
306
|
+
|
|
307
|
+
const result = await this.client.send(new HeadObjectCommand({
|
|
308
|
+
Bucket: this.bucket,
|
|
309
|
+
Key: key,
|
|
310
|
+
}));
|
|
311
|
+
|
|
312
|
+
return { ...result, key, exists: true, queriedAt: nowIso() };
|
|
313
|
+
} catch (error) {
|
|
314
|
+
const queriedAt = nowIso();
|
|
315
|
+
if (error.name === 'NotFound' || error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
|
|
316
|
+
return { key, exists: false, queriedAt };
|
|
317
|
+
}
|
|
318
|
+
debugS3('Get metadata failed for key %s: %s', key, error.message);
|
|
319
|
+
if (error instanceof StorageError) throw error;
|
|
320
|
+
throw new StorageError(`Get metadata failed: ${error.message}`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
async createFolder(key, _options = {}) {
|
|
325
|
+
let folderKey = key;
|
|
326
|
+
if (!folderKey.endsWith('/')) folderKey += '/';
|
|
327
|
+
|
|
328
|
+
if (await this.fileExists(folderKey)) {
|
|
329
|
+
return { key: folderKey, success: true, message: 'Folder already exists' };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const existingFiles = await this.listFiles(folderKey, { maxKeys: 1 });
|
|
333
|
+
if (existingFiles.Contents?.length) {
|
|
334
|
+
return { key: folderKey, success: true, message: 'Folder already exists with content', skipMarker: true };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
await this.uploadFile(folderKey, Buffer.alloc(0), 'application/x-empty', {
|
|
338
|
+
metadata: { folder: 'true', marker: 'true' }
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
return { key: folderKey, success: true, message: 'Folder created' };
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
async copyFile(sourceKey, destKey, options = {}) {
|
|
345
|
+
this.assertCapability('copy', 'copyFile');
|
|
346
|
+
try {
|
|
347
|
+
const copySource = encodeURIComponent(`${this.bucket}/${sourceKey}`);
|
|
348
|
+
await this.client.send(new CopyObjectCommand({
|
|
349
|
+
Bucket: this.bucket,
|
|
350
|
+
Key: destKey,
|
|
351
|
+
CopySource: copySource,
|
|
352
|
+
MetadataDirective: options.metadata ? 'REPLACE' : 'COPY',
|
|
353
|
+
...(options.metadata && { Metadata: options.metadata })
|
|
354
|
+
}));
|
|
355
|
+
return { sourceKey, destKey, copiedAt: nowIso() };
|
|
356
|
+
} catch (error) {
|
|
357
|
+
debugS3('Copy failed from %s to %s: %s', sourceKey, destKey, error.message);
|
|
358
|
+
if (error instanceof StorageError) throw error;
|
|
359
|
+
throw new StorageError(`Copy failed: ${error.message}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// ---------------------------------------------------------------------------
|
|
364
|
+
// S3 Multipart Upload Operations
|
|
365
|
+
// ---------------------------------------------------------------------------
|
|
366
|
+
|
|
367
|
+
async createMultipartUpload(key, contentType = 'application/octet-stream', metadata = {}) {
|
|
368
|
+
this.assertCapability('multipart', 'createMultipartUpload');
|
|
369
|
+
const result = await this.client.send(new CreateMultipartUploadCommand({
|
|
370
|
+
Bucket: this.bucket,
|
|
371
|
+
Key: key,
|
|
372
|
+
ContentType: contentType,
|
|
373
|
+
Metadata: {
|
|
374
|
+
'upload-source': 'web-portal',
|
|
375
|
+
'uploaded-at': nowIso(),
|
|
376
|
+
...metadata
|
|
377
|
+
}
|
|
378
|
+
}));
|
|
379
|
+
return { uploadId: result.UploadId, key };
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
async uploadPart(key, uploadId, partNumber, buffer) {
|
|
383
|
+
this.assertCapability('multipart', 'uploadPart');
|
|
384
|
+
const result = await this.client.send(new UploadPartCommand({
|
|
385
|
+
Bucket: this.bucket,
|
|
386
|
+
Key: key,
|
|
387
|
+
UploadId: uploadId,
|
|
388
|
+
PartNumber: partNumber,
|
|
389
|
+
Body: buffer,
|
|
390
|
+
ContentLength: buffer.length
|
|
391
|
+
}));
|
|
392
|
+
return { ETag: result.ETag, partNumber };
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
async completeMultipartUpload(key, uploadId, parts) {
|
|
396
|
+
this.assertCapability('multipart', 'completeMultipartUpload');
|
|
397
|
+
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
398
|
+
await this.client.send(new CompleteMultipartUploadCommand({
|
|
399
|
+
Bucket: this.bucket,
|
|
400
|
+
Key: key,
|
|
401
|
+
UploadId: uploadId,
|
|
402
|
+
MultipartUpload: {
|
|
403
|
+
Parts: sorted.map(p => ({ PartNumber: p.partNumber, ETag: p.ETag }))
|
|
404
|
+
}
|
|
405
|
+
}));
|
|
406
|
+
return { key };
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async abortMultipartUpload(key, uploadId) {
|
|
410
|
+
this.assertCapability('multipart', 'abortMultipartUpload');
|
|
411
|
+
await this.client.send(new AbortMultipartUploadCommand({
|
|
412
|
+
Bucket: this.bucket,
|
|
413
|
+
Key: key,
|
|
414
|
+
UploadId: uploadId
|
|
415
|
+
}));
|
|
416
|
+
return { key, abortedAt: nowIso() };
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
async listIncompleteMultipartUploads(prefix = '') {
|
|
420
|
+
this.assertCapability('multipart', 'listIncompleteMultipartUploads');
|
|
421
|
+
try {
|
|
422
|
+
const result = await this.client.send(new ListMultipartUploadsCommand({
|
|
423
|
+
Bucket: this.bucket,
|
|
424
|
+
Prefix: prefix || ''
|
|
425
|
+
}));
|
|
426
|
+
return result.Uploads || [];
|
|
427
|
+
} catch (error) {
|
|
428
|
+
debugS3('Failed to list incomplete multipart uploads: %s', error.message);
|
|
429
|
+
if (error instanceof StorageError) throw error;
|
|
430
|
+
throw new StorageError(`Failed to list incomplete uploads: ${error.message}`);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async cleanupIncompleteMultipartUploads(olderThanDays = 7, prefix = '') {
|
|
435
|
+
this.assertCapability('multipart', 'cleanupIncompleteMultipartUploads');
|
|
436
|
+
try {
|
|
437
|
+
const uploads = await this.listIncompleteMultipartUploads(prefix);
|
|
438
|
+
if (!uploads.length) {
|
|
439
|
+
debugS3('No incomplete multipart uploads found');
|
|
440
|
+
return { abortedCount: 0, uploads: [] };
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
const cutoffDate = new Date();
|
|
444
|
+
cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
|
|
445
|
+
|
|
446
|
+
const uploadsToAbort = uploads.filter(u => u.Initiated && new Date(u.Initiated) < cutoffDate);
|
|
447
|
+
if (!uploadsToAbort.length) {
|
|
448
|
+
debugS3('No incomplete uploads older than %s days', olderThanDays);
|
|
449
|
+
return { abortedCount: 0, uploads: [] };
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
debugS3('Found %s incomplete uploads to clean up', uploadsToAbort.length);
|
|
453
|
+
const aborted = [];
|
|
454
|
+
|
|
455
|
+
for (const upload of uploadsToAbort) {
|
|
456
|
+
try {
|
|
457
|
+
await this.abortMultipartUpload(upload.Key, upload.UploadId);
|
|
458
|
+
aborted.push({ key: upload.Key, uploadId: upload.UploadId, initiated: upload.Initiated });
|
|
459
|
+
debugS3('Aborted incomplete upload: %s (initiated: %s)', upload.Key, upload.Initiated);
|
|
460
|
+
} catch (err) {
|
|
461
|
+
debugS3('Failed to abort upload %s: %s', upload.Key, err.message);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
return { abortedCount: aborted.length, uploads: aborted, cleanedAt: nowIso() };
|
|
466
|
+
} catch (error) {
|
|
467
|
+
debugS3('Cleanup of incomplete multipart uploads failed: %s', error.message);
|
|
468
|
+
if (error instanceof StorageError) throw error;
|
|
469
|
+
throw new StorageError(`Cleanup failed: ${error.message}`);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|