mbkbucket 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/PACKAGE.md +6 -0
- package/.mbkbucket.config.example.json +14 -0
- package/README.md +261 -81
- package/cli.js +977 -0
- package/index.js +18 -140
- package/lib/index.js +7 -7
- package/package.json +11 -4
- package/public/bucketadmin.js +36 -4
- package/src/app.js +110 -0
- package/src/cli/auth-helper.js +223 -0
- package/{lib → src}/config/index.js +26 -19
- package/{lib/routes/api.js → src/controllers/api.controller.js} +125 -221
- package/{lib/routes/index.js → src/controllers/dashboard.controller.js} +8 -36
- package/src/controllers/info.controller.js +37 -0
- package/src/controllers/view.controller.js +276 -0
- package/src/middleware/async-handler.js +38 -0
- package/src/middleware/bucket-resolver.js +43 -0
- package/src/middleware/error-handler.js +34 -0
- package/src/middleware/rate-limiter.js +30 -0
- package/src/middleware/security.js +43 -0
- package/src/routes/api.routes.js +66 -0
- package/src/routes/index.js +22 -0
- package/src/routes/info.routes.js +10 -0
- package/src/routes/view.routes.js +25 -0
- package/{lib/s3.js → src/services/s3.service.js} +115 -191
- package/src/utils/errors.js +76 -0
- package/src/utils/helpers.js +173 -0
- package/{lib/debug.js → src/utils/logger.js} +5 -0
- package/test/config-validation.test.js +1 -1
- package/lib/routes/info.js +0 -28
- package/lib/routes/view.js +0 -531
- /package/{lib → src}/config/validation.js +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand } from '@aws-sdk/client-s3';
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import { mbkautheVar } from "mbkauthe";
|
|
4
|
-
import { createLogger } from "
|
|
4
|
+
import { createLogger } from "../utils/logger.js";
|
|
5
5
|
|
|
6
6
|
dotenv.config();
|
|
7
7
|
const debugS3 = createLogger('s3');
|
|
@@ -11,7 +11,6 @@ function parseJsonEnv(envVar, fallback = {}) {
|
|
|
11
11
|
try {
|
|
12
12
|
return JSON.parse(envVar);
|
|
13
13
|
} catch (e) {
|
|
14
|
-
// Log a concise message including the error and a short excerpt of the env value when available
|
|
15
14
|
console.error(`❌ Error parsing JSON for envVar: ${envVar ? String(envVar).slice(0, 200) : 'undefined'} - ${e.message}`);
|
|
16
15
|
console.error(`💡 Tip: Check that inner objects are NOT quoted as strings. Use {"key":{"field":"value"}} not {"key":"{"field":"value"}"}`);
|
|
17
16
|
return fallback;
|
|
@@ -33,17 +32,14 @@ export function getAvailableBucketNames() {
|
|
|
33
32
|
|
|
34
33
|
// Helper to get default bucket name
|
|
35
34
|
function getDefaultBucketName() {
|
|
36
|
-
// First priority: use mbkautheVar.bucket if configured
|
|
37
35
|
if (mbkautheVar && mbkautheVar.bucket) {
|
|
38
36
|
return mbkautheVar.bucket;
|
|
39
37
|
}
|
|
40
|
-
|
|
41
|
-
// Second priority: use first bucket in BucketConnection
|
|
38
|
+
|
|
42
39
|
if (availableBucketNames.length > 0) {
|
|
43
40
|
return availableBucketNames[0];
|
|
44
41
|
}
|
|
45
|
-
|
|
46
|
-
// Fallback: return null (will cause error in getBucketConfig)
|
|
42
|
+
|
|
47
43
|
return null;
|
|
48
44
|
}
|
|
49
45
|
|
|
@@ -67,9 +63,6 @@ export function resolveBucketName(bucketName) {
|
|
|
67
63
|
return candidate;
|
|
68
64
|
}
|
|
69
65
|
|
|
70
|
-
// (bucketClient singleton exported below, after cache setup)
|
|
71
|
-
|
|
72
|
-
// Helper: get configured APP_NAME; prefer mbkautheVar, then env var
|
|
73
66
|
export function getAppName() {
|
|
74
67
|
const app = (mbkautheVar && mbkautheVar.APP_NAME);
|
|
75
68
|
return (typeof app === 'string' && app.trim()) ? app.trim() : '';
|
|
@@ -77,34 +70,29 @@ export function getAppName() {
|
|
|
77
70
|
|
|
78
71
|
function isRootModeApp(appName) {
|
|
79
72
|
const app = String(appName || '').toLowerCase();
|
|
80
|
-
|
|
81
|
-
return isRootApp;
|
|
73
|
+
return app === 'portal' || app === 'mbkbucket';
|
|
82
74
|
}
|
|
83
75
|
|
|
84
76
|
export function ensureKeyHasAppPrefix(key = '') {
|
|
85
77
|
const app = getAppName();
|
|
86
78
|
if (!key) throw new Error('Key is required');
|
|
87
|
-
|
|
88
|
-
// Security: Detect path traversal attempts
|
|
79
|
+
|
|
89
80
|
if (key.includes('../') || key.includes('..\\') || /\.\.[\\/]/.test(key)) {
|
|
90
81
|
throw new Error('Path traversal detected: invalid key');
|
|
91
82
|
}
|
|
92
|
-
|
|
93
|
-
// Security: Block null bytes and control characters
|
|
83
|
+
|
|
94
84
|
if (/[\x00-\x1f\x7f]/.test(key)) {
|
|
95
85
|
throw new Error('Invalid characters in key');
|
|
96
86
|
}
|
|
97
|
-
|
|
87
|
+
|
|
98
88
|
let cleaned = String(key).replace(/^\/+/, '');
|
|
99
89
|
|
|
100
|
-
// Special case: APP_NAME=portal means operate from bucket root.
|
|
101
90
|
if (isRootModeApp(app)) {
|
|
102
91
|
return cleaned;
|
|
103
92
|
}
|
|
104
93
|
|
|
105
94
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
106
95
|
|
|
107
|
-
// Collapse repeated app prefixes (app/app/... -> app/...)
|
|
108
96
|
const appPrefixRegex = new RegExp(`^(?:${escapeRegExp(app)}\/)+`);
|
|
109
97
|
if (!cleaned.startsWith(`${app}/`)) {
|
|
110
98
|
cleaned = `${app}/${cleaned}`;
|
|
@@ -114,58 +102,49 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
114
102
|
return cleaned;
|
|
115
103
|
}
|
|
116
104
|
|
|
117
|
-
|
|
118
105
|
export function ensurePrefix(prefix = '') {
|
|
119
106
|
const app = getAppName();
|
|
120
|
-
|
|
121
|
-
// Security: Detect path traversal attempts
|
|
107
|
+
|
|
122
108
|
if (prefix && (prefix.includes('../') || prefix.includes('..\\') || /\.\.[\\/]/.test(prefix))) {
|
|
123
109
|
throw new Error('Path traversal detected: invalid prefix');
|
|
124
110
|
}
|
|
125
|
-
|
|
126
|
-
// Security: Block null bytes and control characters
|
|
111
|
+
|
|
127
112
|
if (prefix && /[\x00-\x1f\x7f]/.test(prefix)) {
|
|
128
113
|
throw new Error('Invalid characters in prefix');
|
|
129
114
|
}
|
|
130
|
-
|
|
131
|
-
// Only strip LEADING slashes. Preserve trailing slashes for directory listing.
|
|
115
|
+
|
|
132
116
|
let p = String(prefix || '').replace(/^\/+/, '');
|
|
133
117
|
|
|
134
|
-
// Special case: APP_NAME=portal means operate from bucket root.
|
|
135
118
|
if (isRootModeApp(app)) {
|
|
136
119
|
return p;
|
|
137
120
|
}
|
|
138
121
|
|
|
139
122
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
140
|
-
|
|
123
|
+
|
|
141
124
|
if (!p) return `${app}`;
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
if (p === app) return app; // If p is exactly "app", we return "app". (Caller should append / if listing root)
|
|
125
|
+
|
|
126
|
+
if (p === app) return app;
|
|
145
127
|
if (p === `${app}/`) return `${app}/`;
|
|
146
128
|
|
|
147
129
|
if (p.startsWith(`${app}/`)) {
|
|
148
|
-
// collapse repeated app/ sequences
|
|
149
130
|
const appPrefixRegex = new RegExp(`^(?:${escapeRegExp(app)}\/)+`);
|
|
150
131
|
return p.replace(appPrefixRegex, `${app}/`);
|
|
151
132
|
}
|
|
152
133
|
return `${app}/${p}`;
|
|
153
|
-
}
|
|
134
|
+
}
|
|
154
135
|
|
|
155
136
|
// ---------------------------------------------------------------------------
|
|
156
|
-
// Singleton S3 client cache
|
|
157
|
-
// Avoids creating a new S3Client (and losing connection pooling) on every call.
|
|
137
|
+
// Singleton S3 client cache
|
|
158
138
|
// ---------------------------------------------------------------------------
|
|
159
139
|
const _clientCache = new Map();
|
|
160
140
|
const _configCache = new Map();
|
|
161
141
|
|
|
162
142
|
export function getBucketConfig(bucketName) {
|
|
163
143
|
bucketName = resolveBucketName(bucketName);
|
|
164
|
-
|
|
144
|
+
|
|
165
145
|
const cacheKey = bucketName;
|
|
166
146
|
if (_configCache.has(cacheKey)) return _configCache.get(cacheKey);
|
|
167
147
|
|
|
168
|
-
// Look up bucket config from BucketConnection JSON
|
|
169
148
|
if (!hasBucketConfigs) {
|
|
170
149
|
console.error(`[mbkbucket] ❌ BucketConnection is not configured or failed to parse`);
|
|
171
150
|
console.error(`[mbkbucket] Please set BucketConnection in your .env file with this format:`);
|
|
@@ -175,9 +154,9 @@ export function getBucketConfig(bucketName) {
|
|
|
175
154
|
console.error(`[mbkbucket] Right: {"r2":{"BUCKET_NAME":"..."}} ✓`);
|
|
176
155
|
throw new Error(`BucketConnection environment variable is not set or empty. Please configure it with your bucket connections.`);
|
|
177
156
|
}
|
|
178
|
-
|
|
157
|
+
|
|
179
158
|
const cfg = allBucketConfigs[bucketName];
|
|
180
|
-
|
|
159
|
+
|
|
181
160
|
if (!cfg) {
|
|
182
161
|
throw new Error(`Bucket '${bucketName}' not found in BucketConnection. Available buckets: ${availableBucketNames.join(', ')}`);
|
|
183
162
|
}
|
|
@@ -186,10 +165,9 @@ export function getBucketConfig(bucketName) {
|
|
|
186
165
|
return cfg;
|
|
187
166
|
}
|
|
188
167
|
|
|
189
|
-
// Function to get S3 client for specific bucket — returns cached singleton
|
|
190
168
|
export function getBucketClient(bucketName) {
|
|
191
169
|
bucketName = resolveBucketName(bucketName);
|
|
192
|
-
|
|
170
|
+
|
|
193
171
|
if (_clientCache.has(bucketName)) return _clientCache.get(bucketName);
|
|
194
172
|
|
|
195
173
|
const config = getBucketConfig(bucketName);
|
|
@@ -218,46 +196,55 @@ export function getBucketClient(bucketName) {
|
|
|
218
196
|
return client;
|
|
219
197
|
}
|
|
220
198
|
|
|
221
|
-
|
|
222
|
-
|
|
199
|
+
/**
|
|
200
|
+
* Resolve both client and config in a single call — avoids redundant
|
|
201
|
+
* resolveBucketName + cache lookups in every S3 operation.
|
|
202
|
+
*/
|
|
203
|
+
export function getBucketClientAndConfig(bucketName) {
|
|
204
|
+
const resolved = resolveBucketName(bucketName);
|
|
205
|
+
return {
|
|
206
|
+
client: getBucketClient(resolved),
|
|
207
|
+
config: getBucketConfig(resolved),
|
|
208
|
+
bucketName: resolved
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Default client singleton (lazily created on first access, not at import time)
|
|
213
|
+
let _defaultClient = null;
|
|
214
|
+
export const bucketClient = new Proxy({}, {
|
|
215
|
+
get(_, prop) {
|
|
216
|
+
if (!_defaultClient) _defaultClient = getBucketClient();
|
|
217
|
+
const val = _defaultClient[prop];
|
|
218
|
+
return typeof val === 'function' ? val.bind(_defaultClient) : val;
|
|
219
|
+
}
|
|
220
|
+
});
|
|
223
221
|
|
|
224
|
-
// Health check for connection (
|
|
225
|
-
async function checkHealth() {
|
|
222
|
+
// Health check for connection (lazy — only runs when called explicitly)
|
|
223
|
+
export async function checkHealth() {
|
|
226
224
|
try {
|
|
227
225
|
const startTime = Date.now();
|
|
228
|
-
|
|
229
|
-
const config = getBucketConfig();
|
|
226
|
+
const { client, config } = getBucketClientAndConfig();
|
|
230
227
|
const bucketName = config.BUCKET_NAME;
|
|
231
228
|
|
|
232
|
-
|
|
233
|
-
const command = new ListObjectsV2Command({
|
|
234
|
-
Bucket: bucketName,
|
|
235
|
-
MaxKeys: 1,
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
const client = getBucketClient();
|
|
239
|
-
const result = await client.send(command);
|
|
240
|
-
const responseTime = Date.now() - startTime;
|
|
241
|
-
|
|
229
|
+
await client.send(new ListObjectsV2Command({ Bucket: bucketName, MaxKeys: 1 }));
|
|
242
230
|
return {
|
|
243
231
|
status: 'healthy',
|
|
244
|
-
responseTime,
|
|
232
|
+
responseTime: Date.now() - startTime,
|
|
245
233
|
bucket: bucketName,
|
|
246
234
|
region: config.region || 'auto',
|
|
247
235
|
checkedAt: new Date().toISOString()
|
|
248
236
|
};
|
|
249
237
|
} catch (error) {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
bucket: (function(){try{return getBucketConfig().BUCKET_NAME}catch(e){return 'unknown'}})(),
|
|
254
|
-
checkedAt: new Date().toISOString()
|
|
255
|
-
};
|
|
238
|
+
let bucket = 'unknown';
|
|
239
|
+
try { bucket = getBucketConfig().BUCKET_NAME; } catch {}
|
|
240
|
+
return { status: 'unhealthy', error: error.message, bucket, checkedAt: new Date().toISOString() };
|
|
256
241
|
}
|
|
257
242
|
}
|
|
258
243
|
|
|
259
|
-
|
|
260
|
-
|
|
244
|
+
/**
|
|
245
|
+
* Run S3 health check and log results. Should be called explicitly at app startup.
|
|
246
|
+
*/
|
|
247
|
+
export async function runHealthCheck() {
|
|
261
248
|
try {
|
|
262
249
|
debugS3('mbkautheVar snapshot: %O', { APP_NAME: mbkautheVar?.APP_NAME, bucket: mbkautheVar?.bucket });
|
|
263
250
|
const health = await checkHealth();
|
|
@@ -269,35 +256,31 @@ async function checkHealth() {
|
|
|
269
256
|
} catch (err) {
|
|
270
257
|
console.error("[mbkbucket] S3 connection test error:", err.message);
|
|
271
258
|
}
|
|
272
|
-
}
|
|
259
|
+
}
|
|
273
260
|
|
|
274
261
|
// Upload file with enhanced features
|
|
275
262
|
export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
276
263
|
try {
|
|
277
|
-
// Validate inputs
|
|
278
264
|
if (!key || !fileBuffer) {
|
|
279
265
|
throw new Error('Key and file buffer are required');
|
|
280
266
|
}
|
|
281
267
|
|
|
282
|
-
// Enforce app prefix (never allow root keys)
|
|
283
268
|
const keyToUpload = ensureKeyHasAppPrefix(key);
|
|
284
269
|
|
|
285
270
|
const {
|
|
286
271
|
metadata = {},
|
|
287
|
-
cacheControl = 'public, max-age=31536000',
|
|
272
|
+
cacheControl = 'public, max-age=31536000',
|
|
288
273
|
storageClass = 'STANDARD',
|
|
289
274
|
serverSideEncryption = 'AES256',
|
|
290
275
|
bucketName,
|
|
291
|
-
preventOverwrite = false
|
|
276
|
+
preventOverwrite = false
|
|
292
277
|
} = options;
|
|
293
|
-
|
|
294
|
-
const client =
|
|
295
|
-
const config = getBucketConfig(bucketName);
|
|
278
|
+
|
|
279
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
296
280
|
const bucket = config.BUCKET_NAME;
|
|
297
281
|
|
|
298
282
|
debugS3('Uploading file to bucket=%s key=%s', bucket, keyToUpload);
|
|
299
283
|
|
|
300
|
-
// Add default metadata (supports metadata)
|
|
301
284
|
const uploadedAt = new Date().toISOString();
|
|
302
285
|
const defaultMetadata = {
|
|
303
286
|
'uploaded-at': uploadedAt,
|
|
@@ -316,31 +299,29 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
316
299
|
ServerSideEncryption: serverSideEncryption,
|
|
317
300
|
StorageClass: storageClass,
|
|
318
301
|
};
|
|
319
|
-
|
|
320
|
-
// Add conditional upload to prevent overwriting (atomic operation)
|
|
302
|
+
|
|
321
303
|
if (preventOverwrite) {
|
|
322
|
-
commandParams.IfNoneMatch = '*';
|
|
304
|
+
commandParams.IfNoneMatch = '*';
|
|
323
305
|
}
|
|
324
306
|
|
|
325
307
|
const command = new PutObjectCommand(commandParams);
|
|
326
|
-
|
|
308
|
+
|
|
327
309
|
const result = await client.send(command);
|
|
328
|
-
|
|
310
|
+
|
|
329
311
|
return {
|
|
330
312
|
...result,
|
|
331
313
|
fileSize: fileBuffer.length,
|
|
332
314
|
key: keyToUpload,
|
|
333
315
|
contentType,
|
|
334
316
|
uploadedAt
|
|
335
|
-
};
|
|
317
|
+
};
|
|
336
318
|
} catch (error) {
|
|
337
319
|
console.error(`Upload failed for key ${key}:`, error);
|
|
338
|
-
|
|
339
|
-
// Handle precondition failed (file already exists)
|
|
320
|
+
|
|
340
321
|
if (error.name === 'PreconditionFailed' || error.$metadata?.httpStatusCode === 412) {
|
|
341
322
|
throw new Error('File already exists');
|
|
342
323
|
}
|
|
343
|
-
|
|
324
|
+
|
|
344
325
|
throw new Error(`Upload failed: ${error.message}`);
|
|
345
326
|
}
|
|
346
327
|
}
|
|
@@ -353,7 +334,6 @@ export async function downloadFile(key, options = {}) {
|
|
|
353
334
|
throw new Error('Key is required');
|
|
354
335
|
}
|
|
355
336
|
|
|
356
|
-
// Ensure key is app-prefixed
|
|
357
337
|
keyToDownload = ensureKeyHasAppPrefix(key);
|
|
358
338
|
|
|
359
339
|
const {
|
|
@@ -364,12 +344,10 @@ export async function downloadFile(key, options = {}) {
|
|
|
364
344
|
responseContentType = null,
|
|
365
345
|
bucketName
|
|
366
346
|
} = options;
|
|
367
|
-
|
|
368
|
-
const client =
|
|
369
|
-
const config = getBucketConfig(bucketName);
|
|
347
|
+
|
|
348
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
370
349
|
const bucket = config.BUCKET_NAME;
|
|
371
350
|
|
|
372
|
-
// Log range requests for debugging
|
|
373
351
|
if (range) {
|
|
374
352
|
debugS3('Range request for %s: %s', keyToDownload, range);
|
|
375
353
|
}
|
|
@@ -383,23 +361,20 @@ export async function downloadFile(key, options = {}) {
|
|
|
383
361
|
...(responseCacheControl && { ResponseCacheControl: responseCacheControl }),
|
|
384
362
|
...(responseContentType && { ResponseContentType: responseContentType }),
|
|
385
363
|
});
|
|
386
|
-
|
|
364
|
+
|
|
387
365
|
const result = await client.send(command);
|
|
388
|
-
|
|
366
|
+
|
|
389
367
|
return {
|
|
390
368
|
...result,
|
|
391
369
|
key: keyToDownload,
|
|
392
|
-
};
|
|
370
|
+
};
|
|
393
371
|
} catch (error) {
|
|
394
372
|
console.error(`Download failed for key ${key}:`, error);
|
|
395
373
|
|
|
396
|
-
// Some S3-compatible endpoints return 304 Not Modified as a response to conditional GETs.
|
|
397
|
-
// The SDK surface may throw an error with metadata httpStatusCode=304 — treat this as a not-modified condition.
|
|
398
374
|
if (error && error.$metadata && error.$metadata.httpStatusCode === 304) {
|
|
399
375
|
return { notModified: true, key: keyToDownload };
|
|
400
376
|
}
|
|
401
377
|
|
|
402
|
-
// Handle specific S3 errors
|
|
403
378
|
if (error.name === 'NoSuchKey') {
|
|
404
379
|
throw new Error(`File not found: ${key}`);
|
|
405
380
|
} else if (error.name === 'AccessDenied') {
|
|
@@ -417,18 +392,17 @@ export async function deleteFile(key, bucketName) {
|
|
|
417
392
|
throw new Error('Key is required');
|
|
418
393
|
}
|
|
419
394
|
const keyToDelete = ensureKeyHasAppPrefix(key);
|
|
420
|
-
|
|
421
|
-
const client =
|
|
422
|
-
const config = getBucketConfig(bucketName);
|
|
395
|
+
|
|
396
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
423
397
|
const bucket = config.BUCKET_NAME;
|
|
424
398
|
|
|
425
399
|
const command = new DeleteObjectCommand({
|
|
426
400
|
Bucket: bucket,
|
|
427
401
|
Key: keyToDelete,
|
|
428
402
|
});
|
|
429
|
-
|
|
403
|
+
|
|
430
404
|
const result = await client.send(command);
|
|
431
|
-
|
|
405
|
+
|
|
432
406
|
return {
|
|
433
407
|
...result,
|
|
434
408
|
key: keyToDelete,
|
|
@@ -438,7 +412,7 @@ export async function deleteFile(key, bucketName) {
|
|
|
438
412
|
console.error(`Delete failed:`, error);
|
|
439
413
|
throw new Error(`Delete failed: ${error.message}`);
|
|
440
414
|
}
|
|
441
|
-
}
|
|
415
|
+
}
|
|
442
416
|
|
|
443
417
|
// Batch delete multiple files
|
|
444
418
|
export async function deleteFiles(keys, bucketName) {
|
|
@@ -446,12 +420,10 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
446
420
|
if (!keys || !Array.isArray(keys) || keys.length === 0) {
|
|
447
421
|
throw new Error('Keys array is required and must not be empty');
|
|
448
422
|
}
|
|
449
|
-
|
|
450
|
-
const client =
|
|
451
|
-
const config = getBucketConfig(bucketName);
|
|
423
|
+
|
|
424
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
452
425
|
const bucket = config.BUCKET_NAME;
|
|
453
426
|
|
|
454
|
-
// S3 allows max 1000 objects per delete request
|
|
455
427
|
const maxBatchSize = 1000;
|
|
456
428
|
const results = [];
|
|
457
429
|
let deletedCount = 0;
|
|
@@ -459,7 +431,7 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
459
431
|
|
|
460
432
|
for (let i = 0; i < keys.length; i += maxBatchSize) {
|
|
461
433
|
const batch = keys.slice(i, i + maxBatchSize).map(k => ({ Key: ensureKeyHasAppPrefix(k) }));
|
|
462
|
-
|
|
434
|
+
|
|
463
435
|
const command = new DeleteObjectsCommand({
|
|
464
436
|
Bucket: bucket,
|
|
465
437
|
Delete: {
|
|
@@ -486,12 +458,7 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
486
458
|
}
|
|
487
459
|
}
|
|
488
460
|
|
|
489
|
-
//
|
|
490
|
-
/**
|
|
491
|
-
* Delete all objects under a folder prefix (recursive).
|
|
492
|
-
* Accepts either a folder key (may include trailing slash) or a relative prefix.
|
|
493
|
-
* Normalizes the prefix to include the configured APP_NAME and ensures trailing slash.
|
|
494
|
-
*/
|
|
461
|
+
// Delete all objects under a folder prefix (recursive)
|
|
495
462
|
export async function deleteFolder(prefix, bucketName) {
|
|
496
463
|
try {
|
|
497
464
|
if (!prefix) {
|
|
@@ -501,14 +468,12 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
501
468
|
const cleaned = String(prefix).replace(/^\/+/, '').replace(/\/+$/g, '');
|
|
502
469
|
const effectivePrefix = ensurePrefix(cleaned) + '/';
|
|
503
470
|
|
|
504
|
-
const client =
|
|
505
|
-
const config = getBucketConfig(bucketName);
|
|
471
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
506
472
|
const bucket = config.BUCKET_NAME;
|
|
507
473
|
|
|
508
474
|
const keysToDelete = [];
|
|
509
475
|
let continuationToken = null;
|
|
510
476
|
|
|
511
|
-
// Paginate through all objects under the prefix
|
|
512
477
|
do {
|
|
513
478
|
const listCmd = new ListObjectsV2Command({
|
|
514
479
|
Bucket: bucket,
|
|
@@ -525,7 +490,6 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
525
490
|
} while (continuationToken);
|
|
526
491
|
|
|
527
492
|
if (!keysToDelete.length) {
|
|
528
|
-
// Nothing found under this prefix; attempt to remove a zero-byte folder marker if present
|
|
529
493
|
const folderMarkerKey = effectivePrefix;
|
|
530
494
|
const exists = await fileExists(folderMarkerKey, bucketName);
|
|
531
495
|
if (exists) {
|
|
@@ -535,7 +499,6 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
535
499
|
return { deletedCount: 0, deletedAt: new Date().toISOString(), prefix: effectivePrefix };
|
|
536
500
|
}
|
|
537
501
|
|
|
538
|
-
// Use the existing deleteFiles helper which handles batching and error aggregation
|
|
539
502
|
const result = await deleteFiles(keysToDelete, bucketName);
|
|
540
503
|
|
|
541
504
|
return {
|
|
@@ -560,15 +523,12 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
560
523
|
startAfter = null,
|
|
561
524
|
bucketName
|
|
562
525
|
} = options;
|
|
563
|
-
|
|
564
|
-
// Enforce non-root prefix: default to APP_NAME and normalize
|
|
526
|
+
|
|
565
527
|
const effectivePrefix = ensurePrefix(prefix);
|
|
566
528
|
|
|
567
|
-
const client =
|
|
568
|
-
const config = getBucketConfig(bucketName);
|
|
529
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
569
530
|
const bucket = config.BUCKET_NAME;
|
|
570
531
|
|
|
571
|
-
|
|
572
532
|
const command = new ListObjectsV2Command({
|
|
573
533
|
Bucket: bucket,
|
|
574
534
|
Prefix: effectivePrefix,
|
|
@@ -578,9 +538,9 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
578
538
|
...(fetchOwner && { FetchOwner: fetchOwner }),
|
|
579
539
|
...(startAfter && { StartAfter: startAfter }),
|
|
580
540
|
});
|
|
581
|
-
|
|
541
|
+
|
|
582
542
|
const result = await client.send(command);
|
|
583
|
-
|
|
543
|
+
|
|
584
544
|
return {
|
|
585
545
|
...result,
|
|
586
546
|
requestedAt: new Date().toISOString(),
|
|
@@ -601,19 +561,18 @@ export async function getFileMetadata(key, bucketName) {
|
|
|
601
561
|
throw new Error('Key is required');
|
|
602
562
|
}
|
|
603
563
|
const keyToCheck = ensureKeyHasAppPrefix(key);
|
|
604
|
-
|
|
605
|
-
const client =
|
|
606
|
-
const config = getBucketConfig(bucketName);
|
|
564
|
+
|
|
565
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
607
566
|
const bucket = config.BUCKET_NAME;
|
|
608
567
|
|
|
609
568
|
const command = new HeadObjectCommand({
|
|
610
569
|
Bucket: bucket,
|
|
611
570
|
Key: keyToCheck,
|
|
612
571
|
});
|
|
613
|
-
|
|
572
|
+
|
|
614
573
|
const result = await client.send(command);
|
|
615
574
|
const queriedAt = new Date().toISOString();
|
|
616
|
-
|
|
575
|
+
|
|
617
576
|
return {
|
|
618
577
|
...result,
|
|
619
578
|
key: keyToCheck,
|
|
@@ -629,11 +588,11 @@ export async function getFileMetadata(key, bucketName) {
|
|
|
629
588
|
queriedAt
|
|
630
589
|
};
|
|
631
590
|
}
|
|
632
|
-
|
|
591
|
+
|
|
633
592
|
console.error(`Get metadata failed for key ${key}:`, error);
|
|
634
593
|
throw new Error(`Get metadata failed: ${error.message}`);
|
|
635
594
|
}
|
|
636
|
-
}
|
|
595
|
+
}
|
|
637
596
|
|
|
638
597
|
// Check if file exists
|
|
639
598
|
export async function fileExists(key, bucketName) {
|
|
@@ -655,23 +614,16 @@ export async function getFileSize(key, bucketName) {
|
|
|
655
614
|
}
|
|
656
615
|
}
|
|
657
616
|
|
|
658
|
-
// Export health check for external use
|
|
659
|
-
export { checkHealth };
|
|
660
|
-
|
|
661
617
|
// ---------------------------------------------------------------------------
|
|
662
618
|
// S3 Multipart Upload helpers
|
|
663
619
|
// ---------------------------------------------------------------------------
|
|
664
620
|
|
|
665
|
-
/**
|
|
666
|
-
* Initiate a multipart upload. Returns the S3 UploadId.
|
|
667
|
-
*/
|
|
668
621
|
export async function createMultipartUpload(key, contentType = 'application/octet-stream', metadata = {}, bucketName) {
|
|
669
622
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
670
|
-
const
|
|
671
|
-
const client = getBucketClient(bucketName);
|
|
623
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
672
624
|
const uploadedAt = new Date().toISOString();
|
|
673
625
|
const command = new CreateMultipartUploadCommand({
|
|
674
|
-
Bucket:
|
|
626
|
+
Bucket: config.BUCKET_NAME,
|
|
675
627
|
Key: keyToUse,
|
|
676
628
|
ContentType: contentType,
|
|
677
629
|
Metadata: {
|
|
@@ -684,16 +636,11 @@ export async function createMultipartUpload(key, contentType = 'application/octe
|
|
|
684
636
|
return { uploadId: result.UploadId, key: keyToUse };
|
|
685
637
|
}
|
|
686
638
|
|
|
687
|
-
/**
|
|
688
|
-
* Upload a single part. partNumber is 1-based.
|
|
689
|
-
* Returns the ETag needed for CompleteMultipartUpload.
|
|
690
|
-
*/
|
|
691
639
|
export async function uploadPart(key, uploadId, partNumber, buffer, bucketName) {
|
|
692
640
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
693
|
-
const
|
|
694
|
-
const client = getBucketClient(bucketName);
|
|
641
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
695
642
|
const command = new UploadPartCommand({
|
|
696
|
-
Bucket:
|
|
643
|
+
Bucket: config.BUCKET_NAME,
|
|
697
644
|
Key: keyToUse,
|
|
698
645
|
UploadId: uploadId,
|
|
699
646
|
PartNumber: partNumber,
|
|
@@ -704,17 +651,12 @@ export async function uploadPart(key, uploadId, partNumber, buffer, bucketName)
|
|
|
704
651
|
return { ETag: result.ETag, partNumber };
|
|
705
652
|
}
|
|
706
653
|
|
|
707
|
-
/**
|
|
708
|
-
* Complete a multipart upload.
|
|
709
|
-
* parts must be an array of { partNumber, ETag } sorted ascending by partNumber.
|
|
710
|
-
*/
|
|
711
654
|
export async function completeMultipartUpload(key, uploadId, parts, bucketName) {
|
|
712
655
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
713
|
-
const
|
|
714
|
-
const client = getBucketClient(bucketName);
|
|
656
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
715
657
|
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
716
658
|
const command = new CompleteMultipartUploadCommand({
|
|
717
|
-
Bucket:
|
|
659
|
+
Bucket: config.BUCKET_NAME,
|
|
718
660
|
Key: keyToUse,
|
|
719
661
|
UploadId: uploadId,
|
|
720
662
|
MultipartUpload: {
|
|
@@ -725,15 +667,11 @@ export async function completeMultipartUpload(key, uploadId, parts, bucketName)
|
|
|
725
667
|
return { key: keyToUse };
|
|
726
668
|
}
|
|
727
669
|
|
|
728
|
-
/**
|
|
729
|
-
* Abort an in-progress multipart upload, freeing all stored parts.
|
|
730
|
-
*/
|
|
731
670
|
export async function abortMultipartUpload(key, uploadId, bucketName) {
|
|
732
671
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
733
|
-
const
|
|
734
|
-
const client = getBucketClient(bucketName);
|
|
672
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
735
673
|
const command = new AbortMultipartUploadCommand({
|
|
736
|
-
Bucket:
|
|
674
|
+
Bucket: config.BUCKET_NAME,
|
|
737
675
|
Key: keyToUse,
|
|
738
676
|
UploadId: uploadId
|
|
739
677
|
});
|
|
@@ -746,10 +684,10 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
746
684
|
try {
|
|
747
685
|
const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner');
|
|
748
686
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
749
|
-
|
|
687
|
+
|
|
750
688
|
let command;
|
|
751
|
-
const
|
|
752
|
-
const resolvedBucketName =
|
|
689
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
690
|
+
const resolvedBucketName = config.BUCKET_NAME;
|
|
753
691
|
switch (operation) {
|
|
754
692
|
case 'getObject':
|
|
755
693
|
command = new GetObjectCommand({ Bucket: resolvedBucketName, Key: keyToUse });
|
|
@@ -760,9 +698,9 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
760
698
|
default:
|
|
761
699
|
throw new Error(`Unsupported operation: ${operation}`);
|
|
762
700
|
}
|
|
763
|
-
|
|
764
|
-
const signedUrl = await getSignedUrl(
|
|
765
|
-
|
|
701
|
+
|
|
702
|
+
const signedUrl = await getSignedUrl(client, command, { expiresIn });
|
|
703
|
+
|
|
766
704
|
return {
|
|
767
705
|
url: signedUrl,
|
|
768
706
|
key,
|
|
@@ -781,23 +719,16 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
781
719
|
// Multipart Upload Cleanup
|
|
782
720
|
// ---------------------------------------------------------------------------
|
|
783
721
|
|
|
784
|
-
/**
|
|
785
|
-
* List incomplete multipart uploads
|
|
786
|
-
* @param {string} prefix - Optional prefix to filter uploads
|
|
787
|
-
* @param {string} bucketName - Bucket name
|
|
788
|
-
* @returns {Promise<Array>} Array of incomplete multipart uploads
|
|
789
|
-
*/
|
|
790
722
|
export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
791
723
|
try {
|
|
792
724
|
const effectivePrefix = ensurePrefix(prefix || '');
|
|
793
|
-
const
|
|
794
|
-
|
|
795
|
-
|
|
725
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
726
|
+
|
|
796
727
|
const command = new ListMultipartUploadsCommand({
|
|
797
|
-
Bucket:
|
|
728
|
+
Bucket: config.BUCKET_NAME,
|
|
798
729
|
Prefix: effectivePrefix
|
|
799
730
|
});
|
|
800
|
-
|
|
731
|
+
|
|
801
732
|
const result = await client.send(command);
|
|
802
733
|
return result.Uploads || [];
|
|
803
734
|
} catch (error) {
|
|
@@ -806,36 +737,29 @@ export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
|
806
737
|
}
|
|
807
738
|
}
|
|
808
739
|
|
|
809
|
-
/**
|
|
810
|
-
* Clean up old incomplete multipart uploads
|
|
811
|
-
* @param {number} olderThanDays - Delete uploads older than this many days (default: 7)
|
|
812
|
-
* @param {string} prefix - Optional prefix to filter uploads
|
|
813
|
-
* @param {string} bucketName - Bucket name
|
|
814
|
-
* @returns {Promise<Object>} Cleanup results with count of aborted uploads
|
|
815
|
-
*/
|
|
816
740
|
export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefix = '', bucketName) {
|
|
817
741
|
try {
|
|
818
742
|
const uploads = await listIncompleteMultipartUploads(prefix, bucketName);
|
|
819
|
-
|
|
743
|
+
|
|
820
744
|
if (!uploads.length) {
|
|
821
745
|
debugS3('No incomplete multipart uploads found');
|
|
822
746
|
return { abortedCount: 0, uploads: [] };
|
|
823
747
|
}
|
|
824
|
-
|
|
748
|
+
|
|
825
749
|
const cutoffDate = new Date();
|
|
826
750
|
cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
|
|
827
|
-
|
|
751
|
+
|
|
828
752
|
const uploadsToAbort = uploads.filter(upload => {
|
|
829
753
|
return upload.Initiated && new Date(upload.Initiated) < cutoffDate;
|
|
830
754
|
});
|
|
831
|
-
|
|
755
|
+
|
|
832
756
|
if (!uploadsToAbort.length) {
|
|
833
757
|
debugS3('No incomplete uploads older than %s days', olderThanDays);
|
|
834
758
|
return { abortedCount: 0, uploads: [] };
|
|
835
759
|
}
|
|
836
|
-
|
|
760
|
+
|
|
837
761
|
debugS3('Found %s incomplete uploads to clean up', uploadsToAbort.length);
|
|
838
|
-
|
|
762
|
+
|
|
839
763
|
const aborted = [];
|
|
840
764
|
for (const upload of uploadsToAbort) {
|
|
841
765
|
try {
|
|
@@ -850,7 +774,7 @@ export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefi
|
|
|
850
774
|
console.error(`[mbkbucket] Failed to abort upload ${upload.Key}:`, err);
|
|
851
775
|
}
|
|
852
776
|
}
|
|
853
|
-
|
|
777
|
+
|
|
854
778
|
return {
|
|
855
779
|
abortedCount: aborted.length,
|
|
856
780
|
uploads: aborted,
|