mbkbucket 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +4 -2
- package/.github/PACKAGE.md +6 -0
- package/.mbkbucket.config.example.json +14 -0
- package/LICENSE +161 -335
- package/README.md +262 -81
- package/cli.js +977 -0
- package/index.js +18 -140
- package/lib/index.js +7 -7
- package/package.json +18 -6
- package/public/bucketadmin.js +41 -6
- package/src/app.js +110 -0
- package/src/cli/auth-helper.js +223 -0
- package/{lib → src}/config/index.js +26 -19
- package/{lib/routes/api.js → src/controllers/api.controller.js} +125 -221
- package/{lib/routes/index.js → src/controllers/dashboard.controller.js} +8 -36
- package/src/controllers/info.controller.js +37 -0
- package/src/controllers/view.controller.js +276 -0
- package/src/middleware/async-handler.js +38 -0
- package/src/middleware/bucket-resolver.js +43 -0
- package/src/middleware/error-handler.js +34 -0
- package/src/middleware/rate-limiter.js +30 -0
- package/src/middleware/security.js +43 -0
- package/src/routes/api.routes.js +66 -0
- package/src/routes/index.js +22 -0
- package/src/routes/info.routes.js +10 -0
- package/src/routes/view.routes.js +25 -0
- package/{lib/s3.js → src/services/s3.service.js} +116 -190
- package/src/utils/errors.js +76 -0
- package/src/utils/helpers.js +173 -0
- package/{lib/debug.js → src/utils/logger.js} +5 -0
- package/test/config-validation.test.js +1 -1
- package/views/bucket.handlebars +0 -1
- package/lib/routes/info.js +0 -28
- package/lib/routes/view.js +0 -531
- /package/{lib → src}/config/validation.js +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand } from '@aws-sdk/client-s3';
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import { mbkautheVar } from "mbkauthe";
|
|
4
|
-
import { createLogger } from "
|
|
4
|
+
import { createLogger } from "../utils/logger.js";
|
|
5
5
|
|
|
6
6
|
dotenv.config();
|
|
7
7
|
const debugS3 = createLogger('s3');
|
|
@@ -11,7 +11,6 @@ function parseJsonEnv(envVar, fallback = {}) {
|
|
|
11
11
|
try {
|
|
12
12
|
return JSON.parse(envVar);
|
|
13
13
|
} catch (e) {
|
|
14
|
-
// Log a concise message including the error and a short excerpt of the env value when available
|
|
15
14
|
console.error(`❌ Error parsing JSON for envVar: ${envVar ? String(envVar).slice(0, 200) : 'undefined'} - ${e.message}`);
|
|
16
15
|
console.error(`💡 Tip: Check that inner objects are NOT quoted as strings. Use {"key":{"field":"value"}} not {"key":"{"field":"value"}"}`);
|
|
17
16
|
return fallback;
|
|
@@ -33,17 +32,14 @@ export function getAvailableBucketNames() {
|
|
|
33
32
|
|
|
34
33
|
// Helper to get default bucket name
|
|
35
34
|
function getDefaultBucketName() {
|
|
36
|
-
// First priority: use mbkautheVar.bucket if configured
|
|
37
35
|
if (mbkautheVar && mbkautheVar.bucket) {
|
|
38
36
|
return mbkautheVar.bucket;
|
|
39
37
|
}
|
|
40
|
-
|
|
41
|
-
// Second priority: use first bucket in BucketConnection
|
|
38
|
+
|
|
42
39
|
if (availableBucketNames.length > 0) {
|
|
43
40
|
return availableBucketNames[0];
|
|
44
41
|
}
|
|
45
|
-
|
|
46
|
-
// Fallback: return null (will cause error in getBucketConfig)
|
|
42
|
+
|
|
47
43
|
return null;
|
|
48
44
|
}
|
|
49
45
|
|
|
@@ -67,42 +63,36 @@ export function resolveBucketName(bucketName) {
|
|
|
67
63
|
return candidate;
|
|
68
64
|
}
|
|
69
65
|
|
|
70
|
-
// (bucketClient singleton exported below, after cache setup)
|
|
71
|
-
|
|
72
|
-
// Helper: get configured APP_NAME; prefer mbkautheVar, then env var
|
|
73
66
|
export function getAppName() {
|
|
74
67
|
const app = (mbkautheVar && mbkautheVar.APP_NAME);
|
|
75
68
|
return (typeof app === 'string' && app.trim()) ? app.trim() : '';
|
|
76
69
|
}
|
|
77
70
|
|
|
78
71
|
function isRootModeApp(appName) {
|
|
79
|
-
|
|
72
|
+
const app = String(appName || '').toLowerCase();
|
|
73
|
+
return app === 'portal' || app === 'mbkbucket';
|
|
80
74
|
}
|
|
81
75
|
|
|
82
76
|
export function ensureKeyHasAppPrefix(key = '') {
|
|
83
77
|
const app = getAppName();
|
|
84
78
|
if (!key) throw new Error('Key is required');
|
|
85
|
-
|
|
86
|
-
// Security: Detect path traversal attempts
|
|
79
|
+
|
|
87
80
|
if (key.includes('../') || key.includes('..\\') || /\.\.[\\/]/.test(key)) {
|
|
88
81
|
throw new Error('Path traversal detected: invalid key');
|
|
89
82
|
}
|
|
90
|
-
|
|
91
|
-
// Security: Block null bytes and control characters
|
|
83
|
+
|
|
92
84
|
if (/[\x00-\x1f\x7f]/.test(key)) {
|
|
93
85
|
throw new Error('Invalid characters in key');
|
|
94
86
|
}
|
|
95
|
-
|
|
87
|
+
|
|
96
88
|
let cleaned = String(key).replace(/^\/+/, '');
|
|
97
89
|
|
|
98
|
-
// Special case: APP_NAME=portal means operate from bucket root.
|
|
99
90
|
if (isRootModeApp(app)) {
|
|
100
91
|
return cleaned;
|
|
101
92
|
}
|
|
102
93
|
|
|
103
94
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
104
95
|
|
|
105
|
-
// Collapse repeated app prefixes (app/app/... -> app/...)
|
|
106
96
|
const appPrefixRegex = new RegExp(`^(?:${escapeRegExp(app)}\/)+`);
|
|
107
97
|
if (!cleaned.startsWith(`${app}/`)) {
|
|
108
98
|
cleaned = `${app}/${cleaned}`;
|
|
@@ -112,58 +102,49 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
112
102
|
return cleaned;
|
|
113
103
|
}
|
|
114
104
|
|
|
115
|
-
|
|
116
105
|
export function ensurePrefix(prefix = '') {
|
|
117
106
|
const app = getAppName();
|
|
118
|
-
|
|
119
|
-
// Security: Detect path traversal attempts
|
|
107
|
+
|
|
120
108
|
if (prefix && (prefix.includes('../') || prefix.includes('..\\') || /\.\.[\\/]/.test(prefix))) {
|
|
121
109
|
throw new Error('Path traversal detected: invalid prefix');
|
|
122
110
|
}
|
|
123
|
-
|
|
124
|
-
// Security: Block null bytes and control characters
|
|
111
|
+
|
|
125
112
|
if (prefix && /[\x00-\x1f\x7f]/.test(prefix)) {
|
|
126
113
|
throw new Error('Invalid characters in prefix');
|
|
127
114
|
}
|
|
128
|
-
|
|
129
|
-
// Only strip LEADING slashes. Preserve trailing slashes for directory listing.
|
|
115
|
+
|
|
130
116
|
let p = String(prefix || '').replace(/^\/+/, '');
|
|
131
117
|
|
|
132
|
-
// Special case: APP_NAME=portal means operate from bucket root.
|
|
133
118
|
if (isRootModeApp(app)) {
|
|
134
119
|
return p;
|
|
135
120
|
}
|
|
136
121
|
|
|
137
122
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
138
|
-
|
|
123
|
+
|
|
139
124
|
if (!p) return `${app}`;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
if (p === app) return app; // If p is exactly "app", we return "app". (Caller should append / if listing root)
|
|
125
|
+
|
|
126
|
+
if (p === app) return app;
|
|
143
127
|
if (p === `${app}/`) return `${app}/`;
|
|
144
128
|
|
|
145
129
|
if (p.startsWith(`${app}/`)) {
|
|
146
|
-
// collapse repeated app/ sequences
|
|
147
130
|
const appPrefixRegex = new RegExp(`^(?:${escapeRegExp(app)}\/)+`);
|
|
148
131
|
return p.replace(appPrefixRegex, `${app}/`);
|
|
149
132
|
}
|
|
150
133
|
return `${app}/${p}`;
|
|
151
|
-
}
|
|
134
|
+
}
|
|
152
135
|
|
|
153
136
|
// ---------------------------------------------------------------------------
|
|
154
|
-
// Singleton S3 client cache
|
|
155
|
-
// Avoids creating a new S3Client (and losing connection pooling) on every call.
|
|
137
|
+
// Singleton S3 client cache
|
|
156
138
|
// ---------------------------------------------------------------------------
|
|
157
139
|
const _clientCache = new Map();
|
|
158
140
|
const _configCache = new Map();
|
|
159
141
|
|
|
160
142
|
export function getBucketConfig(bucketName) {
|
|
161
143
|
bucketName = resolveBucketName(bucketName);
|
|
162
|
-
|
|
144
|
+
|
|
163
145
|
const cacheKey = bucketName;
|
|
164
146
|
if (_configCache.has(cacheKey)) return _configCache.get(cacheKey);
|
|
165
147
|
|
|
166
|
-
// Look up bucket config from BucketConnection JSON
|
|
167
148
|
if (!hasBucketConfigs) {
|
|
168
149
|
console.error(`[mbkbucket] ❌ BucketConnection is not configured or failed to parse`);
|
|
169
150
|
console.error(`[mbkbucket] Please set BucketConnection in your .env file with this format:`);
|
|
@@ -173,9 +154,9 @@ export function getBucketConfig(bucketName) {
|
|
|
173
154
|
console.error(`[mbkbucket] Right: {"r2":{"BUCKET_NAME":"..."}} ✓`);
|
|
174
155
|
throw new Error(`BucketConnection environment variable is not set or empty. Please configure it with your bucket connections.`);
|
|
175
156
|
}
|
|
176
|
-
|
|
157
|
+
|
|
177
158
|
const cfg = allBucketConfigs[bucketName];
|
|
178
|
-
|
|
159
|
+
|
|
179
160
|
if (!cfg) {
|
|
180
161
|
throw new Error(`Bucket '${bucketName}' not found in BucketConnection. Available buckets: ${availableBucketNames.join(', ')}`);
|
|
181
162
|
}
|
|
@@ -184,10 +165,9 @@ export function getBucketConfig(bucketName) {
|
|
|
184
165
|
return cfg;
|
|
185
166
|
}
|
|
186
167
|
|
|
187
|
-
// Function to get S3 client for specific bucket — returns cached singleton
|
|
188
168
|
export function getBucketClient(bucketName) {
|
|
189
169
|
bucketName = resolveBucketName(bucketName);
|
|
190
|
-
|
|
170
|
+
|
|
191
171
|
if (_clientCache.has(bucketName)) return _clientCache.get(bucketName);
|
|
192
172
|
|
|
193
173
|
const config = getBucketConfig(bucketName);
|
|
@@ -216,46 +196,55 @@ export function getBucketClient(bucketName) {
|
|
|
216
196
|
return client;
|
|
217
197
|
}
|
|
218
198
|
|
|
219
|
-
|
|
220
|
-
|
|
199
|
+
/**
|
|
200
|
+
* Resolve both client and config in a single call — avoids redundant
|
|
201
|
+
* resolveBucketName + cache lookups in every S3 operation.
|
|
202
|
+
*/
|
|
203
|
+
export function getBucketClientAndConfig(bucketName) {
|
|
204
|
+
const resolved = resolveBucketName(bucketName);
|
|
205
|
+
return {
|
|
206
|
+
client: getBucketClient(resolved),
|
|
207
|
+
config: getBucketConfig(resolved),
|
|
208
|
+
bucketName: resolved
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Default client singleton (lazily created on first access, not at import time)
|
|
213
|
+
let _defaultClient = null;
|
|
214
|
+
export const bucketClient = new Proxy({}, {
|
|
215
|
+
get(_, prop) {
|
|
216
|
+
if (!_defaultClient) _defaultClient = getBucketClient();
|
|
217
|
+
const val = _defaultClient[prop];
|
|
218
|
+
return typeof val === 'function' ? val.bind(_defaultClient) : val;
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
221
|
|
|
222
|
-
// Health check for connection (
|
|
223
|
-
async function checkHealth() {
|
|
222
|
+
// Health check for connection (lazy — only runs when called explicitly)
|
|
223
|
+
export async function checkHealth() {
|
|
224
224
|
try {
|
|
225
225
|
const startTime = Date.now();
|
|
226
|
-
|
|
227
|
-
const config = getBucketConfig();
|
|
226
|
+
const { client, config } = getBucketClientAndConfig();
|
|
228
227
|
const bucketName = config.BUCKET_NAME;
|
|
229
228
|
|
|
230
|
-
|
|
231
|
-
const command = new ListObjectsV2Command({
|
|
232
|
-
Bucket: bucketName,
|
|
233
|
-
MaxKeys: 1,
|
|
234
|
-
});
|
|
235
|
-
|
|
236
|
-
const client = getBucketClient();
|
|
237
|
-
const result = await client.send(command);
|
|
238
|
-
const responseTime = Date.now() - startTime;
|
|
239
|
-
|
|
229
|
+
await client.send(new ListObjectsV2Command({ Bucket: bucketName, MaxKeys: 1 }));
|
|
240
230
|
return {
|
|
241
231
|
status: 'healthy',
|
|
242
|
-
responseTime,
|
|
232
|
+
responseTime: Date.now() - startTime,
|
|
243
233
|
bucket: bucketName,
|
|
244
234
|
region: config.region || 'auto',
|
|
245
235
|
checkedAt: new Date().toISOString()
|
|
246
236
|
};
|
|
247
237
|
} catch (error) {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
bucket: (function(){try{return getBucketConfig().BUCKET_NAME}catch(e){return 'unknown'}})(),
|
|
252
|
-
checkedAt: new Date().toISOString()
|
|
253
|
-
};
|
|
238
|
+
let bucket = 'unknown';
|
|
239
|
+
try { bucket = getBucketConfig().BUCKET_NAME; } catch {}
|
|
240
|
+
return { status: 'unhealthy', error: error.message, bucket, checkedAt: new Date().toISOString() };
|
|
254
241
|
}
|
|
255
242
|
}
|
|
256
243
|
|
|
257
|
-
|
|
258
|
-
|
|
244
|
+
/**
|
|
245
|
+
* Run S3 health check and log results. Should be called explicitly at app startup.
|
|
246
|
+
*/
|
|
247
|
+
export async function runHealthCheck() {
|
|
259
248
|
try {
|
|
260
249
|
debugS3('mbkautheVar snapshot: %O', { APP_NAME: mbkautheVar?.APP_NAME, bucket: mbkautheVar?.bucket });
|
|
261
250
|
const health = await checkHealth();
|
|
@@ -267,35 +256,31 @@ async function checkHealth() {
|
|
|
267
256
|
} catch (err) {
|
|
268
257
|
console.error("[mbkbucket] S3 connection test error:", err.message);
|
|
269
258
|
}
|
|
270
|
-
}
|
|
259
|
+
}
|
|
271
260
|
|
|
272
261
|
// Upload file with enhanced features
|
|
273
262
|
export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
274
263
|
try {
|
|
275
|
-
// Validate inputs
|
|
276
264
|
if (!key || !fileBuffer) {
|
|
277
265
|
throw new Error('Key and file buffer are required');
|
|
278
266
|
}
|
|
279
267
|
|
|
280
|
-
// Enforce app prefix (never allow root keys)
|
|
281
268
|
const keyToUpload = ensureKeyHasAppPrefix(key);
|
|
282
269
|
|
|
283
270
|
const {
|
|
284
271
|
metadata = {},
|
|
285
|
-
cacheControl = 'public, max-age=31536000',
|
|
272
|
+
cacheControl = 'public, max-age=31536000',
|
|
286
273
|
storageClass = 'STANDARD',
|
|
287
274
|
serverSideEncryption = 'AES256',
|
|
288
275
|
bucketName,
|
|
289
|
-
preventOverwrite = false
|
|
276
|
+
preventOverwrite = false
|
|
290
277
|
} = options;
|
|
291
|
-
|
|
292
|
-
const client =
|
|
293
|
-
const config = getBucketConfig(bucketName);
|
|
278
|
+
|
|
279
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
294
280
|
const bucket = config.BUCKET_NAME;
|
|
295
281
|
|
|
296
282
|
debugS3('Uploading file to bucket=%s key=%s', bucket, keyToUpload);
|
|
297
283
|
|
|
298
|
-
// Add default metadata (supports metadata)
|
|
299
284
|
const uploadedAt = new Date().toISOString();
|
|
300
285
|
const defaultMetadata = {
|
|
301
286
|
'uploaded-at': uploadedAt,
|
|
@@ -314,31 +299,29 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
314
299
|
ServerSideEncryption: serverSideEncryption,
|
|
315
300
|
StorageClass: storageClass,
|
|
316
301
|
};
|
|
317
|
-
|
|
318
|
-
// Add conditional upload to prevent overwriting (atomic operation)
|
|
302
|
+
|
|
319
303
|
if (preventOverwrite) {
|
|
320
|
-
commandParams.IfNoneMatch = '*';
|
|
304
|
+
commandParams.IfNoneMatch = '*';
|
|
321
305
|
}
|
|
322
306
|
|
|
323
307
|
const command = new PutObjectCommand(commandParams);
|
|
324
|
-
|
|
308
|
+
|
|
325
309
|
const result = await client.send(command);
|
|
326
|
-
|
|
310
|
+
|
|
327
311
|
return {
|
|
328
312
|
...result,
|
|
329
313
|
fileSize: fileBuffer.length,
|
|
330
314
|
key: keyToUpload,
|
|
331
315
|
contentType,
|
|
332
316
|
uploadedAt
|
|
333
|
-
};
|
|
317
|
+
};
|
|
334
318
|
} catch (error) {
|
|
335
319
|
console.error(`Upload failed for key ${key}:`, error);
|
|
336
|
-
|
|
337
|
-
// Handle precondition failed (file already exists)
|
|
320
|
+
|
|
338
321
|
if (error.name === 'PreconditionFailed' || error.$metadata?.httpStatusCode === 412) {
|
|
339
322
|
throw new Error('File already exists');
|
|
340
323
|
}
|
|
341
|
-
|
|
324
|
+
|
|
342
325
|
throw new Error(`Upload failed: ${error.message}`);
|
|
343
326
|
}
|
|
344
327
|
}
|
|
@@ -351,7 +334,6 @@ export async function downloadFile(key, options = {}) {
|
|
|
351
334
|
throw new Error('Key is required');
|
|
352
335
|
}
|
|
353
336
|
|
|
354
|
-
// Ensure key is app-prefixed
|
|
355
337
|
keyToDownload = ensureKeyHasAppPrefix(key);
|
|
356
338
|
|
|
357
339
|
const {
|
|
@@ -362,12 +344,10 @@ export async function downloadFile(key, options = {}) {
|
|
|
362
344
|
responseContentType = null,
|
|
363
345
|
bucketName
|
|
364
346
|
} = options;
|
|
365
|
-
|
|
366
|
-
const client =
|
|
367
|
-
const config = getBucketConfig(bucketName);
|
|
347
|
+
|
|
348
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
368
349
|
const bucket = config.BUCKET_NAME;
|
|
369
350
|
|
|
370
|
-
// Log range requests for debugging
|
|
371
351
|
if (range) {
|
|
372
352
|
debugS3('Range request for %s: %s', keyToDownload, range);
|
|
373
353
|
}
|
|
@@ -381,23 +361,20 @@ export async function downloadFile(key, options = {}) {
|
|
|
381
361
|
...(responseCacheControl && { ResponseCacheControl: responseCacheControl }),
|
|
382
362
|
...(responseContentType && { ResponseContentType: responseContentType }),
|
|
383
363
|
});
|
|
384
|
-
|
|
364
|
+
|
|
385
365
|
const result = await client.send(command);
|
|
386
|
-
|
|
366
|
+
|
|
387
367
|
return {
|
|
388
368
|
...result,
|
|
389
369
|
key: keyToDownload,
|
|
390
|
-
};
|
|
370
|
+
};
|
|
391
371
|
} catch (error) {
|
|
392
372
|
console.error(`Download failed for key ${key}:`, error);
|
|
393
373
|
|
|
394
|
-
// Some S3-compatible endpoints return 304 Not Modified as a response to conditional GETs.
|
|
395
|
-
// The SDK surface may throw an error with metadata httpStatusCode=304 — treat this as a not-modified condition.
|
|
396
374
|
if (error && error.$metadata && error.$metadata.httpStatusCode === 304) {
|
|
397
375
|
return { notModified: true, key: keyToDownload };
|
|
398
376
|
}
|
|
399
377
|
|
|
400
|
-
// Handle specific S3 errors
|
|
401
378
|
if (error.name === 'NoSuchKey') {
|
|
402
379
|
throw new Error(`File not found: ${key}`);
|
|
403
380
|
} else if (error.name === 'AccessDenied') {
|
|
@@ -415,18 +392,17 @@ export async function deleteFile(key, bucketName) {
|
|
|
415
392
|
throw new Error('Key is required');
|
|
416
393
|
}
|
|
417
394
|
const keyToDelete = ensureKeyHasAppPrefix(key);
|
|
418
|
-
|
|
419
|
-
const client =
|
|
420
|
-
const config = getBucketConfig(bucketName);
|
|
395
|
+
|
|
396
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
421
397
|
const bucket = config.BUCKET_NAME;
|
|
422
398
|
|
|
423
399
|
const command = new DeleteObjectCommand({
|
|
424
400
|
Bucket: bucket,
|
|
425
401
|
Key: keyToDelete,
|
|
426
402
|
});
|
|
427
|
-
|
|
403
|
+
|
|
428
404
|
const result = await client.send(command);
|
|
429
|
-
|
|
405
|
+
|
|
430
406
|
return {
|
|
431
407
|
...result,
|
|
432
408
|
key: keyToDelete,
|
|
@@ -436,7 +412,7 @@ export async function deleteFile(key, bucketName) {
|
|
|
436
412
|
console.error(`Delete failed:`, error);
|
|
437
413
|
throw new Error(`Delete failed: ${error.message}`);
|
|
438
414
|
}
|
|
439
|
-
}
|
|
415
|
+
}
|
|
440
416
|
|
|
441
417
|
// Batch delete multiple files
|
|
442
418
|
export async function deleteFiles(keys, bucketName) {
|
|
@@ -444,12 +420,10 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
444
420
|
if (!keys || !Array.isArray(keys) || keys.length === 0) {
|
|
445
421
|
throw new Error('Keys array is required and must not be empty');
|
|
446
422
|
}
|
|
447
|
-
|
|
448
|
-
const client =
|
|
449
|
-
const config = getBucketConfig(bucketName);
|
|
423
|
+
|
|
424
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
450
425
|
const bucket = config.BUCKET_NAME;
|
|
451
426
|
|
|
452
|
-
// S3 allows max 1000 objects per delete request
|
|
453
427
|
const maxBatchSize = 1000;
|
|
454
428
|
const results = [];
|
|
455
429
|
let deletedCount = 0;
|
|
@@ -457,7 +431,7 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
457
431
|
|
|
458
432
|
for (let i = 0; i < keys.length; i += maxBatchSize) {
|
|
459
433
|
const batch = keys.slice(i, i + maxBatchSize).map(k => ({ Key: ensureKeyHasAppPrefix(k) }));
|
|
460
|
-
|
|
434
|
+
|
|
461
435
|
const command = new DeleteObjectsCommand({
|
|
462
436
|
Bucket: bucket,
|
|
463
437
|
Delete: {
|
|
@@ -484,12 +458,7 @@ export async function deleteFiles(keys, bucketName) {
|
|
|
484
458
|
}
|
|
485
459
|
}
|
|
486
460
|
|
|
487
|
-
//
|
|
488
|
-
/**
|
|
489
|
-
* Delete all objects under a folder prefix (recursive).
|
|
490
|
-
* Accepts either a folder key (may include trailing slash) or a relative prefix.
|
|
491
|
-
* Normalizes the prefix to include the configured APP_NAME and ensures trailing slash.
|
|
492
|
-
*/
|
|
461
|
+
// Delete all objects under a folder prefix (recursive)
|
|
493
462
|
export async function deleteFolder(prefix, bucketName) {
|
|
494
463
|
try {
|
|
495
464
|
if (!prefix) {
|
|
@@ -499,14 +468,12 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
499
468
|
const cleaned = String(prefix).replace(/^\/+/, '').replace(/\/+$/g, '');
|
|
500
469
|
const effectivePrefix = ensurePrefix(cleaned) + '/';
|
|
501
470
|
|
|
502
|
-
const client =
|
|
503
|
-
const config = getBucketConfig(bucketName);
|
|
471
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
504
472
|
const bucket = config.BUCKET_NAME;
|
|
505
473
|
|
|
506
474
|
const keysToDelete = [];
|
|
507
475
|
let continuationToken = null;
|
|
508
476
|
|
|
509
|
-
// Paginate through all objects under the prefix
|
|
510
477
|
do {
|
|
511
478
|
const listCmd = new ListObjectsV2Command({
|
|
512
479
|
Bucket: bucket,
|
|
@@ -523,7 +490,6 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
523
490
|
} while (continuationToken);
|
|
524
491
|
|
|
525
492
|
if (!keysToDelete.length) {
|
|
526
|
-
// Nothing found under this prefix; attempt to remove a zero-byte folder marker if present
|
|
527
493
|
const folderMarkerKey = effectivePrefix;
|
|
528
494
|
const exists = await fileExists(folderMarkerKey, bucketName);
|
|
529
495
|
if (exists) {
|
|
@@ -533,7 +499,6 @@ export async function deleteFolder(prefix, bucketName) {
|
|
|
533
499
|
return { deletedCount: 0, deletedAt: new Date().toISOString(), prefix: effectivePrefix };
|
|
534
500
|
}
|
|
535
501
|
|
|
536
|
-
// Use the existing deleteFiles helper which handles batching and error aggregation
|
|
537
502
|
const result = await deleteFiles(keysToDelete, bucketName);
|
|
538
503
|
|
|
539
504
|
return {
|
|
@@ -558,15 +523,12 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
558
523
|
startAfter = null,
|
|
559
524
|
bucketName
|
|
560
525
|
} = options;
|
|
561
|
-
|
|
562
|
-
// Enforce non-root prefix: default to APP_NAME and normalize
|
|
526
|
+
|
|
563
527
|
const effectivePrefix = ensurePrefix(prefix);
|
|
564
528
|
|
|
565
|
-
const client =
|
|
566
|
-
const config = getBucketConfig(bucketName);
|
|
529
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
567
530
|
const bucket = config.BUCKET_NAME;
|
|
568
531
|
|
|
569
|
-
|
|
570
532
|
const command = new ListObjectsV2Command({
|
|
571
533
|
Bucket: bucket,
|
|
572
534
|
Prefix: effectivePrefix,
|
|
@@ -576,9 +538,9 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
576
538
|
...(fetchOwner && { FetchOwner: fetchOwner }),
|
|
577
539
|
...(startAfter && { StartAfter: startAfter }),
|
|
578
540
|
});
|
|
579
|
-
|
|
541
|
+
|
|
580
542
|
const result = await client.send(command);
|
|
581
|
-
|
|
543
|
+
|
|
582
544
|
return {
|
|
583
545
|
...result,
|
|
584
546
|
requestedAt: new Date().toISOString(),
|
|
@@ -599,19 +561,18 @@ export async function getFileMetadata(key, bucketName) {
|
|
|
599
561
|
throw new Error('Key is required');
|
|
600
562
|
}
|
|
601
563
|
const keyToCheck = ensureKeyHasAppPrefix(key);
|
|
602
|
-
|
|
603
|
-
const client =
|
|
604
|
-
const config = getBucketConfig(bucketName);
|
|
564
|
+
|
|
565
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
605
566
|
const bucket = config.BUCKET_NAME;
|
|
606
567
|
|
|
607
568
|
const command = new HeadObjectCommand({
|
|
608
569
|
Bucket: bucket,
|
|
609
570
|
Key: keyToCheck,
|
|
610
571
|
});
|
|
611
|
-
|
|
572
|
+
|
|
612
573
|
const result = await client.send(command);
|
|
613
574
|
const queriedAt = new Date().toISOString();
|
|
614
|
-
|
|
575
|
+
|
|
615
576
|
return {
|
|
616
577
|
...result,
|
|
617
578
|
key: keyToCheck,
|
|
@@ -627,11 +588,11 @@ export async function getFileMetadata(key, bucketName) {
|
|
|
627
588
|
queriedAt
|
|
628
589
|
};
|
|
629
590
|
}
|
|
630
|
-
|
|
591
|
+
|
|
631
592
|
console.error(`Get metadata failed for key ${key}:`, error);
|
|
632
593
|
throw new Error(`Get metadata failed: ${error.message}`);
|
|
633
594
|
}
|
|
634
|
-
}
|
|
595
|
+
}
|
|
635
596
|
|
|
636
597
|
// Check if file exists
|
|
637
598
|
export async function fileExists(key, bucketName) {
|
|
@@ -653,23 +614,16 @@ export async function getFileSize(key, bucketName) {
|
|
|
653
614
|
}
|
|
654
615
|
}
|
|
655
616
|
|
|
656
|
-
// Export health check for external use
|
|
657
|
-
export { checkHealth };
|
|
658
|
-
|
|
659
617
|
// ---------------------------------------------------------------------------
|
|
660
618
|
// S3 Multipart Upload helpers
|
|
661
619
|
// ---------------------------------------------------------------------------
|
|
662
620
|
|
|
663
|
-
/**
|
|
664
|
-
* Initiate a multipart upload. Returns the S3 UploadId.
|
|
665
|
-
*/
|
|
666
621
|
export async function createMultipartUpload(key, contentType = 'application/octet-stream', metadata = {}, bucketName) {
|
|
667
622
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
668
|
-
const
|
|
669
|
-
const client = getBucketClient(bucketName);
|
|
623
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
670
624
|
const uploadedAt = new Date().toISOString();
|
|
671
625
|
const command = new CreateMultipartUploadCommand({
|
|
672
|
-
Bucket:
|
|
626
|
+
Bucket: config.BUCKET_NAME,
|
|
673
627
|
Key: keyToUse,
|
|
674
628
|
ContentType: contentType,
|
|
675
629
|
Metadata: {
|
|
@@ -682,16 +636,11 @@ export async function createMultipartUpload(key, contentType = 'application/octe
|
|
|
682
636
|
return { uploadId: result.UploadId, key: keyToUse };
|
|
683
637
|
}
|
|
684
638
|
|
|
685
|
-
/**
|
|
686
|
-
* Upload a single part. partNumber is 1-based.
|
|
687
|
-
* Returns the ETag needed for CompleteMultipartUpload.
|
|
688
|
-
*/
|
|
689
639
|
export async function uploadPart(key, uploadId, partNumber, buffer, bucketName) {
|
|
690
640
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
691
|
-
const
|
|
692
|
-
const client = getBucketClient(bucketName);
|
|
641
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
693
642
|
const command = new UploadPartCommand({
|
|
694
|
-
Bucket:
|
|
643
|
+
Bucket: config.BUCKET_NAME,
|
|
695
644
|
Key: keyToUse,
|
|
696
645
|
UploadId: uploadId,
|
|
697
646
|
PartNumber: partNumber,
|
|
@@ -702,17 +651,12 @@ export async function uploadPart(key, uploadId, partNumber, buffer, bucketName)
|
|
|
702
651
|
return { ETag: result.ETag, partNumber };
|
|
703
652
|
}
|
|
704
653
|
|
|
705
|
-
/**
|
|
706
|
-
* Complete a multipart upload.
|
|
707
|
-
* parts must be an array of { partNumber, ETag } sorted ascending by partNumber.
|
|
708
|
-
*/
|
|
709
654
|
export async function completeMultipartUpload(key, uploadId, parts, bucketName) {
|
|
710
655
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
711
|
-
const
|
|
712
|
-
const client = getBucketClient(bucketName);
|
|
656
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
713
657
|
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
714
658
|
const command = new CompleteMultipartUploadCommand({
|
|
715
|
-
Bucket:
|
|
659
|
+
Bucket: config.BUCKET_NAME,
|
|
716
660
|
Key: keyToUse,
|
|
717
661
|
UploadId: uploadId,
|
|
718
662
|
MultipartUpload: {
|
|
@@ -723,15 +667,11 @@ export async function completeMultipartUpload(key, uploadId, parts, bucketName)
|
|
|
723
667
|
return { key: keyToUse };
|
|
724
668
|
}
|
|
725
669
|
|
|
726
|
-
/**
|
|
727
|
-
* Abort an in-progress multipart upload, freeing all stored parts.
|
|
728
|
-
*/
|
|
729
670
|
export async function abortMultipartUpload(key, uploadId, bucketName) {
|
|
730
671
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
731
|
-
const
|
|
732
|
-
const client = getBucketClient(bucketName);
|
|
672
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
733
673
|
const command = new AbortMultipartUploadCommand({
|
|
734
|
-
Bucket:
|
|
674
|
+
Bucket: config.BUCKET_NAME,
|
|
735
675
|
Key: keyToUse,
|
|
736
676
|
UploadId: uploadId
|
|
737
677
|
});
|
|
@@ -744,10 +684,10 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
744
684
|
try {
|
|
745
685
|
const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner');
|
|
746
686
|
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
747
|
-
|
|
687
|
+
|
|
748
688
|
let command;
|
|
749
|
-
const
|
|
750
|
-
const resolvedBucketName =
|
|
689
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
690
|
+
const resolvedBucketName = config.BUCKET_NAME;
|
|
751
691
|
switch (operation) {
|
|
752
692
|
case 'getObject':
|
|
753
693
|
command = new GetObjectCommand({ Bucket: resolvedBucketName, Key: keyToUse });
|
|
@@ -758,9 +698,9 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
758
698
|
default:
|
|
759
699
|
throw new Error(`Unsupported operation: ${operation}`);
|
|
760
700
|
}
|
|
761
|
-
|
|
762
|
-
const signedUrl = await getSignedUrl(
|
|
763
|
-
|
|
701
|
+
|
|
702
|
+
const signedUrl = await getSignedUrl(client, command, { expiresIn });
|
|
703
|
+
|
|
764
704
|
return {
|
|
765
705
|
url: signedUrl,
|
|
766
706
|
key,
|
|
@@ -779,23 +719,16 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
779
719
|
// Multipart Upload Cleanup
|
|
780
720
|
// ---------------------------------------------------------------------------
|
|
781
721
|
|
|
782
|
-
/**
|
|
783
|
-
* List incomplete multipart uploads
|
|
784
|
-
* @param {string} prefix - Optional prefix to filter uploads
|
|
785
|
-
* @param {string} bucketName - Bucket name
|
|
786
|
-
* @returns {Promise<Array>} Array of incomplete multipart uploads
|
|
787
|
-
*/
|
|
788
722
|
export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
789
723
|
try {
|
|
790
724
|
const effectivePrefix = ensurePrefix(prefix || '');
|
|
791
|
-
const
|
|
792
|
-
|
|
793
|
-
|
|
725
|
+
const { client, config } = getBucketClientAndConfig(bucketName);
|
|
726
|
+
|
|
794
727
|
const command = new ListMultipartUploadsCommand({
|
|
795
|
-
Bucket:
|
|
728
|
+
Bucket: config.BUCKET_NAME,
|
|
796
729
|
Prefix: effectivePrefix
|
|
797
730
|
});
|
|
798
|
-
|
|
731
|
+
|
|
799
732
|
const result = await client.send(command);
|
|
800
733
|
return result.Uploads || [];
|
|
801
734
|
} catch (error) {
|
|
@@ -804,36 +737,29 @@ export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
|
804
737
|
}
|
|
805
738
|
}
|
|
806
739
|
|
|
807
|
-
/**
|
|
808
|
-
* Clean up old incomplete multipart uploads
|
|
809
|
-
* @param {number} olderThanDays - Delete uploads older than this many days (default: 7)
|
|
810
|
-
* @param {string} prefix - Optional prefix to filter uploads
|
|
811
|
-
* @param {string} bucketName - Bucket name
|
|
812
|
-
* @returns {Promise<Object>} Cleanup results with count of aborted uploads
|
|
813
|
-
*/
|
|
814
740
|
export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefix = '', bucketName) {
|
|
815
741
|
try {
|
|
816
742
|
const uploads = await listIncompleteMultipartUploads(prefix, bucketName);
|
|
817
|
-
|
|
743
|
+
|
|
818
744
|
if (!uploads.length) {
|
|
819
745
|
debugS3('No incomplete multipart uploads found');
|
|
820
746
|
return { abortedCount: 0, uploads: [] };
|
|
821
747
|
}
|
|
822
|
-
|
|
748
|
+
|
|
823
749
|
const cutoffDate = new Date();
|
|
824
750
|
cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
|
|
825
|
-
|
|
751
|
+
|
|
826
752
|
const uploadsToAbort = uploads.filter(upload => {
|
|
827
753
|
return upload.Initiated && new Date(upload.Initiated) < cutoffDate;
|
|
828
754
|
});
|
|
829
|
-
|
|
755
|
+
|
|
830
756
|
if (!uploadsToAbort.length) {
|
|
831
757
|
debugS3('No incomplete uploads older than %s days', olderThanDays);
|
|
832
758
|
return { abortedCount: 0, uploads: [] };
|
|
833
759
|
}
|
|
834
|
-
|
|
760
|
+
|
|
835
761
|
debugS3('Found %s incomplete uploads to clean up', uploadsToAbort.length);
|
|
836
|
-
|
|
762
|
+
|
|
837
763
|
const aborted = [];
|
|
838
764
|
for (const upload of uploadsToAbort) {
|
|
839
765
|
try {
|
|
@@ -848,7 +774,7 @@ export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefi
|
|
|
848
774
|
console.error(`[mbkbucket] Failed to abort upload ${upload.Key}:`, err);
|
|
849
775
|
}
|
|
850
776
|
}
|
|
851
|
-
|
|
777
|
+
|
|
852
778
|
return {
|
|
853
779
|
abortedCount: aborted.length,
|
|
854
780
|
uploads: aborted,
|