mbkbucket 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +5 -2
- package/README.md +59 -14
- package/index.d.ts +67 -3
- package/index.js +4 -79
- package/lib/bucket.js +658 -405
- package/lib/config/index.js +134 -1
- package/lib/s3.js +297 -54
- package/package.json +2 -1
- package/public/bucketadmin.js +139 -33
- package/views/bucketadmincontent.handlebars +3 -2
- package/views/mbkbucket_info.handlebars +226 -0
- package/views/info.handlebars +0 -1056
package/lib/config/index.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { createRequire } from "module";
|
|
2
|
+
import dotenv from "dotenv";
|
|
3
|
+
|
|
4
|
+
dotenv.config();
|
|
2
5
|
|
|
3
6
|
// Load package.json from mbkbucket package (not parent project)
|
|
4
7
|
const require = createRequire(import.meta.url);
|
|
@@ -54,4 +57,134 @@ async function checkVersion() {
|
|
|
54
57
|
console.warn(`[mbkbucket] Failed to check for updates: ${error.message}`);
|
|
55
58
|
}
|
|
56
59
|
}
|
|
57
|
-
|
|
60
|
+
|
|
61
|
+
// Comprehensive validation function
|
|
62
|
+
function validateConfiguration() {
|
|
63
|
+
const errors = [];
|
|
64
|
+
|
|
65
|
+
// Parse and validate mbkbucketVar
|
|
66
|
+
let mbkbucketVar;
|
|
67
|
+
try {
|
|
68
|
+
if (!process.env.mbkbucketVar) {
|
|
69
|
+
// If mbkbucketVar is not defined, use empty object (defaults will be applied later)
|
|
70
|
+
mbkbucketVar = {};
|
|
71
|
+
console.log(`[mbkbucket] mbkbucketVar not defined, using defaults`);
|
|
72
|
+
} else {
|
|
73
|
+
mbkbucketVar = JSON.parse(process.env.mbkbucketVar);
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
errors.push("Invalid JSON in process.env.mbkbucketVar");
|
|
77
|
+
throw new Error(`[mbkbucket] Configuration Error:\n - ${errors.join('\n - ')}`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (!mbkbucketVar || typeof mbkbucketVar !== 'object') {
|
|
81
|
+
errors.push("mbkbucketVar must be a valid object");
|
|
82
|
+
throw new Error(`[mbkbucket] Configuration Error:\n - ${errors.join('\n - ')}`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Apply defaults for optional keys (booleans now)
|
|
86
|
+
const defaults = {
|
|
87
|
+
p_view_inline: true,
|
|
88
|
+
publiView_enabled: false
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const usedDefaults = [];
|
|
92
|
+
Object.entries(defaults).forEach(([key, defaultVal]) => {
|
|
93
|
+
const current = mbkbucketVar[key];
|
|
94
|
+
if (current === undefined || (typeof current === 'string' && current.trim() === '')) {
|
|
95
|
+
mbkbucketVar[key] = defaultVal;
|
|
96
|
+
usedDefaults.push(key);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Helper to normalize various truthy/falsy representations
|
|
101
|
+
function normalizeBool(val) {
|
|
102
|
+
if (typeof val === 'boolean') return val;
|
|
103
|
+
if (typeof val === 'string') {
|
|
104
|
+
const norm = val.trim().toLowerCase();
|
|
105
|
+
if (['true', 't', '1', 'yes', 'y', 'on'].includes(norm)) return true;
|
|
106
|
+
if (['false', 'f', '0', 'no', 'n', 'off'].includes(norm)) return false;
|
|
107
|
+
return val; // return original so later validation will flag it
|
|
108
|
+
}
|
|
109
|
+
return val;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Normalize boolean-like values early so validation works on cleaned values;
|
|
113
|
+
// results are actual booleans.
|
|
114
|
+
['p_view_inline', 'publiView_enabled'].forEach(k => {
|
|
115
|
+
if (k in mbkbucketVar) {
|
|
116
|
+
mbkbucketVar[k] = normalizeBool(mbkbucketVar[k]);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Validate that applied values are proper booleans (after defaults and normalization)
|
|
121
|
+
// Note: p_view_inline and publiView_enabled are optional - defaults are applied above
|
|
122
|
+
['p_view_inline', 'publiView_enabled'].forEach(key => {
|
|
123
|
+
if (mbkbucketVar[key] !== undefined && typeof mbkbucketVar[key] !== 'boolean') {
|
|
124
|
+
errors.push(`mbkbucketVar.${key} must be a boolean if provided`);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
if (errors.length > 0) {
|
|
129
|
+
throw new Error(`[mbkbucket] Configuration Validation Failed:\n - ${errors.join('\n - ')}`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const summary = usedDefaults.length > 0 ? ` (defaults applied: ${usedDefaults.join(', ')})` : '';
|
|
133
|
+
console.log(`[mbkbucket] Configuration loaded${summary}`);
|
|
134
|
+
return mbkbucketVar;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Validate BucketConnection configuration
|
|
138
|
+
function validateBucketConnection() {
|
|
139
|
+
if (!process.env.BucketConnection) {
|
|
140
|
+
console.warn(`[mbkbucket] ⚠️ BucketConnection not configured. You need to set this environment variable.`);
|
|
141
|
+
console.warn(`[mbkbucket] Example format:`);
|
|
142
|
+
console.warn(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"my-bucket","ACCESS_KEY_ID":"key","SECRET_ACCESS_KEY":"secret","ENDPOINT":"https://..."}}`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
const bucketConfig = JSON.parse(process.env.BucketConnection);
|
|
148
|
+
|
|
149
|
+
if (!bucketConfig || typeof bucketConfig !== 'object' || Object.keys(bucketConfig).length === 0) {
|
|
150
|
+
throw new Error('BucketConnection must be a non-empty object');
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Validate each bucket configuration
|
|
154
|
+
const bucketNames = Object.keys(bucketConfig);
|
|
155
|
+
for (const bucketName of bucketNames) {
|
|
156
|
+
const config = bucketConfig[bucketName];
|
|
157
|
+
|
|
158
|
+
if (!config || typeof config !== 'object') {
|
|
159
|
+
throw new Error(`Bucket '${bucketName}' configuration must be an object, not a string. Remove quotes around the inner JSON object.`);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const requiredFields = ['BUCKET_NAME', 'ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'ENDPOINT'];
|
|
163
|
+
const missingFields = requiredFields.filter(field => !config[field]);
|
|
164
|
+
|
|
165
|
+
if (missingFields.length > 0) {
|
|
166
|
+
throw new Error(`Bucket '${bucketName}' is missing required fields: ${missingFields.join(', ')}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
console.log(`[mbkbucket] ✓ BucketConnection validated (${bucketNames.length} bucket(s): ${bucketNames.join(', ')})`);
|
|
171
|
+
return bucketConfig;
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.error(`[mbkbucket] ❌ BucketConnection validation failed: ${error.message}`);
|
|
174
|
+
console.error(`[mbkbucket] Current value (excerpt): ${process.env.BucketConnection.slice(0, 200)}...`);
|
|
175
|
+
console.error(`[mbkbucket] Correct format example:`);
|
|
176
|
+
console.error(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"my-bucket","ACCESS_KEY_ID":"key","SECRET_ACCESS_KEY":"secret","ENDPOINT":"https://endpoint.com"}}`);
|
|
177
|
+
console.error(`[mbkbucket] ⚠️ Note: Inner objects should NOT be quoted as strings!`);
|
|
178
|
+
throw new Error(`[mbkbucket] Invalid BucketConnection configuration`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Parse and validate mbkbucketVar once
|
|
183
|
+
const mbkbucketVar = validateConfiguration();
|
|
184
|
+
console.log(`[mbkbucket] mbkbucketVar configuration:`, mbkbucketVar);
|
|
185
|
+
|
|
186
|
+
// Validate BucketConnection
|
|
187
|
+
validateBucketConnection();
|
|
188
|
+
|
|
189
|
+
export { packageJson, appVersion, getLatestVersion, checkVersion };
|
|
190
|
+
export { mbkbucketVar };
|
package/lib/s3.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
1
|
+
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand } from '@aws-sdk/client-s3';
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import { mbkautheVar } from "mbkauthe";
|
|
4
4
|
|
|
@@ -11,35 +11,32 @@ function parseJsonEnv(envVar, fallback = {}) {
|
|
|
11
11
|
} catch (e) {
|
|
12
12
|
// Log a concise message including the error and a short excerpt of the env value when available
|
|
13
13
|
console.error(`❌ Error parsing JSON for envVar: ${envVar ? String(envVar).slice(0, 200) : 'undefined'} - ${e.message}`);
|
|
14
|
+
console.error(`💡 Tip: Check that inner objects are NOT quoted as strings. Use {"key":{"field":"value"}} not {"key":"{"field":"value"}"}`);
|
|
14
15
|
return fallback;
|
|
15
16
|
}
|
|
16
17
|
}
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const e2_1BucketConfig = parseJsonEnv(process.env.E2_1_Bucket, {});
|
|
21
|
-
|
|
22
|
-
// Function to get bucket config by name or from single configured env var key via mbkautheVar.bucket
|
|
23
|
-
export function getBucketConfig(bucketName = 'r2') {
|
|
24
|
-
// If a single configured bucket is set (the value is the ENV VAR name that contains the JSON config)
|
|
25
|
-
const configuredEnvKey = (mbkautheVar && mbkautheVar.bucket);
|
|
26
|
-
if (configuredEnvKey) {
|
|
27
|
-
const envVal = process.env[configuredEnvKey];
|
|
28
|
-
if (!envVal) throw new Error(`Configured bucket env var '${configuredEnvKey}' is not set`);
|
|
29
|
-
const parsed = parseJsonEnv(envVal, null);
|
|
30
|
-
if (!parsed || typeof parsed !== 'object') {
|
|
31
|
-
throw new Error(`Configured bucket env var '${configuredEnvKey}' contains invalid JSON`);
|
|
32
|
-
}
|
|
33
|
-
return parsed;
|
|
34
|
-
}
|
|
19
|
+
// Parse BucketConnection JSON that contains all bucket configurations
|
|
20
|
+
const allBucketConfigs = parseJsonEnv(process.env.BucketConnection, {});
|
|
35
21
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
22
|
+
// Helper to get default bucket name
|
|
23
|
+
function getDefaultBucketName() {
|
|
24
|
+
// First priority: use mbkautheVar.bucket if configured
|
|
25
|
+
if (mbkautheVar && mbkautheVar.bucket) {
|
|
26
|
+
return mbkautheVar.bucket;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Second priority: use first bucket in BucketConnection
|
|
30
|
+
const buckets = Object.keys(allBucketConfigs);
|
|
31
|
+
if (buckets.length > 0) {
|
|
32
|
+
return buckets[0];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Fallback: return null (will cause error in getBucketConfig)
|
|
36
|
+
return null;
|
|
39
37
|
}
|
|
40
38
|
|
|
41
|
-
//
|
|
42
|
-
export const bucketClient = getBucketClient();
|
|
39
|
+
// (bucketClient singleton exported below, after cache setup)
|
|
43
40
|
|
|
44
41
|
// Helper: get configured APP_NAME; prefer mbkautheVar, then env var
|
|
45
42
|
export function getAppName() {
|
|
@@ -51,6 +48,17 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
51
48
|
const app = getAppName();
|
|
52
49
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
53
50
|
if (!key) throw new Error('Key is required');
|
|
51
|
+
|
|
52
|
+
// Security: Detect path traversal attempts
|
|
53
|
+
if (key.includes('../') || key.includes('..\\') || /\.\.[\\/]/.test(key)) {
|
|
54
|
+
throw new Error('Path traversal detected: invalid key');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Security: Block null bytes and control characters
|
|
58
|
+
if (/[\x00-\x1f\x7f]/.test(key)) {
|
|
59
|
+
throw new Error('Invalid characters in key');
|
|
60
|
+
}
|
|
61
|
+
|
|
54
62
|
let cleaned = String(key).replace(/^\/+/, '');
|
|
55
63
|
// Collapse repeated app prefixes (app/app/... -> app/...)
|
|
56
64
|
const appPrefixRegex = new RegExp(`^(?:${app}\/)+`);
|
|
@@ -66,6 +74,17 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
66
74
|
export function ensurePrefix(prefix = '') {
|
|
67
75
|
const app = getAppName();
|
|
68
76
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
77
|
+
|
|
78
|
+
// Security: Detect path traversal attempts
|
|
79
|
+
if (prefix && (prefix.includes('../') || prefix.includes('..\\') || /\.\.[\\/]/.test(prefix))) {
|
|
80
|
+
throw new Error('Path traversal detected: invalid prefix');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Security: Block null bytes and control characters
|
|
84
|
+
if (prefix && /[\x00-\x1f\x7f]/.test(prefix)) {
|
|
85
|
+
throw new Error('Invalid characters in prefix');
|
|
86
|
+
}
|
|
87
|
+
|
|
69
88
|
// Only strip LEADING slashes. Preserve trailing slashes for directory listing.
|
|
70
89
|
let p = String(prefix || '').replace(/^\/+/, '');
|
|
71
90
|
|
|
@@ -83,11 +102,57 @@ export function ensurePrefix(prefix = '') {
|
|
|
83
102
|
return `${app}/${p}`;
|
|
84
103
|
}
|
|
85
104
|
|
|
86
|
-
//
|
|
87
|
-
|
|
88
|
-
|
|
105
|
+
// ---------------------------------------------------------------------------
|
|
106
|
+
// Singleton S3 client cache — one reused client per bucket key.
|
|
107
|
+
// Avoids creating a new S3Client (and losing connection pooling) on every call.
|
|
108
|
+
// ---------------------------------------------------------------------------
|
|
109
|
+
const _clientCache = new Map();
|
|
110
|
+
const _configCache = new Map();
|
|
111
|
+
|
|
112
|
+
export function getBucketConfig(bucketName) {
|
|
113
|
+
// Resolve default bucket name if not provided
|
|
114
|
+
if (!bucketName) {
|
|
115
|
+
bucketName = getDefaultBucketName();
|
|
116
|
+
if (!bucketName) {
|
|
117
|
+
throw new Error('No bucket name provided and no default bucket configured. Please set mbkautheVar.bucket or configure BucketConnection.');
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const cacheKey = bucketName;
|
|
122
|
+
if (_configCache.has(cacheKey)) return _configCache.get(cacheKey);
|
|
123
|
+
|
|
124
|
+
// Look up bucket config from BucketConnection JSON
|
|
125
|
+
if (!allBucketConfigs || Object.keys(allBucketConfigs).length === 0) {
|
|
126
|
+
console.error(`[mbkbucket] ❌ BucketConnection is not configured or failed to parse`);
|
|
127
|
+
console.error(`[mbkbucket] Please set BucketConnection in your .env file with this format:`);
|
|
128
|
+
console.error(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"...","ACCESS_KEY_ID":"...","SECRET_ACCESS_KEY":"...","ENDPOINT":"https://..."}}`);
|
|
129
|
+
console.error(`[mbkbucket] ⚠️ Important: Inner objects must NOT be quoted as strings!`);
|
|
130
|
+
console.error(`[mbkbucket] Wrong: {"r2":"{\\"BUCKET_NAME\\":\\"...\\"}"} ❌`);
|
|
131
|
+
console.error(`[mbkbucket] Right: {"r2":{"BUCKET_NAME":"..."}} ✓`);
|
|
132
|
+
throw new Error(`BucketConnection environment variable is not set or empty. Please configure it with your bucket connections.`);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const cfg = allBucketConfigs[bucketName];
|
|
136
|
+
|
|
137
|
+
if (!cfg) {
|
|
138
|
+
throw new Error(`Bucket '${bucketName}' not found in BucketConnection. Available buckets: ${Object.keys(allBucketConfigs).join(', ')}`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
_configCache.set(cacheKey, cfg);
|
|
142
|
+
return cfg;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Function to get S3 client for specific bucket — returns cached singleton
|
|
146
|
+
export function getBucketClient(bucketName) {
|
|
147
|
+
// Resolve default bucket name if not provided
|
|
148
|
+
if (!bucketName) {
|
|
149
|
+
bucketName = getDefaultBucketName();
|
|
150
|
+
}
|
|
89
151
|
|
|
90
|
-
return
|
|
152
|
+
if (_clientCache.has(bucketName)) return _clientCache.get(bucketName);
|
|
153
|
+
|
|
154
|
+
const config = getBucketConfig(bucketName);
|
|
155
|
+
const client = new S3Client({
|
|
91
156
|
region: config.region || 'auto',
|
|
92
157
|
endpoint: config.ENDPOINT,
|
|
93
158
|
credentials: {
|
|
@@ -107,10 +172,16 @@ export function getBucketClient(bucketName = 'r2') {
|
|
|
107
172
|
useAccelerateEndpoint: false,
|
|
108
173
|
forcePathStyle: true,
|
|
109
174
|
});
|
|
175
|
+
|
|
176
|
+
_clientCache.set(bucketName, client);
|
|
177
|
+
return client;
|
|
110
178
|
}
|
|
111
179
|
|
|
112
|
-
//
|
|
113
|
-
|
|
180
|
+
// Default client singleton — reused by any external consumers that import it
|
|
181
|
+
export const bucketClient = getBucketClient();
|
|
182
|
+
|
|
183
|
+
// Health check for connection (moved up for startup test)
|
|
184
|
+
async function checkHealth() {
|
|
114
185
|
try {
|
|
115
186
|
const startTime = Date.now();
|
|
116
187
|
|
|
@@ -148,14 +219,14 @@ async function checkR2Health() {
|
|
|
148
219
|
(async () => {
|
|
149
220
|
try {
|
|
150
221
|
console.log('[mbkbucket] mbkautheVar snapshot:', { APP_NAME: mbkautheVar?.APP_NAME, bucket: mbkautheVar?.bucket });
|
|
151
|
-
const health = await
|
|
222
|
+
const health = await checkHealth();
|
|
152
223
|
if (health.status === 'healthy') {
|
|
153
224
|
console.log(`[mbkbucket] Connected to bucket: ${health.bucket} (${health.responseTime}ms)`);
|
|
154
225
|
} else {
|
|
155
|
-
console.error(`[mbkbucket]
|
|
226
|
+
console.error(`[mbkbucket] S3 connection failed:`, health.error);
|
|
156
227
|
}
|
|
157
228
|
} catch (err) {
|
|
158
|
-
console.error("[mbkbucket]
|
|
229
|
+
console.error("[mbkbucket] S3 connection test error:", err.message);
|
|
159
230
|
}
|
|
160
231
|
})();
|
|
161
232
|
|
|
@@ -175,7 +246,8 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
175
246
|
cacheControl = 'public, max-age=31536000', // 1 year default
|
|
176
247
|
storageClass = 'STANDARD',
|
|
177
248
|
serverSideEncryption = 'AES256',
|
|
178
|
-
bucketName
|
|
249
|
+
bucketName,
|
|
250
|
+
preventOverwrite = false // New option to prevent race conditions
|
|
179
251
|
} = options;
|
|
180
252
|
|
|
181
253
|
const client = getBucketClient(bucketName);
|
|
@@ -184,7 +256,7 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
184
256
|
|
|
185
257
|
console.log(`[mbkbucket] Uploading file to bucket=${bucket} key=${keyToUpload}`);
|
|
186
258
|
|
|
187
|
-
// Add default metadata (
|
|
259
|
+
// Add default metadata (supports metadata)
|
|
188
260
|
const defaultMetadata = {
|
|
189
261
|
'uploaded-at': new Date().toISOString(),
|
|
190
262
|
'file-size': fileBuffer.length.toString(),
|
|
@@ -192,7 +264,7 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
192
264
|
...metadata
|
|
193
265
|
};
|
|
194
266
|
|
|
195
|
-
const
|
|
267
|
+
const commandParams = {
|
|
196
268
|
Bucket: bucket,
|
|
197
269
|
Key: keyToUpload,
|
|
198
270
|
Body: fileBuffer,
|
|
@@ -201,8 +273,14 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
201
273
|
Metadata: defaultMetadata,
|
|
202
274
|
ServerSideEncryption: serverSideEncryption,
|
|
203
275
|
StorageClass: storageClass,
|
|
204
|
-
|
|
205
|
-
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
// Add conditional upload to prevent overwriting (atomic operation)
|
|
279
|
+
if (preventOverwrite) {
|
|
280
|
+
commandParams.IfNoneMatch = '*'; // Only upload if object doesn't exist
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const command = new PutObjectCommand(commandParams);
|
|
206
284
|
|
|
207
285
|
const result = await client.send(command);
|
|
208
286
|
|
|
@@ -215,6 +293,12 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
215
293
|
};
|
|
216
294
|
} catch (error) {
|
|
217
295
|
console.error(`Upload failed for key ${key}:`, error);
|
|
296
|
+
|
|
297
|
+
// Handle precondition failed (file already exists)
|
|
298
|
+
if (error.name === 'PreconditionFailed' || error.$metadata?.httpStatusCode === 412) {
|
|
299
|
+
throw new Error('File already exists');
|
|
300
|
+
}
|
|
301
|
+
|
|
218
302
|
throw new Error(`Upload failed: ${error.message}`);
|
|
219
303
|
}
|
|
220
304
|
}
|
|
@@ -236,7 +320,7 @@ export async function downloadFile(key, options = {}) {
|
|
|
236
320
|
ifModifiedSince = null,
|
|
237
321
|
responseCacheControl = null,
|
|
238
322
|
responseContentType = null,
|
|
239
|
-
bucketName
|
|
323
|
+
bucketName
|
|
240
324
|
} = options;
|
|
241
325
|
|
|
242
326
|
const client = getBucketClient(bucketName);
|
|
@@ -262,16 +346,9 @@ export async function downloadFile(key, options = {}) {
|
|
|
262
346
|
const result = await client.send(command);
|
|
263
347
|
const downloadTime = Date.now() - startTime;
|
|
264
348
|
|
|
265
|
-
// Log performance for large files
|
|
266
|
-
if (result.ContentLength > 10 * 1024 * 1024) {
|
|
267
|
-
console.log(`[mbkbucket] Large file download: ${keyToDownload}, Size: ${result.ContentLength} bytes, Time: ${downloadTime}ms`);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
349
|
return {
|
|
271
350
|
...result,
|
|
272
351
|
key: keyToDownload,
|
|
273
|
-
downloadedAt: new Date().toISOString(),
|
|
274
|
-
downloadTime
|
|
275
352
|
};
|
|
276
353
|
} catch (error) {
|
|
277
354
|
console.error(`Download failed for key ${key}:`, error);
|
|
@@ -294,7 +371,7 @@ export async function downloadFile(key, options = {}) {
|
|
|
294
371
|
}
|
|
295
372
|
|
|
296
373
|
// Delete file with enhanced error handling
|
|
297
|
-
export async function deleteFile(key, bucketName
|
|
374
|
+
export async function deleteFile(key, bucketName) {
|
|
298
375
|
try {
|
|
299
376
|
if (!key) {
|
|
300
377
|
throw new Error('Key is required');
|
|
@@ -324,7 +401,7 @@ export async function deleteFile(key, bucketName = 'r2') {
|
|
|
324
401
|
}
|
|
325
402
|
|
|
326
403
|
// Batch delete multiple files
|
|
327
|
-
export async function deleteFiles(keys, bucketName
|
|
404
|
+
export async function deleteFiles(keys, bucketName) {
|
|
328
405
|
try {
|
|
329
406
|
if (!keys || !Array.isArray(keys) || keys.length === 0) {
|
|
330
407
|
throw new Error('Keys array is required and must not be empty');
|
|
@@ -371,7 +448,7 @@ export async function deleteFiles(keys, bucketName = 'r2') {
|
|
|
371
448
|
* Accepts either a folder key (may include trailing slash) or a relative prefix.
|
|
372
449
|
* Normalizes the prefix to include the configured APP_NAME and ensures trailing slash.
|
|
373
450
|
*/
|
|
374
|
-
export async function deleteFolder(prefix, bucketName
|
|
451
|
+
export async function deleteFolder(prefix, bucketName) {
|
|
375
452
|
try {
|
|
376
453
|
if (!prefix) {
|
|
377
454
|
throw new Error('Prefix is required');
|
|
@@ -437,7 +514,7 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
437
514
|
delimiter = null,
|
|
438
515
|
fetchOwner = false,
|
|
439
516
|
startAfter = null,
|
|
440
|
-
bucketName
|
|
517
|
+
bucketName
|
|
441
518
|
} = options;
|
|
442
519
|
|
|
443
520
|
// Enforce non-root prefix: default to APP_NAME and normalize
|
|
@@ -447,7 +524,6 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
447
524
|
const config = getBucketConfig(bucketName);
|
|
448
525
|
const bucket = config.BUCKET_NAME;
|
|
449
526
|
|
|
450
|
-
console.log(`[mbkbucket] Listing objects in bucket=${bucket} prefix=${effectivePrefix}`);
|
|
451
527
|
|
|
452
528
|
const command = new ListObjectsV2Command({
|
|
453
529
|
Bucket: bucket,
|
|
@@ -475,7 +551,7 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
475
551
|
}
|
|
476
552
|
|
|
477
553
|
// Get file metadata without downloading
|
|
478
|
-
export async function getFileMetadata(key, bucketName
|
|
554
|
+
export async function getFileMetadata(key, bucketName) {
|
|
479
555
|
try {
|
|
480
556
|
if (!key) {
|
|
481
557
|
throw new Error('Key is required');
|
|
@@ -514,7 +590,7 @@ export async function getFileMetadata(key, bucketName = 'r2') {
|
|
|
514
590
|
}
|
|
515
591
|
|
|
516
592
|
// Check if file exists
|
|
517
|
-
export async function fileExists(key, bucketName
|
|
593
|
+
export async function fileExists(key, bucketName) {
|
|
518
594
|
try {
|
|
519
595
|
const metadata = await getFileMetadata(key, bucketName);
|
|
520
596
|
return metadata.exists;
|
|
@@ -524,7 +600,7 @@ export async function fileExists(key, bucketName = 'r2') {
|
|
|
524
600
|
}
|
|
525
601
|
|
|
526
602
|
// Get file size without downloading
|
|
527
|
-
export async function getFileSize(key, bucketName
|
|
603
|
+
export async function getFileSize(key, bucketName) {
|
|
528
604
|
try {
|
|
529
605
|
const metadata = await getFileMetadata(key, bucketName);
|
|
530
606
|
return metadata.exists ? metadata.ContentLength : null;
|
|
@@ -534,7 +610,89 @@ export async function getFileSize(key, bucketName = 'r2') {
|
|
|
534
610
|
}
|
|
535
611
|
|
|
536
612
|
// Export health check for external use
|
|
537
|
-
export {
|
|
613
|
+
export { checkHealth };
|
|
614
|
+
|
|
615
|
+
// ---------------------------------------------------------------------------
|
|
616
|
+
// S3 Multipart Upload helpers
|
|
617
|
+
// ---------------------------------------------------------------------------
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Initiate a multipart upload. Returns the S3 UploadId.
|
|
621
|
+
*/
|
|
622
|
+
export async function createMultipartUpload(key, contentType = 'application/octet-stream', metadata = {}) {
|
|
623
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
624
|
+
const cfg = getBucketConfig();
|
|
625
|
+
const client = getBucketClient();
|
|
626
|
+
const command = new CreateMultipartUploadCommand({
|
|
627
|
+
Bucket: cfg.BUCKET_NAME,
|
|
628
|
+
Key: keyToUse,
|
|
629
|
+
ContentType: contentType,
|
|
630
|
+
Metadata: {
|
|
631
|
+
'upload-source': 'web-portal',
|
|
632
|
+
'uploaded-at': new Date().toISOString(),
|
|
633
|
+
...metadata
|
|
634
|
+
}
|
|
635
|
+
});
|
|
636
|
+
const result = await client.send(command);
|
|
637
|
+
return { uploadId: result.UploadId, key: keyToUse };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Upload a single part. partNumber is 1-based.
|
|
642
|
+
* Returns the ETag needed for CompleteMultipartUpload.
|
|
643
|
+
*/
|
|
644
|
+
export async function uploadPart(key, uploadId, partNumber, buffer) {
|
|
645
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
646
|
+
const cfg = getBucketConfig();
|
|
647
|
+
const client = getBucketClient();
|
|
648
|
+
const command = new UploadPartCommand({
|
|
649
|
+
Bucket: cfg.BUCKET_NAME,
|
|
650
|
+
Key: keyToUse,
|
|
651
|
+
UploadId: uploadId,
|
|
652
|
+
PartNumber: partNumber,
|
|
653
|
+
Body: buffer,
|
|
654
|
+
ContentLength: buffer.length
|
|
655
|
+
});
|
|
656
|
+
const result = await client.send(command);
|
|
657
|
+
return { ETag: result.ETag, partNumber };
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Complete a multipart upload.
|
|
662
|
+
* parts must be an array of { partNumber, ETag } sorted ascending by partNumber.
|
|
663
|
+
*/
|
|
664
|
+
export async function completeMultipartUpload(key, uploadId, parts) {
|
|
665
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
666
|
+
const cfg = getBucketConfig();
|
|
667
|
+
const client = getBucketClient();
|
|
668
|
+
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
669
|
+
const command = new CompleteMultipartUploadCommand({
|
|
670
|
+
Bucket: cfg.BUCKET_NAME,
|
|
671
|
+
Key: keyToUse,
|
|
672
|
+
UploadId: uploadId,
|
|
673
|
+
MultipartUpload: {
|
|
674
|
+
Parts: sorted.map(p => ({ PartNumber: p.partNumber, ETag: p.ETag }))
|
|
675
|
+
}
|
|
676
|
+
});
|
|
677
|
+
await client.send(command);
|
|
678
|
+
return { key: keyToUse };
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Abort an in-progress multipart upload, freeing all stored parts.
|
|
683
|
+
*/
|
|
684
|
+
export async function abortMultipartUpload(key, uploadId) {
|
|
685
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
686
|
+
const cfg = getBucketConfig();
|
|
687
|
+
const client = getBucketClient();
|
|
688
|
+
const command = new AbortMultipartUploadCommand({
|
|
689
|
+
Bucket: cfg.BUCKET_NAME,
|
|
690
|
+
Key: keyToUse,
|
|
691
|
+
UploadId: uploadId
|
|
692
|
+
});
|
|
693
|
+
await client.send(command);
|
|
694
|
+
return { key: keyToUse, abortedAt: new Date().toISOString() };
|
|
695
|
+
}
|
|
538
696
|
|
|
539
697
|
// Generate signed URL for temporary access
|
|
540
698
|
export async function generateSignedUrl(key, operation = 'getObject', expiresIn = 3600) {
|
|
@@ -571,3 +729,88 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
571
729
|
throw new Error(`Generate signed URL failed: ${error.message}`);
|
|
572
730
|
}
|
|
573
731
|
}
|
|
732
|
+
|
|
733
|
+
// ---------------------------------------------------------------------------
|
|
734
|
+
// Multipart Upload Cleanup
|
|
735
|
+
// ---------------------------------------------------------------------------
|
|
736
|
+
|
|
737
|
+
/**
|
|
738
|
+
* List incomplete multipart uploads
|
|
739
|
+
* @param {string} prefix - Optional prefix to filter uploads
|
|
740
|
+
* @param {string} bucketName - Bucket name
|
|
741
|
+
* @returns {Promise<Array>} Array of incomplete multipart uploads
|
|
742
|
+
*/
|
|
743
|
+
export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
744
|
+
try {
|
|
745
|
+
const effectivePrefix = prefix ? ensurePrefix(prefix) : getAppName();
|
|
746
|
+
const cfg = getBucketConfig(bucketName);
|
|
747
|
+
const client = getBucketClient(bucketName);
|
|
748
|
+
|
|
749
|
+
const command = new ListMultipartUploadsCommand({
|
|
750
|
+
Bucket: cfg.BUCKET_NAME,
|
|
751
|
+
Prefix: effectivePrefix
|
|
752
|
+
});
|
|
753
|
+
|
|
754
|
+
const result = await client.send(command);
|
|
755
|
+
return result.Uploads || [];
|
|
756
|
+
} catch (error) {
|
|
757
|
+
console.error('Failed to list incomplete multipart uploads:', error);
|
|
758
|
+
throw new Error(`Failed to list incomplete uploads: ${error.message}`);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* Clean up old incomplete multipart uploads
|
|
764
|
+
* @param {number} olderThanDays - Delete uploads older than this many days (default: 7)
|
|
765
|
+
* @param {string} prefix - Optional prefix to filter uploads
|
|
766
|
+
* @param {string} bucketName - Bucket name
|
|
767
|
+
* @returns {Promise<Object>} Cleanup results with count of aborted uploads
|
|
768
|
+
*/
|
|
769
|
+
export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefix = '', bucketName) {
|
|
770
|
+
try {
|
|
771
|
+
const uploads = await listIncompleteMultipartUploads(prefix, bucketName);
|
|
772
|
+
|
|
773
|
+
if (!uploads.length) {
|
|
774
|
+
console.log('[mbkbucket] No incomplete multipart uploads found');
|
|
775
|
+
return { abortedCount: 0, uploads: [] };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
const cutoffDate = new Date();
|
|
779
|
+
cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
|
|
780
|
+
|
|
781
|
+
const uploadsToAbort = uploads.filter(upload => {
|
|
782
|
+
return upload.Initiated && new Date(upload.Initiated) < cutoffDate;
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
if (!uploadsToAbort.length) {
|
|
786
|
+
console.log(`[mbkbucket] No incomplete uploads older than ${olderThanDays} days`);
|
|
787
|
+
return { abortedCount: 0, uploads: [] };
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
console.log(`[mbkbucket] Found ${uploadsToAbort.length} incomplete uploads to clean up`);
|
|
791
|
+
|
|
792
|
+
const aborted = [];
|
|
793
|
+
for (const upload of uploadsToAbort) {
|
|
794
|
+
try {
|
|
795
|
+
await abortMultipartUpload(upload.Key, upload.UploadId);
|
|
796
|
+
aborted.push({
|
|
797
|
+
key: upload.Key,
|
|
798
|
+
uploadId: upload.UploadId,
|
|
799
|
+
initiated: upload.Initiated
|
|
800
|
+
});
|
|
801
|
+
console.log(`[mbkbucket] Aborted incomplete upload: ${upload.Key} (initiated: ${upload.Initiated})`);
|
|
802
|
+
} catch (err) {
|
|
803
|
+
console.error(`[mbkbucket] Failed to abort upload ${upload.Key}:`, err);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
return {
|
|
808
|
+
abortedCount: aborted.length,
|
|
809
|
+
uploads: aborted,
|
|
810
|
+
cleanedAt: new Date().toISOString()
|
|
811
|
+
};
|
|
812
|
+
} catch (error) {
|
|
813
|
+
console.error('Cleanup of incomplete multipart uploads failed:', error);
|
|
814
|
+
throw new Error(`Cleanup failed: ${error.message}`);
|
|
815
|
+
}
|
|
816
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mbkbucket",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "S3 Bucket manager for mbktech.org apps",
|
|
5
5
|
"author": "Muhammad Bin Khalid",
|
|
6
6
|
"homepage": "https://github.com/MIbnEKhalid/mbkbucket#readme",
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
},
|
|
15
15
|
"dependencies": {
|
|
16
16
|
"@aws-sdk/client-s3": "^3.968.0",
|
|
17
|
+
"@aws-sdk/s3-request-presigner": "^3.1000.0",
|
|
17
18
|
"compression": "^1.8.0",
|
|
18
19
|
"dotenv": "^16.6.1",
|
|
19
20
|
"express": "^4.22.1",
|