mbkbucket 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +5 -2
- package/README.md +59 -14
- package/index.d.ts +67 -3
- package/index.js +5 -79
- package/lib/bucket.js +719 -411
- package/lib/config/index.js +134 -1
- package/lib/s3.js +304 -56
- package/package.json +2 -1
- package/public/bucketadmin.css +164 -19
- package/public/bucketadmin.js +317 -94
- package/views/bucket.handlebars +1 -1
- package/views/bucketadmincontent.handlebars +17 -14
- package/views/mbkbucket_info.handlebars +226 -0
- package/views/info.handlebars +0 -1056
package/lib/config/index.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { createRequire } from "module";
|
|
2
|
+
import dotenv from "dotenv";
|
|
3
|
+
|
|
4
|
+
dotenv.config();
|
|
2
5
|
|
|
3
6
|
// Load package.json from mbkbucket package (not parent project)
|
|
4
7
|
const require = createRequire(import.meta.url);
|
|
@@ -54,4 +57,134 @@ async function checkVersion() {
|
|
|
54
57
|
console.warn(`[mbkbucket] Failed to check for updates: ${error.message}`);
|
|
55
58
|
}
|
|
56
59
|
}
|
|
57
|
-
|
|
60
|
+
|
|
61
|
+
// Comprehensive validation function
|
|
62
|
+
function validateConfiguration() {
|
|
63
|
+
const errors = [];
|
|
64
|
+
|
|
65
|
+
// Parse and validate mbkbucketVar
|
|
66
|
+
let mbkbucketVar;
|
|
67
|
+
try {
|
|
68
|
+
if (!process.env.mbkbucketVar) {
|
|
69
|
+
// If mbkbucketVar is not defined, use empty object (defaults will be applied later)
|
|
70
|
+
mbkbucketVar = {};
|
|
71
|
+
console.log(`[mbkbucket] mbkbucketVar not defined, using defaults`);
|
|
72
|
+
} else {
|
|
73
|
+
mbkbucketVar = JSON.parse(process.env.mbkbucketVar);
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
errors.push("Invalid JSON in process.env.mbkbucketVar");
|
|
77
|
+
throw new Error(`[mbkbucket] Configuration Error:\n - ${errors.join('\n - ')}`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (!mbkbucketVar || typeof mbkbucketVar !== 'object') {
|
|
81
|
+
errors.push("mbkbucketVar must be a valid object");
|
|
82
|
+
throw new Error(`[mbkbucket] Configuration Error:\n - ${errors.join('\n - ')}`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Apply defaults for optional keys (booleans now)
|
|
86
|
+
const defaults = {
|
|
87
|
+
p_view_inline: true,
|
|
88
|
+
publiView_enabled: false
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const usedDefaults = [];
|
|
92
|
+
Object.entries(defaults).forEach(([key, defaultVal]) => {
|
|
93
|
+
const current = mbkbucketVar[key];
|
|
94
|
+
if (current === undefined || (typeof current === 'string' && current.trim() === '')) {
|
|
95
|
+
mbkbucketVar[key] = defaultVal;
|
|
96
|
+
usedDefaults.push(key);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Helper to normalize various truthy/falsy representations
|
|
101
|
+
function normalizeBool(val) {
|
|
102
|
+
if (typeof val === 'boolean') return val;
|
|
103
|
+
if (typeof val === 'string') {
|
|
104
|
+
const norm = val.trim().toLowerCase();
|
|
105
|
+
if (['true', 't', '1', 'yes', 'y', 'on'].includes(norm)) return true;
|
|
106
|
+
if (['false', 'f', '0', 'no', 'n', 'off'].includes(norm)) return false;
|
|
107
|
+
return val; // return original so later validation will flag it
|
|
108
|
+
}
|
|
109
|
+
return val;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Normalize boolean-like values early so validation works on cleaned values;
|
|
113
|
+
// results are actual booleans.
|
|
114
|
+
['p_view_inline', 'publiView_enabled'].forEach(k => {
|
|
115
|
+
if (k in mbkbucketVar) {
|
|
116
|
+
mbkbucketVar[k] = normalizeBool(mbkbucketVar[k]);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Validate that applied values are proper booleans (after defaults and normalization)
|
|
121
|
+
// Note: p_view_inline and publiView_enabled are optional - defaults are applied above
|
|
122
|
+
['p_view_inline', 'publiView_enabled'].forEach(key => {
|
|
123
|
+
if (mbkbucketVar[key] !== undefined && typeof mbkbucketVar[key] !== 'boolean') {
|
|
124
|
+
errors.push(`mbkbucketVar.${key} must be a boolean if provided`);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
if (errors.length > 0) {
|
|
129
|
+
throw new Error(`[mbkbucket] Configuration Validation Failed:\n - ${errors.join('\n - ')}`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const summary = usedDefaults.length > 0 ? ` (defaults applied: ${usedDefaults.join(', ')})` : '';
|
|
133
|
+
console.log(`[mbkbucket] Configuration loaded${summary}`);
|
|
134
|
+
return mbkbucketVar;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Validate BucketConnection configuration
|
|
138
|
+
function validateBucketConnection() {
|
|
139
|
+
if (!process.env.BucketConnection) {
|
|
140
|
+
console.warn(`[mbkbucket] ⚠️ BucketConnection not configured. You need to set this environment variable.`);
|
|
141
|
+
console.warn(`[mbkbucket] Example format:`);
|
|
142
|
+
console.warn(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"my-bucket","ACCESS_KEY_ID":"key","SECRET_ACCESS_KEY":"secret","ENDPOINT":"https://..."}}`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
const bucketConfig = JSON.parse(process.env.BucketConnection);
|
|
148
|
+
|
|
149
|
+
if (!bucketConfig || typeof bucketConfig !== 'object' || Object.keys(bucketConfig).length === 0) {
|
|
150
|
+
throw new Error('BucketConnection must be a non-empty object');
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Validate each bucket configuration
|
|
154
|
+
const bucketNames = Object.keys(bucketConfig);
|
|
155
|
+
for (const bucketName of bucketNames) {
|
|
156
|
+
const config = bucketConfig[bucketName];
|
|
157
|
+
|
|
158
|
+
if (!config || typeof config !== 'object') {
|
|
159
|
+
throw new Error(`Bucket '${bucketName}' configuration must be an object, not a string. Remove quotes around the inner JSON object.`);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const requiredFields = ['BUCKET_NAME', 'ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'ENDPOINT'];
|
|
163
|
+
const missingFields = requiredFields.filter(field => !config[field]);
|
|
164
|
+
|
|
165
|
+
if (missingFields.length > 0) {
|
|
166
|
+
throw new Error(`Bucket '${bucketName}' is missing required fields: ${missingFields.join(', ')}`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
console.log(`[mbkbucket] ✓ BucketConnection validated (${bucketNames.length} bucket(s): ${bucketNames.join(', ')})`);
|
|
171
|
+
return bucketConfig;
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.error(`[mbkbucket] ❌ BucketConnection validation failed: ${error.message}`);
|
|
174
|
+
console.error(`[mbkbucket] Current value (excerpt): ${process.env.BucketConnection.slice(0, 200)}...`);
|
|
175
|
+
console.error(`[mbkbucket] Correct format example:`);
|
|
176
|
+
console.error(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"my-bucket","ACCESS_KEY_ID":"key","SECRET_ACCESS_KEY":"secret","ENDPOINT":"https://endpoint.com"}}`);
|
|
177
|
+
console.error(`[mbkbucket] ⚠️ Note: Inner objects should NOT be quoted as strings!`);
|
|
178
|
+
throw new Error(`[mbkbucket] Invalid BucketConnection configuration`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Parse and validate mbkbucketVar once
|
|
183
|
+
const mbkbucketVar = validateConfiguration();
|
|
184
|
+
console.log(`[mbkbucket] mbkbucketVar configuration:`, mbkbucketVar);
|
|
185
|
+
|
|
186
|
+
// Validate BucketConnection
|
|
187
|
+
validateBucketConnection();
|
|
188
|
+
|
|
189
|
+
export { packageJson, appVersion, getLatestVersion, checkVersion };
|
|
190
|
+
export { mbkbucketVar };
|
package/lib/s3.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
1
|
+
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, HeadObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, ListMultipartUploadsCommand } from '@aws-sdk/client-s3';
|
|
2
2
|
import dotenv from "dotenv";
|
|
3
3
|
import { mbkautheVar } from "mbkauthe";
|
|
4
4
|
|
|
@@ -11,35 +11,32 @@ function parseJsonEnv(envVar, fallback = {}) {
|
|
|
11
11
|
} catch (e) {
|
|
12
12
|
// Log a concise message including the error and a short excerpt of the env value when available
|
|
13
13
|
console.error(`❌ Error parsing JSON for envVar: ${envVar ? String(envVar).slice(0, 200) : 'undefined'} - ${e.message}`);
|
|
14
|
+
console.error(`💡 Tip: Check that inner objects are NOT quoted as strings. Use {"key":{"field":"value"}} not {"key":"{"field":"value"}"}`);
|
|
14
15
|
return fallback;
|
|
15
16
|
}
|
|
16
17
|
}
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const e2_1BucketConfig = parseJsonEnv(process.env.E2_1_Bucket, {});
|
|
21
|
-
|
|
22
|
-
// Function to get bucket config by name or from single configured env var key via mbkautheVar.bucket
|
|
23
|
-
export function getBucketConfig(bucketName = 'r2') {
|
|
24
|
-
// If a single configured bucket is set (the value is the ENV VAR name that contains the JSON config)
|
|
25
|
-
const configuredEnvKey = (mbkautheVar && mbkautheVar.bucket);
|
|
26
|
-
if (configuredEnvKey) {
|
|
27
|
-
const envVal = process.env[configuredEnvKey];
|
|
28
|
-
if (!envVal) throw new Error(`Configured bucket env var '${configuredEnvKey}' is not set`);
|
|
29
|
-
const parsed = parseJsonEnv(envVal, null);
|
|
30
|
-
if (!parsed || typeof parsed !== 'object') {
|
|
31
|
-
throw new Error(`Configured bucket env var '${configuredEnvKey}' contains invalid JSON`);
|
|
32
|
-
}
|
|
33
|
-
return parsed;
|
|
34
|
-
}
|
|
19
|
+
// Parse BucketConnection JSON that contains all bucket configurations
|
|
20
|
+
const allBucketConfigs = parseJsonEnv(process.env.BucketConnection, {});
|
|
35
21
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
22
|
+
// Helper to get default bucket name
|
|
23
|
+
function getDefaultBucketName() {
|
|
24
|
+
// First priority: use mbkautheVar.bucket if configured
|
|
25
|
+
if (mbkautheVar && mbkautheVar.bucket) {
|
|
26
|
+
return mbkautheVar.bucket;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Second priority: use first bucket in BucketConnection
|
|
30
|
+
const buckets = Object.keys(allBucketConfigs);
|
|
31
|
+
if (buckets.length > 0) {
|
|
32
|
+
return buckets[0];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Fallback: return null (will cause error in getBucketConfig)
|
|
36
|
+
return null;
|
|
39
37
|
}
|
|
40
38
|
|
|
41
|
-
//
|
|
42
|
-
export const bucketClient = getBucketClient();
|
|
39
|
+
// (bucketClient singleton exported below, after cache setup)
|
|
43
40
|
|
|
44
41
|
// Helper: get configured APP_NAME; prefer mbkautheVar, then env var
|
|
45
42
|
export function getAppName() {
|
|
@@ -51,6 +48,17 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
51
48
|
const app = getAppName();
|
|
52
49
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
53
50
|
if (!key) throw new Error('Key is required');
|
|
51
|
+
|
|
52
|
+
// Security: Detect path traversal attempts
|
|
53
|
+
if (key.includes('../') || key.includes('..\\') || /\.\.[\\/]/.test(key)) {
|
|
54
|
+
throw new Error('Path traversal detected: invalid key');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Security: Block null bytes and control characters
|
|
58
|
+
if (/[\x00-\x1f\x7f]/.test(key)) {
|
|
59
|
+
throw new Error('Invalid characters in key');
|
|
60
|
+
}
|
|
61
|
+
|
|
54
62
|
let cleaned = String(key).replace(/^\/+/, '');
|
|
55
63
|
// Collapse repeated app prefixes (app/app/... -> app/...)
|
|
56
64
|
const appPrefixRegex = new RegExp(`^(?:${app}\/)+`);
|
|
@@ -66,10 +74,26 @@ export function ensureKeyHasAppPrefix(key = '') {
|
|
|
66
74
|
export function ensurePrefix(prefix = '') {
|
|
67
75
|
const app = getAppName();
|
|
68
76
|
if (!app) throw new Error('APP_NAME is not configured; set mbkautheVar.APP_NAME or MBKAUTHE_APP_NAME');
|
|
69
|
-
|
|
77
|
+
|
|
78
|
+
// Security: Detect path traversal attempts
|
|
79
|
+
if (prefix && (prefix.includes('../') || prefix.includes('..\\') || /\.\.[\\/]/.test(prefix))) {
|
|
80
|
+
throw new Error('Path traversal detected: invalid prefix');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Security: Block null bytes and control characters
|
|
84
|
+
if (prefix && /[\x00-\x1f\x7f]/.test(prefix)) {
|
|
85
|
+
throw new Error('Invalid characters in prefix');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Only strip LEADING slashes. Preserve trailing slashes for directory listing.
|
|
89
|
+
let p = String(prefix || '').replace(/^\/+/, '');
|
|
90
|
+
|
|
70
91
|
if (!p) return `${app}`;
|
|
92
|
+
|
|
71
93
|
// If prefix already equals app or already starts with app/, normalize and return single app prefix
|
|
72
|
-
if (p === app) return app;
|
|
94
|
+
if (p === app) return app; // If p is exactly "app", we return "app". (Caller should append / if listing root)
|
|
95
|
+
if (p === `${app}/`) return `${app}/`;
|
|
96
|
+
|
|
73
97
|
if (p.startsWith(`${app}/`)) {
|
|
74
98
|
// collapse repeated app/ sequences
|
|
75
99
|
const appPrefixRegex = new RegExp(`^(?:${app}\/)+`);
|
|
@@ -78,11 +102,57 @@ export function ensurePrefix(prefix = '') {
|
|
|
78
102
|
return `${app}/${p}`;
|
|
79
103
|
}
|
|
80
104
|
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
|
|
105
|
+
// ---------------------------------------------------------------------------
|
|
106
|
+
// Singleton S3 client cache — one reused client per bucket key.
|
|
107
|
+
// Avoids creating a new S3Client (and losing connection pooling) on every call.
|
|
108
|
+
// ---------------------------------------------------------------------------
|
|
109
|
+
const _clientCache = new Map();
|
|
110
|
+
const _configCache = new Map();
|
|
111
|
+
|
|
112
|
+
export function getBucketConfig(bucketName) {
|
|
113
|
+
// Resolve default bucket name if not provided
|
|
114
|
+
if (!bucketName) {
|
|
115
|
+
bucketName = getDefaultBucketName();
|
|
116
|
+
if (!bucketName) {
|
|
117
|
+
throw new Error('No bucket name provided and no default bucket configured. Please set mbkautheVar.bucket or configure BucketConnection.');
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const cacheKey = bucketName;
|
|
122
|
+
if (_configCache.has(cacheKey)) return _configCache.get(cacheKey);
|
|
123
|
+
|
|
124
|
+
// Look up bucket config from BucketConnection JSON
|
|
125
|
+
if (!allBucketConfigs || Object.keys(allBucketConfigs).length === 0) {
|
|
126
|
+
console.error(`[mbkbucket] ❌ BucketConnection is not configured or failed to parse`);
|
|
127
|
+
console.error(`[mbkbucket] Please set BucketConnection in your .env file with this format:`);
|
|
128
|
+
console.error(`[mbkbucket] BucketConnection={"r2":{"BUCKET_NAME":"...","ACCESS_KEY_ID":"...","SECRET_ACCESS_KEY":"...","ENDPOINT":"https://..."}}`);
|
|
129
|
+
console.error(`[mbkbucket] ⚠️ Important: Inner objects must NOT be quoted as strings!`);
|
|
130
|
+
console.error(`[mbkbucket] Wrong: {"r2":"{\\"BUCKET_NAME\\":\\"...\\"}"} ❌`);
|
|
131
|
+
console.error(`[mbkbucket] Right: {"r2":{"BUCKET_NAME":"..."}} ✓`);
|
|
132
|
+
throw new Error(`BucketConnection environment variable is not set or empty. Please configure it with your bucket connections.`);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const cfg = allBucketConfigs[bucketName];
|
|
136
|
+
|
|
137
|
+
if (!cfg) {
|
|
138
|
+
throw new Error(`Bucket '${bucketName}' not found in BucketConnection. Available buckets: ${Object.keys(allBucketConfigs).join(', ')}`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
_configCache.set(cacheKey, cfg);
|
|
142
|
+
return cfg;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Function to get S3 client for specific bucket — returns cached singleton
|
|
146
|
+
export function getBucketClient(bucketName) {
|
|
147
|
+
// Resolve default bucket name if not provided
|
|
148
|
+
if (!bucketName) {
|
|
149
|
+
bucketName = getDefaultBucketName();
|
|
150
|
+
}
|
|
84
151
|
|
|
85
|
-
return
|
|
152
|
+
if (_clientCache.has(bucketName)) return _clientCache.get(bucketName);
|
|
153
|
+
|
|
154
|
+
const config = getBucketConfig(bucketName);
|
|
155
|
+
const client = new S3Client({
|
|
86
156
|
region: config.region || 'auto',
|
|
87
157
|
endpoint: config.ENDPOINT,
|
|
88
158
|
credentials: {
|
|
@@ -102,10 +172,16 @@ export function getBucketClient(bucketName = 'r2') {
|
|
|
102
172
|
useAccelerateEndpoint: false,
|
|
103
173
|
forcePathStyle: true,
|
|
104
174
|
});
|
|
175
|
+
|
|
176
|
+
_clientCache.set(bucketName, client);
|
|
177
|
+
return client;
|
|
105
178
|
}
|
|
106
179
|
|
|
107
|
-
//
|
|
108
|
-
|
|
180
|
+
// Default client singleton — reused by any external consumers that import it
|
|
181
|
+
export const bucketClient = getBucketClient();
|
|
182
|
+
|
|
183
|
+
// Health check for connection (moved up for startup test)
|
|
184
|
+
async function checkHealth() {
|
|
109
185
|
try {
|
|
110
186
|
const startTime = Date.now();
|
|
111
187
|
|
|
@@ -143,14 +219,14 @@ async function checkR2Health() {
|
|
|
143
219
|
(async () => {
|
|
144
220
|
try {
|
|
145
221
|
console.log('[mbkbucket] mbkautheVar snapshot:', { APP_NAME: mbkautheVar?.APP_NAME, bucket: mbkautheVar?.bucket });
|
|
146
|
-
const health = await
|
|
222
|
+
const health = await checkHealth();
|
|
147
223
|
if (health.status === 'healthy') {
|
|
148
224
|
console.log(`[mbkbucket] Connected to bucket: ${health.bucket} (${health.responseTime}ms)`);
|
|
149
225
|
} else {
|
|
150
|
-
console.error(`[mbkbucket]
|
|
226
|
+
console.error(`[mbkbucket] S3 connection failed:`, health.error);
|
|
151
227
|
}
|
|
152
228
|
} catch (err) {
|
|
153
|
-
console.error("[mbkbucket]
|
|
229
|
+
console.error("[mbkbucket] S3 connection test error:", err.message);
|
|
154
230
|
}
|
|
155
231
|
})();
|
|
156
232
|
|
|
@@ -170,7 +246,8 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
170
246
|
cacheControl = 'public, max-age=31536000', // 1 year default
|
|
171
247
|
storageClass = 'STANDARD',
|
|
172
248
|
serverSideEncryption = 'AES256',
|
|
173
|
-
bucketName
|
|
249
|
+
bucketName,
|
|
250
|
+
preventOverwrite = false // New option to prevent race conditions
|
|
174
251
|
} = options;
|
|
175
252
|
|
|
176
253
|
const client = getBucketClient(bucketName);
|
|
@@ -179,7 +256,7 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
179
256
|
|
|
180
257
|
console.log(`[mbkbucket] Uploading file to bucket=${bucket} key=${keyToUpload}`);
|
|
181
258
|
|
|
182
|
-
// Add default metadata (
|
|
259
|
+
// Add default metadata (supports metadata)
|
|
183
260
|
const defaultMetadata = {
|
|
184
261
|
'uploaded-at': new Date().toISOString(),
|
|
185
262
|
'file-size': fileBuffer.length.toString(),
|
|
@@ -187,7 +264,7 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
187
264
|
...metadata
|
|
188
265
|
};
|
|
189
266
|
|
|
190
|
-
const
|
|
267
|
+
const commandParams = {
|
|
191
268
|
Bucket: bucket,
|
|
192
269
|
Key: keyToUpload,
|
|
193
270
|
Body: fileBuffer,
|
|
@@ -196,8 +273,14 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
196
273
|
Metadata: defaultMetadata,
|
|
197
274
|
ServerSideEncryption: serverSideEncryption,
|
|
198
275
|
StorageClass: storageClass,
|
|
199
|
-
|
|
200
|
-
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
// Add conditional upload to prevent overwriting (atomic operation)
|
|
279
|
+
if (preventOverwrite) {
|
|
280
|
+
commandParams.IfNoneMatch = '*'; // Only upload if object doesn't exist
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const command = new PutObjectCommand(commandParams);
|
|
201
284
|
|
|
202
285
|
const result = await client.send(command);
|
|
203
286
|
|
|
@@ -210,6 +293,12 @@ export async function uploadFile(key, fileBuffer, contentType, options = {}) {
|
|
|
210
293
|
};
|
|
211
294
|
} catch (error) {
|
|
212
295
|
console.error(`Upload failed for key ${key}:`, error);
|
|
296
|
+
|
|
297
|
+
// Handle precondition failed (file already exists)
|
|
298
|
+
if (error.name === 'PreconditionFailed' || error.$metadata?.httpStatusCode === 412) {
|
|
299
|
+
throw new Error('File already exists');
|
|
300
|
+
}
|
|
301
|
+
|
|
213
302
|
throw new Error(`Upload failed: ${error.message}`);
|
|
214
303
|
}
|
|
215
304
|
}
|
|
@@ -231,7 +320,7 @@ export async function downloadFile(key, options = {}) {
|
|
|
231
320
|
ifModifiedSince = null,
|
|
232
321
|
responseCacheControl = null,
|
|
233
322
|
responseContentType = null,
|
|
234
|
-
bucketName
|
|
323
|
+
bucketName
|
|
235
324
|
} = options;
|
|
236
325
|
|
|
237
326
|
const client = getBucketClient(bucketName);
|
|
@@ -257,16 +346,9 @@ export async function downloadFile(key, options = {}) {
|
|
|
257
346
|
const result = await client.send(command);
|
|
258
347
|
const downloadTime = Date.now() - startTime;
|
|
259
348
|
|
|
260
|
-
// Log performance for large files
|
|
261
|
-
if (result.ContentLength > 10 * 1024 * 1024) {
|
|
262
|
-
console.log(`[mbkbucket] Large file download: ${keyToDownload}, Size: ${result.ContentLength} bytes, Time: ${downloadTime}ms`);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
349
|
return {
|
|
266
350
|
...result,
|
|
267
351
|
key: keyToDownload,
|
|
268
|
-
downloadedAt: new Date().toISOString(),
|
|
269
|
-
downloadTime
|
|
270
352
|
};
|
|
271
353
|
} catch (error) {
|
|
272
354
|
console.error(`Download failed for key ${key}:`, error);
|
|
@@ -289,7 +371,7 @@ export async function downloadFile(key, options = {}) {
|
|
|
289
371
|
}
|
|
290
372
|
|
|
291
373
|
// Delete file with enhanced error handling
|
|
292
|
-
export async function deleteFile(key, bucketName
|
|
374
|
+
export async function deleteFile(key, bucketName) {
|
|
293
375
|
try {
|
|
294
376
|
if (!key) {
|
|
295
377
|
throw new Error('Key is required');
|
|
@@ -319,7 +401,7 @@ export async function deleteFile(key, bucketName = 'r2') {
|
|
|
319
401
|
}
|
|
320
402
|
|
|
321
403
|
// Batch delete multiple files
|
|
322
|
-
export async function deleteFiles(keys, bucketName
|
|
404
|
+
export async function deleteFiles(keys, bucketName) {
|
|
323
405
|
try {
|
|
324
406
|
if (!keys || !Array.isArray(keys) || keys.length === 0) {
|
|
325
407
|
throw new Error('Keys array is required and must not be empty');
|
|
@@ -366,7 +448,7 @@ export async function deleteFiles(keys, bucketName = 'r2') {
|
|
|
366
448
|
* Accepts either a folder key (may include trailing slash) or a relative prefix.
|
|
367
449
|
* Normalizes the prefix to include the configured APP_NAME and ensures trailing slash.
|
|
368
450
|
*/
|
|
369
|
-
export async function deleteFolder(prefix, bucketName
|
|
451
|
+
export async function deleteFolder(prefix, bucketName) {
|
|
370
452
|
try {
|
|
371
453
|
if (!prefix) {
|
|
372
454
|
throw new Error('Prefix is required');
|
|
@@ -432,7 +514,7 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
432
514
|
delimiter = null,
|
|
433
515
|
fetchOwner = false,
|
|
434
516
|
startAfter = null,
|
|
435
|
-
bucketName
|
|
517
|
+
bucketName
|
|
436
518
|
} = options;
|
|
437
519
|
|
|
438
520
|
// Enforce non-root prefix: default to APP_NAME and normalize
|
|
@@ -442,7 +524,6 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
442
524
|
const config = getBucketConfig(bucketName);
|
|
443
525
|
const bucket = config.BUCKET_NAME;
|
|
444
526
|
|
|
445
|
-
console.log(`[mbkbucket] Listing objects in bucket=${bucket} prefix=${effectivePrefix}`);
|
|
446
527
|
|
|
447
528
|
const command = new ListObjectsV2Command({
|
|
448
529
|
Bucket: bucket,
|
|
@@ -470,7 +551,7 @@ export async function listfiles(prefix = '', options = {}) {
|
|
|
470
551
|
}
|
|
471
552
|
|
|
472
553
|
// Get file metadata without downloading
|
|
473
|
-
export async function getFileMetadata(key, bucketName
|
|
554
|
+
export async function getFileMetadata(key, bucketName) {
|
|
474
555
|
try {
|
|
475
556
|
if (!key) {
|
|
476
557
|
throw new Error('Key is required');
|
|
@@ -509,7 +590,7 @@ export async function getFileMetadata(key, bucketName = 'r2') {
|
|
|
509
590
|
}
|
|
510
591
|
|
|
511
592
|
// Check if file exists
|
|
512
|
-
export async function fileExists(key, bucketName
|
|
593
|
+
export async function fileExists(key, bucketName) {
|
|
513
594
|
try {
|
|
514
595
|
const metadata = await getFileMetadata(key, bucketName);
|
|
515
596
|
return metadata.exists;
|
|
@@ -519,7 +600,7 @@ export async function fileExists(key, bucketName = 'r2') {
|
|
|
519
600
|
}
|
|
520
601
|
|
|
521
602
|
// Get file size without downloading
|
|
522
|
-
export async function getFileSize(key, bucketName
|
|
603
|
+
export async function getFileSize(key, bucketName) {
|
|
523
604
|
try {
|
|
524
605
|
const metadata = await getFileMetadata(key, bucketName);
|
|
525
606
|
return metadata.exists ? metadata.ContentLength : null;
|
|
@@ -529,7 +610,89 @@ export async function getFileSize(key, bucketName = 'r2') {
|
|
|
529
610
|
}
|
|
530
611
|
|
|
531
612
|
// Export health check for external use
|
|
532
|
-
export {
|
|
613
|
+
export { checkHealth };
|
|
614
|
+
|
|
615
|
+
// ---------------------------------------------------------------------------
|
|
616
|
+
// S3 Multipart Upload helpers
|
|
617
|
+
// ---------------------------------------------------------------------------
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Initiate a multipart upload. Returns the S3 UploadId.
|
|
621
|
+
*/
|
|
622
|
+
export async function createMultipartUpload(key, contentType = 'application/octet-stream', metadata = {}) {
|
|
623
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
624
|
+
const cfg = getBucketConfig();
|
|
625
|
+
const client = getBucketClient();
|
|
626
|
+
const command = new CreateMultipartUploadCommand({
|
|
627
|
+
Bucket: cfg.BUCKET_NAME,
|
|
628
|
+
Key: keyToUse,
|
|
629
|
+
ContentType: contentType,
|
|
630
|
+
Metadata: {
|
|
631
|
+
'upload-source': 'web-portal',
|
|
632
|
+
'uploaded-at': new Date().toISOString(),
|
|
633
|
+
...metadata
|
|
634
|
+
}
|
|
635
|
+
});
|
|
636
|
+
const result = await client.send(command);
|
|
637
|
+
return { uploadId: result.UploadId, key: keyToUse };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Upload a single part. partNumber is 1-based.
|
|
642
|
+
* Returns the ETag needed for CompleteMultipartUpload.
|
|
643
|
+
*/
|
|
644
|
+
export async function uploadPart(key, uploadId, partNumber, buffer) {
|
|
645
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
646
|
+
const cfg = getBucketConfig();
|
|
647
|
+
const client = getBucketClient();
|
|
648
|
+
const command = new UploadPartCommand({
|
|
649
|
+
Bucket: cfg.BUCKET_NAME,
|
|
650
|
+
Key: keyToUse,
|
|
651
|
+
UploadId: uploadId,
|
|
652
|
+
PartNumber: partNumber,
|
|
653
|
+
Body: buffer,
|
|
654
|
+
ContentLength: buffer.length
|
|
655
|
+
});
|
|
656
|
+
const result = await client.send(command);
|
|
657
|
+
return { ETag: result.ETag, partNumber };
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Complete a multipart upload.
|
|
662
|
+
* parts must be an array of { partNumber, ETag } sorted ascending by partNumber.
|
|
663
|
+
*/
|
|
664
|
+
export async function completeMultipartUpload(key, uploadId, parts) {
|
|
665
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
666
|
+
const cfg = getBucketConfig();
|
|
667
|
+
const client = getBucketClient();
|
|
668
|
+
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
669
|
+
const command = new CompleteMultipartUploadCommand({
|
|
670
|
+
Bucket: cfg.BUCKET_NAME,
|
|
671
|
+
Key: keyToUse,
|
|
672
|
+
UploadId: uploadId,
|
|
673
|
+
MultipartUpload: {
|
|
674
|
+
Parts: sorted.map(p => ({ PartNumber: p.partNumber, ETag: p.ETag }))
|
|
675
|
+
}
|
|
676
|
+
});
|
|
677
|
+
await client.send(command);
|
|
678
|
+
return { key: keyToUse };
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Abort an in-progress multipart upload, freeing all stored parts.
|
|
683
|
+
*/
|
|
684
|
+
export async function abortMultipartUpload(key, uploadId) {
|
|
685
|
+
const keyToUse = ensureKeyHasAppPrefix(key);
|
|
686
|
+
const cfg = getBucketConfig();
|
|
687
|
+
const client = getBucketClient();
|
|
688
|
+
const command = new AbortMultipartUploadCommand({
|
|
689
|
+
Bucket: cfg.BUCKET_NAME,
|
|
690
|
+
Key: keyToUse,
|
|
691
|
+
UploadId: uploadId
|
|
692
|
+
});
|
|
693
|
+
await client.send(command);
|
|
694
|
+
return { key: keyToUse, abortedAt: new Date().toISOString() };
|
|
695
|
+
}
|
|
533
696
|
|
|
534
697
|
// Generate signed URL for temporary access
|
|
535
698
|
export async function generateSignedUrl(key, operation = 'getObject', expiresIn = 3600) {
|
|
@@ -566,3 +729,88 @@ export async function generateSignedUrl(key, operation = 'getObject', expiresIn
|
|
|
566
729
|
throw new Error(`Generate signed URL failed: ${error.message}`);
|
|
567
730
|
}
|
|
568
731
|
}
|
|
732
|
+
|
|
733
|
+
// ---------------------------------------------------------------------------
|
|
734
|
+
// Multipart Upload Cleanup
|
|
735
|
+
// ---------------------------------------------------------------------------
|
|
736
|
+
|
|
737
|
+
/**
|
|
738
|
+
* List incomplete multipart uploads
|
|
739
|
+
* @param {string} prefix - Optional prefix to filter uploads
|
|
740
|
+
* @param {string} bucketName - Bucket name
|
|
741
|
+
* @returns {Promise<Array>} Array of incomplete multipart uploads
|
|
742
|
+
*/
|
|
743
|
+
export async function listIncompleteMultipartUploads(prefix = '', bucketName) {
|
|
744
|
+
try {
|
|
745
|
+
const effectivePrefix = prefix ? ensurePrefix(prefix) : getAppName();
|
|
746
|
+
const cfg = getBucketConfig(bucketName);
|
|
747
|
+
const client = getBucketClient(bucketName);
|
|
748
|
+
|
|
749
|
+
const command = new ListMultipartUploadsCommand({
|
|
750
|
+
Bucket: cfg.BUCKET_NAME,
|
|
751
|
+
Prefix: effectivePrefix
|
|
752
|
+
});
|
|
753
|
+
|
|
754
|
+
const result = await client.send(command);
|
|
755
|
+
return result.Uploads || [];
|
|
756
|
+
} catch (error) {
|
|
757
|
+
console.error('Failed to list incomplete multipart uploads:', error);
|
|
758
|
+
throw new Error(`Failed to list incomplete uploads: ${error.message}`);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* Clean up old incomplete multipart uploads
|
|
764
|
+
* @param {number} olderThanDays - Delete uploads older than this many days (default: 7)
|
|
765
|
+
* @param {string} prefix - Optional prefix to filter uploads
|
|
766
|
+
* @param {string} bucketName - Bucket name
|
|
767
|
+
* @returns {Promise<Object>} Cleanup results with count of aborted uploads
|
|
768
|
+
*/
|
|
769
|
+
export async function cleanupIncompleteMultipartUploads(olderThanDays = 7, prefix = '', bucketName) {
|
|
770
|
+
try {
|
|
771
|
+
const uploads = await listIncompleteMultipartUploads(prefix, bucketName);
|
|
772
|
+
|
|
773
|
+
if (!uploads.length) {
|
|
774
|
+
console.log('[mbkbucket] No incomplete multipart uploads found');
|
|
775
|
+
return { abortedCount: 0, uploads: [] };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
const cutoffDate = new Date();
|
|
779
|
+
cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
|
|
780
|
+
|
|
781
|
+
const uploadsToAbort = uploads.filter(upload => {
|
|
782
|
+
return upload.Initiated && new Date(upload.Initiated) < cutoffDate;
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
if (!uploadsToAbort.length) {
|
|
786
|
+
console.log(`[mbkbucket] No incomplete uploads older than ${olderThanDays} days`);
|
|
787
|
+
return { abortedCount: 0, uploads: [] };
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
console.log(`[mbkbucket] Found ${uploadsToAbort.length} incomplete uploads to clean up`);
|
|
791
|
+
|
|
792
|
+
const aborted = [];
|
|
793
|
+
for (const upload of uploadsToAbort) {
|
|
794
|
+
try {
|
|
795
|
+
await abortMultipartUpload(upload.Key, upload.UploadId);
|
|
796
|
+
aborted.push({
|
|
797
|
+
key: upload.Key,
|
|
798
|
+
uploadId: upload.UploadId,
|
|
799
|
+
initiated: upload.Initiated
|
|
800
|
+
});
|
|
801
|
+
console.log(`[mbkbucket] Aborted incomplete upload: ${upload.Key} (initiated: ${upload.Initiated})`);
|
|
802
|
+
} catch (err) {
|
|
803
|
+
console.error(`[mbkbucket] Failed to abort upload ${upload.Key}:`, err);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
return {
|
|
808
|
+
abortedCount: aborted.length,
|
|
809
|
+
uploads: aborted,
|
|
810
|
+
cleanedAt: new Date().toISOString()
|
|
811
|
+
};
|
|
812
|
+
} catch (error) {
|
|
813
|
+
console.error('Cleanup of incomplete multipart uploads failed:', error);
|
|
814
|
+
throw new Error(`Cleanup failed: ${error.message}`);
|
|
815
|
+
}
|
|
816
|
+
}
|