@onlineapps/conn-base-storage 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API.md +618 -0
- package/README.md +341 -0
- package/SHARED_URL_ADDRESSING.md +258 -0
- package/coverage/base.css +224 -0
- package/coverage/block-navigation.js +87 -0
- package/coverage/clover.xml +213 -0
- package/coverage/coverage-final.json +3 -0
- package/coverage/favicon.png +0 -0
- package/coverage/index.html +131 -0
- package/coverage/index.js.html +1579 -0
- package/coverage/internal-url-adapter.js.html +604 -0
- package/coverage/lcov-report/base.css +224 -0
- package/coverage/lcov-report/block-navigation.js +87 -0
- package/coverage/lcov-report/favicon.png +0 -0
- package/coverage/lcov-report/index.html +131 -0
- package/coverage/lcov-report/index.js.html +1579 -0
- package/coverage/lcov-report/internal-url-adapter.js.html +604 -0
- package/coverage/lcov-report/prettify.css +1 -0
- package/coverage/lcov-report/prettify.js +2 -0
- package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/coverage/lcov-report/sorter.js +210 -0
- package/coverage/lcov.info +434 -0
- package/coverage/prettify.css +1 -0
- package/coverage/prettify.js +2 -0
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +210 -0
- package/jest.config.js +13 -0
- package/jest.integration.config.js +9 -0
- package/package.json +33 -0
- package/src/index.js +853 -0
- package/src/internal-url-adapter.js +174 -0
- package/src/sharedUrlAdapter.js +258 -0
- package/test/component/storage.component.test.js +363 -0
- package/test/integration/setup.js +3 -0
- package/test/integration/storage.integration.test.js +224 -0
- package/test/unit/internal-url-adapter.test.js +211 -0
- package/test/unit/legacy.storage.test.js.bak +614 -0
- package/test/unit/storage.extended.unit.test.js +435 -0
- package/test/unit/storage.unit.test.js +373 -0
package/src/index.js
ADDED
|
@@ -0,0 +1,853 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module @onlineapps/conn-base-storage
|
|
3
|
+
* @description MinIO storage connector with automatic SHA256 fingerprinting for OA Drive.
|
|
4
|
+
* Provides immutable content storage with verification, caching, and presigned URLs.
|
|
5
|
+
*
|
|
6
|
+
* @see {@link https://github.com/onlineapps/oa-drive/tree/main/shared/connector/conn-base-storage|GitHub Repository}
|
|
7
|
+
* @author OA Drive Team
|
|
8
|
+
* @license MIT
|
|
9
|
+
* @since 1.0.0
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const Minio = require('minio');
|
|
13
|
+
const crypto = require('crypto');
|
|
14
|
+
const winston = require('winston');
|
|
15
|
+
const SharedUrlAdapter = require('./sharedUrlAdapter');
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Storage connector for MinIO with automatic fingerprinting
|
|
19
|
+
*
|
|
20
|
+
* @class StorageConnector
|
|
21
|
+
*
|
|
22
|
+
* @example <caption>Basic Usage</caption>
|
|
23
|
+
* const storage = new StorageConnector({
|
|
24
|
+
* endPoint: 'minio.example.com',
|
|
25
|
+
* port: 9000,
|
|
26
|
+
* accessKey: 'minioadmin',
|
|
27
|
+
* secretKey: 'minioadmin'
|
|
28
|
+
* });
|
|
29
|
+
*
|
|
30
|
+
* await storage.initialize();
|
|
31
|
+
* const result = await storage.uploadWithFingerprint(
|
|
32
|
+
* 'documents',
|
|
33
|
+
* { type: 'invoice', amount: 1500 },
|
|
34
|
+
* 'invoices'
|
|
35
|
+
* );
|
|
36
|
+
*
|
|
37
|
+
* @example <caption>With Caching</caption>
|
|
38
|
+
* const storage = new StorageConnector({
|
|
39
|
+
* cacheEnabled: true,
|
|
40
|
+
* maxCacheSize: 200
|
|
41
|
+
* });
|
|
42
|
+
*/
|
|
43
|
+
class StorageConnector {
|
|
44
|
+
/**
|
|
45
|
+
* Creates a new StorageConnector instance
|
|
46
|
+
*
|
|
47
|
+
* @constructor
|
|
48
|
+
* @param {Object} [config={}] - Configuration options
|
|
49
|
+
* @param {string} [config.endPoint='localhost'] - MinIO server endpoint
|
|
50
|
+
* @param {number} [config.port=9000] - MinIO server port
|
|
51
|
+
* @param {boolean} [config.useSSL=false] - Use SSL/TLS
|
|
52
|
+
* @param {string} [config.accessKey='minioadmin'] - Access key
|
|
53
|
+
* @param {string} [config.secretKey='minioadmin'] - Secret key
|
|
54
|
+
* @param {string} [config.defaultBucket='api-storage'] - Default bucket name
|
|
55
|
+
* @param {boolean} [config.cacheEnabled=true] - Enable content caching
|
|
56
|
+
* @param {number} [config.maxCacheSize=100] - Maximum cache entries
|
|
57
|
+
* @param {string} [config.logLevel='info'] - Logging level
|
|
58
|
+
*
|
|
59
|
+
* @example <caption>Full Configuration</caption>
|
|
60
|
+
* const storage = new StorageConnector({
|
|
61
|
+
* endPoint: 'minio.example.com',
|
|
62
|
+
* port: 9000,
|
|
63
|
+
* useSSL: true,
|
|
64
|
+
* accessKey: 'AKIAIOSFODNN7EXAMPLE',
|
|
65
|
+
* secretKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
|
|
66
|
+
* defaultBucket: 'production-storage',
|
|
67
|
+
* cacheEnabled: true,
|
|
68
|
+
* maxCacheSize: 500
|
|
69
|
+
* });
|
|
70
|
+
*/
|
|
71
|
+
constructor(config = {}) {
|
|
72
|
+
// Logger setup
|
|
73
|
+
this.logger = winston.createLogger({
|
|
74
|
+
level: config.logLevel || 'info',
|
|
75
|
+
format: winston.format.combine(
|
|
76
|
+
winston.format.timestamp(),
|
|
77
|
+
winston.format.json()
|
|
78
|
+
),
|
|
79
|
+
transports: [
|
|
80
|
+
new winston.transports.Console()
|
|
81
|
+
]
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
// MinIO client configuration
|
|
85
|
+
const minioConfig = {
|
|
86
|
+
endPoint: config.endPoint || process.env.MINIO_ENDPOINT || 'localhost',
|
|
87
|
+
port: parseInt(config.port || process.env.MINIO_PORT || 9000),
|
|
88
|
+
useSSL: config.useSSL || process.env.MINIO_USE_SSL === 'true',
|
|
89
|
+
accessKey: config.accessKey || process.env.MINIO_ACCESS_KEY || 'minioadmin',
|
|
90
|
+
secretKey: config.secretKey || process.env.MINIO_SECRET_KEY || 'minioadmin'
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
this.client = new Minio.Client(minioConfig);
|
|
94
|
+
this.defaultBucket = config.defaultBucket || process.env.MINIO_DEFAULT_BUCKET || 'api-storage';
|
|
95
|
+
|
|
96
|
+
// Store config for access in tests
|
|
97
|
+
this.config = {
|
|
98
|
+
...minioConfig,
|
|
99
|
+
bucketName: this.validateBucketName(config.bucketName || 'oa-drive-storage'),
|
|
100
|
+
cacheMaxSize: config.cacheMaxSize || 100
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
// MinIO client
|
|
104
|
+
this.minioClient = this.client;
|
|
105
|
+
|
|
106
|
+
// Cache configuration
|
|
107
|
+
this.cacheEnabled = config.cacheEnabled !== false;
|
|
108
|
+
this.cache = new Map();
|
|
109
|
+
this.contentCache = new Map();
|
|
110
|
+
this.maxCacheSize = config.maxCacheSize || 100; // Max items in cache
|
|
111
|
+
|
|
112
|
+
// Track initialization
|
|
113
|
+
this.initialized = false;
|
|
114
|
+
|
|
115
|
+
// Initialize shared URL adapter
|
|
116
|
+
this.sharedUrl = new SharedUrlAdapter(this);
|
|
117
|
+
|
|
118
|
+
this.logger.info('StorageConnector initialized', {
|
|
119
|
+
endpoint: minioConfig.endPoint,
|
|
120
|
+
port: minioConfig.port,
|
|
121
|
+
defaultBucket: this.defaultBucket
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Initialize connector and ensure bucket exists
|
|
127
|
+
*
|
|
128
|
+
* @async
|
|
129
|
+
* @method initialize
|
|
130
|
+
* @returns {Promise<boolean>} Returns true when initialized
|
|
131
|
+
*
|
|
132
|
+
* @throws {Error} If initialization fails
|
|
133
|
+
*
|
|
134
|
+
* @example
|
|
135
|
+
* try {
|
|
136
|
+
* await storage.initialize();
|
|
137
|
+
* console.log('Storage ready');
|
|
138
|
+
* } catch (error) {
|
|
139
|
+
* console.error('Failed to initialize:', error);
|
|
140
|
+
* }
|
|
141
|
+
*/
|
|
142
|
+
async initialize() {
|
|
143
|
+
try {
|
|
144
|
+
// Check if default bucket exists
|
|
145
|
+
const bucketExists = await this.client.bucketExists(this.defaultBucket);
|
|
146
|
+
|
|
147
|
+
if (!bucketExists) {
|
|
148
|
+
// Create bucket with versioning
|
|
149
|
+
await this.client.makeBucket(this.defaultBucket, 'us-east-1');
|
|
150
|
+
this.logger.info(`Bucket ${this.defaultBucket} created`);
|
|
151
|
+
|
|
152
|
+
// Set bucket policy for read access
|
|
153
|
+
const policy = {
|
|
154
|
+
Version: '2012-10-17',
|
|
155
|
+
Statement: [
|
|
156
|
+
{
|
|
157
|
+
Effect: 'Allow',
|
|
158
|
+
Principal: { AWS: ['*'] },
|
|
159
|
+
Action: ['s3:GetObject'],
|
|
160
|
+
Resource: [`arn:aws:s3:::${this.defaultBucket}/*`]
|
|
161
|
+
}
|
|
162
|
+
]
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
await this.client.setBucketPolicy(
|
|
166
|
+
this.defaultBucket,
|
|
167
|
+
JSON.stringify(policy)
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
this.logger.info('StorageConnector ready');
|
|
172
|
+
this.initialized = true;
|
|
173
|
+
return true;
|
|
174
|
+
} catch (error) {
|
|
175
|
+
this.logger.error('Failed to initialize StorageConnector', error);
|
|
176
|
+
throw error;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Generate SHA256 fingerprint for content
|
|
182
|
+
*
|
|
183
|
+
* @method generateFingerprint
|
|
184
|
+
* @param {string|Buffer|Object} content - Content to fingerprint
|
|
185
|
+
* @returns {string} SHA256 hash in hex format
|
|
186
|
+
*
|
|
187
|
+
* @example <caption>String Content</caption>
|
|
188
|
+
* const hash = storage.generateFingerprint('Hello World');
|
|
189
|
+
* // Returns: SHA256 hash
|
|
190
|
+
*
|
|
191
|
+
* @example <caption>Object Content</caption>
|
|
192
|
+
* const hash = storage.generateFingerprint({ name: 'John', age: 30 });
|
|
193
|
+
* // Returns consistent hash for the object
|
|
194
|
+
*/
|
|
195
|
+
generateFingerprint(content) {
|
|
196
|
+
// Handle undefined and null
|
|
197
|
+
if (content === undefined || content === null) {
|
|
198
|
+
content = String(content);
|
|
199
|
+
} else if (typeof content === 'symbol') {
|
|
200
|
+
content = content.toString();
|
|
201
|
+
} else if (typeof content !== 'string' && !Buffer.isBuffer(content)) {
|
|
202
|
+
// Handle circular references
|
|
203
|
+
try {
|
|
204
|
+
content = JSON.stringify(content);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
// If circular reference, use a deterministic string representation
|
|
207
|
+
content = '[Circular Reference]';
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return crypto
|
|
212
|
+
.createHash('sha256')
|
|
213
|
+
.update(content)
|
|
214
|
+
.digest('hex');
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Upload content with automatic fingerprinting
|
|
219
|
+
*
|
|
220
|
+
* @async
|
|
221
|
+
* @method uploadWithFingerprint
|
|
222
|
+
* @param {string} bucket - Bucket name
|
|
223
|
+
* @param {string|Buffer|Object} content - Content to upload
|
|
224
|
+
* @param {string} [pathPrefix=''] - Path prefix for organization
|
|
225
|
+
* @returns {Promise<UploadResult>} Upload result with fingerprint
|
|
226
|
+
* @returns {string} result.fingerprint - SHA256 fingerprint
|
|
227
|
+
* @returns {string} result.bucket - Bucket name
|
|
228
|
+
* @returns {string} result.path - Storage path
|
|
229
|
+
* @returns {string} result.url - Public URL
|
|
230
|
+
* @returns {boolean} result.existed - Whether object already existed
|
|
231
|
+
*
|
|
232
|
+
* @throws {Error} If upload fails
|
|
233
|
+
*
|
|
234
|
+
* @example <caption>Upload JSON Object</caption>
|
|
235
|
+
* const result = await storage.uploadWithFingerprint(
|
|
236
|
+
* 'invoices',
|
|
237
|
+
* { invoiceId: 'INV-123', amount: 1500 },
|
|
238
|
+
* '2024/01'
|
|
239
|
+
* );
|
|
240
|
+
* console.log(`Stored at: ${result.url}`);
|
|
241
|
+
* console.log(`Fingerprint: ${result.fingerprint}`);
|
|
242
|
+
*
|
|
243
|
+
* @example <caption>Upload String Content</caption>
|
|
244
|
+
* const result = await storage.uploadWithFingerprint(
|
|
245
|
+
* 'documents',
|
|
246
|
+
* 'Important document content',
|
|
247
|
+
* 'contracts'
|
|
248
|
+
* );
|
|
249
|
+
*
|
|
250
|
+
* @example <caption>Upload Buffer</caption>
|
|
251
|
+
* const buffer = Buffer.from('Binary data');
|
|
252
|
+
* const result = await storage.uploadWithFingerprint(
|
|
253
|
+
* 'files',
|
|
254
|
+
* buffer
|
|
255
|
+
* );
|
|
256
|
+
*/
|
|
257
|
+
async uploadWithFingerprint(bucket, content, pathPrefix = '') {
|
|
258
|
+
try {
|
|
259
|
+
// Ensure bucket is provided
|
|
260
|
+
bucket = bucket || this.defaultBucket;
|
|
261
|
+
|
|
262
|
+
// Convert content to Buffer if needed
|
|
263
|
+
let buffer;
|
|
264
|
+
if (Buffer.isBuffer(content)) {
|
|
265
|
+
buffer = content;
|
|
266
|
+
} else if (typeof content === 'object') {
|
|
267
|
+
buffer = Buffer.from(JSON.stringify(content));
|
|
268
|
+
} else {
|
|
269
|
+
buffer = Buffer.from(content);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Generate fingerprint
|
|
273
|
+
const fingerprint = this.generateFingerprint(buffer);
|
|
274
|
+
|
|
275
|
+
// Build path with fingerprint
|
|
276
|
+
const path = pathPrefix
|
|
277
|
+
? `${pathPrefix}/${fingerprint}.json`
|
|
278
|
+
: `${fingerprint}.json`;
|
|
279
|
+
|
|
280
|
+
// Check if object already exists (immutable)
|
|
281
|
+
try {
|
|
282
|
+
await this.client.statObject(bucket, path);
|
|
283
|
+
this.logger.debug(`Object already exists: ${bucket}/${path}`);
|
|
284
|
+
|
|
285
|
+
return {
|
|
286
|
+
fingerprint,
|
|
287
|
+
bucket,
|
|
288
|
+
path,
|
|
289
|
+
url: this.getPublicUrl(bucket, path),
|
|
290
|
+
existed: true
|
|
291
|
+
};
|
|
292
|
+
} catch (err) {
|
|
293
|
+
// Object doesn't exist, proceed with upload
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Upload to MinIO
|
|
297
|
+
await this.client.putObject(
|
|
298
|
+
bucket,
|
|
299
|
+
path,
|
|
300
|
+
buffer,
|
|
301
|
+
buffer.length,
|
|
302
|
+
{
|
|
303
|
+
'Content-Type': 'application/json',
|
|
304
|
+
'x-amz-meta-fingerprint': fingerprint,
|
|
305
|
+
'x-amz-meta-uploaded-at': new Date().toISOString()
|
|
306
|
+
}
|
|
307
|
+
);
|
|
308
|
+
|
|
309
|
+
this.logger.info(`Uploaded content to ${bucket}/${path}`, { fingerprint });
|
|
310
|
+
|
|
311
|
+
return {
|
|
312
|
+
fingerprint,
|
|
313
|
+
bucket,
|
|
314
|
+
path,
|
|
315
|
+
url: this.getPublicUrl(bucket, path),
|
|
316
|
+
existed: false
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
} catch (error) {
|
|
320
|
+
this.logger.error('Upload failed', error);
|
|
321
|
+
throw error;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Download content and verify fingerprint
|
|
327
|
+
*
|
|
328
|
+
* @async
|
|
329
|
+
* @method downloadWithVerification
|
|
330
|
+
* @param {string} bucket - Bucket name
|
|
331
|
+
* @param {string} path - Object path
|
|
332
|
+
* @param {string} expectedFingerprint - Expected SHA256 fingerprint
|
|
333
|
+
* @returns {Promise<string>} Content as string
|
|
334
|
+
*
|
|
335
|
+
* @throws {Error} If download fails or fingerprint mismatch
|
|
336
|
+
*
|
|
337
|
+
* @example <caption>Download with Verification</caption>
|
|
338
|
+
* const content = await storage.downloadWithVerification(
|
|
339
|
+
* 'invoices',
|
|
340
|
+
* '2024/01/abc123.json',
|
|
341
|
+
* 'abc123...'
|
|
342
|
+
* );
|
|
343
|
+
* const data = JSON.parse(content);
|
|
344
|
+
*
|
|
345
|
+
* @example <caption>Handle Verification Failure</caption>
|
|
346
|
+
* try {
|
|
347
|
+
* const content = await storage.downloadWithVerification(
|
|
348
|
+
* 'documents',
|
|
349
|
+
* 'contract.json',
|
|
350
|
+
* expectedHash
|
|
351
|
+
* );
|
|
352
|
+
* } catch (error) {
|
|
353
|
+
* if (error.message.includes('Fingerprint mismatch')) {
|
|
354
|
+
* console.error('Content has been tampered!');
|
|
355
|
+
* }
|
|
356
|
+
* }
|
|
357
|
+
*/
|
|
358
|
+
async downloadWithVerification(bucket, path, expectedFingerprint) {
|
|
359
|
+
try {
|
|
360
|
+
bucket = bucket || this.defaultBucket;
|
|
361
|
+
|
|
362
|
+
// Check cache first
|
|
363
|
+
const cacheKey = `${bucket}/${path}/${expectedFingerprint}`;
|
|
364
|
+
if (this.cacheEnabled && this.contentCache.has(cacheKey)) {
|
|
365
|
+
this.logger.debug(`Cache hit for ${cacheKey}`);
|
|
366
|
+
return this.contentCache.get(cacheKey);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Download from MinIO
|
|
370
|
+
const stream = await this.client.getObject(bucket, path);
|
|
371
|
+
|
|
372
|
+
// Collect stream data
|
|
373
|
+
const chunks = [];
|
|
374
|
+
for await (const chunk of stream) {
|
|
375
|
+
chunks.push(chunk);
|
|
376
|
+
}
|
|
377
|
+
const buffer = Buffer.concat(chunks);
|
|
378
|
+
|
|
379
|
+
// Verify fingerprint if provided
|
|
380
|
+
if (expectedFingerprint) {
|
|
381
|
+
const actualFingerprint = this.generateFingerprint(buffer);
|
|
382
|
+
|
|
383
|
+
if (actualFingerprint !== expectedFingerprint) {
|
|
384
|
+
throw new Error(
|
|
385
|
+
`Fingerprint mismatch! Expected: ${expectedFingerprint}, Got: ${actualFingerprint}`
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
this.logger.debug(`Fingerprint verified for ${bucket}/${path}`);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// Convert to string
|
|
393
|
+
const content = buffer.toString('utf8');
|
|
394
|
+
|
|
395
|
+
// Update cache
|
|
396
|
+
if (this.cacheEnabled && expectedFingerprint) {
|
|
397
|
+
this.updateCache(cacheKey, content);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
return content;
|
|
401
|
+
|
|
402
|
+
} catch (error) {
|
|
403
|
+
this.logger.error(`Download failed for ${bucket}/${path}`, error);
|
|
404
|
+
throw error;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Get pre-signed URL for direct access
|
|
410
|
+
*
|
|
411
|
+
* @async
|
|
412
|
+
* @method getPresignedUrl
|
|
413
|
+
* @param {string} bucket - Bucket name
|
|
414
|
+
* @param {string} path - Object path
|
|
415
|
+
* @param {number} [expiry=3600] - URL expiry in seconds
|
|
416
|
+
* @returns {Promise<string>} Pre-signed URL
|
|
417
|
+
*
|
|
418
|
+
* @throws {Error} If URL generation fails
|
|
419
|
+
*
|
|
420
|
+
* @example <caption>Generate Temporary URL</caption>
|
|
421
|
+
* const url = await storage.getPresignedUrl(
|
|
422
|
+
* 'invoices',
|
|
423
|
+
* '2024/01/invoice-123.json',
|
|
424
|
+
* 7200 // 2 hours
|
|
425
|
+
* );
|
|
426
|
+
* // Share URL with external service
|
|
427
|
+
*
|
|
428
|
+
* @example <caption>Short-lived URL</caption>
|
|
429
|
+
* const url = await storage.getPresignedUrl(
|
|
430
|
+
* 'temp',
|
|
431
|
+
* 'report.pdf',
|
|
432
|
+
* 300 // 5 minutes
|
|
433
|
+
* );
|
|
434
|
+
*/
|
|
435
|
+
async getPresignedUrl(bucket, path, expiry = 3600) {
|
|
436
|
+
try {
|
|
437
|
+
bucket = bucket || this.defaultBucket;
|
|
438
|
+
|
|
439
|
+
const url = await this.client.presignedGetObject(
|
|
440
|
+
bucket,
|
|
441
|
+
path,
|
|
442
|
+
expiry
|
|
443
|
+
);
|
|
444
|
+
|
|
445
|
+
this.logger.debug(`Generated presigned URL for ${bucket}/${path}`);
|
|
446
|
+
return url;
|
|
447
|
+
|
|
448
|
+
} catch (error) {
|
|
449
|
+
this.logger.error('Failed to generate presigned URL', error);
|
|
450
|
+
throw error;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* List objects with fingerprint metadata
|
|
456
|
+
*
|
|
457
|
+
* @async
|
|
458
|
+
* @method listWithFingerprints
|
|
459
|
+
* @param {string} bucket - Bucket name
|
|
460
|
+
* @param {string} [prefix=''] - Path prefix to filter
|
|
461
|
+
* @returns {Promise<Array<ObjectInfo>>} List of objects with metadata
|
|
462
|
+
*
|
|
463
|
+
* @example <caption>List All Objects</caption>
|
|
464
|
+
* const objects = await storage.listWithFingerprints('invoices');
|
|
465
|
+
* objects.forEach(obj => {
|
|
466
|
+
* console.log(`${obj.name}: ${obj.fingerprint}`);
|
|
467
|
+
* });
|
|
468
|
+
*
|
|
469
|
+
* @example <caption>List with Prefix</caption>
|
|
470
|
+
* const objects = await storage.listWithFingerprints(
|
|
471
|
+
* 'documents',
|
|
472
|
+
* '2024/01/'
|
|
473
|
+
* );
|
|
474
|
+
*/
|
|
475
|
+
async listWithFingerprints(bucket, prefix = '') {
|
|
476
|
+
try {
|
|
477
|
+
bucket = bucket || this.defaultBucket;
|
|
478
|
+
|
|
479
|
+
const objects = [];
|
|
480
|
+
const stream = this.client.listObjectsV2(bucket, prefix, true);
|
|
481
|
+
|
|
482
|
+
for await (const obj of stream) {
|
|
483
|
+
// Get metadata for each object
|
|
484
|
+
const stat = await this.client.statObject(bucket, obj.name);
|
|
485
|
+
|
|
486
|
+
objects.push({
|
|
487
|
+
name: obj.name,
|
|
488
|
+
size: obj.size,
|
|
489
|
+
lastModified: obj.lastModified,
|
|
490
|
+
fingerprint: stat.metaData['x-amz-meta-fingerprint'],
|
|
491
|
+
uploadedAt: stat.metaData['x-amz-meta-uploaded-at']
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
return objects;
|
|
496
|
+
|
|
497
|
+
} catch (error) {
|
|
498
|
+
this.logger.error('Failed to list objects', error);
|
|
499
|
+
throw error;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Delete object (use with caution - breaks immutability)
|
|
505
|
+
*
|
|
506
|
+
* @async
|
|
507
|
+
* @method deleteObject
|
|
508
|
+
* @param {string} bucket - Bucket name
|
|
509
|
+
* @param {string} path - Object path
|
|
510
|
+
* @returns {Promise<boolean>} Returns true on successful deletion
|
|
511
|
+
*
|
|
512
|
+
* @throws {Error} If deletion fails
|
|
513
|
+
*
|
|
514
|
+
* @warning This breaks immutability - use only when necessary
|
|
515
|
+
*
|
|
516
|
+
* @example
|
|
517
|
+
* await storage.deleteObject('temp', 'temporary-file.json');
|
|
518
|
+
*/
|
|
519
|
+
async deleteObject(bucket, path) {
|
|
520
|
+
try {
|
|
521
|
+
bucket = bucket || this.defaultBucket;
|
|
522
|
+
|
|
523
|
+
await this.client.removeObject(bucket, path);
|
|
524
|
+
this.logger.warn(`Deleted object: ${bucket}/${path}`);
|
|
525
|
+
|
|
526
|
+
// Clear from cache
|
|
527
|
+
const cacheKeys = Array.from(this.contentCache.keys())
|
|
528
|
+
.filter(key => key.startsWith(`${bucket}/${path}/`));
|
|
529
|
+
|
|
530
|
+
cacheKeys.forEach(key => this.contentCache.delete(key));
|
|
531
|
+
|
|
532
|
+
return true;
|
|
533
|
+
|
|
534
|
+
} catch (error) {
|
|
535
|
+
this.logger.error(`Failed to delete ${bucket}/${path}`, error);
|
|
536
|
+
throw error;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Get public URL for an object
|
|
542
|
+
*
|
|
543
|
+
* @method getPublicUrl
|
|
544
|
+
* @param {string} bucket - Bucket name
|
|
545
|
+
* @param {string} path - Object path
|
|
546
|
+
* @returns {string} Public URL
|
|
547
|
+
*
|
|
548
|
+
* @example
|
|
549
|
+
* const url = storage.getPublicUrl('invoices', '2024/invoice-123.json');
|
|
550
|
+
* // Returns: http://localhost:9000/invoices/2024/invoice-123.json
|
|
551
|
+
*/
|
|
552
|
+
getPublicUrl(bucket, path) {
|
|
553
|
+
const protocol = this.client.useSSL ? 'https' : 'http';
|
|
554
|
+
const port = this.client.port === 80 || this.client.port === 443
|
|
555
|
+
? ''
|
|
556
|
+
: `:${this.client.port}`;
|
|
557
|
+
|
|
558
|
+
return `${protocol}://${this.client.host}${port}/${bucket}/${path}`;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* Update cache with LRU eviction
|
|
563
|
+
*
|
|
564
|
+
* @private
|
|
565
|
+
* @method updateCache
|
|
566
|
+
* @param {string} key - Cache key
|
|
567
|
+
* @param {*} value - Value to cache
|
|
568
|
+
* @returns {void}
|
|
569
|
+
*/
|
|
570
|
+
updateCache(key, value) {
|
|
571
|
+
// Remove oldest entry if cache is full
|
|
572
|
+
if (this.contentCache.size >= this.maxCacheSize) {
|
|
573
|
+
const firstKey = this.contentCache.keys().next().value;
|
|
574
|
+
this.contentCache.delete(firstKey);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// Add new entry
|
|
578
|
+
this.contentCache.set(key, value);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Create bucket if not exists
|
|
584
|
+
*
|
|
585
|
+
* @async
|
|
586
|
+
* @method ensureBucket
|
|
587
|
+
* @param {string} bucketName - Bucket name to ensure
|
|
588
|
+
* @returns {Promise<boolean>} Returns true when bucket exists
|
|
589
|
+
*
|
|
590
|
+
* @throws {Error} If bucket creation fails
|
|
591
|
+
*
|
|
592
|
+
* @example
|
|
593
|
+
* await storage.ensureBucket('new-bucket');
|
|
594
|
+
*/
|
|
595
|
+
async ensureBucket(bucketName) {
|
|
596
|
+
try {
|
|
597
|
+
const exists = await this.client.bucketExists(bucketName);
|
|
598
|
+
|
|
599
|
+
if (!exists) {
|
|
600
|
+
await this.client.makeBucket(bucketName, 'us-east-1');
|
|
601
|
+
this.logger.info(`Bucket ${bucketName} created`);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
return true;
|
|
605
|
+
} catch (error) {
|
|
606
|
+
this.logger.error(`Failed to ensure bucket ${bucketName}`, error);
|
|
607
|
+
throw error;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* Validate bucket name according to S3 standards
|
|
613
|
+
*
|
|
614
|
+
* @method validateBucketName
|
|
615
|
+
* @param {string} name - Bucket name to validate
|
|
616
|
+
* @returns {string} Valid bucket name or default
|
|
617
|
+
*
|
|
618
|
+
* @example
|
|
619
|
+
* const validName = storage.validateBucketName('My-Bucket');
|
|
620
|
+
* // Returns: 'oa-drive-storage' (invalid characters)
|
|
621
|
+
*/
|
|
622
|
+
validateBucketName(name) {
|
|
623
|
+
// S3 bucket naming rules
|
|
624
|
+
const validName = /^[a-z0-9][a-z0-9\-\.]*[a-z0-9]$/;
|
|
625
|
+
const minLength = 3;
|
|
626
|
+
const maxLength = 63;
|
|
627
|
+
|
|
628
|
+
if (!name ||
|
|
629
|
+
name.length < minLength ||
|
|
630
|
+
name.length > maxLength ||
|
|
631
|
+
!validName.test(name)) {
|
|
632
|
+
return 'oa-drive-storage'; // Return default if invalid
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
return name;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Get object name with fingerprint appended
|
|
640
|
+
*
|
|
641
|
+
* @method getObjectNameWithFingerprint
|
|
642
|
+
* @param {string} originalName - Original file name
|
|
643
|
+
* @param {string} fingerprint - SHA256 fingerprint
|
|
644
|
+
* @returns {string} Name with fingerprint
|
|
645
|
+
*
|
|
646
|
+
* @example
|
|
647
|
+
* const name = storage.getObjectNameWithFingerprint('document.pdf', 'abc123...');
|
|
648
|
+
* // Returns: 'document-abc123...pdf'
|
|
649
|
+
*/
|
|
650
|
+
getObjectNameWithFingerprint(originalName, fingerprint) {
|
|
651
|
+
const parts = originalName.split('.');
|
|
652
|
+
const extension = parts.length > 1 ? '.' + parts.pop() : '';
|
|
653
|
+
const baseName = parts.join('.');
|
|
654
|
+
return `${baseName}-${fingerprint}${extension}`;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/**
|
|
658
|
+
* Extract fingerprint from object name
|
|
659
|
+
*
|
|
660
|
+
* @method extractFingerprintFromName
|
|
661
|
+
* @param {string} objectName - Object name with fingerprint
|
|
662
|
+
* @returns {string|null} Extracted fingerprint or null
|
|
663
|
+
*
|
|
664
|
+
* @example
|
|
665
|
+
* const fp = storage.extractFingerprintFromName('document-abc123def456.pdf');
|
|
666
|
+
* // Returns: 'abc123def456' if valid format
|
|
667
|
+
*/
|
|
668
|
+
extractFingerprintFromName(objectName) {
|
|
669
|
+
// Match pattern: name-<64-char-hex>.extension
|
|
670
|
+
const match = objectName.match(/-([a-f0-9]{64})(\.[^.]+)?$/);
|
|
671
|
+
return match ? match[1] : null;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Add item to cache with size limit
|
|
676
|
+
*
|
|
677
|
+
* @private
|
|
678
|
+
* @method addToCache
|
|
679
|
+
* @param {string} key - Cache key
|
|
680
|
+
* @param {*} value - Value to cache
|
|
681
|
+
* @returns {void}
|
|
682
|
+
*/
|
|
683
|
+
addToCache(key, value) {
|
|
684
|
+
// Remove oldest if at max size
|
|
685
|
+
if (this.cache.size >= this.config.cacheMaxSize) {
|
|
686
|
+
const firstKey = this.cache.keys().next().value;
|
|
687
|
+
this.cache.delete(firstKey);
|
|
688
|
+
}
|
|
689
|
+
this.cache.set(key, value);
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
/**
|
|
693
|
+
* Get item from cache
|
|
694
|
+
*
|
|
695
|
+
* @private
|
|
696
|
+
* @method getFromCache
|
|
697
|
+
* @param {string} key - Cache key
|
|
698
|
+
* @returns {*} Cached value or undefined
|
|
699
|
+
*/
|
|
700
|
+
getFromCache(key) {
|
|
701
|
+
return this.cache.get(key);
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Clear all caches
|
|
706
|
+
*
|
|
707
|
+
* @method clearCache
|
|
708
|
+
* @returns {void}
|
|
709
|
+
*
|
|
710
|
+
* @example
|
|
711
|
+
* storage.clearCache();
|
|
712
|
+
* // All cached content cleared
|
|
713
|
+
*/
|
|
714
|
+
clearCache() {
|
|
715
|
+
this.cache.clear();
|
|
716
|
+
this.contentCache.clear();
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Get internal URL for service-to-service communication
|
|
721
|
+
*
|
|
722
|
+
* @method getInternalUrl
|
|
723
|
+
* @param {string} bucket - Bucket name
|
|
724
|
+
* @param {string} path - Object path
|
|
725
|
+
* @returns {string} Internal URL with internal:// protocol
|
|
726
|
+
*
|
|
727
|
+
* @example
|
|
728
|
+
* const url = storage.getInternalUrl('invoices', 'invoice-123.json');
|
|
729
|
+
* // Returns: 'internal://storage/invoices/invoice-123.json'
|
|
730
|
+
*/
|
|
731
|
+
getInternalUrl(bucket, path) {
|
|
732
|
+
return `internal://storage/${bucket}/${path}`;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/**
|
|
736
|
+
* Get public URL for an object
|
|
737
|
+
*
|
|
738
|
+
* @deprecated Use getInternalUrl() for service-to-service communication
|
|
739
|
+
* @method getPublicUrl
|
|
740
|
+
* @param {string} bucket - Bucket name
|
|
741
|
+
* @param {string} path - Object path
|
|
742
|
+
* @returns {string} Internal URL for backward compatibility
|
|
743
|
+
*/
|
|
744
|
+
getPublicUrl(bucket, path) {
|
|
745
|
+
// Pro zpětnou kompatibilitu, vrací internal URL
|
|
746
|
+
return this.getInternalUrl(bucket, path);
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/**
|
|
750
|
+
* Get presigned URL for temporary access
|
|
751
|
+
*
|
|
752
|
+
* @async
|
|
753
|
+
* @method getPresignedUrl
|
|
754
|
+
* @param {string} bucket - Bucket name
|
|
755
|
+
* @param {string} objectName - Object name
|
|
756
|
+
* @param {number} [expiry=86400] - URL expiry in seconds (default 24 hours)
|
|
757
|
+
* @returns {Promise<string>} Pre-signed URL
|
|
758
|
+
*
|
|
759
|
+
* @throws {Error} If URL generation fails
|
|
760
|
+
*
|
|
761
|
+
* @example
|
|
762
|
+
* const url = await storage.getPresignedUrl('documents', 'contract.pdf', 3600);
|
|
763
|
+
*/
|
|
764
|
+
async getPresignedUrl(bucket, objectName, expiry = 86400) {
|
|
765
|
+
try {
|
|
766
|
+
return await this.client.presignedUrl('GET', bucket, objectName, expiry);
|
|
767
|
+
} catch (error) {
|
|
768
|
+
this.logger.error('Failed to generate presigned URL', error);
|
|
769
|
+
throw error;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* List objects with extracted fingerprints
|
|
775
|
+
*
|
|
776
|
+
* @async
|
|
777
|
+
* @method listObjectsWithFingerprints
|
|
778
|
+
* @param {string} bucket - Bucket name
|
|
779
|
+
* @param {string} [prefix=''] - Path prefix to filter
|
|
780
|
+
* @returns {Promise<Array<Object>>} List of objects with fingerprints
|
|
781
|
+
*
|
|
782
|
+
* @example
|
|
783
|
+
* const objects = await storage.listObjectsWithFingerprints('invoices', '2024/');
|
|
784
|
+
*/
|
|
785
|
+
async listObjectsWithFingerprints(bucket, prefix = '') {
|
|
786
|
+
try {
|
|
787
|
+
bucket = bucket || this.defaultBucket;
|
|
788
|
+
const objects = [];
|
|
789
|
+
|
|
790
|
+
return new Promise((resolve, reject) => {
|
|
791
|
+
const stream = this.client.listObjectsV2(bucket, prefix, true);
|
|
792
|
+
|
|
793
|
+
stream.on('data', obj => {
|
|
794
|
+
objects.push({
|
|
795
|
+
name: obj.name,
|
|
796
|
+
size: obj.size,
|
|
797
|
+
lastModified: obj.lastModified,
|
|
798
|
+
fingerprint: this.extractFingerprintFromName(obj.name)
|
|
799
|
+
});
|
|
800
|
+
});
|
|
801
|
+
|
|
802
|
+
stream.on('error', reject);
|
|
803
|
+
stream.on('end', () => resolve(objects));
|
|
804
|
+
});
|
|
805
|
+
} catch (error) {
|
|
806
|
+
this.logger.error('Failed to list objects', error);
|
|
807
|
+
throw error;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
/**
|
|
813
|
+
* @typedef {Object} UploadResult
|
|
814
|
+
* @property {string} fingerprint - SHA256 fingerprint of content
|
|
815
|
+
* @property {string} bucket - Bucket name
|
|
816
|
+
* @property {string} path - Storage path
|
|
817
|
+
* @property {string} url - Public or internal URL
|
|
818
|
+
* @property {boolean} existed - Whether object already existed
|
|
819
|
+
*/
|
|
820
|
+
|
|
821
|
+
/**
|
|
822
|
+
* @typedef {Object} ObjectInfo
|
|
823
|
+
* @property {string} name - Object name
|
|
824
|
+
* @property {number} size - Object size in bytes
|
|
825
|
+
* @property {Date} lastModified - Last modification date
|
|
826
|
+
* @property {string} [fingerprint] - SHA256 fingerprint if available
|
|
827
|
+
* @property {string} [uploadedAt] - Upload timestamp
|
|
828
|
+
*/
|
|
829
|
+
|
|
830
|
+
// Export main class
|
|
831
|
+
module.exports = StorageConnector;
|
|
832
|
+
|
|
833
|
+
/**
|
|
834
|
+
* Factory function to create storage instance
|
|
835
|
+
*
|
|
836
|
+
* @function create
|
|
837
|
+
* @param {Object} config - Configuration object
|
|
838
|
+
* @returns {StorageConnector} New storage connector instance
|
|
839
|
+
*
|
|
840
|
+
* @example
|
|
841
|
+
* const storage = StorageConnector.create({
|
|
842
|
+
* endPoint: 'minio.example.com',
|
|
843
|
+
* accessKey: 'key',
|
|
844
|
+
* secretKey: 'secret'
|
|
845
|
+
* });
|
|
846
|
+
*/
|
|
847
|
+
module.exports.create = (config) => new StorageConnector(config);
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* Current version
|
|
851
|
+
* @constant {string}
|
|
852
|
+
*/
|
|
853
|
+
module.exports.VERSION = '1.0.0';
|