@hdriel/aws-utils 1.1.2 โ 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +751 -0
- package/dist/index.cjs +0 -43
- package/dist/index.d.cts +0 -10
- package/dist/index.d.ts +0 -10
- package/dist/index.js +0 -43
- package/package.json +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
# S3 Utility Package
|
|
2
|
+
|
|
3
|
+
A powerful, type-safe wrapper around AWS S3 SDK v3 that simplifies S3 operations with advanced features like streaming, file uploads, directory management, and LocalStack support.
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
## Features
|
|
7
|
+
|
|
8
|
+
โจ **Simplified API** - Clean, intuitive methods for common S3 operations
|
|
9
|
+
๐ **Directory Management** - Create, list, and delete directories with ease
|
|
10
|
+
๐ค **Advanced File Uploads** - Multer integration with Express.js middleware
|
|
11
|
+
๐ฌ **Video Streaming** - Built-in support for range requests and video streaming
|
|
12
|
+
๐ฆ **Zip Streaming** - Stream multiple files as a zip archive
|
|
13
|
+
๐ท๏ธ **File Tagging & Versioning** - Tag files and manage versions
|
|
14
|
+
๐ **Presigned URLs** - Generate temporary signed URLs
|
|
15
|
+
๐งช **LocalStack Support** - First-class support for local S3 testing
|
|
16
|
+
โก **Connection Pooling** - Optimized HTTP/HTTPS agents for better performance
|
|
17
|
+
๐ **Pagination** - Built-in pagination for large directory listings
|
|
18
|
+
|
|
19
|
+
# FULL DEMO PROJECT EXAMPLE:
|
|
20
|
+
please see this project code before using: [aws-utils-demo github link!](https://github.com/hdriel/aws-utils-demo)
|
|
21
|
+

|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npm install @hdriel/aws-utils
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Quick Start
|
|
31
|
+
|
|
32
|
+
First load this file somewhere on starting server
|
|
33
|
+
```typescript
|
|
34
|
+
// aws-utils-config.ts
|
|
35
|
+
import env from './dotenv.ts';
|
|
36
|
+
import { AWSConfigSharingUtil } from '@hdriel/aws-utils';
|
|
37
|
+
|
|
38
|
+
AWSConfigSharingUtil.setConfig({
|
|
39
|
+
accessKeyId: env?.AWS_ACCESS_KEY_ID,
|
|
40
|
+
secretAccessKey: env?.AWS_SECRET_ACCESS_KEY,
|
|
41
|
+
region: env?.AWS_REGION,
|
|
42
|
+
endpoint: env?.AWS_ENDPOINT,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// console.log('AWSConfigSharingUtil configuration');
|
|
46
|
+
// console.table(AWSConfigSharingUtil.getConfig());
|
|
47
|
+
```
|
|
48
|
+
on your server files:
|
|
49
|
+
```typescript
|
|
50
|
+
import './aws-utils-config';
|
|
51
|
+
...
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
then write your code...
|
|
55
|
+
|
|
56
|
+
for example:
|
|
57
|
+
|
|
58
|
+
```typescript
|
|
59
|
+
import { S3Util, S3LocalstackUtil } from '@hdriel/aws-utils';
|
|
60
|
+
|
|
61
|
+
// Initialize S3 utility
|
|
62
|
+
|
|
63
|
+
// for localstack usage
|
|
64
|
+
const s3 = new S3LocalstackUtil({ bucket: 'demo' });
|
|
65
|
+
const directoryTreeInfo = await s3.directoryListPaginated('/', { pageSize: 100, pageNumber: 0 });
|
|
66
|
+
console.log('Directory tree info', JSON.stringify(directoryTreeInfo, null, 2));
|
|
67
|
+
|
|
68
|
+
// OR
|
|
69
|
+
|
|
70
|
+
// for production usage
|
|
71
|
+
const s3 = new S3Util({ bucket: 'demo' });
|
|
72
|
+
const directoryTreeInfo = await s3.directoryListPaginated('/', { pageSize: 100, pageNumber: 0 });
|
|
73
|
+
console.log('Directory tree info', JSON.stringify(directoryTreeInfo, null, 2));
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
// Usage examples
|
|
77
|
+
// Initialize bucket (creates if doesn't exist)
|
|
78
|
+
await s3.initBucket();
|
|
79
|
+
|
|
80
|
+
// Upload a file
|
|
81
|
+
await s3.uploadFile('/documents/file.pdf', fileBuffer);
|
|
82
|
+
|
|
83
|
+
// Check if file exists
|
|
84
|
+
const exists = await s3.fileExists('/documents/file.pdf');
|
|
85
|
+
|
|
86
|
+
// Get file content
|
|
87
|
+
const content = await s3.fileContent('/documents/file.pdf', 'utf8');
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Configuration Options
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
interface S3UtilProps {
|
|
94
|
+
bucket: string; // Required: S3 bucket name
|
|
95
|
+
logger?: Logger; // Optional: Logger instance
|
|
96
|
+
reqId?: string; // Optional: Request ID for logging
|
|
97
|
+
accessKeyId?: string; // AWS credentials
|
|
98
|
+
secretAccessKey?: string; // AWS credentials
|
|
99
|
+
endpoint?: string; // Custom endpoint (e.g., LocalStack)
|
|
100
|
+
region?: string; // AWS region (default: from config)
|
|
101
|
+
s3ForcePathStyle?: boolean; // Use path-style URLs (default: true)
|
|
102
|
+
maxUploadFileSizeRestriction?: string; // Max upload size (default: '10GB')
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
----
|
|
106
|
+
|
|
107
|
+
# FULL DEMO PROJECT EXAMPLE:
|
|
108
|
+
please see this project code before using: [aws-utils-demo github link!](https://github.com/hdriel/aws-utils-demo)
|
|
109
|
+

|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
## Core Features
|
|
116
|
+
|
|
117
|
+
### ๐ชฃ Bucket Operations
|
|
118
|
+
|
|
119
|
+
#### Initialize Bucket
|
|
120
|
+
```typescript
|
|
121
|
+
// Create private bucket
|
|
122
|
+
await s3.initBucket('private');
|
|
123
|
+
|
|
124
|
+
// Create public bucket
|
|
125
|
+
await s3.initBucket('public-read');
|
|
126
|
+
|
|
127
|
+
// With location constraint
|
|
128
|
+
await s3.initBucket('private', {
|
|
129
|
+
includeConstraintLocation: true
|
|
130
|
+
});
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
#### Bucket Information
|
|
134
|
+
```typescript
|
|
135
|
+
const info = await s3.bucketInfo();
|
|
136
|
+
console.log(info);
|
|
137
|
+
// {
|
|
138
|
+
// name: 'my-bucket',
|
|
139
|
+
// region: 'us-east-1',
|
|
140
|
+
// exists: true,
|
|
141
|
+
// creationDate: Date,
|
|
142
|
+
// versioning: 'Enabled',
|
|
143
|
+
// encryption: { enabled: true, type: 'AES256' },
|
|
144
|
+
// publicAccessBlock: { ... },
|
|
145
|
+
// policy: { ... }
|
|
146
|
+
// }
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
#### Check Bucket Exists
|
|
150
|
+
```typescript
|
|
151
|
+
const exists = await s3.isBucketExists();
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
#### Delete Bucket
|
|
155
|
+
```typescript
|
|
156
|
+
// Delete bucket (must be empty)
|
|
157
|
+
await s3.destroyBucket();
|
|
158
|
+
|
|
159
|
+
// Force delete with all contents
|
|
160
|
+
await s3.destroyBucket(true);
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
#### List All Buckets
|
|
164
|
+
```typescript
|
|
165
|
+
const buckets = await s3.getBucketList();
|
|
166
|
+
|
|
167
|
+
// Include public access configuration
|
|
168
|
+
const bucketsWithAccess = await s3.getBucketList({
|
|
169
|
+
includePublicAccess: true
|
|
170
|
+
});
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### ๐ Directory Operations
|
|
174
|
+
|
|
175
|
+
#### Create Directory
|
|
176
|
+
```typescript
|
|
177
|
+
await s3.createDirectory('/uploads/images');
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
#### List Directory Contents
|
|
181
|
+
```typescript
|
|
182
|
+
const { directories, files } = await s3.directoryList('/uploads');
|
|
183
|
+
|
|
184
|
+
console.log('Subdirectories:', directories);
|
|
185
|
+
console.log('Files:', files);
|
|
186
|
+
// files: [
|
|
187
|
+
// {
|
|
188
|
+
// Key: '/uploads/image.jpg',
|
|
189
|
+
// Name: 'image.jpg',
|
|
190
|
+
// Size: 12345,
|
|
191
|
+
// LastModified: Date,
|
|
192
|
+
// Location: 'https://...'
|
|
193
|
+
// }
|
|
194
|
+
// ]
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
#### Paginated Directory Listing
|
|
198
|
+
```typescript
|
|
199
|
+
// Get second page with 50 items per page
|
|
200
|
+
const result = await s3.directoryListPaginated('/uploads', {
|
|
201
|
+
pageSize: 50,
|
|
202
|
+
pageNumber: 1
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
console.log(result.directories); // Array of directory names
|
|
206
|
+
console.log(result.files); // Array of file objects
|
|
207
|
+
console.log(result.totalFetched); // Number of items returned
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
#### Delete Directory
|
|
211
|
+
```typescript
|
|
212
|
+
// Delete directory and all contents
|
|
213
|
+
await s3.deleteDirectory('/uploads/temp');
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
#### Check Directory Exists
|
|
217
|
+
```typescript
|
|
218
|
+
const exists = await s3.directoryExists('/uploads/images');
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### ๐ File Operations
|
|
222
|
+
|
|
223
|
+
#### Upload File
|
|
224
|
+
```typescript
|
|
225
|
+
import { ACLs } from '@hdriel/aws-utils';
|
|
226
|
+
|
|
227
|
+
// Upload buffer
|
|
228
|
+
await s3.uploadFile('/documents/file.pdf', buffer);
|
|
229
|
+
|
|
230
|
+
// Upload with public access
|
|
231
|
+
await s3.uploadFile('/public/image.jpg', buffer, ACLs.public_read);
|
|
232
|
+
|
|
233
|
+
// Upload with version tag
|
|
234
|
+
await s3.uploadFile('/docs/v2.pdf', buffer, ACLs.private, '2.0.0');
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
#### Check File Exists
|
|
238
|
+
```typescript
|
|
239
|
+
const exists = await s3.fileExists('/documents/file.pdf');
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
#### Get File Content
|
|
243
|
+
```typescript
|
|
244
|
+
// As buffer
|
|
245
|
+
const buffer = await s3.fileContent('/documents/file.pdf');
|
|
246
|
+
|
|
247
|
+
// As base64 string
|
|
248
|
+
const base64 = await s3.fileContent('/image.jpg', 'base64');
|
|
249
|
+
|
|
250
|
+
// As UTF-8 string
|
|
251
|
+
const text = await s3.fileContent('/data.json', 'utf8');
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
#### File Information
|
|
255
|
+
```typescript
|
|
256
|
+
const info = await s3.fileInfo('/documents/file.pdf');
|
|
257
|
+
console.log(info.ContentLength);
|
|
258
|
+
console.log(info.ContentType);
|
|
259
|
+
console.log(info.LastModified);
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
#### List Files
|
|
263
|
+
```typescript
|
|
264
|
+
// List all files in directory
|
|
265
|
+
const files = await s3.fileListInfo('/documents');
|
|
266
|
+
|
|
267
|
+
// List files with prefix
|
|
268
|
+
const pdfFiles = await s3.fileListInfo('/documents', 'report-');
|
|
269
|
+
|
|
270
|
+
// Paginated file listing
|
|
271
|
+
const { files, totalFetched } = await s3.fileListInfoPaginated('/documents', {
|
|
272
|
+
fileNamePrefix: 'invoice-',
|
|
273
|
+
pageSize: 100,
|
|
274
|
+
pageNumber: 0
|
|
275
|
+
});
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
#### File Size
|
|
279
|
+
```typescript
|
|
280
|
+
const bytes = await s3.sizeOf('/large-file.zip');
|
|
281
|
+
const kb = await s3.sizeOf('/large-file.zip', 'KB');
|
|
282
|
+
const mb = await s3.sizeOf('/large-file.zip', 'MB');
|
|
283
|
+
const gb = await s3.sizeOf('/large-file.zip', 'GB');
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
#### Delete File
|
|
287
|
+
```typescript
|
|
288
|
+
await s3.deleteFile('/documents/old-file.pdf');
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
#### Generate Presigned URL
|
|
292
|
+
```typescript
|
|
293
|
+
// Expires in 15 minutes (default)
|
|
294
|
+
const url = await s3.fileUrl('/private/document.pdf');
|
|
295
|
+
|
|
296
|
+
// Custom expiration
|
|
297
|
+
const url = await s3.fileUrl('/private/document.pdf', '1h');
|
|
298
|
+
const url = await s3.fileUrl('/private/document.pdf', 3600); // seconds
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
#### File Tagging
|
|
302
|
+
```typescript
|
|
303
|
+
// Tag file with version
|
|
304
|
+
await s3.taggingFile('/documents/file.pdf', '1.0.0');
|
|
305
|
+
|
|
306
|
+
// Get file version
|
|
307
|
+
const version = await s3.fileVersion('/documents/file.pdf');
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
### ๐ฌ Streaming & Express.js Integration
|
|
311
|
+
|
|
312
|
+
#### Stream File Download
|
|
313
|
+
```typescript
|
|
314
|
+
import express from 'express';
|
|
315
|
+
|
|
316
|
+
const app = express();
|
|
317
|
+
|
|
318
|
+
// Stream single file
|
|
319
|
+
app.get('/download/:file',
|
|
320
|
+
await s3.getStreamFileCtrl({
|
|
321
|
+
filePath: '/documents/file.pdf',
|
|
322
|
+
filename: 'download.pdf',
|
|
323
|
+
forDownloading: true
|
|
324
|
+
})
|
|
325
|
+
);
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
#### Stream Zip Archive
|
|
329
|
+
```typescript
|
|
330
|
+
// Download multiple files as zip
|
|
331
|
+
app.get('/download-all',
|
|
332
|
+
await s3.getStreamZipFileCtr({
|
|
333
|
+
filePath: [
|
|
334
|
+
'/documents/file1.pdf',
|
|
335
|
+
'/documents/file2.pdf',
|
|
336
|
+
'/images/photo.jpg'
|
|
337
|
+
],
|
|
338
|
+
filename: 'archive.zip',
|
|
339
|
+
compressionLevel: 5 // 0-9, lower = faster
|
|
340
|
+
})
|
|
341
|
+
);
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
#### Stream Video with Range Support
|
|
345
|
+
```typescript
|
|
346
|
+
// Video streaming with range requests
|
|
347
|
+
app.get('/video/:id',
|
|
348
|
+
await s3.getStreamVideoFileCtrl({
|
|
349
|
+
fileKey: '/videos/movie.mp4',
|
|
350
|
+
contentType: 'video/mp4',
|
|
351
|
+
bufferMB: 5,
|
|
352
|
+
streamTimeoutMS: 30000,
|
|
353
|
+
allowedWhitelist: ['https://myapp.com']
|
|
354
|
+
})
|
|
355
|
+
);
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
#### View Image
|
|
359
|
+
```typescript
|
|
360
|
+
// Serve image with caching
|
|
361
|
+
app.get('/image',
|
|
362
|
+
s3.getImageFileViewCtrl({
|
|
363
|
+
queryField: 'path', // ?path=/images/photo.jpg
|
|
364
|
+
cachingAge: 31536000 // 1 year
|
|
365
|
+
})
|
|
366
|
+
);
|
|
367
|
+
|
|
368
|
+
// With fixed file path
|
|
369
|
+
app.get('/logo',
|
|
370
|
+
s3.getImageFileViewCtrl({
|
|
371
|
+
fileKey: '/public/logo.png'
|
|
372
|
+
})
|
|
373
|
+
);
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
#### View PDF
|
|
377
|
+
```typescript
|
|
378
|
+
app.get('/pdf',
|
|
379
|
+
s3.getPdfFileViewCtrl({
|
|
380
|
+
queryField: 'document',
|
|
381
|
+
cachingAge: 86400 // 1 day
|
|
382
|
+
})
|
|
383
|
+
);
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
### ๐ค File Upload Middleware
|
|
387
|
+
|
|
388
|
+
#### Single File Upload
|
|
389
|
+
```typescript
|
|
390
|
+
import express from 'express';
|
|
391
|
+
|
|
392
|
+
const app = express();
|
|
393
|
+
|
|
394
|
+
app.post('/upload',
|
|
395
|
+
s3.uploadSingleFile('file', '/uploads', {
|
|
396
|
+
maxFileSize: '5MB',
|
|
397
|
+
fileType: ['image', 'application'],
|
|
398
|
+
fileExt: ['jpg', 'png', 'pdf']
|
|
399
|
+
}),
|
|
400
|
+
(req, res) => {
|
|
401
|
+
console.log(req.s3File);
|
|
402
|
+
// {
|
|
403
|
+
// key: '/uploads/photo.jpg',
|
|
404
|
+
// location: 'https://...',
|
|
405
|
+
// size: 12345,
|
|
406
|
+
// mimetype: 'image/jpeg',
|
|
407
|
+
// ...
|
|
408
|
+
// }
|
|
409
|
+
res.json({ file: req.s3File });
|
|
410
|
+
}
|
|
411
|
+
);
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
#### Multiple Files Upload
|
|
415
|
+
```typescript
|
|
416
|
+
app.post('/upload-multiple',
|
|
417
|
+
s3.uploadMultipleFiles('photos', '/uploads/gallery', {
|
|
418
|
+
maxFileSize: '10MB',
|
|
419
|
+
maxFilesCount: 5,
|
|
420
|
+
fileType: ['image']
|
|
421
|
+
}),
|
|
422
|
+
(req, res) => {
|
|
423
|
+
console.log(req.s3Files); // Array of uploaded files
|
|
424
|
+
res.json({ files: req.s3Files });
|
|
425
|
+
}
|
|
426
|
+
);
|
|
427
|
+
```
|
|
428
|
+
|
|
429
|
+
#### Upload with Custom Filename
|
|
430
|
+
```typescript
|
|
431
|
+
app.post('/upload',
|
|
432
|
+
s3.uploadSingleFile('file', '/uploads', {
|
|
433
|
+
filename: async (req, file) => {
|
|
434
|
+
const timestamp = Date.now();
|
|
435
|
+
const ext = path.extname(file.originalname);
|
|
436
|
+
return `${req.user.id}-${timestamp}${ext}`;
|
|
437
|
+
}
|
|
438
|
+
}),
|
|
439
|
+
(req, res) => {
|
|
440
|
+
res.json({ file: req.s3File });
|
|
441
|
+
}
|
|
442
|
+
);
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
#### Upload with Custom Metadata
|
|
446
|
+
```typescript
|
|
447
|
+
app.post('/upload',
|
|
448
|
+
s3.uploadSingleFile('file', '/uploads', {
|
|
449
|
+
metadata: async (req, file) => ({
|
|
450
|
+
userId: req.user.id,
|
|
451
|
+
uploadDate: new Date().toISOString(),
|
|
452
|
+
originalName: file.originalname
|
|
453
|
+
})
|
|
454
|
+
}),
|
|
455
|
+
(req, res) => {
|
|
456
|
+
res.json({ file: req.s3File });
|
|
457
|
+
}
|
|
458
|
+
);
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
#### Upload Any Files (Mixed Fields)
|
|
462
|
+
```typescript
|
|
463
|
+
app.post('/upload-any',
|
|
464
|
+
s3.uploadAnyFiles('/uploads', 10, {
|
|
465
|
+
maxFileSize: '20MB'
|
|
466
|
+
}),
|
|
467
|
+
(req, res) => {
|
|
468
|
+
console.log(req.s3AllFiles); // All uploaded files
|
|
469
|
+
res.json({ files: req.s3AllFiles });
|
|
470
|
+
}
|
|
471
|
+
);
|
|
472
|
+
```
|
|
473
|
+
|
|
474
|
+
### Upload Options
|
|
475
|
+
|
|
476
|
+
```typescript
|
|
477
|
+
interface S3UploadOptions {
|
|
478
|
+
acl?: 'private' | 'public-read' | 'public-read-write';
|
|
479
|
+
maxFileSize?: string | number; // '5MB', '1GB', or bytes
|
|
480
|
+
maxFilesCount?: number; // For multiple file uploads
|
|
481
|
+
filename?: string | ((req, file) => string | Promise<string>);
|
|
482
|
+
fileType?: Array<'image' | 'video' | 'audio' | 'application' | 'text'>;
|
|
483
|
+
fileExt?: string[]; // ['jpg', 'png', 'pdf']
|
|
484
|
+
metadata?: object | ((req, file) => object | Promise<object>);
|
|
485
|
+
}
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
## ๐งช LocalStack Support
|
|
489
|
+
|
|
490
|
+
For local development and testing with LocalStack:
|
|
491
|
+
|
|
492
|
+
```typescript
|
|
493
|
+
import { S3LocalstackUtil } from '@hdriel/aws-utils';
|
|
494
|
+
|
|
495
|
+
const s3 = new S3LocalstackUtil({
|
|
496
|
+
bucket: 'test-bucket',
|
|
497
|
+
endpoint: 'http://localhost:4566',
|
|
498
|
+
region: 'us-east-1',
|
|
499
|
+
accessKeyId: 'test',
|
|
500
|
+
secretAccessKey: 'test',
|
|
501
|
+
s3ForcePathStyle: true
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
// Use same API as S3Util
|
|
505
|
+
await s3.initBucket();
|
|
506
|
+
await s3.uploadFile('/test.txt', Buffer.from('Hello LocalStack!'));
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
### LocalStack Docker Setup
|
|
510
|
+
|
|
511
|
+
```yaml
|
|
512
|
+
# docker-compose.yml
|
|
513
|
+
services:
|
|
514
|
+
localstack:
|
|
515
|
+
image: localstack/localstack
|
|
516
|
+
ports:
|
|
517
|
+
- "127.0.0.1:4566:4566" # LocalStack Gateway
|
|
518
|
+
- "127.0.0.1:4510-4559:4510-4559" # external services port range
|
|
519
|
+
environment:
|
|
520
|
+
# LocalStack configuration: https://docs.localstack.cloud/references/configuration/
|
|
521
|
+
- CLEAR_TMP_FOLDER=0
|
|
522
|
+
- DEBUG=${DEBUG:-1}
|
|
523
|
+
- PERSISTENCE=${PERSISTENCE:-1}
|
|
524
|
+
- LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR:-}
|
|
525
|
+
- LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY:-} # only required for Pro
|
|
526
|
+
- SERVICES=s3,lambda,sns,sqs,iam
|
|
527
|
+
- DATA_DIR=/tmp/localstack/data
|
|
528
|
+
- START_WEB=1
|
|
529
|
+
- DOCKER_HOST=unix:///var/run/docker.sock
|
|
530
|
+
- DEFAULT_REGION=us-east-1
|
|
531
|
+
- AWS_DEFAULT_REGION=us-east-1
|
|
532
|
+
- AWS_EXECUTION_ENV=True
|
|
533
|
+
- ENV=${NODE_ENV}
|
|
534
|
+
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-xxxxxxxxx}
|
|
535
|
+
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-xxxxxxxxxxxxxxxxxxxxx}
|
|
536
|
+
- HOSTNAME_EXTERNAL=localhost
|
|
537
|
+
volumes:
|
|
538
|
+
- "/var/run/docker.sock:/var/run/docker.sock"
|
|
539
|
+
- "${VOLUME_DIR_LOCALSTACK:-./docker-data/aws-localstack}:/var/lib/localstack"
|
|
540
|
+
- "${VOLUME_DIR_LOCALSTACK:-./docker-data/aws-localstack}/aws-s3:/tmp/localstack"
|
|
541
|
+
- "${VOLUME_DIR_LOCALSTACK:-./docker-data/aws-localstack}/aws-bootstrap:/opt/bootstrap/"
|
|
542
|
+
networks:
|
|
543
|
+
- app-network
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
## ๐ง Advanced Usage
|
|
547
|
+
|
|
548
|
+
### Dynamic Bucket Switching
|
|
549
|
+
|
|
550
|
+
```typescript
|
|
551
|
+
const s3 = new S3Util({
|
|
552
|
+
bucket: 'default-bucket',
|
|
553
|
+
// ... other config
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
// Switch to different bucket
|
|
557
|
+
s3.changeBucket('another-bucket');
|
|
558
|
+
|
|
559
|
+
// Operations now use 'another-bucket'
|
|
560
|
+
await s3.fileExists('/file.txt');
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
### Custom Logger Integration
|
|
564
|
+
|
|
565
|
+
```typescript
|
|
566
|
+
import { Logger } from 'stack-trace-logger';
|
|
567
|
+
|
|
568
|
+
const logger = new Logger('S3Service');
|
|
569
|
+
|
|
570
|
+
const s3 = new S3Util({
|
|
571
|
+
bucket: 'my-bucket',
|
|
572
|
+
logger,
|
|
573
|
+
reqId: 'request-123'
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
// All operations will log with your logger
|
|
577
|
+
await s3.uploadFile('/test.txt', buffer);
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
### Connection Pooling Configuration
|
|
581
|
+
|
|
582
|
+
The utility includes optimized HTTP/HTTPS agents:
|
|
583
|
+
|
|
584
|
+
```typescript
|
|
585
|
+
// Default configuration (already included):
|
|
586
|
+
// - keepAlive: true
|
|
587
|
+
// - maxSockets: 300
|
|
588
|
+
// - connectionTimeout: 3000ms
|
|
589
|
+
// - socketTimeout: 30000ms
|
|
590
|
+
```
|
|
591
|
+
|
|
592
|
+
### Batch Operations
|
|
593
|
+
|
|
594
|
+
```typescript
|
|
595
|
+
// Upload multiple files in parallel
|
|
596
|
+
const files = [
|
|
597
|
+
{ path: '/docs/file1.pdf', data: buffer1 },
|
|
598
|
+
{ path: '/docs/file2.pdf', data: buffer2 },
|
|
599
|
+
{ path: '/docs/file3.pdf', data: buffer3 }
|
|
600
|
+
];
|
|
601
|
+
|
|
602
|
+
await Promise.all(
|
|
603
|
+
files.map(file => s3.uploadFile(file.path, file.data))
|
|
604
|
+
);
|
|
605
|
+
|
|
606
|
+
// Delete multiple files
|
|
607
|
+
const filesToDelete = ['/old/file1.txt', '/old/file2.txt'];
|
|
608
|
+
await Promise.all(
|
|
609
|
+
filesToDelete.map(path => s3.deleteFile(path))
|
|
610
|
+
);
|
|
611
|
+
```
|
|
612
|
+
|
|
613
|
+
## ๐ Complete Express.js Example
|
|
614
|
+
|
|
615
|
+
```typescript
|
|
616
|
+
import express from 'express';
|
|
617
|
+
import { S3Util, ACLs } from '@hdriel/aws-utils';
|
|
618
|
+
|
|
619
|
+
const app = express();
|
|
620
|
+
const s3 = new S3Util({
|
|
621
|
+
bucket: process.env.S3_BUCKET!,
|
|
622
|
+
region: process.env.AWS_REGION,
|
|
623
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
624
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
// Initialize bucket on startup
|
|
628
|
+
(async () => {
|
|
629
|
+
await s3.initBucket();
|
|
630
|
+
console.log('S3 bucket initialized');
|
|
631
|
+
})();
|
|
632
|
+
|
|
633
|
+
// Upload endpoint
|
|
634
|
+
app.post('/api/upload',
|
|
635
|
+
s3.uploadSingleFile('file', '/uploads', {
|
|
636
|
+
maxFileSize: '10MB',
|
|
637
|
+
fileType: ['image', 'application'],
|
|
638
|
+
filename: async (req, file) => {
|
|
639
|
+
const timestamp = Date.now();
|
|
640
|
+
const sanitized = file.originalname.replace(/[^a-zA-Z0-9.-]/g, '_');
|
|
641
|
+
return `${timestamp}-${sanitized}`;
|
|
642
|
+
}
|
|
643
|
+
}),
|
|
644
|
+
async (req, res) => {
|
|
645
|
+
const { key, location, size } = req.s3File!;
|
|
646
|
+
|
|
647
|
+
// Generate temporary URL
|
|
648
|
+
const url = await s3.fileUrl(key, '1h');
|
|
649
|
+
|
|
650
|
+
res.json({ key, location, size, temporaryUrl: url });
|
|
651
|
+
}
|
|
652
|
+
);
|
|
653
|
+
|
|
654
|
+
// Download endpoint
|
|
655
|
+
app.get('/api/download/:key(*)',
|
|
656
|
+
async (req, res, next) => {
|
|
657
|
+
const key = decodeURIComponent(req.params.key);
|
|
658
|
+
const ctrl = await s3.getStreamFileCtrl({
|
|
659
|
+
filePath: key,
|
|
660
|
+
forDownloading: true
|
|
661
|
+
});
|
|
662
|
+
ctrl(req, res, next);
|
|
663
|
+
}
|
|
664
|
+
);
|
|
665
|
+
|
|
666
|
+
// List files endpoint
|
|
667
|
+
app.get('/api/files', async (req, res) => {
|
|
668
|
+
const { page = '0', size = '50' } = req.query;
|
|
669
|
+
|
|
670
|
+
const result = await s3.directoryListPaginated('/uploads', {
|
|
671
|
+
pageNumber: parseInt(page as string),
|
|
672
|
+
pageSize: parseInt(size as string)
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
res.json(result);
|
|
676
|
+
});
|
|
677
|
+
|
|
678
|
+
// Delete file endpoint
|
|
679
|
+
app.delete('/api/files/:key(*)', async (req, res) => {
|
|
680
|
+
const key = decodeURIComponent(req.params.key);
|
|
681
|
+
await s3.deleteFile(key);
|
|
682
|
+
res.json({ success: true });
|
|
683
|
+
});
|
|
684
|
+
|
|
685
|
+
// Video streaming endpoint
|
|
686
|
+
app.get('/api/video/:id',
|
|
687
|
+
async (req, res, next) => {
|
|
688
|
+
const videoPath = `/videos/${req.params.id}.mp4`;
|
|
689
|
+
const ctrl = await s3.getStreamVideoFileCtrl({
|
|
690
|
+
fileKey: videoPath,
|
|
691
|
+
contentType: 'video/mp4',
|
|
692
|
+
bufferMB: 5
|
|
693
|
+
});
|
|
694
|
+
ctrl(req, res, next);
|
|
695
|
+
}
|
|
696
|
+
);
|
|
697
|
+
|
|
698
|
+
app.listen(3000, () => {
|
|
699
|
+
console.log('Server running on port 3000');
|
|
700
|
+
});
|
|
701
|
+
```
|
|
702
|
+
|
|
703
|
+
## ๐ Performance Tips
|
|
704
|
+
|
|
705
|
+
1. **Use Pagination**: For large directories, always use paginated methods
|
|
706
|
+
2. **Stream Large Files**: Use streaming methods instead of loading entire files into memory
|
|
707
|
+
3. **Connection Pooling**: The built-in connection pooling is optimized for concurrent requests
|
|
708
|
+
4. **Batch Operations**: Use `Promise.all()` for parallel operations when possible
|
|
709
|
+
5. **Presigned URLs**: Generate presigned URLs for direct client uploads/downloads when appropriate
|
|
710
|
+
|
|
711
|
+
## ๐ก๏ธ Error Handling
|
|
712
|
+
|
|
713
|
+
```typescript
|
|
714
|
+
try {
|
|
715
|
+
await s3.uploadFile('/docs/file.pdf', buffer);
|
|
716
|
+
} catch (error) {
|
|
717
|
+
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
|
718
|
+
console.error('File not found');
|
|
719
|
+
} else {
|
|
720
|
+
console.error('Upload failed:', error);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
## ๐ TypeScript Support
|
|
726
|
+
|
|
727
|
+
This package is written in TypeScript and includes full type definitions:
|
|
728
|
+
|
|
729
|
+
```typescript
|
|
730
|
+
import type {
|
|
731
|
+
ContentFile,
|
|
732
|
+
FileUploadResponse,
|
|
733
|
+
TreeDirectoryItem,
|
|
734
|
+
UploadedS3File,
|
|
735
|
+
S3UploadOptions
|
|
736
|
+
} from '@hdriel/aws-utils';
|
|
737
|
+
```
|
|
738
|
+
|
|
739
|
+
## ๐ค Author
|
|
740
|
+
|
|
741
|
+
[Hadriel Benjo](https://github.com/hdriel)
|
|
742
|
+
|
|
743
|
+
## ๐ Links
|
|
744
|
+
|
|
745
|
+
- [AWS S3 Documentation](https://docs.aws.amazon.com/s3/)
|
|
746
|
+
- [LocalStack Documentation](https://docs.localstack.cloud/user-guide/aws/s3/)
|
|
747
|
+
- [GitHub Repository](#)
|
|
748
|
+
|
|
749
|
+
---
|
|
750
|
+
|
|
751
|
+
Made with โค๏ธ for developers who want powerful S3 utilities without the complexity.
|
package/dist/index.cjs
CHANGED
|
@@ -882,49 +882,6 @@ var S3Directory = class extends S3Bucket {
|
|
|
882
882
|
};
|
|
883
883
|
});
|
|
884
884
|
}
|
|
885
|
-
/**
|
|
886
|
-
* Get all files recursively (example for search/indexing)
|
|
887
|
-
* @param directoryPath
|
|
888
|
-
*/
|
|
889
|
-
directoryListRecursive(directoryPath) {
|
|
890
|
-
return __async(this, null, function* () {
|
|
891
|
-
var _a2;
|
|
892
|
-
let normalizedPath = getNormalizedPath(directoryPath);
|
|
893
|
-
if (normalizedPath !== "/" && directoryPath !== "" && directoryPath !== void 0) normalizedPath += "/";
|
|
894
|
-
else normalizedPath = "/";
|
|
895
|
-
const allDirectories = [];
|
|
896
|
-
const allFiles = [];
|
|
897
|
-
let ContinuationToken = void 0;
|
|
898
|
-
do {
|
|
899
|
-
const result = yield this.execute(
|
|
900
|
-
new import_client_s33.ListObjectsV2Command({
|
|
901
|
-
Bucket: this.bucket,
|
|
902
|
-
Prefix: normalizedPath,
|
|
903
|
-
ContinuationToken
|
|
904
|
-
})
|
|
905
|
-
);
|
|
906
|
-
if (result.Contents) {
|
|
907
|
-
for (const content of result.Contents) {
|
|
908
|
-
const fullPath = content.Key;
|
|
909
|
-
const relativePath = fullPath.replace(normalizedPath, "");
|
|
910
|
-
const filename = fullPath.split("/").pop();
|
|
911
|
-
if (fullPath.endsWith("/")) {
|
|
912
|
-
allDirectories.push(relativePath.slice(0, -1));
|
|
913
|
-
} else {
|
|
914
|
-
allFiles.push(__spreadProps(__spreadValues({}, content), {
|
|
915
|
-
Name: filename,
|
|
916
|
-
Path: fullPath,
|
|
917
|
-
Location: content.Key ? `${this.link}${(_a2 = content.Key) == null ? void 0 : _a2.replace(/^\//, "")}` : "",
|
|
918
|
-
LastModified: content.LastModified ? new Date(content.LastModified) : null
|
|
919
|
-
}));
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
ContinuationToken = result.NextContinuationToken;
|
|
924
|
-
} while (ContinuationToken);
|
|
925
|
-
return { directories: allDirectories, files: allFiles };
|
|
926
|
-
});
|
|
927
|
-
}
|
|
928
885
|
/**
|
|
929
886
|
* Get tree files recursively (example for build file explorer UI)
|
|
930
887
|
* @param directoryPath - the directory start from
|
package/dist/index.d.cts
CHANGED
|
@@ -207,16 +207,6 @@ declare class S3Directory extends S3Bucket {
|
|
|
207
207
|
files: ContentFile[];
|
|
208
208
|
totalFetched: number;
|
|
209
209
|
}>;
|
|
210
|
-
/**
|
|
211
|
-
* Get all files recursively (example for search/indexing)
|
|
212
|
-
* @param directoryPath
|
|
213
|
-
*/
|
|
214
|
-
directoryListRecursive(directoryPath?: string): Promise<{
|
|
215
|
-
directories: string[];
|
|
216
|
-
files: Array<ContentFile & {
|
|
217
|
-
Name: string;
|
|
218
|
-
}>;
|
|
219
|
-
}>;
|
|
220
210
|
/**
|
|
221
211
|
* Get tree files recursively (example for build file explorer UI)
|
|
222
212
|
* @param directoryPath - the directory start from
|
package/dist/index.d.ts
CHANGED
|
@@ -207,16 +207,6 @@ declare class S3Directory extends S3Bucket {
|
|
|
207
207
|
files: ContentFile[];
|
|
208
208
|
totalFetched: number;
|
|
209
209
|
}>;
|
|
210
|
-
/**
|
|
211
|
-
* Get all files recursively (example for search/indexing)
|
|
212
|
-
* @param directoryPath
|
|
213
|
-
*/
|
|
214
|
-
directoryListRecursive(directoryPath?: string): Promise<{
|
|
215
|
-
directories: string[];
|
|
216
|
-
files: Array<ContentFile & {
|
|
217
|
-
Name: string;
|
|
218
|
-
}>;
|
|
219
|
-
}>;
|
|
220
210
|
/**
|
|
221
211
|
* Get tree files recursively (example for build file explorer UI)
|
|
222
212
|
* @param directoryPath - the directory start from
|
package/dist/index.js
CHANGED
|
@@ -871,49 +871,6 @@ var S3Directory = class extends S3Bucket {
|
|
|
871
871
|
};
|
|
872
872
|
});
|
|
873
873
|
}
|
|
874
|
-
/**
|
|
875
|
-
* Get all files recursively (example for search/indexing)
|
|
876
|
-
* @param directoryPath
|
|
877
|
-
*/
|
|
878
|
-
directoryListRecursive(directoryPath) {
|
|
879
|
-
return __async(this, null, function* () {
|
|
880
|
-
var _a2;
|
|
881
|
-
let normalizedPath = getNormalizedPath(directoryPath);
|
|
882
|
-
if (normalizedPath !== "/" && directoryPath !== "" && directoryPath !== void 0) normalizedPath += "/";
|
|
883
|
-
else normalizedPath = "/";
|
|
884
|
-
const allDirectories = [];
|
|
885
|
-
const allFiles = [];
|
|
886
|
-
let ContinuationToken = void 0;
|
|
887
|
-
do {
|
|
888
|
-
const result = yield this.execute(
|
|
889
|
-
new ListObjectsV2Command2({
|
|
890
|
-
Bucket: this.bucket,
|
|
891
|
-
Prefix: normalizedPath,
|
|
892
|
-
ContinuationToken
|
|
893
|
-
})
|
|
894
|
-
);
|
|
895
|
-
if (result.Contents) {
|
|
896
|
-
for (const content of result.Contents) {
|
|
897
|
-
const fullPath = content.Key;
|
|
898
|
-
const relativePath = fullPath.replace(normalizedPath, "");
|
|
899
|
-
const filename = fullPath.split("/").pop();
|
|
900
|
-
if (fullPath.endsWith("/")) {
|
|
901
|
-
allDirectories.push(relativePath.slice(0, -1));
|
|
902
|
-
} else {
|
|
903
|
-
allFiles.push(__spreadProps(__spreadValues({}, content), {
|
|
904
|
-
Name: filename,
|
|
905
|
-
Path: fullPath,
|
|
906
|
-
Location: content.Key ? `${this.link}${(_a2 = content.Key) == null ? void 0 : _a2.replace(/^\//, "")}` : "",
|
|
907
|
-
LastModified: content.LastModified ? new Date(content.LastModified) : null
|
|
908
|
-
}));
|
|
909
|
-
}
|
|
910
|
-
}
|
|
911
|
-
}
|
|
912
|
-
ContinuationToken = result.NextContinuationToken;
|
|
913
|
-
} while (ContinuationToken);
|
|
914
|
-
return { directories: allDirectories, files: allFiles };
|
|
915
|
-
});
|
|
916
|
-
}
|
|
917
874
|
/**
|
|
918
875
|
* Get tree files recursively (example for build file explorer UI)
|
|
919
876
|
* @param directoryPath - the directory start from
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hdriel/aws-utils",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.4",
|
|
4
4
|
"description": "Simplified AWS SDK (v3) utilities for S3 (upload, download, streaming) with TypeScript support",
|
|
5
5
|
"author": "Hadriel Benjo (https://github.com/hdriel)",
|
|
6
6
|
"type": "module",
|