express-storage 2.0.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +366 -34
- package/dist/cjs/config/index.d.ts +10 -0
- package/dist/cjs/config/index.d.ts.map +1 -0
- package/dist/cjs/config/index.js +19 -0
- package/dist/cjs/config/index.js.map +1 -0
- package/dist/cjs/drivers/azure.driver.d.ts +27 -42
- package/dist/cjs/drivers/azure.driver.d.ts.map +1 -1
- package/dist/cjs/drivers/azure.driver.js +206 -212
- package/dist/cjs/drivers/azure.driver.js.map +1 -1
- package/dist/cjs/drivers/base.driver.d.ts +69 -103
- package/dist/cjs/drivers/base.driver.d.ts.map +1 -1
- package/dist/cjs/drivers/base.driver.js +170 -167
- package/dist/cjs/drivers/base.driver.js.map +1 -1
- package/dist/cjs/drivers/gcs.driver.d.ts +20 -38
- package/dist/cjs/drivers/gcs.driver.d.ts.map +1 -1
- package/dist/cjs/drivers/gcs.driver.js +160 -176
- package/dist/cjs/drivers/gcs.driver.js.map +1 -1
- package/dist/cjs/drivers/index.d.ts +15 -0
- package/dist/cjs/drivers/index.d.ts.map +1 -0
- package/dist/cjs/drivers/index.js +26 -0
- package/dist/cjs/drivers/index.js.map +1 -0
- package/dist/cjs/drivers/local.driver.d.ts +24 -45
- package/dist/cjs/drivers/local.driver.d.ts.map +1 -1
- package/dist/cjs/drivers/local.driver.js +266 -338
- package/dist/cjs/drivers/local.driver.js.map +1 -1
- package/dist/cjs/drivers/s3.driver.d.ts +19 -39
- package/dist/cjs/drivers/s3.driver.d.ts.map +1 -1
- package/dist/cjs/drivers/s3.driver.js +205 -197
- package/dist/cjs/drivers/s3.driver.js.map +1 -1
- package/dist/cjs/factory/driver.factory.d.ts +32 -51
- package/dist/cjs/factory/driver.factory.d.ts.map +1 -1
- package/dist/cjs/factory/driver.factory.js +75 -155
- package/dist/cjs/factory/driver.factory.js.map +1 -1
- package/dist/cjs/index.d.ts +11 -15
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +14 -47
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/storage-manager.d.ts +107 -125
- package/dist/cjs/storage-manager.d.ts.map +1 -1
- package/dist/cjs/storage-manager.js +346 -416
- package/dist/cjs/storage-manager.js.map +1 -1
- package/dist/cjs/types/storage.types.d.ts +250 -107
- package/dist/cjs/types/storage.types.d.ts.map +1 -1
- package/dist/cjs/utils/file.utils.d.ts +62 -8
- package/dist/cjs/utils/file.utils.d.ts.map +1 -1
- package/dist/cjs/utils/file.utils.js +196 -29
- package/dist/cjs/utils/file.utils.js.map +1 -1
- package/dist/cjs/utils/index.d.ts +12 -0
- package/dist/cjs/utils/index.d.ts.map +1 -0
- package/dist/cjs/utils/index.js +36 -0
- package/dist/cjs/utils/index.js.map +1 -0
- package/dist/cjs/utils/rate-limiter.d.ts +40 -0
- package/dist/cjs/utils/rate-limiter.d.ts.map +1 -0
- package/dist/cjs/utils/rate-limiter.js +87 -0
- package/dist/cjs/utils/rate-limiter.js.map +1 -0
- package/dist/esm/config/index.d.ts +10 -0
- package/dist/esm/config/index.d.ts.map +1 -0
- package/dist/esm/config/index.js +10 -0
- package/dist/esm/config/index.js.map +1 -0
- package/dist/esm/drivers/azure.driver.d.ts +27 -42
- package/dist/esm/drivers/azure.driver.d.ts.map +1 -1
- package/dist/esm/drivers/azure.driver.js +172 -210
- package/dist/esm/drivers/azure.driver.js.map +1 -1
- package/dist/esm/drivers/base.driver.d.ts +69 -103
- package/dist/esm/drivers/base.driver.d.ts.map +1 -1
- package/dist/esm/drivers/base.driver.js +171 -168
- package/dist/esm/drivers/base.driver.js.map +1 -1
- package/dist/esm/drivers/gcs.driver.d.ts +20 -38
- package/dist/esm/drivers/gcs.driver.d.ts.map +1 -1
- package/dist/esm/drivers/gcs.driver.js +126 -174
- package/dist/esm/drivers/gcs.driver.js.map +1 -1
- package/dist/esm/drivers/index.d.ts +15 -0
- package/dist/esm/drivers/index.d.ts.map +1 -0
- package/dist/esm/drivers/index.js +15 -0
- package/dist/esm/drivers/index.js.map +1 -0
- package/dist/esm/drivers/local.driver.d.ts +24 -45
- package/dist/esm/drivers/local.driver.d.ts.map +1 -1
- package/dist/esm/drivers/local.driver.js +266 -338
- package/dist/esm/drivers/local.driver.js.map +1 -1
- package/dist/esm/drivers/s3.driver.d.ts +19 -39
- package/dist/esm/drivers/s3.driver.d.ts.map +1 -1
- package/dist/esm/drivers/s3.driver.js +171 -195
- package/dist/esm/drivers/s3.driver.js.map +1 -1
- package/dist/esm/factory/driver.factory.d.ts +32 -51
- package/dist/esm/factory/driver.factory.d.ts.map +1 -1
- package/dist/esm/factory/driver.factory.js +73 -158
- package/dist/esm/factory/driver.factory.js.map +1 -1
- package/dist/esm/index.d.ts +11 -15
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +12 -19
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/storage-manager.d.ts +107 -125
- package/dist/esm/storage-manager.d.ts.map +1 -1
- package/dist/esm/storage-manager.js +348 -418
- package/dist/esm/storage-manager.js.map +1 -1
- package/dist/esm/types/storage.types.d.ts +250 -107
- package/dist/esm/types/storage.types.d.ts.map +1 -1
- package/dist/esm/utils/file.utils.d.ts +62 -8
- package/dist/esm/utils/file.utils.d.ts.map +1 -1
- package/dist/esm/utils/file.utils.js +190 -29
- package/dist/esm/utils/file.utils.js.map +1 -1
- package/dist/esm/utils/index.d.ts +12 -0
- package/dist/esm/utils/index.d.ts.map +1 -0
- package/dist/esm/utils/index.js +11 -0
- package/dist/esm/utils/index.js.map +1 -0
- package/dist/esm/utils/rate-limiter.d.ts +40 -0
- package/dist/esm/utils/rate-limiter.d.ts.map +1 -0
- package/dist/esm/utils/rate-limiter.js +82 -0
- package/dist/esm/utils/rate-limiter.js.map +1 -0
- package/package.json +83 -48
- package/src/config/index.ts +17 -0
- package/src/drivers/azure.driver.ts +434 -0
- package/src/drivers/base.driver.ts +436 -0
- package/src/drivers/gcs.driver.ts +366 -0
- package/src/drivers/index.ts +15 -0
- package/src/drivers/local.driver.ts +626 -0
- package/src/drivers/s3.driver.ts +459 -0
- package/src/factory/driver.factory.ts +101 -0
- package/src/index.ts +72 -0
- package/src/storage-manager.ts +801 -0
- package/src/types/storage.types.ts +561 -0
- package/src/utils/config.utils.ts +229 -0
- package/src/utils/file.utils.ts +536 -0
- package/src/utils/index.ts +35 -0
- package/src/utils/rate-limiter.ts +94 -0
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Express Storage
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**Express.js file upload middleware for AWS S3, Google Cloud Storage, Azure Blob Storage, and local disk — one unified API, zero vendor lock-in.**
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Express Storage is a TypeScript-first file upload library for Node.js and Express. Upload files to AWS S3, Google Cloud Storage (GCS), Azure Blob Storage, or local disk using a single API. Switch cloud providers by changing one environment variable — no code changes needed. Built-in presigned URL support, file validation, streaming uploads, and security protection make it a production-ready alternative to multer-s3 that works with every major cloud provider.
|
|
6
6
|
|
|
7
7
|
[](https://www.npmjs.com/package/express-storage)
|
|
8
8
|
[](https://www.npmjs.com/package/express-storage)
|
|
@@ -14,21 +14,45 @@ Stop writing separate upload code for every storage provider. Express Storage gi
|
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
17
|
-
##
|
|
17
|
+
## Table of Contents
|
|
18
|
+
|
|
19
|
+
- [Features](#features)
|
|
20
|
+
- [Quick Start](#quick-start)
|
|
21
|
+
- [Supported Storage Providers](#supported-storage-providers)
|
|
22
|
+
- [Error Codes](#error-codes)
|
|
23
|
+
- [Security Features](#security-features)
|
|
24
|
+
- [Presigned URLs: Client-Side Uploads](#presigned-urls-client-side-uploads)
|
|
25
|
+
- [Large File Uploads](#large-file-uploads)
|
|
26
|
+
- [API Reference](#api-reference)
|
|
27
|
+
- [Environment Variables](#environment-variables)
|
|
28
|
+
- [Lifecycle Hooks](#lifecycle-hooks)
|
|
29
|
+
- [Type-Safe Results](#type-safe-results)
|
|
30
|
+
- [Configurable Concurrency](#configurable-concurrency)
|
|
31
|
+
- [Lifecycle Management](#lifecycle-management)
|
|
32
|
+
- [Custom Rate Limiting](#custom-rate-limiting)
|
|
33
|
+
- [Utilities](#utilities)
|
|
34
|
+
- [Real-World Examples](#real-world-examples)
|
|
35
|
+
- [Migrating Between Providers](#migrating-between-providers)
|
|
36
|
+
- [Migrating from v2 to v3](#migrating-from-v2-to-v3)
|
|
37
|
+
- [Why Express Storage over Alternatives?](#why-express-storage-over-alternatives)
|
|
38
|
+
- [TypeScript Support](#typescript-support)
|
|
39
|
+
- [Contributing](#contributing)
|
|
18
40
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
You start with local storage, then realize you need S3 for production. You copy-paste upload code from Stack Overflow, then discover it's vulnerable to path traversal attacks. You build presigned URL support, then learn Azure handles it completely differently than AWS.
|
|
22
|
-
|
|
23
|
-
**Express Storage solves these problems once, so you don't have to.**
|
|
41
|
+
---
|
|
24
42
|
|
|
25
|
-
|
|
43
|
+
## Features
|
|
26
44
|
|
|
27
|
-
- **One API, Four Providers** — Write upload code once. Deploy to
|
|
28
|
-
- **
|
|
29
|
-
- **
|
|
30
|
-
- **
|
|
45
|
+
- **One API, Four Providers** — Write upload code once. Deploy to AWS S3, GCS, Azure, or local disk.
|
|
46
|
+
- **Presigned URLs** — Client-side uploads that bypass your server, with per-provider constraint enforcement.
|
|
47
|
+
- **File Validation** — Size limits, MIME type checks, and extension filtering before storage.
|
|
48
|
+
- **Security Built In** — Path traversal prevention, filename sanitization, null byte protection.
|
|
49
|
+
- **TypeScript Native** — Full type safety with discriminated unions. No `any` types.
|
|
50
|
+
- **Streaming Uploads** — Automatic multipart/streaming for files over 100MB.
|
|
31
51
|
- **Zero Config Switching** — Change `FILE_DRIVER=local` to `FILE_DRIVER=s3` and you're done.
|
|
52
|
+
- **Lifecycle Hooks** — Tap into upload/delete events for logging, virus scanning, or audit trails.
|
|
53
|
+
- **Batch Operations** — Upload or delete multiple files in parallel with concurrency control and `AbortSignal` support.
|
|
54
|
+
- **Custom Rate Limiting** — Built-in in-memory limiter or plug in your own (Redis, Memcached, etc.).
|
|
55
|
+
- **Lightweight** — Install only the cloud SDK you need. No dependency bloat.
|
|
32
56
|
|
|
33
57
|
---
|
|
34
58
|
|
|
@@ -40,6 +64,21 @@ You start with local storage, then realize you need S3 for production. You copy-
|
|
|
40
64
|
npm install express-storage
|
|
41
65
|
```
|
|
42
66
|
|
|
67
|
+
Then install only the cloud SDK you need:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
# For AWS S3
|
|
71
|
+
npm install @aws-sdk/client-s3 @aws-sdk/lib-storage @aws-sdk/s3-request-presigner
|
|
72
|
+
|
|
73
|
+
# For Google Cloud Storage
|
|
74
|
+
npm install @google-cloud/storage
|
|
75
|
+
|
|
76
|
+
# For Azure Blob Storage
|
|
77
|
+
npm install @azure/storage-blob @azure/identity
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Local storage works out of the box with no additional dependencies.
|
|
81
|
+
|
|
43
82
|
### Basic Setup
|
|
44
83
|
|
|
45
84
|
```typescript
|
|
@@ -58,7 +97,7 @@ app.post("/upload", upload.single("file"), async (req, res) => {
|
|
|
58
97
|
});
|
|
59
98
|
|
|
60
99
|
if (result.success) {
|
|
61
|
-
res.json({ url: result.fileUrl });
|
|
100
|
+
res.json({ reference: result.reference, url: result.fileUrl });
|
|
62
101
|
} else {
|
|
63
102
|
res.status(400).json({ error: result.error });
|
|
64
103
|
}
|
|
@@ -109,6 +148,52 @@ That's it. Your upload code stays the same regardless of which provider you choo
|
|
|
109
148
|
|
|
110
149
|
---
|
|
111
150
|
|
|
151
|
+
## Error Codes
|
|
152
|
+
|
|
153
|
+
Every error result includes a `code` field for programmatic error handling — no more parsing error strings:
|
|
154
|
+
|
|
155
|
+
```typescript
|
|
156
|
+
const result = await storage.uploadFile(file, {
|
|
157
|
+
maxSize: 5 * 1024 * 1024,
|
|
158
|
+
allowedMimeTypes: ["image/jpeg", "image/png"],
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
if (!result.success) {
|
|
162
|
+
switch (result.code) {
|
|
163
|
+
case "FILE_TOO_LARGE":
|
|
164
|
+
res.status(413).json({ error: "File is too large" });
|
|
165
|
+
break;
|
|
166
|
+
case "INVALID_MIME_TYPE":
|
|
167
|
+
res.status(415).json({ error: "Unsupported file type" });
|
|
168
|
+
break;
|
|
169
|
+
case "RATE_LIMITED":
|
|
170
|
+
res.status(429).json({ error: "Too many requests" });
|
|
171
|
+
break;
|
|
172
|
+
default:
|
|
173
|
+
res.status(400).json({ error: result.error });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
| Code | When |
|
|
179
|
+
| -------------------------- | -------------------------------------------------------------- |
|
|
180
|
+
| `NO_FILE` | No file provided to upload |
|
|
181
|
+
| `FILE_EMPTY` | File has zero bytes |
|
|
182
|
+
| `FILE_TOO_LARGE` | File exceeds `maxSize` or `maxFileSize` |
|
|
183
|
+
| `INVALID_MIME_TYPE` | MIME type not in `allowedMimeTypes` |
|
|
184
|
+
| `INVALID_EXTENSION` | Extension not in `allowedExtensions` |
|
|
185
|
+
| `INVALID_FILENAME` | Filename is empty, too long, or contains illegal characters |
|
|
186
|
+
| `INVALID_INPUT` | Bad argument (e.g., non-numeric fileSize, missing fileName) |
|
|
187
|
+
| `PATH_TRAVERSAL` | Path contains `..`, `\0`, or other traversal sequences |
|
|
188
|
+
| `FILE_NOT_FOUND` | File doesn't exist (delete, validate, view) |
|
|
189
|
+
| `VALIDATION_FAILED` | Post-upload validation failed (content type or size mismatch) |
|
|
190
|
+
| `RATE_LIMITED` | Presigned URL rate limit exceeded |
|
|
191
|
+
| `HOOK_ABORTED` | A `beforeUpload` or `beforeDelete` hook threw |
|
|
192
|
+
| `PRESIGNED_NOT_SUPPORTED` | Local driver doesn't support presigned URLs |
|
|
193
|
+
| `PROVIDER_ERROR` | Cloud provider SDK error (network, auth, permissions) |
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
112
197
|
## Security Features
|
|
113
198
|
|
|
114
199
|
File uploads are one of the most exploited attack vectors in web applications. Express Storage protects you by default.
|
|
@@ -303,9 +388,6 @@ const result = await storage.uploadFile(file, validation?, options?);
|
|
|
303
388
|
|
|
304
389
|
// Multiple files (processed in parallel with concurrency limits)
|
|
305
390
|
const results = await storage.uploadFiles(files, validation?, options?);
|
|
306
|
-
|
|
307
|
-
// Generic upload (auto-detects single vs multiple)
|
|
308
|
-
const result = await storage.upload(input, validation?, options?);
|
|
309
391
|
```
|
|
310
392
|
|
|
311
393
|
### Presigned URL Methods
|
|
@@ -328,12 +410,17 @@ const results = await storage.generateViewUrls(references);
|
|
|
328
410
|
### File Management
|
|
329
411
|
|
|
330
412
|
```typescript
|
|
331
|
-
// Delete single file
|
|
332
|
-
const
|
|
413
|
+
// Delete single file (returns DeleteResult with error details on failure)
|
|
414
|
+
const result = await storage.deleteFile(reference);
|
|
415
|
+
if (!result.success) console.log(result.error, result.code);
|
|
333
416
|
|
|
334
417
|
// Delete multiple files
|
|
335
418
|
const results = await storage.deleteFiles(references);
|
|
336
419
|
|
|
420
|
+
// Get file metadata without downloading
|
|
421
|
+
const info = await storage.getMetadata(reference);
|
|
422
|
+
if (info) console.log(info.name, info.size, info.contentType, info.lastModified);
|
|
423
|
+
|
|
337
424
|
// List files with pagination
|
|
338
425
|
const result = await storage.listFiles(prefix?, maxResults?, continuationToken?);
|
|
339
426
|
```
|
|
@@ -407,6 +494,145 @@ interface FileValidationOptions {
|
|
|
407
494
|
|
|
408
495
|
---
|
|
409
496
|
|
|
497
|
+
## Lifecycle Hooks
|
|
498
|
+
|
|
499
|
+
Hooks let you tap into the upload/delete lifecycle without modifying drivers. Perfect for logging, virus scanning, metrics, or audit trails.
|
|
500
|
+
|
|
501
|
+
```typescript
|
|
502
|
+
const storage = new StorageManager({
|
|
503
|
+
driver: "s3",
|
|
504
|
+
hooks: {
|
|
505
|
+
beforeUpload: async (file) => {
|
|
506
|
+
await virusScan(file.buffer); // Throw to abort upload
|
|
507
|
+
},
|
|
508
|
+
afterUpload: (result, file) => {
|
|
509
|
+
auditLog("file_uploaded", { result, originalName: file.originalname });
|
|
510
|
+
},
|
|
511
|
+
beforeDelete: async (reference) => {
|
|
512
|
+
await checkPermissions(reference);
|
|
513
|
+
},
|
|
514
|
+
afterDelete: (reference, success) => {
|
|
515
|
+
if (success) auditLog("file_deleted", { reference });
|
|
516
|
+
},
|
|
517
|
+
onError: (error, context) => {
|
|
518
|
+
metrics.increment("storage.error", { operation: context.operation });
|
|
519
|
+
},
|
|
520
|
+
},
|
|
521
|
+
});
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
All hooks are optional and async-safe. `beforeUpload` and `beforeDelete` can throw to abort the operation — the error message is included in the result.
|
|
525
|
+
|
|
526
|
+
---
|
|
527
|
+
|
|
528
|
+
## Type-Safe Results
|
|
529
|
+
|
|
530
|
+
All result types use TypeScript discriminated unions. Check `result.success` and TypeScript narrows the type automatically:
|
|
531
|
+
|
|
532
|
+
```typescript
|
|
533
|
+
const result = await storage.uploadFile(file);
|
|
534
|
+
|
|
535
|
+
if (result.success) {
|
|
536
|
+
console.log(result.reference); // stored file path (for delete/view/getMetadata)
|
|
537
|
+
console.log(result.fileUrl); // URL to access the file
|
|
538
|
+
} else {
|
|
539
|
+
console.log(result.error); // TypeScript knows this exists
|
|
540
|
+
}
|
|
541
|
+
```
|
|
542
|
+
|
|
543
|
+
This applies to all result types: `FileUploadResult`, `DeleteResult`, `PresignedUrlResult`, `BlobValidationResult`, and `ListFilesResult`.
|
|
544
|
+
|
|
545
|
+
---
|
|
546
|
+
|
|
547
|
+
## Configurable Concurrency
|
|
548
|
+
|
|
549
|
+
Control how many parallel operations run in batch methods:
|
|
550
|
+
|
|
551
|
+
```typescript
|
|
552
|
+
const storage = new StorageManager({
|
|
553
|
+
driver: "s3",
|
|
554
|
+
concurrency: 5, // Applies to uploadFiles, deleteFiles, generateUploadUrls, etc.
|
|
555
|
+
});
|
|
556
|
+
```
|
|
557
|
+
|
|
558
|
+
Default is 10. Lower it for rate-limited APIs or resource-constrained environments.
|
|
559
|
+
|
|
560
|
+
### Cancellable Batch Operations
|
|
561
|
+
|
|
562
|
+
All batch methods accept an `AbortSignal` for cancelling long-running operations mid-flight:
|
|
563
|
+
|
|
564
|
+
```typescript
|
|
565
|
+
const controller = new AbortController();
|
|
566
|
+
|
|
567
|
+
// Cancel after 5 seconds
|
|
568
|
+
setTimeout(() => controller.abort(), 5000);
|
|
569
|
+
|
|
570
|
+
try {
|
|
571
|
+
const results = await storage.uploadFiles(files, validation, options, {
|
|
572
|
+
signal: controller.signal,
|
|
573
|
+
});
|
|
574
|
+
} catch (error) {
|
|
575
|
+
console.log("Upload batch was cancelled");
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// Also works with deleteFiles, generateUploadUrls, generateViewUrls
|
|
579
|
+
await storage.deleteFiles(references, { signal: controller.signal });
|
|
580
|
+
```
|
|
581
|
+
|
|
582
|
+
---
|
|
583
|
+
|
|
584
|
+
## Lifecycle Management
|
|
585
|
+
|
|
586
|
+
Clean up resources when you're done with a StorageManager instance:
|
|
587
|
+
|
|
588
|
+
```typescript
|
|
589
|
+
const storage = new StorageManager({ driver: "s3", rateLimiter: { maxRequests: 100 } });
|
|
590
|
+
|
|
591
|
+
// ... use storage ...
|
|
592
|
+
|
|
593
|
+
// Release resources (clears factory cache entry and rate limiter)
|
|
594
|
+
storage.destroy();
|
|
595
|
+
```
|
|
596
|
+
|
|
597
|
+
This is especially useful in tests, serverless functions, or any environment where StorageManager instances are created and discarded frequently.
|
|
598
|
+
|
|
599
|
+
---
|
|
600
|
+
|
|
601
|
+
## Custom Rate Limiting
|
|
602
|
+
|
|
603
|
+
The built-in rate limiter works for single-process apps. For clustered deployments, provide your own adapter:
|
|
604
|
+
|
|
605
|
+
```typescript
|
|
606
|
+
import { StorageManager, RateLimiterAdapter } from "express-storage";
|
|
607
|
+
// or: import { RateLimiterAdapter } from "express-storage"; // types are always at top level
|
|
608
|
+
|
|
609
|
+
// Built-in in-memory limiter
|
|
610
|
+
const storage = new StorageManager({
|
|
611
|
+
driver: "s3",
|
|
612
|
+
rateLimiter: { maxRequests: 100, windowMs: 60000 },
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
// Custom Redis-backed limiter
|
|
616
|
+
class RedisRateLimiter implements RateLimiterAdapter {
|
|
617
|
+
async tryAcquire() {
|
|
618
|
+
/* Redis INCR + EXPIRE */
|
|
619
|
+
}
|
|
620
|
+
async getRemainingRequests() {
|
|
621
|
+
/* ... */
|
|
622
|
+
}
|
|
623
|
+
async getResetTime() {
|
|
624
|
+
/* ... */
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const storage = new StorageManager({
|
|
629
|
+
driver: "s3",
|
|
630
|
+
rateLimiter: new RedisRateLimiter(redisClient),
|
|
631
|
+
});
|
|
632
|
+
```
|
|
633
|
+
|
|
634
|
+
---
|
|
635
|
+
|
|
410
636
|
## Utilities
|
|
411
637
|
|
|
412
638
|
Express Storage includes battle-tested utilities you can use directly.
|
|
@@ -414,7 +640,7 @@ Express Storage includes battle-tested utilities you can use directly.
|
|
|
414
640
|
### Retry with Exponential Backoff
|
|
415
641
|
|
|
416
642
|
```typescript
|
|
417
|
-
import { withRetry } from "express-storage";
|
|
643
|
+
import { withRetry } from "express-storage/utils";
|
|
418
644
|
|
|
419
645
|
const result = await withRetry(() => storage.uploadFile(file), {
|
|
420
646
|
maxAttempts: 3,
|
|
@@ -432,7 +658,7 @@ import {
|
|
|
432
658
|
isDocumentFile,
|
|
433
659
|
getFileExtension,
|
|
434
660
|
formatFileSize,
|
|
435
|
-
} from "express-storage";
|
|
661
|
+
} from "express-storage/utils";
|
|
436
662
|
|
|
437
663
|
isImageFile("image/jpeg"); // true
|
|
438
664
|
isDocumentFile("application/pdf"); // true
|
|
@@ -443,7 +669,7 @@ formatFileSize(1048576); // '1 MB'
|
|
|
443
669
|
### Custom Logging
|
|
444
670
|
|
|
445
671
|
```typescript
|
|
446
|
-
import { StorageManager, Logger } from "express-storage";
|
|
672
|
+
import { StorageManager, type Logger } from "express-storage";
|
|
447
673
|
|
|
448
674
|
const logger: Logger = {
|
|
449
675
|
debug: (msg, ...args) => console.debug(`[Storage] ${msg}`, ...args),
|
|
@@ -476,7 +702,7 @@ app.post("/users/:id/avatar", upload.single("avatar"), async (req, res) => {
|
|
|
476
702
|
);
|
|
477
703
|
|
|
478
704
|
if (result.success) {
|
|
479
|
-
await db.users.update(req.params.id, { avatarUrl: result.fileUrl });
|
|
705
|
+
await db.users.update(req.params.id, { reference: result.reference, avatarUrl: result.fileUrl });
|
|
480
706
|
res.json({ avatarUrl: result.fileUrl });
|
|
481
707
|
} else {
|
|
482
708
|
res.status(400).json({ error: result.error });
|
|
@@ -553,7 +779,7 @@ app.post("/gallery/upload", upload.array("photos", 20), async (req, res) => {
|
|
|
553
779
|
uploaded: successful.length,
|
|
554
780
|
failed: failed.length,
|
|
555
781
|
files: successful.map((r) => ({
|
|
556
|
-
|
|
782
|
+
reference: r.reference,
|
|
557
783
|
url: r.fileUrl,
|
|
558
784
|
})),
|
|
559
785
|
errors: failed.map((r) => r.error),
|
|
@@ -600,27 +826,130 @@ AZURE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=...
|
|
|
600
826
|
|
|
601
827
|
---
|
|
602
828
|
|
|
829
|
+
## Migrating from v2 to v3
|
|
830
|
+
|
|
831
|
+
v3 has breaking changes in dependencies, types, and configuration. Most apps require minimal code changes.
|
|
832
|
+
|
|
833
|
+
### What Changed
|
|
834
|
+
|
|
835
|
+
1. **Cloud SDKs are optional peer dependencies.** Install only what you need — no more downloading all SDKs.
|
|
836
|
+
2. **Result types are discriminated unions.** `result.fileName` is guaranteed when `result.success === true`. Code that accessed properties without checking `success` may need updates.
|
|
837
|
+
3. **Presigned driver subclasses removed.** `S3PresignedStorageDriver`, `GCSPresignedStorageDriver`, and `AzurePresignedStorageDriver` are no longer exported. Use the base driver classes or `StorageManager` (the `'s3-presigned'` driver string still works).
|
|
838
|
+
4. **`rateLimit` option renamed to `rateLimiter`.** Now accepts either options or a custom adapter.
|
|
839
|
+
5. **`getRateLimitStatus()` is async.** Returns a Promise.
|
|
840
|
+
6. **`deleteFile()` returns `DeleteResult`** instead of `boolean`. Check `result.success` instead of the boolean value.
|
|
841
|
+
7. **`IStorageDriver.delete()` returns `DeleteResult`** instead of `boolean`. Custom drivers must be updated.
|
|
842
|
+
8. **`ensureDirectoryExists()` is async.** Returns a `Promise<void>` — add `await` to existing calls.
|
|
843
|
+
9. **Presigned URL methods return stricter types.** `generateUploadUrl()` returns `PresignedUploadUrlResult` (guarantees `uploadUrl`, `fileName`, `reference`, `expiresIn` on success). `generateViewUrl()` returns `PresignedViewUrlResult` (guarantees `viewUrl`, `reference`, `expiresIn` on success).
|
|
844
|
+
|
|
845
|
+
### Migration Steps
|
|
846
|
+
|
|
847
|
+
1. Update the package:
|
|
848
|
+
|
|
849
|
+
```bash
|
|
850
|
+
npm install express-storage@3
|
|
851
|
+
```
|
|
852
|
+
|
|
853
|
+
2. Install the SDK for your provider:
|
|
854
|
+
|
|
855
|
+
```bash
|
|
856
|
+
# If you use S3
|
|
857
|
+
npm install @aws-sdk/client-s3 @aws-sdk/lib-storage @aws-sdk/s3-request-presigner
|
|
858
|
+
|
|
859
|
+
# If you use GCS
|
|
860
|
+
npm install @google-cloud/storage
|
|
861
|
+
|
|
862
|
+
# If you use Azure
|
|
863
|
+
npm install @azure/storage-blob @azure/identity
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
3. Update result type access — `fileName` is now `reference`:
|
|
867
|
+
|
|
868
|
+
```typescript
|
|
869
|
+
// Before (v2)
|
|
870
|
+
const name = result.fileName!;
|
|
871
|
+
|
|
872
|
+
// After (v3) — "reference" is the stored file path used for all subsequent operations
|
|
873
|
+
if (result.success) {
|
|
874
|
+
const ref = result.reference; // pass to deleteFile(), getMetadata(), generateViewUrl()
|
|
875
|
+
const url = result.fileUrl; // URL to access the file
|
|
876
|
+
}
|
|
877
|
+
```
|
|
878
|
+
|
|
879
|
+
4. Update rate limiting config (if used):
|
|
880
|
+
|
|
881
|
+
```typescript
|
|
882
|
+
// Before (v2)
|
|
883
|
+
new StorageManager({ driver: "s3", rateLimit: { maxRequests: 100 } });
|
|
884
|
+
|
|
885
|
+
// After (v3)
|
|
886
|
+
new StorageManager({ driver: "s3", rateLimiter: { maxRequests: 100 } });
|
|
887
|
+
```
|
|
888
|
+
|
|
889
|
+
If you forget to install a required SDK, you'll get a clear error message telling you exactly what to install.
|
|
890
|
+
|
|
891
|
+
---
|
|
892
|
+
|
|
893
|
+
## Why Express Storage over Alternatives?
|
|
894
|
+
|
|
895
|
+
If you're evaluating file upload libraries for Express.js, here's how Express Storage compares:
|
|
896
|
+
|
|
897
|
+
| Feature | **Express Storage** | **multer-s3** | **express-fileupload** | **uploadfs** |
|
|
898
|
+
| --------------------------- | ------------------- | ------------- | ---------------------- | ------------ |
|
|
899
|
+
| AWS S3 | Yes | Yes | Manual | Yes |
|
|
900
|
+
| Google Cloud Storage | Yes | No | No | Yes |
|
|
901
|
+
| Azure Blob Storage | Yes | No | No | Yes |
|
|
902
|
+
| Local disk | Yes | No | Yes | Yes |
|
|
903
|
+
| Presigned URLs | Yes | No | No | No |
|
|
904
|
+
| File validation | Yes | No | Partial | No |
|
|
905
|
+
| TypeScript (native) | Yes | No | @types | No |
|
|
906
|
+
| Streaming uploads | Yes | Yes | No | No |
|
|
907
|
+
| Switch providers at runtime | Yes (env var) | No | No | No |
|
|
908
|
+
| Path traversal protection | Yes | No | No | No |
|
|
909
|
+
| Lifecycle hooks | Yes | No | No | No |
|
|
910
|
+
| Batch operations | Yes | No | No | No |
|
|
911
|
+
| Rate limiting | Yes | No | No | No |
|
|
912
|
+
|
|
913
|
+
**multer-s3** is great if you only need S3. Express Storage covers S3 *plus* GCS, Azure, and local disk with the same code — and adds presigned URLs, validation, and security that multer-s3 doesn't provide.
|
|
914
|
+
|
|
915
|
+
---
|
|
916
|
+
|
|
603
917
|
## TypeScript Support
|
|
604
918
|
|
|
605
919
|
Express Storage is written in TypeScript and exports all types:
|
|
606
920
|
|
|
607
921
|
```typescript
|
|
922
|
+
// Core — what most users need
|
|
608
923
|
import {
|
|
609
924
|
StorageManager,
|
|
610
|
-
|
|
925
|
+
InMemoryRateLimiter,
|
|
611
926
|
FileUploadResult,
|
|
612
|
-
|
|
927
|
+
DeleteResult,
|
|
928
|
+
PresignedUploadUrlResult,
|
|
929
|
+
StorageOptions,
|
|
613
930
|
FileValidationOptions,
|
|
614
931
|
UploadOptions,
|
|
615
|
-
Logger,
|
|
616
932
|
} from "express-storage";
|
|
617
933
|
|
|
618
|
-
//
|
|
934
|
+
// Utilities — standalone helpers (import separately to keep your bundle small)
|
|
935
|
+
import { withRetry, formatFileSize, withConcurrencyLimit } from "express-storage/utils";
|
|
936
|
+
|
|
937
|
+
// Drivers — for custom driver implementations or direct driver use
|
|
938
|
+
import { BaseStorageDriver, createDriver } from "express-storage/drivers";
|
|
939
|
+
|
|
940
|
+
// Config — environment variable loading and validation
|
|
941
|
+
import { validateStorageConfig, loadAndValidateConfig } from "express-storage/config";
|
|
942
|
+
|
|
943
|
+
// Discriminated unions — TypeScript narrows automatically
|
|
619
944
|
const result: FileUploadResult = await storage.uploadFile(file);
|
|
620
945
|
|
|
621
946
|
if (result.success) {
|
|
622
|
-
|
|
623
|
-
console.log(result.
|
|
947
|
+
// TypeScript knows: result is FileUploadSuccess
|
|
948
|
+
console.log(result.reference); // string — stored file path
|
|
949
|
+
console.log(result.fileUrl); // string — URL to access
|
|
950
|
+
} else {
|
|
951
|
+
// TypeScript knows: result is FileUploadError
|
|
952
|
+
console.log(result.error); // string (guaranteed)
|
|
624
953
|
}
|
|
625
954
|
```
|
|
626
955
|
|
|
@@ -628,17 +957,20 @@ if (result.success) {
|
|
|
628
957
|
|
|
629
958
|
## Contributing
|
|
630
959
|
|
|
631
|
-
Contributions are welcome!
|
|
960
|
+
Contributions are welcome!
|
|
632
961
|
|
|
633
962
|
```bash
|
|
634
963
|
# Clone the repository
|
|
635
964
|
git clone https://github.com/th3hero/express-storage.git
|
|
636
965
|
|
|
637
|
-
# Install dependencies
|
|
966
|
+
# Install dependencies (includes all cloud SDKs for development)
|
|
638
967
|
npm install
|
|
639
968
|
|
|
640
|
-
# Run
|
|
641
|
-
npm
|
|
969
|
+
# Run tests
|
|
970
|
+
npm test
|
|
971
|
+
|
|
972
|
+
# Run tests in watch mode
|
|
973
|
+
npm run test:watch
|
|
642
974
|
|
|
643
975
|
# Build for production
|
|
644
976
|
npm run build
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* express-storage/config
|
|
3
|
+
*
|
|
4
|
+
* Configuration loading, validation, and environment variable utilities.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* import { loadAndValidateConfig, validateStorageConfig } from 'express-storage/config';
|
|
8
|
+
*/
|
|
9
|
+
export { loadAndValidateConfig, validateStorageConfig, initializeDotenv, resetDotenvInitialization, loadEnvironmentConfig, environmentToStorageConfig, } from '../utils/config.utils.js';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,0BAA0B,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* express-storage/config
|
|
4
|
+
*
|
|
5
|
+
* Configuration loading, validation, and environment variable utilities.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* import { loadAndValidateConfig, validateStorageConfig } from 'express-storage/config';
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.environmentToStorageConfig = exports.loadEnvironmentConfig = exports.resetDotenvInitialization = exports.initializeDotenv = exports.validateStorageConfig = exports.loadAndValidateConfig = void 0;
|
|
12
|
+
var config_utils_js_1 = require("../utils/config.utils.js");
|
|
13
|
+
Object.defineProperty(exports, "loadAndValidateConfig", { enumerable: true, get: function () { return config_utils_js_1.loadAndValidateConfig; } });
|
|
14
|
+
Object.defineProperty(exports, "validateStorageConfig", { enumerable: true, get: function () { return config_utils_js_1.validateStorageConfig; } });
|
|
15
|
+
Object.defineProperty(exports, "initializeDotenv", { enumerable: true, get: function () { return config_utils_js_1.initializeDotenv; } });
|
|
16
|
+
Object.defineProperty(exports, "resetDotenvInitialization", { enumerable: true, get: function () { return config_utils_js_1.resetDotenvInitialization; } });
|
|
17
|
+
Object.defineProperty(exports, "loadEnvironmentConfig", { enumerable: true, get: function () { return config_utils_js_1.loadEnvironmentConfig; } });
|
|
18
|
+
Object.defineProperty(exports, "environmentToStorageConfig", { enumerable: true, get: function () { return config_utils_js_1.environmentToStorageConfig; } });
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;AAEH,4DAOkC;AANhC,wHAAA,qBAAqB,OAAA;AACrB,wHAAA,qBAAqB,OAAA;AACrB,mHAAA,gBAAgB,OAAA;AAChB,4HAAA,yBAAyB,OAAA;AACzB,wHAAA,qBAAqB,OAAA;AACrB,6HAAA,0BAA0B,OAAA"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { ContainerClient } from '@azure/storage-blob';
|
|
2
1
|
import { BaseStorageDriver } from './base.driver.js';
|
|
3
|
-
import { FileUploadResult, PresignedUrlResult, StorageConfig, BlobValidationOptions, BlobValidationResult, ListFilesResult, UploadOptions } from '../types/storage.types.js';
|
|
2
|
+
import { FileUploadResult, PresignedUrlResult, StorageConfig, BlobValidationOptions, BlobValidationResult, ListFilesResult, UploadOptions, FileInfo, DeleteResult } from '../types/storage.types.js';
|
|
4
3
|
/**
|
|
5
4
|
* AzureStorageDriver - Handles file operations with Azure Blob Storage.
|
|
6
5
|
*
|
|
@@ -11,17 +10,24 @@ import { FileUploadResult, PresignedUrlResult, StorageConfig, BlobValidationOpti
|
|
|
11
10
|
*
|
|
12
11
|
* Important: SAS URL generation requires an account key.
|
|
13
12
|
* Managed Identity works great for direct uploads but can't create presigned URLs.
|
|
13
|
+
*
|
|
14
|
+
* When driver is 'azure-presigned', upload() returns SAS URLs instead of
|
|
15
|
+
* uploading directly. Always call validateAndConfirmUpload() after client
|
|
16
|
+
* uploads — Azure doesn't enforce constraints on SAS URLs.
|
|
17
|
+
*
|
|
18
|
+
* Required packages: @azure/storage-blob, @azure/identity
|
|
14
19
|
*/
|
|
15
20
|
export declare class AzureStorageDriver extends BaseStorageDriver {
|
|
16
|
-
private
|
|
17
|
-
|
|
18
|
-
private containerName;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
+
private _blobServiceClient?;
|
|
22
|
+
private _containerClient?;
|
|
23
|
+
private readonly containerName;
|
|
24
|
+
private readonly accountName;
|
|
25
|
+
private readonly accountKey?;
|
|
21
26
|
constructor(config: StorageConfig);
|
|
27
|
+
private ensureContainerClient;
|
|
28
|
+
destroy(): void;
|
|
22
29
|
/**
|
|
23
|
-
* Uploads a file
|
|
24
|
-
* Handles both memory and disk storage from Multer.
|
|
30
|
+
* Uploads a file to Azure, or returns a SAS URL when in presigned mode.
|
|
25
31
|
*
|
|
26
32
|
* For large files (>100MB), uses streaming upload to reduce
|
|
27
33
|
* memory usage and improve reliability.
|
|
@@ -34,55 +40,34 @@ export declare class AzureStorageDriver extends BaseStorageDriver {
|
|
|
34
40
|
* or content type. Always call validateAndConfirmUpload() after the
|
|
35
41
|
* client uploads to verify the file is what you expected.
|
|
36
42
|
*/
|
|
37
|
-
generateUploadUrl(fileName: string, contentType?: string,
|
|
43
|
+
generateUploadUrl(fileName: string, contentType?: string, _fileSize?: number): Promise<PresignedUrlResult>;
|
|
38
44
|
/**
|
|
39
45
|
* Creates a SAS URL for downloading/viewing a file.
|
|
40
46
|
*/
|
|
41
47
|
generateViewUrl(fileName: string): Promise<PresignedUrlResult>;
|
|
48
|
+
/**
|
|
49
|
+
* Generates a SAS URL for a blob with the specified permissions.
|
|
50
|
+
*/
|
|
51
|
+
private generateSasUrl;
|
|
42
52
|
/**
|
|
43
53
|
* Deletes a file from Azure Blob Storage.
|
|
44
|
-
* Returns false if the file doesn't exist, throws on real errors.
|
|
45
54
|
*/
|
|
46
|
-
delete(fileName: string): Promise<
|
|
55
|
+
delete(fileName: string): Promise<DeleteResult>;
|
|
47
56
|
/**
|
|
48
57
|
* Validates an upload against expected values and deletes invalid files.
|
|
58
|
+
* Uses shared validation logic from BaseStorageDriver.
|
|
49
59
|
*
|
|
50
60
|
* This is CRITICAL for Azure presigned uploads because Azure doesn't
|
|
51
|
-
* enforce constraints at the URL level.
|
|
52
|
-
* executable when you expected a 1MB image.
|
|
53
|
-
*
|
|
54
|
-
* Always call this after presigned uploads with your expected values.
|
|
61
|
+
* enforce constraints at the URL level.
|
|
55
62
|
*/
|
|
56
63
|
validateAndConfirmUpload(reference: string, options?: BlobValidationOptions): Promise<BlobValidationResult>;
|
|
57
64
|
/**
|
|
58
|
-
*
|
|
65
|
+
* Returns metadata about a file from Azure without downloading it.
|
|
59
66
|
*/
|
|
60
|
-
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* AzurePresignedStorageDriver - Azure driver that returns SAS URLs from upload().
|
|
64
|
-
*
|
|
65
|
-
* Use this when you want clients to upload directly to Azure without
|
|
66
|
-
* the file passing through your server.
|
|
67
|
-
*
|
|
68
|
-
* Critical: Always call validateAndConfirmUpload() after clients upload!
|
|
69
|
-
* Azure doesn't enforce any constraints on SAS URLs.
|
|
70
|
-
*/
|
|
71
|
-
export declare class AzurePresignedStorageDriver extends AzureStorageDriver {
|
|
72
|
-
constructor(config: StorageConfig);
|
|
73
|
-
private hasAccountKey;
|
|
67
|
+
getMetadata(reference: string): Promise<FileInfo | null>;
|
|
74
68
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* The returned fileUrl is the SAS upload URL.
|
|
78
|
-
* After the client uploads, use validateAndConfirmUpload() to verify
|
|
79
|
-
* the file and get a view URL.
|
|
80
|
-
*
|
|
81
|
-
* Note: The `options` parameter (metadata, cacheControl, etc.) is NOT applied
|
|
82
|
-
* when using presigned uploads. These options must be set by the client when
|
|
83
|
-
* making the actual upload request to Azure, or configured via container settings.
|
|
84
|
-
* For server-side uploads with full options support, use the regular 'azure' driver.
|
|
69
|
+
* Lists files in the container with optional prefix filtering and pagination.
|
|
85
70
|
*/
|
|
86
|
-
|
|
71
|
+
listFiles(prefix?: string, maxResults?: number, continuationToken?: string): Promise<ListFilesResult>;
|
|
87
72
|
}
|
|
88
73
|
//# sourceMappingURL=azure.driver.d.ts.map
|