@procedures/s3 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/LICENSE +21 -0
- package/README.md +886 -2
- package/dist/.tsbuildinfo +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/module.d.ts +16 -0
- package/dist/module.d.ts.map +1 -0
- package/dist/module.js +151 -0
- package/dist/module.js.map +1 -0
- package/dist/procedures/s3/delete.d.ts +26 -0
- package/dist/procedures/s3/delete.d.ts.map +1 -0
- package/dist/procedures/s3/delete.js +40 -0
- package/dist/procedures/s3/delete.js.map +1 -0
- package/dist/procedures/s3/download.d.ts +36 -0
- package/dist/procedures/s3/download.d.ts.map +1 -0
- package/dist/procedures/s3/download.js +143 -0
- package/dist/procedures/s3/download.js.map +1 -0
- package/dist/procedures/s3/index.d.ts +13 -0
- package/dist/procedures/s3/index.d.ts.map +1 -0
- package/dist/procedures/s3/index.js +13 -0
- package/dist/procedures/s3/index.js.map +1 -0
- package/dist/procedures/s3/list-all.d.ts +24 -0
- package/dist/procedures/s3/list-all.d.ts.map +1 -0
- package/dist/procedures/s3/list-all.js +57 -0
- package/dist/procedures/s3/list-all.js.map +1 -0
- package/dist/procedures/s3/list.d.ts +28 -0
- package/dist/procedures/s3/list.d.ts.map +1 -0
- package/dist/procedures/s3/list.js +56 -0
- package/dist/procedures/s3/list.js.map +1 -0
- package/dist/procedures/s3/multipart-abort.d.ts +21 -0
- package/dist/procedures/s3/multipart-abort.d.ts.map +1 -0
- package/dist/procedures/s3/multipart-abort.js +33 -0
- package/dist/procedures/s3/multipart-abort.js.map +1 -0
- package/dist/procedures/s3/multipart-complete.d.ts +26 -0
- package/dist/procedures/s3/multipart-complete.d.ts.map +1 -0
- package/dist/procedures/s3/multipart-complete.js +51 -0
- package/dist/procedures/s3/multipart-complete.js.map +1 -0
- package/dist/procedures/s3/multipart-init.d.ts +24 -0
- package/dist/procedures/s3/multipart-init.d.ts.map +1 -0
- package/dist/procedures/s3/multipart-init.js +42 -0
- package/dist/procedures/s3/multipart-init.js.map +1 -0
- package/dist/procedures/s3/multipart-upload.d.ts +24 -0
- package/dist/procedures/s3/multipart-upload.d.ts.map +1 -0
- package/dist/procedures/s3/multipart-upload.js +44 -0
- package/dist/procedures/s3/multipart-upload.js.map +1 -0
- package/dist/procedures/s3/upload.d.ts +30 -0
- package/dist/procedures/s3/upload.d.ts.map +1 -0
- package/dist/procedures/s3/upload.js +50 -0
- package/dist/procedures/s3/upload.js.map +1 -0
- package/dist/register.d.ts +8 -0
- package/dist/register.d.ts.map +1 -0
- package/dist/register.js +14 -0
- package/dist/register.js.map +1 -0
- package/dist/s3-client.d.ts +38 -0
- package/dist/s3-client.d.ts.map +1 -0
- package/dist/s3-client.js +83 -0
- package/dist/s3-client.js.map +1 -0
- package/dist/types.d.ts +198 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +197 -0
- package/dist/types.js.map +1 -0
- package/package.json +77 -3
- package/src/index.ts +73 -0
- package/src/module.ts +211 -0
- package/src/procedures/s3/delete.ts +45 -0
- package/src/procedures/s3/download.ts +155 -0
- package/src/procedures/s3/index.ts +13 -0
- package/src/procedures/s3/list-all.ts +63 -0
- package/src/procedures/s3/list.ts +63 -0
- package/src/procedures/s3/multipart-abort.ts +40 -0
- package/src/procedures/s3/multipart-complete.ts +58 -0
- package/src/procedures/s3/multipart-init.ts +50 -0
- package/src/procedures/s3/multipart-upload.ts +53 -0
- package/src/procedures/s3/upload.ts +56 -0
- package/src/register.ts +17 -0
- package/src/s3-client.ts +94 -0
- package/src/types.ts +425 -0
package/README.md
CHANGED
|
@@ -1,3 +1,887 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @procedures/s3
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@procedures/s3)
|
|
4
|
+
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://www.typescriptlang.org/)
|
|
7
|
+
[](https://nodejs.org/)
|
|
8
|
+
|
|
9
|
+
AWS S3 operations as RPC procedures. Upload, download, list, delete, and perform multipart uploads via `client.call()`.
|
|
10
|
+
|
|
11
|
+
## Overview
|
|
12
|
+
|
|
13
|
+
`client-s3` provides comprehensive AWS S3 access through the client procedure ecosystem with:
|
|
14
|
+
- **Standard operations**: Upload, download, list, delete objects
|
|
15
|
+
- **Multipart uploads**: Large file uploads with resumable parts
|
|
16
|
+
- **Environment-based config**: AWS credentials from environment variables
|
|
17
|
+
- **Zod validation**: Runtime type safety on all inputs
|
|
18
|
+
- **MinIO/LocalStack support**: Custom S3 endpoints for testing
|
|
19
|
+
|
|
20
|
+
The package auto-registers 9 procedures:
|
|
21
|
+
- **Standard**: `s3.upload`, `s3.download`, `s3.list`, `s3.listAll`, `s3.delete`
|
|
22
|
+
- **Multipart**: `s3.multipart.init`, `s3.multipart.upload`, `s3.multipart.complete`, `s3.multipart.abort`
|
|
23
|
+
|
|
24
|
+
## Architecture
|
|
25
|
+
|
|
26
|
+
```mermaid
|
|
27
|
+
graph TB
|
|
28
|
+
subgraph "Client Ecosystem"
|
|
29
|
+
Client["@procedures/client"]
|
|
30
|
+
Registry["Procedure Registry"]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
subgraph "client-s3 Package"
|
|
34
|
+
Register["register.ts<br/>(Auto-registers procedures)"]
|
|
35
|
+
Types["types.ts<br/>(Zod Schemas + Types)"]
|
|
36
|
+
S3Client["s3-client.ts<br/>(Singleton S3Client)"]
|
|
37
|
+
|
|
38
|
+
subgraph "Procedure Categories"
|
|
39
|
+
StandardOps["Standard Operations<br/>upload, download, list, delete"]
|
|
40
|
+
MultipartOps["Multipart Operations<br/>init, upload, complete, abort"]
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
subgraph "AWS SDK"
|
|
45
|
+
AWSS3["@aws-sdk/client-s3"]
|
|
46
|
+
Commands["PutObjectCommand<br/>GetObjectCommand<br/>ListObjectsV2Command<br/>DeleteObjectCommand<br/>CreateMultipartUploadCommand<br/>UploadPartCommand<br/>CompleteMultipartUploadCommand<br/>AbortMultipartUploadCommand"]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
subgraph "Configuration"
|
|
50
|
+
EnvVars["Environment Variables<br/>AWS_ACCESS_KEY_ID<br/>AWS_SECRET_ACCESS_KEY<br/>AWS_REGION<br/>S3_ENDPOINT"]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
subgraph "S3 Storage"
|
|
54
|
+
S3["AWS S3<br/>or<br/>MinIO/LocalStack"]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
Register --> Client
|
|
58
|
+
Register --> Registry
|
|
59
|
+
|
|
60
|
+
StandardOps --> S3Client
|
|
61
|
+
MultipartOps --> S3Client
|
|
62
|
+
|
|
63
|
+
S3Client --> EnvVars
|
|
64
|
+
S3Client --> AWSS3
|
|
65
|
+
AWSS3 --> Commands
|
|
66
|
+
Commands --> S3
|
|
67
|
+
|
|
68
|
+
Types -.validates.-> StandardOps
|
|
69
|
+
Types -.validates.-> MultipartOps
|
|
70
|
+
|
|
71
|
+
style Register fill:#e1f5ff
|
|
72
|
+
style S3Client fill:#fff4e1
|
|
73
|
+
style Types fill:#f0f0f0
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Package Dependency Flow
|
|
77
|
+
|
|
78
|
+
```mermaid
|
|
79
|
+
graph LR
|
|
80
|
+
App["Your Application"]
|
|
81
|
+
ClientS3["@procedures/s3"]
|
|
82
|
+
Client["@procedures/client"]
|
|
83
|
+
Zod["zod"]
|
|
84
|
+
AWSSDK["@aws-sdk/client-s3"]
|
|
85
|
+
|
|
86
|
+
App --> ClientS3
|
|
87
|
+
ClientS3 --> Client
|
|
88
|
+
ClientS3 --> Zod
|
|
89
|
+
ClientS3 --> AWSSDK
|
|
90
|
+
|
|
91
|
+
App -.client.call.-> Client
|
|
92
|
+
Client -.executes.-> ClientS3
|
|
93
|
+
ClientS3 -.validates.-> Zod
|
|
94
|
+
ClientS3 -.S3 ops.-> AWSSDK
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Installation
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
npm install @procedures/s3
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Peer Dependencies
|
|
104
|
+
|
|
105
|
+
```json
|
|
106
|
+
{
|
|
107
|
+
"@procedures/client": "github:mark1russell7/client#main",
|
|
108
|
+
"@aws-sdk/client-s3": "^3.705.0",
|
|
109
|
+
"zod": "^3.24.0"
|
|
110
|
+
}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Configuration
|
|
114
|
+
|
|
115
|
+
### Environment Variables
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
# Static credentials (optional — omit to use an IAM role / SSO / profile)
|
|
119
|
+
AWS_ACCESS_KEY_ID=your-access-key
|
|
120
|
+
AWS_SECRET_ACCESS_KEY=your-secret-key
|
|
121
|
+
AWS_SESSION_TOKEN=your-session-token # For STS/SSO temporary credentials
|
|
122
|
+
|
|
123
|
+
# Optional
|
|
124
|
+
AWS_REGION=us-east-1 # Default: us-east-1
|
|
125
|
+
S3_ENDPOINT=http://localhost:9000 # For MinIO/LocalStack
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The package supplies explicit static credentials **only when both
|
|
129
|
+
`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are set** (adding
|
|
130
|
+
`AWS_SESSION_TOKEN` when present). If they are not set, it falls back to the
|
|
131
|
+
AWS SDK default credential provider chain — SSO, IAM roles/instance profiles
|
|
132
|
+
(IMDS), and shared config profiles all work with no code changes. Never
|
|
133
|
+
hardcode credentials in your code.
|
|
134
|
+
|
|
135
|
+
### S3 Client Configuration
|
|
136
|
+
|
|
137
|
+
The package uses a singleton S3Client that's automatically configured from environment variables:
|
|
138
|
+
|
|
139
|
+
```typescript
|
|
140
|
+
import { getS3Config, getS3Client, resetS3Client } from "@procedures/s3";
|
|
141
|
+
|
|
142
|
+
// Get current config (from environment)
|
|
143
|
+
const config = getS3Config();
|
|
144
|
+
console.log(config.region);
|
|
145
|
+
|
|
146
|
+
// Get singleton S3 client
|
|
147
|
+
const s3 = getS3Client();
|
|
148
|
+
|
|
149
|
+
// Reset client (useful for testing or credential refresh)
|
|
150
|
+
resetS3Client();
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## API Reference
|
|
154
|
+
|
|
155
|
+
### Standard Operations
|
|
156
|
+
|
|
157
|
+
#### `s3.upload`
|
|
158
|
+
|
|
159
|
+
Upload content to an S3 bucket.
|
|
160
|
+
|
|
161
|
+
```typescript
|
|
162
|
+
const result = await client.exec(
|
|
163
|
+
["s3", "upload"],
|
|
164
|
+
{
|
|
165
|
+
bucket: "my-bucket",
|
|
166
|
+
key: "files/document.pdf",
|
|
167
|
+
body: fileContent, // String or base64 encoded
|
|
168
|
+
contentType: "application/pdf",
|
|
169
|
+
base64: true, // If body is base64 encoded
|
|
170
|
+
metadata: {
|
|
171
|
+
uploadedBy: "user123",
|
|
172
|
+
timestamp: "2024-03-15T10:30:00Z"
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
console.log(result.etag);
|
|
178
|
+
console.log(result.location); // s3://my-bucket/files/document.pdf
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
**Input:**
|
|
182
|
+
```typescript
|
|
183
|
+
interface S3UploadInput {
|
|
184
|
+
bucket: string; // S3 bucket name
|
|
185
|
+
key: string; // Object key (path in bucket)
|
|
186
|
+
body: string; // Content (string or base64)
|
|
187
|
+
contentType?: string; // MIME type
|
|
188
|
+
base64?: boolean; // If true, body is base64 encoded
|
|
189
|
+
metadata?: Record<string, string>; // Custom metadata
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
**Output:**
|
|
194
|
+
```typescript
|
|
195
|
+
interface S3UploadOutput {
|
|
196
|
+
etag: string; // Object ETag
|
|
197
|
+
versionId?: string; // Version ID (if versioning enabled)
|
|
198
|
+
location: string; // Full S3 URI
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
#### `s3.download`
|
|
203
|
+
|
|
204
|
+
Download content from an S3 bucket.
|
|
205
|
+
|
|
206
|
+
```typescript
|
|
207
|
+
const result = await client.exec(
|
|
208
|
+
["s3", "download"],
|
|
209
|
+
{
|
|
210
|
+
bucket: "my-bucket",
|
|
211
|
+
key: "files/document.pdf",
|
|
212
|
+
encoding: "utf8", // Omit for binary (returns base64)
|
|
213
|
+
versionId: "abc123" // Optional specific version
|
|
214
|
+
}
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
console.log(result.body); // Content as string or base64
|
|
218
|
+
console.log(result.contentType);
|
|
219
|
+
console.log(result.contentLength);
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
**Input:**
|
|
223
|
+
```typescript
|
|
224
|
+
interface S3DownloadInput {
|
|
225
|
+
bucket: string; // S3 bucket name
|
|
226
|
+
key: string; // Object key
|
|
227
|
+
encoding?: string; // Text encoding (utf8, ascii, etc.)
|
|
228
|
+
// Omit for binary (returns base64)
|
|
229
|
+
versionId?: string; // Specific version to download
|
|
230
|
+
}
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
**Output:**
|
|
234
|
+
```typescript
|
|
235
|
+
interface S3DownloadOutput {
|
|
236
|
+
body: string; // Content (string if encoding specified, base64 otherwise)
|
|
237
|
+
contentLength: number; // Size in bytes
|
|
238
|
+
contentType?: string; // MIME type
|
|
239
|
+
etag?: string; // Object ETag
|
|
240
|
+
lastModified?: string; // ISO timestamp
|
|
241
|
+
}
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
#### `s3.list`
|
|
245
|
+
|
|
246
|
+
List objects in an S3 bucket.
|
|
247
|
+
|
|
248
|
+
```typescript
|
|
249
|
+
const result = await client.exec(
|
|
250
|
+
["s3", "list"],
|
|
251
|
+
{
|
|
252
|
+
bucket: "my-bucket",
|
|
253
|
+
prefix: "files/",
|
|
254
|
+
maxKeys: 100,
|
|
255
|
+
continuationToken: undefined, // For pagination
|
|
256
|
+
delimiter: "/" // For hierarchical listing
|
|
257
|
+
}
|
|
258
|
+
);
|
|
259
|
+
|
|
260
|
+
console.log(`Found ${result.keyCount} objects`);
|
|
261
|
+
result.contents.forEach(obj => {
|
|
262
|
+
console.log(`${obj.key} - ${obj.size} bytes`);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
if (result.isTruncated) {
|
|
266
|
+
console.log("More results available with token:", result.nextContinuationToken);
|
|
267
|
+
}
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
**Input:**
|
|
271
|
+
```typescript
|
|
272
|
+
interface S3ListInput {
|
|
273
|
+
bucket: string; // S3 bucket name
|
|
274
|
+
prefix?: string; // Filter by prefix
|
|
275
|
+
maxKeys?: number; // Max objects to return (default: 1000)
|
|
276
|
+
continuationToken?: string; // Pagination token
|
|
277
|
+
delimiter?: string; // Delimiter for hierarchical listing (e.g., "/")
|
|
278
|
+
}
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
**Output:**
|
|
282
|
+
```typescript
|
|
283
|
+
interface S3ListOutput {
|
|
284
|
+
contents: S3ListObject[]; // List of objects
|
|
285
|
+
commonPrefixes: string[]; // Common prefixes (when using delimiter)
|
|
286
|
+
isTruncated: boolean; // More results available
|
|
287
|
+
nextContinuationToken?: string; // Token for next page
|
|
288
|
+
keyCount: number; // Number of keys returned
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
interface S3ListObject {
|
|
292
|
+
key: string; // Object key
|
|
293
|
+
size: number; // Size in bytes
|
|
294
|
+
lastModified: string; // ISO timestamp
|
|
295
|
+
etag: string; // Object ETag
|
|
296
|
+
storageClass?: string; // Storage class (STANDARD, GLACIER, etc.)
|
|
297
|
+
}
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
#### `s3.listAll`
|
|
301
|
+
|
|
302
|
+
List **every** object under a bucket/prefix, following `continuationToken`
|
|
303
|
+
automatically. Use this when a bucket/prefix may contain more than 1000 keys —
|
|
304
|
+
`s3.list` returns at most one page, whereas `s3.listAll` loops until the result
|
|
305
|
+
is no longer truncated and returns the aggregated contents.
|
|
306
|
+
|
|
307
|
+
```typescript
|
|
308
|
+
const result = await client.exec(
|
|
309
|
+
["s3", "listAll"],
|
|
310
|
+
{
|
|
311
|
+
bucket: "my-bucket",
|
|
312
|
+
prefix: "snapshots/",
|
|
313
|
+
maxKeys: 1000, // Optional: page size per underlying request
|
|
314
|
+
delimiter: "/" // Optional: hierarchical listing
|
|
315
|
+
}
|
|
316
|
+
);
|
|
317
|
+
|
|
318
|
+
console.log(`Found ${result.keyCount} objects across ${result.pageCount} pages`);
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
**Input:**
|
|
322
|
+
```typescript
|
|
323
|
+
interface S3ListAllInput {
|
|
324
|
+
bucket: string; // S3 bucket name
|
|
325
|
+
prefix?: string; // Filter by prefix
|
|
326
|
+
maxKeys?: number; // Page size per request (default/max 1000)
|
|
327
|
+
delimiter?: string; // Delimiter for hierarchical listing (e.g., "/")
|
|
328
|
+
}
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
**Output:**
|
|
332
|
+
```typescript
|
|
333
|
+
interface S3ListAllOutput {
|
|
334
|
+
contents: S3ListObject[]; // All objects (not capped at 1000)
|
|
335
|
+
commonPrefixes: string[]; // Deduplicated common prefixes across pages
|
|
336
|
+
keyCount: number; // Total number of keys returned
|
|
337
|
+
pageCount: number; // Number of ListObjectsV2 requests made
|
|
338
|
+
}
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
> Warning: `s3.listAll` buffers the full key set in memory and issues one request
|
|
342
|
+
> per 1000 keys. For very large buckets prefer paging with `s3.list` +
|
|
343
|
+
> `continuationToken`.
|
|
344
|
+
|
|
345
|
+
#### `s3.delete`
|
|
346
|
+
|
|
347
|
+
Delete an object from an S3 bucket.
|
|
348
|
+
|
|
349
|
+
```typescript
|
|
350
|
+
const result = await client.exec(
|
|
351
|
+
["s3", "delete"],
|
|
352
|
+
{
|
|
353
|
+
bucket: "my-bucket",
|
|
354
|
+
key: "files/old-document.pdf",
|
|
355
|
+
versionId: "abc123" // Optional specific version
|
|
356
|
+
}
|
|
357
|
+
);
|
|
358
|
+
|
|
359
|
+
console.log(result.deleted); // true
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
**Input:**
|
|
363
|
+
```typescript
|
|
364
|
+
interface S3DeleteInput {
|
|
365
|
+
bucket: string; // S3 bucket name
|
|
366
|
+
key: string; // Object key
|
|
367
|
+
versionId?: string; // Specific version to delete
|
|
368
|
+
}
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
**Output:**
|
|
372
|
+
```typescript
|
|
373
|
+
interface S3DeleteOutput {
|
|
374
|
+
deleted: boolean; // Whether deletion was successful
|
|
375
|
+
deleteMarker?: boolean; // Delete marker created (if versioning enabled)
|
|
376
|
+
versionId?: string; // Version ID of delete marker
|
|
377
|
+
}
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
### Multipart Upload Operations
|
|
381
|
+
|
|
382
|
+
For large files (>5MB), use multipart upload:
|
|
383
|
+
|
|
384
|
+
#### `s3.multipart.init`
|
|
385
|
+
|
|
386
|
+
Initialize a multipart upload.
|
|
387
|
+
|
|
388
|
+
```typescript
|
|
389
|
+
const result = await client.exec(
|
|
390
|
+
["s3", "multipart", "init"],
|
|
391
|
+
{
|
|
392
|
+
bucket: "my-bucket",
|
|
393
|
+
key: "large-files/video.mp4",
|
|
394
|
+
contentType: "video/mp4",
|
|
395
|
+
metadata: {
|
|
396
|
+
originalName: "vacation-video.mp4",
|
|
397
|
+
uploadedBy: "user123"
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
);
|
|
401
|
+
|
|
402
|
+
console.log(result.uploadId); // Save this for subsequent part uploads
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
**Input:**
|
|
406
|
+
```typescript
|
|
407
|
+
interface S3MultipartInitInput {
|
|
408
|
+
bucket: string; // S3 bucket name
|
|
409
|
+
key: string; // Object key
|
|
410
|
+
contentType?: string; // MIME type
|
|
411
|
+
metadata?: Record<string, string>; // Custom metadata
|
|
412
|
+
}
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
**Output:**
|
|
416
|
+
```typescript
|
|
417
|
+
interface S3MultipartInitOutput {
|
|
418
|
+
uploadId: string; // Upload ID for subsequent operations
|
|
419
|
+
bucket: string; // Bucket name
|
|
420
|
+
key: string; // Object key
|
|
421
|
+
}
|
|
422
|
+
```
|
|
423
|
+
|
|
424
|
+
#### `s3.multipart.upload`
|
|
425
|
+
|
|
426
|
+
Upload a part in a multipart upload.
|
|
427
|
+
|
|
428
|
+
```typescript
|
|
429
|
+
const result = await client.exec(
|
|
430
|
+
["s3", "multipart", "upload"],
|
|
431
|
+
{
|
|
432
|
+
bucket: "my-bucket",
|
|
433
|
+
key: "large-files/video.mp4",
|
|
434
|
+
uploadId: uploadId, // From init
|
|
435
|
+
partNumber: 1, // Part number (1-10000)
|
|
436
|
+
body: partData // Base64 encoded part data
|
|
437
|
+
}
|
|
438
|
+
);
|
|
439
|
+
|
|
440
|
+
console.log(result.etag); // Save etag for complete
|
|
441
|
+
console.log(result.partNumber);
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
**Input:**
|
|
445
|
+
```typescript
|
|
446
|
+
interface S3MultipartUploadInput {
|
|
447
|
+
bucket: string; // S3 bucket name
|
|
448
|
+
key: string; // Object key
|
|
449
|
+
uploadId: string; // Upload ID from init
|
|
450
|
+
partNumber: number; // Part number (1-10000)
|
|
451
|
+
body: string; // Part content (base64 encoded)
|
|
452
|
+
}
|
|
453
|
+
```
|
|
454
|
+
|
|
455
|
+
**Output:**
|
|
456
|
+
```typescript
|
|
457
|
+
interface S3MultipartUploadOutput {
|
|
458
|
+
etag: string; // Part ETag (required for complete)
|
|
459
|
+
partNumber: number; // Part number
|
|
460
|
+
}
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
#### `s3.multipart.complete`
|
|
464
|
+
|
|
465
|
+
Complete a multipart upload by assembling parts.
|
|
466
|
+
|
|
467
|
+
```typescript
|
|
468
|
+
const result = await client.exec(
|
|
469
|
+
["s3", "multipart", "complete"],
|
|
470
|
+
{
|
|
471
|
+
bucket: "my-bucket",
|
|
472
|
+
key: "large-files/video.mp4",
|
|
473
|
+
uploadId: uploadId,
|
|
474
|
+
parts: [
|
|
475
|
+
{ partNumber: 1, etag: "etag1" },
|
|
476
|
+
{ partNumber: 2, etag: "etag2" },
|
|
477
|
+
{ partNumber: 3, etag: "etag3" }
|
|
478
|
+
]
|
|
479
|
+
}
|
|
480
|
+
);
|
|
481
|
+
|
|
482
|
+
console.log(result.location); // Final object location
|
|
483
|
+
console.log(result.etag); // Final object ETag
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
**Input:**
|
|
487
|
+
```typescript
|
|
488
|
+
interface S3MultipartCompleteInput {
|
|
489
|
+
bucket: string; // S3 bucket name
|
|
490
|
+
key: string; // Object key
|
|
491
|
+
uploadId: string; // Upload ID
|
|
492
|
+
parts: Array<{ // Uploaded parts (ordered by partNumber)
|
|
493
|
+
etag: string;
|
|
494
|
+
partNumber: number;
|
|
495
|
+
}>;
|
|
496
|
+
}
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
**Output:**
|
|
500
|
+
```typescript
|
|
501
|
+
interface S3MultipartCompleteOutput {
|
|
502
|
+
location: string; // Object location URL
|
|
503
|
+
bucket: string; // Bucket name
|
|
504
|
+
key: string; // Object key
|
|
505
|
+
etag: string; // Final object ETag
|
|
506
|
+
}
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
#### `s3.multipart.abort`
|
|
510
|
+
|
|
511
|
+
Abort a multipart upload and clean up parts.
|
|
512
|
+
|
|
513
|
+
```typescript
|
|
514
|
+
const result = await client.exec(
|
|
515
|
+
["s3", "multipart", "abort"],
|
|
516
|
+
{
|
|
517
|
+
bucket: "my-bucket",
|
|
518
|
+
key: "large-files/video.mp4",
|
|
519
|
+
uploadId: uploadId
|
|
520
|
+
}
|
|
521
|
+
);
|
|
522
|
+
|
|
523
|
+
console.log(result.aborted); // true
|
|
524
|
+
```
|
|
525
|
+
|
|
526
|
+
**Input:**
|
|
527
|
+
```typescript
|
|
528
|
+
interface S3MultipartAbortInput {
|
|
529
|
+
bucket: string; // S3 bucket name
|
|
530
|
+
key: string; // Object key
|
|
531
|
+
uploadId: string; // Upload ID to abort
|
|
532
|
+
}
|
|
533
|
+
```
|
|
534
|
+
|
|
535
|
+
**Output:**
|
|
536
|
+
```typescript
|
|
537
|
+
interface S3MultipartAbortOutput {
|
|
538
|
+
aborted: boolean; // Whether abort was successful
|
|
539
|
+
}
|
|
540
|
+
```
|
|
541
|
+
|
|
542
|
+
## Usage Examples
|
|
543
|
+
|
|
544
|
+
### Basic Upload and Download
|
|
545
|
+
|
|
546
|
+
```typescript
|
|
547
|
+
import { Client, LocalTransport, PROCEDURE_REGISTRY } from "@procedures/client";
|
|
548
|
+
import "@procedures/s3"; // registers its procedures
|
|
549
|
+
|
|
550
|
+
const client = new Client(new LocalTransport({ registry: PROCEDURE_REGISTRY }));
|
|
551
|
+
|
|
552
|
+
// Upload text content
|
|
553
|
+
await client.exec(
|
|
554
|
+
["s3", "upload"],
|
|
555
|
+
{
|
|
556
|
+
bucket: "my-bucket",
|
|
557
|
+
key: "data/config.json",
|
|
558
|
+
body: JSON.stringify({ env: "production" }),
|
|
559
|
+
contentType: "application/json"
|
|
560
|
+
}
|
|
561
|
+
);
|
|
562
|
+
|
|
563
|
+
// Upload binary file (base64 encoded)
|
|
564
|
+
const fileBuffer = await fs.promises.readFile("./image.png");
|
|
565
|
+
const base64Content = fileBuffer.toString("base64");
|
|
566
|
+
|
|
567
|
+
await client.exec(
|
|
568
|
+
["s3", "upload"],
|
|
569
|
+
{
|
|
570
|
+
bucket: "my-bucket",
|
|
571
|
+
key: "images/photo.png",
|
|
572
|
+
body: base64Content,
|
|
573
|
+
base64: true,
|
|
574
|
+
contentType: "image/png",
|
|
575
|
+
metadata: {
|
|
576
|
+
uploadedBy: "user123",
|
|
577
|
+
originalName: "vacation-photo.png"
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
);
|
|
581
|
+
|
|
582
|
+
// Download text content
|
|
583
|
+
const { body } = await client.exec(
|
|
584
|
+
["s3", "download"],
|
|
585
|
+
{
|
|
586
|
+
bucket: "my-bucket",
|
|
587
|
+
key: "data/config.json",
|
|
588
|
+
encoding: "utf8"
|
|
589
|
+
}
|
|
590
|
+
);
|
|
591
|
+
|
|
592
|
+
const config = JSON.parse(body);
|
|
593
|
+
|
|
594
|
+
// Download binary content (returns base64)
|
|
595
|
+
const { body: imageBase64 } = await client.exec(
|
|
596
|
+
["s3", "download"],
|
|
597
|
+
{
|
|
598
|
+
bucket: "my-bucket",
|
|
599
|
+
key: "images/photo.png"
|
|
600
|
+
}
|
|
601
|
+
);
|
|
602
|
+
|
|
603
|
+
const imageBuffer = Buffer.from(imageBase64, "base64");
|
|
604
|
+
```
|
|
605
|
+
|
|
606
|
+
### List and Delete Objects
|
|
607
|
+
|
|
608
|
+
```typescript
|
|
609
|
+
// List all objects in a prefix
|
|
610
|
+
const { contents } = await client.exec(
|
|
611
|
+
["s3", "list"],
|
|
612
|
+
{
|
|
613
|
+
bucket: "my-bucket",
|
|
614
|
+
prefix: "data/",
|
|
615
|
+
maxKeys: 1000
|
|
616
|
+
}
|
|
617
|
+
);
|
|
618
|
+
|
|
619
|
+
console.log(`Found ${contents.length} objects`);
|
|
620
|
+
|
|
621
|
+
// List with pagination
|
|
622
|
+
let continuationToken: string | undefined;
|
|
623
|
+
do {
|
|
624
|
+
const result = await client.exec(
|
|
625
|
+
["s3", "list"],
|
|
626
|
+
{
|
|
627
|
+
bucket: "my-bucket",
|
|
628
|
+
prefix: "logs/",
|
|
629
|
+
maxKeys: 100,
|
|
630
|
+
continuationToken
|
|
631
|
+
}
|
|
632
|
+
);
|
|
633
|
+
|
|
634
|
+
result.contents.forEach(obj => {
|
|
635
|
+
console.log(`${obj.key} - ${obj.size} bytes`);
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
continuationToken = result.nextContinuationToken;
|
|
639
|
+
} while (continuationToken);
|
|
640
|
+
|
|
641
|
+
// Delete objects
|
|
642
|
+
for (const obj of contents) {
|
|
643
|
+
await client.exec(
|
|
644
|
+
["s3", "delete"],
|
|
645
|
+
{
|
|
646
|
+
bucket: "my-bucket",
|
|
647
|
+
key: obj.key
|
|
648
|
+
}
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
```
|
|
652
|
+
|
|
653
|
+
### Hierarchical Listing (Folders)
|
|
654
|
+
|
|
655
|
+
```typescript
|
|
656
|
+
// List "folders" in a bucket
|
|
657
|
+
const { contents, commonPrefixes } = await client.exec(
|
|
658
|
+
["s3", "list"],
|
|
659
|
+
{
|
|
660
|
+
bucket: "my-bucket",
|
|
661
|
+
prefix: "data/",
|
|
662
|
+
delimiter: "/"
|
|
663
|
+
}
|
|
664
|
+
);
|
|
665
|
+
|
|
666
|
+
console.log("Files in data/:");
|
|
667
|
+
contents.forEach(obj => {
|
|
668
|
+
console.log(` ${obj.key}`);
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
console.log("Subdirectories:");
|
|
672
|
+
commonPrefixes.forEach(prefix => {
|
|
673
|
+
console.log(` ${prefix}`);
|
|
674
|
+
});
|
|
675
|
+
```
|
|
676
|
+
|
|
677
|
+
### Multipart Upload (Large Files)
|
|
678
|
+
|
|
679
|
+
```typescript
|
|
680
|
+
const PART_SIZE = 5 * 1024 * 1024; // 5MB parts
|
|
681
|
+
|
|
682
|
+
// 1. Initialize upload
|
|
683
|
+
const { uploadId } = await client.exec(
|
|
684
|
+
["s3", "multipart", "init"],
|
|
685
|
+
{
|
|
686
|
+
bucket: "my-bucket",
|
|
687
|
+
key: "videos/large-file.mp4",
|
|
688
|
+
contentType: "video/mp4"
|
|
689
|
+
}
|
|
690
|
+
);
|
|
691
|
+
|
|
692
|
+
try {
|
|
693
|
+
// 2. Upload parts
|
|
694
|
+
const file = await fs.promises.readFile("./large-file.mp4");
|
|
695
|
+
const parts = [];
|
|
696
|
+
|
|
697
|
+
for (let i = 0; i < file.length; i += PART_SIZE) {
|
|
698
|
+
const partNumber = Math.floor(i / PART_SIZE) + 1;
|
|
699
|
+
const partData = file.slice(i, Math.min(i + PART_SIZE, file.length));
|
|
700
|
+
const base64Part = partData.toString("base64");
|
|
701
|
+
|
|
702
|
+
const { etag } = await client.exec(
|
|
703
|
+
["s3", "multipart", "upload"],
|
|
704
|
+
{
|
|
705
|
+
bucket: "my-bucket",
|
|
706
|
+
key: "videos/large-file.mp4",
|
|
707
|
+
uploadId,
|
|
708
|
+
partNumber,
|
|
709
|
+
body: base64Part
|
|
710
|
+
}
|
|
711
|
+
);
|
|
712
|
+
|
|
713
|
+
parts.push({ partNumber, etag });
|
|
714
|
+
console.log(`Uploaded part ${partNumber}`);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// 3. Complete upload
|
|
718
|
+
const result = await client.exec(
|
|
719
|
+
["s3", "multipart", "complete"],
|
|
720
|
+
{
|
|
721
|
+
bucket: "my-bucket",
|
|
722
|
+
key: "videos/large-file.mp4",
|
|
723
|
+
uploadId,
|
|
724
|
+
parts
|
|
725
|
+
}
|
|
726
|
+
);
|
|
727
|
+
|
|
728
|
+
console.log(`Upload complete: ${result.location}`);
|
|
729
|
+
|
|
730
|
+
} catch (error) {
|
|
731
|
+
// 4. Abort on error
|
|
732
|
+
await client.exec(
|
|
733
|
+
["s3", "multipart", "abort"],
|
|
734
|
+
{
|
|
735
|
+
bucket: "my-bucket",
|
|
736
|
+
key: "videos/large-file.mp4",
|
|
737
|
+
uploadId
|
|
738
|
+
}
|
|
739
|
+
);
|
|
740
|
+
throw error;
|
|
741
|
+
}
|
|
742
|
+
```
|
|
743
|
+
|
|
744
|
+
### Backup Script Example
|
|
745
|
+
|
|
746
|
+
```typescript
|
|
747
|
+
import { glob } from "fast-glob";
|
|
748
|
+
|
|
749
|
+
// Backup directory to S3
|
|
750
|
+
const files = await glob("./data/**/*", { onlyFiles: true });
|
|
751
|
+
|
|
752
|
+
for (const file of files) {
|
|
753
|
+
const content = await fs.promises.readFile(file);
|
|
754
|
+
const s3Key = file.replace("./data/", "backups/");
|
|
755
|
+
|
|
756
|
+
await client.exec(
|
|
757
|
+
["s3", "upload"],
|
|
758
|
+
{
|
|
759
|
+
bucket: "my-backup-bucket",
|
|
760
|
+
key: s3Key,
|
|
761
|
+
body: content.toString("base64"),
|
|
762
|
+
base64: true,
|
|
763
|
+
metadata: {
|
|
764
|
+
backupDate: new Date().toISOString(),
|
|
765
|
+
originalPath: file
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
);
|
|
769
|
+
|
|
770
|
+
console.log(`Backed up: ${file} -> s3://my-backup-bucket/${s3Key}`);
|
|
771
|
+
}
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
### Using with MinIO or LocalStack
|
|
775
|
+
|
|
776
|
+
```bash
|
|
777
|
+
# Set custom endpoint for local S3-compatible storage
|
|
778
|
+
export S3_ENDPOINT=http://localhost:9000
|
|
779
|
+
export AWS_ACCESS_KEY_ID=minioadmin
|
|
780
|
+
export AWS_SECRET_ACCESS_KEY=minioadmin
|
|
781
|
+
export AWS_REGION=us-east-1
|
|
782
|
+
```
|
|
783
|
+
|
|
784
|
+
```typescript
|
|
785
|
+
// Works the same way with MinIO/LocalStack
|
|
786
|
+
await client.exec(
|
|
787
|
+
["s3", "upload"],
|
|
788
|
+
{
|
|
789
|
+
bucket: "test-bucket",
|
|
790
|
+
key: "test.txt",
|
|
791
|
+
body: "Hello MinIO!"
|
|
792
|
+
}
|
|
793
|
+
);
|
|
794
|
+
```
|
|
795
|
+
|
|
796
|
+
## Integration with Ecosystem
|
|
797
|
+
|
|
798
|
+
### Bundle Packages
|
|
799
|
+
|
|
800
|
+
`client-s3` can be included in bundle packages:
|
|
801
|
+
|
|
802
|
+
```typescript
|
|
803
|
+
// In bundle-data/src/register.ts
|
|
804
|
+
import "@procedures/s3/register.js";
|
|
805
|
+
import "@procedures/mongo"; // registers its procedures
|
|
806
|
+
import "@procedures/fs"; // registers its procedures
|
|
807
|
+
```
|
|
808
|
+
|
|
809
|
+
## Dependencies
|
|
810
|
+
|
|
811
|
+
### Runtime Dependencies
|
|
812
|
+
|
|
813
|
+
- **@procedures/client** - Core client procedure system
|
|
814
|
+
- **@aws-sdk/client-s3** (^3.705.0) - AWS SDK for S3
|
|
815
|
+
- **zod** (^3.24.0) - Runtime type validation
|
|
816
|
+
|
|
817
|
+
### Development Dependencies
|
|
818
|
+
|
|
819
|
+
- **@mark1russell7/cue** - Configuration and type utilities
|
|
820
|
+
- **typescript** (^5.9.3) - TypeScript compiler
|
|
821
|
+
- **@types/node** (^22.0.0) - Node.js type definitions
|
|
822
|
+
|
|
823
|
+
## Package Structure
|
|
824
|
+
|
|
825
|
+
```
|
|
826
|
+
client-s3/
|
|
827
|
+
├── src/
|
|
828
|
+
│ ├── types.ts # Zod schemas + TypeScript types
|
|
829
|
+
│ ├── s3-client.ts # S3Client singleton factory
|
|
830
|
+
│ ├── register.ts # Auto-registration
|
|
831
|
+
│ ├── index.ts # Main entry point
|
|
832
|
+
│ ├── procedures/
|
|
833
|
+
│ │ └── s3/
|
|
834
|
+
│ │ ├── index.ts # Procedure exports
|
|
835
|
+
│ │ ├── upload.ts # s3.upload
|
|
836
|
+
│ │ ├── download.ts # s3.download
|
|
837
|
+
│ │ ├── list.ts # s3.list
|
|
838
|
+
│ │ ├── delete.ts # s3.delete
|
|
839
|
+
│ │ ├── multipart-init.ts # s3.multipart.init
|
|
840
|
+
│ │ ├── multipart-upload.ts # s3.multipart.upload
|
|
841
|
+
│ │ ├── multipart-complete.ts # s3.multipart.complete
|
|
842
|
+
│ │ └── multipart-abort.ts # s3.multipart.abort
|
|
843
|
+
├── dist/ # Compiled output
|
|
844
|
+
├── package.json
|
|
845
|
+
├── tsconfig.json
|
|
846
|
+
└── README.md
|
|
847
|
+
```
|
|
848
|
+
|
|
849
|
+
## Development
|
|
850
|
+
|
|
851
|
+
### Build
|
|
852
|
+
|
|
853
|
+
```bash
|
|
854
|
+
npm run build
|
|
855
|
+
```
|
|
856
|
+
|
|
857
|
+
### Type Check
|
|
858
|
+
|
|
859
|
+
```bash
|
|
860
|
+
npm run typecheck
|
|
861
|
+
```
|
|
862
|
+
|
|
863
|
+
### Clean
|
|
864
|
+
|
|
865
|
+
```bash
|
|
866
|
+
npm run clean
|
|
867
|
+
```
|
|
868
|
+
|
|
869
|
+
## Security
|
|
870
|
+
|
|
871
|
+
- Never hardcode AWS credentials in code
|
|
872
|
+
- Prefer IAM roles / instance profiles when running on AWS infrastructure: leave
|
|
873
|
+
`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` unset and the SDK default credential
|
|
874
|
+
provider chain resolves role credentials automatically
|
|
875
|
+
- Use static `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` (plus `AWS_SESSION_TOKEN`
|
|
876
|
+
for temporary SSO/STS credentials) only where an IAM role is not available
|
|
877
|
+
- Restrict S3 bucket policies to minimum required permissions
|
|
878
|
+
- Enable S3 bucket versioning for important data
|
|
879
|
+
- Consider S3 bucket encryption at rest
|
|
880
|
+
|
|
881
|
+
## License
|
|
882
|
+
|
|
883
|
+
MIT
|
|
884
|
+
|
|
885
|
+
## Author
|
|
886
|
+
|
|
887
|
+
Mark Russell <marktheprogrammer17@gmail.com>
|