@directus/storage-driver-s3 12.0.7 → 12.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +55 -53
- package/dist/index.js +314 -385
- package/package.json +12 -11
package/dist/index.d.ts
CHANGED
|
@@ -1,58 +1,60 @@
|
|
|
1
|
-
import { ObjectCannedACL, ServerSideEncryption } from
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { ObjectCannedACL, ServerSideEncryption } from "@aws-sdk/client-s3";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
3
|
+
import { TusDriver } from "@directus/storage";
|
|
4
|
+
import { ChunkedUploadContext, ReadOptions } from "@directus/types";
|
|
4
5
|
|
|
6
|
+
//#region src/index.d.ts
|
|
5
7
|
type DriverS3Config = {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
8
|
+
root?: string;
|
|
9
|
+
key?: string;
|
|
10
|
+
secret?: string;
|
|
11
|
+
bucket: string;
|
|
12
|
+
acl?: ObjectCannedACL;
|
|
13
|
+
serverSideEncryption?: ServerSideEncryption;
|
|
14
|
+
endpoint?: string;
|
|
15
|
+
region?: string;
|
|
16
|
+
forcePathStyle?: boolean;
|
|
17
|
+
tus?: {
|
|
18
|
+
chunkSize?: number;
|
|
19
|
+
};
|
|
20
|
+
connectionTimeout?: number;
|
|
21
|
+
socketTimeout?: number;
|
|
22
|
+
maxSockets?: number;
|
|
23
|
+
keepAlive?: boolean;
|
|
22
24
|
};
|
|
23
25
|
declare class DriverS3 implements TusDriver {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
26
|
+
private config;
|
|
27
|
+
private readonly client;
|
|
28
|
+
private readonly root;
|
|
29
|
+
private partUploadSemaphore;
|
|
30
|
+
private readonly preferredPartSize;
|
|
31
|
+
maxMultipartParts: 10000;
|
|
32
|
+
minPartSize: 5242880;
|
|
33
|
+
maxUploadSize: 5497558138880;
|
|
34
|
+
constructor(config: DriverS3Config);
|
|
35
|
+
private getClient;
|
|
36
|
+
private fullPath;
|
|
37
|
+
read(filepath: string, options?: ReadOptions): Promise<Readable>;
|
|
38
|
+
stat(filepath: string): Promise<{
|
|
39
|
+
size: number;
|
|
40
|
+
modified: Date;
|
|
41
|
+
}>;
|
|
42
|
+
exists(filepath: string): Promise<boolean>;
|
|
43
|
+
move(src: string, dest: string): Promise<void>;
|
|
44
|
+
copy(src: string, dest: string): Promise<void>;
|
|
45
|
+
write(filepath: string, content: Readable, type?: string): Promise<void>;
|
|
46
|
+
delete(filepath: string): Promise<void>;
|
|
47
|
+
list(prefix?: string): AsyncGenerator<string, void, unknown>;
|
|
48
|
+
get tusExtensions(): string[];
|
|
49
|
+
createChunkedUpload(filepath: string, context: ChunkedUploadContext): Promise<ChunkedUploadContext>;
|
|
50
|
+
deleteChunkedUpload(filepath: string, context: ChunkedUploadContext): Promise<void>;
|
|
51
|
+
finishChunkedUpload(filepath: string, context: ChunkedUploadContext): Promise<void>;
|
|
52
|
+
writeChunk(filepath: string, content: Readable, offset: number, context: ChunkedUploadContext): Promise<number>;
|
|
53
|
+
private uploadPart;
|
|
54
|
+
private uploadParts;
|
|
55
|
+
private retrieveParts;
|
|
56
|
+
private finishMultipartUpload;
|
|
57
|
+
private calcOptimalPartSize;
|
|
56
58
|
}
|
|
57
|
-
|
|
58
|
-
export { DriverS3,
|
|
59
|
+
//#endregion
|
|
60
|
+
export { DriverS3, DriverS3 as default, DriverS3Config };
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
3
|
-
AbortMultipartUploadCommand,
|
|
4
|
-
CompleteMultipartUploadCommand,
|
|
5
|
-
CopyObjectCommand,
|
|
6
|
-
CreateMultipartUploadCommand,
|
|
7
|
-
DeleteObjectCommand,
|
|
8
|
-
DeleteObjectsCommand,
|
|
9
|
-
GetObjectCommand,
|
|
10
|
-
HeadObjectCommand,
|
|
11
|
-
ListObjectsV2Command,
|
|
12
|
-
ListPartsCommand,
|
|
13
|
-
S3Client,
|
|
14
|
-
UploadPartCommand
|
|
15
|
-
} from "@aws-sdk/client-s3";
|
|
1
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListPartsCommand, S3Client, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
16
2
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
17
3
|
import { normalizePath } from "@directus/utils";
|
|
18
4
|
import { isReadableStream } from "@directus/utils/node";
|
|
@@ -20,375 +6,318 @@ import { Semaphore } from "@shopify/semaphore";
|
|
|
20
6
|
import { NodeHttpHandler } from "@smithy/node-http-handler";
|
|
21
7
|
import { ERRORS, StreamSplitter, TUS_RESUMABLE } from "@tus/utils";
|
|
22
8
|
import ms from "ms";
|
|
23
|
-
import fs, { promises
|
|
24
|
-
import { Agent
|
|
25
|
-
import { Agent as
|
|
26
|
-
import os from "os";
|
|
27
|
-
import { join } from "path";
|
|
28
|
-
import { promises as
|
|
9
|
+
import fs, { promises } from "node:fs";
|
|
10
|
+
import { Agent } from "node:http";
|
|
11
|
+
import { Agent as Agent$1 } from "node:https";
|
|
12
|
+
import os from "node:os";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { promises as promises$1 } from "node:stream";
|
|
15
|
+
|
|
16
|
+
//#region src/index.ts
|
|
29
17
|
var DriverS3 = class {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
promises.push(Promise.reject(error));
|
|
334
|
-
} finally {
|
|
335
|
-
await Promise.all(promises);
|
|
336
|
-
}
|
|
337
|
-
return bytesUploaded;
|
|
338
|
-
}
|
|
339
|
-
async retrieveParts(key, uploadId, partNumberMarker) {
|
|
340
|
-
const data = await this.client.send(
|
|
341
|
-
new ListPartsCommand({
|
|
342
|
-
Bucket: this.config.bucket,
|
|
343
|
-
Key: key,
|
|
344
|
-
UploadId: uploadId,
|
|
345
|
-
PartNumberMarker: partNumberMarker
|
|
346
|
-
})
|
|
347
|
-
);
|
|
348
|
-
let parts = data.Parts ?? [];
|
|
349
|
-
if (data.IsTruncated) {
|
|
350
|
-
const rest = await this.retrieveParts(key, uploadId, data.NextPartNumberMarker);
|
|
351
|
-
parts = [...parts, ...rest];
|
|
352
|
-
}
|
|
353
|
-
if (!partNumberMarker) {
|
|
354
|
-
parts.sort((a, b) => a.PartNumber - b.PartNumber);
|
|
355
|
-
}
|
|
356
|
-
return parts;
|
|
357
|
-
}
|
|
358
|
-
async finishMultipartUpload(key, uploadId, parts) {
|
|
359
|
-
const command = new CompleteMultipartUploadCommand({
|
|
360
|
-
Bucket: this.config.bucket,
|
|
361
|
-
Key: key,
|
|
362
|
-
UploadId: uploadId,
|
|
363
|
-
MultipartUpload: {
|
|
364
|
-
Parts: parts.map((part) => {
|
|
365
|
-
return {
|
|
366
|
-
ETag: part.ETag,
|
|
367
|
-
PartNumber: part.PartNumber
|
|
368
|
-
};
|
|
369
|
-
})
|
|
370
|
-
}
|
|
371
|
-
});
|
|
372
|
-
const response = await this.client.send(command);
|
|
373
|
-
return response.Location;
|
|
374
|
-
}
|
|
375
|
-
calcOptimalPartSize(size) {
|
|
376
|
-
if (size === void 0) {
|
|
377
|
-
size = this.maxUploadSize;
|
|
378
|
-
}
|
|
379
|
-
let optimalPartSize;
|
|
380
|
-
if (size <= this.preferredPartSize) {
|
|
381
|
-
optimalPartSize = size;
|
|
382
|
-
} else if (size <= this.preferredPartSize * this.maxMultipartParts) {
|
|
383
|
-
optimalPartSize = this.preferredPartSize;
|
|
384
|
-
} else {
|
|
385
|
-
optimalPartSize = Math.ceil(size / this.maxMultipartParts);
|
|
386
|
-
}
|
|
387
|
-
return optimalPartSize;
|
|
388
|
-
}
|
|
389
|
-
};
|
|
390
|
-
var index_default = DriverS3;
|
|
391
|
-
export {
|
|
392
|
-
DriverS3,
|
|
393
|
-
index_default as default
|
|
18
|
+
config;
|
|
19
|
+
client;
|
|
20
|
+
root;
|
|
21
|
+
partUploadSemaphore;
|
|
22
|
+
preferredPartSize;
|
|
23
|
+
maxMultipartParts = 1e4;
|
|
24
|
+
minPartSize = 5242880;
|
|
25
|
+
maxUploadSize = 5497558138880;
|
|
26
|
+
constructor(config) {
|
|
27
|
+
this.config = config;
|
|
28
|
+
this.client = this.getClient();
|
|
29
|
+
this.root = this.config.root ? normalizePath(this.config.root, { removeLeading: true }) : "";
|
|
30
|
+
this.preferredPartSize = config.tus?.chunkSize ?? this.minPartSize;
|
|
31
|
+
this.partUploadSemaphore = new Semaphore(60);
|
|
32
|
+
}
|
|
33
|
+
getClient() {
|
|
34
|
+
const connectionTimeout = ms(String(this.config.connectionTimeout ?? 5e3));
|
|
35
|
+
const socketTimeout = ms(String(this.config.socketTimeout ?? 12e4));
|
|
36
|
+
const maxSockets = this.config.maxSockets ?? 500;
|
|
37
|
+
const keepAlive = this.config.keepAlive ?? true;
|
|
38
|
+
const s3ClientConfig = { requestHandler: new NodeHttpHandler({
|
|
39
|
+
connectionTimeout,
|
|
40
|
+
socketTimeout,
|
|
41
|
+
httpAgent: new Agent({
|
|
42
|
+
maxSockets,
|
|
43
|
+
keepAlive
|
|
44
|
+
}),
|
|
45
|
+
httpsAgent: new Agent$1({
|
|
46
|
+
maxSockets,
|
|
47
|
+
keepAlive
|
|
48
|
+
})
|
|
49
|
+
}) };
|
|
50
|
+
if (this.config.key && !this.config.secret || this.config.secret && !this.config.key) throw new Error("Both `key` and `secret` are required when defined");
|
|
51
|
+
if (this.config.key && this.config.secret) s3ClientConfig.credentials = {
|
|
52
|
+
accessKeyId: this.config.key,
|
|
53
|
+
secretAccessKey: this.config.secret
|
|
54
|
+
};
|
|
55
|
+
if (this.config.endpoint) {
|
|
56
|
+
const protocol = this.config.endpoint.startsWith("http://") ? "http:" : "https:";
|
|
57
|
+
s3ClientConfig.endpoint = {
|
|
58
|
+
hostname: this.config.endpoint.replace("https://", "").replace("http://", ""),
|
|
59
|
+
protocol,
|
|
60
|
+
path: "/"
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (this.config.region) s3ClientConfig.region = this.config.region;
|
|
64
|
+
if (this.config.forcePathStyle !== void 0) s3ClientConfig.forcePathStyle = this.config.forcePathStyle;
|
|
65
|
+
return new S3Client(s3ClientConfig);
|
|
66
|
+
}
|
|
67
|
+
fullPath(filepath) {
|
|
68
|
+
return normalizePath(join(this.root, filepath));
|
|
69
|
+
}
|
|
70
|
+
async read(filepath, options) {
|
|
71
|
+
const { range } = options ?? {};
|
|
72
|
+
const commandInput = {
|
|
73
|
+
Key: this.fullPath(filepath),
|
|
74
|
+
Bucket: this.config.bucket
|
|
75
|
+
};
|
|
76
|
+
if (range) commandInput.Range = `bytes=${range.start ?? ""}-${range.end ?? ""}`;
|
|
77
|
+
const { Body: stream$1 } = await this.client.send(new GetObjectCommand(commandInput));
|
|
78
|
+
if (!stream$1 || !isReadableStream(stream$1)) throw new Error(`No stream returned for file "${filepath}"`);
|
|
79
|
+
return stream$1;
|
|
80
|
+
}
|
|
81
|
+
async stat(filepath) {
|
|
82
|
+
const { ContentLength, LastModified } = await this.client.send(new HeadObjectCommand({
|
|
83
|
+
Key: this.fullPath(filepath),
|
|
84
|
+
Bucket: this.config.bucket
|
|
85
|
+
}));
|
|
86
|
+
return {
|
|
87
|
+
size: ContentLength,
|
|
88
|
+
modified: LastModified
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
async exists(filepath) {
|
|
92
|
+
try {
|
|
93
|
+
await this.stat(filepath);
|
|
94
|
+
return true;
|
|
95
|
+
} catch {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async move(src, dest) {
|
|
100
|
+
await this.copy(src, dest);
|
|
101
|
+
await this.delete(src);
|
|
102
|
+
}
|
|
103
|
+
async copy(src, dest) {
|
|
104
|
+
const params = {
|
|
105
|
+
Key: this.fullPath(dest),
|
|
106
|
+
Bucket: this.config.bucket,
|
|
107
|
+
CopySource: `/${this.config.bucket}/${this.fullPath(src)}`
|
|
108
|
+
};
|
|
109
|
+
if (this.config.serverSideEncryption) params.ServerSideEncryption = this.config.serverSideEncryption;
|
|
110
|
+
if (this.config.acl) params.ACL = this.config.acl;
|
|
111
|
+
await this.client.send(new CopyObjectCommand(params));
|
|
112
|
+
}
|
|
113
|
+
async write(filepath, content, type) {
|
|
114
|
+
const params = {
|
|
115
|
+
Key: this.fullPath(filepath),
|
|
116
|
+
Body: content,
|
|
117
|
+
Bucket: this.config.bucket
|
|
118
|
+
};
|
|
119
|
+
if (type) params.ContentType = type;
|
|
120
|
+
if (this.config.acl) params.ACL = this.config.acl;
|
|
121
|
+
if (this.config.serverSideEncryption) params.ServerSideEncryption = this.config.serverSideEncryption;
|
|
122
|
+
await new Upload({
|
|
123
|
+
client: this.client,
|
|
124
|
+
params
|
|
125
|
+
}).done();
|
|
126
|
+
}
|
|
127
|
+
async delete(filepath) {
|
|
128
|
+
await this.client.send(new DeleteObjectCommand({
|
|
129
|
+
Key: this.fullPath(filepath),
|
|
130
|
+
Bucket: this.config.bucket
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
async *list(prefix = "") {
|
|
134
|
+
let Prefix = this.fullPath(prefix);
|
|
135
|
+
if (Prefix === ".") Prefix = "";
|
|
136
|
+
let continuationToken = void 0;
|
|
137
|
+
do {
|
|
138
|
+
const listObjectsV2CommandInput = {
|
|
139
|
+
Bucket: this.config.bucket,
|
|
140
|
+
Prefix,
|
|
141
|
+
MaxKeys: 1e3
|
|
142
|
+
};
|
|
143
|
+
if (continuationToken) listObjectsV2CommandInput.ContinuationToken = continuationToken;
|
|
144
|
+
const response = await this.client.send(new ListObjectsV2Command(listObjectsV2CommandInput));
|
|
145
|
+
continuationToken = response.NextContinuationToken;
|
|
146
|
+
if (response.Contents) for (const object of response.Contents) {
|
|
147
|
+
if (!object.Key) continue;
|
|
148
|
+
if (object.Key.endsWith("/")) continue;
|
|
149
|
+
yield object.Key.substring(this.root.length);
|
|
150
|
+
}
|
|
151
|
+
} while (continuationToken);
|
|
152
|
+
}
|
|
153
|
+
get tusExtensions() {
|
|
154
|
+
return [
|
|
155
|
+
"creation",
|
|
156
|
+
"termination",
|
|
157
|
+
"expiration"
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
async createChunkedUpload(filepath, context) {
|
|
161
|
+
const command = new CreateMultipartUploadCommand({
|
|
162
|
+
Bucket: this.config.bucket,
|
|
163
|
+
Key: this.fullPath(filepath),
|
|
164
|
+
Metadata: { "tus-version": TUS_RESUMABLE },
|
|
165
|
+
...context.metadata?.["contentType"] ? { ContentType: context.metadata["contentType"] } : {},
|
|
166
|
+
...context.metadata?.["cacheControl"] ? { CacheControl: context.metadata["cacheControl"] } : {}
|
|
167
|
+
});
|
|
168
|
+
const res = await this.client.send(command);
|
|
169
|
+
context.metadata["upload-id"] = res.UploadId;
|
|
170
|
+
return context;
|
|
171
|
+
}
|
|
172
|
+
async deleteChunkedUpload(filepath, context) {
|
|
173
|
+
const key = this.fullPath(filepath);
|
|
174
|
+
try {
|
|
175
|
+
const { "upload-id": uploadId } = context.metadata;
|
|
176
|
+
if (uploadId) await this.client.send(new AbortMultipartUploadCommand({
|
|
177
|
+
Bucket: this.config.bucket,
|
|
178
|
+
Key: key,
|
|
179
|
+
UploadId: uploadId
|
|
180
|
+
}));
|
|
181
|
+
} catch (error) {
|
|
182
|
+
if (error?.code && [
|
|
183
|
+
"NotFound",
|
|
184
|
+
"NoSuchKey",
|
|
185
|
+
"NoSuchUpload"
|
|
186
|
+
].includes(error.Code)) throw ERRORS.FILE_NOT_FOUND;
|
|
187
|
+
throw error;
|
|
188
|
+
}
|
|
189
|
+
await this.client.send(new DeleteObjectsCommand({
|
|
190
|
+
Bucket: this.config.bucket,
|
|
191
|
+
Delete: { Objects: [{ Key: key }] }
|
|
192
|
+
}));
|
|
193
|
+
}
|
|
194
|
+
async finishChunkedUpload(filepath, context) {
|
|
195
|
+
const key = this.fullPath(filepath);
|
|
196
|
+
const uploadId = context.metadata["upload-id"];
|
|
197
|
+
const size = context.size;
|
|
198
|
+
const chunkSize = this.calcOptimalPartSize(size);
|
|
199
|
+
const expectedParts = Math.ceil(size / chunkSize);
|
|
200
|
+
let parts = await this.retrieveParts(key, uploadId);
|
|
201
|
+
let retries = 0;
|
|
202
|
+
while (parts.length !== expectedParts && retries < 3) {
|
|
203
|
+
++retries;
|
|
204
|
+
await new Promise((resolve) => setTimeout(resolve, 500 * retries));
|
|
205
|
+
parts = await this.retrieveParts(key, uploadId);
|
|
206
|
+
}
|
|
207
|
+
if (parts.length !== expectedParts) throw {
|
|
208
|
+
status_code: 500,
|
|
209
|
+
body: "Failed to upload all parts to S3."
|
|
210
|
+
};
|
|
211
|
+
await this.finishMultipartUpload(key, uploadId, parts);
|
|
212
|
+
}
|
|
213
|
+
async writeChunk(filepath, content, offset, context) {
|
|
214
|
+
const key = this.fullPath(filepath);
|
|
215
|
+
const uploadId = context.metadata["upload-id"];
|
|
216
|
+
const size = context.size;
|
|
217
|
+
const parts = await this.retrieveParts(key, uploadId);
|
|
218
|
+
const nextPartNumber = (parts.length > 0 ? parts[parts.length - 1].PartNumber : 0) + 1;
|
|
219
|
+
const requestedOffset = offset;
|
|
220
|
+
const bytesUploaded = await this.uploadParts(key, uploadId, size, content, nextPartNumber, offset);
|
|
221
|
+
return requestedOffset + bytesUploaded;
|
|
222
|
+
}
|
|
223
|
+
async uploadPart(key, uploadId, readStream, partNumber) {
|
|
224
|
+
return (await this.client.send(new UploadPartCommand({
|
|
225
|
+
Bucket: this.config.bucket,
|
|
226
|
+
Key: key,
|
|
227
|
+
UploadId: uploadId,
|
|
228
|
+
PartNumber: partNumber,
|
|
229
|
+
Body: readStream
|
|
230
|
+
}))).ETag;
|
|
231
|
+
}
|
|
232
|
+
async uploadParts(key, uploadId, size, readStream, currentPartNumber, offset) {
|
|
233
|
+
const promises$2 = [];
|
|
234
|
+
let pendingChunkFilepath = null;
|
|
235
|
+
let bytesUploaded = 0;
|
|
236
|
+
let permit = void 0;
|
|
237
|
+
const splitterStream = new StreamSplitter({
|
|
238
|
+
chunkSize: this.calcOptimalPartSize(size),
|
|
239
|
+
directory: os.tmpdir()
|
|
240
|
+
}).on("beforeChunkStarted", async () => {
|
|
241
|
+
permit = await this.partUploadSemaphore.acquire();
|
|
242
|
+
}).on("chunkStarted", (filepath) => {
|
|
243
|
+
pendingChunkFilepath = filepath;
|
|
244
|
+
}).on("chunkFinished", ({ path, size: partSize }) => {
|
|
245
|
+
pendingChunkFilepath = null;
|
|
246
|
+
const partNumber = currentPartNumber++;
|
|
247
|
+
const acquiredPermit = permit;
|
|
248
|
+
offset += partSize;
|
|
249
|
+
const isFinalPart = size === offset;
|
|
250
|
+
const deferred = new Promise(async (resolve, reject) => {
|
|
251
|
+
try {
|
|
252
|
+
const readable = fs.createReadStream(path);
|
|
253
|
+
readable.on("error", reject);
|
|
254
|
+
if (partSize >= this.minPartSize || isFinalPart) {
|
|
255
|
+
await this.uploadPart(key, uploadId, readable, partNumber);
|
|
256
|
+
bytesUploaded += partSize;
|
|
257
|
+
}
|
|
258
|
+
resolve();
|
|
259
|
+
} catch (error) {
|
|
260
|
+
reject(error);
|
|
261
|
+
} finally {
|
|
262
|
+
promises.rm(path).catch(() => {});
|
|
263
|
+
acquiredPermit?.release();
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
promises$2.push(deferred);
|
|
267
|
+
}).on("chunkError", () => {
|
|
268
|
+
permit?.release();
|
|
269
|
+
});
|
|
270
|
+
try {
|
|
271
|
+
await promises$1.pipeline(readStream, splitterStream);
|
|
272
|
+
} catch (error) {
|
|
273
|
+
if (pendingChunkFilepath !== null) try {
|
|
274
|
+
await promises.rm(pendingChunkFilepath);
|
|
275
|
+
} catch {}
|
|
276
|
+
promises$2.push(Promise.reject(error));
|
|
277
|
+
} finally {
|
|
278
|
+
await Promise.all(promises$2);
|
|
279
|
+
}
|
|
280
|
+
return bytesUploaded;
|
|
281
|
+
}
|
|
282
|
+
async retrieveParts(key, uploadId, partNumberMarker) {
|
|
283
|
+
const data = await this.client.send(new ListPartsCommand({
|
|
284
|
+
Bucket: this.config.bucket,
|
|
285
|
+
Key: key,
|
|
286
|
+
UploadId: uploadId,
|
|
287
|
+
PartNumberMarker: partNumberMarker
|
|
288
|
+
}));
|
|
289
|
+
let parts = data.Parts ?? [];
|
|
290
|
+
if (data.IsTruncated) {
|
|
291
|
+
const rest = await this.retrieveParts(key, uploadId, data.NextPartNumberMarker);
|
|
292
|
+
parts = [...parts, ...rest];
|
|
293
|
+
}
|
|
294
|
+
if (!partNumberMarker) parts.sort((a, b) => a.PartNumber - b.PartNumber);
|
|
295
|
+
return parts;
|
|
296
|
+
}
|
|
297
|
+
async finishMultipartUpload(key, uploadId, parts) {
|
|
298
|
+
const command = new CompleteMultipartUploadCommand({
|
|
299
|
+
Bucket: this.config.bucket,
|
|
300
|
+
Key: key,
|
|
301
|
+
UploadId: uploadId,
|
|
302
|
+
MultipartUpload: { Parts: parts.map((part) => {
|
|
303
|
+
return {
|
|
304
|
+
ETag: part.ETag,
|
|
305
|
+
PartNumber: part.PartNumber
|
|
306
|
+
};
|
|
307
|
+
}) }
|
|
308
|
+
});
|
|
309
|
+
return (await this.client.send(command)).Location;
|
|
310
|
+
}
|
|
311
|
+
calcOptimalPartSize(size) {
|
|
312
|
+
if (size === void 0) size = this.maxUploadSize;
|
|
313
|
+
let optimalPartSize;
|
|
314
|
+
if (size <= this.preferredPartSize) optimalPartSize = size;
|
|
315
|
+
else if (size <= this.preferredPartSize * this.maxMultipartParts) optimalPartSize = this.preferredPartSize;
|
|
316
|
+
else optimalPartSize = Math.ceil(size / this.maxMultipartParts);
|
|
317
|
+
return optimalPartSize;
|
|
318
|
+
}
|
|
394
319
|
};
|
|
320
|
+
var src_default = DriverS3;
|
|
321
|
+
|
|
322
|
+
//#endregion
|
|
323
|
+
export { DriverS3, src_default as default };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@directus/storage-driver-s3",
|
|
3
|
-
"version": "12.0.
|
|
3
|
+
"version": "12.0.9",
|
|
4
4
|
"description": "S3 file storage abstraction for `@directus/storage`",
|
|
5
5
|
"homepage": "https://directus.io",
|
|
6
6
|
"repository": {
|
|
@@ -21,27 +21,28 @@
|
|
|
21
21
|
"dist"
|
|
22
22
|
],
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@aws-sdk/client-s3": "3.
|
|
25
|
-
"@aws-sdk/lib-storage": "3.
|
|
24
|
+
"@aws-sdk/client-s3": "3.858.0",
|
|
25
|
+
"@aws-sdk/lib-storage": "3.858.0",
|
|
26
26
|
"@shopify/semaphore": "3.1.0",
|
|
27
|
-
"@smithy/node-http-handler": "4.0
|
|
27
|
+
"@smithy/node-http-handler": "4.1.0",
|
|
28
28
|
"@tus/utils": "0.5.1",
|
|
29
29
|
"ms": "2.1.3",
|
|
30
|
-
"@directus/storage": "12.0.
|
|
31
|
-
"@directus/utils": "13.0.
|
|
30
|
+
"@directus/storage": "12.0.2",
|
|
31
|
+
"@directus/utils": "13.0.10"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"@directus/tsconfig": "3.0.0",
|
|
35
|
-
"@ngneat/falso": "
|
|
35
|
+
"@ngneat/falso": "8.0.2",
|
|
36
36
|
"@types/ms": "2.1.0",
|
|
37
37
|
"@vitest/coverage-v8": "3.2.4",
|
|
38
|
-
"
|
|
38
|
+
"tsdown": "0.14.2",
|
|
39
39
|
"typescript": "5.8.3",
|
|
40
|
-
"vitest": "3.2.4"
|
|
40
|
+
"vitest": "3.2.4",
|
|
41
|
+
"@directus/types": "13.2.3"
|
|
41
42
|
},
|
|
42
43
|
"scripts": {
|
|
43
|
-
"build": "
|
|
44
|
-
"dev": "
|
|
44
|
+
"build": "tsdown src/index.ts --dts",
|
|
45
|
+
"dev": "tsdown src/index.ts --dts --watch",
|
|
45
46
|
"test": "vitest run",
|
|
46
47
|
"test:coverage": "vitest run --coverage"
|
|
47
48
|
}
|