@travetto/model-s3 8.0.0-alpha.3 → 8.0.0-alpha.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/__index__.ts +1 -1
- package/package.json +19 -19
- package/src/config.ts +19 -11
- package/src/service.ts +146 -89
- package/support/service.s3.ts +2 -2
package/README.md
CHANGED
|
@@ -13,11 +13,11 @@ npm install @travetto/model-s3
|
|
|
13
13
|
yarn add @travetto/model-s3
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.").
|
|
16
|
+
This module provides an [s3](https://aws.amazon.com/documentation/s3/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and stream against [s3](https://aws.amazon.com/documentation/s3/).
|
|
17
17
|
|
|
18
18
|
Supported features:
|
|
19
|
-
* [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L11)
|
|
20
19
|
* [Blob](https://github.com/travetto/travetto/tree/main/module/model/src/types/blob.ts#L8)
|
|
20
|
+
* [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/types/crud.ts#L10)
|
|
21
21
|
* [Expiry](https://github.com/travetto/travetto/tree/main/module/model/src/types/expiry.ts#L10)
|
|
22
22
|
|
|
23
23
|
Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
|
|
@@ -65,11 +65,11 @@ export class S3ModelConfig {
|
|
|
65
65
|
modifyStorage?: boolean;
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
|
-
* Provide
|
|
68
|
+
* Provide base URL for public access
|
|
69
69
|
*/
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
@Url()
|
|
71
|
+
@Required(false)
|
|
72
|
+
publicBaseUrl: string;
|
|
73
73
|
|
|
74
74
|
/**
|
|
75
75
|
* Produces the s3 config from the provide details, post construction
|
|
@@ -81,10 +81,18 @@ export class S3ModelConfig {
|
|
|
81
81
|
this.bucket ??= 'app';
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
if (!this.publicBaseUrl) {
|
|
85
|
+
if (this.endpoint?.includes('localhost')) {
|
|
86
|
+
this.publicBaseUrl = this.endpoint;
|
|
87
|
+
} else {
|
|
88
|
+
this.publicBaseUrl = `https://${this.bucket}.s3.amazonaws.com`;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
84
92
|
if (!this.accessKeyId && !this.secretAccessKey) {
|
|
85
|
-
const
|
|
86
|
-
this.accessKeyId =
|
|
87
|
-
this.secretAccessKey =
|
|
93
|
+
const credentials = await fromIni({ profile: this.profile })();
|
|
94
|
+
this.accessKeyId = credentials.accessKeyId;
|
|
95
|
+
this.secretAccessKey = credentials.secretAccessKey;
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
this.config = {
|
|
@@ -106,4 +114,4 @@ export class S3ModelConfig {
|
|
|
106
114
|
|
|
107
115
|
Additionally, you can see that the class is registered with the [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) annotation, and so these values can be overridden using the standard [Configuration](https://github.com/travetto/travetto/tree/main/module/config#readme "Configuration support") resolution paths.
|
|
108
116
|
|
|
109
|
-
**Note**: Do not commit your `accessKeyId` or `secretAccessKey` values to your source repository, especially if it is public facing.
|
|
117
|
+
**Note**: Do not commit your `accessKeyId` or `secretAccessKey` values to your source repository, especially if it is public facing. Not only is it a security risk, but Amazon will scan public repos, looking for keys, and if found will react swiftly.
|
package/__index__.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export * from './src/config.ts';
|
|
2
|
-
export * from './src/service.ts';
|
|
2
|
+
export * from './src/service.ts';
|
package/package.json
CHANGED
|
@@ -1,39 +1,42 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@travetto/model-s3",
|
|
3
|
-
"version": "8.0.0-alpha.
|
|
4
|
-
"type": "module",
|
|
3
|
+
"version": "8.0.0-alpha.30",
|
|
5
4
|
"description": "S3 backing for the travetto model module.",
|
|
6
5
|
"keywords": [
|
|
7
|
-
"s3",
|
|
8
6
|
"model",
|
|
9
|
-
"
|
|
10
|
-
"travetto"
|
|
7
|
+
"s3",
|
|
8
|
+
"travetto",
|
|
9
|
+
"typescript"
|
|
11
10
|
],
|
|
12
11
|
"homepage": "https://travetto.io",
|
|
13
12
|
"license": "MIT",
|
|
14
13
|
"author": {
|
|
15
|
-
"
|
|
16
|
-
"
|
|
14
|
+
"name": "Travetto Framework",
|
|
15
|
+
"email": "travetto.framework@gmail.com"
|
|
16
|
+
},
|
|
17
|
+
"repository": {
|
|
18
|
+
"url": "git+https://github.com/travetto/travetto.git",
|
|
19
|
+
"directory": "module/model-s3"
|
|
17
20
|
},
|
|
18
21
|
"files": [
|
|
19
22
|
"__index__.ts",
|
|
20
23
|
"support",
|
|
21
24
|
"src"
|
|
22
25
|
],
|
|
26
|
+
"type": "module",
|
|
23
27
|
"main": "__index__.ts",
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"directory": "module/model-s3"
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
27
30
|
},
|
|
28
31
|
"dependencies": {
|
|
29
|
-
"@aws-sdk/client-s3": "^3.
|
|
30
|
-
"@aws-sdk/credential-provider-ini": "^3.
|
|
31
|
-
"@aws-sdk/s3-request-presigner": "^3.
|
|
32
|
-
"@travetto/config": "^8.0.0-alpha.
|
|
33
|
-
"@travetto/model": "^8.0.0-alpha.
|
|
32
|
+
"@aws-sdk/client-s3": "^3.1116.0",
|
|
33
|
+
"@aws-sdk/credential-provider-ini": "^3.973.15",
|
|
34
|
+
"@aws-sdk/s3-request-presigner": "^3.1116.0",
|
|
35
|
+
"@travetto/config": "^8.0.0-alpha.27",
|
|
36
|
+
"@travetto/model": "^8.0.0-alpha.28"
|
|
34
37
|
},
|
|
35
38
|
"peerDependencies": {
|
|
36
|
-
"@travetto/cli": "^8.0.0-alpha.
|
|
39
|
+
"@travetto/cli": "^8.0.0-alpha.33"
|
|
37
40
|
},
|
|
38
41
|
"peerDependenciesMeta": {
|
|
39
42
|
"@travetto/cli": {
|
|
@@ -42,8 +45,5 @@
|
|
|
42
45
|
},
|
|
43
46
|
"travetto": {
|
|
44
47
|
"displayName": "S3 Model Support"
|
|
45
|
-
},
|
|
46
|
-
"publishConfig": {
|
|
47
|
-
"access": "public"
|
|
48
48
|
}
|
|
49
49
|
}
|
package/src/config.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { fromIni } from '@aws-sdk/credential-provider-ini';
|
|
2
1
|
import type s3 from '@aws-sdk/client-s3';
|
|
2
|
+
import { fromIni } from '@aws-sdk/credential-provider-ini';
|
|
3
3
|
|
|
4
4
|
import { Config, EnvVar } from '@travetto/config';
|
|
5
|
-
import { Required } from '@travetto/schema';
|
|
6
|
-
import { Runtime } from '@travetto/runtime';
|
|
7
5
|
import { PostConstruct } from '@travetto/di';
|
|
6
|
+
import { Runtime } from '@travetto/runtime';
|
|
7
|
+
import { Required, Url } from '@travetto/schema';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* S3 Support as an Asset Source
|
|
@@ -33,11 +33,11 @@ export class S3ModelConfig {
|
|
|
33
33
|
modifyStorage?: boolean;
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
|
-
* Provide
|
|
36
|
+
* Provide base URL for public access
|
|
37
37
|
*/
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
@Url()
|
|
39
|
+
@Required(false)
|
|
40
|
+
publicBaseUrl: string;
|
|
41
41
|
|
|
42
42
|
/**
|
|
43
43
|
* Produces the s3 config from the provide details, post construction
|
|
@@ -49,10 +49,18 @@ export class S3ModelConfig {
|
|
|
49
49
|
this.bucket ??= 'app';
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
if (!this.publicBaseUrl) {
|
|
53
|
+
if (this.endpoint?.includes('localhost')) {
|
|
54
|
+
this.publicBaseUrl = this.endpoint;
|
|
55
|
+
} else {
|
|
56
|
+
this.publicBaseUrl = `https://${this.bucket}.s3.amazonaws.com`;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
52
60
|
if (!this.accessKeyId && !this.secretAccessKey) {
|
|
53
|
-
const
|
|
54
|
-
this.accessKeyId =
|
|
55
|
-
this.secretAccessKey =
|
|
61
|
+
const credentials = await fromIni({ profile: this.profile })();
|
|
62
|
+
this.accessKeyId = credentials.accessKeyId;
|
|
63
|
+
this.secretAccessKey = credentials.secretAccessKey;
|
|
56
64
|
}
|
|
57
65
|
|
|
58
66
|
this.config = {
|
|
@@ -69,4 +77,4 @@ export class S3ModelConfig {
|
|
|
69
77
|
this.config.forcePathStyle ??= true;
|
|
70
78
|
}
|
|
71
79
|
}
|
|
72
|
-
}
|
|
80
|
+
}
|
package/src/service.ts
CHANGED
|
@@ -1,18 +1,42 @@
|
|
|
1
1
|
import { Agent } from 'node:https';
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { type CompletedPart, type CreateMultipartUploadRequest, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3';
|
|
4
|
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
4
5
|
import type { MetadataBearer } from '@aws-sdk/types';
|
|
5
6
|
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
|
6
|
-
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
7
7
|
|
|
8
|
+
import { Injectable, PostConstruct } from '@travetto/di';
|
|
8
9
|
import {
|
|
9
|
-
|
|
10
|
-
type ModelBlobSupport,
|
|
10
|
+
ExistsError,
|
|
11
|
+
type ModelBlobSupport,
|
|
12
|
+
type ModelCrudSupport,
|
|
13
|
+
ModelCrudUtil,
|
|
14
|
+
type ModelExpirySupport,
|
|
15
|
+
ModelExpiryUtil,
|
|
16
|
+
type ModelListOptions,
|
|
17
|
+
ModelRegistryIndex,
|
|
18
|
+
type ModelStorageSupport,
|
|
19
|
+
ModelStorageUtil,
|
|
20
|
+
type ModelType,
|
|
21
|
+
NotFoundError,
|
|
22
|
+
type OptionalId
|
|
11
23
|
} from '@travetto/model';
|
|
12
|
-
import { Injectable, PostConstruct } from '@travetto/di';
|
|
13
24
|
import {
|
|
14
|
-
|
|
15
|
-
|
|
25
|
+
asFull,
|
|
26
|
+
type BinaryArray,
|
|
27
|
+
type BinaryMetadata,
|
|
28
|
+
BinaryMetadataUtil,
|
|
29
|
+
type BinaryType,
|
|
30
|
+
BinaryUtil,
|
|
31
|
+
type ByteRange,
|
|
32
|
+
type Class,
|
|
33
|
+
CodecUtil,
|
|
34
|
+
castTo,
|
|
35
|
+
JSONUtil,
|
|
36
|
+
RuntimeError,
|
|
37
|
+
type TimeSpan,
|
|
38
|
+
TimeUtil,
|
|
39
|
+
TypedObject
|
|
16
40
|
} from '@travetto/runtime';
|
|
17
41
|
|
|
18
42
|
import type { S3ModelConfig } from './config.ts';
|
|
@@ -21,11 +45,12 @@ function isMetadataBearer(value: unknown): value is MetadataBearer {
|
|
|
21
45
|
return !!value && typeof value === 'object' && '$metadata' in value;
|
|
22
46
|
}
|
|
23
47
|
|
|
24
|
-
function
|
|
48
|
+
function hasLowerContentType<T>(value: T): value is T & { contenttype?: string } {
|
|
25
49
|
return value !== undefined && value !== null && Object.hasOwn(value, 'contenttype');
|
|
26
50
|
}
|
|
27
51
|
|
|
28
|
-
type S3Metadata = Pick<
|
|
52
|
+
type S3Metadata = Pick<
|
|
53
|
+
CreateMultipartUploadRequest,
|
|
29
54
|
'ContentType' | 'Metadata' | 'ContentEncoding' | 'ContentLanguage' | 'CacheControl' | 'ContentDisposition'
|
|
30
55
|
>;
|
|
31
56
|
|
|
@@ -34,12 +59,13 @@ type S3Metadata = Pick<CreateMultipartUploadRequest,
|
|
|
34
59
|
*/
|
|
35
60
|
@Injectable()
|
|
36
61
|
export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, ModelStorageSupport, ModelExpirySupport {
|
|
37
|
-
|
|
38
62
|
idSource = ModelCrudUtil.uuidSource();
|
|
39
63
|
client: S3;
|
|
40
64
|
config: S3ModelConfig;
|
|
41
65
|
|
|
42
|
-
constructor(config: S3ModelConfig) {
|
|
66
|
+
constructor(config: S3ModelConfig) {
|
|
67
|
+
this.config = config;
|
|
68
|
+
}
|
|
43
69
|
|
|
44
70
|
#getMetadata(metadata: BinaryMetadata): S3Metadata {
|
|
45
71
|
return {
|
|
@@ -48,8 +74,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
48
74
|
...(metadata.contentLanguage ? { ContentLanguage: metadata.contentLanguage } : {}),
|
|
49
75
|
...(metadata.cacheControl ? { CacheControl: metadata.cacheControl } : {}),
|
|
50
76
|
Metadata: TypedObject.fromEntries(
|
|
51
|
-
TypedObject.entries(metadata)
|
|
52
|
-
.map(([key, value]) => [key, typeof value === 'string' ? value : JSONUtil.toUTF8(value)] as const)
|
|
77
|
+
TypedObject.entries(metadata).map(([key, value]) => [key, typeof value === 'string' ? value : JSONUtil.toUTF8(value)] as const)
|
|
53
78
|
)
|
|
54
79
|
};
|
|
55
80
|
}
|
|
@@ -73,12 +98,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
73
98
|
return this.#basicKey(key);
|
|
74
99
|
}
|
|
75
100
|
|
|
76
|
-
#query<U extends object>(cls: string | Class, id: string, extra: U = asFull({})):
|
|
101
|
+
#query<U extends object>(cls: string | Class, id: string, extra: U = asFull({})): U & { Key: string; Bucket: string } {
|
|
77
102
|
const key = this.#resolveKey(cls, id);
|
|
78
103
|
return { Key: key, Bucket: this.config.bucket, ...extra };
|
|
79
104
|
}
|
|
80
105
|
|
|
81
|
-
#queryBlob<U extends object>(id: string, extra: U = asFull({})):
|
|
106
|
+
#queryBlob<U extends object>(id: string, extra: U = asFull({})): U & { Key: string; Bucket: string } {
|
|
82
107
|
const key = this.#basicKey(id);
|
|
83
108
|
return { Key: key, Bucket: this.config.bucket, ...extra };
|
|
84
109
|
}
|
|
@@ -93,16 +118,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
93
118
|
return {};
|
|
94
119
|
}
|
|
95
120
|
|
|
96
|
-
async
|
|
121
|
+
async *#iterateBucket(cls?: string | Class, options?: ModelListOptions): AsyncIterable<{ Key: string; id: string }[]> {
|
|
97
122
|
let Marker: string | undefined;
|
|
98
|
-
|
|
123
|
+
const batchSize = options?.batchSizeHint ?? 100;
|
|
124
|
+
const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
|
|
125
|
+
let produced = 0;
|
|
126
|
+
for (; !options?.abort?.aborted && produced < maxCount;) {
|
|
99
127
|
const items = await this.client.listObjects({
|
|
100
128
|
Bucket: this.config.bucket,
|
|
101
129
|
Prefix: cls ? this.#resolveKey(cls) : this.config.namespace,
|
|
102
|
-
Marker
|
|
130
|
+
Marker,
|
|
131
|
+
MaxKeys: batchSize
|
|
103
132
|
});
|
|
104
|
-
|
|
105
|
-
|
|
133
|
+
|
|
134
|
+
let toSend = items.Contents?.map(item => ({ Key: item.Key!, id: item.Key!.split(':').pop()! }));
|
|
135
|
+
if (toSend) {
|
|
136
|
+
produced += toSend.length;
|
|
137
|
+
if (produced > maxCount) {
|
|
138
|
+
toSend = toSend.slice(0, maxCount - (produced - toSend.length));
|
|
139
|
+
}
|
|
140
|
+
yield toSend;
|
|
106
141
|
}
|
|
107
142
|
if (items.NextMarker) {
|
|
108
143
|
Marker = items.NextMarker;
|
|
@@ -123,12 +158,16 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
123
158
|
let total = 0;
|
|
124
159
|
let i = 1;
|
|
125
160
|
const flush = async (): Promise<void> => {
|
|
126
|
-
if (!total) {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
161
|
+
if (!total) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const part = await this.client.uploadPart(
|
|
165
|
+
this.#queryBlob(id, {
|
|
166
|
+
Body: BinaryUtil.binaryArrayToUint8Array(BinaryUtil.combineBinaryArrays(buffers)),
|
|
167
|
+
PartNumber: i,
|
|
168
|
+
UploadId
|
|
169
|
+
})
|
|
170
|
+
);
|
|
132
171
|
parts.push({ PartNumber: i, ETag: part.ETag });
|
|
133
172
|
i += 1;
|
|
134
173
|
buffers = [];
|
|
@@ -145,10 +184,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
145
184
|
}
|
|
146
185
|
await flush();
|
|
147
186
|
|
|
148
|
-
await this.client.completeMultipartUpload(
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
187
|
+
await this.client.completeMultipartUpload(
|
|
188
|
+
this.#queryBlob(id, {
|
|
189
|
+
UploadId,
|
|
190
|
+
MultipartUpload: { Parts: parts }
|
|
191
|
+
})
|
|
192
|
+
);
|
|
152
193
|
} catch (error) {
|
|
153
194
|
await this.client.abortMultipartUpload(this.#queryBlob(id, { UploadId }));
|
|
154
195
|
throw error;
|
|
@@ -182,14 +223,18 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
182
223
|
async initializeClient(): Promise<void> {
|
|
183
224
|
this.client = new S3({
|
|
184
225
|
...this.config.config,
|
|
185
|
-
...('requestHandler' in this.config.config
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
226
|
+
...('requestHandler' in this.config.config
|
|
227
|
+
? {
|
|
228
|
+
requestHandler: new NodeHttpHandler({
|
|
229
|
+
...this.config.config.requestHandler,
|
|
230
|
+
...('httpsAgent' in this.config.config.requestHandler!
|
|
231
|
+
? {
|
|
232
|
+
httpsAgent: new Agent({ ...(this.config.config.requestHandler?.httpsAgent ?? {}) })
|
|
233
|
+
}
|
|
234
|
+
: {})
|
|
235
|
+
})
|
|
236
|
+
}
|
|
237
|
+
: {})
|
|
193
238
|
});
|
|
194
239
|
ModelStorageUtil.storageInitialization(this);
|
|
195
240
|
}
|
|
@@ -232,10 +277,8 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
232
277
|
}
|
|
233
278
|
throw new NotFoundError(cls, id);
|
|
234
279
|
} catch (error) {
|
|
235
|
-
if (isMetadataBearer(error)) {
|
|
236
|
-
|
|
237
|
-
error = new NotFoundError(cls, id);
|
|
238
|
-
}
|
|
280
|
+
if (isMetadataBearer(error) && error.$metadata.httpStatusCode === 404) {
|
|
281
|
+
throw new NotFoundError(cls, id);
|
|
239
282
|
}
|
|
240
283
|
throw error;
|
|
241
284
|
}
|
|
@@ -247,12 +290,14 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
247
290
|
prepped = await ModelCrudUtil.preStore(cls, item, this);
|
|
248
291
|
}
|
|
249
292
|
const content = JSONUtil.toBinaryArray(prepped);
|
|
250
|
-
await this.client.putObject(
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
293
|
+
await this.client.putObject(
|
|
294
|
+
this.#query(cls, prepped.id, {
|
|
295
|
+
Body: BinaryUtil.binaryArrayToUint8Array(content),
|
|
296
|
+
ContentType: 'application/json',
|
|
297
|
+
ContentLength: content.byteLength,
|
|
298
|
+
...this.#getExpiryConfig(cls, prepped)
|
|
299
|
+
})
|
|
300
|
+
);
|
|
256
301
|
return prepped;
|
|
257
302
|
}
|
|
258
303
|
|
|
@@ -293,17 +338,9 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
293
338
|
await this.client.deleteObject(this.#query(cls, id));
|
|
294
339
|
}
|
|
295
340
|
|
|
296
|
-
async *
|
|
297
|
-
for await (const batch of this.#iterateBucket(cls)) {
|
|
298
|
-
|
|
299
|
-
try {
|
|
300
|
-
yield await this.get(cls, id);
|
|
301
|
-
} catch (error) {
|
|
302
|
-
if (!(error instanceof NotFoundError)) {
|
|
303
|
-
throw error;
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
}
|
|
341
|
+
async *list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
|
|
342
|
+
for await (const batch of this.#iterateBucket(cls, options)) {
|
|
343
|
+
yield ModelCrudUtil.filterOutNotFound(batch.map(item => this.get(cls, item.id)));
|
|
307
344
|
}
|
|
308
345
|
}
|
|
309
346
|
|
|
@@ -314,7 +351,13 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
314
351
|
|
|
315
352
|
// Blob support
|
|
316
353
|
async upsertBlob(location: string, input: BinaryType, metadata?: BinaryMetadata, overwrite = true): Promise<void> {
|
|
317
|
-
if (
|
|
354
|
+
if (
|
|
355
|
+
!overwrite &&
|
|
356
|
+
(await this.getBlobMetadata(location).then(
|
|
357
|
+
() => true,
|
|
358
|
+
() => false
|
|
359
|
+
))
|
|
360
|
+
) {
|
|
318
361
|
return;
|
|
319
362
|
}
|
|
320
363
|
|
|
@@ -322,11 +365,12 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
322
365
|
|
|
323
366
|
const length = BinaryMetadataUtil.readLength(resolved);
|
|
324
367
|
|
|
325
|
-
if (length && length < this.config.chunkSize) {
|
|
368
|
+
if (length && length < this.config.chunkSize) {
|
|
369
|
+
// If smaller than chunk size
|
|
326
370
|
const blob = this.#queryBlob(location, {
|
|
327
371
|
Body: BinaryUtil.toReadable(input),
|
|
328
372
|
ContentLength: length,
|
|
329
|
-
...this.#getMetadata(resolved)
|
|
373
|
+
...this.#getMetadata(resolved)
|
|
330
374
|
});
|
|
331
375
|
// Upload to s3
|
|
332
376
|
await this.client.putObject(blob);
|
|
@@ -337,18 +381,26 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
337
381
|
|
|
338
382
|
async #getObject(location: string, range?: Required<ByteRange>): Promise<BinaryType> {
|
|
339
383
|
// Read from s3
|
|
340
|
-
const result = await this.client.getObject(
|
|
341
|
-
|
|
342
|
-
|
|
384
|
+
const result = await this.client.getObject(
|
|
385
|
+
this.#queryBlob(
|
|
386
|
+
location,
|
|
387
|
+
range
|
|
388
|
+
? {
|
|
389
|
+
Range: `bytes=${range.start}-${range.end}`
|
|
390
|
+
}
|
|
391
|
+
: {}
|
|
392
|
+
)
|
|
393
|
+
);
|
|
343
394
|
|
|
344
395
|
const body: BinaryType | string | undefined = castTo(result.Body);
|
|
345
396
|
|
|
346
397
|
switch (typeof body) {
|
|
347
|
-
case 'undefined':
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
CodecUtil.fromUTF8String(body);
|
|
351
|
-
default:
|
|
398
|
+
case 'undefined':
|
|
399
|
+
throw new RuntimeError('Unable to read type: undefined');
|
|
400
|
+
case 'string':
|
|
401
|
+
return body.endsWith('=') ? CodecUtil.fromBase64String(body) : CodecUtil.fromUTF8String(body);
|
|
402
|
+
default:
|
|
403
|
+
return body;
|
|
352
404
|
}
|
|
353
405
|
}
|
|
354
406
|
|
|
@@ -358,15 +410,13 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
358
410
|
return BinaryMetadataUtil.makeBlob(() => this.#getObject(location, final), { ...metadata, range: final });
|
|
359
411
|
}
|
|
360
412
|
|
|
361
|
-
async headBlob(location: string): Promise<{ Metadata?: BinaryMetadata
|
|
413
|
+
async headBlob(location: string): Promise<{ Metadata?: BinaryMetadata; ContentLength?: number; ContentType?: string }> {
|
|
362
414
|
const query = this.#queryBlob(location);
|
|
363
415
|
try {
|
|
364
|
-
return
|
|
416
|
+
return await this.client.headObject(query);
|
|
365
417
|
} catch (error) {
|
|
366
|
-
if (isMetadataBearer(error)) {
|
|
367
|
-
|
|
368
|
-
error = new NotFoundError('Blob', location);
|
|
369
|
-
}
|
|
418
|
+
if (isMetadataBearer(error) && error.$metadata.httpStatusCode === 404) {
|
|
419
|
+
throw new NotFoundError('Blob', location);
|
|
370
420
|
}
|
|
371
421
|
throw error;
|
|
372
422
|
}
|
|
@@ -377,13 +427,13 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
377
427
|
|
|
378
428
|
if (blob) {
|
|
379
429
|
const metadata: BinaryMetadata = {
|
|
380
|
-
contentType: '',
|
|
430
|
+
contentType: blob.ContentType ?? '',
|
|
381
431
|
...blob.Metadata,
|
|
382
|
-
size: blob.ContentLength
|
|
432
|
+
size: blob.ContentLength!
|
|
383
433
|
};
|
|
384
|
-
if (
|
|
385
|
-
metadata
|
|
386
|
-
delete metadata
|
|
434
|
+
if (hasLowerContentType(metadata)) {
|
|
435
|
+
metadata.contentType = metadata.contenttype!;
|
|
436
|
+
delete metadata.contenttype;
|
|
387
437
|
}
|
|
388
438
|
return metadata;
|
|
389
439
|
} else {
|
|
@@ -406,12 +456,19 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
406
456
|
}
|
|
407
457
|
|
|
408
458
|
// Signed urls
|
|
409
|
-
async getBlobReadUrl(location: string, expiresIn: TimeSpan = '1h'): Promise<string> {
|
|
410
|
-
|
|
411
|
-
this
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
459
|
+
async getBlobReadUrl(location: string, expiresIn: TimeSpan | false = '1h'): Promise<string> {
|
|
460
|
+
if (expiresIn === false) {
|
|
461
|
+
const key = this.#basicKey(location);
|
|
462
|
+
const baseUrl = this.config.publicBaseUrl;
|
|
463
|
+
if (this.config.config.forcePathStyle) {
|
|
464
|
+
return `${baseUrl}/${this.config.bucket}/${key}`;
|
|
465
|
+
} else {
|
|
466
|
+
return `${baseUrl}/${key}`;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return await getSignedUrl(this.client, new GetObjectCommand(this.#queryBlob(location)), {
|
|
470
|
+
expiresIn: TimeUtil.duration(expiresIn, 's')
|
|
471
|
+
});
|
|
415
472
|
}
|
|
416
473
|
|
|
417
474
|
async getBlobWriteUrl(location: string, metadata: BinaryMetadata, expiresIn: TimeSpan = '1h'): Promise<string> {
|
|
@@ -422,7 +479,7 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
422
479
|
...this.#queryBlob(location),
|
|
423
480
|
...base,
|
|
424
481
|
...(metadata.size ? { ContentLength: metadata.size } : {}),
|
|
425
|
-
...(
|
|
482
|
+
...(metadata.hash && metadata.hash !== '-1' ? { ChecksumSHA256: metadata.hash } : {})
|
|
426
483
|
}),
|
|
427
484
|
{
|
|
428
485
|
expiresIn: TimeUtil.duration(expiresIn, 's'),
|
|
@@ -455,4 +512,4 @@ export class S3ModelService implements ModelCrudSupport, ModelBlobSupport, Model
|
|
|
455
512
|
await this.client.deleteBucket({ Bucket: this.config.bucket });
|
|
456
513
|
}
|
|
457
514
|
}
|
|
458
|
-
}
|
|
515
|
+
}
|
package/support/service.s3.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ServiceDescriptor } from '@travetto/cli';
|
|
2
2
|
|
|
3
|
-
const version = '4.
|
|
3
|
+
const version = '4.12.4';
|
|
4
4
|
|
|
5
5
|
export const service: ServiceDescriptor = {
|
|
6
6
|
name: 's3',
|
|
@@ -11,4 +11,4 @@ export const service: ServiceDescriptor = {
|
|
|
11
11
|
env: {
|
|
12
12
|
COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: 'app'
|
|
13
13
|
}
|
|
14
|
-
};
|
|
14
|
+
};
|