@twin.org/blob-storage-connector-aws-s3 0.0.2-next.4 → 0.0.3-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/index.js +6 -0
- package/dist/es/index.js.map +1 -0
- package/dist/es/models/IS3BlobStorageConnectorConfig.js +4 -0
- package/dist/es/models/IS3BlobStorageConnectorConfig.js.map +1 -0
- package/dist/es/models/IS3BlobStorageConnectorConstructorOptions.js +2 -0
- package/dist/es/models/IS3BlobStorageConnectorConstructorOptions.js.map +1 -0
- package/dist/{esm/index.mjs → es/s3BlobStorageConnector.js} +57 -39
- package/dist/es/s3BlobStorageConnector.js.map +1 -0
- package/dist/types/index.d.ts +3 -3
- package/dist/types/models/IS3BlobStorageConnectorConstructorOptions.d.ts +5 -1
- package/dist/types/s3BlobStorageConnector.d.ts +7 -2
- package/docs/changelog.md +36 -0
- package/docs/reference/classes/S3BlobStorageConnector.md +19 -5
- package/docs/reference/interfaces/IS3BlobStorageConnectorConstructorOptions.md +8 -0
- package/locales/en.json +0 -1
- package/package.json +13 -10
- package/dist/cjs/index.cjs +0 -204
package/dist/es/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// Copyright 2024 IOTA Stiftung.
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0.
|
|
3
|
+
export * from "./s3BlobStorageConnector.js";
|
|
4
|
+
export * from "./models/IS3BlobStorageConnectorConfig.js";
|
|
5
|
+
export * from "./models/IS3BlobStorageConnectorConstructorOptions.js";
|
|
6
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,2CAA2C,CAAC;AAC1D,cAAc,uDAAuD,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nexport * from \"./s3BlobStorageConnector.js\";\nexport * from \"./models/IS3BlobStorageConnectorConfig.js\";\nexport * from \"./models/IS3BlobStorageConnectorConstructorOptions.js\";\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IS3BlobStorageConnectorConfig.js","sourceRoot":"","sources":["../../../src/models/IS3BlobStorageConnectorConfig.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\n\n/**\n * Configuration for the S3 Blob Storage Connector.\n */\nexport interface IS3BlobStorageConnectorConfig {\n\t/**\n\t * The AWS region.\n\t */\n\tregion: string;\n\n\t/**\n\t * The S3 bucket name.\n\t */\n\tbucketName: string;\n\n\t/**\n\t * The authentication mode.\n\t * - \"credentials\": Use access key ID and secret access key.\n\t * - \"pod\": Use IAM role attached to the pod (e.g., in EKS).\n\t * @default credentials\n\t */\n\tauthMode?: \"credentials\" | \"pod\";\n\n\t/**\n\t * The AWS access key ID.\n\t */\n\taccessKeyId?: string;\n\n\t/**\n\t * The AWS secret access key.\n\t */\n\tsecretAccessKey?: string;\n\n\t/**\n\t * Optional endpoint for S3-compatible storage (e.g., MinIO).\n\t */\n\tendpoint?: string;\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IS3BlobStorageConnectorConstructorOptions.js","sourceRoot":"","sources":["../../../src/models/IS3BlobStorageConnectorConstructorOptions.ts"],"names":[],"mappings":"","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport type { IS3BlobStorageConnectorConfig } from \"./IS3BlobStorageConnectorConfig.js\";\n\n/**\n * Options for the S3 Blob Storage Connector constructor.\n */\nexport interface IS3BlobStorageConnectorConstructorOptions {\n\t/**\n\t * The keys to use from the context ids to create partitions.\n\t */\n\tpartitionContextIds?: string[];\n\n\t/**\n\t * The configuration for the connector.\n\t */\n\tconfig: IS3BlobStorageConnectorConfig;\n}\n"]}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { S3Client, ListBucketsCommand, CreateBucketCommand, PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
|
2
|
-
import { Guards, ComponentFactory, BaseError, Converter, Urn, GeneralError } from '@twin.org/core';
|
|
3
|
-
import { Sha256 } from '@twin.org/crypto';
|
|
4
|
-
|
|
5
1
|
// Copyright 2024 IOTA Stiftung.
|
|
6
2
|
// SPDX-License-Identifier: Apache-2.0.
|
|
3
|
+
import { CreateBucketCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListBucketsCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
|
4
|
+
import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
|
|
5
|
+
import { BaseError, ComponentFactory, Converter, GeneralError, Guards, Urn } from "@twin.org/core";
|
|
6
|
+
import { Sha256 } from "@twin.org/crypto";
|
|
7
7
|
/**
|
|
8
8
|
* Class for performing blob storage operations on S3.
|
|
9
9
|
* See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/ for more information.
|
|
10
10
|
*/
|
|
11
|
-
class S3BlobStorageConnector {
|
|
11
|
+
export class S3BlobStorageConnector {
|
|
12
12
|
/**
|
|
13
13
|
* The namespace for the items.
|
|
14
14
|
*/
|
|
@@ -16,12 +16,17 @@ class S3BlobStorageConnector {
|
|
|
16
16
|
/**
|
|
17
17
|
* Runtime name for the class.
|
|
18
18
|
*/
|
|
19
|
-
CLASS_NAME = "S3BlobStorageConnector";
|
|
19
|
+
static CLASS_NAME = "S3BlobStorageConnector";
|
|
20
20
|
/**
|
|
21
21
|
* The configuration for the connector.
|
|
22
22
|
* @internal
|
|
23
23
|
*/
|
|
24
24
|
_config;
|
|
25
|
+
/**
|
|
26
|
+
* The keys to use from the context ids to create partitions.
|
|
27
|
+
* @internal
|
|
28
|
+
*/
|
|
29
|
+
_partitionContextIds;
|
|
25
30
|
/**
|
|
26
31
|
* The S3 client.
|
|
27
32
|
* @internal
|
|
@@ -32,20 +37,21 @@ class S3BlobStorageConnector {
|
|
|
32
37
|
* @param options The options for the connector.
|
|
33
38
|
*/
|
|
34
39
|
constructor(options) {
|
|
35
|
-
Guards.object(
|
|
36
|
-
Guards.object(
|
|
37
|
-
Guards.stringValue(
|
|
38
|
-
Guards.stringValue(
|
|
40
|
+
Guards.object(S3BlobStorageConnector.CLASS_NAME, "options", options);
|
|
41
|
+
Guards.object(S3BlobStorageConnector.CLASS_NAME, "options.config", options.config);
|
|
42
|
+
Guards.stringValue(S3BlobStorageConnector.CLASS_NAME, "options.config.region", options.config.region);
|
|
43
|
+
Guards.stringValue(S3BlobStorageConnector.CLASS_NAME, "options.config.bucketName", options.config.bucketName);
|
|
39
44
|
options.config.authMode ??= "credentials";
|
|
40
45
|
let credentials;
|
|
41
46
|
if (options.config.authMode === "credentials") {
|
|
42
|
-
Guards.stringValue(
|
|
43
|
-
Guards.stringValue(
|
|
47
|
+
Guards.stringValue(S3BlobStorageConnector.CLASS_NAME, "options.config.accessKeyId", options.config.accessKeyId);
|
|
48
|
+
Guards.stringValue(S3BlobStorageConnector.CLASS_NAME, "options.config.secretAccessKey", options.config.secretAccessKey);
|
|
44
49
|
credentials = {
|
|
45
50
|
accessKeyId: options.config.accessKeyId,
|
|
46
51
|
secretAccessKey: options.config.secretAccessKey
|
|
47
52
|
};
|
|
48
53
|
}
|
|
54
|
+
this._partitionContextIds = options.partitionContextIds;
|
|
49
55
|
this._config = options.config;
|
|
50
56
|
this._s3Client = new S3Client({
|
|
51
57
|
region: this._config.region,
|
|
@@ -54,6 +60,13 @@ class S3BlobStorageConnector {
|
|
|
54
60
|
forcePathStyle: true
|
|
55
61
|
});
|
|
56
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Returns the class name of the component.
|
|
65
|
+
* @returns The class name of the component.
|
|
66
|
+
*/
|
|
67
|
+
className() {
|
|
68
|
+
return S3BlobStorageConnector.CLASS_NAME;
|
|
69
|
+
}
|
|
57
70
|
/**
|
|
58
71
|
* Bootstrap the component by creating and initializing any resources it needs.
|
|
59
72
|
* @param nodeLoggingComponentType The node logging component type.
|
|
@@ -62,21 +75,13 @@ class S3BlobStorageConnector {
|
|
|
62
75
|
async bootstrap(nodeLoggingComponentType) {
|
|
63
76
|
const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
|
|
64
77
|
try {
|
|
65
|
-
await nodeLogging?.log({
|
|
66
|
-
level: "info",
|
|
67
|
-
source: this.CLASS_NAME,
|
|
68
|
-
message: "bucketCreating",
|
|
69
|
-
data: {
|
|
70
|
-
bucket: this._config.bucketName
|
|
71
|
-
}
|
|
72
|
-
});
|
|
73
78
|
const listBucketsCommand = new ListBucketsCommand({});
|
|
74
79
|
const bucketsList = await this._s3Client.send(listBucketsCommand);
|
|
75
80
|
const bucketExists = bucketsList.Buckets?.some(bucket => bucket.Name === this._config.bucketName);
|
|
76
81
|
if (bucketExists) {
|
|
77
82
|
await nodeLogging?.log({
|
|
78
83
|
level: "info",
|
|
79
|
-
source:
|
|
84
|
+
source: S3BlobStorageConnector.CLASS_NAME,
|
|
80
85
|
message: "bucketExists",
|
|
81
86
|
data: {
|
|
82
87
|
bucket: this._config.bucketName
|
|
@@ -84,21 +89,21 @@ class S3BlobStorageConnector {
|
|
|
84
89
|
});
|
|
85
90
|
}
|
|
86
91
|
else {
|
|
87
|
-
await this._s3Client.send(new CreateBucketCommand({ Bucket: this._config.bucketName }));
|
|
88
92
|
await nodeLogging?.log({
|
|
89
93
|
level: "info",
|
|
90
|
-
source:
|
|
91
|
-
message: "
|
|
94
|
+
source: S3BlobStorageConnector.CLASS_NAME,
|
|
95
|
+
message: "bucketCreating",
|
|
92
96
|
data: {
|
|
93
97
|
bucket: this._config.bucketName
|
|
94
98
|
}
|
|
95
99
|
});
|
|
100
|
+
await this._s3Client.send(new CreateBucketCommand({ Bucket: this._config.bucketName }));
|
|
96
101
|
}
|
|
97
102
|
}
|
|
98
103
|
catch (err) {
|
|
99
104
|
await nodeLogging?.log({
|
|
100
105
|
level: "error",
|
|
101
|
-
source:
|
|
106
|
+
source: S3BlobStorageConnector.CLASS_NAME,
|
|
102
107
|
message: "bucketCreateFailed",
|
|
103
108
|
data: {
|
|
104
109
|
bucket: this._config.bucketName
|
|
@@ -115,19 +120,21 @@ class S3BlobStorageConnector {
|
|
|
115
120
|
* @returns The id of the stored blob in urn format.
|
|
116
121
|
*/
|
|
117
122
|
async set(blob) {
|
|
118
|
-
Guards.uint8Array(
|
|
123
|
+
Guards.uint8Array(S3BlobStorageConnector.CLASS_NAME, "blob", blob);
|
|
119
124
|
try {
|
|
120
125
|
const id = Converter.bytesToHex(Sha256.sum256(blob));
|
|
126
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
127
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
121
128
|
const command = new PutObjectCommand({
|
|
122
129
|
Bucket: this._config.bucketName,
|
|
123
|
-
Key: id
|
|
130
|
+
Key: `${partitionKey ?? "root"}/${id}`,
|
|
124
131
|
Body: blob
|
|
125
132
|
});
|
|
126
133
|
await this._s3Client.send(command);
|
|
127
134
|
return `blob:${new Urn(S3BlobStorageConnector.NAMESPACE, id).toString()}`;
|
|
128
135
|
}
|
|
129
136
|
catch (err) {
|
|
130
|
-
throw new GeneralError(
|
|
137
|
+
throw new GeneralError(S3BlobStorageConnector.CLASS_NAME, "setBlobFailed", undefined, err);
|
|
131
138
|
}
|
|
132
139
|
}
|
|
133
140
|
/**
|
|
@@ -136,10 +143,12 @@ class S3BlobStorageConnector {
|
|
|
136
143
|
* @returns The data for the blob if it can be found or undefined.
|
|
137
144
|
*/
|
|
138
145
|
async get(id) {
|
|
139
|
-
Urn.guard(
|
|
146
|
+
Urn.guard(S3BlobStorageConnector.CLASS_NAME, "id", id);
|
|
140
147
|
const urnParsed = Urn.fromValidString(id);
|
|
148
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
149
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
141
150
|
if (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {
|
|
142
|
-
throw new GeneralError(
|
|
151
|
+
throw new GeneralError(S3BlobStorageConnector.CLASS_NAME, "namespaceMismatch", {
|
|
143
152
|
namespace: S3BlobStorageConnector.NAMESPACE,
|
|
144
153
|
id
|
|
145
154
|
});
|
|
@@ -148,14 +157,22 @@ class S3BlobStorageConnector {
|
|
|
148
157
|
const key = urnParsed.namespaceSpecific(1);
|
|
149
158
|
const command = new GetObjectCommand({
|
|
150
159
|
Bucket: this._config.bucketName,
|
|
151
|
-
Key: key
|
|
160
|
+
Key: `${partitionKey ?? "root"}/${key}`
|
|
152
161
|
});
|
|
153
162
|
const response = await this._s3Client.send(command);
|
|
154
163
|
if (response.Body) {
|
|
155
164
|
return new Uint8Array(await response.Body.transformToByteArray());
|
|
156
165
|
}
|
|
157
166
|
}
|
|
158
|
-
catch {
|
|
167
|
+
catch (err) {
|
|
168
|
+
if (BaseError.isErrorName(err, "NoSuchKey")) {
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
throw new GeneralError(S3BlobStorageConnector.CLASS_NAME, "getBlobFailed", {
|
|
172
|
+
id,
|
|
173
|
+
namespace: S3BlobStorageConnector.NAMESPACE
|
|
174
|
+
}, err);
|
|
175
|
+
}
|
|
159
176
|
}
|
|
160
177
|
/**
|
|
161
178
|
* Remove the blob.
|
|
@@ -163,10 +180,12 @@ class S3BlobStorageConnector {
|
|
|
163
180
|
* @returns True if the blob was found.
|
|
164
181
|
*/
|
|
165
182
|
async remove(id) {
|
|
166
|
-
Urn.guard(
|
|
183
|
+
Urn.guard(S3BlobStorageConnector.CLASS_NAME, "id", id);
|
|
167
184
|
const urnParsed = Urn.fromValidString(id);
|
|
185
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
186
|
+
const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
|
|
168
187
|
if (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {
|
|
169
|
-
throw new GeneralError(
|
|
188
|
+
throw new GeneralError(S3BlobStorageConnector.CLASS_NAME, "namespaceMismatch", {
|
|
170
189
|
namespace: S3BlobStorageConnector.NAMESPACE,
|
|
171
190
|
id
|
|
172
191
|
});
|
|
@@ -175,7 +194,7 @@ class S3BlobStorageConnector {
|
|
|
175
194
|
const key = urnParsed.namespaceSpecific(1);
|
|
176
195
|
const headCommand = new HeadObjectCommand({
|
|
177
196
|
Bucket: this._config.bucketName,
|
|
178
|
-
Key: key
|
|
197
|
+
Key: `${partitionKey ?? "root"}/${key}`
|
|
179
198
|
});
|
|
180
199
|
try {
|
|
181
200
|
await this._s3Client.send(headCommand);
|
|
@@ -188,15 +207,14 @@ class S3BlobStorageConnector {
|
|
|
188
207
|
}
|
|
189
208
|
const deleteCommand = new DeleteObjectCommand({
|
|
190
209
|
Bucket: this._config.bucketName,
|
|
191
|
-
Key: key
|
|
210
|
+
Key: `${partitionKey ?? "root"}/${key}`
|
|
192
211
|
});
|
|
193
212
|
await this._s3Client.send(deleteCommand);
|
|
194
213
|
return true;
|
|
195
214
|
}
|
|
196
215
|
catch (err) {
|
|
197
|
-
throw new GeneralError(
|
|
216
|
+
throw new GeneralError(S3BlobStorageConnector.CLASS_NAME, "removeBlobFailed", { id }, err);
|
|
198
217
|
}
|
|
199
218
|
}
|
|
200
219
|
}
|
|
201
|
-
|
|
202
|
-
export { S3BlobStorageConnector };
|
|
220
|
+
//# sourceMappingURL=s3BlobStorageConnector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3BlobStorageConnector.js","sourceRoot":"","sources":["../../src/s3BlobStorageConnector.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EACN,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,QAAQ,EACR,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAM1C;;;GAGG;AACH,MAAM,OAAO,sBAAsB;IAClC;;OAEG;IACI,MAAM,CAAU,SAAS,GAAW,IAAI,CAAC;IAEhD;;OAEG;IACI,MAAM,CAAU,UAAU,4BAA4C;IAE7E;;;OAGG;IACc,OAAO,CAAgC;IAExD;;;OAGG;IACc,oBAAoB,CAAY;IAEjD;;;OAGG;IACc,SAAS,CAAW;IAErC;;;OAGG;IACH,YAAY,OAAkD;QAC7D,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,UAAU,aAAmB,OAAO,CAAC,CAAC;QAC3E,MAAM,CAAC,MAAM,CACZ,sBAAsB,CAAC,UAAU,oBAEjC,OAAO,CAAC,MAAM,CACd,CAAC;QACF,MAAM,CAAC,WAAW,CACjB,sBAAsB,CAAC,UAAU,2BAEjC,OAAO,CAAC,MAAM,CAAC,MAAM,CACrB,CAAC;QACF,MAAM,CAAC,WAAW,CACjB,sBAAsB,CAAC,UAAU,+BAEjC,OAAO,CAAC,MAAM,CAAC,UAAU,CACzB,CAAC;QAEF,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,aAAa,CAAC;QAE1C,IAAI,WAAW,CAAC;QAChB,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YAC/C,MAAM,CAAC,WAAW,CACjB,sBAAsB,CAAC,UAAU,gCAEjC,OAAO,CAAC,MAAM,CAAC,WAAW,CAC1B,CAAC;YACF,MAAM,CAAC,WAAW,CACjB,sBAAsB,CAAC,UAAU,oCAEjC,OAAO,CAAC,MAAM,CAAC,eAAe,CAC9B,CAAC;YACF,WAAW,GAAG;gBACb,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW;gBACvC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe;aAC/C,CAAC;QACH,CAAC;QAED,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAExD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC;YAC7B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC3B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YAC/B,WAAW;YACX,cAAc,EAAE,IAAI;SACpB,CAAC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,SAAS;QACf,OAAO,sBAAsB,CAAC,UAAU,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,SAAS,CAAC,wBAAiC;QACvD,MAAM,WAAW,GAAG,gBAAgB,CAAC,WAAW,CAAoB,wBAAwB,CAAC,CAAC;QAE9F,IAAI,CAAC;YACJ,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;YACtD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClE,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAC7C,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,CACjD,CAAC;YAEF,IAAI,YAAY,EAAE,CAAC;gBAClB,MAAM,WAAW,EAAE,GAAG,CAAC;oBACtB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,sBAAsB,CAAC,UAAU;oBACzC,OAAO,EAAE,cAAc;oBACvB,IAAI,EAAE;wBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;qBAC/B;iBACD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,MAAM,WAAW,EAAE,GAAG,CAAC;oBACtB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,sBAAsB,CAAC,UAAU;oBACzC,OAAO,EAAE,gBAAgB;oBACzB,IAAI,EAAE;wBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;qBAC/B;iBACD,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACzF,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,WAAW,EAAE,GAAG,CAAC;gBACtB,KAAK,EAAE,OAAO;gBACd,MAAM,EAAE,sBAAsB,CAAC,UAAU;gBACzC,OAAO,EAAE,oBAAoB;gBAC7B,IAAI,EAAE;oBACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;iBAC/B;gBACD,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC;aAC/B,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,GAAG,CAAC,IAAgB;QAChC,MAAM,CAAC,UAAU,CAAC,sBAAsB,CAAC,UAAU,UAAgB,IAAI,CAAC,CAAC;QAEzE,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAErD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,CAAC;YACxD,MAAM,YAAY,GAAG,eAAe,CAAC,kBAAkB,CACtD,UAAU,EACV,IAAI,CAAC,oBAAoB,CACzB,CAAC;YAEF,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBAC/B,GAAG,EAAE,GAAG,YAAY,IAAI,MAAM,IAAI,EAAE,EAAE;gBACtC,IAAI,EAAE,IAAI;aACV,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEnC,OAAO,QAAQ,IAAI,GAAG,CAAC,sBAAsB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAC5F,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,GAAG,CAAC,EAAU;QAC1B,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,UAAU,QAAc,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAE1C,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,CAAC;QACxD,MAAM,YAAY,GAAG,eAAe,CAAC,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAE/F,IAAI,SAAS,CAAC,eAAe,EAAE,KAAK,sBAAsB,CAAC,SAAS,EAAE,CAAC;YACtE,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,UAAU,EAAE,mBAAmB,EAAE;gBAC9E,SAAS,EAAE,sBAAsB,CAAC,SAAS;gBAC3C,EAAE;aACF,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBAC/B,GAAG,EAAE,GAAG,YAAY,IAAI,MAAM,IAAI,GAAG,EAAE;aACvC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEpD,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnB,OAAO,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;YACnE,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,SAAS,CAAC,WAAW,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC7C,OAAO,SAAS,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,YAAY,CACrB,sBAAsB,CAAC,UAAU,EACjC,eAAe,EACf;gBACC,EAAE;gBACF,SAAS,EAAE,sBAAsB,CAAC,SAAS;aAC3C,EACD,GAAG,CACH,CAAC;QACH,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,MAAM,CAAC,EAAU;QAC7B,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,UAAU,QAAc,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAE1C,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,CAAC;QACxD,MAAM,YAAY,GAAG,eAAe,CAAC,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAE/F,IAAI,SAAS,CAAC,eAAe,EAAE,KAAK,sBAAsB,CAAC,SAAS,EAAE,CAAC;YACtE,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,UAAU,EAAE,mBAAmB,EAAE;gBAC9E,SAAS,EAAE,sBAAsB,CAAC,SAAS;gBAC3C,EAAE;aACF,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAE3C,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC;gBACzC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBAC/B,GAAG,EAAE,GAAG,YAAY,IAAI,MAAM,IAAI,GAAG,EAAE;aACvC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;oBAC9C,OAAO,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,KAAK,CAAC;YACb,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC;gBAC7C,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBAC/B,GAAG,EAAE,GAAG,YAAY,IAAI,MAAM,IAAI,GAAG,EAAE;aACvC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEzC,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,UAAU,EAAE,kBAAkB,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAC5F,CAAC;IACF,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport {\n\tCreateBucketCommand,\n\tDeleteObjectCommand,\n\tGetObjectCommand,\n\tHeadObjectCommand,\n\tListBucketsCommand,\n\tPutObjectCommand,\n\tS3Client\n} from \"@aws-sdk/client-s3\";\nimport type { IBlobStorageConnector } from \"@twin.org/blob-storage-models\";\nimport { ContextIdHelper, ContextIdStore } from \"@twin.org/context\";\nimport { BaseError, ComponentFactory, Converter, GeneralError, Guards, Urn } from \"@twin.org/core\";\nimport { Sha256 } from \"@twin.org/crypto\";\nimport type { ILoggingComponent } from \"@twin.org/logging-models\";\nimport { nameof } from \"@twin.org/nameof\";\nimport type { IS3BlobStorageConnectorConfig } from \"./models/IS3BlobStorageConnectorConfig.js\";\nimport type { IS3BlobStorageConnectorConstructorOptions } from \"./models/IS3BlobStorageConnectorConstructorOptions.js\";\n\n/**\n * Class for performing blob storage operations on S3.\n * See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/ for more information.\n */\nexport class S3BlobStorageConnector implements IBlobStorageConnector {\n\t/**\n\t * The namespace for the items.\n\t */\n\tpublic static readonly NAMESPACE: string = \"s3\";\n\n\t/**\n\t * Runtime name for the class.\n\t */\n\tpublic static readonly CLASS_NAME: string = nameof<S3BlobStorageConnector>();\n\n\t/**\n\t * The configuration for the connector.\n\t * @internal\n\t */\n\tprivate readonly _config: IS3BlobStorageConnectorConfig;\n\n\t/**\n\t * The keys to use from the context ids to create partitions.\n\t * @internal\n\t */\n\tprivate readonly _partitionContextIds?: string[];\n\n\t/**\n\t * The S3 client.\n\t * @internal\n\t */\n\tprivate readonly _s3Client: S3Client;\n\n\t/**\n\t * Create a new instance of S3BlobStorageConnector.\n\t * @param options The options for the connector.\n\t */\n\tconstructor(options: IS3BlobStorageConnectorConstructorOptions) {\n\t\tGuards.object(S3BlobStorageConnector.CLASS_NAME, nameof(options), options);\n\t\tGuards.object<IS3BlobStorageConnectorConfig>(\n\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\tnameof(options.config),\n\t\t\toptions.config\n\t\t);\n\t\tGuards.stringValue(\n\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\tnameof(options.config.region),\n\t\t\toptions.config.region\n\t\t);\n\t\tGuards.stringValue(\n\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\tnameof(options.config.bucketName),\n\t\t\toptions.config.bucketName\n\t\t);\n\n\t\toptions.config.authMode ??= \"credentials\";\n\n\t\tlet credentials;\n\t\tif (options.config.authMode === \"credentials\") {\n\t\t\tGuards.stringValue(\n\t\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\t\tnameof(options.config.accessKeyId),\n\t\t\t\toptions.config.accessKeyId\n\t\t\t);\n\t\t\tGuards.stringValue(\n\t\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\t\tnameof(options.config.secretAccessKey),\n\t\t\t\toptions.config.secretAccessKey\n\t\t\t);\n\t\t\tcredentials = {\n\t\t\t\taccessKeyId: options.config.accessKeyId,\n\t\t\t\tsecretAccessKey: options.config.secretAccessKey\n\t\t\t};\n\t\t}\n\n\t\tthis._partitionContextIds = options.partitionContextIds;\n\n\t\tthis._config = options.config;\n\t\tthis._s3Client = new S3Client({\n\t\t\tregion: this._config.region,\n\t\t\tendpoint: this._config.endpoint,\n\t\t\tcredentials,\n\t\t\tforcePathStyle: true\n\t\t});\n\t}\n\n\t/**\n\t * Returns the class name of the component.\n\t * @returns The class name of the component.\n\t */\n\tpublic className(): string {\n\t\treturn S3BlobStorageConnector.CLASS_NAME;\n\t}\n\n\t/**\n\t * Bootstrap the component by creating and initializing any resources it needs.\n\t * @param nodeLoggingComponentType The node logging component type.\n\t * @returns True if the bootstrapping process was successful.\n\t */\n\tpublic async bootstrap(nodeLoggingComponentType?: string): Promise<boolean> {\n\t\tconst nodeLogging = ComponentFactory.getIfExists<ILoggingComponent>(nodeLoggingComponentType);\n\n\t\ttry {\n\t\t\tconst listBucketsCommand = new ListBucketsCommand({});\n\t\t\tconst bucketsList = await this._s3Client.send(listBucketsCommand);\n\t\t\tconst bucketExists = bucketsList.Buckets?.some(\n\t\t\t\tbucket => bucket.Name === this._config.bucketName\n\t\t\t);\n\n\t\t\tif (bucketExists) {\n\t\t\t\tawait nodeLogging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: S3BlobStorageConnector.CLASS_NAME,\n\t\t\t\t\tmessage: \"bucketExists\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tbucket: this._config.bucketName\n\t\t\t\t\t}\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tawait nodeLogging?.log({\n\t\t\t\t\tlevel: \"info\",\n\t\t\t\t\tsource: S3BlobStorageConnector.CLASS_NAME,\n\t\t\t\t\tmessage: \"bucketCreating\",\n\t\t\t\t\tdata: {\n\t\t\t\t\t\tbucket: this._config.bucketName\n\t\t\t\t\t}\n\t\t\t\t});\n\t\t\t\tawait this._s3Client.send(new CreateBucketCommand({ Bucket: this._config.bucketName }));\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tawait nodeLogging?.log({\n\t\t\t\tlevel: \"error\",\n\t\t\t\tsource: S3BlobStorageConnector.CLASS_NAME,\n\t\t\t\tmessage: \"bucketCreateFailed\",\n\t\t\t\tdata: {\n\t\t\t\t\tbucket: this._config.bucketName\n\t\t\t\t},\n\t\t\t\terror: BaseError.fromError(err)\n\t\t\t});\n\n\t\t\treturn false;\n\t\t}\n\n\t\treturn true;\n\t}\n\n\t/**\n\t * Set the blob.\n\t * @param blob The data for the blob.\n\t * @returns The id of the stored blob in urn format.\n\t */\n\tpublic async set(blob: Uint8Array): Promise<string> {\n\t\tGuards.uint8Array(S3BlobStorageConnector.CLASS_NAME, nameof(blob), blob);\n\n\t\ttry {\n\t\t\tconst id = Converter.bytesToHex(Sha256.sum256(blob));\n\n\t\t\tconst contextIds = await ContextIdStore.getContextIds();\n\t\t\tconst partitionKey = ContextIdHelper.combinedContextKey(\n\t\t\t\tcontextIds,\n\t\t\t\tthis._partitionContextIds\n\t\t\t);\n\n\t\t\tconst command = new PutObjectCommand({\n\t\t\t\tBucket: this._config.bucketName,\n\t\t\t\tKey: `${partitionKey ?? \"root\"}/${id}`,\n\t\t\t\tBody: blob\n\t\t\t});\n\n\t\t\tawait this._s3Client.send(command);\n\n\t\t\treturn `blob:${new Urn(S3BlobStorageConnector.NAMESPACE, id).toString()}`;\n\t\t} catch (err) {\n\t\t\tthrow new GeneralError(S3BlobStorageConnector.CLASS_NAME, \"setBlobFailed\", undefined, err);\n\t\t}\n\t}\n\n\t/**\n\t * Get the blob.\n\t * @param id The id of the blob to get in urn format.\n\t * @returns The data for the blob if it can be found or undefined.\n\t */\n\tpublic async get(id: string): Promise<Uint8Array | undefined> {\n\t\tUrn.guard(S3BlobStorageConnector.CLASS_NAME, nameof(id), id);\n\t\tconst urnParsed = Urn.fromValidString(id);\n\n\t\tconst contextIds = await ContextIdStore.getContextIds();\n\t\tconst partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);\n\n\t\tif (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {\n\t\t\tthrow new GeneralError(S3BlobStorageConnector.CLASS_NAME, \"namespaceMismatch\", {\n\t\t\t\tnamespace: S3BlobStorageConnector.NAMESPACE,\n\t\t\t\tid\n\t\t\t});\n\t\t}\n\n\t\ttry {\n\t\t\tconst key = urnParsed.namespaceSpecific(1);\n\t\t\tconst command = new GetObjectCommand({\n\t\t\t\tBucket: this._config.bucketName,\n\t\t\t\tKey: `${partitionKey ?? \"root\"}/${key}`\n\t\t\t});\n\n\t\t\tconst response = await this._s3Client.send(command);\n\n\t\t\tif (response.Body) {\n\t\t\t\treturn new Uint8Array(await response.Body.transformToByteArray());\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tif (BaseError.isErrorName(err, \"NoSuchKey\")) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\t\t\tthrow new GeneralError(\n\t\t\t\tS3BlobStorageConnector.CLASS_NAME,\n\t\t\t\t\"getBlobFailed\",\n\t\t\t\t{\n\t\t\t\t\tid,\n\t\t\t\t\tnamespace: S3BlobStorageConnector.NAMESPACE\n\t\t\t\t},\n\t\t\t\terr\n\t\t\t);\n\t\t}\n\t}\n\n\t/**\n\t * Remove the blob.\n\t * @param id The id of the blob to remove in urn format.\n\t * @returns True if the blob was found.\n\t */\n\tpublic async remove(id: string): Promise<boolean> {\n\t\tUrn.guard(S3BlobStorageConnector.CLASS_NAME, nameof(id), id);\n\t\tconst urnParsed = Urn.fromValidString(id);\n\n\t\tconst contextIds = await ContextIdStore.getContextIds();\n\t\tconst partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);\n\n\t\tif (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {\n\t\t\tthrow new GeneralError(S3BlobStorageConnector.CLASS_NAME, \"namespaceMismatch\", {\n\t\t\t\tnamespace: S3BlobStorageConnector.NAMESPACE,\n\t\t\t\tid\n\t\t\t});\n\t\t}\n\n\t\ttry {\n\t\t\tconst key = urnParsed.namespaceSpecific(1);\n\n\t\t\tconst headCommand = new HeadObjectCommand({\n\t\t\t\tBucket: this._config.bucketName,\n\t\t\t\tKey: `${partitionKey ?? \"root\"}/${key}`\n\t\t\t});\n\n\t\t\ttry {\n\t\t\t\tawait this._s3Client.send(headCommand);\n\t\t\t} catch (error) {\n\t\t\t\tif (BaseError.isErrorName(error, \"NotFound\")) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tconst deleteCommand = new DeleteObjectCommand({\n\t\t\t\tBucket: this._config.bucketName,\n\t\t\t\tKey: `${partitionKey ?? \"root\"}/${key}`\n\t\t\t});\n\n\t\t\tawait this._s3Client.send(deleteCommand);\n\n\t\t\treturn true;\n\t\t} catch (err) {\n\t\t\tthrow new GeneralError(S3BlobStorageConnector.CLASS_NAME, \"removeBlobFailed\", { id }, err);\n\t\t}\n\t}\n}\n"]}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from "./s3BlobStorageConnector";
|
|
2
|
-
export * from "./models/IS3BlobStorageConnectorConfig";
|
|
3
|
-
export * from "./models/IS3BlobStorageConnectorConstructorOptions";
|
|
1
|
+
export * from "./s3BlobStorageConnector.js";
|
|
2
|
+
export * from "./models/IS3BlobStorageConnectorConfig.js";
|
|
3
|
+
export * from "./models/IS3BlobStorageConnectorConstructorOptions.js";
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
import type { IS3BlobStorageConnectorConfig } from "./IS3BlobStorageConnectorConfig";
|
|
1
|
+
import type { IS3BlobStorageConnectorConfig } from "./IS3BlobStorageConnectorConfig.js";
|
|
2
2
|
/**
|
|
3
3
|
* Options for the S3 Blob Storage Connector constructor.
|
|
4
4
|
*/
|
|
5
5
|
export interface IS3BlobStorageConnectorConstructorOptions {
|
|
6
|
+
/**
|
|
7
|
+
* The keys to use from the context ids to create partitions.
|
|
8
|
+
*/
|
|
9
|
+
partitionContextIds?: string[];
|
|
6
10
|
/**
|
|
7
11
|
* The configuration for the connector.
|
|
8
12
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { IBlobStorageConnector } from "@twin.org/blob-storage-models";
|
|
2
|
-
import type { IS3BlobStorageConnectorConstructorOptions } from "./models/IS3BlobStorageConnectorConstructorOptions";
|
|
2
|
+
import type { IS3BlobStorageConnectorConstructorOptions } from "./models/IS3BlobStorageConnectorConstructorOptions.js";
|
|
3
3
|
/**
|
|
4
4
|
* Class for performing blob storage operations on S3.
|
|
5
5
|
* See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/ for more information.
|
|
@@ -12,12 +12,17 @@ export declare class S3BlobStorageConnector implements IBlobStorageConnector {
|
|
|
12
12
|
/**
|
|
13
13
|
* Runtime name for the class.
|
|
14
14
|
*/
|
|
15
|
-
readonly CLASS_NAME: string;
|
|
15
|
+
static readonly CLASS_NAME: string;
|
|
16
16
|
/**
|
|
17
17
|
* Create a new instance of S3BlobStorageConnector.
|
|
18
18
|
* @param options The options for the connector.
|
|
19
19
|
*/
|
|
20
20
|
constructor(options: IS3BlobStorageConnectorConstructorOptions);
|
|
21
|
+
/**
|
|
22
|
+
* Returns the class name of the component.
|
|
23
|
+
* @returns The class name of the component.
|
|
24
|
+
*/
|
|
25
|
+
className(): string;
|
|
21
26
|
/**
|
|
22
27
|
* Bootstrap the component by creating and initializing any resources it needs.
|
|
23
28
|
* @param nodeLoggingComponentType The node logging component type.
|
package/docs/changelog.md
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
# @twin.org/blob-storage-connector-aws-s3 - Changelog
|
|
2
2
|
|
|
3
|
+
## [0.0.3-next.1](https://github.com/twinfoundation/blob-storage/compare/blob-storage-connector-aws-s3-v0.0.3-next.0...blob-storage-connector-aws-s3-v0.0.3-next.1) (2025-11-11)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* add AWS pod authentication mode ([c34b11c](https://github.com/twinfoundation/blob-storage/commit/c34b11cb32d3310a9e59840341a2b0b4221bc780))
|
|
9
|
+
* add context id features ([#30](https://github.com/twinfoundation/blob-storage/issues/30)) ([fbf1c92](https://github.com/twinfoundation/blob-storage/commit/fbf1c9276424c841ef5ef3f4de8469ab3fba7e9c))
|
|
10
|
+
* add validate-locales ([f20fcec](https://github.com/twinfoundation/blob-storage/commit/f20fceced91e39a0c9edb770b2e43ce944c92f3c))
|
|
11
|
+
* Blob storage connector for AWS S3 ([#7](https://github.com/twinfoundation/blob-storage/issues/7)) ([1d7f1e7](https://github.com/twinfoundation/blob-storage/commit/1d7f1e7d323926f7f31229d38eb5de429f6e1554))
|
|
12
|
+
* eslint migration to flat config ([e4239dd](https://github.com/twinfoundation/blob-storage/commit/e4239dd1c721955cff7f0357255d2bba15319972))
|
|
13
|
+
* update dependencies ([56f0094](https://github.com/twinfoundation/blob-storage/commit/56f0094b68d8bd22864cd899ac1b61d95540f719))
|
|
14
|
+
* update framework core ([ff339fe](https://github.com/twinfoundation/blob-storage/commit/ff339fe7e3f09ddff429907834bdf43617e9c05e))
|
|
15
|
+
* update to support fully qualified data type names ([3297d69](https://github.com/twinfoundation/blob-storage/commit/3297d69d332058b0f0141002087f56ba230620e1))
|
|
16
|
+
* use shared store mechanism ([#12](https://github.com/twinfoundation/blob-storage/issues/12)) ([cae8110](https://github.com/twinfoundation/blob-storage/commit/cae8110681847a1ac4fcac968b8196694e49c320))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### Dependencies
|
|
20
|
+
|
|
21
|
+
* The following workspace dependencies were updated
|
|
22
|
+
* dependencies
|
|
23
|
+
* @twin.org/blob-storage-models bumped from 0.0.3-next.0 to 0.0.3-next.1
|
|
24
|
+
|
|
25
|
+
## [0.0.2-next.5](https://github.com/twinfoundation/blob-storage/compare/blob-storage-connector-aws-s3-v0.0.2-next.4...blob-storage-connector-aws-s3-v0.0.2-next.5) (2025-10-09)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### Features
|
|
29
|
+
|
|
30
|
+
* add validate-locales ([f20fcec](https://github.com/twinfoundation/blob-storage/commit/f20fceced91e39a0c9edb770b2e43ce944c92f3c))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
### Dependencies
|
|
34
|
+
|
|
35
|
+
* The following workspace dependencies were updated
|
|
36
|
+
* dependencies
|
|
37
|
+
* @twin.org/blob-storage-models bumped from 0.0.2-next.4 to 0.0.2-next.5
|
|
38
|
+
|
|
3
39
|
## [0.0.2-next.4](https://github.com/twinfoundation/blob-storage/compare/blob-storage-connector-aws-s3-v0.0.2-next.3...blob-storage-connector-aws-s3-v0.0.2-next.4) (2025-10-02)
|
|
4
40
|
|
|
5
41
|
|
|
@@ -39,15 +39,29 @@ The namespace for the items.
|
|
|
39
39
|
|
|
40
40
|
### CLASS\_NAME
|
|
41
41
|
|
|
42
|
-
> `readonly` **CLASS\_NAME**: `string`
|
|
42
|
+
> `readonly` `static` **CLASS\_NAME**: `string`
|
|
43
43
|
|
|
44
44
|
Runtime name for the class.
|
|
45
45
|
|
|
46
|
+
## Methods
|
|
47
|
+
|
|
48
|
+
### className()
|
|
49
|
+
|
|
50
|
+
> **className**(): `string`
|
|
51
|
+
|
|
52
|
+
Returns the class name of the component.
|
|
53
|
+
|
|
54
|
+
#### Returns
|
|
55
|
+
|
|
56
|
+
`string`
|
|
57
|
+
|
|
58
|
+
The class name of the component.
|
|
59
|
+
|
|
46
60
|
#### Implementation of
|
|
47
61
|
|
|
48
|
-
`IBlobStorageConnector.
|
|
62
|
+
`IBlobStorageConnector.className`
|
|
49
63
|
|
|
50
|
-
|
|
64
|
+
***
|
|
51
65
|
|
|
52
66
|
### bootstrap()
|
|
53
67
|
|
|
@@ -103,7 +117,7 @@ The id of the stored blob in urn format.
|
|
|
103
117
|
|
|
104
118
|
### get()
|
|
105
119
|
|
|
106
|
-
> **get**(`id`): `Promise`\<`
|
|
120
|
+
> **get**(`id`): `Promise`\<`Uint8Array`\<`ArrayBufferLike`\> \| `undefined`\>
|
|
107
121
|
|
|
108
122
|
Get the blob.
|
|
109
123
|
|
|
@@ -117,7 +131,7 @@ The id of the blob to get in urn format.
|
|
|
117
131
|
|
|
118
132
|
#### Returns
|
|
119
133
|
|
|
120
|
-
`Promise`\<`
|
|
134
|
+
`Promise`\<`Uint8Array`\<`ArrayBufferLike`\> \| `undefined`\>
|
|
121
135
|
|
|
122
136
|
The data for the blob if it can be found or undefined.
|
|
123
137
|
|
|
@@ -4,6 +4,14 @@ Options for the S3 Blob Storage Connector constructor.
|
|
|
4
4
|
|
|
5
5
|
## Properties
|
|
6
6
|
|
|
7
|
+
### partitionContextIds?
|
|
8
|
+
|
|
9
|
+
> `optional` **partitionContextIds**: `string`[]
|
|
10
|
+
|
|
11
|
+
The keys to use from the context ids to create partitions.
|
|
12
|
+
|
|
13
|
+
***
|
|
14
|
+
|
|
7
15
|
### config
|
|
8
16
|
|
|
9
17
|
> **config**: [`IS3BlobStorageConnectorConfig`](IS3BlobStorageConnectorConfig.md)
|
package/locales/en.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@twin.org/blob-storage-connector-aws-s3",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.3-next.1",
|
|
4
4
|
"description": "Blob Storage connector implementation using AWS S3",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -14,27 +14,26 @@
|
|
|
14
14
|
"node": ">=20.0.0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@aws-sdk/client-s3": "3.
|
|
18
|
-
"@twin.org/blob-storage-models": "0.0.
|
|
17
|
+
"@aws-sdk/client-s3": "3.928.0",
|
|
18
|
+
"@twin.org/blob-storage-models": "0.0.3-next.1",
|
|
19
|
+
"@twin.org/context": "next",
|
|
19
20
|
"@twin.org/core": "next",
|
|
20
21
|
"@twin.org/crypto": "next",
|
|
21
22
|
"@twin.org/logging-models": "next",
|
|
22
23
|
"@twin.org/nameof": "next"
|
|
23
24
|
},
|
|
24
|
-
"main": "./dist/
|
|
25
|
-
"module": "./dist/esm/index.mjs",
|
|
25
|
+
"main": "./dist/es/index.js",
|
|
26
26
|
"types": "./dist/types/index.d.ts",
|
|
27
27
|
"exports": {
|
|
28
28
|
".": {
|
|
29
29
|
"types": "./dist/types/index.d.ts",
|
|
30
|
-
"
|
|
31
|
-
"
|
|
30
|
+
"import": "./dist/es/index.js",
|
|
31
|
+
"default": "./dist/es/index.js"
|
|
32
32
|
},
|
|
33
33
|
"./locales/*.json": "./locales/*.json"
|
|
34
34
|
},
|
|
35
35
|
"files": [
|
|
36
|
-
"dist/
|
|
37
|
-
"dist/esm",
|
|
36
|
+
"dist/es",
|
|
38
37
|
"dist/types",
|
|
39
38
|
"locales",
|
|
40
39
|
"docs"
|
|
@@ -53,5 +52,9 @@
|
|
|
53
52
|
"connector",
|
|
54
53
|
"adapter",
|
|
55
54
|
"integration"
|
|
56
|
-
]
|
|
55
|
+
],
|
|
56
|
+
"bugs": {
|
|
57
|
+
"url": "git+https://github.com/twinfoundation/blob-storage/issues"
|
|
58
|
+
},
|
|
59
|
+
"homepage": "https://twindev.org"
|
|
57
60
|
}
|
package/dist/cjs/index.cjs
DELETED
|
@@ -1,204 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var clientS3 = require('@aws-sdk/client-s3');
|
|
4
|
-
var core = require('@twin.org/core');
|
|
5
|
-
var crypto = require('@twin.org/crypto');
|
|
6
|
-
|
|
7
|
-
// Copyright 2024 IOTA Stiftung.
|
|
8
|
-
// SPDX-License-Identifier: Apache-2.0.
|
|
9
|
-
/**
|
|
10
|
-
* Class for performing blob storage operations on S3.
|
|
11
|
-
* See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/ for more information.
|
|
12
|
-
*/
|
|
13
|
-
class S3BlobStorageConnector {
|
|
14
|
-
/**
|
|
15
|
-
* The namespace for the items.
|
|
16
|
-
*/
|
|
17
|
-
static NAMESPACE = "s3";
|
|
18
|
-
/**
|
|
19
|
-
* Runtime name for the class.
|
|
20
|
-
*/
|
|
21
|
-
CLASS_NAME = "S3BlobStorageConnector";
|
|
22
|
-
/**
|
|
23
|
-
* The configuration for the connector.
|
|
24
|
-
* @internal
|
|
25
|
-
*/
|
|
26
|
-
_config;
|
|
27
|
-
/**
|
|
28
|
-
* The S3 client.
|
|
29
|
-
* @internal
|
|
30
|
-
*/
|
|
31
|
-
_s3Client;
|
|
32
|
-
/**
|
|
33
|
-
* Create a new instance of S3BlobStorageConnector.
|
|
34
|
-
* @param options The options for the connector.
|
|
35
|
-
*/
|
|
36
|
-
constructor(options) {
|
|
37
|
-
core.Guards.object(this.CLASS_NAME, "options", options);
|
|
38
|
-
core.Guards.object(this.CLASS_NAME, "options.config", options.config);
|
|
39
|
-
core.Guards.stringValue(this.CLASS_NAME, "options.config.region", options.config.region);
|
|
40
|
-
core.Guards.stringValue(this.CLASS_NAME, "options.config.bucketName", options.config.bucketName);
|
|
41
|
-
options.config.authMode ??= "credentials";
|
|
42
|
-
let credentials;
|
|
43
|
-
if (options.config.authMode === "credentials") {
|
|
44
|
-
core.Guards.stringValue(this.CLASS_NAME, "options.config.accessKeyId", options.config.accessKeyId);
|
|
45
|
-
core.Guards.stringValue(this.CLASS_NAME, "options.config.secretAccessKey", options.config.secretAccessKey);
|
|
46
|
-
credentials = {
|
|
47
|
-
accessKeyId: options.config.accessKeyId,
|
|
48
|
-
secretAccessKey: options.config.secretAccessKey
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
this._config = options.config;
|
|
52
|
-
this._s3Client = new clientS3.S3Client({
|
|
53
|
-
region: this._config.region,
|
|
54
|
-
endpoint: this._config.endpoint,
|
|
55
|
-
credentials,
|
|
56
|
-
forcePathStyle: true
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Bootstrap the component by creating and initializing any resources it needs.
|
|
61
|
-
* @param nodeLoggingComponentType The node logging component type.
|
|
62
|
-
* @returns True if the bootstrapping process was successful.
|
|
63
|
-
*/
|
|
64
|
-
async bootstrap(nodeLoggingComponentType) {
|
|
65
|
-
const nodeLogging = core.ComponentFactory.getIfExists(nodeLoggingComponentType);
|
|
66
|
-
try {
|
|
67
|
-
await nodeLogging?.log({
|
|
68
|
-
level: "info",
|
|
69
|
-
source: this.CLASS_NAME,
|
|
70
|
-
message: "bucketCreating",
|
|
71
|
-
data: {
|
|
72
|
-
bucket: this._config.bucketName
|
|
73
|
-
}
|
|
74
|
-
});
|
|
75
|
-
const listBucketsCommand = new clientS3.ListBucketsCommand({});
|
|
76
|
-
const bucketsList = await this._s3Client.send(listBucketsCommand);
|
|
77
|
-
const bucketExists = bucketsList.Buckets?.some(bucket => bucket.Name === this._config.bucketName);
|
|
78
|
-
if (bucketExists) {
|
|
79
|
-
await nodeLogging?.log({
|
|
80
|
-
level: "info",
|
|
81
|
-
source: this.CLASS_NAME,
|
|
82
|
-
message: "bucketExists",
|
|
83
|
-
data: {
|
|
84
|
-
bucket: this._config.bucketName
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
else {
|
|
89
|
-
await this._s3Client.send(new clientS3.CreateBucketCommand({ Bucket: this._config.bucketName }));
|
|
90
|
-
await nodeLogging?.log({
|
|
91
|
-
level: "info",
|
|
92
|
-
source: this.CLASS_NAME,
|
|
93
|
-
message: "bucketCreated",
|
|
94
|
-
data: {
|
|
95
|
-
bucket: this._config.bucketName
|
|
96
|
-
}
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
catch (err) {
|
|
101
|
-
await nodeLogging?.log({
|
|
102
|
-
level: "error",
|
|
103
|
-
source: this.CLASS_NAME,
|
|
104
|
-
message: "bucketCreateFailed",
|
|
105
|
-
data: {
|
|
106
|
-
bucket: this._config.bucketName
|
|
107
|
-
},
|
|
108
|
-
error: core.BaseError.fromError(err)
|
|
109
|
-
});
|
|
110
|
-
return false;
|
|
111
|
-
}
|
|
112
|
-
return true;
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Set the blob.
|
|
116
|
-
* @param blob The data for the blob.
|
|
117
|
-
* @returns The id of the stored blob in urn format.
|
|
118
|
-
*/
|
|
119
|
-
async set(blob) {
|
|
120
|
-
core.Guards.uint8Array(this.CLASS_NAME, "blob", blob);
|
|
121
|
-
try {
|
|
122
|
-
const id = core.Converter.bytesToHex(crypto.Sha256.sum256(blob));
|
|
123
|
-
const command = new clientS3.PutObjectCommand({
|
|
124
|
-
Bucket: this._config.bucketName,
|
|
125
|
-
Key: id,
|
|
126
|
-
Body: blob
|
|
127
|
-
});
|
|
128
|
-
await this._s3Client.send(command);
|
|
129
|
-
return `blob:${new core.Urn(S3BlobStorageConnector.NAMESPACE, id).toString()}`;
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
throw new core.GeneralError(this.CLASS_NAME, "setBlobFailed", undefined, err);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Get the blob.
|
|
137
|
-
* @param id The id of the blob to get in urn format.
|
|
138
|
-
* @returns The data for the blob if it can be found or undefined.
|
|
139
|
-
*/
|
|
140
|
-
async get(id) {
|
|
141
|
-
core.Urn.guard(this.CLASS_NAME, "id", id);
|
|
142
|
-
const urnParsed = core.Urn.fromValidString(id);
|
|
143
|
-
if (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {
|
|
144
|
-
throw new core.GeneralError(this.CLASS_NAME, "namespaceMismatch", {
|
|
145
|
-
namespace: S3BlobStorageConnector.NAMESPACE,
|
|
146
|
-
id
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
try {
|
|
150
|
-
const key = urnParsed.namespaceSpecific(1);
|
|
151
|
-
const command = new clientS3.GetObjectCommand({
|
|
152
|
-
Bucket: this._config.bucketName,
|
|
153
|
-
Key: key
|
|
154
|
-
});
|
|
155
|
-
const response = await this._s3Client.send(command);
|
|
156
|
-
if (response.Body) {
|
|
157
|
-
return new Uint8Array(await response.Body.transformToByteArray());
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
catch { }
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Remove the blob.
|
|
164
|
-
* @param id The id of the blob to remove in urn format.
|
|
165
|
-
* @returns True if the blob was found.
|
|
166
|
-
*/
|
|
167
|
-
async remove(id) {
|
|
168
|
-
core.Urn.guard(this.CLASS_NAME, "id", id);
|
|
169
|
-
const urnParsed = core.Urn.fromValidString(id);
|
|
170
|
-
if (urnParsed.namespaceMethod() !== S3BlobStorageConnector.NAMESPACE) {
|
|
171
|
-
throw new core.GeneralError(this.CLASS_NAME, "namespaceMismatch", {
|
|
172
|
-
namespace: S3BlobStorageConnector.NAMESPACE,
|
|
173
|
-
id
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
try {
|
|
177
|
-
const key = urnParsed.namespaceSpecific(1);
|
|
178
|
-
const headCommand = new clientS3.HeadObjectCommand({
|
|
179
|
-
Bucket: this._config.bucketName,
|
|
180
|
-
Key: key
|
|
181
|
-
});
|
|
182
|
-
try {
|
|
183
|
-
await this._s3Client.send(headCommand);
|
|
184
|
-
}
|
|
185
|
-
catch (error) {
|
|
186
|
-
if (core.BaseError.isErrorName(error, "NotFound")) {
|
|
187
|
-
return false;
|
|
188
|
-
}
|
|
189
|
-
throw error;
|
|
190
|
-
}
|
|
191
|
-
const deleteCommand = new clientS3.DeleteObjectCommand({
|
|
192
|
-
Bucket: this._config.bucketName,
|
|
193
|
-
Key: key
|
|
194
|
-
});
|
|
195
|
-
await this._s3Client.send(deleteCommand);
|
|
196
|
-
return true;
|
|
197
|
-
}
|
|
198
|
-
catch (err) {
|
|
199
|
-
throw new core.GeneralError(this.CLASS_NAME, "removeBlobFailed", { id }, err);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
exports.S3BlobStorageConnector = S3BlobStorageConnector;
|