@adobe/helix-config 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/package.json +13 -5
- package/src/index.js +1 -1
- package/src/storage/config-store.js +86 -0
- package/src/storage/index.js +12 -0
- package/src/storage/storage.js +612 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
# [2.3.0](https://github.com/adobe/helix-config/compare/v2.2.0...v2.3.0) (2024-04-04)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* add storage api ([#36](https://github.com/adobe/helix-config/issues/36)) ([85475ce](https://github.com/adobe/helix-config/commit/85475ce91ebc42bd6f8f85e2834b54d23b8db944))
|
|
7
|
+
|
|
1
8
|
# [2.2.0](https://github.com/adobe/helix-config/compare/v2.1.0...v2.2.0) (2024-03-28)
|
|
2
9
|
|
|
3
10
|
|
package/package.json
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adobe/helix-config",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "Helix Config",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
7
7
|
"exports": {
|
|
8
|
-
".": "./src/index.js"
|
|
8
|
+
".": "./src/index.js",
|
|
9
|
+
"./storage": "./src/storage/index.js"
|
|
9
10
|
},
|
|
10
11
|
"type": "module",
|
|
11
12
|
"scripts": {
|
|
@@ -43,16 +44,23 @@
|
|
|
43
44
|
"json-schema-to-typescript": "13.1.2",
|
|
44
45
|
"junit-report-builder": "3.2.1",
|
|
45
46
|
"lint-staged": "15.2.2",
|
|
46
|
-
"mocha": "10.
|
|
47
|
+
"mocha": "10.4.0",
|
|
47
48
|
"mocha-multi-reporters": "1.5.1",
|
|
48
|
-
"
|
|
49
|
+
"nock": "13.5.4",
|
|
50
|
+
"semantic-release": "23.0.0",
|
|
51
|
+
"xml2js": "0.6.2"
|
|
49
52
|
},
|
|
50
53
|
"lint-staged": {
|
|
51
54
|
"*.js": "eslint",
|
|
52
55
|
"*.cjs": "eslint"
|
|
53
56
|
},
|
|
54
57
|
"dependencies": {
|
|
58
|
+
"@adobe/fetch": "4.1.1",
|
|
55
59
|
"@adobe/helix-shared-config": "10.4.0",
|
|
56
|
-
"@adobe/helix-shared-
|
|
60
|
+
"@adobe/helix-shared-process-queue": "3.0.3",
|
|
61
|
+
"@adobe/helix-shared-utils": "3.0.1",
|
|
62
|
+
"@aws-sdk/client-s3": "3.540.0",
|
|
63
|
+
"@aws-sdk/node-http-handler": "3.374.0",
|
|
64
|
+
"mime": "4.0.1"
|
|
57
65
|
}
|
|
58
66
|
}
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright
|
|
2
|
+
* Copyright 2024 Adobe. All rights reserved.
|
|
3
3
|
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
5
|
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2023 Adobe. All rights reserved.
|
|
3
|
+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
|
+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
*
|
|
7
|
+
* Unless required by applicable law or agreed to in writing, software distributed under
|
|
8
|
+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
|
|
9
|
+
* OF ANY KIND, either express or implied. See the License for the specific language
|
|
10
|
+
* governing permissions and limitations under the License.
|
|
11
|
+
*/
|
|
12
|
+
import { Response } from '@adobe/fetch';
|
|
13
|
+
import { HelixStorage } from './storage.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* General purpose config store.
|
|
17
|
+
*/
|
|
18
|
+
export class ConfigStore {
|
|
19
|
+
/**
|
|
20
|
+
* @param {string} org the org id
|
|
21
|
+
* @param {string} type store type (org, sites, profiles, secrets, users)
|
|
22
|
+
* @param {string} name config name
|
|
23
|
+
* @param {string }ref the ref
|
|
24
|
+
*/
|
|
25
|
+
constructor(org, type = 'org', name = 'config') {
|
|
26
|
+
if (!org) {
|
|
27
|
+
throw new Error('org required');
|
|
28
|
+
}
|
|
29
|
+
if (org.includes('/') || type.includes('/') || name.includes('/')) {
|
|
30
|
+
throw new Error('orgId, type and name must not contain slashes');
|
|
31
|
+
}
|
|
32
|
+
this.org = org;
|
|
33
|
+
this.type = type;
|
|
34
|
+
this.name = name;
|
|
35
|
+
this.key = type === org
|
|
36
|
+
? `/orgs/${this.org}/config.json`
|
|
37
|
+
: `/orgs/${this.org}/${this.type}/${this.name}.json`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async create(ctx) {
|
|
41
|
+
const storage = HelixStorage.fromContext(ctx).configBus();
|
|
42
|
+
if (await storage.head(this.key)) {
|
|
43
|
+
return new Response('', { status: 409 });
|
|
44
|
+
}
|
|
45
|
+
await storage.put(this.key, JSON.stringify(ctx.data), 'application/json');
|
|
46
|
+
await this.purge(ctx, null, ctx.data);
|
|
47
|
+
return new Response('', { status: 201 });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async read(ctx) {
|
|
51
|
+
const storage = HelixStorage.fromContext(ctx).configBus();
|
|
52
|
+
const buf = await storage.get(this.key);
|
|
53
|
+
if (!buf) {
|
|
54
|
+
return new Response('', { status: 404 });
|
|
55
|
+
}
|
|
56
|
+
return new Response(buf, {
|
|
57
|
+
headers: {
|
|
58
|
+
'content-type': 'application/json',
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async update(ctx) {
|
|
64
|
+
const storage = HelixStorage.fromContext(ctx).configBus();
|
|
65
|
+
const buf = await storage.get(this.key);
|
|
66
|
+
const old = buf ? JSON.parse(buf) : null;
|
|
67
|
+
await storage.put(this.key, JSON.stringify(ctx.data), 'application/json');
|
|
68
|
+
await this.purge(ctx, old, ctx.data);
|
|
69
|
+
return new Response('', { status: 202 });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async remove(ctx) {
|
|
73
|
+
const storage = HelixStorage.fromContext(ctx).configBus();
|
|
74
|
+
const buf = await storage.get(this.key);
|
|
75
|
+
if (buf) {
|
|
76
|
+
await storage.remove(this.key);
|
|
77
|
+
await this.purge(ctx, JSON.parse(buf), null);
|
|
78
|
+
}
|
|
79
|
+
return new Response('', { status: 204 });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// eslint-disable-next-line class-methods-use-this,no-unused-vars
|
|
83
|
+
async purge(ctx, oldConfig, newConfig) {
|
|
84
|
+
// override in subclass
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2024 Adobe. All rights reserved.
|
|
3
|
+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
|
+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
*
|
|
7
|
+
* Unless required by applicable law or agreed to in writing, software distributed under
|
|
8
|
+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
|
|
9
|
+
* OF ANY KIND, either express or implied. See the License for the specific language
|
|
10
|
+
* governing permissions and limitations under the License.
|
|
11
|
+
*/
|
|
12
|
+
export { ConfigStore } from './config-store.js';
|
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2022 Adobe. All rights reserved.
|
|
3
|
+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
* you may not use this file except in compliance with the License. You may obtain a copy
|
|
5
|
+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
*
|
|
7
|
+
* Unless required by applicable law or agreed to in writing, software distributed under
|
|
8
|
+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
|
|
9
|
+
* OF ANY KIND, either express or implied. See the License for the specific language
|
|
10
|
+
* governing permissions and limitations under the License.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/* eslint-disable max-classes-per-file,no-param-reassign */
|
|
14
|
+
import { Agent } from 'node:https';
|
|
15
|
+
import { promisify } from 'util';
|
|
16
|
+
import zlib from 'zlib';
|
|
17
|
+
import {
|
|
18
|
+
CopyObjectCommand,
|
|
19
|
+
DeleteObjectCommand,
|
|
20
|
+
DeleteObjectsCommand,
|
|
21
|
+
GetObjectCommand,
|
|
22
|
+
HeadObjectCommand,
|
|
23
|
+
ListObjectsV2Command,
|
|
24
|
+
PutObjectCommand,
|
|
25
|
+
S3Client,
|
|
26
|
+
} from '@aws-sdk/client-s3';
|
|
27
|
+
import { NodeHttpHandler } from '@aws-sdk/node-http-handler';
|
|
28
|
+
import { Response } from '@adobe/fetch';
|
|
29
|
+
import mime from 'mime';
|
|
30
|
+
import processQueue from '@adobe/helix-shared-process-queue';
|
|
31
|
+
|
|
32
|
+
const gzip = promisify(zlib.gzip);
|
|
33
|
+
const gunzip = promisify(zlib.gunzip);
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @typedef {import('@aws-sdk/client-s3').CommandInput} CommandInput
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @typedef ObjectInfo
|
|
41
|
+
* @property {string} key
|
|
42
|
+
* @property {string} path the path to the object, w/o the prefix
|
|
43
|
+
* @property {string} lastModified
|
|
44
|
+
* @property {number} contentLength
|
|
45
|
+
* @property {string} contentType
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* @callback ObjectFilter
|
|
50
|
+
* @param {ObjectInfo} info of the object to filter
|
|
51
|
+
* @returns {boolean} {@code true} if the object is accepted
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* result object headers
|
|
56
|
+
*/
|
|
57
|
+
const AWS_META_HEADERS = [
|
|
58
|
+
'CacheControl',
|
|
59
|
+
'ContentType',
|
|
60
|
+
'ContentEncoding',
|
|
61
|
+
'Expires',
|
|
62
|
+
];
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Sanitizes the input key or path and returns a bucket relative key (without leading / ).
|
|
66
|
+
* @param {string} keyOrPath
|
|
67
|
+
* @returns {string}
|
|
68
|
+
*/
|
|
69
|
+
function sanitizeKey(keyOrPath) {
|
|
70
|
+
if (keyOrPath.charAt(0) === '/') {
|
|
71
|
+
return keyOrPath.substring(1);
|
|
72
|
+
}
|
|
73
|
+
return keyOrPath;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Bucket class
|
|
78
|
+
*/
|
|
79
|
+
class Bucket {
|
|
80
|
+
constructor(opts) {
|
|
81
|
+
Object.assign(this, {
|
|
82
|
+
_s3: opts.s3,
|
|
83
|
+
_r2: opts.r2,
|
|
84
|
+
_log: opts.log,
|
|
85
|
+
_clients: [opts.s3],
|
|
86
|
+
_bucket: opts.bucketId,
|
|
87
|
+
});
|
|
88
|
+
if (opts.r2) {
|
|
89
|
+
this._clients.push(opts.r2);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
get client() {
|
|
94
|
+
return this._s3;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
get bucket() {
|
|
98
|
+
return this._bucket;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
get log() {
|
|
102
|
+
return this._log;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Return an object contents.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} key object key
|
|
109
|
+
* @param {object} [meta] output object to receive metadata if specified
|
|
110
|
+
* @returns object contents as a Buffer or null if no found.
|
|
111
|
+
* @throws an error if the object could not be loaded due to an unexpected error.
|
|
112
|
+
*/
|
|
113
|
+
async get(key, meta = null) {
|
|
114
|
+
const { log } = this;
|
|
115
|
+
const input = {
|
|
116
|
+
Bucket: this.bucket,
|
|
117
|
+
Key: sanitizeKey(key),
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
const result = await this.client.send(new GetObjectCommand(input));
|
|
122
|
+
log.info(`object downloaded from: ${input.Bucket}/${input.Key}`);
|
|
123
|
+
|
|
124
|
+
const buf = await new Response(result.Body, {}).buffer();
|
|
125
|
+
if (meta) {
|
|
126
|
+
Object.assign(meta, result.Metadata);
|
|
127
|
+
for (const name of AWS_META_HEADERS) {
|
|
128
|
+
if (name in result) {
|
|
129
|
+
meta[name] = result[name];
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (result.ContentEncoding === 'gzip') {
|
|
134
|
+
return await gunzip(buf);
|
|
135
|
+
}
|
|
136
|
+
return buf;
|
|
137
|
+
} catch (e) {
|
|
138
|
+
/* c8 ignore next 3 */
|
|
139
|
+
if (e.$metadata.httpStatusCode !== 404) {
|
|
140
|
+
throw e;
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async head(path) {
|
|
147
|
+
const input = {
|
|
148
|
+
Bucket: this._bucket,
|
|
149
|
+
Key: sanitizeKey(path),
|
|
150
|
+
};
|
|
151
|
+
try {
|
|
152
|
+
const result = await this.client.send(new HeadObjectCommand(input));
|
|
153
|
+
this.log.info(`Object metadata downloaded from: ${input.Bucket}/${input.Key}`);
|
|
154
|
+
return result;
|
|
155
|
+
} catch (e) {
|
|
156
|
+
/* c8 ignore next 3 */
|
|
157
|
+
if (e.$metadata.httpStatusCode !== 404) {
|
|
158
|
+
throw e;
|
|
159
|
+
}
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Return an object's metadata.
|
|
166
|
+
*
|
|
167
|
+
* @param {string} key object key
|
|
168
|
+
* @returns object metadata or null
|
|
169
|
+
* @throws an error if the object could not be loaded due to an unexpected error.
|
|
170
|
+
*/
|
|
171
|
+
async metadata(key) {
|
|
172
|
+
const res = await this.head(key);
|
|
173
|
+
return res?.Metadata;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Internal helper for sending a command to both S3 and R2 clients.
|
|
178
|
+
* @param {function} CommandConstructor constructor of command to send to the client
|
|
179
|
+
* @param {CommandInput} input command input
|
|
180
|
+
* @returns {Promise<*>} the command result
|
|
181
|
+
*/
|
|
182
|
+
async sendToS3andR2(CommandConstructor, input) {
|
|
183
|
+
// send cmd to s3 and r2 (mirror) in parallel
|
|
184
|
+
const tasks = this._clients.map((c) => c.send(new CommandConstructor(input)));
|
|
185
|
+
const result = await Promise.allSettled(tasks);
|
|
186
|
+
|
|
187
|
+
const rejected = result.filter(({ status }) => status === 'rejected');
|
|
188
|
+
if (!rejected.length) {
|
|
189
|
+
return result[0].value;
|
|
190
|
+
} else {
|
|
191
|
+
// at least 1 cmd failed
|
|
192
|
+
/* c8 ignore next */
|
|
193
|
+
const type = result[0].status === 'rejected' ? 'S3' : 'R2';
|
|
194
|
+
const err = rejected[0].reason;
|
|
195
|
+
err.message = `[${type}] ${err.message}`;
|
|
196
|
+
throw err;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Store an object contents, along with metadata.
|
|
202
|
+
*
|
|
203
|
+
* @param {string} path object key
|
|
204
|
+
* @param {Buffer|string} body data to store
|
|
205
|
+
* @param {string} [contentType] content type. defaults to 'application/octet-stream'
|
|
206
|
+
* @param {object} [meta] metadata to store with the object. defaults to '{}'
|
|
207
|
+
* @param {boolean} [compress = true]
|
|
208
|
+
* @returns result obtained from S3
|
|
209
|
+
*/
|
|
210
|
+
async put(path, body, contentType = 'application/octet-stream', meta = {}, compress = true) {
|
|
211
|
+
const input = {
|
|
212
|
+
Body: body,
|
|
213
|
+
Bucket: this.bucket,
|
|
214
|
+
ContentType: contentType,
|
|
215
|
+
Metadata: meta,
|
|
216
|
+
Key: sanitizeKey(path),
|
|
217
|
+
};
|
|
218
|
+
if (compress) {
|
|
219
|
+
input.ContentEncoding = 'gzip';
|
|
220
|
+
input.Body = await gzip(body);
|
|
221
|
+
}
|
|
222
|
+
// write to s3 and r2 (mirror) in parallel
|
|
223
|
+
const res = await this.sendToS3andR2(PutObjectCommand, input);
|
|
224
|
+
this.log.info(`object uploaded to: ${input.Bucket}/${input.Key}`);
|
|
225
|
+
return res;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Updates the metadata
|
|
230
|
+
* @param {string} path
|
|
231
|
+
* @param {object} meta
|
|
232
|
+
* @param {object} opts
|
|
233
|
+
* @returns {Promise<*>}
|
|
234
|
+
*/
|
|
235
|
+
async putMeta(path, meta, opts = {}) {
|
|
236
|
+
const key = sanitizeKey(path);
|
|
237
|
+
const input = {
|
|
238
|
+
Bucket: this._bucket,
|
|
239
|
+
Key: key,
|
|
240
|
+
CopySource: `${this.bucket}/${key}`,
|
|
241
|
+
Metadata: meta,
|
|
242
|
+
MetadataDirective: 'REPLACE',
|
|
243
|
+
...opts,
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
// write to s3 and r2 (mirror) in parallel
|
|
247
|
+
const result = await this.sendToS3andR2(CopyObjectCommand, input);
|
|
248
|
+
this.log.info(`Metadata updated for: ${input.CopySource}`);
|
|
249
|
+
return result;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Copy an object in the same bucket.
|
|
254
|
+
*
|
|
255
|
+
* @param {string} src source key
|
|
256
|
+
* @param {string} dst destination key
|
|
257
|
+
* @returns result obtained from S3
|
|
258
|
+
*/
|
|
259
|
+
async copy(src, dst) {
|
|
260
|
+
const input = {
|
|
261
|
+
Bucket: this.bucket,
|
|
262
|
+
CopySource: `${this.bucket}/${sanitizeKey(src)}`,
|
|
263
|
+
Key: sanitizeKey(dst),
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
try {
|
|
267
|
+
// write to s3 and r2 (mirror) in parallel
|
|
268
|
+
await this.sendToS3andR2(CopyObjectCommand, input);
|
|
269
|
+
this.log.info(`object copied from ${input.CopySource} to: ${input.Bucket}/${input.Key}`);
|
|
270
|
+
} catch (e) {
|
|
271
|
+
/* c8 ignore next 3 */
|
|
272
|
+
if (e.Code !== 'NoSuchKey') {
|
|
273
|
+
throw e;
|
|
274
|
+
}
|
|
275
|
+
const e2 = new Error(`source does not exist: ${input.CopySource}`);
|
|
276
|
+
e2.status = 404;
|
|
277
|
+
throw e2;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Remove object(s)
|
|
283
|
+
*
|
|
284
|
+
* @param {string|string[]} path source key(s)
|
|
285
|
+
* @returns result obtained from S3
|
|
286
|
+
*/
|
|
287
|
+
async remove(path) {
|
|
288
|
+
if (Array.isArray(path)) {
|
|
289
|
+
const input = {
|
|
290
|
+
Bucket: this.bucket,
|
|
291
|
+
Delete: {
|
|
292
|
+
Objects: path.map((p) => ({ Key: sanitizeKey(p) })),
|
|
293
|
+
},
|
|
294
|
+
};
|
|
295
|
+
// delete on s3 and r2 (mirror) in parallel
|
|
296
|
+
try {
|
|
297
|
+
const result = await this.sendToS3andR2(DeleteObjectsCommand, input);
|
|
298
|
+
this.log.info(`${result.Deleted.length} objects deleted from bucket ${input.Bucket}.`);
|
|
299
|
+
return result;
|
|
300
|
+
} catch (e) {
|
|
301
|
+
const msg = `removing ${input.Delete.Objects.length} objects from bucket ${input.Bucket} failed: ${e.message}`;
|
|
302
|
+
this.log.error(msg);
|
|
303
|
+
const e2 = new Error(msg);
|
|
304
|
+
e2.status = e.$metadata?.httpStatusCode;
|
|
305
|
+
throw e2;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const input = {
|
|
310
|
+
Bucket: this.bucket,
|
|
311
|
+
Key: sanitizeKey(path),
|
|
312
|
+
};
|
|
313
|
+
// delete on s3 and r2 (mirror) in parallel
|
|
314
|
+
try {
|
|
315
|
+
const result = await this.sendToS3andR2(DeleteObjectCommand, input);
|
|
316
|
+
this.log.info(`object deleted: ${input.Bucket}/${input.Key}`);
|
|
317
|
+
return result;
|
|
318
|
+
} catch (e) {
|
|
319
|
+
const msg = `removing ${input.Bucket}/${input.Key} from storage failed: ${e.message}`;
|
|
320
|
+
this.log.error(msg);
|
|
321
|
+
const e2 = new Error(msg);
|
|
322
|
+
e2.status = e.$metadata.httpStatusCode;
|
|
323
|
+
throw e2;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Returns a list of object below the given prefix
|
|
329
|
+
* @param {string} prefix
|
|
330
|
+
* @returns {Promise<ObjectInfo[]>}
|
|
331
|
+
*/
|
|
332
|
+
async list(prefix) {
|
|
333
|
+
let ContinuationToken;
|
|
334
|
+
const objects = [];
|
|
335
|
+
do {
|
|
336
|
+
// eslint-disable-next-line no-await-in-loop
|
|
337
|
+
const result = await this.client.send(new ListObjectsV2Command({
|
|
338
|
+
Bucket: this.bucket,
|
|
339
|
+
ContinuationToken,
|
|
340
|
+
Prefix: prefix,
|
|
341
|
+
}));
|
|
342
|
+
ContinuationToken = result.IsTruncated ? result.NextContinuationToken : '';
|
|
343
|
+
(result.Contents || []).forEach((content) => {
|
|
344
|
+
const key = content.Key;
|
|
345
|
+
objects.push({
|
|
346
|
+
key,
|
|
347
|
+
lastModified: content.LastModified,
|
|
348
|
+
contentLength: content.Size,
|
|
349
|
+
contentType: mime.getType(key),
|
|
350
|
+
path: `${key.substring(prefix.length)}`,
|
|
351
|
+
});
|
|
352
|
+
});
|
|
353
|
+
} while (ContinuationToken);
|
|
354
|
+
return objects;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async listFolders(prefix) {
|
|
358
|
+
let ContinuationToken;
|
|
359
|
+
const folders = [];
|
|
360
|
+
do {
|
|
361
|
+
// eslint-disable-next-line no-await-in-loop
|
|
362
|
+
const result = await this.client.send(new ListObjectsV2Command({
|
|
363
|
+
Bucket: this.bucket,
|
|
364
|
+
ContinuationToken,
|
|
365
|
+
Prefix: prefix,
|
|
366
|
+
Delimiter: '/',
|
|
367
|
+
}));
|
|
368
|
+
ContinuationToken = result.IsTruncated ? result.NextContinuationToken : '';
|
|
369
|
+
(result.CommonPrefixes || []).forEach(({ Prefix }) => {
|
|
370
|
+
folders.push(Prefix);
|
|
371
|
+
});
|
|
372
|
+
} while (ContinuationToken);
|
|
373
|
+
return folders;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Copies the tree below src to dst.
|
|
378
|
+
* @param {string} src Source prefix
|
|
379
|
+
* @param {string} dst Destination prefix
|
|
380
|
+
* @param {ObjectFilter} filter Filter function
|
|
381
|
+
* @returns {Promise<*[]>}
|
|
382
|
+
*/
|
|
383
|
+
async copyDeep(src, dst, filter = () => true) {
|
|
384
|
+
const { log } = this;
|
|
385
|
+
const tasks = [];
|
|
386
|
+
const Prefix = sanitizeKey(src);
|
|
387
|
+
const dstPrefix = sanitizeKey(dst);
|
|
388
|
+
this.log.info(`fetching list of files to copy ${this.bucket}/${Prefix} => ${dstPrefix}`);
|
|
389
|
+
(await this.list(Prefix)).forEach((obj) => {
|
|
390
|
+
const {
|
|
391
|
+
path, key, contentLength, contentType,
|
|
392
|
+
} = obj;
|
|
393
|
+
if (filter(obj)) {
|
|
394
|
+
tasks.push({
|
|
395
|
+
src: key,
|
|
396
|
+
path,
|
|
397
|
+
contentLength,
|
|
398
|
+
contentType,
|
|
399
|
+
dst: `${dstPrefix}${path}`,
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
let errors = 0;
|
|
405
|
+
const changes = [];
|
|
406
|
+
await processQueue(tasks, async (task) => {
|
|
407
|
+
log.info(`copy to ${task.dst}`);
|
|
408
|
+
const input = {
|
|
409
|
+
Bucket: this.bucket,
|
|
410
|
+
CopySource: `${this.bucket}/${task.src}`,
|
|
411
|
+
Key: task.dst,
|
|
412
|
+
};
|
|
413
|
+
try {
|
|
414
|
+
// write to s3 and r2 (mirror) in parallel
|
|
415
|
+
await this.sendToS3andR2(CopyObjectCommand, input);
|
|
416
|
+
changes.push(task);
|
|
417
|
+
} catch (e) {
|
|
418
|
+
// at least 1 cmd failed
|
|
419
|
+
log.warn(`error while copying ${task.dst}: ${e}`);
|
|
420
|
+
errors += 1;
|
|
421
|
+
}
|
|
422
|
+
}, 64);
|
|
423
|
+
log.info(`copied ${changes.length} files to ${dst} (${errors} errors)`);
|
|
424
|
+
return changes;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
async rmdir(src) {
|
|
428
|
+
const { log } = this;
|
|
429
|
+
src = sanitizeKey(src);
|
|
430
|
+
log.info(`fetching list of files to delete from ${this.bucket}/${src}`);
|
|
431
|
+
const items = await this.list(src);
|
|
432
|
+
|
|
433
|
+
let oks = 0;
|
|
434
|
+
let errors = 0;
|
|
435
|
+
await processQueue(items, async (item) => {
|
|
436
|
+
const { key } = item;
|
|
437
|
+
log.info(`deleting ${this.bucket}/${key}`);
|
|
438
|
+
const input = {
|
|
439
|
+
Bucket: this.bucket,
|
|
440
|
+
Key: key,
|
|
441
|
+
};
|
|
442
|
+
|
|
443
|
+
try {
|
|
444
|
+
// delete on s3 and r2 (mirror) in parallel
|
|
445
|
+
await this.sendToS3andR2(DeleteObjectCommand, input);
|
|
446
|
+
oks += 1;
|
|
447
|
+
} catch (e) {
|
|
448
|
+
// at least 1 cmd failed
|
|
449
|
+
log.warn(`error while deleting ${key}: ${e.$metadata.httpStatusCode}`);
|
|
450
|
+
errors += 1;
|
|
451
|
+
}
|
|
452
|
+
}, 64);
|
|
453
|
+
log.info(`deleted ${oks} files (${errors} errors)`);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* The Helix Storage provides a factory for simplified bucket operations to S3 and R2
|
|
459
|
+
*/
|
|
460
|
+
export class HelixStorage {
|
|
461
|
+
static fromContext(context) {
|
|
462
|
+
if (!context.attributes.storage) {
|
|
463
|
+
const {
|
|
464
|
+
HELIX_HTTP_CONNECTION_TIMEOUT: connectionTimeout = 5000,
|
|
465
|
+
HELIX_HTTP_SOCKET_TIMEOUT: socketTimeout = 15000,
|
|
466
|
+
HELIX_HTTP_S3_KEEP_ALIVE: keepAlive,
|
|
467
|
+
CLOUDFLARE_ACCOUNT_ID: r2AccountId,
|
|
468
|
+
CLOUDFLARE_R2_ACCESS_KEY_ID: r2AccessKeyId,
|
|
469
|
+
CLOUDFLARE_R2_SECRET_ACCESS_KEY: r2SecretAccessKey,
|
|
470
|
+
} = context.env;
|
|
471
|
+
|
|
472
|
+
context.attributes.storage = new HelixStorage({
|
|
473
|
+
connectionTimeout,
|
|
474
|
+
socketTimeout,
|
|
475
|
+
r2AccountId,
|
|
476
|
+
r2AccessKeyId,
|
|
477
|
+
r2SecretAccessKey,
|
|
478
|
+
keepAlive: String(keepAlive) === 'true',
|
|
479
|
+
log: context.log,
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
return context.attributes.storage;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
static AWS_S3_SYSTEM_HEADERS = {
|
|
486
|
+
'content-type': 'ContentType',
|
|
487
|
+
'content-disposition': 'ContentDisposition',
|
|
488
|
+
'content-encoding': 'ContentEncoding',
|
|
489
|
+
'content-language': 'ContentLanguage',
|
|
490
|
+
};
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Create an instance
|
|
494
|
+
*
|
|
495
|
+
* @param {object} [opts] options
|
|
496
|
+
* @param {string} [opts.region] AWS region
|
|
497
|
+
* @param {string} [opts.accessKeyId] AWS access key
|
|
498
|
+
* @param {string} [opts.secretAccessKey] AWS secret access key
|
|
499
|
+
* @param {strong} [opts.r2AccountId]
|
|
500
|
+
* @param {strong} [opts.r2AccessKeyId]
|
|
501
|
+
* @param {strong} [opts.r2SecretAccessKey]
|
|
502
|
+
* @param {object} [opts.log] logger
|
|
503
|
+
*/
|
|
504
|
+
constructor(opts = {}) {
|
|
505
|
+
const {
|
|
506
|
+
region, accessKeyId, secretAccessKey,
|
|
507
|
+
connectionTimeout, socketTimeout,
|
|
508
|
+
r2AccountId, r2AccessKeyId, r2SecretAccessKey,
|
|
509
|
+
log = console,
|
|
510
|
+
keepAlive = true,
|
|
511
|
+
} = opts;
|
|
512
|
+
|
|
513
|
+
if (region && accessKeyId && secretAccessKey) {
|
|
514
|
+
log.debug('Creating S3Client with credentials');
|
|
515
|
+
this._s3 = new S3Client({
|
|
516
|
+
region,
|
|
517
|
+
credentials: {
|
|
518
|
+
accessKeyId,
|
|
519
|
+
secretAccessKey,
|
|
520
|
+
},
|
|
521
|
+
requestHandler: new NodeHttpHandler({
|
|
522
|
+
httpsAgent: new Agent({
|
|
523
|
+
keepAlive,
|
|
524
|
+
}),
|
|
525
|
+
connectionTimeout,
|
|
526
|
+
socketTimeout,
|
|
527
|
+
}),
|
|
528
|
+
});
|
|
529
|
+
} else {
|
|
530
|
+
log.debug('Creating S3Client without credentials');
|
|
531
|
+
this._s3 = new S3Client({
|
|
532
|
+
requestHandler: new NodeHttpHandler({
|
|
533
|
+
httpsAgent: new Agent({
|
|
534
|
+
keepAlive,
|
|
535
|
+
}),
|
|
536
|
+
connectionTimeout,
|
|
537
|
+
socketTimeout,
|
|
538
|
+
}),
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
// initializing the R2 client which is used for mirroring all S3 writes to R2
|
|
543
|
+
log.debug('Creating R2 S3Client');
|
|
544
|
+
this._r2 = new S3Client({
|
|
545
|
+
endpoint: `https://${r2AccountId}.r2.cloudflarestorage.com`,
|
|
546
|
+
region: 'us-east-1', // https://github.com/aws/aws-sdk-js-v3/issues/1845#issuecomment-754832210
|
|
547
|
+
credentials: {
|
|
548
|
+
accessKeyId: r2AccessKeyId,
|
|
549
|
+
secretAccessKey: r2SecretAccessKey,
|
|
550
|
+
},
|
|
551
|
+
requestHandler: new NodeHttpHandler({
|
|
552
|
+
httpsAgent: new Agent({
|
|
553
|
+
keepAlive,
|
|
554
|
+
}),
|
|
555
|
+
connectionTimeout,
|
|
556
|
+
socketTimeout,
|
|
557
|
+
}),
|
|
558
|
+
});
|
|
559
|
+
this._log = log;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* creates a bucket instance that allows to perform storage related operations.
|
|
564
|
+
* @param bucketId
|
|
565
|
+
* @returns {Bucket}
|
|
566
|
+
*/
|
|
567
|
+
bucket(bucketId) {
|
|
568
|
+
if (!this._s3) {
|
|
569
|
+
throw new Error('storage already closed.');
|
|
570
|
+
}
|
|
571
|
+
if (!bucketId) {
|
|
572
|
+
throw new Error('bucketId is required.');
|
|
573
|
+
}
|
|
574
|
+
return new Bucket({
|
|
575
|
+
bucketId,
|
|
576
|
+
s3: this._s3,
|
|
577
|
+
r2: this._r2,
|
|
578
|
+
log: this._log,
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* @returns {Bucket}
|
|
584
|
+
*/
|
|
585
|
+
contentBus() {
|
|
586
|
+
return this.bucket('helix-content-bus');
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* @returns {Bucket}
|
|
591
|
+
*/
|
|
592
|
+
codeBus() {
|
|
593
|
+
return this.bucket('helix-code-bus');
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* @returns {Bucket}
|
|
598
|
+
*/
|
|
599
|
+
configBus() {
|
|
600
|
+
return this.bucket('helix-config-bus');
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Close this storage. Destroys the S3 client used.
|
|
605
|
+
*/
|
|
606
|
+
close() {
|
|
607
|
+
this._s3?.destroy();
|
|
608
|
+
this._r2?.destroy();
|
|
609
|
+
delete this._s3;
|
|
610
|
+
delete this._r2;
|
|
611
|
+
}
|
|
612
|
+
}
|