@atproto/aws 0.2.28 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/s3.d.ts +11 -2
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +75 -81
- package/dist/s3.js.map +1 -1
- package/package.json +8 -7
- package/src/s3.ts +98 -59
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @atproto/aws
|
|
2
2
|
|
|
3
|
+
## 0.2.30
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [[`8dd77bad2`](https://github.com/bluesky-social/atproto/commit/8dd77bad2fdee20e39d3787198d960c19d8df3d0)]:
|
|
8
|
+
- @atproto/repo@0.8.10
|
|
9
|
+
|
|
10
|
+
## 0.2.29
|
|
11
|
+
|
|
12
|
+
### Patch Changes
|
|
13
|
+
|
|
14
|
+
- [#4169](https://github.com/bluesky-social/atproto/pull/4169) [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Optimistically attempt to move files before checking for their existence, resulting in faster `makePermanent` calls
|
|
15
|
+
|
|
16
|
+
- [#4169](https://github.com/bluesky-social/atproto/pull/4169) [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099) Thanks [@matthieusieben](https://github.com/matthieusieben)! - `S3BlobStore`'s `deleteMany` now supports any number of input (and will process deletes by chunks internally)
|
|
17
|
+
|
|
18
|
+
- [#4169](https://github.com/bluesky-social/atproto/pull/4169) [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Update `@aws-sdk` dependencies
|
|
19
|
+
|
|
20
|
+
- [#4169](https://github.com/bluesky-social/atproto/pull/4169) [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Set a timeout (of 10 seconds by default) on every `S3BlobStore` requests
|
|
21
|
+
|
|
22
|
+
- Updated dependencies [[`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099), [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099), [`055a413fb`](https://github.com/bluesky-social/atproto/commit/055a413fba4fab510ec899377154f1204ab12099)]:
|
|
23
|
+
- @atproto/repo@0.8.9
|
|
24
|
+
- @atproto/common-web@0.4.3
|
|
25
|
+
- @atproto/common@0.4.12
|
|
26
|
+
- @atproto/crypto@0.4.4
|
|
27
|
+
|
|
3
28
|
## 0.2.28
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
package/dist/s3.d.ts
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
import stream from 'node:stream';
|
|
2
|
-
import
|
|
2
|
+
import { S3ClientConfig } from '@aws-sdk/client-s3';
|
|
3
3
|
import { CID } from 'multiformats/cid';
|
|
4
4
|
import { BlobStore } from '@atproto/repo';
|
|
5
5
|
export type S3Config = {
|
|
6
6
|
bucket: string;
|
|
7
|
+
/**
|
|
8
|
+
* The maximum time any request to S3 (including individual blob chunks
|
|
9
|
+
* uploads) can take, in milliseconds.
|
|
10
|
+
*/
|
|
11
|
+
requestTimeoutMs?: number;
|
|
12
|
+
/**
|
|
13
|
+
* The maximum total time a blob upload can take, in milliseconds.
|
|
14
|
+
*/
|
|
7
15
|
uploadTimeoutMs?: number;
|
|
8
|
-
} & Omit<
|
|
16
|
+
} & Omit<S3ClientConfig, 'apiVersion' | 'requestHandler'>;
|
|
9
17
|
export declare class S3BlobStore implements BlobStore {
|
|
10
18
|
did: string;
|
|
11
19
|
private client;
|
|
@@ -17,6 +25,7 @@ export declare class S3BlobStore implements BlobStore {
|
|
|
17
25
|
private getTmpPath;
|
|
18
26
|
private getStoredPath;
|
|
19
27
|
private getQuarantinedPath;
|
|
28
|
+
private uploadBytes;
|
|
20
29
|
putTemp(bytes: Uint8Array | stream.Readable): Promise<string>;
|
|
21
30
|
makePermanent(key: string, cid: CID): Promise<void>;
|
|
22
31
|
putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>;
|
package/dist/s3.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,
|
|
1
|
+
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAiB,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAElE,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAGtC,OAAO,EAAqB,SAAS,EAAE,MAAM,eAAe,CAAA;AAE5D,MAAM,MAAM,QAAQ,GAAG;IACrB,MAAM,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB,GAAG,IAAI,CAAC,cAAc,EAAE,YAAY,GAAG,gBAAgB,CAAC,CAAA;AAEzD,qBAAa,WAAY,YAAW,SAAS;IAMlC,GAAG,EAAE,MAAM;IALpB,OAAO,CAAC,MAAM,CAAI;IAClB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,eAAe,CAAQ;gBAGtB,GAAG,EAAE,MAAM,EAClB,GAAG,EAAE,QAAQ;IAqBf,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,IAClB,KAAK,MAAM;IAKrB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,kBAAkB;YAIZ,WAAW;IAoCnB,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAM7D,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBnD,YAAY,CAChB,GAAG,EAAE,GAAG,EACR,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,QAAQ,GAClC,OAAO,CAAC,IAAI,CAAC;IAIV,UAAU,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnC,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;YAO7B,SAAS;IAYjB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC;IAKvC,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;IAK7C,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/B,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAatC,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC;IAIrC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;YAI9B,MAAM;YAYN,SAAS;YAOT,cAAc;YASd,IAAI;CA8BnB;AAED,eAAe,WAAW,CAAA"}
|
package/dist/s3.js
CHANGED
|
@@ -1,45 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
3
|
exports.S3BlobStore = void 0;
|
|
37
|
-
const
|
|
4
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
38
5
|
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
6
|
+
const common_web_1 = require("@atproto/common-web");
|
|
39
7
|
const crypto_1 = require("@atproto/crypto");
|
|
40
8
|
const repo_1 = require("@atproto/repo");
|
|
41
|
-
// @NOTE we use Upload rather than client.putObject because stream
|
|
42
|
-
// length is not known in advance. See also aws/aws-sdk-js-v3#2348.
|
|
43
9
|
class S3BlobStore {
|
|
44
10
|
constructor(did, cfg) {
|
|
45
11
|
Object.defineProperty(this, "did", {
|
|
@@ -66,12 +32,17 @@ class S3BlobStore {
|
|
|
66
32
|
writable: true,
|
|
67
33
|
value: void 0
|
|
68
34
|
});
|
|
69
|
-
const { bucket, uploadTimeoutMs, ...rest } = cfg;
|
|
35
|
+
const { bucket, uploadTimeoutMs = 10 * common_web_1.SECOND, requestTimeoutMs = uploadTimeoutMs, ...rest } = cfg;
|
|
70
36
|
this.bucket = bucket;
|
|
71
|
-
this.uploadTimeoutMs = uploadTimeoutMs
|
|
72
|
-
this.client = new
|
|
37
|
+
this.uploadTimeoutMs = uploadTimeoutMs;
|
|
38
|
+
this.client = new client_s3_1.S3({
|
|
73
39
|
...rest,
|
|
74
40
|
apiVersion: '2006-03-01',
|
|
41
|
+
// Ensures that all requests timeout under "requestTimeoutMs".
|
|
42
|
+
//
|
|
43
|
+
// @NOTE This will also apply to the upload of each individual chunk
|
|
44
|
+
// when using Upload from @aws-sdk/lib-storage.
|
|
45
|
+
requestHandler: { requestTimeout: requestTimeoutMs },
|
|
75
46
|
});
|
|
76
47
|
}
|
|
77
48
|
static creator(cfg) {
|
|
@@ -91,17 +62,23 @@ class S3BlobStore {
|
|
|
91
62
|
getQuarantinedPath(cid) {
|
|
92
63
|
return `quarantine/${this.did}/${cid.toString()}`;
|
|
93
64
|
}
|
|
94
|
-
async
|
|
95
|
-
|
|
96
|
-
//
|
|
65
|
+
async uploadBytes(path, bytes) {
|
|
66
|
+
// @NOTE we use Upload rather than client.putObject because stream length is
|
|
67
|
+
// not known in advance. See also aws/aws-sdk-js-v3#2348.
|
|
68
|
+
//
|
|
69
|
+
// See also https://github.com/aws/aws-sdk-js-v3/issues/6426, wherein Upload
|
|
70
|
+
// may hang the s3 connection under certain circumstances. We don't have a
|
|
71
|
+
// good way to avoid this, so we use timeouts defensively on all s3
|
|
72
|
+
// requests.
|
|
73
|
+
const abortSignal = AbortSignal.timeout(this.uploadTimeoutMs);
|
|
97
74
|
const abortController = new AbortController();
|
|
98
|
-
|
|
75
|
+
abortSignal.addEventListener('abort', () => abortController.abort());
|
|
99
76
|
const upload = new lib_storage_1.Upload({
|
|
100
77
|
client: this.client,
|
|
101
78
|
params: {
|
|
102
79
|
Bucket: this.bucket,
|
|
103
80
|
Body: bytes,
|
|
104
|
-
Key:
|
|
81
|
+
Key: path,
|
|
105
82
|
},
|
|
106
83
|
// @ts-ignore native implementation fine in node >=15
|
|
107
84
|
abortController,
|
|
@@ -109,44 +86,46 @@ class S3BlobStore {
|
|
|
109
86
|
try {
|
|
110
87
|
await upload.done();
|
|
111
88
|
}
|
|
112
|
-
|
|
113
|
-
|
|
89
|
+
catch (err) {
|
|
90
|
+
// Translate aws-sdk's abort error to something more specific
|
|
91
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
92
|
+
throw new Error('Blob upload timed out', { cause: err });
|
|
93
|
+
}
|
|
94
|
+
throw err;
|
|
114
95
|
}
|
|
96
|
+
}
|
|
97
|
+
async putTemp(bytes) {
|
|
98
|
+
const key = this.genKey();
|
|
99
|
+
await this.uploadBytes(this.getTmpPath(key), bytes);
|
|
115
100
|
return key;
|
|
116
101
|
}
|
|
117
102
|
async makePermanent(key, cid) {
|
|
118
|
-
|
|
119
|
-
|
|
103
|
+
try {
|
|
104
|
+
// @NOTE we normally call this method when we know the file is temporary.
|
|
105
|
+
// Because of this, we optimistically move the file, allowing to make
|
|
106
|
+
// fewer network requests in the happy path.
|
|
120
107
|
await this.move({
|
|
121
108
|
from: this.getTmpPath(key),
|
|
122
109
|
to: this.getStoredPath(cid),
|
|
123
110
|
});
|
|
124
111
|
}
|
|
125
|
-
|
|
126
|
-
//
|
|
127
|
-
|
|
112
|
+
catch (err) {
|
|
113
|
+
// If the optimistic move failed because the temp file was not found,
|
|
114
|
+
// check if the permanent file already exists. If it does, we can assume
|
|
115
|
+
// that another process made the file permanent concurrently, and we can
|
|
116
|
+
// no-op.
|
|
117
|
+
if (err instanceof repo_1.BlobNotFoundError) {
|
|
118
|
+
// Blob was not found from temp storage...
|
|
119
|
+
const alreadyHas = await this.hasStored(cid);
|
|
120
|
+
// already saved, so we no-op
|
|
121
|
+
if (alreadyHas)
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
throw err;
|
|
128
125
|
}
|
|
129
126
|
}
|
|
130
127
|
async putPermanent(cid, bytes) {
|
|
131
|
-
|
|
132
|
-
const abortController = new AbortController();
|
|
133
|
-
const timeout = setTimeout(() => abortController.abort(), this.uploadTimeoutMs);
|
|
134
|
-
const upload = new lib_storage_1.Upload({
|
|
135
|
-
client: this.client,
|
|
136
|
-
params: {
|
|
137
|
-
Bucket: this.bucket,
|
|
138
|
-
Body: bytes,
|
|
139
|
-
Key: this.getStoredPath(cid),
|
|
140
|
-
},
|
|
141
|
-
// @ts-ignore native implementation fine in node >=15
|
|
142
|
-
abortController,
|
|
143
|
-
});
|
|
144
|
-
try {
|
|
145
|
-
await upload.done();
|
|
146
|
-
}
|
|
147
|
-
finally {
|
|
148
|
-
clearTimeout(timeout);
|
|
149
|
-
}
|
|
128
|
+
await this.uploadBytes(this.getStoredPath(cid), bytes);
|
|
150
129
|
}
|
|
151
130
|
async quarantine(cid) {
|
|
152
131
|
await this.move({
|
|
@@ -184,8 +163,18 @@ class S3BlobStore {
|
|
|
184
163
|
await this.deleteKey(this.getStoredPath(cid));
|
|
185
164
|
}
|
|
186
165
|
async deleteMany(cids) {
|
|
187
|
-
const
|
|
188
|
-
|
|
166
|
+
const errors = [];
|
|
167
|
+
for (const chunk of (0, common_web_1.chunkArray)(cids, 500)) {
|
|
168
|
+
try {
|
|
169
|
+
const keys = chunk.map((cid) => this.getStoredPath(cid));
|
|
170
|
+
await this.deleteManyKeys(keys);
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
errors.push(err);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (errors.length)
|
|
177
|
+
throw (0, common_web_1.aggregateErrors)(errors);
|
|
189
178
|
}
|
|
190
179
|
async hasStored(cid) {
|
|
191
180
|
return this.hasKey(this.getStoredPath(cid));
|
|
@@ -226,24 +215,29 @@ class S3BlobStore {
|
|
|
226
215
|
CopySource: `${this.bucket}/${keys.from}`,
|
|
227
216
|
Key: keys.to,
|
|
228
217
|
});
|
|
218
|
+
}
|
|
219
|
+
catch (cause) {
|
|
220
|
+
if (cause instanceof client_s3_1.NoSuchKey) {
|
|
221
|
+
// Already deleted, possibly by a concurrently running process
|
|
222
|
+
throw new repo_1.BlobNotFoundError(undefined, { cause });
|
|
223
|
+
}
|
|
224
|
+
throw cause;
|
|
225
|
+
}
|
|
226
|
+
try {
|
|
229
227
|
await this.client.deleteObject({
|
|
230
228
|
Bucket: this.bucket,
|
|
231
229
|
Key: keys.from,
|
|
232
230
|
});
|
|
233
231
|
}
|
|
234
232
|
catch (err) {
|
|
235
|
-
|
|
233
|
+
if (err instanceof client_s3_1.NoSuchKey) {
|
|
234
|
+
// Already deleted, possibly by a concurrently running process
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
throw err;
|
|
236
238
|
}
|
|
237
239
|
}
|
|
238
240
|
}
|
|
239
241
|
exports.S3BlobStore = S3BlobStore;
|
|
240
|
-
const handleErr = (err) => {
|
|
241
|
-
if (err?.['Code'] === 'NoSuchKey') {
|
|
242
|
-
throw new repo_1.BlobNotFoundError();
|
|
243
|
-
}
|
|
244
|
-
else {
|
|
245
|
-
throw err;
|
|
246
|
-
}
|
|
247
|
-
};
|
|
248
242
|
exports.default = S3BlobStore;
|
|
249
243
|
//# sourceMappingURL=s3.js.map
|
package/dist/s3.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":";;;AACA,kDAAkE;AAClE,sDAA6C;AAE7C,oDAAyE;AACzE,4CAA2C;AAC3C,wCAA4D;AAe5D,MAAa,WAAW;IAKtB,YACS,GAAW,EAClB,GAAa;QADb;;;;mBAAO,GAAG;WAAQ;QALZ;;;;;WAAU;QACV;;;;;WAAc;QACd;;;;;WAAuB;QAM7B,MAAM,EACJ,MAAM,EACN,eAAe,GAAG,EAAE,GAAG,mBAAM,EAC7B,gBAAgB,GAAG,eAAe,EAClC,GAAG,IAAI,EACR,GAAG,GAAG,CAAA;QACP,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,cAAE,CAAC;YACnB,GAAG,IAAI;YACP,UAAU,EAAE,YAAY;YACxB,8DAA8D;YAC9D,EAAE;YACF,oEAAoE;YACpE,+CAA+C;YAC/C,cAAc,EAAE,EAAE,cAAc,EAAE,gBAAgB,EAAE;SACrD,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,GAAa;QAC1B,OAAO,CAAC,GAAW,EAAE,EAAE;YACrB,OAAO,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAClC,CAAC,CAAA;IACH,CAAC;IAEO,MAAM;QACZ,OAAO,IAAA,kBAAS,EAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;IAChC,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,OAAO,OAAO,IAAI,CAAC,GAAG,IAAI,GAAG,EAAE,CAAA;IACjC,CAAC;IAEO,aAAa,CAAC,GAAQ;QAC5B,OAAO,UAAU,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAA;IAC/C,CAAC;IAEO,kBAAkB,CAAC,GAAQ;QACjC,OAAO,cAAc,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAA;IACnD,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,KAAmC;QACzE,4EAA4E;QAC5E,yDAAyD;QACzD,EAAE;QACF,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,YAAY;QAEZ,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7D,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAC7C,WAAW,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAA;QAEpE,MAAM,MAAM,GAAG,IAAI,oBAAM,CAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,IAAI;aACV;YACD,qDAAqD;YACrD,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6DAA6D;YAC7D,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,GAAG,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAmC;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QACzB,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QACnD,OAAO,GAAG,CAAA;IACZ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,GAAW,EAAE,GAAQ;QACvC,IAAI,CAAC;YACH,yEAAyE;YACzE,qEAAqE;YACrE,4CAA4C;YAC5C,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAC1B,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;aAC5B,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,SAAS;YACT,IAAI,GAAG,YAAY,wBAAiB,EAAE,CAAC;gBACrC,0CAA0C;gBAC1C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAC5C,6BAA6B;gBAC7B,IAAI,UAAU;oBAAE,OAAM;YACxB,CAAC;YAED,MAAM,GAAG,CAAA;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,GAAQ,EACR,KAAmC;QAEnC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;IACxD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAQ;QACvB,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;YAC7B,EAAE,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAQ;QACzB,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC;YAClC,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;SAC5B,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAQ;QAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACtC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;SAC7B,CAAC,CAAA;QACF,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,IAAI,CAAA;QACjB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,wBAAiB,EAAE,CAAA;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,GAAQ;QACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,GAAG,CAAC,oBAAoB,EAAE,CAAA;IACnC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAQ;QACtB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,GAAsB,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAQ;QACnB,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAW;QAC1B,MAAM,MAAM,GAAc,EAAE,CAAA;QAC5B,KAAK,MAAM,KAAK,IAAI,IAAA,uBAAU,EAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAA;gBACxD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,MAAM;YAAE,MAAM,IAAA,4BAAe,EAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAQ;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAW;QAC9B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,GAAG;aACT,CAAC,CAAA;YACF,OAAO,GAAG,CAAC,SAAS,CAAC,cAAc,KAAK,GAAG,CAAA;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,EAAE,GAAG;SACT,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAc;QACzC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;YAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE;gBACN,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;aACvC;SACF,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,IAAkC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE;gBACzC,GAAG,EAAE,IAAI,CAAC,EAAE;aACb,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,qBAAS,EAAE,CAAC;gBAC/B,8DAA8D;gBAC9D,MAAM,IAAI,wBAAiB,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;YACnD,CAAC;YAED,MAAM,KAAK,CAAA;QACb,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;gBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,IAAI;aACf,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,qBAAS,EAAE,CAAC;gBAC7B,8DAA8D;gBAC9D,OAAM;YACR,CAAC;YAED,MAAM,GAAG,CAAA;QACX,CAAC;IACH,CAAC;CACF;AAnPD,kCAmPC;AAED,kBAAe,WAAW,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atproto/aws",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.30",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Shared AWS cloud API helpers for atproto services",
|
|
6
6
|
"keywords": [
|
|
@@ -19,17 +19,18 @@
|
|
|
19
19
|
"node": ">=18.7.0"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@aws-sdk/client-cloudfront": "^3.
|
|
23
|
-
"@aws-sdk/client-kms": "^3.
|
|
24
|
-
"@aws-sdk/client-s3": "^3.
|
|
25
|
-
"@aws-sdk/lib-storage": "
|
|
22
|
+
"@aws-sdk/client-cloudfront": "^3.879.0",
|
|
23
|
+
"@aws-sdk/client-kms": "^3.879.0",
|
|
24
|
+
"@aws-sdk/client-s3": "^3.879.0",
|
|
25
|
+
"@aws-sdk/lib-storage": "3.879.0",
|
|
26
26
|
"@noble/curves": "^1.7.0",
|
|
27
27
|
"key-encoder": "^2.0.3",
|
|
28
28
|
"multiformats": "^9.9.0",
|
|
29
29
|
"uint8arrays": "3.0.0",
|
|
30
|
-
"@atproto/common": "^0.4.
|
|
30
|
+
"@atproto/common": "^0.4.12",
|
|
31
|
+
"@atproto/common-web": "^0.4.3",
|
|
31
32
|
"@atproto/crypto": "^0.4.4",
|
|
32
|
-
"@atproto/repo": "^0.8.
|
|
33
|
+
"@atproto/repo": "^0.8.10"
|
|
33
34
|
},
|
|
34
35
|
"devDependencies": {
|
|
35
36
|
"typescript": "^5.6.3"
|
package/src/s3.ts
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
import stream from 'node:stream'
|
|
2
|
-
import
|
|
2
|
+
import { NoSuchKey, S3, S3ClientConfig } from '@aws-sdk/client-s3'
|
|
3
3
|
import { Upload } from '@aws-sdk/lib-storage'
|
|
4
4
|
import { CID } from 'multiformats/cid'
|
|
5
|
+
import { SECOND, aggregateErrors, chunkArray } from '@atproto/common-web'
|
|
5
6
|
import { randomStr } from '@atproto/crypto'
|
|
6
7
|
import { BlobNotFoundError, BlobStore } from '@atproto/repo'
|
|
7
8
|
|
|
8
|
-
export type S3Config = {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
9
|
+
export type S3Config = {
|
|
10
|
+
bucket: string
|
|
11
|
+
/**
|
|
12
|
+
* The maximum time any request to S3 (including individual blob chunks
|
|
13
|
+
* uploads) can take, in milliseconds.
|
|
14
|
+
*/
|
|
15
|
+
requestTimeoutMs?: number
|
|
16
|
+
/**
|
|
17
|
+
* The maximum total time a blob upload can take, in milliseconds.
|
|
18
|
+
*/
|
|
19
|
+
uploadTimeoutMs?: number
|
|
20
|
+
} & Omit<S3ClientConfig, 'apiVersion' | 'requestHandler'>
|
|
15
21
|
|
|
16
22
|
export class S3BlobStore implements BlobStore {
|
|
17
|
-
private client:
|
|
23
|
+
private client: S3
|
|
18
24
|
private bucket: string
|
|
19
25
|
private uploadTimeoutMs: number
|
|
20
26
|
|
|
@@ -22,12 +28,22 @@ export class S3BlobStore implements BlobStore {
|
|
|
22
28
|
public did: string,
|
|
23
29
|
cfg: S3Config,
|
|
24
30
|
) {
|
|
25
|
-
const {
|
|
31
|
+
const {
|
|
32
|
+
bucket,
|
|
33
|
+
uploadTimeoutMs = 10 * SECOND,
|
|
34
|
+
requestTimeoutMs = uploadTimeoutMs,
|
|
35
|
+
...rest
|
|
36
|
+
} = cfg
|
|
26
37
|
this.bucket = bucket
|
|
27
|
-
this.uploadTimeoutMs = uploadTimeoutMs
|
|
28
|
-
this.client = new
|
|
38
|
+
this.uploadTimeoutMs = uploadTimeoutMs
|
|
39
|
+
this.client = new S3({
|
|
29
40
|
...rest,
|
|
30
41
|
apiVersion: '2006-03-01',
|
|
42
|
+
// Ensures that all requests timeout under "requestTimeoutMs".
|
|
43
|
+
//
|
|
44
|
+
// @NOTE This will also apply to the upload of each individual chunk
|
|
45
|
+
// when using Upload from @aws-sdk/lib-storage.
|
|
46
|
+
requestHandler: { requestTimeout: requestTimeoutMs },
|
|
31
47
|
})
|
|
32
48
|
}
|
|
33
49
|
|
|
@@ -53,42 +69,70 @@ export class S3BlobStore implements BlobStore {
|
|
|
53
69
|
return `quarantine/${this.did}/${cid.toString()}`
|
|
54
70
|
}
|
|
55
71
|
|
|
56
|
-
async
|
|
57
|
-
|
|
58
|
-
//
|
|
72
|
+
private async uploadBytes(path: string, bytes: Uint8Array | stream.Readable) {
|
|
73
|
+
// @NOTE we use Upload rather than client.putObject because stream length is
|
|
74
|
+
// not known in advance. See also aws/aws-sdk-js-v3#2348.
|
|
75
|
+
//
|
|
76
|
+
// See also https://github.com/aws/aws-sdk-js-v3/issues/6426, wherein Upload
|
|
77
|
+
// may hang the s3 connection under certain circumstances. We don't have a
|
|
78
|
+
// good way to avoid this, so we use timeouts defensively on all s3
|
|
79
|
+
// requests.
|
|
80
|
+
|
|
81
|
+
const abortSignal = AbortSignal.timeout(this.uploadTimeoutMs)
|
|
59
82
|
const abortController = new AbortController()
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
this.uploadTimeoutMs,
|
|
63
|
-
)
|
|
83
|
+
abortSignal.addEventListener('abort', () => abortController.abort())
|
|
84
|
+
|
|
64
85
|
const upload = new Upload({
|
|
65
86
|
client: this.client,
|
|
66
87
|
params: {
|
|
67
88
|
Bucket: this.bucket,
|
|
68
89
|
Body: bytes,
|
|
69
|
-
Key:
|
|
90
|
+
Key: path,
|
|
70
91
|
},
|
|
71
92
|
// @ts-ignore native implementation fine in node >=15
|
|
72
93
|
abortController,
|
|
73
94
|
})
|
|
95
|
+
|
|
74
96
|
try {
|
|
75
97
|
await upload.done()
|
|
76
|
-
}
|
|
77
|
-
|
|
98
|
+
} catch (err) {
|
|
99
|
+
// Translate aws-sdk's abort error to something more specific
|
|
100
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
101
|
+
throw new Error('Blob upload timed out', { cause: err })
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
throw err
|
|
78
105
|
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async putTemp(bytes: Uint8Array | stream.Readable): Promise<string> {
|
|
109
|
+
const key = this.genKey()
|
|
110
|
+
await this.uploadBytes(this.getTmpPath(key), bytes)
|
|
79
111
|
return key
|
|
80
112
|
}
|
|
81
113
|
|
|
82
114
|
async makePermanent(key: string, cid: CID): Promise<void> {
|
|
83
|
-
|
|
84
|
-
|
|
115
|
+
try {
|
|
116
|
+
// @NOTE we normally call this method when we know the file is temporary.
|
|
117
|
+
// Because of this, we optimistically move the file, allowing to make
|
|
118
|
+
// fewer network requests in the happy path.
|
|
85
119
|
await this.move({
|
|
86
120
|
from: this.getTmpPath(key),
|
|
87
121
|
to: this.getStoredPath(cid),
|
|
88
122
|
})
|
|
89
|
-
}
|
|
90
|
-
//
|
|
91
|
-
|
|
123
|
+
} catch (err) {
|
|
124
|
+
// If the optimistic move failed because the temp file was not found,
|
|
125
|
+
// check if the permanent file already exists. If it does, we can assume
|
|
126
|
+
// that another process made the file permanent concurrently, and we can
|
|
127
|
+
// no-op.
|
|
128
|
+
if (err instanceof BlobNotFoundError) {
|
|
129
|
+
// Blob was not found from temp storage...
|
|
130
|
+
const alreadyHas = await this.hasStored(cid)
|
|
131
|
+
// already saved, so we no-op
|
|
132
|
+
if (alreadyHas) return
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
throw err
|
|
92
136
|
}
|
|
93
137
|
}
|
|
94
138
|
|
|
@@ -96,27 +140,7 @@ export class S3BlobStore implements BlobStore {
|
|
|
96
140
|
cid: CID,
|
|
97
141
|
bytes: Uint8Array | stream.Readable,
|
|
98
142
|
): Promise<void> {
|
|
99
|
-
|
|
100
|
-
const abortController = new AbortController()
|
|
101
|
-
const timeout = setTimeout(
|
|
102
|
-
() => abortController.abort(),
|
|
103
|
-
this.uploadTimeoutMs,
|
|
104
|
-
)
|
|
105
|
-
const upload = new Upload({
|
|
106
|
-
client: this.client,
|
|
107
|
-
params: {
|
|
108
|
-
Bucket: this.bucket,
|
|
109
|
-
Body: bytes,
|
|
110
|
-
Key: this.getStoredPath(cid),
|
|
111
|
-
},
|
|
112
|
-
// @ts-ignore native implementation fine in node >=15
|
|
113
|
-
abortController,
|
|
114
|
-
})
|
|
115
|
-
try {
|
|
116
|
-
await upload.done()
|
|
117
|
-
} finally {
|
|
118
|
-
clearTimeout(timeout)
|
|
119
|
-
}
|
|
143
|
+
await this.uploadBytes(this.getStoredPath(cid), bytes)
|
|
120
144
|
}
|
|
121
145
|
|
|
122
146
|
async quarantine(cid: CID): Promise<void> {
|
|
@@ -160,8 +184,16 @@ export class S3BlobStore implements BlobStore {
|
|
|
160
184
|
}
|
|
161
185
|
|
|
162
186
|
async deleteMany(cids: CID[]): Promise<void> {
|
|
163
|
-
const
|
|
164
|
-
|
|
187
|
+
const errors: unknown[] = []
|
|
188
|
+
for (const chunk of chunkArray(cids, 500)) {
|
|
189
|
+
try {
|
|
190
|
+
const keys = chunk.map((cid) => this.getStoredPath(cid))
|
|
191
|
+
await this.deleteManyKeys(keys)
|
|
192
|
+
} catch (err) {
|
|
193
|
+
errors.push(err)
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (errors.length) throw aggregateErrors(errors)
|
|
165
197
|
}
|
|
166
198
|
|
|
167
199
|
async hasStored(cid: CID): Promise<boolean> {
|
|
@@ -207,21 +239,28 @@ export class S3BlobStore implements BlobStore {
|
|
|
207
239
|
CopySource: `${this.bucket}/${keys.from}`,
|
|
208
240
|
Key: keys.to,
|
|
209
241
|
})
|
|
242
|
+
} catch (cause) {
|
|
243
|
+
if (cause instanceof NoSuchKey) {
|
|
244
|
+
// Already deleted, possibly by a concurrently running process
|
|
245
|
+
throw new BlobNotFoundError(undefined, { cause })
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
throw cause
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
try {
|
|
210
252
|
await this.client.deleteObject({
|
|
211
253
|
Bucket: this.bucket,
|
|
212
254
|
Key: keys.from,
|
|
213
255
|
})
|
|
214
256
|
} catch (err) {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}
|
|
257
|
+
if (err instanceof NoSuchKey) {
|
|
258
|
+
// Already deleted, possibly by a concurrently running process
|
|
259
|
+
return
|
|
260
|
+
}
|
|
219
261
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
throw new BlobNotFoundError()
|
|
223
|
-
} else {
|
|
224
|
-
throw err
|
|
262
|
+
throw err
|
|
263
|
+
}
|
|
225
264
|
}
|
|
226
265
|
}
|
|
227
266
|
|