@syncular/server 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +3 -3
- package/dist/bucket-blob-store.d.ts +77 -0
- package/dist/bucket-blob-store.js +266 -0
- package/dist/bucket-segment-store.d.ts +77 -0
- package/dist/bucket-segment-store.js +445 -0
- package/dist/context.d.ts +7 -0
- package/dist/d1-storage.js +8 -2
- package/dist/handler.js +21 -7
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/object-bucket.d.ts +78 -0
- package/dist/object-bucket.js +18 -0
- package/dist/operations-realtime.js +3 -0
- package/dist/operations.js +4 -4
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +11 -0
- package/dist/realtime.js +38 -56
- package/dist/s3-blob-store.d.ts +9 -64
- package/dist/s3-blob-store.js +25 -359
- package/dist/s3-object-bucket.d.ts +39 -0
- package/dist/s3-object-bucket.js +164 -0
- package/dist/s3-segment-store.d.ts +9 -61
- package/dist/s3-segment-store.js +25 -503
- package/dist/seed.js +4 -3
- package/package.json +2 -2
- package/src/blob-handlers.ts +4 -4
- package/src/bucket-blob-store.ts +351 -0
- package/src/bucket-segment-store.ts +582 -0
- package/src/context.ts +10 -0
- package/src/d1-storage.ts +12 -2
- package/src/handler.ts +38 -10
- package/src/index.ts +4 -0
- package/src/object-bucket.ts +83 -0
- package/src/operations-realtime.ts +7 -0
- package/src/operations.ts +8 -4
- package/src/push.ts +40 -9
- package/src/realtime.ts +45 -55
- package/src/s3-blob-store.ts +25 -466
- package/src/s3-object-bucket.ts +233 -0
- package/src/s3-segment-store.ts +25 -646
- package/src/seed.ts +4 -3
package/dist/s3-blob-store.js
CHANGED
|
@@ -1,104 +1,28 @@
|
|
|
1
|
-
import { EMPTY_PAYLOAD_SHA256, presignUrl, sha256Hex, signRequest, } from './sigv4.js';
|
|
2
|
-
const BLOB_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
3
|
-
const BLOB_META_HEADER = 'x-amz-meta-syncular-blob';
|
|
4
|
-
/**
|
|
5
|
-
* Stats accumulator (mirrors `S3SegmentStore`): a small counter object under
|
|
6
|
-
* a fixed key, updated read-modify-write on `put` under an ETag
|
|
7
|
-
* compare-and-swap. A LIST-free point-lookup store cannot count objects on
|
|
8
|
-
* demand cheaply, so `stats()` reads the accumulator. The counters are
|
|
9
|
-
* APPROXIMATE — a crash between the blob PUT and the accumulator CAS, or a
|
|
10
|
-
* sweep DELETE the accumulator does not decrement, drifts them — so the
|
|
11
|
-
* admin surface labels S3 blob stats `approximate: true`. A health gauge,
|
|
12
|
-
* not an invoice.
|
|
13
|
-
*/
|
|
14
|
-
const STATS_KEY_SUFFIX = 'stats/blobs.json';
|
|
15
|
-
const STATS_CAS_ATTEMPTS = 16;
|
|
16
|
-
function emptyStats() {
|
|
17
|
-
return { count: 0, bytes: 0 };
|
|
18
|
-
}
|
|
19
|
-
function parseStatsJson(json) {
|
|
20
|
-
const parsed = JSON.parse(json);
|
|
21
|
-
return {
|
|
22
|
-
count: Number(parsed.count ?? 0),
|
|
23
|
-
bytes: Number(parsed.bytes ?? 0),
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
// Runtime-neutral base64url: `Buffer` is not present on
|
|
27
|
-
// Cloudflare Workers without `nodejs_compat`, so the metadata header is
|
|
28
|
-
// (de)coded with `btoa`/`atob`, available in every runtime.
|
|
29
|
-
function utf8ToBase64url(text) {
|
|
30
|
-
let binary = '';
|
|
31
|
-
for (const byte of new TextEncoder().encode(text)) {
|
|
32
|
-
binary += String.fromCharCode(byte);
|
|
33
|
-
}
|
|
34
|
-
return btoa(binary)
|
|
35
|
-
.replaceAll('+', '-')
|
|
36
|
-
.replaceAll('/', '_')
|
|
37
|
-
.replace(/=+$/, '');
|
|
38
|
-
}
|
|
39
|
-
function base64urlToUtf8(value) {
|
|
40
|
-
const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
|
|
41
|
-
const bytes = new Uint8Array(binary.length);
|
|
42
|
-
for (let i = 0; i < binary.length; i++)
|
|
43
|
-
bytes[i] = binary.charCodeAt(i);
|
|
44
|
-
return new TextDecoder().decode(bytes);
|
|
45
|
-
}
|
|
46
|
-
function metaToJson(meta) {
|
|
47
|
-
return JSON.stringify(meta);
|
|
48
|
-
}
|
|
49
|
-
function parseMetaJson(json, source) {
|
|
50
|
-
let parsed;
|
|
51
|
-
try {
|
|
52
|
-
parsed = JSON.parse(json);
|
|
53
|
-
}
|
|
54
|
-
catch {
|
|
55
|
-
throw new Error(`S3BlobStore: corrupt blob metadata in ${source}`);
|
|
56
|
-
}
|
|
57
|
-
if (typeof parsed !== 'object' || parsed === null) {
|
|
58
|
-
throw new Error(`S3BlobStore: corrupt blob metadata in ${source}`);
|
|
59
|
-
}
|
|
60
|
-
const r = parsed;
|
|
61
|
-
if (typeof r.byteLength !== 'number' || typeof r.createdAtMs !== 'number') {
|
|
62
|
-
throw new Error(`S3BlobStore: bad blob metadata in ${source}`);
|
|
63
|
-
}
|
|
64
|
-
return {
|
|
65
|
-
byteLength: r.byteLength,
|
|
66
|
-
...(typeof r.mediaType === 'string' ? { mediaType: r.mediaType } : {}),
|
|
67
|
-
createdAtMs: r.createdAtMs,
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
1
|
/**
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
2
|
+
* S3-compatible blob store (SPEC.md §5.9.2 durable objects, §5.9.5 delegated
|
|
3
|
+
* presign) — AWS S3, Cloudflare R2's S3 endpoint, MinIO. The blob twin of
|
|
4
|
+
* `s3-segment-store.ts`.
|
|
5
|
+
*
|
|
6
|
+
* The store logic (content-addressed key layout, idempotent re-put, orphan
|
|
7
|
+
* sweep, CAS stats accumulator) is `BucketBlobStore`, shared with
|
|
8
|
+
* `R2BlobStore` in `@syncular/server-workers`; this class adds the S3 REST
|
|
9
|
+
* transport (`S3ObjectBucket`) and the SigV4 presign methods, which a bucket
|
|
10
|
+
* binding cannot mint.
|
|
75
11
|
*/
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const nextToken = truncated && tokenMatch !== null
|
|
84
|
-
? unescapeXml(tokenMatch[1] ?? '')
|
|
85
|
-
: undefined;
|
|
86
|
-
return { objects, nextToken };
|
|
87
|
-
}
|
|
88
|
-
function unescapeXml(value) {
|
|
89
|
-
return value
|
|
90
|
-
.replaceAll('<', '<')
|
|
91
|
-
.replaceAll('>', '>')
|
|
92
|
-
.replaceAll('"', '"')
|
|
93
|
-
.replaceAll(''', "'")
|
|
94
|
-
.replaceAll('&', '&');
|
|
95
|
-
}
|
|
96
|
-
export class S3BlobStore {
|
|
97
|
-
#config;
|
|
12
|
+
import { BucketBlobStore, } from './bucket-blob-store.js';
|
|
13
|
+
import { S3ObjectBucket } from './s3-object-bucket.js';
|
|
14
|
+
import { presignUrl } from './sigv4.js';
|
|
15
|
+
const BLOB_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
16
|
+
export class S3BlobStore extends BucketBlobStore {
|
|
17
|
+
#bucket;
|
|
18
|
+
#region;
|
|
98
19
|
#credentials;
|
|
99
|
-
#prefix;
|
|
100
20
|
constructor(config) {
|
|
101
|
-
|
|
21
|
+
const options = { keyPrefix: config.keyPrefix };
|
|
22
|
+
const bucket = new S3ObjectBucket(config);
|
|
23
|
+
super(bucket, options);
|
|
24
|
+
this.#bucket = bucket;
|
|
25
|
+
this.#region = config.region;
|
|
102
26
|
this.#credentials = {
|
|
103
27
|
accessKeyId: config.accessKeyId,
|
|
104
28
|
secretAccessKey: config.secretAccessKey,
|
|
@@ -106,264 +30,6 @@ export class S3BlobStore {
|
|
|
106
30
|
? { sessionToken: config.sessionToken }
|
|
107
31
|
: {}),
|
|
108
32
|
};
|
|
109
|
-
this.#prefix = config.keyPrefix ?? '';
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* `{keyPrefix}blob/{partition}/sha256/{hex}` — the §5.9.5 "signed key embeds
|
|
113
|
-
* the blobId". Blobs are content-addressed, but download authorization and
|
|
114
|
-
* the reference index are per-partition, so the same bytes uploaded under
|
|
115
|
-
* two partitions are two objects (a partition MUST NOT read another's
|
|
116
|
-
* attachment by guessing a content address). The partition is folded into
|
|
117
|
-
* the key as a path segment.
|
|
118
|
-
*/
|
|
119
|
-
objectKeyFor(partition, blobId) {
|
|
120
|
-
return `${this.#prefix}blob/${partition}/${blobId.replace(':', '/')}`;
|
|
121
|
-
}
|
|
122
|
-
#statsKey() {
|
|
123
|
-
return `${this.#prefix}${STATS_KEY_SUFFIX}`;
|
|
124
|
-
}
|
|
125
|
-
#urlFor(key) {
|
|
126
|
-
const endpoint = this.#config.endpoint.replace(/\/+$/, '');
|
|
127
|
-
return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
|
|
128
|
-
}
|
|
129
|
-
async #request(method, key, options) {
|
|
130
|
-
const url = this.#urlFor(key);
|
|
131
|
-
if (options?.query !== undefined) {
|
|
132
|
-
for (const [name, value] of options.query) {
|
|
133
|
-
url.searchParams.set(name, value);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
const body = options?.body;
|
|
137
|
-
const headers = await signRequest({
|
|
138
|
-
method,
|
|
139
|
-
url,
|
|
140
|
-
region: this.#config.region,
|
|
141
|
-
credentials: this.#credentials,
|
|
142
|
-
nowMs: Date.now(),
|
|
143
|
-
payloadHash: body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
|
|
144
|
-
...(options?.headers !== undefined ? { headers: options.headers } : {}),
|
|
145
|
-
});
|
|
146
|
-
const response = await fetch(url, {
|
|
147
|
-
method,
|
|
148
|
-
headers,
|
|
149
|
-
...(body !== undefined ? { body: body } : {}),
|
|
150
|
-
});
|
|
151
|
-
if (response.status === 404) {
|
|
152
|
-
await response.arrayBuffer();
|
|
153
|
-
return undefined;
|
|
154
|
-
}
|
|
155
|
-
if (!response.ok) {
|
|
156
|
-
const detail = (await response.text()).slice(0, 300);
|
|
157
|
-
throw new Error(`S3BlobStore: ${method} ${key} failed with ${response.status}: ${detail}`);
|
|
158
|
-
}
|
|
159
|
-
return response;
|
|
160
|
-
}
|
|
161
|
-
async #readStats() {
|
|
162
|
-
const response = await this.#request('GET', this.#statsKey());
|
|
163
|
-
if (response === undefined) {
|
|
164
|
-
return { stats: emptyStats(), etag: undefined };
|
|
165
|
-
}
|
|
166
|
-
const etag = response.headers.get('etag') ?? undefined;
|
|
167
|
-
const stats = parseStatsJson(await response.text());
|
|
168
|
-
return { stats, etag };
|
|
169
|
-
}
|
|
170
|
-
async #writeStatsCas(stats, etag) {
|
|
171
|
-
const url = this.#urlFor(this.#statsKey());
|
|
172
|
-
const body = new TextEncoder().encode(JSON.stringify(stats));
|
|
173
|
-
const conditional = etag === undefined ? { 'if-none-match': '*' } : { 'if-match': etag };
|
|
174
|
-
const headers = await signRequest({
|
|
175
|
-
method: 'PUT',
|
|
176
|
-
url,
|
|
177
|
-
region: this.#config.region,
|
|
178
|
-
credentials: this.#credentials,
|
|
179
|
-
nowMs: Date.now(),
|
|
180
|
-
payloadHash: await sha256Hex(body),
|
|
181
|
-
headers: { 'content-type': 'application/json', ...conditional },
|
|
182
|
-
});
|
|
183
|
-
const response = await fetch(url, {
|
|
184
|
-
method: 'PUT',
|
|
185
|
-
headers,
|
|
186
|
-
body: body,
|
|
187
|
-
});
|
|
188
|
-
if (response.status === 412) {
|
|
189
|
-
await response.arrayBuffer();
|
|
190
|
-
return false;
|
|
191
|
-
}
|
|
192
|
-
if (!response.ok) {
|
|
193
|
-
const detail = (await response.text()).slice(0, 300);
|
|
194
|
-
throw new Error(`S3BlobStore: stats PUT failed with ${response.status}: ${detail}`);
|
|
195
|
-
}
|
|
196
|
-
await response.arrayBuffer();
|
|
197
|
-
return true;
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Fold `delta` into the stats accumulator under an ETag CAS retry loop.
|
|
201
|
-
* Best-effort: exhausting the budget leaves the counters drifted (they are
|
|
202
|
-
* documented APPROXIMATE) rather than failing an op that already touched
|
|
203
|
-
* the bytes.
|
|
204
|
-
*/
|
|
205
|
-
async #bumpStats(delta) {
|
|
206
|
-
for (let attempt = 0; attempt < STATS_CAS_ATTEMPTS; attempt++) {
|
|
207
|
-
const { stats, etag } = await this.#readStats();
|
|
208
|
-
const next = {
|
|
209
|
-
count: stats.count + delta.count,
|
|
210
|
-
bytes: stats.bytes + delta.bytes,
|
|
211
|
-
};
|
|
212
|
-
if (await this.#writeStatsCas(next, etag))
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
// Lost the CAS race `STATS_CAS_ATTEMPTS` times — leave counters as-is;
|
|
216
|
-
// `stats()` stays APPROXIMATE by contract.
|
|
217
|
-
}
|
|
218
|
-
async put(partition, blobId, bytes, nowMs, mediaType) {
|
|
219
|
-
const record = {
|
|
220
|
-
blobId,
|
|
221
|
-
partition,
|
|
222
|
-
byteLength: bytes.length,
|
|
223
|
-
...(mediaType !== undefined ? { mediaType } : {}),
|
|
224
|
-
createdAtMs: nowMs,
|
|
225
|
-
};
|
|
226
|
-
const objectKey = this.objectKeyFor(partition, blobId);
|
|
227
|
-
// Idempotent re-put (same content-address ⇒ same key): count each distinct
|
|
228
|
-
// blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
|
|
229
|
-
// is not reset by a re-upload (§5.9.2 upload age).
|
|
230
|
-
const preexisting = await this.#request('HEAD', objectKey);
|
|
231
|
-
if (preexisting !== undefined) {
|
|
232
|
-
const meta = preexisting.headers.get(BLOB_META_HEADER);
|
|
233
|
-
await preexisting.arrayBuffer();
|
|
234
|
-
if (meta !== null) {
|
|
235
|
-
const existing = parseMetaJson(base64urlToUtf8(meta), blobId);
|
|
236
|
-
return {
|
|
237
|
-
blobId,
|
|
238
|
-
partition,
|
|
239
|
-
byteLength: existing.byteLength,
|
|
240
|
-
...(existing.mediaType !== undefined
|
|
241
|
-
? { mediaType: existing.mediaType }
|
|
242
|
-
: {}),
|
|
243
|
-
createdAtMs: existing.createdAtMs,
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
const meta = {
|
|
248
|
-
byteLength: bytes.length,
|
|
249
|
-
...(mediaType !== undefined ? { mediaType } : {}),
|
|
250
|
-
createdAtMs: nowMs,
|
|
251
|
-
};
|
|
252
|
-
const response = await this.#request('PUT', objectKey, {
|
|
253
|
-
body: bytes,
|
|
254
|
-
headers: {
|
|
255
|
-
'content-type': mediaType ?? 'application/octet-stream',
|
|
256
|
-
[BLOB_META_HEADER]: utf8ToBase64url(metaToJson(meta)),
|
|
257
|
-
},
|
|
258
|
-
});
|
|
259
|
-
await response?.arrayBuffer();
|
|
260
|
-
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
261
|
-
return record;
|
|
262
|
-
}
|
|
263
|
-
async has(partition, blobId) {
|
|
264
|
-
if (!BLOB_ID_PATTERN.test(blobId))
|
|
265
|
-
return false;
|
|
266
|
-
const response = await this.#request('HEAD', this.objectKeyFor(partition, blobId));
|
|
267
|
-
if (response === undefined)
|
|
268
|
-
return false;
|
|
269
|
-
await response.arrayBuffer();
|
|
270
|
-
return true;
|
|
271
|
-
}
|
|
272
|
-
async get(partition, blobId) {
|
|
273
|
-
if (!BLOB_ID_PATTERN.test(blobId))
|
|
274
|
-
return undefined;
|
|
275
|
-
const response = await this.#request('GET', this.objectKeyFor(partition, blobId));
|
|
276
|
-
if (response === undefined)
|
|
277
|
-
return undefined;
|
|
278
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
279
|
-
const rawMeta = response.headers.get(BLOB_META_HEADER);
|
|
280
|
-
const meta = rawMeta !== null
|
|
281
|
-
? parseMetaJson(base64urlToUtf8(rawMeta), blobId)
|
|
282
|
-
: { byteLength: bytes.length, createdAtMs: 0 };
|
|
283
|
-
return {
|
|
284
|
-
record: {
|
|
285
|
-
blobId,
|
|
286
|
-
partition,
|
|
287
|
-
byteLength: meta.byteLength,
|
|
288
|
-
...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
|
|
289
|
-
createdAtMs: meta.createdAtMs,
|
|
290
|
-
},
|
|
291
|
-
bytes,
|
|
292
|
-
};
|
|
293
|
-
}
|
|
294
|
-
/**
|
|
295
|
-
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page `ListObjectsV2`
|
|
296
|
-
* over `{prefix}blob/{partition}/`, HEAD each object for its `createdAtMs`,
|
|
297
|
-
* and DELETE the ones both older than `olderThanMs` AND absent from
|
|
298
|
-
* `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
|
|
299
|
-
* referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
|
|
300
|
-
*/
|
|
301
|
-
async sweepOrphans(partition, olderThanMs, referencedBlobIds) {
|
|
302
|
-
const listPrefix = `${this.#prefix}blob/${partition}/`;
|
|
303
|
-
const swept = [];
|
|
304
|
-
let deletedBytes = 0;
|
|
305
|
-
let continuationToken;
|
|
306
|
-
do {
|
|
307
|
-
const query = [
|
|
308
|
-
['list-type', '2'],
|
|
309
|
-
['prefix', listPrefix],
|
|
310
|
-
];
|
|
311
|
-
if (continuationToken !== undefined) {
|
|
312
|
-
query.push(['continuation-token', continuationToken]);
|
|
313
|
-
}
|
|
314
|
-
// ListObjectsV2 is a bucket-level GET (`GET /{bucket}?list-type=2`).
|
|
315
|
-
const response = await this.#request('GET', '', { query });
|
|
316
|
-
if (response === undefined)
|
|
317
|
-
break;
|
|
318
|
-
const { objects, nextToken } = parseListObjectsV2(await response.text());
|
|
319
|
-
continuationToken = nextToken;
|
|
320
|
-
for (const { key } of objects) {
|
|
321
|
-
// `{listPrefix}sha256/{hex}` → `sha256:{hex}`.
|
|
322
|
-
const rest = key.slice(listPrefix.length);
|
|
323
|
-
if (!rest.startsWith('sha256/'))
|
|
324
|
-
continue;
|
|
325
|
-
const blobId = `sha256:${rest.slice('sha256/'.length)}`;
|
|
326
|
-
if (!BLOB_ID_PATTERN.test(blobId))
|
|
327
|
-
continue;
|
|
328
|
-
if (referencedBlobIds.has(blobId))
|
|
329
|
-
continue;
|
|
330
|
-
const head = await this.#request('HEAD', key);
|
|
331
|
-
if (head === undefined)
|
|
332
|
-
continue;
|
|
333
|
-
const rawMeta = head.headers.get(BLOB_META_HEADER);
|
|
334
|
-
const lengthHeader = head.headers.get('content-length');
|
|
335
|
-
await head.arrayBuffer();
|
|
336
|
-
const meta = rawMeta !== null
|
|
337
|
-
? parseMetaJson(base64urlToUtf8(rawMeta), blobId)
|
|
338
|
-
: {
|
|
339
|
-
byteLength: Number(lengthHeader ?? 0),
|
|
340
|
-
createdAtMs: 0,
|
|
341
|
-
};
|
|
342
|
-
if (meta.createdAtMs >= olderThanMs)
|
|
343
|
-
continue;
|
|
344
|
-
const del = await this.#request('DELETE', key);
|
|
345
|
-
await del?.arrayBuffer();
|
|
346
|
-
swept.push(blobId);
|
|
347
|
-
deletedBytes += meta.byteLength;
|
|
348
|
-
}
|
|
349
|
-
} while (continuationToken !== undefined);
|
|
350
|
-
if (swept.length > 0) {
|
|
351
|
-
await this.#bumpStats({ count: -swept.length, bytes: -deletedBytes });
|
|
352
|
-
}
|
|
353
|
-
return swept;
|
|
354
|
-
}
|
|
355
|
-
/**
|
|
356
|
-
* Store-wide counters from the pointer-object accumulator (never a LIST on
|
|
357
|
-
* the read path). `approximate: true` for the same reasons `S3SegmentStore`
|
|
358
|
-
* documents: best-effort ETag-CAS maintenance and sweep deletes that may
|
|
359
|
-
* miss the decrement. Exact within a single writer; a health gauge under
|
|
360
|
-
* concurrency. Note this is store-WIDE, not partition-scoped, so a shared
|
|
361
|
-
* bucket reports the whole bucket — the parameter is accepted for interface
|
|
362
|
-
* parity but the accumulator is one object per store.
|
|
363
|
-
*/
|
|
364
|
-
async stats(_partition) {
|
|
365
|
-
const { stats } = await this.#readStats();
|
|
366
|
-
return { count: stats.count, bytes: stats.bytes, approximate: true };
|
|
367
33
|
}
|
|
368
34
|
/**
|
|
369
35
|
* SigV4 presigned GET for a blob object (§5.9.5 delegated presign). The
|
|
@@ -381,8 +47,8 @@ export class S3BlobStore {
|
|
|
381
47
|
const ttlSeconds = options?.ttlSeconds ?? 900;
|
|
382
48
|
const nowMs = options?.nowMs ?? Date.now();
|
|
383
49
|
const url = await presignUrl({
|
|
384
|
-
url: this.#urlFor(this.objectKeyFor(partition, blobId)),
|
|
385
|
-
region: this.#
|
|
50
|
+
url: this.#bucket.urlFor(this.objectKeyFor(partition, blobId)),
|
|
51
|
+
region: this.#region,
|
|
386
52
|
credentials: this.#credentials,
|
|
387
53
|
nowMs,
|
|
388
54
|
expiresSeconds: ttlSeconds,
|
|
@@ -412,8 +78,8 @@ export class S3BlobStore {
|
|
|
412
78
|
const nowMs = options?.nowMs ?? Date.now();
|
|
413
79
|
const url = await presignUrl({
|
|
414
80
|
method: 'PUT',
|
|
415
|
-
url: this.#urlFor(this.objectKeyFor(partition, blobId)),
|
|
416
|
-
region: this.#
|
|
81
|
+
url: this.#bucket.urlFor(this.objectKeyFor(partition, blobId)),
|
|
82
|
+
region: this.#region,
|
|
417
83
|
credentials: this.#credentials,
|
|
418
84
|
nowMs,
|
|
419
85
|
expiresSeconds: ttlSeconds,
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `ObjectBucket` over the S3 REST API: hand-rolled SigV4 over `fetch`
|
|
3
|
+
* (sigv4.ts), zero dependencies, path-style requests
|
|
4
|
+
* (`{endpoint}/{bucket}/{key}`) that AWS S3, Cloudflare R2's S3 endpoint, and
|
|
5
|
+
* MinIO all accept.
|
|
6
|
+
*
|
|
7
|
+
* This module owns the transport only. The segment and blob store logic lives
|
|
8
|
+
* in `bucket-segment-store.ts` / `bucket-blob-store.ts` and runs over any
|
|
9
|
+
* `ObjectBucket`, so the same objects are reachable through an R2 binding
|
|
10
|
+
* (`R2SegmentStore` in `@syncular/server-workers`) with no copy between them.
|
|
11
|
+
*/
|
|
12
|
+
import type { BucketListPage, BucketObject, BucketObjectHead, BucketWriteOptions, ObjectBucket } from './object-bucket.js';
|
|
13
|
+
export interface S3ObjectBucketConfig {
|
|
14
|
+
/**
|
|
15
|
+
* Endpoint origin, no bucket, no trailing slash:
|
|
16
|
+
* AWS `https://s3.<region>.amazonaws.com`,
|
|
17
|
+
* R2 `https://<account-id>.r2.cloudflarestorage.com`,
|
|
18
|
+
* MinIO `http://127.0.0.1:9000`.
|
|
19
|
+
*/
|
|
20
|
+
readonly endpoint: string;
|
|
21
|
+
/** AWS region; R2 uses `auto`. */
|
|
22
|
+
readonly region: string;
|
|
23
|
+
readonly bucket: string;
|
|
24
|
+
readonly accessKeyId: string;
|
|
25
|
+
readonly secretAccessKey: string;
|
|
26
|
+
/** Optional STS session token. */
|
|
27
|
+
readonly sessionToken?: string;
|
|
28
|
+
}
|
|
29
|
+
export declare class S3ObjectBucket implements ObjectBucket {
|
|
30
|
+
#private;
|
|
31
|
+
constructor(config: S3ObjectBucketConfig);
|
|
32
|
+
/** The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`). */
|
|
33
|
+
urlFor(key: string): URL;
|
|
34
|
+
get(key: string): Promise<BucketObject | undefined>;
|
|
35
|
+
head(key: string): Promise<BucketObjectHead | undefined>;
|
|
36
|
+
put(key: string, options: BucketWriteOptions): Promise<boolean>;
|
|
37
|
+
delete(key: string): Promise<void>;
|
|
38
|
+
list(prefix: string, cursor?: string): Promise<BucketListPage>;
|
|
39
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { EMPTY_PAYLOAD_SHA256, sha256Hex, signRequest, } from './sigv4.js';
|
|
2
|
+
const META_HEADER_PREFIX = 'x-amz-meta-';
|
|
3
|
+
/** User metadata carried as `x-amz-meta-*` headers (lowercased by `Headers`). */
|
|
4
|
+
function metadataOf(headers) {
|
|
5
|
+
const metadata = {};
|
|
6
|
+
for (const [name, value] of headers) {
|
|
7
|
+
if (name.startsWith(META_HEADER_PREFIX)) {
|
|
8
|
+
metadata[name.slice(META_HEADER_PREFIX.length)] = value;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
return metadata;
|
|
12
|
+
}
|
|
13
|
+
function sizeOf(headers) {
|
|
14
|
+
const value = headers.get('content-length');
|
|
15
|
+
if (value === null)
|
|
16
|
+
return undefined;
|
|
17
|
+
const size = Number(value);
|
|
18
|
+
return Number.isFinite(size) ? size : undefined;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Extract `<Key>` values (and the `<NextContinuationToken>`, if the listing
|
|
22
|
+
* is truncated) from a `ListObjectsV2` XML body. A tiny regex reader; the
|
|
23
|
+
* only XML the stores ever parse, and only on the GC path. `ListObjectsV2`
|
|
24
|
+
* returns keys XML-escaped for `& < >`; we unescape those.
|
|
25
|
+
*/
|
|
26
|
+
function parseListObjectsV2(xml) {
|
|
27
|
+
const objects = [];
|
|
28
|
+
for (const m of xml.matchAll(/<Key>([\s\S]*?)<\/Key>/g)) {
|
|
29
|
+
objects.push({ key: unescapeXml(m[1] ?? '') });
|
|
30
|
+
}
|
|
31
|
+
const truncated = /<IsTruncated>\s*true\s*<\/IsTruncated>/i.test(xml);
|
|
32
|
+
const tokenMatch = /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/.exec(xml);
|
|
33
|
+
const nextToken = truncated && tokenMatch !== null
|
|
34
|
+
? unescapeXml(tokenMatch[1] ?? '')
|
|
35
|
+
: undefined;
|
|
36
|
+
return { objects, nextToken };
|
|
37
|
+
}
|
|
38
|
+
function unescapeXml(value) {
|
|
39
|
+
return value
|
|
40
|
+
.replaceAll('<', '<')
|
|
41
|
+
.replaceAll('>', '>')
|
|
42
|
+
.replaceAll('"', '"')
|
|
43
|
+
.replaceAll(''', "'")
|
|
44
|
+
.replaceAll('&', '&');
|
|
45
|
+
}
|
|
46
|
+
export class S3ObjectBucket {
|
|
47
|
+
#config;
|
|
48
|
+
#credentials;
|
|
49
|
+
constructor(config) {
|
|
50
|
+
this.#config = config;
|
|
51
|
+
this.#credentials = {
|
|
52
|
+
accessKeyId: config.accessKeyId,
|
|
53
|
+
secretAccessKey: config.secretAccessKey,
|
|
54
|
+
...(config.sessionToken !== undefined
|
|
55
|
+
? { sessionToken: config.sessionToken }
|
|
56
|
+
: {}),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`). */
|
|
60
|
+
urlFor(key) {
|
|
61
|
+
const endpoint = this.#config.endpoint.replace(/\/+$/, '');
|
|
62
|
+
return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
|
|
63
|
+
}
|
|
64
|
+
async #request(method, key, options) {
|
|
65
|
+
const url = this.urlFor(key);
|
|
66
|
+
if (options?.query !== undefined) {
|
|
67
|
+
for (const [name, value] of options.query) {
|
|
68
|
+
url.searchParams.set(name, value);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const body = options?.body;
|
|
72
|
+
const headers = await signRequest({
|
|
73
|
+
method,
|
|
74
|
+
url,
|
|
75
|
+
region: this.#config.region,
|
|
76
|
+
credentials: this.#credentials,
|
|
77
|
+
nowMs: Date.now(),
|
|
78
|
+
payloadHash: body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
|
|
79
|
+
...(options?.headers !== undefined ? { headers: options.headers } : {}),
|
|
80
|
+
});
|
|
81
|
+
const response = await fetch(url, {
|
|
82
|
+
method,
|
|
83
|
+
headers,
|
|
84
|
+
...(body !== undefined ? { body: body } : {}),
|
|
85
|
+
});
|
|
86
|
+
if (response.status === 404) {
|
|
87
|
+
await response.arrayBuffer();
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
if (options?.allow412 === true && response.status === 412)
|
|
91
|
+
return response;
|
|
92
|
+
if (!response.ok) {
|
|
93
|
+
const detail = (await response.text()).slice(0, 300);
|
|
94
|
+
throw new Error(`S3ObjectBucket: ${method} ${key} failed with ${response.status}: ${detail}`);
|
|
95
|
+
}
|
|
96
|
+
return response;
|
|
97
|
+
}
|
|
98
|
+
async get(key) {
|
|
99
|
+
const response = await this.#request('GET', key);
|
|
100
|
+
if (response === undefined)
|
|
101
|
+
return undefined;
|
|
102
|
+
return {
|
|
103
|
+
body: response.body,
|
|
104
|
+
etag: response.headers.get('etag') ?? undefined,
|
|
105
|
+
size: sizeOf(response.headers),
|
|
106
|
+
metadata: metadataOf(response.headers),
|
|
107
|
+
arrayBuffer: () => response.arrayBuffer(),
|
|
108
|
+
text: () => response.text(),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
async head(key) {
|
|
112
|
+
const response = await this.#request('HEAD', key);
|
|
113
|
+
if (response === undefined)
|
|
114
|
+
return undefined;
|
|
115
|
+
await response.arrayBuffer();
|
|
116
|
+
return {
|
|
117
|
+
etag: response.headers.get('etag') ?? undefined,
|
|
118
|
+
size: sizeOf(response.headers),
|
|
119
|
+
metadata: metadataOf(response.headers),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
async put(key, options) {
|
|
123
|
+
const headers = {
|
|
124
|
+
'content-type': options.contentType,
|
|
125
|
+
};
|
|
126
|
+
if (options.createOnly === true)
|
|
127
|
+
headers['if-none-match'] = '*';
|
|
128
|
+
if (options.ifMatch !== undefined)
|
|
129
|
+
headers['if-match'] = options.ifMatch;
|
|
130
|
+
for (const [name, value] of Object.entries(options.metadata ?? {})) {
|
|
131
|
+
headers[`${META_HEADER_PREFIX}${name}`] = value;
|
|
132
|
+
}
|
|
133
|
+
const response = await this.#request('PUT', key, {
|
|
134
|
+
body: options.body,
|
|
135
|
+
headers,
|
|
136
|
+
allow412: true,
|
|
137
|
+
});
|
|
138
|
+
if (response === undefined) {
|
|
139
|
+
throw new Error(`S3ObjectBucket: PUT ${key} failed with 404`);
|
|
140
|
+
}
|
|
141
|
+
await response.arrayBuffer();
|
|
142
|
+
// A failed `If-Match` / `If-None-Match: *` is a `412`: the caller's CAS
|
|
143
|
+
// read superseded, so it re-reads instead of overwriting.
|
|
144
|
+
return response.status !== 412;
|
|
145
|
+
}
|
|
146
|
+
async delete(key) {
|
|
147
|
+
const response = await this.#request('DELETE', key);
|
|
148
|
+
await response?.arrayBuffer();
|
|
149
|
+
}
|
|
150
|
+
async list(prefix, cursor) {
|
|
151
|
+
const query = [
|
|
152
|
+
['list-type', '2'],
|
|
153
|
+
['prefix', prefix],
|
|
154
|
+
];
|
|
155
|
+
if (cursor !== undefined)
|
|
156
|
+
query.push(['continuation-token', cursor]);
|
|
157
|
+
// ListObjectsV2 is a bucket-level GET (`GET /{bucket}?list-type=2`).
|
|
158
|
+
const response = await this.#request('GET', '', { query });
|
|
159
|
+
if (response === undefined)
|
|
160
|
+
return { keys: [], cursor: undefined };
|
|
161
|
+
const { objects, nextToken } = parseListObjectsV2(await response.text());
|
|
162
|
+
return { keys: objects.map((object) => object.key), cursor: nextToken };
|
|
163
|
+
}
|
|
164
|
+
}
|