@syncular/server-storage-s3 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +53 -0
- package/src/index.test.ts +522 -0
- package/src/index.ts +324 -0
package/package.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@syncular/server-storage-s3",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "S3-compatible blob storage adapter for Syncular server",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"author": "Benjamin Kniffler",
|
|
7
|
+
"homepage": "https://syncular.dev",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/syncular/syncular.git",
|
|
11
|
+
"directory": "packages/server-storage-s3"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/syncular/syncular/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"sync",
|
|
18
|
+
"offline-first",
|
|
19
|
+
"blob",
|
|
20
|
+
"storage",
|
|
21
|
+
"s3"
|
|
22
|
+
],
|
|
23
|
+
"private": false,
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public"
|
|
26
|
+
},
|
|
27
|
+
"type": "module",
|
|
28
|
+
"exports": {
|
|
29
|
+
".": {
|
|
30
|
+
"bun": "./src/index.ts",
|
|
31
|
+
"import": {
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"default": "./dist/index.js"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"scripts": {
|
|
38
|
+
"test": "bun test --pass-with-no-tests",
|
|
39
|
+
"tsgo": "tsgo --noEmit",
|
|
40
|
+
"build": "tsgo",
|
|
41
|
+
"release": "bunx syncular-publish"
|
|
42
|
+
},
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"@syncular/core": "0.0.0"
|
|
45
|
+
},
|
|
46
|
+
"devDependencies": {
|
|
47
|
+
"@syncular/config": "0.0.0"
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
50
|
+
"dist",
|
|
51
|
+
"src"
|
|
52
|
+
]
|
|
53
|
+
}
|
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import {
|
|
3
|
+
createS3BlobStorageAdapter,
|
|
4
|
+
type GetSignedUrlFn,
|
|
5
|
+
type S3ClientLike,
|
|
6
|
+
type S3Commands,
|
|
7
|
+
} from './index';
|
|
8
|
+
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// Helpers
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
|
|
13
|
+
const TEST_BUCKET = 'test-bucket';
|
|
14
|
+
const TEST_HASH =
|
|
15
|
+
'sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
|
|
16
|
+
const TEST_HEX =
|
|
17
|
+
'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
|
|
18
|
+
|
|
19
|
+
/** Compute the expected base64 of a hex hash (used in checksum headers). */
|
|
20
|
+
function hexToBase64(hex: string): string {
|
|
21
|
+
return Buffer.from(hex, 'hex').toString('base64');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
// Tag types so the mock client can identify which command was sent
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
const PUT_TAG = Symbol('PutObjectCommand');
|
|
29
|
+
const GET_TAG = Symbol('GetObjectCommand');
|
|
30
|
+
const HEAD_TAG = Symbol('HeadObjectCommand');
|
|
31
|
+
const DELETE_TAG = Symbol('DeleteObjectCommand');
|
|
32
|
+
|
|
33
|
+
interface MockCommand {
|
|
34
|
+
__tag: symbol;
|
|
35
|
+
input: Record<string, unknown>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function createMockCommands(): S3Commands {
|
|
39
|
+
return {
|
|
40
|
+
PutObjectCommand: class {
|
|
41
|
+
__tag = PUT_TAG;
|
|
42
|
+
input: Record<string, unknown>;
|
|
43
|
+
constructor(input: Record<string, unknown>) {
|
|
44
|
+
this.input = input;
|
|
45
|
+
}
|
|
46
|
+
} as unknown as S3Commands['PutObjectCommand'],
|
|
47
|
+
|
|
48
|
+
GetObjectCommand: class {
|
|
49
|
+
__tag = GET_TAG;
|
|
50
|
+
input: Record<string, unknown>;
|
|
51
|
+
constructor(input: Record<string, unknown>) {
|
|
52
|
+
this.input = input;
|
|
53
|
+
}
|
|
54
|
+
} as unknown as S3Commands['GetObjectCommand'],
|
|
55
|
+
|
|
56
|
+
HeadObjectCommand: class {
|
|
57
|
+
__tag = HEAD_TAG;
|
|
58
|
+
input: Record<string, unknown>;
|
|
59
|
+
constructor(input: Record<string, unknown>) {
|
|
60
|
+
this.input = input;
|
|
61
|
+
}
|
|
62
|
+
} as unknown as S3Commands['HeadObjectCommand'],
|
|
63
|
+
|
|
64
|
+
DeleteObjectCommand: class {
|
|
65
|
+
__tag = DELETE_TAG;
|
|
66
|
+
input: Record<string, unknown>;
|
|
67
|
+
constructor(input: Record<string, unknown>) {
|
|
68
|
+
this.input = input;
|
|
69
|
+
}
|
|
70
|
+
} as unknown as S3Commands['DeleteObjectCommand'],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// ---------------------------------------------------------------------------
|
|
75
|
+
// Mock S3 client
|
|
76
|
+
// ---------------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
interface SendCall {
|
|
79
|
+
tag: symbol;
|
|
80
|
+
input: Record<string, unknown>;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function createMockS3Client(options?: {
|
|
84
|
+
/** Value returned by send(). Can be a function of the command tag. */
|
|
85
|
+
response?:
|
|
86
|
+
| Record<string, unknown>
|
|
87
|
+
| ((tag: symbol) => Record<string, unknown>);
|
|
88
|
+
/** When true, send() rejects with a NotFound-style error. */
|
|
89
|
+
notFound?: boolean;
|
|
90
|
+
}) {
|
|
91
|
+
const calls: SendCall[] = [];
|
|
92
|
+
|
|
93
|
+
const client: S3ClientLike = {
|
|
94
|
+
async send(command: unknown) {
|
|
95
|
+
const cmd = command as MockCommand;
|
|
96
|
+
calls.push({ tag: cmd.__tag, input: cmd.input });
|
|
97
|
+
|
|
98
|
+
if (options?.notFound) {
|
|
99
|
+
const err = new Error('NotFound') as Error & {
|
|
100
|
+
name: string;
|
|
101
|
+
$metadata: { httpStatusCode: number };
|
|
102
|
+
};
|
|
103
|
+
err.name = 'NotFound';
|
|
104
|
+
err.$metadata = { httpStatusCode: 404 };
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (typeof options?.response === 'function') {
|
|
109
|
+
return options.response(cmd.__tag);
|
|
110
|
+
}
|
|
111
|
+
return options?.response ?? {};
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
return { client, calls };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// Mock getSignedUrl
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
interface SignedUrlCall {
|
|
123
|
+
command: MockCommand;
|
|
124
|
+
options: { expiresIn: number };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function createMockGetSignedUrl(): {
|
|
128
|
+
fn: GetSignedUrlFn;
|
|
129
|
+
calls: SignedUrlCall[];
|
|
130
|
+
} {
|
|
131
|
+
const calls: SignedUrlCall[] = [];
|
|
132
|
+
const fn: GetSignedUrlFn = async (_client, command, options) => {
|
|
133
|
+
const cmd = command as MockCommand;
|
|
134
|
+
calls.push({ command: cmd, options });
|
|
135
|
+
return `https://s3.example.com/presigned/${cmd.input.Key as string}`;
|
|
136
|
+
};
|
|
137
|
+
return { fn, calls };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ---------------------------------------------------------------------------
|
|
141
|
+
// Tests
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
describe('createS3BlobStorageAdapter', () => {
|
|
145
|
+
// ---- signUpload ----
|
|
146
|
+
describe('signUpload', () => {
|
|
147
|
+
test('returns presigned URL with correct method and headers', async () => {
|
|
148
|
+
const commands = createMockCommands();
|
|
149
|
+
const { client } = createMockS3Client();
|
|
150
|
+
const { fn: getSignedUrl, calls: signCalls } = createMockGetSignedUrl();
|
|
151
|
+
|
|
152
|
+
const adapter = createS3BlobStorageAdapter({
|
|
153
|
+
client,
|
|
154
|
+
bucket: TEST_BUCKET,
|
|
155
|
+
commands,
|
|
156
|
+
getSignedUrl,
|
|
157
|
+
requireChecksum: false,
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
const result = await adapter.signUpload({
|
|
161
|
+
hash: TEST_HASH,
|
|
162
|
+
size: 1024,
|
|
163
|
+
mimeType: 'image/png',
|
|
164
|
+
expiresIn: 300,
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
expect(result.method).toBe('PUT');
|
|
168
|
+
expect(result.url).toContain(TEST_HEX);
|
|
169
|
+
expect(result.headers).toBeDefined();
|
|
170
|
+
expect(result.headers!['Content-Type']).toBe('image/png');
|
|
171
|
+
expect(result.headers!['Content-Length']).toBe('1024');
|
|
172
|
+
// No checksum header when requireChecksum=false
|
|
173
|
+
expect(result.headers!['x-amz-checksum-sha256']).toBeUndefined();
|
|
174
|
+
|
|
175
|
+
// Verify the presigner was called with the right expiresIn
|
|
176
|
+
expect(signCalls).toHaveLength(1);
|
|
177
|
+
expect(signCalls[0]!.options.expiresIn).toBe(300);
|
|
178
|
+
|
|
179
|
+
// Verify PutObjectCommand was constructed with correct bucket/key
|
|
180
|
+
const cmdInput = signCalls[0]!.command.input;
|
|
181
|
+
expect(cmdInput.Bucket).toBe(TEST_BUCKET);
|
|
182
|
+
expect(cmdInput.Key).toBe(TEST_HEX);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test('includes checksum header when requireChecksum=true', async () => {
|
|
186
|
+
const commands = createMockCommands();
|
|
187
|
+
const { client } = createMockS3Client();
|
|
188
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
189
|
+
|
|
190
|
+
const adapter = createS3BlobStorageAdapter({
|
|
191
|
+
client,
|
|
192
|
+
bucket: TEST_BUCKET,
|
|
193
|
+
commands,
|
|
194
|
+
getSignedUrl,
|
|
195
|
+
requireChecksum: true,
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
const result = await adapter.signUpload({
|
|
199
|
+
hash: TEST_HASH,
|
|
200
|
+
size: 512,
|
|
201
|
+
mimeType: 'application/octet-stream',
|
|
202
|
+
expiresIn: 60,
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const expectedBase64 = hexToBase64(TEST_HEX);
|
|
206
|
+
expect(result.headers!['x-amz-checksum-sha256']).toBe(expectedBase64);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
// ---- signDownload ----
|
|
211
|
+
describe('signDownload', () => {
|
|
212
|
+
test('returns presigned URL', async () => {
|
|
213
|
+
const commands = createMockCommands();
|
|
214
|
+
const { client } = createMockS3Client();
|
|
215
|
+
const { fn: getSignedUrl, calls: signCalls } = createMockGetSignedUrl();
|
|
216
|
+
|
|
217
|
+
const adapter = createS3BlobStorageAdapter({
|
|
218
|
+
client,
|
|
219
|
+
bucket: TEST_BUCKET,
|
|
220
|
+
commands,
|
|
221
|
+
getSignedUrl,
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
const url = await adapter.signDownload({
|
|
225
|
+
hash: TEST_HASH,
|
|
226
|
+
expiresIn: 120,
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
expect(url).toContain(TEST_HEX);
|
|
230
|
+
expect(signCalls).toHaveLength(1);
|
|
231
|
+
expect(signCalls[0]!.options.expiresIn).toBe(120);
|
|
232
|
+
|
|
233
|
+
// Verify GetObjectCommand was used
|
|
234
|
+
expect(signCalls[0]!.command.__tag).toBe(GET_TAG);
|
|
235
|
+
expect(signCalls[0]!.command.input.Bucket).toBe(TEST_BUCKET);
|
|
236
|
+
expect(signCalls[0]!.command.input.Key).toBe(TEST_HEX);
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
// ---- exists ----
|
|
241
|
+
describe('exists', () => {
|
|
242
|
+
test('returns true when HeadObject succeeds', async () => {
|
|
243
|
+
const commands = createMockCommands();
|
|
244
|
+
const { client } = createMockS3Client();
|
|
245
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
246
|
+
|
|
247
|
+
const adapter = createS3BlobStorageAdapter({
|
|
248
|
+
client,
|
|
249
|
+
bucket: TEST_BUCKET,
|
|
250
|
+
commands,
|
|
251
|
+
getSignedUrl,
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
expect(await adapter.exists(TEST_HASH)).toBe(true);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
test('returns false on NotFound error', async () => {
|
|
258
|
+
const commands = createMockCommands();
|
|
259
|
+
const { client } = createMockS3Client({ notFound: true });
|
|
260
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
261
|
+
|
|
262
|
+
const adapter = createS3BlobStorageAdapter({
|
|
263
|
+
client,
|
|
264
|
+
bucket: TEST_BUCKET,
|
|
265
|
+
commands,
|
|
266
|
+
getSignedUrl,
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
expect(await adapter.exists(TEST_HASH)).toBe(false);
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// ---- delete ----
|
|
274
|
+
describe('delete', () => {
|
|
275
|
+
test('calls DeleteObjectCommand with correct bucket and key', async () => {
|
|
276
|
+
const commands = createMockCommands();
|
|
277
|
+
const { client, calls } = createMockS3Client();
|
|
278
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
279
|
+
|
|
280
|
+
const adapter = createS3BlobStorageAdapter({
|
|
281
|
+
client,
|
|
282
|
+
bucket: TEST_BUCKET,
|
|
283
|
+
commands,
|
|
284
|
+
getSignedUrl,
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
await adapter.delete(TEST_HASH);
|
|
288
|
+
|
|
289
|
+
expect(calls).toHaveLength(1);
|
|
290
|
+
expect(calls[0]!.tag).toBe(DELETE_TAG);
|
|
291
|
+
expect(calls[0]!.input.Bucket).toBe(TEST_BUCKET);
|
|
292
|
+
expect(calls[0]!.input.Key).toBe(TEST_HEX);
|
|
293
|
+
});
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
// ---- getMetadata ----
|
|
297
|
+
describe('getMetadata', () => {
|
|
298
|
+
test('returns size and mimeType from HeadObject', async () => {
|
|
299
|
+
const commands = createMockCommands();
|
|
300
|
+
const { client } = createMockS3Client({
|
|
301
|
+
response: { ContentLength: 2048, ContentType: 'image/jpeg' },
|
|
302
|
+
});
|
|
303
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
304
|
+
|
|
305
|
+
const adapter = createS3BlobStorageAdapter({
|
|
306
|
+
client,
|
|
307
|
+
bucket: TEST_BUCKET,
|
|
308
|
+
commands,
|
|
309
|
+
getSignedUrl,
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
const meta = await adapter.getMetadata!(TEST_HASH);
|
|
313
|
+
expect(meta).toEqual({ size: 2048, mimeType: 'image/jpeg' });
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test('returns null on NotFound', async () => {
|
|
317
|
+
const commands = createMockCommands();
|
|
318
|
+
const { client } = createMockS3Client({ notFound: true });
|
|
319
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
320
|
+
|
|
321
|
+
const adapter = createS3BlobStorageAdapter({
|
|
322
|
+
client,
|
|
323
|
+
bucket: TEST_BUCKET,
|
|
324
|
+
commands,
|
|
325
|
+
getSignedUrl,
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
const meta = await adapter.getMetadata!(TEST_HASH);
|
|
329
|
+
expect(meta).toBeNull();
|
|
330
|
+
});
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
// ---- put ----
|
|
334
|
+
describe('put', () => {
|
|
335
|
+
test('calls PutObjectCommand with Body and correct key', async () => {
|
|
336
|
+
const commands = createMockCommands();
|
|
337
|
+
const { client, calls } = createMockS3Client();
|
|
338
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
339
|
+
|
|
340
|
+
const adapter = createS3BlobStorageAdapter({
|
|
341
|
+
client,
|
|
342
|
+
bucket: TEST_BUCKET,
|
|
343
|
+
commands,
|
|
344
|
+
getSignedUrl,
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
const data = new Uint8Array([1, 2, 3, 4]);
|
|
348
|
+
await adapter.put!(TEST_HASH, data);
|
|
349
|
+
|
|
350
|
+
expect(calls).toHaveLength(1);
|
|
351
|
+
expect(calls[0]!.tag).toBe(PUT_TAG);
|
|
352
|
+
expect(calls[0]!.input.Bucket).toBe(TEST_BUCKET);
|
|
353
|
+
expect(calls[0]!.input.Key).toBe(TEST_HEX);
|
|
354
|
+
expect(calls[0]!.input.Body).toBe(data);
|
|
355
|
+
expect(calls[0]!.input.ContentLength).toBe(4);
|
|
356
|
+
expect(calls[0]!.input.ContentType).toBe('application/octet-stream');
|
|
357
|
+
});
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
// ---- get ----
|
|
361
|
+
describe('get', () => {
|
|
362
|
+
test('returns Uint8Array from transformToByteArray', async () => {
|
|
363
|
+
const expectedBytes = new Uint8Array([10, 20, 30]);
|
|
364
|
+
const commands = createMockCommands();
|
|
365
|
+
const { client } = createMockS3Client({
|
|
366
|
+
response: {
|
|
367
|
+
Body: {
|
|
368
|
+
transformToByteArray: async () => expectedBytes,
|
|
369
|
+
},
|
|
370
|
+
},
|
|
371
|
+
});
|
|
372
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
373
|
+
|
|
374
|
+
const adapter = createS3BlobStorageAdapter({
|
|
375
|
+
client,
|
|
376
|
+
bucket: TEST_BUCKET,
|
|
377
|
+
commands,
|
|
378
|
+
getSignedUrl,
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
const result = await adapter.get!(TEST_HASH);
|
|
382
|
+
expect(result).toBe(expectedBytes);
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
test('returns null on NotFound', async () => {
|
|
386
|
+
const commands = createMockCommands();
|
|
387
|
+
const { client } = createMockS3Client({ notFound: true });
|
|
388
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
389
|
+
|
|
390
|
+
const adapter = createS3BlobStorageAdapter({
|
|
391
|
+
client,
|
|
392
|
+
bucket: TEST_BUCKET,
|
|
393
|
+
commands,
|
|
394
|
+
getSignedUrl,
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
const result = await adapter.get!(TEST_HASH);
|
|
398
|
+
expect(result).toBeNull();
|
|
399
|
+
});
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
// ---- getStream ----
|
|
403
|
+
describe('getStream', () => {
|
|
404
|
+
test('returns ReadableStream from transformToWebStream', async () => {
|
|
405
|
+
const mockStream = new ReadableStream<Uint8Array>({
|
|
406
|
+
start(controller) {
|
|
407
|
+
controller.enqueue(new Uint8Array([5, 6, 7]));
|
|
408
|
+
controller.close();
|
|
409
|
+
},
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
const commands = createMockCommands();
|
|
413
|
+
const { client } = createMockS3Client({
|
|
414
|
+
response: {
|
|
415
|
+
Body: {
|
|
416
|
+
transformToWebStream: () => mockStream,
|
|
417
|
+
},
|
|
418
|
+
},
|
|
419
|
+
});
|
|
420
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
421
|
+
|
|
422
|
+
const adapter = createS3BlobStorageAdapter({
|
|
423
|
+
client,
|
|
424
|
+
bucket: TEST_BUCKET,
|
|
425
|
+
commands,
|
|
426
|
+
getSignedUrl,
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
const result = await adapter.getStream!(TEST_HASH);
|
|
430
|
+
expect(result).toBe(mockStream);
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
test('returns null on NotFound', async () => {
|
|
434
|
+
const commands = createMockCommands();
|
|
435
|
+
const { client } = createMockS3Client({ notFound: true });
|
|
436
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
437
|
+
|
|
438
|
+
const adapter = createS3BlobStorageAdapter({
|
|
439
|
+
client,
|
|
440
|
+
bucket: TEST_BUCKET,
|
|
441
|
+
commands,
|
|
442
|
+
getSignedUrl,
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
const result = await adapter.getStream!(TEST_HASH);
|
|
446
|
+
expect(result).toBeNull();
|
|
447
|
+
});
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
// ---- key prefix ----
|
|
451
|
+
describe('key prefix', () => {
|
|
452
|
+
test('prepends keyPrefix to all keys', async () => {
|
|
453
|
+
const commands = createMockCommands();
|
|
454
|
+
const { client, calls } = createMockS3Client();
|
|
455
|
+
const { fn: getSignedUrl, calls: signCalls } = createMockGetSignedUrl();
|
|
456
|
+
|
|
457
|
+
const adapter = createS3BlobStorageAdapter({
|
|
458
|
+
client,
|
|
459
|
+
bucket: TEST_BUCKET,
|
|
460
|
+
keyPrefix: 'blobs/',
|
|
461
|
+
commands,
|
|
462
|
+
getSignedUrl,
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
// exists -> HeadObjectCommand
|
|
466
|
+
await adapter.exists(TEST_HASH);
|
|
467
|
+
expect(calls[0]!.input.Key).toBe(`blobs/${TEST_HEX}`);
|
|
468
|
+
|
|
469
|
+
// delete -> DeleteObjectCommand
|
|
470
|
+
await adapter.delete(TEST_HASH);
|
|
471
|
+
expect(calls[1]!.input.Key).toBe(`blobs/${TEST_HEX}`);
|
|
472
|
+
|
|
473
|
+
// signUpload -> PutObjectCommand via presigner
|
|
474
|
+
await adapter.signUpload({
|
|
475
|
+
hash: TEST_HASH,
|
|
476
|
+
size: 100,
|
|
477
|
+
mimeType: 'text/plain',
|
|
478
|
+
expiresIn: 60,
|
|
479
|
+
});
|
|
480
|
+
expect(signCalls[0]!.command.input.Key).toBe(`blobs/${TEST_HEX}`);
|
|
481
|
+
|
|
482
|
+
// signDownload -> GetObjectCommand via presigner
|
|
483
|
+
await adapter.signDownload({ hash: TEST_HASH, expiresIn: 60 });
|
|
484
|
+
expect(signCalls[1]!.command.input.Key).toBe(`blobs/${TEST_HEX}`);
|
|
485
|
+
});
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
// ---- hash stripping ----
|
|
489
|
+
describe('hash stripping', () => {
|
|
490
|
+
test('strips "sha256:" prefix from hash to form the key', async () => {
|
|
491
|
+
const commands = createMockCommands();
|
|
492
|
+
const { client, calls } = createMockS3Client();
|
|
493
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
494
|
+
|
|
495
|
+
const adapter = createS3BlobStorageAdapter({
|
|
496
|
+
client,
|
|
497
|
+
bucket: TEST_BUCKET,
|
|
498
|
+
commands,
|
|
499
|
+
getSignedUrl,
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
await adapter.exists('sha256:deadbeef');
|
|
503
|
+
expect(calls[0]!.input.Key).toBe('deadbeef');
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
test('leaves hashes without "sha256:" prefix unchanged', async () => {
|
|
507
|
+
const commands = createMockCommands();
|
|
508
|
+
const { client, calls } = createMockS3Client();
|
|
509
|
+
const { fn: getSignedUrl } = createMockGetSignedUrl();
|
|
510
|
+
|
|
511
|
+
const adapter = createS3BlobStorageAdapter({
|
|
512
|
+
client,
|
|
513
|
+
bucket: TEST_BUCKET,
|
|
514
|
+
commands,
|
|
515
|
+
getSignedUrl,
|
|
516
|
+
});
|
|
517
|
+
|
|
518
|
+
await adapter.exists('deadbeef');
|
|
519
|
+
expect(calls[0]!.input.Key).toBe('deadbeef');
|
|
520
|
+
});
|
|
521
|
+
});
|
|
522
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S3-compatible blob storage adapter.
|
|
3
|
+
*
|
|
4
|
+
* Works with AWS S3, Cloudflare R2, MinIO, and other S3-compatible services.
|
|
5
|
+
* Requires @aws-sdk/client-s3 and @aws-sdk/s3-request-presigner as peer dependencies.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
BlobSignDownloadOptions,
|
|
10
|
+
BlobSignedUpload,
|
|
11
|
+
BlobSignUploadOptions,
|
|
12
|
+
BlobStorageAdapter,
|
|
13
|
+
} from '@syncular/core';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* S3 client interface (minimal subset of @aws-sdk/client-s3).
|
|
17
|
+
* This allows users to pass in their own configured S3 client.
|
|
18
|
+
*/
|
|
19
|
+
export interface S3ClientLike {
|
|
20
|
+
send(command: unknown): Promise<unknown>;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Function to create presigned URLs.
|
|
25
|
+
* This should be getSignedUrl from @aws-sdk/s3-request-presigner.
|
|
26
|
+
*/
|
|
27
|
+
export type GetSignedUrlFn = (
|
|
28
|
+
client: S3ClientLike,
|
|
29
|
+
command: unknown,
|
|
30
|
+
options: { expiresIn: number }
|
|
31
|
+
) => Promise<string>;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* S3 command constructors.
|
|
35
|
+
* These should be imported from @aws-sdk/client-s3.
|
|
36
|
+
*/
|
|
37
|
+
export interface S3Commands {
|
|
38
|
+
PutObjectCommand: new (input: {
|
|
39
|
+
Bucket: string;
|
|
40
|
+
Key: string;
|
|
41
|
+
ContentLength?: number;
|
|
42
|
+
ContentType?: string;
|
|
43
|
+
ChecksumSHA256?: string;
|
|
44
|
+
Body?: Uint8Array | ReadableStream<Uint8Array>;
|
|
45
|
+
}) => unknown;
|
|
46
|
+
GetObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
|
|
47
|
+
HeadObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
|
|
48
|
+
DeleteObjectCommand: new (input: { Bucket: string; Key: string }) => unknown;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface S3BlobStorageAdapterOptions {
|
|
52
|
+
/** S3 client instance */
|
|
53
|
+
client: S3ClientLike;
|
|
54
|
+
/** S3 bucket name */
|
|
55
|
+
bucket: string;
|
|
56
|
+
/** Optional key prefix for all blobs */
|
|
57
|
+
keyPrefix?: string;
|
|
58
|
+
/** S3 command constructors */
|
|
59
|
+
commands: S3Commands;
|
|
60
|
+
/** getSignedUrl function from @aws-sdk/s3-request-presigner */
|
|
61
|
+
getSignedUrl: GetSignedUrlFn;
|
|
62
|
+
/**
|
|
63
|
+
* Whether to require SHA-256 checksum validation on upload.
|
|
64
|
+
* Supported by S3 and R2. Default: true.
|
|
65
|
+
*/
|
|
66
|
+
requireChecksum?: boolean;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Create an S3-compatible blob storage adapter.
|
|
71
|
+
*
|
|
72
|
+
* @example
|
|
73
|
+
* ```typescript
|
|
74
|
+
* import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
|
75
|
+
* import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
76
|
+
*
|
|
77
|
+
* const adapter = createS3BlobStorageAdapter({
|
|
78
|
+
* client: new S3Client({ region: 'us-east-1' }),
|
|
79
|
+
* bucket: 'my-bucket',
|
|
80
|
+
* keyPrefix: 'blobs/',
|
|
81
|
+
* commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
|
|
82
|
+
* getSignedUrl,
|
|
83
|
+
* });
|
|
84
|
+
* ```
|
|
85
|
+
*/
|
|
86
|
+
export function createS3BlobStorageAdapter(
|
|
87
|
+
options: S3BlobStorageAdapterOptions
|
|
88
|
+
): BlobStorageAdapter {
|
|
89
|
+
const {
|
|
90
|
+
client,
|
|
91
|
+
bucket,
|
|
92
|
+
keyPrefix = '',
|
|
93
|
+
commands,
|
|
94
|
+
getSignedUrl,
|
|
95
|
+
requireChecksum = true,
|
|
96
|
+
} = options;
|
|
97
|
+
|
|
98
|
+
function getKey(hash: string): string {
|
|
99
|
+
// Remove "sha256:" prefix and use hex as key
|
|
100
|
+
const hex = hash.startsWith('sha256:') ? hash.slice(7) : hash;
|
|
101
|
+
return `${keyPrefix}${hex}`;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
name: 's3',
|
|
106
|
+
|
|
107
|
+
async signUpload(opts: BlobSignUploadOptions): Promise<BlobSignedUpload> {
|
|
108
|
+
const key = getKey(opts.hash);
|
|
109
|
+
|
|
110
|
+
// Extract hex hash for checksum (S3 expects base64-encoded SHA-256)
|
|
111
|
+
const hexHash = opts.hash.startsWith('sha256:')
|
|
112
|
+
? opts.hash.slice(7)
|
|
113
|
+
: opts.hash;
|
|
114
|
+
|
|
115
|
+
// Convert hex to base64 for S3 checksum header
|
|
116
|
+
const checksumBase64 = hexToBase64(hexHash);
|
|
117
|
+
|
|
118
|
+
const commandInput: {
|
|
119
|
+
Bucket: string;
|
|
120
|
+
Key: string;
|
|
121
|
+
ContentLength: number;
|
|
122
|
+
ContentType: string;
|
|
123
|
+
ChecksumSHA256?: string;
|
|
124
|
+
} = {
|
|
125
|
+
Bucket: bucket,
|
|
126
|
+
Key: key,
|
|
127
|
+
ContentLength: opts.size,
|
|
128
|
+
ContentType: opts.mimeType,
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
if (requireChecksum) {
|
|
132
|
+
commandInput.ChecksumSHA256 = checksumBase64;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const command = new commands.PutObjectCommand(commandInput);
|
|
136
|
+
const url = await getSignedUrl(client, command, {
|
|
137
|
+
expiresIn: opts.expiresIn,
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const headers: Record<string, string> = {
|
|
141
|
+
'Content-Type': opts.mimeType,
|
|
142
|
+
'Content-Length': String(opts.size),
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
if (requireChecksum) {
|
|
146
|
+
headers['x-amz-checksum-sha256'] = checksumBase64;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
url,
|
|
151
|
+
method: 'PUT',
|
|
152
|
+
headers,
|
|
153
|
+
};
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async signDownload(opts: BlobSignDownloadOptions): Promise<string> {
|
|
157
|
+
const key = getKey(opts.hash);
|
|
158
|
+
const command = new commands.GetObjectCommand({
|
|
159
|
+
Bucket: bucket,
|
|
160
|
+
Key: key,
|
|
161
|
+
});
|
|
162
|
+
return getSignedUrl(client, command, { expiresIn: opts.expiresIn });
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
async exists(hash: string): Promise<boolean> {
|
|
166
|
+
const key = getKey(hash);
|
|
167
|
+
try {
|
|
168
|
+
const command = new commands.HeadObjectCommand({
|
|
169
|
+
Bucket: bucket,
|
|
170
|
+
Key: key,
|
|
171
|
+
});
|
|
172
|
+
await client.send(command);
|
|
173
|
+
return true;
|
|
174
|
+
} catch (err) {
|
|
175
|
+
// Check for NotFound error
|
|
176
|
+
if (isNotFoundError(err)) {
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
throw err;
|
|
180
|
+
}
|
|
181
|
+
},
|
|
182
|
+
|
|
183
|
+
async delete(hash: string): Promise<void> {
|
|
184
|
+
const key = getKey(hash);
|
|
185
|
+
const command = new commands.DeleteObjectCommand({
|
|
186
|
+
Bucket: bucket,
|
|
187
|
+
Key: key,
|
|
188
|
+
});
|
|
189
|
+
await client.send(command);
|
|
190
|
+
},
|
|
191
|
+
|
|
192
|
+
async getMetadata(
|
|
193
|
+
hash: string
|
|
194
|
+
): Promise<{ size: number; mimeType?: string } | null> {
|
|
195
|
+
const key = getKey(hash);
|
|
196
|
+
try {
|
|
197
|
+
const command = new commands.HeadObjectCommand({
|
|
198
|
+
Bucket: bucket,
|
|
199
|
+
Key: key,
|
|
200
|
+
});
|
|
201
|
+
const response = (await client.send(command)) as {
|
|
202
|
+
ContentLength?: number;
|
|
203
|
+
ContentType?: string;
|
|
204
|
+
};
|
|
205
|
+
return {
|
|
206
|
+
size: response.ContentLength ?? 0,
|
|
207
|
+
mimeType: response.ContentType,
|
|
208
|
+
};
|
|
209
|
+
} catch (err) {
|
|
210
|
+
if (isNotFoundError(err)) {
|
|
211
|
+
return null;
|
|
212
|
+
}
|
|
213
|
+
throw err;
|
|
214
|
+
}
|
|
215
|
+
},
|
|
216
|
+
|
|
217
|
+
async put(hash: string, data: Uint8Array): Promise<void> {
|
|
218
|
+
const key = getKey(hash);
|
|
219
|
+
const command = new commands.PutObjectCommand({
|
|
220
|
+
Bucket: bucket,
|
|
221
|
+
Key: key,
|
|
222
|
+
Body: data,
|
|
223
|
+
ContentLength: data.length,
|
|
224
|
+
ContentType: 'application/octet-stream',
|
|
225
|
+
});
|
|
226
|
+
await client.send(command);
|
|
227
|
+
},
|
|
228
|
+
|
|
229
|
+
async get(hash: string): Promise<Uint8Array | null> {
|
|
230
|
+
const key = getKey(hash);
|
|
231
|
+
try {
|
|
232
|
+
const command = new commands.GetObjectCommand({
|
|
233
|
+
Bucket: bucket,
|
|
234
|
+
Key: key,
|
|
235
|
+
});
|
|
236
|
+
const response = (await client.send(command)) as {
|
|
237
|
+
Body?: { transformToByteArray(): Promise<Uint8Array> };
|
|
238
|
+
};
|
|
239
|
+
if (!response.Body) return null;
|
|
240
|
+
return response.Body.transformToByteArray();
|
|
241
|
+
} catch (err) {
|
|
242
|
+
if (isNotFoundError(err)) {
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
throw err;
|
|
246
|
+
}
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
async getStream(hash: string): Promise<ReadableStream<Uint8Array> | null> {
|
|
250
|
+
const key = getKey(hash);
|
|
251
|
+
try {
|
|
252
|
+
const command = new commands.GetObjectCommand({
|
|
253
|
+
Bucket: bucket,
|
|
254
|
+
Key: key,
|
|
255
|
+
});
|
|
256
|
+
const response = (await client.send(command)) as {
|
|
257
|
+
Body?: { transformToWebStream(): ReadableStream<Uint8Array> };
|
|
258
|
+
};
|
|
259
|
+
if (!response.Body) return null;
|
|
260
|
+
return response.Body.transformToWebStream();
|
|
261
|
+
} catch (err) {
|
|
262
|
+
if (isNotFoundError(err)) {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
throw err;
|
|
266
|
+
}
|
|
267
|
+
},
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function isNotFoundError(err: unknown): boolean {
|
|
272
|
+
if (typeof err !== 'object' || err === null) return false;
|
|
273
|
+
const e = err as { name?: string; $metadata?: { httpStatusCode?: number } };
|
|
274
|
+
return (
|
|
275
|
+
e.name === 'NotFound' ||
|
|
276
|
+
e.name === 'NoSuchKey' ||
|
|
277
|
+
e.$metadata?.httpStatusCode === 404
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function hexToBase64(hex: string): string {
|
|
282
|
+
const bytes = new Uint8Array(hex.length / 2);
|
|
283
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
284
|
+
bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Use Buffer if available (Node/Bun), otherwise manual base64
|
|
288
|
+
if (typeof Buffer !== 'undefined') {
|
|
289
|
+
return Buffer.from(bytes).toString('base64');
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// Manual base64 encoding
|
|
293
|
+
const chars =
|
|
294
|
+
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
|
|
295
|
+
let result = '';
|
|
296
|
+
const len = bytes.length;
|
|
297
|
+
const remainder = len % 3;
|
|
298
|
+
|
|
299
|
+
for (let i = 0; i < len - remainder; i += 3) {
|
|
300
|
+
const a = bytes[i]!;
|
|
301
|
+
const b = bytes[i + 1]!;
|
|
302
|
+
const c = bytes[i + 2]!;
|
|
303
|
+
result +=
|
|
304
|
+
chars.charAt((a >> 2) & 0x3f) +
|
|
305
|
+
chars.charAt(((a << 4) | (b >> 4)) & 0x3f) +
|
|
306
|
+
chars.charAt(((b << 2) | (c >> 6)) & 0x3f) +
|
|
307
|
+
chars.charAt(c & 0x3f);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
if (remainder === 1) {
|
|
311
|
+
const a = bytes[len - 1]!;
|
|
312
|
+
result += `${chars.charAt((a >> 2) & 0x3f) + chars.charAt((a << 4) & 0x3f)}==`;
|
|
313
|
+
} else if (remainder === 2) {
|
|
314
|
+
const a = bytes[len - 2]!;
|
|
315
|
+
const b = bytes[len - 1]!;
|
|
316
|
+
result +=
|
|
317
|
+
chars.charAt((a >> 2) & 0x3f) +
|
|
318
|
+
chars.charAt(((a << 4) | (b >> 4)) & 0x3f) +
|
|
319
|
+
chars.charAt((b << 2) & 0x3f) +
|
|
320
|
+
'=';
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
return result;
|
|
324
|
+
}
|