lopata 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -1
- package/src/bindings/r2.ts +289 -167
- package/src/cli/dev.ts +44 -0
- package/src/s3/chunked.ts +129 -0
- package/src/s3/headers.ts +131 -0
- package/src/s3/proxy.ts +699 -0
- package/src/s3/xml.ts +348 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lopata",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -34,6 +34,10 @@
|
|
|
34
34
|
"prepublishOnly": "bun scripts/build-assets.ts"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
|
+
"@aws-sdk/client-s3": "^3.1036.0",
|
|
38
|
+
"@aws-sdk/lib-storage": "^3.1036.0",
|
|
39
|
+
"@aws-sdk/s3-presigned-post": "^3.1036.0",
|
|
40
|
+
"@aws-sdk/s3-request-presigner": "^3.1036.0",
|
|
37
41
|
"@biomejs/biome": "^2.3.13",
|
|
38
42
|
"@cloudflare/sandbox": "^0.7.4",
|
|
39
43
|
"@types/bun": "latest",
|
package/src/bindings/r2.ts
CHANGED
|
@@ -45,13 +45,22 @@ export interface R2Checksums {
|
|
|
45
45
|
sha512?: ArrayBuffer
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
export interface R2HTTPMetadata {
|
|
49
|
+
contentType?: string
|
|
50
|
+
contentLanguage?: string
|
|
51
|
+
contentDisposition?: string
|
|
52
|
+
contentEncoding?: string
|
|
53
|
+
cacheControl?: string
|
|
54
|
+
cacheExpiry?: Date
|
|
55
|
+
}
|
|
56
|
+
|
|
48
57
|
export interface R2GetOptions {
|
|
49
58
|
onlyIf?: R2Conditional
|
|
50
59
|
range?: R2Range
|
|
51
60
|
}
|
|
52
61
|
|
|
53
62
|
export interface R2PutOptions {
|
|
54
|
-
httpMetadata?:
|
|
63
|
+
httpMetadata?: R2HTTPMetadata
|
|
55
64
|
customMetadata?: Record<string, string>
|
|
56
65
|
onlyIf?: R2Conditional
|
|
57
66
|
md5?: ArrayBuffer | string
|
|
@@ -75,7 +84,7 @@ interface R2ObjectMeta {
|
|
|
75
84
|
etag: string
|
|
76
85
|
version: string
|
|
77
86
|
uploaded: Date
|
|
78
|
-
httpMetadata:
|
|
87
|
+
httpMetadata: R2HTTPMetadata
|
|
79
88
|
customMetadata: Record<string, string>
|
|
80
89
|
checksums: R2Checksums
|
|
81
90
|
range?: { offset: number; length: number }
|
|
@@ -90,7 +99,7 @@ export class R2Object {
|
|
|
90
99
|
readonly httpEtag: string
|
|
91
100
|
readonly version: string
|
|
92
101
|
readonly uploaded: Date
|
|
93
|
-
readonly httpMetadata:
|
|
102
|
+
readonly httpMetadata: R2HTTPMetadata
|
|
94
103
|
readonly customMetadata: Record<string, string>
|
|
95
104
|
readonly checksums: R2Checksums
|
|
96
105
|
readonly storageClass: string
|
|
@@ -111,8 +120,10 @@ export class R2Object {
|
|
|
111
120
|
}
|
|
112
121
|
|
|
113
122
|
writeHttpMetadata(headers: Headers): void {
|
|
114
|
-
for (const [
|
|
115
|
-
|
|
123
|
+
for (const [header, field] of HTTP_METADATA_FIELDS) {
|
|
124
|
+
const v = this.httpMetadata[field]
|
|
125
|
+
if (!v) continue
|
|
126
|
+
headers.set(header, v instanceof Date ? v.toUTCString() : v)
|
|
116
127
|
}
|
|
117
128
|
}
|
|
118
129
|
}
|
|
@@ -120,30 +131,48 @@ export class R2Object {
|
|
|
120
131
|
// --- R2ObjectBody ---
|
|
121
132
|
|
|
122
133
|
export class R2ObjectBody extends R2Object {
|
|
123
|
-
private data: ArrayBuffer
|
|
124
134
|
readonly bodyUsed: boolean = false
|
|
135
|
+
private filePath: string
|
|
136
|
+
private rangeOffset: number
|
|
137
|
+
private rangeLength: number
|
|
138
|
+
private totalSize: number
|
|
125
139
|
|
|
126
|
-
constructor(meta: R2ObjectMeta,
|
|
140
|
+
constructor(meta: R2ObjectMeta, filePath: string, rangeOffset: number, rangeLength: number, totalSize: number) {
|
|
127
141
|
super(meta)
|
|
128
|
-
this.
|
|
142
|
+
this.filePath = filePath
|
|
143
|
+
this.rangeOffset = rangeOffset
|
|
144
|
+
this.rangeLength = rangeLength
|
|
145
|
+
this.totalSize = totalSize
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private isFullFile(): boolean {
|
|
149
|
+
return this.rangeOffset === 0 && this.rangeLength === this.totalSize
|
|
129
150
|
}
|
|
130
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Full-file bodies stream directly from disk (O(chunk) memory). Range bodies
|
|
154
|
+
* read the slice eagerly — ranges are typically small (< a few MB) so this
|
|
155
|
+
* doesn't undermine the memory guarantees the streaming write path gives us.
|
|
156
|
+
*/
|
|
131
157
|
get body(): ReadableStream<Uint8Array> {
|
|
132
|
-
|
|
158
|
+
if (this.isFullFile()) return Bun.file(this.filePath).stream()
|
|
159
|
+
const slice = Bun.file(this.filePath).slice(this.rangeOffset, this.rangeOffset + this.rangeLength)
|
|
133
160
|
return new ReadableStream({
|
|
134
|
-
start(controller) {
|
|
135
|
-
|
|
161
|
+
async start(controller) {
|
|
162
|
+
const buf = await slice.arrayBuffer()
|
|
163
|
+
controller.enqueue(new Uint8Array(buf))
|
|
136
164
|
controller.close()
|
|
137
165
|
},
|
|
138
166
|
})
|
|
139
167
|
}
|
|
140
168
|
|
|
141
169
|
async arrayBuffer(): Promise<ArrayBuffer> {
|
|
142
|
-
return this.
|
|
170
|
+
if (this.isFullFile()) return Bun.file(this.filePath).arrayBuffer()
|
|
171
|
+
return Bun.file(this.filePath).slice(this.rangeOffset, this.rangeOffset + this.rangeLength).arrayBuffer()
|
|
143
172
|
}
|
|
144
173
|
|
|
145
174
|
async text(): Promise<string> {
|
|
146
|
-
return new TextDecoder().decode(this.
|
|
175
|
+
return new TextDecoder().decode(await this.arrayBuffer())
|
|
147
176
|
}
|
|
148
177
|
|
|
149
178
|
async json<T = unknown>(): Promise<T> {
|
|
@@ -151,7 +180,7 @@ export class R2ObjectBody extends R2Object {
|
|
|
151
180
|
}
|
|
152
181
|
|
|
153
182
|
async blob(): Promise<Blob> {
|
|
154
|
-
return new Blob([this.
|
|
183
|
+
return new Blob([await this.arrayBuffer()])
|
|
155
184
|
}
|
|
156
185
|
}
|
|
157
186
|
|
|
@@ -175,12 +204,51 @@ function rowToMeta(row: R2Row): R2ObjectMeta {
|
|
|
175
204
|
etag: row.etag,
|
|
176
205
|
version: row.version ?? row.etag,
|
|
177
206
|
uploaded: new Date(row.uploaded),
|
|
178
|
-
httpMetadata:
|
|
207
|
+
httpMetadata: deserializeHttpMetadata(row.http_metadata),
|
|
179
208
|
customMetadata: row.custom_metadata ? JSON.parse(row.custom_metadata) : {},
|
|
180
209
|
checksums: row.checksums ? deserializeChecksums(JSON.parse(row.checksums)) : {},
|
|
181
210
|
}
|
|
182
211
|
}
|
|
183
212
|
|
|
213
|
+
/**
|
|
214
|
+
* (httpHeader, R2HTTPMetadata field). Drives serialization, deserialization,
|
|
215
|
+
* and HTTP header round-tripping — keep this list as the single source of truth.
|
|
216
|
+
* cacheExpiry is a Date; all others are strings.
|
|
217
|
+
*/
|
|
218
|
+
export const HTTP_METADATA_FIELDS: ReadonlyArray<
|
|
219
|
+
readonly [httpHeader: string, field: keyof R2HTTPMetadata]
|
|
220
|
+
> = [
|
|
221
|
+
['content-type', 'contentType'],
|
|
222
|
+
['content-language', 'contentLanguage'],
|
|
223
|
+
['content-disposition', 'contentDisposition'],
|
|
224
|
+
['content-encoding', 'contentEncoding'],
|
|
225
|
+
['cache-control', 'cacheControl'],
|
|
226
|
+
['expires', 'cacheExpiry'],
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
function serializeHttpMetadata(m: R2HTTPMetadata | undefined): string | null {
|
|
230
|
+
if (!m) return null
|
|
231
|
+
const obj: Record<string, string> = {}
|
|
232
|
+
for (const [, field] of HTTP_METADATA_FIELDS) {
|
|
233
|
+
const v = m[field]
|
|
234
|
+
if (!v) continue
|
|
235
|
+
obj[field] = v instanceof Date ? v.toISOString() : v
|
|
236
|
+
}
|
|
237
|
+
return Object.keys(obj).length === 0 ? null : JSON.stringify(obj)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function deserializeHttpMetadata(s: string | null): R2HTTPMetadata {
|
|
241
|
+
if (!s) return {}
|
|
242
|
+
const obj = JSON.parse(s) as Record<string, string>
|
|
243
|
+
const result: R2HTTPMetadata = {}
|
|
244
|
+
for (const [, field] of HTTP_METADATA_FIELDS) {
|
|
245
|
+
const v = obj[field]
|
|
246
|
+
if (!v) continue
|
|
247
|
+
;(result[field] as string | Date) = field === 'cacheExpiry' ? new Date(v) : v
|
|
248
|
+
}
|
|
249
|
+
return result
|
|
250
|
+
}
|
|
251
|
+
|
|
184
252
|
function serializeChecksums(c: R2Checksums): Record<string, string> {
|
|
185
253
|
const result: Record<string, string> = {}
|
|
186
254
|
for (const [k, v] of Object.entries(c)) {
|
|
@@ -201,6 +269,69 @@ function deserializeChecksums(c: Record<string, string>): R2Checksums {
|
|
|
201
269
|
return result
|
|
202
270
|
}
|
|
203
271
|
|
|
272
|
+
/**
|
|
273
|
+
* Compute the S3-compatible multipart ETag: hex(md5(concat-of-part-md5-bytes)) + "-" + N.
|
|
274
|
+
* Each part's stored etag is hex MD5 of the part body; we concatenate the raw (binary)
|
|
275
|
+
* digests and hash the result.
|
|
276
|
+
*/
|
|
277
|
+
function computeMultipartEtag(partRows: Array<{ etag: string }>): string {
|
|
278
|
+
const buf = Buffer.concat(partRows.map((r) => Buffer.from(r.etag, 'hex')))
|
|
279
|
+
const hasher = new Bun.CryptoHasher('md5')
|
|
280
|
+
hasher.update(buf)
|
|
281
|
+
return `${hasher.digest('hex')}-${partRows.length}`
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Write `value` to `filePath` while hashing it. No intermediate buffering for
|
|
286
|
+
* streams/Blobs — bytes flow directly from the source to disk. Returns MD5 hex
|
|
287
|
+
* etag and total byte count.
|
|
288
|
+
*/
|
|
289
|
+
async function writeValueToFile(
|
|
290
|
+
value: string | ArrayBuffer | ArrayBufferView | ReadableStream | Blob | null,
|
|
291
|
+
filePath: string,
|
|
292
|
+
): Promise<{ etag: string; size: number }> {
|
|
293
|
+
mkdirSync(dirname(filePath), { recursive: true })
|
|
294
|
+
const hasher = new Bun.CryptoHasher('md5')
|
|
295
|
+
|
|
296
|
+
if (value === null) {
|
|
297
|
+
await Bun.write(filePath, '')
|
|
298
|
+
return { etag: hasher.digest('hex'), size: 0 }
|
|
299
|
+
}
|
|
300
|
+
if (typeof value === 'string') {
|
|
301
|
+
const bytes = new TextEncoder().encode(value)
|
|
302
|
+
hasher.update(bytes)
|
|
303
|
+
await Bun.write(filePath, bytes)
|
|
304
|
+
return { etag: hasher.digest('hex'), size: bytes.byteLength }
|
|
305
|
+
}
|
|
306
|
+
if (value instanceof ArrayBuffer) {
|
|
307
|
+
hasher.update(new Uint8Array(value))
|
|
308
|
+
await Bun.write(filePath, value)
|
|
309
|
+
return { etag: hasher.digest('hex'), size: value.byteLength }
|
|
310
|
+
}
|
|
311
|
+
if (ArrayBuffer.isView(value)) {
|
|
312
|
+
const view = new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
313
|
+
hasher.update(view)
|
|
314
|
+
await Bun.write(filePath, view)
|
|
315
|
+
return { etag: hasher.digest('hex'), size: view.byteLength }
|
|
316
|
+
}
|
|
317
|
+
const stream = value instanceof Blob ? value.stream() : value
|
|
318
|
+
const writer = Bun.file(filePath).writer()
|
|
319
|
+
let size = 0
|
|
320
|
+
const reader = stream.getReader()
|
|
321
|
+
try {
|
|
322
|
+
while (true) {
|
|
323
|
+
const { done, value: chunk } = await reader.read()
|
|
324
|
+
if (done) break
|
|
325
|
+
hasher.update(chunk)
|
|
326
|
+
writer.write(chunk)
|
|
327
|
+
size += chunk.byteLength
|
|
328
|
+
}
|
|
329
|
+
} finally {
|
|
330
|
+
await writer.end()
|
|
331
|
+
}
|
|
332
|
+
return { etag: hasher.digest('hex'), size }
|
|
333
|
+
}
|
|
334
|
+
|
|
204
335
|
// --- Conditional check ---
|
|
205
336
|
|
|
206
337
|
function evaluateConditional(cond: R2Conditional, etag: string, uploaded: Date): boolean {
|
|
@@ -264,7 +395,10 @@ export class R2MultipartUpload {
|
|
|
264
395
|
this.limits = limits
|
|
265
396
|
}
|
|
266
397
|
|
|
267
|
-
async uploadPart(
|
|
398
|
+
async uploadPart(
|
|
399
|
+
partNumber: number,
|
|
400
|
+
data: ArrayBuffer | ArrayBufferView | string | ReadableStream | Blob,
|
|
401
|
+
): Promise<{ partNumber: number; etag: string }> {
|
|
268
402
|
// Verify upload exists and is not aborted/completed
|
|
269
403
|
const upload = this.db
|
|
270
404
|
.query<MultipartRow, [string, string]>(
|
|
@@ -273,47 +407,15 @@ export class R2MultipartUpload {
|
|
|
273
407
|
.get(this.uploadId, this.bucket)
|
|
274
408
|
if (!upload) throw new Error('Multipart upload not found or already completed/aborted')
|
|
275
409
|
|
|
276
|
-
let buf: ArrayBuffer
|
|
277
|
-
if (typeof data === 'string') {
|
|
278
|
-
buf = new TextEncoder().encode(data).buffer as ArrayBuffer
|
|
279
|
-
} else if (data instanceof Uint8Array) {
|
|
280
|
-
buf = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer
|
|
281
|
-
} else if (data instanceof ArrayBuffer) {
|
|
282
|
-
buf = data
|
|
283
|
-
} else {
|
|
284
|
-
// ReadableStream
|
|
285
|
-
const chunks: Uint8Array[] = []
|
|
286
|
-
const reader = data.getReader()
|
|
287
|
-
while (true) {
|
|
288
|
-
const { done, value } = await reader.read()
|
|
289
|
-
if (done) break
|
|
290
|
-
chunks.push(value)
|
|
291
|
-
}
|
|
292
|
-
const total = chunks.reduce((s, c) => s + c.length, 0)
|
|
293
|
-
const combined = new Uint8Array(total)
|
|
294
|
-
let offset = 0
|
|
295
|
-
for (const c of chunks) {
|
|
296
|
-
combined.set(c, offset)
|
|
297
|
-
offset += c.length
|
|
298
|
-
}
|
|
299
|
-
buf = combined.buffer as ArrayBuffer
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
const hasher = new Bun.CryptoHasher('md5')
|
|
303
|
-
hasher.update(new Uint8Array(buf))
|
|
304
|
-
const etag = hasher.digest('hex')
|
|
305
|
-
|
|
306
|
-
// Store part data to a temp file
|
|
307
410
|
const partDir = join(this.baseDir, '__multipart__', this.uploadId)
|
|
308
|
-
mkdirSync(partDir, { recursive: true })
|
|
309
411
|
const partPath = join(partDir, `part-${partNumber}`)
|
|
310
|
-
await
|
|
412
|
+
const { etag, size } = await writeValueToFile(data, partPath)
|
|
311
413
|
|
|
312
414
|
// Upsert part record
|
|
313
415
|
this.db.run(
|
|
314
416
|
`INSERT OR REPLACE INTO r2_multipart_parts (upload_id, part_number, etag, size, file_path)
|
|
315
417
|
VALUES (?, ?, ?, ?, ?)`,
|
|
316
|
-
[this.uploadId, partNumber, etag,
|
|
418
|
+
[this.uploadId, partNumber, etag, size, partPath],
|
|
317
419
|
)
|
|
318
420
|
|
|
319
421
|
return { partNumber, etag }
|
|
@@ -327,66 +429,69 @@ export class R2MultipartUpload {
|
|
|
327
429
|
.get(this.uploadId, this.bucket)
|
|
328
430
|
if (!upload) throw new Error('Multipart upload not found or already completed/aborted')
|
|
329
431
|
|
|
330
|
-
//
|
|
432
|
+
// Load all parts in one query and index by number — avoids N+1 for uploads with many parts.
|
|
433
|
+
const allRows = this.db
|
|
434
|
+
.query<MultipartPartRow, [string]>(`SELECT * FROM r2_multipart_parts WHERE upload_id = ?`)
|
|
435
|
+
.all(this.uploadId)
|
|
436
|
+
const byNumber = new Map(allRows.map((r) => [r.part_number, r]))
|
|
331
437
|
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber)
|
|
332
|
-
|
|
333
|
-
// Load all part data
|
|
334
|
-
const allParts: Uint8Array[] = []
|
|
335
|
-
let totalSize = 0
|
|
438
|
+
const partRows: MultipartPartRow[] = []
|
|
336
439
|
for (const p of sorted) {
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
.get(this.uploadId, p.partNumber)
|
|
342
|
-
if (!partRow) throw new Error(`Part ${p.partNumber} not found`)
|
|
343
|
-
if (partRow.etag !== p.etag) throw new Error(`Part ${p.partNumber} etag mismatch`)
|
|
344
|
-
const data = await Bun.file(partRow.file_path).arrayBuffer()
|
|
345
|
-
allParts.push(new Uint8Array(data))
|
|
346
|
-
totalSize += data.byteLength
|
|
440
|
+
const row = byNumber.get(p.partNumber)
|
|
441
|
+
if (!row) throw new Error(`Part ${p.partNumber} not found`)
|
|
442
|
+
if (row.etag !== p.etag) throw new Error(`Part ${p.partNumber} etag mismatch`)
|
|
443
|
+
partRows.push(row)
|
|
347
444
|
}
|
|
348
445
|
|
|
349
|
-
//
|
|
350
|
-
const combined = new Uint8Array(totalSize)
|
|
351
|
-
let offset = 0
|
|
352
|
-
for (const part of allParts) {
|
|
353
|
-
combined.set(part, offset)
|
|
354
|
-
offset += part.length
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
// Write final file
|
|
446
|
+
// Stream each part's bytes through a FileSink — O(1) memory regardless of total size.
|
|
358
447
|
const filePath = join(this.baseDir, this.key)
|
|
359
448
|
mkdirSync(dirname(filePath), { recursive: true })
|
|
360
|
-
|
|
449
|
+
const writer = Bun.file(filePath).writer()
|
|
450
|
+
let totalSize = 0
|
|
451
|
+
try {
|
|
452
|
+
for (const row of partRows) {
|
|
453
|
+
const partStream = Bun.file(row.file_path).stream()
|
|
454
|
+
const reader = partStream.getReader()
|
|
455
|
+
while (true) {
|
|
456
|
+
const { done, value } = await reader.read()
|
|
457
|
+
if (done) break
|
|
458
|
+
writer.write(value)
|
|
459
|
+
totalSize += value.byteLength
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
} finally {
|
|
463
|
+
await writer.end()
|
|
464
|
+
}
|
|
361
465
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
const etag = hasher.digest('hex')
|
|
466
|
+
// S3/CF multipart ETag: hex(md5(concat-of-part-md5-bytes)) + "-" + partCount.
|
|
467
|
+
const etag = computeMultipartEtag(partRows)
|
|
365
468
|
const uploaded = new Date()
|
|
366
469
|
const version = crypto.randomUUID()
|
|
367
470
|
|
|
368
|
-
const httpMeta =
|
|
471
|
+
const httpMeta = deserializeHttpMetadata(upload.http_metadata)
|
|
369
472
|
const customMeta = upload.custom_metadata ? JSON.parse(upload.custom_metadata) : {}
|
|
370
473
|
|
|
371
|
-
//
|
|
372
|
-
this.db.
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
474
|
+
// Atomically: persist the final object and remove multipart DB rows.
|
|
475
|
+
this.db.transaction(() => {
|
|
476
|
+
this.db.run(
|
|
477
|
+
`INSERT OR REPLACE INTO r2_objects (bucket, key, size, etag, version, uploaded, http_metadata, custom_metadata, checksums)
|
|
478
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
479
|
+
[
|
|
480
|
+
this.bucket,
|
|
481
|
+
this.key,
|
|
482
|
+
totalSize,
|
|
483
|
+
etag,
|
|
484
|
+
version,
|
|
485
|
+
uploaded.toISOString(),
|
|
486
|
+
upload.http_metadata,
|
|
487
|
+
upload.custom_metadata,
|
|
488
|
+
null,
|
|
489
|
+
],
|
|
490
|
+
)
|
|
491
|
+
this.db.run(`DELETE FROM r2_multipart_parts WHERE upload_id = ?`, [this.uploadId])
|
|
492
|
+
this.db.run(`DELETE FROM r2_multipart_uploads WHERE upload_id = ?`, [this.uploadId])
|
|
493
|
+
})()
|
|
494
|
+
this.removePartFiles()
|
|
390
495
|
|
|
391
496
|
return new R2Object({
|
|
392
497
|
key: this.key,
|
|
@@ -401,18 +506,31 @@ export class R2MultipartUpload {
|
|
|
401
506
|
}
|
|
402
507
|
|
|
403
508
|
async abort(): Promise<void> {
|
|
404
|
-
this.
|
|
509
|
+
this.db.transaction(() => {
|
|
510
|
+
this.db.run(`DELETE FROM r2_multipart_parts WHERE upload_id = ?`, [this.uploadId])
|
|
511
|
+
this.db.run(`DELETE FROM r2_multipart_uploads WHERE upload_id = ?`, [this.uploadId])
|
|
512
|
+
})()
|
|
513
|
+
this.removePartFiles()
|
|
405
514
|
}
|
|
406
515
|
|
|
407
|
-
|
|
408
|
-
|
|
516
|
+
/** Inspect parts that have been uploaded so far (used by S3 ListParts). */
|
|
517
|
+
listParts(): Array<{ partNumber: number; etag: string; size: number; lastModified: Date }> {
|
|
518
|
+
const rows = this.db
|
|
519
|
+
.query<MultipartPartRow, [string]>(
|
|
520
|
+
`SELECT * FROM r2_multipart_parts WHERE upload_id = ? ORDER BY part_number`,
|
|
521
|
+
)
|
|
522
|
+
.all(this.uploadId)
|
|
523
|
+
return rows.map((r) => ({
|
|
524
|
+
partNumber: r.part_number,
|
|
525
|
+
etag: r.etag,
|
|
526
|
+
size: r.size,
|
|
527
|
+
lastModified: new Date(),
|
|
528
|
+
}))
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
private removePartFiles(): void {
|
|
409
532
|
const partDir = join(this.baseDir, '__multipart__', this.uploadId)
|
|
410
|
-
if (existsSync(partDir)) {
|
|
411
|
-
rmSync(partDir, { recursive: true, force: true })
|
|
412
|
-
}
|
|
413
|
-
// Delete DB records
|
|
414
|
-
this.db.run(`DELETE FROM r2_multipart_parts WHERE upload_id = ?`, [this.uploadId])
|
|
415
|
-
this.db.run(`DELETE FROM r2_multipart_uploads WHERE upload_id = ?`, [this.uploadId])
|
|
533
|
+
if (existsSync(partDir)) rmSync(partDir, { recursive: true, force: true })
|
|
416
534
|
}
|
|
417
535
|
}
|
|
418
536
|
|
|
@@ -447,6 +565,11 @@ export class FileR2Bucket {
|
|
|
447
565
|
} catch {
|
|
448
566
|
// Column already exists
|
|
449
567
|
}
|
|
568
|
+
try {
|
|
569
|
+
this.db.run(`ALTER TABLE r2_objects ADD COLUMN tags TEXT`)
|
|
570
|
+
} catch {
|
|
571
|
+
// Column already exists
|
|
572
|
+
}
|
|
450
573
|
}
|
|
451
574
|
|
|
452
575
|
private ensureMultipartTables(): void {
|
|
@@ -494,34 +617,9 @@ export class FileR2Bucket {
|
|
|
494
617
|
return join(this.baseDir, key)
|
|
495
618
|
}
|
|
496
619
|
|
|
497
|
-
private async readValue(
|
|
498
|
-
value: string | ArrayBuffer | ReadableStream | Blob | null,
|
|
499
|
-
): Promise<ArrayBuffer> {
|
|
500
|
-
if (value === null) return new ArrayBuffer(0)
|
|
501
|
-
if (typeof value === 'string') return new TextEncoder().encode(value).buffer as ArrayBuffer
|
|
502
|
-
if (value instanceof ArrayBuffer) return value
|
|
503
|
-
if (value instanceof Blob) return await value.arrayBuffer()
|
|
504
|
-
// ReadableStream
|
|
505
|
-
const chunks: Uint8Array[] = []
|
|
506
|
-
const reader = value.getReader()
|
|
507
|
-
while (true) {
|
|
508
|
-
const { done, value: chunk } = await reader.read()
|
|
509
|
-
if (done) break
|
|
510
|
-
chunks.push(chunk)
|
|
511
|
-
}
|
|
512
|
-
const total = chunks.reduce((s, c) => s + c.length, 0)
|
|
513
|
-
const buf = new Uint8Array(total)
|
|
514
|
-
let offset = 0
|
|
515
|
-
for (const c of chunks) {
|
|
516
|
-
buf.set(c, offset)
|
|
517
|
-
offset += c.length
|
|
518
|
-
}
|
|
519
|
-
return buf.buffer as ArrayBuffer
|
|
520
|
-
}
|
|
521
|
-
|
|
522
620
|
async put(
|
|
523
621
|
key: string,
|
|
524
|
-
value: string | ArrayBuffer | ReadableStream | Blob | null,
|
|
622
|
+
value: string | ArrayBuffer | ArrayBufferView | ReadableStream | Blob | null,
|
|
525
623
|
options?: R2PutOptions,
|
|
526
624
|
): Promise<R2Object | null> {
|
|
527
625
|
this.validateKey(key)
|
|
@@ -538,19 +636,12 @@ export class FileR2Bucket {
|
|
|
538
636
|
}
|
|
539
637
|
}
|
|
540
638
|
|
|
541
|
-
const data = await this.readValue(value)
|
|
542
|
-
|
|
543
639
|
const fp = this.filePath(key)
|
|
544
|
-
|
|
545
|
-
await Bun.write(fp, data)
|
|
546
|
-
|
|
547
|
-
const hasher = new Bun.CryptoHasher('md5')
|
|
548
|
-
hasher.update(new Uint8Array(data))
|
|
549
|
-
const etag = hasher.digest('hex')
|
|
640
|
+
const { etag, size } = await writeValueToFile(value, fp)
|
|
550
641
|
const uploaded = new Date()
|
|
551
642
|
const version = crypto.randomUUID()
|
|
552
643
|
|
|
553
|
-
// Build checksums
|
|
644
|
+
// Build checksums. md5 is always computed; user-supplied sha* are stored as-is.
|
|
554
645
|
const checksums: R2Checksums = { md5: Buffer.from(etag, 'hex').buffer as ArrayBuffer }
|
|
555
646
|
for (const algo of ['sha1', 'sha256', 'sha384', 'sha512'] as const) {
|
|
556
647
|
const provided = options?.[algo]
|
|
@@ -565,11 +656,11 @@ export class FileR2Bucket {
|
|
|
565
656
|
[
|
|
566
657
|
this.bucket,
|
|
567
658
|
key,
|
|
568
|
-
|
|
659
|
+
size,
|
|
569
660
|
etag,
|
|
570
661
|
version,
|
|
571
662
|
uploaded.toISOString(),
|
|
572
|
-
options?.httpMetadata
|
|
663
|
+
serializeHttpMetadata(options?.httpMetadata),
|
|
573
664
|
options?.customMetadata ? JSON.stringify(options.customMetadata) : null,
|
|
574
665
|
JSON.stringify(serializeChecksums(checksums)),
|
|
575
666
|
],
|
|
@@ -577,7 +668,7 @@ export class FileR2Bucket {
|
|
|
577
668
|
|
|
578
669
|
return new R2Object({
|
|
579
670
|
key,
|
|
580
|
-
size
|
|
671
|
+
size,
|
|
581
672
|
etag,
|
|
582
673
|
version,
|
|
583
674
|
uploaded,
|
|
@@ -609,34 +700,23 @@ export class FileR2Bucket {
|
|
|
609
700
|
}
|
|
610
701
|
}
|
|
611
702
|
|
|
612
|
-
const
|
|
613
|
-
|
|
614
|
-
let
|
|
615
|
-
|
|
616
|
-
// Handle range reads
|
|
703
|
+
const totalSize = row.size
|
|
704
|
+
let offset = 0
|
|
705
|
+
let length = totalSize
|
|
617
706
|
if (options?.range) {
|
|
618
707
|
const range = options.range
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
if ('suffix' in range && range.suffix !== undefined) {
|
|
623
|
-
// suffix: last N bytes
|
|
624
|
-
offset = Math.max(0, data.byteLength - range.suffix)
|
|
625
|
-
length = data.byteLength - offset
|
|
708
|
+
if (range.suffix !== undefined) {
|
|
709
|
+
offset = Math.max(0, totalSize - range.suffix)
|
|
710
|
+
length = totalSize - offset
|
|
626
711
|
} else {
|
|
627
712
|
offset = range.offset ?? 0
|
|
628
|
-
length = range.length ??
|
|
629
|
-
|
|
630
|
-
if (offset + length > data.byteLength) {
|
|
631
|
-
length = data.byteLength - offset
|
|
632
|
-
}
|
|
713
|
+
length = range.length ?? totalSize - offset
|
|
714
|
+
if (offset + length > totalSize) length = totalSize - offset
|
|
633
715
|
}
|
|
634
|
-
|
|
635
|
-
data = data.slice(offset, offset + length)
|
|
636
716
|
meta.range = { offset, length }
|
|
637
717
|
}
|
|
638
718
|
|
|
639
|
-
return new R2ObjectBody(meta,
|
|
719
|
+
return new R2ObjectBody(meta, this.filePath(key), offset, length, totalSize)
|
|
640
720
|
}
|
|
641
721
|
|
|
642
722
|
async head(key: string): Promise<R2Object | null> {
|
|
@@ -716,7 +796,7 @@ export class FileR2Bucket {
|
|
|
716
796
|
|
|
717
797
|
async createMultipartUpload(
|
|
718
798
|
key: string,
|
|
719
|
-
options?: { httpMetadata?:
|
|
799
|
+
options?: { httpMetadata?: R2HTTPMetadata; customMetadata?: Record<string, string> },
|
|
720
800
|
): Promise<R2MultipartUpload> {
|
|
721
801
|
this.validateKey(key)
|
|
722
802
|
this.validateCustomMetadata(options?.customMetadata)
|
|
@@ -729,7 +809,7 @@ export class FileR2Bucket {
|
|
|
729
809
|
uploadId,
|
|
730
810
|
this.bucket,
|
|
731
811
|
key,
|
|
732
|
-
options?.httpMetadata
|
|
812
|
+
serializeHttpMetadata(options?.httpMetadata),
|
|
733
813
|
options?.customMetadata ? JSON.stringify(options.customMetadata) : null,
|
|
734
814
|
new Date().toISOString(),
|
|
735
815
|
],
|
|
@@ -741,6 +821,48 @@ export class FileR2Bucket {
|
|
|
741
821
|
resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload {
|
|
742
822
|
return new R2MultipartUpload(this.db, this.bucket, this.baseDir, key, uploadId, this.limits)
|
|
743
823
|
}
|
|
824
|
+
|
|
825
|
+
/** Read tags for a key. Returns [] if the object has no tags or doesn't exist. */
|
|
826
|
+
getTags(key: string): Array<{ key: string; value: string }> {
|
|
827
|
+
const row = this.db
|
|
828
|
+
.query<{ tags: string | null }, [string, string]>(
|
|
829
|
+
`SELECT tags FROM r2_objects WHERE bucket = ? AND key = ?`,
|
|
830
|
+
)
|
|
831
|
+
.get(this.bucket, key)
|
|
832
|
+
if (!row?.tags) return []
|
|
833
|
+
return JSON.parse(row.tags) as Array<{ key: string; value: string }>
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/** Replace an object's tag set. No-op if the object does not exist. */
|
|
837
|
+
setTags(key: string, tags: Array<{ key: string; value: string }>): void {
|
|
838
|
+
const serialized = tags.length === 0 ? null : JSON.stringify(tags)
|
|
839
|
+
this.db.run(`UPDATE r2_objects SET tags = ? WHERE bucket = ? AND key = ?`, [
|
|
840
|
+
serialized,
|
|
841
|
+
this.bucket,
|
|
842
|
+
key,
|
|
843
|
+
])
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/** List in-progress multipart uploads in this bucket (used by S3 ListMultipartUploads). */
|
|
847
|
+
listMultipartUploads(prefix?: string): Array<{ key: string; uploadId: string; initiated: Date }> {
|
|
848
|
+
interface Row {
|
|
849
|
+
upload_id: string
|
|
850
|
+
key: string
|
|
851
|
+
created_at: string
|
|
852
|
+
}
|
|
853
|
+
const rows = prefix
|
|
854
|
+
? this.db
|
|
855
|
+
.query<Row, [string, string]>(
|
|
856
|
+
`SELECT upload_id, key, created_at FROM r2_multipart_uploads WHERE bucket = ? AND key LIKE ? ORDER BY key`,
|
|
857
|
+
)
|
|
858
|
+
.all(this.bucket, prefix + '%')
|
|
859
|
+
: this.db
|
|
860
|
+
.query<Row, [string]>(
|
|
861
|
+
`SELECT upload_id, key, created_at FROM r2_multipart_uploads WHERE bucket = ? ORDER BY key`,
|
|
862
|
+
)
|
|
863
|
+
.all(this.bucket)
|
|
864
|
+
return rows.map((r) => ({ key: r.key, uploadId: r.upload_id, initiated: new Date(r.created_at) }))
|
|
865
|
+
}
|
|
744
866
|
}
|
|
745
867
|
|
|
746
868
|
function buildListObject(meta: R2ObjectMeta, include?: ('httpMetadata' | 'customMetadata')[]): R2Object {
|