@sproutboat/wire 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/broker.ts CHANGED
@@ -18,8 +18,9 @@
18
18
  * the only SSRF control today.
19
19
  */
20
20
  import { Database, type Statement } from "bun:sqlite";
21
- import { createHash } from "node:crypto";
22
- import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
21
+ import { createHash, randomBytes } from "node:crypto";
22
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statfsSync, statSync, writeFileSync } from "node:fs";
23
+ import { open } from "node:fs/promises";
23
24
  import { dirname, join, normalize, resolve } from "node:path";
24
25
  import { parseArgs } from "node:util";
25
26
  import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
@@ -84,6 +85,16 @@ export type BrokerOptions = {
84
85
  * data outlives a redeploy.
85
86
  */
86
87
  resourceDir?: string;
88
+ /** Account whose R2 resources this broker may mutate. */
89
+ ownerId?: string;
90
+ /** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
91
+ r2QuotaBytes?: number;
92
+ /** Resource ids belonging to ownerId, used to seed the shared R2 usage ledger. */
93
+ r2ResourceIds?: string[];
94
+ /** Free disk capacity retained while direct R2 uploads are accepted. */
95
+ r2MinFreeBytes?: number;
96
+ /** Test hook for checking the filesystem capacity available to temporary uploads. */
97
+ freeBytes?: (path: string) => number;
87
98
  token?: string;
88
99
  bindings?: Partial<Bindings>;
89
100
  secrets?: Record<string, string>;
@@ -115,7 +126,28 @@ type R2Row = {
115
126
  uploaded: string;
116
127
  http_json: string;
117
128
  custom_json: string;
129
+ blob_id: string | null;
118
130
  };
131
+ type R2UploadRow = {
132
+ key: string;
133
+ uploaded: string;
134
+ http_json: string;
135
+ custom_json: string;
136
+ };
137
+ type R2PartRow = { part_number: number; size: number; etag: string };
138
+ type R2UploadIdentity = { bucket: string; key: string; upload_id: string };
139
+ type R2TransferRow = {
140
+ bucket: string;
141
+ key: string;
142
+ method: string;
143
+ expires: number;
144
+ max_bytes: number;
145
+ expected_sha256: string;
146
+ http_json: string;
147
+ custom_json: string;
148
+ used: number;
149
+ };
150
+ const MULTIPART_EXPIRY_MS = 7 * 24 * 60 * 60 * 1000;
119
151
  const isNumber = (v: JsonValue | undefined): v is number => Number.isFinite(v);
120
152
  const sqlParams = (v: JsonValue | undefined): SqlParam[] => {
121
153
  if (!Array.isArray(v)) return [];
@@ -144,6 +176,8 @@ export type Broker = {
144
176
  * negotiation.
145
177
  */
146
178
  handleFrame(payload: Buffer): Promise<Buffer>;
179
+ /** Serve one reserved direct R2 upload or download request. */
180
+ transfer(request: Request): Promise<Response>;
147
181
  close(): void;
148
182
  };
149
183
 
@@ -230,8 +264,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
230
264
  conn.exec("PRAGMA journal_mode = WAL");
231
265
  conn.exec("PRAGMA synchronous = NORMAL");
232
266
  conn.exec(
233
- "CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (ns, key))",
267
+ "CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, " +
268
+ "expires_at INTEGER, PRIMARY KEY (ns, key))",
234
269
  );
270
+ const kvColumns = conn.query<{ name: string }, []>("PRAGMA table_info(kv)").all();
271
+ if (!kvColumns.some((column) => column.name === "expires_at")) conn.exec("ALTER TABLE kv ADD COLUMN expires_at INTEGER");
235
272
  // #56 — no `body` column: an object's bytes live in their own file under
236
273
  // `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
237
274
  // Doubling every object through SQLite's own page cache on top of the
@@ -241,8 +278,26 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
241
278
  conn.exec(
242
279
  "CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
243
280
  "etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
281
+ "blob_id TEXT, " +
244
282
  "PRIMARY KEY (bucket, key))",
245
283
  );
284
+ const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
285
+ if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
286
+ conn.exec(
287
+ "CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
288
+ "uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
289
+ "PRIMARY KEY (bucket, key, upload_id))",
290
+ );
291
+ conn.exec(
292
+ "CREATE TABLE IF NOT EXISTS r2_part (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
293
+ "part_number INTEGER NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, " +
294
+ "PRIMARY KEY (bucket, key, upload_id, part_number))",
295
+ );
296
+ conn.exec(
297
+ "CREATE TABLE IF NOT EXISTS r2_transfer (token TEXT PRIMARY KEY, bucket TEXT NOT NULL, key TEXT NOT NULL, " +
298
+ "method TEXT NOT NULL, expires INTEGER NOT NULL, max_bytes INTEGER NOT NULL, expected_sha256 TEXT NOT NULL DEFAULT '', " +
299
+ "http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', used INTEGER NOT NULL DEFAULT 0)",
300
+ );
246
301
  conn.exec(
247
302
  "CREATE TABLE IF NOT EXISTS mq (queue TEXT NOT NULL, id TEXT PRIMARY KEY, body TEXT NOT NULL, " +
248
303
  "visible_at INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, dead INTEGER NOT NULL DEFAULT 0)",
@@ -290,6 +345,121 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
290
345
  return conn;
291
346
  };
292
347
 
348
+ const quotaOwner = opts.ownerId;
349
+ const quotaBytes = opts.r2QuotaBytes && opts.r2QuotaBytes > 0 ? opts.r2QuotaBytes : null;
350
+ const quotaDirectory = resourceDir;
351
+ if (quotaOwner && quotaDirectory && quotaBytes) mkdirSync(quotaDirectory, { recursive: true });
352
+ // One ledger for every account, shared by the brokers of all its deployments.
353
+ // Ticket reservations are recorded before an upload begins, so four accepted
354
+ // requests cannot each independently decide that the same remaining bytes fit.
355
+ const quotaDb = quotaOwner && quotaDirectory && quotaBytes
356
+ ? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
357
+ : null;
358
+ if (quotaDb) {
359
+ const owner = quotaOwner!;
360
+ quotaDb.exec("PRAGMA journal_mode = WAL");
361
+ quotaDb.exec(
362
+ "CREATE TABLE IF NOT EXISTS r2_account_quota (owner TEXT PRIMARY KEY, used_bytes INTEGER NOT NULL, reserved_bytes INTEGER NOT NULL)",
363
+ );
364
+ quotaDb.exec(
365
+ "CREATE TABLE IF NOT EXISTS r2_quota_reservation (token TEXT PRIMARY KEY, owner TEXT NOT NULL, bytes INTEGER NOT NULL, expires INTEGER NOT NULL)",
366
+ );
367
+ quotaDb.exec(
368
+ "CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
369
+ );
370
+ const existing = quotaDb.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?").get(owner);
371
+ if (!existing) {
372
+ quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
373
+ }
374
+ quotaDb.query("INSERT OR IGNORE INTO r2_transfer_metric (owner) VALUES (?)").run(owner);
375
+ }
376
+
377
+ const committedR2Bytes = (): number => {
378
+ if (!quotaDirectory) return 0;
379
+ let used = 0;
380
+ for (const id of opts.r2ResourceIds ?? []) {
381
+ // Open through resourceDb so older resource files gain the multipart
382
+ // tables before the aggregate is queried.
383
+ const store = resourceDb(id);
384
+ used += store.query<{ bytes: number }, []>(
385
+ "SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
386
+ ).get()?.bytes ?? 0;
387
+ }
388
+ return used;
389
+ };
390
+
391
+ const availableBytes = (): number => {
392
+ if (!quotaDirectory) return Number.POSITIVE_INFINITY;
393
+ if (opts.freeBytes) return opts.freeBytes(quotaDirectory);
394
+ const stats = statfsSync(quotaDirectory);
395
+ return Number(stats.bavail) * Number(stats.bsize);
396
+ };
397
+
398
+ const releaseExpiredR2Reservations = (now = Date.now()): void => {
399
+ if (!quotaDb || !quotaOwner) return;
400
+ quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
401
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
402
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
403
+ ).get(quotaOwner)?.bytes ?? 0;
404
+ quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
405
+ };
406
+
407
+ const reserveR2 = (token: string, bytes: number, expires: number): "ok" | "quota" | "disk" => {
408
+ if (!quotaDb || !quotaOwner || !quotaBytes) return "ok";
409
+ quotaDb.exec("BEGIN IMMEDIATE");
410
+ try {
411
+ releaseExpiredR2Reservations();
412
+ const used = committedR2Bytes();
413
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
414
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
415
+ ).get(quotaOwner)?.bytes ?? 0;
416
+ quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
417
+ if (used + reserved + bytes > quotaBytes) {
418
+ quotaDb.exec("ROLLBACK");
419
+ return "quota";
420
+ }
421
+ const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
422
+ if (availableBytes() < bytes + floor) {
423
+ quotaDb.exec("ROLLBACK");
424
+ return "disk";
425
+ }
426
+ quotaDb.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)").run(token, quotaOwner, bytes, expires);
427
+ quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved + bytes, quotaOwner);
428
+ quotaDb.exec("COMMIT");
429
+ return "ok";
430
+ } catch (error) {
431
+ quotaDb.exec("ROLLBACK");
432
+ throw error;
433
+ }
434
+ };
435
+
436
+ const settleR2 = (token: string): void => {
437
+ if (!quotaDb || !quotaOwner) return;
438
+ quotaDb.exec("BEGIN IMMEDIATE");
439
+ try {
440
+ const reservation = quotaDb.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?").get(token);
441
+ if (!reservation) {
442
+ quotaDb.exec("COMMIT");
443
+ return;
444
+ }
445
+ quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
446
+ const used = committedR2Bytes();
447
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
448
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
449
+ ).get(quotaOwner)?.bytes ?? 0;
450
+ quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
451
+ quotaDb.exec("COMMIT");
452
+ } catch (error) {
453
+ quotaDb.exec("ROLLBACK");
454
+ throw error;
455
+ }
456
+ };
457
+
458
+ const recordR2TransferMetric = (field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size"): void => {
459
+ if (!quotaDb || !quotaOwner) return;
460
+ quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
461
+ };
462
+
293
463
  /**
294
464
  * Where a KV / R2 / queue binding stores: its resource's own file keyed by
295
465
  * `id` when the config bound it to one, else the per-broker `db` partitioned
@@ -303,29 +473,48 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
303
473
  };
304
474
 
305
475
  type KvStmts = {
306
- get: Statement<{ value: string }, [string, string]>;
307
- put: Statement<unknown, [string, string, string]>;
476
+ get: Statement<{ value: string; expires_at: number | null }, [string, string]>;
477
+ put: Statement<unknown, [string, string, string, number | null]>;
308
478
  del: Statement<unknown, [string, string]>;
309
- list: Statement<{ key: string }, [string, string]>;
479
+ list: Statement<{ key: string }, [string, string, number]>;
310
480
  };
311
481
  const kvStmtCache = new Map<Database, KvStmts>();
312
482
  const kvStmts = (store: Database): KvStmts => {
313
483
  let stmts = kvStmtCache.get(store);
314
484
  if (!stmts) {
315
485
  stmts = {
316
- get: store.query<{ value: string }, [string, string]>("SELECT value FROM kv WHERE ns = ? AND key = ?"),
486
+ get: store.query<{ value: string; expires_at: number | null }, [string, string]>(
487
+ "SELECT value, expires_at FROM kv WHERE ns = ? AND key = ?",
488
+ ),
317
489
  put: store.query(
318
- "INSERT INTO kv (ns, key, value) VALUES (?1, ?2, ?3) ON CONFLICT (ns, key) DO UPDATE SET value = ?3",
490
+ "INSERT INTO kv (ns, key, value, expires_at) VALUES (?1, ?2, ?3, ?4) " +
491
+ "ON CONFLICT (ns, key) DO UPDATE SET value = ?3, expires_at = ?4",
319
492
  ),
320
493
  del: store.query("DELETE FROM kv WHERE ns = ? AND key = ?"),
321
- list: store.query<{ key: string }, [string, string]>(
322
- "SELECT key FROM kv WHERE ns = ? AND key LIKE ? || '%' ORDER BY key LIMIT 1000",
494
+ // #58 — an expired row is filtered out here; a periodic sweep to hard-delete
495
+ // it is a separate concern (ponytail: rows accumulate until touched by a
496
+ // get/list/put on the same key/prefix, add a cron sweep if that matters).
497
+ list: store.query<{ key: string }, [string, string, number]>(
498
+ "SELECT key FROM kv WHERE ns = ? AND key LIKE ? || '%' AND (expires_at IS NULL OR expires_at > ?) ORDER BY key LIMIT 1000",
323
499
  ),
324
500
  };
325
501
  kvStmtCache.set(store, stmts);
326
502
  }
327
503
  return stmts;
328
504
  };
505
+ /** CF's own floor. Cloudflare rejects `expirationTtl < 60`; matched here so a
506
+ * handler ported from Workers gets the same validation error either place. */
507
+ const MIN_KV_TTL_SECONDS = 60;
508
+ /** `{ expirationTtl }` (seconds from now) or `{ expiration }` (absolute unix
509
+ * seconds) -> ms epoch, or `null` for no expiry. */
510
+ const kvExpiresAt = (msg: Frame): number | null => {
511
+ if (isNumber(msg.expirationTtl)) {
512
+ if (msg.expirationTtl < MIN_KV_TTL_SECONDS) throw new Error(`expirationTtl must be at least ${MIN_KV_TTL_SECONDS} seconds`);
513
+ return Date.now() + msg.expirationTtl * 1000;
514
+ }
515
+ if (isNumber(msg.expiration)) return msg.expiration * 1000;
516
+ return null;
517
+ };
329
518
 
330
519
  const bound =
331
520
  (list: string[], kind: string) =>
@@ -428,39 +617,482 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
428
617
  * `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
429
618
  */
430
619
  const r2MemBlobs = new Map<string, Uint8Array>();
431
- const r2BlobId = (bucketPart: string, key: string): string =>
432
- createHash("sha256").update(`${bucketPart}\0${key}`).digest("hex");
620
+ const r2MemParts = new Map<string, Uint8Array>();
621
+ const r2BlobId = (bucketPart: string, key: string, generation?: string | null): string =>
622
+ createHash("sha256")
623
+ .update(generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`)
624
+ .digest("hex");
433
625
  const r2BlobDir = (store: Database): string | null => {
434
626
  const file = store.filename;
435
627
  return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
436
628
  };
437
- const r2BlobPath = (store: Database, bucketPart: string, key: string): string | null => {
629
+ const r2BlobPath = (store: Database, bucketPart: string, key: string, generation?: string | null): string | null => {
438
630
  const dir = r2BlobDir(store);
439
- return dir ? join(dir, `${r2BlobId(bucketPart, key)}.blob`) : null;
631
+ return dir ? join(dir, `${r2BlobId(bucketPart, key, generation)}.blob`) : null;
632
+ };
633
+ const r2BlobMemKey = (bucketPart: string, key: string, generation?: string | null): string =>
634
+ generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`;
635
+ const r2WriteBlob = (
636
+ store: Database,
637
+ bucketPart: string,
638
+ key: string,
639
+ bytes: Uint8Array,
640
+ generation: string,
641
+ ): void => {
642
+ const path = r2BlobPath(store, bucketPart, key, generation);
643
+ if (!path) {
644
+ r2MemBlobs.set(r2BlobMemKey(bucketPart, key, generation), bytes);
645
+ return;
646
+ }
647
+ mkdirSync(dirname(path), { recursive: true });
648
+ const temporary = `${path}.tmp`;
649
+ writeFileSync(temporary, bytes);
650
+ renameSync(temporary, path);
651
+ };
652
+ const r2ReadBlob = (
653
+ store: Database,
654
+ bucketPart: string,
655
+ key: string,
656
+ generation?: string | null,
657
+ ): Uint8Array | null => {
658
+ const path = r2BlobPath(store, bucketPart, key, generation);
659
+ if (!path) return r2MemBlobs.get(r2BlobMemKey(bucketPart, key, generation)) ?? null;
660
+ return existsSync(path) ? readFileSync(path) : null;
440
661
  };
441
- const r2WriteBlob = (store: Database, bucketPart: string, key: string, bytes: Uint8Array): void => {
442
- const path = r2BlobPath(store, bucketPart, key);
662
+ const r2DeleteBlob = (store: Database, bucketPart: string, key: string, generation?: string | null): void => {
663
+ const path = r2BlobPath(store, bucketPart, key, generation);
443
664
  if (!path) {
444
- r2MemBlobs.set(`${bucketPart}\0${key}`, bytes);
665
+ r2MemBlobs.delete(r2BlobMemKey(bucketPart, key, generation));
666
+ return;
667
+ }
668
+ if (existsSync(path)) rmSync(path);
669
+ };
670
+ const r2PartId = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
671
+ createHash("sha256").update(`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`).digest("hex");
672
+ const r2PartPath = (
673
+ store: Database,
674
+ bucketPart: string,
675
+ key: string,
676
+ uploadId: string,
677
+ partNumber: number,
678
+ ): string | null => {
679
+ const file = store.filename;
680
+ if (!file || file === ":memory:") return null;
681
+ return join(dirname(resolve(file)), "r2-parts", `${r2PartId(bucketPart, key, uploadId, partNumber)}.part`);
682
+ };
683
+ const r2PartMemKey = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
684
+ `${bucketPart}\0${key}\0${uploadId}\0${partNumber}`;
685
+ const r2WritePart = (
686
+ store: Database,
687
+ bucketPart: string,
688
+ key: string,
689
+ uploadId: string,
690
+ partNumber: number,
691
+ bytes: Uint8Array,
692
+ ): void => {
693
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
694
+ if (!path) {
695
+ r2MemParts.set(r2PartMemKey(bucketPart, key, uploadId, partNumber), bytes);
445
696
  return;
446
697
  }
447
698
  mkdirSync(dirname(path), { recursive: true });
448
- writeFileSync(path, bytes);
699
+ const temporary = `${path}.tmp`;
700
+ writeFileSync(temporary, bytes);
701
+ renameSync(temporary, path);
449
702
  };
450
- const r2ReadBlob = (store: Database, bucketPart: string, key: string): Uint8Array | null => {
451
- const path = r2BlobPath(store, bucketPart, key);
452
- if (!path) return r2MemBlobs.get(`${bucketPart}\0${key}`) ?? null;
703
+ const r2ReadPart = (
704
+ store: Database,
705
+ bucketPart: string,
706
+ key: string,
707
+ uploadId: string,
708
+ partNumber: number,
709
+ ): Uint8Array | null => {
710
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
711
+ if (!path) return r2MemParts.get(r2PartMemKey(bucketPart, key, uploadId, partNumber)) ?? null;
453
712
  return existsSync(path) ? readFileSync(path) : null;
454
713
  };
455
- const r2DeleteBlob = (store: Database, bucketPart: string, key: string): void => {
456
- const path = r2BlobPath(store, bucketPart, key);
714
+ const r2DeletePart = (
715
+ store: Database,
716
+ bucketPart: string,
717
+ key: string,
718
+ uploadId: string,
719
+ partNumber: number,
720
+ ): void => {
721
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
457
722
  if (!path) {
458
- r2MemBlobs.delete(`${bucketPart}\0${key}`);
723
+ r2MemParts.delete(r2PartMemKey(bucketPart, key, uploadId, partNumber));
459
724
  return;
460
725
  }
461
726
  if (existsSync(path)) rmSync(path);
462
727
  };
463
728
 
729
+ const requireMultipart = (store: Database, bucket: string, key: string, uploadId: string): R2UploadRow => {
730
+ const row = store
731
+ .query<R2UploadRow, [string, string, string]>(
732
+ "SELECT key, uploaded, http_json, custom_json FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?",
733
+ )
734
+ .get(bucket, key, uploadId);
735
+ if (!row) throw new Error("multipart upload not found");
736
+ return row;
737
+ };
738
+
739
+ const multipartParts = (store: Database, bucket: string, key: string, uploadId: string): R2PartRow[] =>
740
+ store
741
+ .query<R2PartRow, [string, string, string]>(
742
+ "SELECT part_number, size, etag FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ? ORDER BY part_number",
743
+ )
744
+ .all(bucket, key, uploadId);
745
+
746
+ const requestedParts = (value: JsonValue | undefined): Array<{ partNumber: number; etag: string }> => {
747
+ if (!Array.isArray(value) || value.length === 0) throw new Error("complete requires at least one uploaded part");
748
+ return value.map((part) => {
749
+ const parsed = jsonObject(part);
750
+ const partNumber = Number(parsed?.partNumber);
751
+ const etag = isString(parsed?.etag) ? parsed.etag : "";
752
+ if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000 || !etag) {
753
+ throw new Error("invalid uploaded part");
754
+ }
755
+ return { partNumber, etag };
756
+ });
757
+ };
758
+
759
+ const validateCompletedParts = (stored: R2PartRow[], requested: Array<{ partNumber: number; etag: string }>): void => {
760
+ if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
761
+ for (let index = 0; index < stored.length; index++) {
762
+ const actual = stored[index];
763
+ const expected = requested[index];
764
+ if (actual.part_number !== expected.partNumber || actual.etag !== expected.etag) {
765
+ throw new Error("uploaded parts do not match");
766
+ }
767
+ if (index > 0 && expected.partNumber <= requested[index - 1].partNumber) {
768
+ throw new Error("uploaded parts must be ordered by partNumber");
769
+ }
770
+ if (index < stored.length - 1 && actual.size !== stored[0].size) {
771
+ throw new Error("all multipart parts except the last must have the same size");
772
+ }
773
+ if (index === stored.length - 1 && stored.length > 1 && actual.size > stored[0].size) {
774
+ throw new Error("the last multipart part cannot be larger than the others");
775
+ }
776
+ }
777
+ };
778
+
779
+ const deleteMultipart = (store: Database, bucket: string, key: string, uploadId: string): void => {
780
+ for (const part of multipartParts(store, bucket, key, uploadId)) {
781
+ r2DeletePart(store, bucket, key, uploadId, part.part_number);
782
+ }
783
+ store.query("DELETE FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
784
+ store.query("DELETE FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
785
+ };
786
+
787
+ const cleanupExpiredMultipart = (store: Database): void => {
788
+ const cutoff = new Date(Date.now() - MULTIPART_EXPIRY_MS).toISOString();
789
+ const expired = store
790
+ .query<R2UploadIdentity, [string]>("SELECT bucket, key, upload_id FROM r2_upload WHERE uploaded < ?")
791
+ .all(cutoff);
792
+ for (const upload of expired) deleteMultipart(store, upload.bucket, upload.key, upload.upload_id);
793
+ };
794
+
795
+ const commitR2Metadata = (
796
+ store: Database,
797
+ bucket: string,
798
+ row: R2Row,
799
+ previous: Pick<R2Row, "blob_id"> | null,
800
+ ): void => {
801
+ try {
802
+ store
803
+ .query(
804
+ "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json, blob_id) " +
805
+ "VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
806
+ "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, " +
807
+ "custom_json=?7, blob_id=?8",
808
+ )
809
+ .run(bucket, row.key, row.size, row.etag, row.uploaded, row.http_json, row.custom_json, row.blob_id);
810
+ } catch (error) {
811
+ if (previous?.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, row.blob_id);
812
+ throw error;
813
+ }
814
+ if (previous && previous.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, previous.blob_id);
815
+ };
816
+
817
+ const putR2 = (
818
+ store: Database,
819
+ bucket: string,
820
+ key: string,
821
+ bytes: Uint8Array,
822
+ httpJson: string,
823
+ customJson: string,
824
+ ): R2Row => {
825
+ const previous = store
826
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
827
+ .get(bucket, key);
828
+ const etag = createHash("sha256").update(bytes).digest("hex");
829
+ const row: R2Row = {
830
+ key,
831
+ size: bytes.byteLength,
832
+ etag,
833
+ uploaded: new Date().toISOString(),
834
+ http_json: httpJson,
835
+ custom_json: customJson,
836
+ blob_id: etag,
837
+ };
838
+ r2WriteBlob(store, bucket, key, bytes, etag);
839
+ commitR2Metadata(store, bucket, row, previous);
840
+ return row;
841
+ };
842
+
843
+ const transferLimit = (): number => {
844
+ const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
845
+ return Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
846
+ };
847
+ let activeUploads = 0;
848
+ const maxActiveUploads = 4;
849
+
850
+ const cleanupTransferTemps = (store: Database): void => {
851
+ const directory = r2BlobDir(store);
852
+ if (!directory || !existsSync(directory)) return;
853
+ const now = Date.now();
854
+ const cutoff = Date.now() - 24 * 60 * 60 * 1000;
855
+ for (const name of readdirSync(directory)) {
856
+ if (!/^[0-9a-f]{24}\.upload\.tmp$/.test(name)) continue;
857
+ const path = join(directory, name);
858
+ try {
859
+ const tokenValue = name.slice(0, 24);
860
+ const ticket = store.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?").get(tokenValue);
861
+ if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
862
+ } catch (error) {
863
+ if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
864
+ }
865
+ }
866
+ };
867
+
868
+ const transferPath = (binding: string, tokenValue: string): string =>
869
+ `/__sb/r2/transfer/${encodeURIComponent(binding)}/${tokenValue}`;
870
+
871
+ const parseRange = (header: string | null, size: number): { start: number; end: number } | null => {
872
+ if (!header) return { start: 0, end: Math.max(0, size - 1) };
873
+ const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
874
+ if (!match || size === 0) return null;
875
+ if (!match[1]) {
876
+ const suffix = Number(match[2]);
877
+ if (!Number.isSafeInteger(suffix) || suffix <= 0) return null;
878
+ return { start: Math.max(0, size - suffix), end: size - 1 };
879
+ }
880
+ const start = Number(match[1]);
881
+ const requestedEnd = match[2] ? Number(match[2]) : size - 1;
882
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return null;
883
+ return { start, end: Math.min(requestedEnd, size - 1) };
884
+ };
885
+
886
+ const transfer = async (request: Request): Promise<Response> => {
887
+ const url = new URL(request.url);
888
+ const match = /^\/__sb\/r2\/transfer\/([A-Z][A-Z0-9_]*)\/([0-9a-f]{24})$/.exec(url.pathname);
889
+ if (!match) return new Response("not found", { status: 404 });
890
+ let store: Database;
891
+ let bucket: string;
892
+ try {
893
+ ({ store, part: bucket } = storeFor("r2", requireR2(match[1])));
894
+ } catch {
895
+ return new Response("not found", { status: 404 });
896
+ }
897
+ const tokenValue = match[2];
898
+ const ticket = store
899
+ .query<R2TransferRow, [string]>(
900
+ "SELECT bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json, used " +
901
+ "FROM r2_transfer WHERE token = ?",
902
+ )
903
+ .get(tokenValue);
904
+ const method = request.method.toUpperCase();
905
+ const allowedMethod = ticket?.method === "upload" ? "PUT" : ticket?.method === "download" ? "GET, HEAD" : "";
906
+ if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
907
+ return new Response("transfer ticket expired or already used", { status: 410 });
908
+ }
909
+ if ((ticket.method === "upload" && method !== "PUT") || (ticket.method === "download" && method !== "GET" && method !== "HEAD")) {
910
+ return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
911
+ }
912
+ if (ticket.method === "download") {
913
+ const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, ticket.key);
914
+ if (!row) return new Response("not found", { status: 404 });
915
+ const etag = `"${row.etag}"`;
916
+ if (request.headers.get("if-none-match") === etag) return new Response(null, { status: 304, headers: { etag } });
917
+ const rangeHeader = request.headers.get("range");
918
+ const range = parseRange(rangeHeader, row.size);
919
+ if (!range) {
920
+ return new Response(null, { status: 416, headers: { "content-range": `bytes */${row.size}`, etag } });
921
+ }
922
+ const path = r2BlobPath(store, bucket, ticket.key, row.blob_id);
923
+ if (!path || !existsSync(path)) return new Response("not found", { status: 404 });
924
+ const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
925
+ if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
926
+ const length = row.size === 0 ? 0 : range.end - range.start + 1;
927
+ const metadata = jsonObject(parseJsonValue(row.http_json));
928
+ const headers = new Headers({
929
+ "accept-ranges": "bytes",
930
+ "content-length": String(length),
931
+ etag,
932
+ });
933
+ if (isString(metadata?.contentType)) headers.set("content-type", metadata.contentType);
934
+ if (rangeHeader) headers.set("content-range", `bytes ${range.start}-${range.end}/${row.size}`);
935
+ const body = method === "HEAD" ? null : Bun.file(path).slice(range.start, range.start + length);
936
+ return new Response(body, { status: rangeHeader ? 206 : 200, headers });
937
+ }
938
+
939
+ const declared = request.headers.has("content-length") ? Number(request.headers.get("content-length")) : NaN;
940
+ if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
941
+ return new Response("invalid content length", { status: 400 });
942
+ }
943
+ if (Number.isFinite(declared) && declared > ticket.max_bytes) return new Response("upload too large", { status: 413 });
944
+ const directory = r2BlobDir(store);
945
+ if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
946
+ if (activeUploads >= maxActiveUploads) {
947
+ return new Response("too many active uploads", { status: 503, headers: { "retry-after": "5" } });
948
+ }
949
+ const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
950
+ if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
951
+ activeUploads++;
952
+ const temporary = join(directory, `${tokenValue}.upload.tmp`);
953
+ await (async () => {
954
+ mkdirSync(directory, { recursive: true });
955
+ const reservation = await open(temporary, "wx");
956
+ await reservation.close();
957
+ })().catch((error) => {
958
+ activeUploads--;
959
+ throw error;
960
+ });
961
+ const writer = Bun.file(temporary).writer({ highWaterMark: 64 * 1024 });
962
+ const hash = createHash("sha256");
963
+ let size = 0;
964
+ try {
965
+ if (request.body) {
966
+ for await (const chunk of request.body) {
967
+ size += chunk.byteLength;
968
+ if (size > ticket.max_bytes) throw new RangeError("upload too large");
969
+ const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
970
+ if (floor > 0 && availableBytes() < chunk.byteLength + floor) {
971
+ const error = new Error("R2 storage is temporarily unavailable");
972
+ Object.assign(error, { code: "SB_R2_DISK_FLOOR" });
973
+ throw error;
974
+ }
975
+ hash.update(chunk);
976
+ const written = await writer.write(chunk);
977
+ if (written !== chunk.byteLength) throw new Error("upload write was incomplete");
978
+ }
979
+ }
980
+ await writer.end();
981
+ const persisted = await open(temporary, "r");
982
+ try {
983
+ await persisted.sync();
984
+ } finally {
985
+ await persisted.close();
986
+ }
987
+ const etag = hash.digest("hex");
988
+ if (ticket.expected_sha256 && ticket.expected_sha256 !== etag) {
989
+ rmSync(temporary);
990
+ settleR2(tokenValue);
991
+ return Response.json({ error: "sha256 mismatch", actual: etag }, { status: 422 });
992
+ }
993
+ const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
994
+ renameSync(temporary, destination);
995
+ const previous = store
996
+ .query<Pick<R2Row, "blob_id" | "size">, [string, string]>("SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?")
997
+ .get(bucket, ticket.key);
998
+ const row: R2Row = {
999
+ key: ticket.key,
1000
+ size,
1001
+ etag,
1002
+ uploaded: new Date().toISOString(),
1003
+ http_json: ticket.http_json,
1004
+ custom_json: ticket.custom_json,
1005
+ blob_id: etag,
1006
+ };
1007
+ commitR2Metadata(store, bucket, row, previous);
1008
+ settleR2(tokenValue);
1009
+ return Response.json({ object: r2Row(row) }, { status: 201, headers: { etag: `"${etag}"` } });
1010
+ } catch (error) {
1011
+ try {
1012
+ await writer.end();
1013
+ } catch {
1014
+ // Preserve the original upload failure while releasing the file sink.
1015
+ }
1016
+ if (existsSync(temporary)) rmSync(temporary);
1017
+ settleR2(tokenValue);
1018
+ if (error instanceof RangeError) {
1019
+ recordR2TransferMetric("rejected_size");
1020
+ return new Response(error.message, { status: 413 });
1021
+ }
1022
+ if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "SB_R2_DISK_FLOOR")) {
1023
+ recordR2TransferMetric("rejected_disk");
1024
+ return new Response("R2 storage is temporarily unavailable", { status: 507 });
1025
+ }
1026
+ throw error;
1027
+ } finally {
1028
+ activeUploads--;
1029
+ }
1030
+ };
1031
+
1032
+ const completeMultipart = (
1033
+ store: Database,
1034
+ bucket: string,
1035
+ key: string,
1036
+ uploadId: string,
1037
+ requested: Array<{ partNumber: number; etag: string }>,
1038
+ ): ReturnType<typeof r2Row> => {
1039
+ const upload = requireMultipart(store, bucket, key, uploadId);
1040
+ const parts = multipartParts(store, bucket, key, uploadId);
1041
+ validateCompletedParts(parts, requested);
1042
+
1043
+ const hash = createHash("sha256");
1044
+ let size = 0;
1045
+ const blobDirectory = r2BlobDir(store);
1046
+ let inMemoryBody: Uint8Array | null = null;
1047
+ let temporary: string | null = null;
1048
+ if (blobDirectory) {
1049
+ mkdirSync(blobDirectory, { recursive: true });
1050
+ temporary = join(blobDirectory, `${uploadId}.tmp`);
1051
+ writeFileSync(temporary, new Uint8Array(0));
1052
+ try {
1053
+ for (const part of parts) {
1054
+ const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
1055
+ if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
1056
+ appendFileSync(temporary, bytes);
1057
+ hash.update(bytes);
1058
+ size += bytes.byteLength;
1059
+ }
1060
+ } catch (error) {
1061
+ if (existsSync(temporary)) rmSync(temporary);
1062
+ throw error;
1063
+ }
1064
+ } else {
1065
+ const chunks: Uint8Array[] = [];
1066
+ for (const part of parts) {
1067
+ const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
1068
+ if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
1069
+ chunks.push(bytes);
1070
+ hash.update(bytes);
1071
+ size += bytes.byteLength;
1072
+ }
1073
+ inMemoryBody = Buffer.concat(chunks);
1074
+ }
1075
+
1076
+ const etag = hash.digest("hex");
1077
+ const previous = store
1078
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
1079
+ .get(bucket, key);
1080
+ if (temporary) renameSync(temporary, r2BlobPath(store, bucket, key, etag)!);
1081
+ else r2WriteBlob(store, bucket, key, inMemoryBody ?? new Uint8Array(0), etag);
1082
+ const row: R2Row = {
1083
+ key,
1084
+ size,
1085
+ etag,
1086
+ uploaded: new Date().toISOString(),
1087
+ http_json: upload.http_json,
1088
+ custom_json: upload.custom_json,
1089
+ blob_id: etag,
1090
+ };
1091
+ commitR2Metadata(store, bucket, row, previous);
1092
+ deleteMultipart(store, bucket, key, uploadId);
1093
+ return r2Row(row);
1094
+ };
1095
+
464
1096
  const r2Row = (r: R2Row) => ({
465
1097
  key: r.key,
466
1098
  size: r.size,
@@ -580,12 +1212,18 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
580
1212
  return { ok: true, op: "pong", echo: msg.msg ?? null, pid: process.pid };
581
1213
  case "kv.get": {
582
1214
  const { store, part } = storeFor("kv", requireKv(msg.ns));
583
- const row = kvStmts(store).get.get(part, str(msg.key));
584
- return row ? { ok: true, found: true, value: row.value } : { ok: true, found: false, value: null };
1215
+ const stmts = kvStmts(store);
1216
+ const row = stmts.get.get(part, str(msg.key));
1217
+ if (!row) return { ok: true, found: false, value: null };
1218
+ if (row.expires_at != null && row.expires_at <= Date.now()) {
1219
+ stmts.del.run(part, str(msg.key));
1220
+ return { ok: true, found: false, value: null };
1221
+ }
1222
+ return { ok: true, found: true, value: row.value };
585
1223
  }
586
1224
  case "kv.put": {
587
1225
  const { store, part } = storeFor("kv", requireKv(msg.ns));
588
- kvStmts(store).put.run(part, str(msg.key), str(msg.value));
1226
+ kvStmts(store).put.run(part, str(msg.key), str(msg.value), kvExpiresAt(msg));
589
1227
  return { ok: true };
590
1228
  }
591
1229
  case "kv.delete": {
@@ -598,7 +1236,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
598
1236
  return {
599
1237
  ok: true,
600
1238
  keys: kvStmts(store)
601
- .list.all(part, str(msg.prefix))
1239
+ .list.all(part, str(msg.prefix), Date.now())
602
1240
  .map((r) => r.key),
603
1241
  };
604
1242
  }
@@ -656,24 +1294,107 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
656
1294
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
657
1295
  const key = str(msg.key);
658
1296
  const bytes = new TextEncoder().encode(str(msg.body));
659
- const etag = createHash("sha256").update(bytes).digest("hex");
1297
+ return {
1298
+ ok: true,
1299
+ object: r2Row(
1300
+ putR2(
1301
+ store,
1302
+ bucket,
1303
+ key,
1304
+ bytes,
1305
+ JSON.stringify(msg.httpMetadata ?? {}),
1306
+ JSON.stringify(msg.customMetadata ?? {}),
1307
+ ),
1308
+ ),
1309
+ };
1310
+ }
1311
+ case "r2.multipart.create": {
1312
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1313
+ cleanupExpiredMultipart(store);
1314
+ const key = str(msg.key);
1315
+ const uploadId = str(msg.uploadId) || newId();
660
1316
  const uploaded = new Date().toISOString();
661
- r2WriteBlob(store, bucket, key, bytes);
662
1317
  store
663
1318
  .query(
664
- "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7) " +
665
- "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, custom_json=?7",
1319
+ "INSERT INTO r2_upload (bucket, key, upload_id, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6) " +
1320
+ "ON CONFLICT (bucket, key, upload_id) DO NOTHING",
666
1321
  )
667
1322
  .run(
668
1323
  bucket,
669
1324
  key,
670
- bytes.byteLength,
671
- etag,
1325
+ uploadId,
672
1326
  uploaded,
673
1327
  JSON.stringify(msg.httpMetadata ?? {}),
674
1328
  JSON.stringify(msg.customMetadata ?? {}),
675
1329
  );
676
- return { ok: true, object: { key, size: bytes.byteLength, etag, uploaded } };
1330
+ return { ok: true, key, uploadId };
1331
+ }
1332
+ case "r2.transfer.create": {
1333
+ const binding = requireR2(msg.bucket);
1334
+ const { store, part: bucket } = storeFor("r2", binding);
1335
+ const key = str(msg.key);
1336
+ const method = msg.method === "download" ? "download" : msg.method === "upload" ? "upload" : "";
1337
+ if (!method) throw new Error("transfer method must be upload or download");
1338
+ const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
1339
+ const expires = Date.now() + expiresIn * 1000;
1340
+ const maxBytes =
1341
+ method === "upload"
1342
+ ? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
1343
+ : 0;
1344
+ const expectedSha256 = str(msg.sha256).toLowerCase();
1345
+ if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
1346
+ throw new Error("sha256 must be 64 hexadecimal characters");
1347
+ }
1348
+ store.query("DELETE FROM r2_transfer WHERE expires < ? OR used = 1").run(Date.now());
1349
+ cleanupTransferTemps(store);
1350
+ const tokenValue = randomBytes(12).toString("hex");
1351
+ const reservation = method === "upload" ? reserveR2(tokenValue, maxBytes, expires) : "ok";
1352
+ if (reservation === "quota") {
1353
+ recordR2TransferMetric("rejected_quota");
1354
+ throw new Error("R2 account quota would be exceeded");
1355
+ }
1356
+ if (reservation === "disk") {
1357
+ recordR2TransferMetric("rejected_disk");
1358
+ throw new Error("R2 storage is temporarily unavailable");
1359
+ }
1360
+ try {
1361
+ store
1362
+ .query(
1363
+ "INSERT INTO r2_transfer (token, bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json) " +
1364
+ "VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
1365
+ )
1366
+ .run(
1367
+ tokenValue,
1368
+ bucket,
1369
+ key,
1370
+ method,
1371
+ expires,
1372
+ maxBytes,
1373
+ expectedSha256,
1374
+ JSON.stringify(msg.httpMetadata ?? {}),
1375
+ JSON.stringify(msg.customMetadata ?? {}),
1376
+ );
1377
+ } catch (error) {
1378
+ if (reservation === "ok" && method === "upload") settleR2(tokenValue);
1379
+ throw error;
1380
+ }
1381
+ recordR2TransferMetric("tickets_issued");
1382
+ return { ok: true, url: transferPath(binding, tokenValue), expiresAt: new Date(expires).toISOString() };
1383
+ }
1384
+ case "r2.multipart.complete": {
1385
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1386
+ const key = str(msg.key);
1387
+ const uploadId = str(msg.uploadId);
1388
+ const object = completeMultipart(store, bucket, key, uploadId, requestedParts(msg.parts));
1389
+ return { ok: true, object };
1390
+ }
1391
+ case "r2.multipart.abort": {
1392
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1393
+ const key = str(msg.key);
1394
+ const uploadId = str(msg.uploadId);
1395
+ requireMultipart(store, bucket, key, uploadId);
1396
+ deleteMultipart(store, bucket, key, uploadId);
1397
+ return { ok: true };
677
1398
  }
678
1399
  case "r2.get":
679
1400
  case "r2.head": {
@@ -681,15 +1402,19 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
681
1402
  const key = str(msg.key);
682
1403
  const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
683
1404
  if (!row) return { ok: true, found: false };
684
- const body =
685
- msg.op === "r2.get" ? new TextDecoder().decode(r2ReadBlob(store, bucket, key) ?? new Uint8Array(0)) : undefined;
1405
+ const body = msg.op === "r2.get"
1406
+ ? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
1407
+ : undefined;
686
1408
  return { ok: true, found: true, object: r2Row(row), body };
687
1409
  }
688
1410
  case "r2.delete": {
689
1411
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
690
1412
  const key = str(msg.key);
1413
+ const row = store
1414
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
1415
+ .get(bucket, key);
691
1416
  store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
692
- r2DeleteBlob(store, bucket, key);
1417
+ if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
693
1418
  return { ok: true };
694
1419
  }
695
1420
  case "r2.list": {
@@ -876,24 +1601,35 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
876
1601
  // second full-object copy through the database's own page cache.
877
1602
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
878
1603
  const key = str(msg.key);
1604
+ const row = putR2(
1605
+ store,
1606
+ bucket,
1607
+ key,
1608
+ binary,
1609
+ JSON.stringify(msg.httpMetadata ?? {}),
1610
+ JSON.stringify(msg.customMetadata ?? {}),
1611
+ );
1612
+ return { reply: { ok: true, object: r2Row(row) } };
1613
+ }
1614
+
1615
+ if (msg.op === "r2.multipart.put") {
1616
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1617
+ const key = str(msg.key);
1618
+ const uploadId = str(msg.uploadId);
1619
+ const partNumber = Number(msg.partNumber);
1620
+ if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
1621
+ throw new Error("partNumber must be an integer between 1 and 10000");
1622
+ }
1623
+ requireMultipart(store, bucket, key, uploadId);
879
1624
  const etag = createHash("sha256").update(binary).digest("hex");
880
- const uploaded = new Date().toISOString();
881
- r2WriteBlob(store, bucket, key, binary);
1625
+ r2WritePart(store, bucket, key, uploadId, partNumber, binary);
882
1626
  store
883
1627
  .query(
884
- "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7) " +
885
- "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, custom_json=?7",
1628
+ "INSERT INTO r2_part (bucket, key, upload_id, part_number, size, etag) VALUES (?1,?2,?3,?4,?5,?6) " +
1629
+ "ON CONFLICT (bucket, key, upload_id, part_number) DO UPDATE SET size=?5, etag=?6",
886
1630
  )
887
- .run(
888
- bucket,
889
- key,
890
- binary.byteLength,
891
- etag,
892
- uploaded,
893
- JSON.stringify(msg.httpMetadata ?? {}),
894
- JSON.stringify(msg.customMetadata ?? {}),
895
- );
896
- return { reply: { ok: true, object: { key, size: binary.byteLength, etag, uploaded } } };
1631
+ .run(bucket, key, uploadId, partNumber, binary.byteLength, etag);
1632
+ return { reply: { ok: true, part: { partNumber, etag } } };
897
1633
  }
898
1634
 
899
1635
  if (msg.op === "r2.get") {
@@ -901,7 +1637,10 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
901
1637
  const key = str(msg.key);
902
1638
  const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
903
1639
  if (!row) return { reply: { ok: true, found: false } };
904
- return { reply: { ok: true, found: true, object: r2Row(row) }, bytes: r2ReadBlob(store, bucket, key) ?? new Uint8Array(0) };
1640
+ return {
1641
+ reply: { ok: true, found: true, object: r2Row(row) },
1642
+ bytes: r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0),
1643
+ };
905
1644
  }
906
1645
 
907
1646
  if (msg.op === "assets.get") {
@@ -939,7 +1678,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
939
1678
  if (!msg) throw new Error("request frame was not a JSON object");
940
1679
  if (token && str(msg.token) !== token) return frameOf(encodeV1({ ok: false, error: "unauthorized" }));
941
1680
  const binary = new Uint8Array(payload.subarray(5 + jsonLen));
1681
+ const id = isSafeInteger(msg.id) ? msg.id : null;
1682
+ const replayKey = id === null ? null : `${str(msg.op)}:${id}`;
1683
+ if (replayKey && REPLAYABLE.has(str(msg.op))) {
1684
+ const seen = replayed.get(replayKey);
1685
+ if (seen) return frameOf(encodeV1(seen));
1686
+ }
942
1687
  const { reply, bytes } = await dispatchBinary(msg, binary);
1688
+ if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
943
1689
  return frameOf(encodeV1(reply, bytes));
944
1690
  } catch (e) {
945
1691
  return frameOf(encodeV1({ ok: false, error: e instanceof Error ? e.message : String(e) }));
@@ -966,6 +1712,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
966
1712
  "d1.backup",
967
1713
  "ratelimit.check",
968
1714
  "r2.put",
1715
+ "r2.multipart.create",
1716
+ "r2.multipart.put",
1717
+ "r2.multipart.complete",
1718
+ "r2.multipart.abort",
1719
+ "r2.transfer.create",
969
1720
  "r2.delete",
970
1721
  "do.storage.put",
971
1722
  "do.storage.delete",
@@ -979,6 +1730,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
979
1730
  * milliseconds, so this only has to outlive a reconnect, not a session. */
980
1731
  const REPLAY_WINDOW = 256;
981
1732
 
1733
+ const rememberReplay = (key: string, reply: Frame): void => {
1734
+ replayed.set(key, reply);
1735
+ if (replayed.size > REPLAY_WINDOW) {
1736
+ const oldest = replayed.keys().next().value;
1737
+ if (oldest !== undefined) replayed.delete(oldest);
1738
+ }
1739
+ };
1740
+
982
1741
  /**
983
1742
  * Run `msg`, or replay what it answered last time. Keyed by the id the sprout
984
1743
  * put on the request; a request without one is always run.
@@ -991,12 +1750,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
991
1750
  const seen = replayed.get(key);
992
1751
  if (seen) return seen;
993
1752
  const reply = await dispatch(msg);
994
- replayed.set(key, reply);
995
- // Insertion-ordered, so the oldest key is the first one out.
996
- if (replayed.size > REPLAY_WINDOW) {
997
- const oldest = replayed.keys().next().value;
998
- if (oldest !== undefined) replayed.delete(oldest);
999
- }
1753
+ rememberReplay(key, reply);
1000
1754
  return reply;
1001
1755
  }
1002
1756
 
@@ -1143,6 +1897,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
1143
1897
  dispatch,
1144
1898
  handlePayload,
1145
1899
  handleFrame,
1900
+ transfer,
1146
1901
  close: () => {
1147
1902
  for (const t of timers) clearInterval(t);
1148
1903
  for (const conn of d1Conns.values()) conn.close();
@@ -1230,17 +1985,33 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
1230
1985
  },
1231
1986
  },
1232
1987
  });
1233
- return { port: server.port, stop: () => server.stop(true) };
1988
+ return { port: server.port ?? port, stop: () => server.stop(true) };
1989
+ }
1990
+
1991
+ /** Start the HTTP listener that serves short-lived direct R2 transfer tickets. */
1992
+ export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
1993
+ const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
1994
+ const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
1995
+ const server = Bun.serve({ hostname, port, maxRequestBodySize, fetch(request, server) {
1996
+ server.timeout(request, 255);
1997
+ return broker.transfer(request);
1998
+ } });
1999
+ return { port: server.port ?? port, stop: () => server.stop(true) };
1234
2000
  }
1235
2001
 
1236
2002
  if (import.meta.main) {
1237
2003
  const { values } = parseArgs({
1238
2004
  options: {
1239
2005
  port: { type: "string" },
2006
+ "transfer-port": { type: "string" },
1240
2007
  token: { type: "string" },
1241
2008
  db: { type: "string" },
1242
2009
  "data-dir": { type: "string" },
1243
2010
  "resource-dir": { type: "string" },
2011
+ "owner-id": { type: "string" },
2012
+ "r2-quota-bytes": { type: "string" },
2013
+ "r2-resource-ids": { type: "string" },
2014
+ "r2-min-free-bytes": { type: "string" },
1244
2015
  bindings: { type: "string" },
1245
2016
  secrets: { type: "string" },
1246
2017
  "sprout-url": { type: "string" },
@@ -1289,6 +2060,10 @@ if (import.meta.main) {
1289
2060
  db: values.db,
1290
2061
  dataDir: values["data-dir"],
1291
2062
  resourceDir: values["resource-dir"],
2063
+ ownerId: values["owner-id"],
2064
+ r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
2065
+ r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
2066
+ r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
1292
2067
  token: values.token ?? process.env.SB_BROKER_TOKEN,
1293
2068
  bindings,
1294
2069
  secrets,
@@ -1299,7 +2074,9 @@ if (import.meta.main) {
1299
2074
  dispatchEnabled: () => dispatchEnabled,
1300
2075
  });
1301
2076
  const { port } = listen(broker, "127.0.0.1", Number(values.port ?? process.env.SB_BROKER_PORT ?? 0));
2077
+ const transferPort = Number(values["transfer-port"] ?? process.env.SB_R2_TRANSFER_PORT ?? 0);
2078
+ const transferServer = transferPort > 0 ? listenTransfers(broker, "127.0.0.1", transferPort) : null;
1302
2079
  console.log(
1303
- `sproutboat broker: 127.0.0.1:${port} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
2080
+ `sproutboat broker: 127.0.0.1:${port} transfer=${transferServer?.port ?? "off"} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
1304
2081
  );
1305
2082
  }