@sproutboat/wire 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/broker.ts CHANGED
@@ -18,8 +18,9 @@
18
18
  * the only SSRF control today.
19
19
  */
20
20
  import { Database, type Statement } from "bun:sqlite";
21
- import { createHash } from "node:crypto";
22
- import { existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
21
+ import { createHash, randomBytes } from "node:crypto";
22
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statfsSync, statSync, writeFileSync } from "node:fs";
23
+ import { open } from "node:fs/promises";
23
24
  import { dirname, join, normalize, resolve } from "node:path";
24
25
  import { parseArgs } from "node:util";
25
26
  import { resolveAssetKey, type AssetManifest } from "@sproutboat/assets";
@@ -84,6 +85,16 @@ export type BrokerOptions = {
84
85
  * data outlives a redeploy.
85
86
  */
86
87
  resourceDir?: string;
88
+ /** Account whose R2 resources this broker may mutate. */
89
+ ownerId?: string;
90
+ /** Account-wide R2 byte ceiling. Zero disables the ceiling for local development. */
91
+ r2QuotaBytes?: number;
92
+ /** Resource ids belonging to ownerId, used to seed the shared R2 usage ledger. */
93
+ r2ResourceIds?: string[];
94
+ /** Free disk capacity retained while direct R2 uploads are accepted. */
95
+ r2MinFreeBytes?: number;
96
+ /** Test hook for checking the filesystem capacity available to temporary uploads. */
97
+ freeBytes?: (path: string) => number;
87
98
  token?: string;
88
99
  bindings?: Partial<Bindings>;
89
100
  secrets?: Record<string, string>;
@@ -110,14 +121,33 @@ type D1Result = {
110
121
  };
111
122
  type R2Row = {
112
123
  key: string;
113
- /** TEXT from a v0 write, BLOB from a v1 one; sqlite hands each back as-is. */
114
- body: string | Uint8Array;
115
124
  size: number;
116
125
  etag: string;
117
126
  uploaded: string;
118
127
  http_json: string;
119
128
  custom_json: string;
129
+ blob_id: string | null;
120
130
  };
131
+ type R2UploadRow = {
132
+ key: string;
133
+ uploaded: string;
134
+ http_json: string;
135
+ custom_json: string;
136
+ };
137
+ type R2PartRow = { part_number: number; size: number; etag: string };
138
+ type R2UploadIdentity = { bucket: string; key: string; upload_id: string };
139
+ type R2TransferRow = {
140
+ bucket: string;
141
+ key: string;
142
+ method: string;
143
+ expires: number;
144
+ max_bytes: number;
145
+ expected_sha256: string;
146
+ http_json: string;
147
+ custom_json: string;
148
+ used: number;
149
+ };
150
+ const MULTIPART_EXPIRY_MS = 7 * 24 * 60 * 60 * 1000;
121
151
  const isNumber = (v: JsonValue | undefined): v is number => Number.isFinite(v);
122
152
  const sqlParams = (v: JsonValue | undefined): SqlParam[] => {
123
153
  if (!Array.isArray(v)) return [];
@@ -146,6 +176,8 @@ export type Broker = {
146
176
  * negotiation.
147
177
  */
148
178
  handleFrame(payload: Buffer): Promise<Buffer>;
179
+ /** Serve one reserved direct R2 upload or download request. */
180
+ transfer(request: Request): Promise<Response>;
149
181
  close(): void;
150
182
  };
151
183
 
@@ -234,11 +266,35 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
234
266
  conn.exec(
235
267
  "CREATE TABLE IF NOT EXISTS kv (ns TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (ns, key))",
236
268
  );
269
+ // #56 — no `body` column: an object's bytes live in their own file under
270
+ // `r2Dir` (or an in-memory map for `:memory:`), never in the SQLite row.
271
+ // Doubling every object through SQLite's own page cache on top of the
272
+ // buffer it already arrived in was the real cost driving #56's "large
273
+ // uploads make memory skyrocket" complaint, not just the JSON-escaping
274
+ // #63 already fixed.
237
275
  conn.exec(
238
- "CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, body TEXT NOT NULL, size INTEGER NOT NULL, " +
276
+ "CREATE TABLE IF NOT EXISTS r2 (bucket TEXT NOT NULL, key TEXT NOT NULL, size INTEGER NOT NULL, " +
239
277
  "etag TEXT NOT NULL, uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
278
+ "blob_id TEXT, " +
240
279
  "PRIMARY KEY (bucket, key))",
241
280
  );
281
+ const r2Columns = conn.query<{ name: string }, []>("PRAGMA table_info(r2)").all();
282
+ if (!r2Columns.some((column) => column.name === "blob_id")) conn.exec("ALTER TABLE r2 ADD COLUMN blob_id TEXT");
283
+ conn.exec(
284
+ "CREATE TABLE IF NOT EXISTS r2_upload (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
285
+ "uploaded TEXT NOT NULL, http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', " +
286
+ "PRIMARY KEY (bucket, key, upload_id))",
287
+ );
288
+ conn.exec(
289
+ "CREATE TABLE IF NOT EXISTS r2_part (bucket TEXT NOT NULL, key TEXT NOT NULL, upload_id TEXT NOT NULL, " +
290
+ "part_number INTEGER NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, " +
291
+ "PRIMARY KEY (bucket, key, upload_id, part_number))",
292
+ );
293
+ conn.exec(
294
+ "CREATE TABLE IF NOT EXISTS r2_transfer (token TEXT PRIMARY KEY, bucket TEXT NOT NULL, key TEXT NOT NULL, " +
295
+ "method TEXT NOT NULL, expires INTEGER NOT NULL, max_bytes INTEGER NOT NULL, expected_sha256 TEXT NOT NULL DEFAULT '', " +
296
+ "http_json TEXT NOT NULL DEFAULT '{}', custom_json TEXT NOT NULL DEFAULT '{}', used INTEGER NOT NULL DEFAULT 0)",
297
+ );
242
298
  conn.exec(
243
299
  "CREATE TABLE IF NOT EXISTS mq (queue TEXT NOT NULL, id TEXT PRIMARY KEY, body TEXT NOT NULL, " +
244
300
  "visible_at INTEGER NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, dead INTEGER NOT NULL DEFAULT 0)",
@@ -286,6 +342,121 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
286
342
  return conn;
287
343
  };
288
344
 
345
+ const quotaOwner = opts.ownerId;
346
+ const quotaBytes = opts.r2QuotaBytes && opts.r2QuotaBytes > 0 ? opts.r2QuotaBytes : null;
347
+ const quotaDirectory = resourceDir;
348
+ if (quotaOwner && quotaDirectory && quotaBytes) mkdirSync(quotaDirectory, { recursive: true });
349
+ // One ledger for every account, shared by the brokers of all its deployments.
350
+ // Ticket reservations are recorded before an upload begins, so four accepted
351
+ // requests cannot each independently decide that the same remaining bytes fit.
352
+ const quotaDb = quotaOwner && quotaDirectory && quotaBytes
353
+ ? new Database(join(quotaDirectory, "r2-quota.sqlite"), { create: true })
354
+ : null;
355
+ if (quotaDb) {
356
+ const owner = quotaOwner!;
357
+ quotaDb.exec("PRAGMA journal_mode = WAL");
358
+ quotaDb.exec(
359
+ "CREATE TABLE IF NOT EXISTS r2_account_quota (owner TEXT PRIMARY KEY, used_bytes INTEGER NOT NULL, reserved_bytes INTEGER NOT NULL)",
360
+ );
361
+ quotaDb.exec(
362
+ "CREATE TABLE IF NOT EXISTS r2_quota_reservation (token TEXT PRIMARY KEY, owner TEXT NOT NULL, bytes INTEGER NOT NULL, expires INTEGER NOT NULL)",
363
+ );
364
+ quotaDb.exec(
365
+ "CREATE TABLE IF NOT EXISTS r2_transfer_metric (owner TEXT PRIMARY KEY, tickets_issued INTEGER NOT NULL DEFAULT 0, rejected_quota INTEGER NOT NULL DEFAULT 0, rejected_disk INTEGER NOT NULL DEFAULT 0, rejected_size INTEGER NOT NULL DEFAULT 0)",
366
+ );
367
+ const existing = quotaDb.query<{ owner: string }, [string]>("SELECT owner FROM r2_account_quota WHERE owner = ?").get(owner);
368
+ if (!existing) {
369
+ quotaDb.query("INSERT INTO r2_account_quota (owner, used_bytes, reserved_bytes) VALUES (?1, 0, 0)").run(owner);
370
+ }
371
+ quotaDb.query("INSERT OR IGNORE INTO r2_transfer_metric (owner) VALUES (?)").run(owner);
372
+ }
373
+
374
+ const committedR2Bytes = (): number => {
375
+ if (!quotaDirectory) return 0;
376
+ let used = 0;
377
+ for (const id of opts.r2ResourceIds ?? []) {
378
+ // Open through resourceDb so older resource files gain the multipart
379
+ // tables before the aggregate is queried.
380
+ const store = resourceDb(id);
381
+ used += store.query<{ bytes: number }, []>(
382
+ "SELECT (SELECT COALESCE(SUM(size), 0) FROM r2) + (SELECT COALESCE(SUM(size), 0) FROM r2_part) AS bytes",
383
+ ).get()?.bytes ?? 0;
384
+ }
385
+ return used;
386
+ };
387
+
388
+ const availableBytes = (): number => {
389
+ if (!quotaDirectory) return Number.POSITIVE_INFINITY;
390
+ if (opts.freeBytes) return opts.freeBytes(quotaDirectory);
391
+ const stats = statfsSync(quotaDirectory);
392
+ return Number(stats.bavail) * Number(stats.bsize);
393
+ };
394
+
395
+ const releaseExpiredR2Reservations = (now = Date.now()): void => {
396
+ if (!quotaDb || !quotaOwner) return;
397
+ quotaDb.query("DELETE FROM r2_quota_reservation WHERE owner = ?1 AND expires < ?2").run(quotaOwner, now);
398
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
399
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
400
+ ).get(quotaOwner)?.bytes ?? 0;
401
+ quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved, quotaOwner);
402
+ };
403
+
404
+ const reserveR2 = (token: string, bytes: number, expires: number): "ok" | "quota" | "disk" => {
405
+ if (!quotaDb || !quotaOwner || !quotaBytes) return "ok";
406
+ quotaDb.exec("BEGIN IMMEDIATE");
407
+ try {
408
+ releaseExpiredR2Reservations();
409
+ const used = committedR2Bytes();
410
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
411
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
412
+ ).get(quotaOwner)?.bytes ?? 0;
413
+ quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
414
+ if (used + reserved + bytes > quotaBytes) {
415
+ quotaDb.exec("ROLLBACK");
416
+ return "quota";
417
+ }
418
+ const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
419
+ if (availableBytes() < bytes + floor) {
420
+ quotaDb.exec("ROLLBACK");
421
+ return "disk";
422
+ }
423
+ quotaDb.query("INSERT INTO r2_quota_reservation (token, owner, bytes, expires) VALUES (?1, ?2, ?3, ?4)").run(token, quotaOwner, bytes, expires);
424
+ quotaDb.query("UPDATE r2_account_quota SET reserved_bytes = ?1 WHERE owner = ?2").run(reserved + bytes, quotaOwner);
425
+ quotaDb.exec("COMMIT");
426
+ return "ok";
427
+ } catch (error) {
428
+ quotaDb.exec("ROLLBACK");
429
+ throw error;
430
+ }
431
+ };
432
+
433
+ const settleR2 = (token: string): void => {
434
+ if (!quotaDb || !quotaOwner) return;
435
+ quotaDb.exec("BEGIN IMMEDIATE");
436
+ try {
437
+ const reservation = quotaDb.query<{ bytes: number }, [string]>("SELECT bytes FROM r2_quota_reservation WHERE token = ?").get(token);
438
+ if (!reservation) {
439
+ quotaDb.exec("COMMIT");
440
+ return;
441
+ }
442
+ quotaDb.query("DELETE FROM r2_quota_reservation WHERE token = ?").run(token);
443
+ const used = committedR2Bytes();
444
+ const reserved = quotaDb.query<{ bytes: number }, [string]>(
445
+ "SELECT COALESCE(SUM(bytes), 0) AS bytes FROM r2_quota_reservation WHERE owner = ?",
446
+ ).get(quotaOwner)?.bytes ?? 0;
447
+ quotaDb.query("UPDATE r2_account_quota SET used_bytes = ?1, reserved_bytes = ?2 WHERE owner = ?3").run(used, reserved, quotaOwner);
448
+ quotaDb.exec("COMMIT");
449
+ } catch (error) {
450
+ quotaDb.exec("ROLLBACK");
451
+ throw error;
452
+ }
453
+ };
454
+
455
+ const recordR2TransferMetric = (field: "tickets_issued" | "rejected_quota" | "rejected_disk" | "rejected_size"): void => {
456
+ if (!quotaDb || !quotaOwner) return;
457
+ quotaDb.query(`UPDATE r2_transfer_metric SET ${field} = ${field} + 1 WHERE owner = ?`).run(quotaOwner);
458
+ };
459
+
289
460
  /**
290
461
  * Where a KV / R2 / queue binding stores: its resource's own file keyed by
291
462
  * `id` when the config bound it to one, else the per-broker `db` partitioned
@@ -414,15 +585,491 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
414
585
  return { path: dest, bytes: statSync(dest).size };
415
586
  };
416
587
 
417
- /** An object body as bytes, whichever storage class it came back in. */
418
- const r2Bytes = (body: string | Uint8Array): Uint8Array =>
419
- body instanceof Uint8Array ? body : new TextEncoder().encode(body);
588
+ /**
589
+ * #56 — an object's bytes never enter SQLite; they live in their own file,
590
+ * named by a hash of bucket+key (so unicode keys and directory traversal
591
+ * are both non-issues), colocated with whichever store file holds that
592
+ * bucket's metadata — the main `db` for a bare-string binding, the
593
+ * resource's own file under `resourceDir` for an account-level one (#74) —
594
+ * so blobs persist across a redeploy exactly when that metadata does.
595
+ * `:memory:` stores (tests, `db: ":memory:"`) fall back to an in-memory map.
596
+ */
597
+ const r2MemBlobs = new Map<string, Uint8Array>();
598
+ const r2MemParts = new Map<string, Uint8Array>();
599
+ const r2BlobId = (bucketPart: string, key: string, generation?: string | null): string =>
600
+ createHash("sha256")
601
+ .update(generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`)
602
+ .digest("hex");
603
+ const r2BlobDir = (store: Database): string | null => {
604
+ const file = store.filename;
605
+ return file && file !== ":memory:" ? join(dirname(resolve(file)), "r2-blobs") : null;
606
+ };
607
+ const r2BlobPath = (store: Database, bucketPart: string, key: string, generation?: string | null): string | null => {
608
+ const dir = r2BlobDir(store);
609
+ return dir ? join(dir, `${r2BlobId(bucketPart, key, generation)}.blob`) : null;
610
+ };
611
+ const r2BlobMemKey = (bucketPart: string, key: string, generation?: string | null): string =>
612
+ generation ? `${bucketPart}\0${key}\0${generation}` : `${bucketPart}\0${key}`;
613
+ const r2WriteBlob = (
614
+ store: Database,
615
+ bucketPart: string,
616
+ key: string,
617
+ bytes: Uint8Array,
618
+ generation: string,
619
+ ): void => {
620
+ const path = r2BlobPath(store, bucketPart, key, generation);
621
+ if (!path) {
622
+ r2MemBlobs.set(r2BlobMemKey(bucketPart, key, generation), bytes);
623
+ return;
624
+ }
625
+ mkdirSync(dirname(path), { recursive: true });
626
+ const temporary = `${path}.tmp`;
627
+ writeFileSync(temporary, bytes);
628
+ renameSync(temporary, path);
629
+ };
630
+ const r2ReadBlob = (
631
+ store: Database,
632
+ bucketPart: string,
633
+ key: string,
634
+ generation?: string | null,
635
+ ): Uint8Array | null => {
636
+ const path = r2BlobPath(store, bucketPart, key, generation);
637
+ if (!path) return r2MemBlobs.get(r2BlobMemKey(bucketPart, key, generation)) ?? null;
638
+ return existsSync(path) ? readFileSync(path) : null;
639
+ };
640
+ const r2DeleteBlob = (store: Database, bucketPart: string, key: string, generation?: string | null): void => {
641
+ const path = r2BlobPath(store, bucketPart, key, generation);
642
+ if (!path) {
643
+ r2MemBlobs.delete(r2BlobMemKey(bucketPart, key, generation));
644
+ return;
645
+ }
646
+ if (existsSync(path)) rmSync(path);
647
+ };
648
+ const r2PartId = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
649
+ createHash("sha256").update(`${bucketPart}\0${key}\0${uploadId}\0${partNumber}`).digest("hex");
650
+ const r2PartPath = (
651
+ store: Database,
652
+ bucketPart: string,
653
+ key: string,
654
+ uploadId: string,
655
+ partNumber: number,
656
+ ): string | null => {
657
+ const file = store.filename;
658
+ if (!file || file === ":memory:") return null;
659
+ return join(dirname(resolve(file)), "r2-parts", `${r2PartId(bucketPart, key, uploadId, partNumber)}.part`);
660
+ };
661
+ const r2PartMemKey = (bucketPart: string, key: string, uploadId: string, partNumber: number): string =>
662
+ `${bucketPart}\0${key}\0${uploadId}\0${partNumber}`;
663
+ const r2WritePart = (
664
+ store: Database,
665
+ bucketPart: string,
666
+ key: string,
667
+ uploadId: string,
668
+ partNumber: number,
669
+ bytes: Uint8Array,
670
+ ): void => {
671
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
672
+ if (!path) {
673
+ r2MemParts.set(r2PartMemKey(bucketPart, key, uploadId, partNumber), bytes);
674
+ return;
675
+ }
676
+ mkdirSync(dirname(path), { recursive: true });
677
+ const temporary = `${path}.tmp`;
678
+ writeFileSync(temporary, bytes);
679
+ renameSync(temporary, path);
680
+ };
681
+ const r2ReadPart = (
682
+ store: Database,
683
+ bucketPart: string,
684
+ key: string,
685
+ uploadId: string,
686
+ partNumber: number,
687
+ ): Uint8Array | null => {
688
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
689
+ if (!path) return r2MemParts.get(r2PartMemKey(bucketPart, key, uploadId, partNumber)) ?? null;
690
+ return existsSync(path) ? readFileSync(path) : null;
691
+ };
692
+ const r2DeletePart = (
693
+ store: Database,
694
+ bucketPart: string,
695
+ key: string,
696
+ uploadId: string,
697
+ partNumber: number,
698
+ ): void => {
699
+ const path = r2PartPath(store, bucketPart, key, uploadId, partNumber);
700
+ if (!path) {
701
+ r2MemParts.delete(r2PartMemKey(bucketPart, key, uploadId, partNumber));
702
+ return;
703
+ }
704
+ if (existsSync(path)) rmSync(path);
705
+ };
706
+
707
+ const requireMultipart = (store: Database, bucket: string, key: string, uploadId: string): R2UploadRow => {
708
+ const row = store
709
+ .query<R2UploadRow, [string, string, string]>(
710
+ "SELECT key, uploaded, http_json, custom_json FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?",
711
+ )
712
+ .get(bucket, key, uploadId);
713
+ if (!row) throw new Error("multipart upload not found");
714
+ return row;
715
+ };
716
+
717
+ const multipartParts = (store: Database, bucket: string, key: string, uploadId: string): R2PartRow[] =>
718
+ store
719
+ .query<R2PartRow, [string, string, string]>(
720
+ "SELECT part_number, size, etag FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ? ORDER BY part_number",
721
+ )
722
+ .all(bucket, key, uploadId);
723
+
724
+ const requestedParts = (value: JsonValue | undefined): Array<{ partNumber: number; etag: string }> => {
725
+ if (!Array.isArray(value) || value.length === 0) throw new Error("complete requires at least one uploaded part");
726
+ return value.map((part) => {
727
+ const parsed = jsonObject(part);
728
+ const partNumber = Number(parsed?.partNumber);
729
+ const etag = isString(parsed?.etag) ? parsed.etag : "";
730
+ if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000 || !etag) {
731
+ throw new Error("invalid uploaded part");
732
+ }
733
+ return { partNumber, etag };
734
+ });
735
+ };
736
+
737
+ const validateCompletedParts = (stored: R2PartRow[], requested: Array<{ partNumber: number; etag: string }>): void => {
738
+ if (stored.length !== requested.length) throw new Error("uploaded parts do not match");
739
+ for (let index = 0; index < stored.length; index++) {
740
+ const actual = stored[index];
741
+ const expected = requested[index];
742
+ if (actual.part_number !== expected.partNumber || actual.etag !== expected.etag) {
743
+ throw new Error("uploaded parts do not match");
744
+ }
745
+ if (index > 0 && expected.partNumber <= requested[index - 1].partNumber) {
746
+ throw new Error("uploaded parts must be ordered by partNumber");
747
+ }
748
+ if (index < stored.length - 1 && actual.size !== stored[0].size) {
749
+ throw new Error("all multipart parts except the last must have the same size");
750
+ }
751
+ if (index === stored.length - 1 && stored.length > 1 && actual.size > stored[0].size) {
752
+ throw new Error("the last multipart part cannot be larger than the others");
753
+ }
754
+ }
755
+ };
756
+
757
+ const deleteMultipart = (store: Database, bucket: string, key: string, uploadId: string): void => {
758
+ for (const part of multipartParts(store, bucket, key, uploadId)) {
759
+ r2DeletePart(store, bucket, key, uploadId, part.part_number);
760
+ }
761
+ store.query("DELETE FROM r2_part WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
762
+ store.query("DELETE FROM r2_upload WHERE bucket = ? AND key = ? AND upload_id = ?").run(bucket, key, uploadId);
763
+ };
764
+
765
+ const cleanupExpiredMultipart = (store: Database): void => {
766
+ const cutoff = new Date(Date.now() - MULTIPART_EXPIRY_MS).toISOString();
767
+ const expired = store
768
+ .query<R2UploadIdentity, [string]>("SELECT bucket, key, upload_id FROM r2_upload WHERE uploaded < ?")
769
+ .all(cutoff);
770
+ for (const upload of expired) deleteMultipart(store, upload.bucket, upload.key, upload.upload_id);
771
+ };
772
+
773
+ const commitR2Metadata = (
774
+ store: Database,
775
+ bucket: string,
776
+ row: R2Row,
777
+ previous: Pick<R2Row, "blob_id"> | null,
778
+ ): void => {
779
+ try {
780
+ store
781
+ .query(
782
+ "INSERT INTO r2 (bucket, key, size, etag, uploaded, http_json, custom_json, blob_id) " +
783
+ "VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
784
+ "ON CONFLICT (bucket, key) DO UPDATE SET size=?3, etag=?4, uploaded=?5, http_json=?6, " +
785
+ "custom_json=?7, blob_id=?8",
786
+ )
787
+ .run(bucket, row.key, row.size, row.etag, row.uploaded, row.http_json, row.custom_json, row.blob_id);
788
+ } catch (error) {
789
+ if (previous?.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, row.blob_id);
790
+ throw error;
791
+ }
792
+ if (previous && previous.blob_id !== row.blob_id) r2DeleteBlob(store, bucket, row.key, previous.blob_id);
793
+ };
794
+
795
+ const putR2 = (
796
+ store: Database,
797
+ bucket: string,
798
+ key: string,
799
+ bytes: Uint8Array,
800
+ httpJson: string,
801
+ customJson: string,
802
+ ): R2Row => {
803
+ const previous = store
804
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
805
+ .get(bucket, key);
806
+ const etag = createHash("sha256").update(bytes).digest("hex");
807
+ const row: R2Row = {
808
+ key,
809
+ size: bytes.byteLength,
810
+ etag,
811
+ uploaded: new Date().toISOString(),
812
+ http_json: httpJson,
813
+ custom_json: customJson,
814
+ blob_id: etag,
815
+ };
816
+ r2WriteBlob(store, bucket, key, bytes, etag);
817
+ commitR2Metadata(store, bucket, row, previous);
818
+ return row;
819
+ };
820
+
821
+ const transferLimit = (): number => {
822
+ const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
823
+ return Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
824
+ };
825
+ let activeUploads = 0;
826
+ const maxActiveUploads = 4;
827
+
828
+ const cleanupTransferTemps = (store: Database): void => {
829
+ const directory = r2BlobDir(store);
830
+ if (!directory || !existsSync(directory)) return;
831
+ const now = Date.now();
832
+ const cutoff = Date.now() - 24 * 60 * 60 * 1000;
833
+ for (const name of readdirSync(directory)) {
834
+ if (!/^[0-9a-f]{24}\.upload\.tmp$/.test(name)) continue;
835
+ const path = join(directory, name);
836
+ try {
837
+ const tokenValue = name.slice(0, 24);
838
+ const ticket = store.query<{ expires: number }, [string]>("SELECT expires FROM r2_transfer WHERE token = ?").get(tokenValue);
839
+ if (!ticket || ticket.expires < now || statSync(path).mtimeMs < cutoff) rmSync(path);
840
+ } catch (error) {
841
+ if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) throw error;
842
+ }
843
+ }
844
+ };
845
+
846
+ const transferPath = (binding: string, tokenValue: string): string =>
847
+ `/__sb/r2/transfer/${encodeURIComponent(binding)}/${tokenValue}`;
420
848
 
421
- /** An object body as text, for the v0 reply shape that carries it in JSON.
422
- * A binary object stored by a v1 client cannot survive this, which is why
423
- * v1 exists — but returning a decoded string beats returning `{"0":...}`. */
424
- const r2Text = (body: string | Uint8Array): string =>
425
- body instanceof Uint8Array ? new TextDecoder().decode(body) : body;
849
+ const parseRange = (header: string | null, size: number): { start: number; end: number } | null => {
850
+ if (!header) return { start: 0, end: Math.max(0, size - 1) };
851
+ const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
852
+ if (!match || size === 0) return null;
853
+ if (!match[1]) {
854
+ const suffix = Number(match[2]);
855
+ if (!Number.isSafeInteger(suffix) || suffix <= 0) return null;
856
+ return { start: Math.max(0, size - suffix), end: size - 1 };
857
+ }
858
+ const start = Number(match[1]);
859
+ const requestedEnd = match[2] ? Number(match[2]) : size - 1;
860
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(requestedEnd) || start < 0 || start >= size || requestedEnd < start) return null;
861
+ return { start, end: Math.min(requestedEnd, size - 1) };
862
+ };
863
+
864
+ const transfer = async (request: Request): Promise<Response> => {
865
+ const url = new URL(request.url);
866
+ const match = /^\/__sb\/r2\/transfer\/([A-Z][A-Z0-9_]*)\/([0-9a-f]{24})$/.exec(url.pathname);
867
+ if (!match) return new Response("not found", { status: 404 });
868
+ let store: Database;
869
+ let bucket: string;
870
+ try {
871
+ ({ store, part: bucket } = storeFor("r2", requireR2(match[1])));
872
+ } catch {
873
+ return new Response("not found", { status: 404 });
874
+ }
875
+ const tokenValue = match[2];
876
+ const ticket = store
877
+ .query<R2TransferRow, [string]>(
878
+ "SELECT bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json, used " +
879
+ "FROM r2_transfer WHERE token = ?",
880
+ )
881
+ .get(tokenValue);
882
+ const method = request.method.toUpperCase();
883
+ const allowedMethod = ticket?.method === "upload" ? "PUT" : ticket?.method === "download" ? "GET, HEAD" : "";
884
+ if (!ticket || ticket.bucket !== bucket || ticket.used || ticket.expires < Date.now()) {
885
+ return new Response("transfer ticket expired or already used", { status: 410 });
886
+ }
887
+ if ((ticket.method === "upload" && method !== "PUT") || (ticket.method === "download" && method !== "GET" && method !== "HEAD")) {
888
+ return new Response("method not allowed", { status: 405, headers: { allow: allowedMethod } });
889
+ }
890
+ if (ticket.method === "download") {
891
+ const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, ticket.key);
892
+ if (!row) return new Response("not found", { status: 404 });
893
+ const etag = `"${row.etag}"`;
894
+ if (request.headers.get("if-none-match") === etag) return new Response(null, { status: 304, headers: { etag } });
895
+ const rangeHeader = request.headers.get("range");
896
+ const range = parseRange(rangeHeader, row.size);
897
+ if (!range) {
898
+ return new Response(null, { status: 416, headers: { "content-range": `bytes */${row.size}`, etag } });
899
+ }
900
+ const path = r2BlobPath(store, bucket, ticket.key, row.blob_id);
901
+ if (!path || !existsSync(path)) return new Response("not found", { status: 404 });
902
+ const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
903
+ if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
904
+ const length = row.size === 0 ? 0 : range.end - range.start + 1;
905
+ const metadata = jsonObject(parseJsonValue(row.http_json));
906
+ const headers = new Headers({
907
+ "accept-ranges": "bytes",
908
+ "content-length": String(length),
909
+ etag,
910
+ });
911
+ if (isString(metadata?.contentType)) headers.set("content-type", metadata.contentType);
912
+ if (rangeHeader) headers.set("content-range", `bytes ${range.start}-${range.end}/${row.size}`);
913
+ const body = method === "HEAD" ? null : Bun.file(path).slice(range.start, range.start + length);
914
+ return new Response(body, { status: rangeHeader ? 206 : 200, headers });
915
+ }
916
+
917
+ const declared = request.headers.has("content-length") ? Number(request.headers.get("content-length")) : NaN;
918
+ if (request.headers.has("content-length") && (!Number.isSafeInteger(declared) || declared < 0)) {
919
+ return new Response("invalid content length", { status: 400 });
920
+ }
921
+ if (Number.isFinite(declared) && declared > ticket.max_bytes) return new Response("upload too large", { status: 413 });
922
+ const directory = r2BlobDir(store);
923
+ if (!directory) return new Response("direct transfers require a file-backed store", { status: 503 });
924
+ if (activeUploads >= maxActiveUploads) {
925
+ return new Response("too many active uploads", { status: 503, headers: { "retry-after": "5" } });
926
+ }
927
+ const claimed = store.query("UPDATE r2_transfer SET used = 1 WHERE token = ? AND used = 0").run(tokenValue);
928
+ if (claimed.changes !== 1) return new Response("transfer ticket already used", { status: 410 });
929
+ activeUploads++;
930
+ const temporary = join(directory, `${tokenValue}.upload.tmp`);
931
+ await (async () => {
932
+ mkdirSync(directory, { recursive: true });
933
+ const reservation = await open(temporary, "wx");
934
+ await reservation.close();
935
+ })().catch((error) => {
936
+ activeUploads--;
937
+ throw error;
938
+ });
939
+ const writer = Bun.file(temporary).writer({ highWaterMark: 64 * 1024 });
940
+ const hash = createHash("sha256");
941
+ let size = 0;
942
+ try {
943
+ if (request.body) {
944
+ for await (const chunk of request.body) {
945
+ size += chunk.byteLength;
946
+ if (size > ticket.max_bytes) throw new RangeError("upload too large");
947
+ const floor = Math.max(opts.r2MinFreeBytes ?? 0, 0);
948
+ if (floor > 0 && availableBytes() < chunk.byteLength + floor) {
949
+ const error = new Error("R2 storage is temporarily unavailable");
950
+ Object.assign(error, { code: "SB_R2_DISK_FLOOR" });
951
+ throw error;
952
+ }
953
+ hash.update(chunk);
954
+ const written = await writer.write(chunk);
955
+ if (written !== chunk.byteLength) throw new Error("upload write was incomplete");
956
+ }
957
+ }
958
+ await writer.end();
959
+ const persisted = await open(temporary, "r");
960
+ try {
961
+ await persisted.sync();
962
+ } finally {
963
+ await persisted.close();
964
+ }
965
+ const etag = hash.digest("hex");
966
+ if (ticket.expected_sha256 && ticket.expected_sha256 !== etag) {
967
+ rmSync(temporary);
968
+ settleR2(tokenValue);
969
+ return Response.json({ error: "sha256 mismatch", actual: etag }, { status: 422 });
970
+ }
971
+ const destination = r2BlobPath(store, bucket, ticket.key, etag)!;
972
+ renameSync(temporary, destination);
973
+ const previous = store
974
+ .query<Pick<R2Row, "blob_id" | "size">, [string, string]>("SELECT blob_id, size FROM r2 WHERE bucket = ? AND key = ?")
975
+ .get(bucket, ticket.key);
976
+ const row: R2Row = {
977
+ key: ticket.key,
978
+ size,
979
+ etag,
980
+ uploaded: new Date().toISOString(),
981
+ http_json: ticket.http_json,
982
+ custom_json: ticket.custom_json,
983
+ blob_id: etag,
984
+ };
985
+ commitR2Metadata(store, bucket, row, previous);
986
+ settleR2(tokenValue);
987
+ return Response.json({ object: r2Row(row) }, { status: 201, headers: { etag: `"${etag}"` } });
988
+ } catch (error) {
989
+ try {
990
+ await writer.end();
991
+ } catch {
992
+ // Preserve the original upload failure while releasing the file sink.
993
+ }
994
+ if (existsSync(temporary)) rmSync(temporary);
995
+ settleR2(tokenValue);
996
+ if (error instanceof RangeError) {
997
+ recordR2TransferMetric("rejected_size");
998
+ return new Response(error.message, { status: 413 });
999
+ }
1000
+ if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "SB_R2_DISK_FLOOR")) {
1001
+ recordR2TransferMetric("rejected_disk");
1002
+ return new Response("R2 storage is temporarily unavailable", { status: 507 });
1003
+ }
1004
+ throw error;
1005
+ } finally {
1006
+ activeUploads--;
1007
+ }
1008
+ };
1009
+
1010
+ const completeMultipart = (
1011
+ store: Database,
1012
+ bucket: string,
1013
+ key: string,
1014
+ uploadId: string,
1015
+ requested: Array<{ partNumber: number; etag: string }>,
1016
+ ): ReturnType<typeof r2Row> => {
1017
+ const upload = requireMultipart(store, bucket, key, uploadId);
1018
+ const parts = multipartParts(store, bucket, key, uploadId);
1019
+ validateCompletedParts(parts, requested);
1020
+
1021
+ const hash = createHash("sha256");
1022
+ let size = 0;
1023
+ const blobDirectory = r2BlobDir(store);
1024
+ let inMemoryBody: Uint8Array | null = null;
1025
+ let temporary: string | null = null;
1026
+ if (blobDirectory) {
1027
+ mkdirSync(blobDirectory, { recursive: true });
1028
+ temporary = join(blobDirectory, `${uploadId}.tmp`);
1029
+ writeFileSync(temporary, new Uint8Array(0));
1030
+ try {
1031
+ for (const part of parts) {
1032
+ const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
1033
+ if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
1034
+ appendFileSync(temporary, bytes);
1035
+ hash.update(bytes);
1036
+ size += bytes.byteLength;
1037
+ }
1038
+ } catch (error) {
1039
+ if (existsSync(temporary)) rmSync(temporary);
1040
+ throw error;
1041
+ }
1042
+ } else {
1043
+ const chunks: Uint8Array[] = [];
1044
+ for (const part of parts) {
1045
+ const bytes = r2ReadPart(store, bucket, key, uploadId, part.part_number);
1046
+ if (!bytes || bytes.byteLength !== part.size) throw new Error(`multipart part ${part.part_number} is missing`);
1047
+ chunks.push(bytes);
1048
+ hash.update(bytes);
1049
+ size += bytes.byteLength;
1050
+ }
1051
+ inMemoryBody = Buffer.concat(chunks);
1052
+ }
1053
+
1054
+ const etag = hash.digest("hex");
1055
+ const previous = store
1056
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
1057
+ .get(bucket, key);
1058
+ if (temporary) renameSync(temporary, r2BlobPath(store, bucket, key, etag)!);
1059
+ else r2WriteBlob(store, bucket, key, inMemoryBody ?? new Uint8Array(0), etag);
1060
+ const row: R2Row = {
1061
+ key,
1062
+ size,
1063
+ etag,
1064
+ uploaded: new Date().toISOString(),
1065
+ http_json: upload.http_json,
1066
+ custom_json: upload.custom_json,
1067
+ blob_id: etag,
1068
+ };
1069
+ commitR2Metadata(store, bucket, row, previous);
1070
+ deleteMultipart(store, bucket, key, uploadId);
1071
+ return r2Row(row);
1072
+ };
426
1073
 
427
1074
  const r2Row = (r: R2Row) => ({
428
1075
  key: r.key,
@@ -617,40 +1264,129 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
617
1264
 
618
1265
  case "r2.put": {
619
1266
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
620
- const body = str(msg.body);
621
- const etag = createHash("sha256").update(body).digest("hex");
1267
+ const key = str(msg.key);
1268
+ const bytes = new TextEncoder().encode(str(msg.body));
1269
+ return {
1270
+ ok: true,
1271
+ object: r2Row(
1272
+ putR2(
1273
+ store,
1274
+ bucket,
1275
+ key,
1276
+ bytes,
1277
+ JSON.stringify(msg.httpMetadata ?? {}),
1278
+ JSON.stringify(msg.customMetadata ?? {}),
1279
+ ),
1280
+ ),
1281
+ };
1282
+ }
1283
+ case "r2.multipart.create": {
1284
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1285
+ cleanupExpiredMultipart(store);
1286
+ const key = str(msg.key);
1287
+ const uploadId = str(msg.uploadId) || newId();
1288
+ const uploaded = new Date().toISOString();
622
1289
  store
623
1290
  .query(
624
- "INSERT INTO r2 (bucket, key, body, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
625
- "ON CONFLICT (bucket, key) DO UPDATE SET body=?3, size=?4, etag=?5, uploaded=?6, http_json=?7, custom_json=?8",
1291
+ "INSERT INTO r2_upload (bucket, key, upload_id, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6) " +
1292
+ "ON CONFLICT (bucket, key, upload_id) DO NOTHING",
626
1293
  )
627
1294
  .run(
628
1295
  bucket,
629
- str(msg.key),
630
- body,
631
- Buffer.byteLength(body),
632
- etag,
633
- new Date().toISOString(),
1296
+ key,
1297
+ uploadId,
1298
+ uploaded,
634
1299
  JSON.stringify(msg.httpMetadata ?? {}),
635
1300
  JSON.stringify(msg.customMetadata ?? {}),
636
1301
  );
637
- return {
638
- ok: true,
639
- object: { key: str(msg.key), size: Buffer.byteLength(body), etag, uploaded: new Date().toISOString() },
640
- };
1302
+ return { ok: true, key, uploadId };
1303
+ }
1304
+ case "r2.transfer.create": {
1305
+ const binding = requireR2(msg.bucket);
1306
+ const { store, part: bucket } = storeFor("r2", binding);
1307
+ const key = str(msg.key);
1308
+ const method = msg.method === "download" ? "download" : msg.method === "upload" ? "upload" : "";
1309
+ if (!method) throw new Error("transfer method must be upload or download");
1310
+ const expiresIn = Math.min(Math.max(Number(msg.expiresIn) || 900, 30), 3600);
1311
+ const expires = Date.now() + expiresIn * 1000;
1312
+ const maxBytes =
1313
+ method === "upload"
1314
+ ? Math.min(Math.max(Number(msg.maxBytes) || 100 * 1024 * 1024, 1), transferLimit())
1315
+ : 0;
1316
+ const expectedSha256 = str(msg.sha256).toLowerCase();
1317
+ if (expectedSha256 && !/^[0-9a-f]{64}$/.test(expectedSha256)) {
1318
+ throw new Error("sha256 must be 64 hexadecimal characters");
1319
+ }
1320
+ store.query("DELETE FROM r2_transfer WHERE expires < ? OR used = 1").run(Date.now());
1321
+ cleanupTransferTemps(store);
1322
+ const tokenValue = randomBytes(12).toString("hex");
1323
+ const reservation = method === "upload" ? reserveR2(tokenValue, maxBytes, expires) : "ok";
1324
+ if (reservation === "quota") {
1325
+ recordR2TransferMetric("rejected_quota");
1326
+ throw new Error("R2 account quota would be exceeded");
1327
+ }
1328
+ if (reservation === "disk") {
1329
+ recordR2TransferMetric("rejected_disk");
1330
+ throw new Error("R2 storage is temporarily unavailable");
1331
+ }
1332
+ try {
1333
+ store
1334
+ .query(
1335
+ "INSERT INTO r2_transfer (token, bucket, key, method, expires, max_bytes, expected_sha256, http_json, custom_json) " +
1336
+ "VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
1337
+ )
1338
+ .run(
1339
+ tokenValue,
1340
+ bucket,
1341
+ key,
1342
+ method,
1343
+ expires,
1344
+ maxBytes,
1345
+ expectedSha256,
1346
+ JSON.stringify(msg.httpMetadata ?? {}),
1347
+ JSON.stringify(msg.customMetadata ?? {}),
1348
+ );
1349
+ } catch (error) {
1350
+ if (reservation === "ok" && method === "upload") settleR2(tokenValue);
1351
+ throw error;
1352
+ }
1353
+ recordR2TransferMetric("tickets_issued");
1354
+ return { ok: true, url: transferPath(binding, tokenValue), expiresAt: new Date(expires).toISOString() };
1355
+ }
1356
+ case "r2.multipart.complete": {
1357
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1358
+ const key = str(msg.key);
1359
+ const uploadId = str(msg.uploadId);
1360
+ const object = completeMultipart(store, bucket, key, uploadId, requestedParts(msg.parts));
1361
+ return { ok: true, object };
1362
+ }
1363
+ case "r2.multipart.abort": {
1364
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1365
+ const key = str(msg.key);
1366
+ const uploadId = str(msg.uploadId);
1367
+ requireMultipart(store, bucket, key, uploadId);
1368
+ deleteMultipart(store, bucket, key, uploadId);
1369
+ return { ok: true };
641
1370
  }
642
1371
  case "r2.get":
643
1372
  case "r2.head": {
644
1373
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
645
- const row = store
646
- .query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
647
- .get(bucket, str(msg.key));
1374
+ const key = str(msg.key);
1375
+ const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
648
1376
  if (!row) return { ok: true, found: false };
649
- return { ok: true, found: true, object: r2Row(row), body: msg.op === "r2.get" ? r2Text(row.body) : undefined };
1377
+ const body = msg.op === "r2.get"
1378
+ ? new TextDecoder().decode(r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0))
1379
+ : undefined;
1380
+ return { ok: true, found: true, object: r2Row(row), body };
650
1381
  }
651
1382
  case "r2.delete": {
652
1383
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
653
- store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, str(msg.key));
1384
+ const key = str(msg.key);
1385
+ const row = store
1386
+ .query<Pick<R2Row, "blob_id">, [string, string]>("SELECT blob_id FROM r2 WHERE bucket = ? AND key = ?")
1387
+ .get(bucket, key);
1388
+ store.query("DELETE FROM r2 WHERE bucket = ? AND key = ?").run(bucket, key);
1389
+ if (row) r2DeleteBlob(store, bucket, key, row.blob_id);
654
1390
  return { ok: true };
655
1391
  }
656
1392
  case "r2.list": {
@@ -832,38 +1568,51 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
832
1568
  */
833
1569
  async function dispatchBinary(msg: Frame, binary: Uint8Array): Promise<{ reply: Frame; bytes?: Uint8Array }> {
834
1570
  if (msg.op === "r2.put") {
835
- // The body arrived as bytes; store it as a blob rather than a string, so
836
- // a put costs one copy instead of an escape, a parse and a re-encode.
1571
+ // The body arrived as bytes; write it straight to its own file rather
1572
+ // than into SQLite (#56) — no escape, no parse, no re-encode, and no
1573
+ // second full-object copy through the database's own page cache.
1574
+ const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1575
+ const key = str(msg.key);
1576
+ const row = putR2(
1577
+ store,
1578
+ bucket,
1579
+ key,
1580
+ binary,
1581
+ JSON.stringify(msg.httpMetadata ?? {}),
1582
+ JSON.stringify(msg.customMetadata ?? {}),
1583
+ );
1584
+ return { reply: { ok: true, object: r2Row(row) } };
1585
+ }
1586
+
1587
+ if (msg.op === "r2.multipart.put") {
837
1588
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
1589
+ const key = str(msg.key);
1590
+ const uploadId = str(msg.uploadId);
1591
+ const partNumber = Number(msg.partNumber);
1592
+ if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > 10000) {
1593
+ throw new Error("partNumber must be an integer between 1 and 10000");
1594
+ }
1595
+ requireMultipart(store, bucket, key, uploadId);
838
1596
  const etag = createHash("sha256").update(binary).digest("hex");
839
- const uploaded = new Date().toISOString();
1597
+ r2WritePart(store, bucket, key, uploadId, partNumber, binary);
840
1598
  store
841
1599
  .query(
842
- "INSERT INTO r2 (bucket, key, body, size, etag, uploaded, http_json, custom_json) VALUES (?1,?2,?3,?4,?5,?6,?7,?8) " +
843
- "ON CONFLICT (bucket, key) DO UPDATE SET body=?3, size=?4, etag=?5, uploaded=?6, http_json=?7, custom_json=?8",
1600
+ "INSERT INTO r2_part (bucket, key, upload_id, part_number, size, etag) VALUES (?1,?2,?3,?4,?5,?6) " +
1601
+ "ON CONFLICT (bucket, key, upload_id, part_number) DO UPDATE SET size=?5, etag=?6",
844
1602
  )
845
- .run(
846
- bucket,
847
- str(msg.key),
848
- binary,
849
- binary.byteLength,
850
- etag,
851
- uploaded,
852
- JSON.stringify(msg.httpMetadata ?? {}),
853
- JSON.stringify(msg.customMetadata ?? {}),
854
- );
855
- return {
856
- reply: { ok: true, object: { key: str(msg.key), size: binary.byteLength, etag, uploaded } },
857
- };
1603
+ .run(bucket, key, uploadId, partNumber, binary.byteLength, etag);
1604
+ return { reply: { ok: true, part: { partNumber, etag } } };
858
1605
  }
859
1606
 
860
1607
  if (msg.op === "r2.get") {
861
1608
  const { store, part: bucket } = storeFor("r2", requireR2(msg.bucket));
862
- const row = store
863
- .query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?")
864
- .get(bucket, str(msg.key));
1609
+ const key = str(msg.key);
1610
+ const row = store.query<R2Row, [string, string]>("SELECT * FROM r2 WHERE bucket = ? AND key = ?").get(bucket, key);
865
1611
  if (!row) return { reply: { ok: true, found: false } };
866
- return { reply: { ok: true, found: true, object: r2Row(row) }, bytes: r2Bytes(row.body) };
1612
+ return {
1613
+ reply: { ok: true, found: true, object: r2Row(row) },
1614
+ bytes: r2ReadBlob(store, bucket, key, row.blob_id) ?? new Uint8Array(0),
1615
+ };
867
1616
  }
868
1617
 
869
1618
  if (msg.op === "assets.get") {
@@ -901,7 +1650,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
901
1650
  if (!msg) throw new Error("request frame was not a JSON object");
902
1651
  if (token && str(msg.token) !== token) return frameOf(encodeV1({ ok: false, error: "unauthorized" }));
903
1652
  const binary = new Uint8Array(payload.subarray(5 + jsonLen));
1653
+ const id = isSafeInteger(msg.id) ? msg.id : null;
1654
+ const replayKey = id === null ? null : `${str(msg.op)}:${id}`;
1655
+ if (replayKey && REPLAYABLE.has(str(msg.op))) {
1656
+ const seen = replayed.get(replayKey);
1657
+ if (seen) return frameOf(encodeV1(seen));
1658
+ }
904
1659
  const { reply, bytes } = await dispatchBinary(msg, binary);
1660
+ if (replayKey && REPLAYABLE.has(str(msg.op))) rememberReplay(replayKey, reply);
905
1661
  return frameOf(encodeV1(reply, bytes));
906
1662
  } catch (e) {
907
1663
  return frameOf(encodeV1({ ok: false, error: e instanceof Error ? e.message : String(e) }));
@@ -928,6 +1684,11 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
928
1684
  "d1.backup",
929
1685
  "ratelimit.check",
930
1686
  "r2.put",
1687
+ "r2.multipart.create",
1688
+ "r2.multipart.put",
1689
+ "r2.multipart.complete",
1690
+ "r2.multipart.abort",
1691
+ "r2.transfer.create",
931
1692
  "r2.delete",
932
1693
  "do.storage.put",
933
1694
  "do.storage.delete",
@@ -941,6 +1702,14 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
941
1702
  * milliseconds, so this only has to outlive a reconnect, not a session. */
942
1703
  const REPLAY_WINDOW = 256;
943
1704
 
1705
+ const rememberReplay = (key: string, reply: Frame): void => {
1706
+ replayed.set(key, reply);
1707
+ if (replayed.size > REPLAY_WINDOW) {
1708
+ const oldest = replayed.keys().next().value;
1709
+ if (oldest !== undefined) replayed.delete(oldest);
1710
+ }
1711
+ };
1712
+
944
1713
  /**
945
1714
  * Run `msg`, or replay what it answered last time. Keyed by the id the sprout
946
1715
  * put on the request; a request without one is always run.
@@ -953,12 +1722,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
953
1722
  const seen = replayed.get(key);
954
1723
  if (seen) return seen;
955
1724
  const reply = await dispatch(msg);
956
- replayed.set(key, reply);
957
- // Insertion-ordered, so the oldest key is the first one out.
958
- if (replayed.size > REPLAY_WINDOW) {
959
- const oldest = replayed.keys().next().value;
960
- if (oldest !== undefined) replayed.delete(oldest);
961
- }
1725
+ rememberReplay(key, reply);
962
1726
  return reply;
963
1727
  }
964
1728
 
@@ -1105,6 +1869,7 @@ export function createBroker(opts: BrokerOptions = {}): Broker {
1105
1869
  dispatch,
1106
1870
  handlePayload,
1107
1871
  handleFrame,
1872
+ transfer,
1108
1873
  close: () => {
1109
1874
  for (const t of timers) clearInterval(t);
1110
1875
  for (const conn of d1Conns.values()) conn.close();
@@ -1192,17 +1957,33 @@ export function listen(broker: Broker, hostname: string, port: number): BrokerSe
1192
1957
  },
1193
1958
  },
1194
1959
  });
1195
- return { port: server.port, stop: () => server.stop(true) };
1960
+ return { port: server.port ?? port, stop: () => server.stop(true) };
1961
+ }
1962
+
1963
+ /** Start the HTTP listener that serves short-lived direct R2 transfer tickets. */
1964
+ export function listenTransfers(broker: Broker, hostname: string, port: number): BrokerServer {
1965
+ const configured = Number(process.env.SB_R2_TRANSFER_MAX_BYTES);
1966
+ const maxRequestBodySize = Number.isSafeInteger(configured) && configured > 0 ? configured : 5 * 1024 * 1024 * 1024;
1967
+ const server = Bun.serve({ hostname, port, maxRequestBodySize, fetch(request, server) {
1968
+ server.timeout(request, 255);
1969
+ return broker.transfer(request);
1970
+ } });
1971
+ return { port: server.port ?? port, stop: () => server.stop(true) };
1196
1972
  }
1197
1973
 
1198
1974
  if (import.meta.main) {
1199
1975
  const { values } = parseArgs({
1200
1976
  options: {
1201
1977
  port: { type: "string" },
1978
+ "transfer-port": { type: "string" },
1202
1979
  token: { type: "string" },
1203
1980
  db: { type: "string" },
1204
1981
  "data-dir": { type: "string" },
1205
1982
  "resource-dir": { type: "string" },
1983
+ "owner-id": { type: "string" },
1984
+ "r2-quota-bytes": { type: "string" },
1985
+ "r2-resource-ids": { type: "string" },
1986
+ "r2-min-free-bytes": { type: "string" },
1206
1987
  bindings: { type: "string" },
1207
1988
  secrets: { type: "string" },
1208
1989
  "sprout-url": { type: "string" },
@@ -1251,6 +2032,10 @@ if (import.meta.main) {
1251
2032
  db: values.db,
1252
2033
  dataDir: values["data-dir"],
1253
2034
  resourceDir: values["resource-dir"],
2035
+ ownerId: values["owner-id"],
2036
+ r2QuotaBytes: Number(values["r2-quota-bytes"] ?? process.env.SPROUTBOAT_R2_QUOTA_BYTES) || undefined,
2037
+ r2ResourceIds: values["r2-resource-ids"] ? String(values["r2-resource-ids"]).split(",").filter(Boolean) : undefined,
2038
+ r2MinFreeBytes: Number(values["r2-min-free-bytes"] ?? process.env.SPROUTBOAT_R2_MIN_FREE_BYTES) || undefined,
1254
2039
  token: values.token ?? process.env.SB_BROKER_TOKEN,
1255
2040
  bindings,
1256
2041
  secrets,
@@ -1261,7 +2046,9 @@ if (import.meta.main) {
1261
2046
  dispatchEnabled: () => dispatchEnabled,
1262
2047
  });
1263
2048
  const { port } = listen(broker, "127.0.0.1", Number(values.port ?? process.env.SB_BROKER_PORT ?? 0));
2049
+ const transferPort = Number(values["transfer-port"] ?? process.env.SB_R2_TRANSFER_PORT ?? 0);
2050
+ const transferServer = transferPort > 0 ? listenTransfers(broker, "127.0.0.1", transferPort) : null;
1264
2051
  console.log(
1265
- `sproutboat broker: 127.0.0.1:${port} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
2052
+ `sproutboat broker: 127.0.0.1:${port} transfer=${transferServer?.port ?? "off"} db=${values.db ?? ":memory:"} sprout=${values["sprout-url"] ?? process.env.SB_SPROUT_URL ?? "(none)"} dispatch=${dispatchEnabled ? "enabled" : "disabled"}`,
1266
2053
  );
1267
2054
  }